Add hls support on server

This commit is contained in:
Chocobozzz 2019-01-29 08:37:25 +01:00 committed by Chocobozzz
parent 4348a27d25
commit 0920929696
81 changed files with 2000 additions and 407 deletions

View File

@ -134,7 +134,7 @@
"ngx-qrcode2": "^0.0.9",
"node-sass": "^4.9.3",
"npm-font-source-sans-pro": "^1.0.2",
"p2p-media-loader-hlsjs": "^0.3.0",
"p2p-media-loader-hlsjs": "^0.4.0",
"path-browserify": "^1.0.0",
"primeng": "^7.0.0",
"process": "^0.11.10",

View File

@ -22,7 +22,9 @@ export abstract class UserEdit extends FormReactive {
}
computeQuotaWithTranscoding () {
const resolutions = this.serverService.getConfig().transcoding.enabledResolutions
const transcodingConfig = this.serverService.getConfig().transcoding
const resolutions = transcodingConfig.enabledResolutions
const higherResolution = VideoResolution.H_1080P
let multiplier = 0
@ -30,6 +32,8 @@ export abstract class UserEdit extends FormReactive {
multiplier += resolution / higherResolution
}
if (transcodingConfig.hls.enabled) multiplier *= 2
return multiplier * parseInt(this.form.value['videoQuota'], 10)
}

View File

@ -51,7 +51,10 @@ export class ServerService {
requiresEmailVerification: false
},
transcoding: {
enabledResolutions: []
enabledResolutions: [],
hls: {
enabled: false
}
},
avatar: {
file: {

View File

@ -3,6 +3,8 @@ import { AuthUser } from '../../core'
import { Video } from '../../shared/video/video.model'
import { Account } from '@app/shared/account/account.model'
import { VideoChannel } from '@app/shared/video-channel/video-channel.model'
import { VideoStreamingPlaylist } from '../../../../../shared/models/videos/video-streaming-playlist.model'
import { VideoStreamingPlaylistType } from '../../../../../shared/models/videos/video-streaming-playlist.type'
export class VideoDetails extends Video implements VideoDetailsServerModel {
descriptionPath: string
@ -19,6 +21,10 @@ export class VideoDetails extends Video implements VideoDetailsServerModel {
likesPercent: number
dislikesPercent: number
trackerUrls: string[]
streamingPlaylists: VideoStreamingPlaylist[]
constructor (hash: VideoDetailsServerModel, translations = {}) {
super(hash, translations)
@ -30,6 +36,9 @@ export class VideoDetails extends Video implements VideoDetailsServerModel {
this.support = hash.support
this.commentsEnabled = hash.commentsEnabled
this.trackerUrls = hash.trackerUrls
this.streamingPlaylists = hash.streamingPlaylists
this.buildLikeAndDislikePercents()
}
@ -53,4 +62,8 @@ export class VideoDetails extends Video implements VideoDetailsServerModel {
this.likesPercent = (this.likes / (this.likes + this.dislikes)) * 100
this.dislikesPercent = (this.dislikes / (this.likes + this.dislikes)) * 100
}
getHlsPlaylist () {
return this.streamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS)
}
}

View File

@ -23,7 +23,7 @@ import { I18n } from '@ngx-translate/i18n-polyfill'
import { environment } from '../../../environments/environment'
import { VideoCaptionService } from '@app/shared/video-caption'
import { MarkdownService } from '@app/shared/renderer'
import { PeertubePlayerManager } from '../../../assets/player/peertube-player-manager'
import { P2PMediaLoaderOptions, PeertubePlayerManager, PlayerMode, WebtorrentOptions } from '../../../assets/player/peertube-player-manager'
@Component({
selector: 'my-video-watch',
@ -424,15 +424,33 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
serverUrl: environment.apiUrl,
videoCaptions: playerCaptions
},
webtorrent: {
videoFiles: this.video.files
}
}
let mode: PlayerMode
const hlsPlaylist = this.video.getHlsPlaylist()
if (hlsPlaylist) {
mode = 'p2p-media-loader'
const p2pMediaLoader = {
playlistUrl: hlsPlaylist.playlistUrl,
segmentsSha256Url: hlsPlaylist.segmentsSha256Url,
redundancyBaseUrls: hlsPlaylist.redundancies.map(r => r.baseUrl),
trackerAnnounce: this.video.trackerUrls,
videoFiles: this.video.files
} as P2PMediaLoaderOptions
Object.assign(options, { p2pMediaLoader })
} else {
mode = 'webtorrent'
const webtorrent = {
videoFiles: this.video.files
} as WebtorrentOptions
Object.assign(options, { webtorrent })
}
this.zone.runOutsideAngular(async () => {
this.player = await PeertubePlayerManager.initialize('webtorrent', options)
this.player = await PeertubePlayerManager.initialize(mode, options)
this.player.on('customError', ({ err }: { err: any }) => this.handleError(err))
})

View File

@ -1,21 +1,21 @@
// FIXME: something weird with our path definition in tsconfig and typings
// @ts-ignore
import * as videojs from 'video.js'
import { P2PMediaLoaderPluginOptions, PlayerNetworkInfo, VideoJSComponentInterface } from './peertube-videojs-typings'
import { P2PMediaLoaderPluginOptions, PlayerNetworkInfo, VideoJSComponentInterface } from '../peertube-videojs-typings'
import { Engine, initHlsJsPlayer, initVideoJsContribHlsJsPlayer } from 'p2p-media-loader-hlsjs'
import { Events } from 'p2p-media-loader-core'
// videojs-hlsjs-plugin needs videojs in window
window['videojs'] = videojs
require('@streamroot/videojs-hlsjs-plugin')
import { Engine, initVideoJsContribHlsJsPlayer } from 'p2p-media-loader-hlsjs'
import { Events } from 'p2p-media-loader-core'
const Plugin: VideoJSComponentInterface = videojs.getPlugin('plugin')
class P2pMediaLoaderPlugin extends Plugin {
private readonly CONSTANTS = {
INFO_SCHEDULER: 1000 // Don't change this
}
private readonly options: P2PMediaLoaderPluginOptions
private hlsjs: any // Don't type hlsjs to not bundle the module
private p2pEngine: Engine
@ -26,16 +26,22 @@ class P2pMediaLoaderPlugin extends Plugin {
totalDownload: 0,
totalUpload: 0
}
private statsHTTPBytes = {
pendingDownload: [] as number[],
pendingUpload: [] as number[],
totalDownload: 0,
totalUpload: 0
}
private networkInfoInterval: any
constructor (player: videojs.Player, options: P2PMediaLoaderPluginOptions) {
super(player, options)
this.options = options
videojs.Html5Hlsjs.addHook('beforeinitialize', (videojsPlayer: any, hlsjs: any) => {
this.hlsjs = hlsjs
this.initialize()
})
initVideoJsContribHlsJsPlayer(player)
@ -44,6 +50,8 @@ class P2pMediaLoaderPlugin extends Plugin {
type: options.type,
src: options.src
})
player.ready(() => this.initialize())
}
dispose () {
@ -51,6 +59,8 @@ class P2pMediaLoaderPlugin extends Plugin {
}
private initialize () {
initHlsJsPlayer(this.hlsjs)
this.p2pEngine = this.player.tech_.options_.hlsjsConfig.loader.getEngine()
// Avoid using constants to not import hls.hs
@ -59,38 +69,55 @@ class P2pMediaLoaderPlugin extends Plugin {
this.trigger('resolutionChange', { auto: this.hlsjs.autoLevelEnabled, resolutionId: data.height })
})
this.p2pEngine.on(Events.SegmentError, (segment, err) => {
console.error('Segment error.', segment, err)
})
this.statsP2PBytes.numPeers = 1 + this.options.redundancyBaseUrls.length
this.runStats()
}
private runStats () {
this.p2pEngine.on(Events.PieceBytesDownloaded, (method: string, size: number) => {
if (method === 'p2p') {
this.statsP2PBytes.pendingDownload.push(size)
this.statsP2PBytes.totalDownload += size
}
const elem = method === 'p2p' ? this.statsP2PBytes : this.statsHTTPBytes
elem.pendingDownload.push(size)
elem.totalDownload += size
})
this.p2pEngine.on(Events.PieceBytesUploaded, (method: string, size: number) => {
if (method === 'p2p') {
this.statsP2PBytes.pendingUpload.push(size)
this.statsP2PBytes.totalUpload += size
}
const elem = method === 'p2p' ? this.statsP2PBytes : this.statsHTTPBytes
elem.pendingUpload.push(size)
elem.totalUpload += size
})
this.p2pEngine.on(Events.PeerConnect, () => this.statsP2PBytes.numPeers++)
this.p2pEngine.on(Events.PeerClose, () => this.statsP2PBytes.numPeers--)
this.networkInfoInterval = setInterval(() => {
let downloadSpeed = this.statsP2PBytes.pendingDownload.reduce((a: number, b: number) => a + b, 0)
let uploadSpeed = this.statsP2PBytes.pendingUpload.reduce((a: number, b: number) => a + b, 0)
const p2pDownloadSpeed = this.arraySum(this.statsP2PBytes.pendingDownload)
const p2pUploadSpeed = this.arraySum(this.statsP2PBytes.pendingUpload)
const httpDownloadSpeed = this.arraySum(this.statsHTTPBytes.pendingDownload)
const httpUploadSpeed = this.arraySum(this.statsHTTPBytes.pendingUpload)
this.statsP2PBytes.pendingDownload = []
this.statsP2PBytes.pendingUpload = []
this.statsHTTPBytes.pendingDownload = []
this.statsHTTPBytes.pendingUpload = []
return this.player.trigger('p2pInfo', {
http: {
downloadSpeed: httpDownloadSpeed,
uploadSpeed: httpUploadSpeed,
downloaded: this.statsHTTPBytes.totalDownload,
uploaded: this.statsHTTPBytes.totalUpload
},
p2p: {
downloadSpeed,
uploadSpeed,
downloadSpeed: p2pDownloadSpeed,
uploadSpeed: p2pUploadSpeed,
numPeers: this.statsP2PBytes.numPeers,
downloaded: this.statsP2PBytes.totalDownload,
uploaded: this.statsP2PBytes.totalUpload
@ -98,6 +125,10 @@ class P2pMediaLoaderPlugin extends Plugin {
} as PlayerNetworkInfo)
}, this.CONSTANTS.INFO_SCHEDULER)
}
private arraySum (data: number[]) {
return data.reduce((a: number, b: number) => a + b, 0)
}
}
videojs.registerPlugin('p2pMediaLoader', P2pMediaLoaderPlugin)

View File

@ -0,0 +1,28 @@
import { basename } from 'path'
import { Segment } from 'p2p-media-loader-core'
function segmentUrlBuilderFactory (baseUrls: string[]) {
return function segmentBuilder (segment: Segment) {
const max = baseUrls.length + 1
const i = getRandomInt(max)
if (i === max - 1) return segment.url
let newBaseUrl = baseUrls[i]
let middlePart = newBaseUrl.endsWith('/') ? '' : '/'
return newBaseUrl + middlePart + basename(segment.url)
}
}
// ---------------------------------------------------------------------------
export {
segmentUrlBuilderFactory
}
// ---------------------------------------------------------------------------
function getRandomInt (max: number) {
return Math.floor(Math.random() * Math.floor(max))
}

View File

@ -0,0 +1,56 @@
import { Segment } from 'p2p-media-loader-core'
import { basename } from 'path'
function segmentValidatorFactory (segmentsSha256Url: string) {
const segmentsJSON = fetchSha256Segments(segmentsSha256Url)
return async function segmentValidator (segment: Segment) {
const segmentName = basename(segment.url)
const hashShouldBe = (await segmentsJSON)[segmentName]
if (hashShouldBe === undefined) {
throw new Error(`Unknown segment name ${segmentName} in segment validator`)
}
const calculatedSha = bufferToEx(await sha256(segment.data))
if (calculatedSha !== hashShouldBe) {
throw new Error(`Hashes does not correspond for segment ${segmentName} (expected: ${hashShouldBe} instead of ${calculatedSha})`)
}
}
}
// ---------------------------------------------------------------------------
export {
segmentValidatorFactory
}
// ---------------------------------------------------------------------------
function fetchSha256Segments (url: string) {
return fetch(url)
.then(res => res.json())
.catch(err => {
console.error('Cannot get sha256 segments', err)
return {}
})
}
function sha256 (data?: ArrayBuffer) {
if (!data) return undefined
return window.crypto.subtle.digest('SHA-256', data)
}
// Thanks: https://stackoverflow.com/a/53307879
function bufferToEx (buffer?: ArrayBuffer) {
if (!buffer) return ''
let s = ''
const h = '0123456789abcdef'
const o = new Uint8Array(buffer)
o.forEach((v: any) => s += h[ v >> 4 ] + h[ v & 15 ])
return s
}

View File

@ -13,8 +13,10 @@ import './videojs-components/p2p-info-button'
import './videojs-components/peertube-load-progress-bar'
import './videojs-components/theater-button'
import { P2PMediaLoaderPluginOptions, UserWatching, VideoJSCaption, VideoJSPluginOptions, videojsUntyped } from './peertube-videojs-typings'
import { buildVideoEmbed, buildVideoLink, copyToClipboard } from './utils'
import { buildVideoEmbed, buildVideoLink, copyToClipboard, getRtcConfig } from './utils'
import { getCompleteLocale, getShortLocale, is18nLocale, isDefaultLocale } from '../../../../shared/models/i18n/i18n'
import { segmentValidatorFactory } from './p2p-media-loader/segment-validator'
import { segmentUrlBuilderFactory } from './p2p-media-loader/segment-url-builder'
// Change 'Playback Rate' to 'Speed' (smaller for our settings menu)
videojsUntyped.getComponent('PlaybackRateMenuButton').prototype.controlText_ = 'Speed'
@ -31,7 +33,10 @@ export type WebtorrentOptions = {
export type P2PMediaLoaderOptions = {
playlistUrl: string
segmentsSha256Url: string
trackerAnnounce: string[]
redundancyBaseUrls: string[]
videoFiles: VideoFile[]
}
export type CommonOptions = {
@ -90,11 +95,11 @@ export class PeertubePlayerManager {
static async initialize (mode: PlayerMode, options: PeertubePlayerManagerOptions) {
let p2pMediaLoader: any
if (mode === 'webtorrent') await import('./webtorrent-plugin')
if (mode === 'webtorrent') await import('./webtorrent/webtorrent-plugin')
if (mode === 'p2p-media-loader') {
[ p2pMediaLoader ] = await Promise.all([
import('p2p-media-loader-hlsjs'),
import('./p2p-media-loader-plugin')
import('./p2p-media-loader/p2p-media-loader-plugin')
])
}
@ -144,11 +149,14 @@ export class PeertubePlayerManager {
const commonOptions = options.common
const webtorrentOptions = options.webtorrent
const p2pMediaLoaderOptions = options.p2pMediaLoader
let autoplay = options.common.autoplay
let html5 = {}
const plugins: VideoJSPluginOptions = {
peertube: {
autoplay: commonOptions.autoplay, // Use peertube plugin autoplay because we get the file by webtorrent
mode,
autoplay, // Use peertube plugin autoplay because we get the file by webtorrent
videoViewUrl: commonOptions.videoViewUrl,
videoDuration: commonOptions.videoDuration,
startTime: commonOptions.startTime,
@ -160,19 +168,35 @@ export class PeertubePlayerManager {
if (p2pMediaLoaderOptions) {
const p2pMediaLoader: P2PMediaLoaderPluginOptions = {
redundancyBaseUrls: options.p2pMediaLoader.redundancyBaseUrls,
type: 'application/x-mpegURL',
src: p2pMediaLoaderOptions.playlistUrl
}
const trackerAnnounce = p2pMediaLoaderOptions.trackerAnnounce
.filter(t => t.startsWith('ws'))
const p2pMediaLoaderConfig = {
// loader: {
// trackerAnnounce: p2pMediaLoaderOptions.trackerAnnounce
// },
loader: {
trackerAnnounce,
segmentValidator: segmentValidatorFactory(options.p2pMediaLoader.segmentsSha256Url),
rtcConfig: getRtcConfig(),
requiredSegmentsPriority: 5,
segmentUrlBuilder: segmentUrlBuilderFactory(options.p2pMediaLoader.redundancyBaseUrls)
},
segments: {
swarmId: p2pMediaLoaderOptions.playlistUrl
}
}
const streamrootHls = {
levelLabelHandler: (level: { height: number, width: number }) => {
const file = p2pMediaLoaderOptions.videoFiles.find(f => f.resolution.id === level.height)
let label = file.resolution.label
if (file.fps >= 50) label += file.fps
return label
},
html5: {
hlsjsConfig: {
liveSyncDurationCount: 7,
@ -187,12 +211,15 @@ export class PeertubePlayerManager {
if (webtorrentOptions) {
const webtorrent = {
autoplay: commonOptions.autoplay,
autoplay,
videoDuration: commonOptions.videoDuration,
playerElement: commonOptions.playerElement,
videoFiles: webtorrentOptions.videoFiles
}
Object.assign(plugins, { webtorrent })
// WebTorrent plugin handles autoplay, because we do some hackish stuff in there
autoplay = false
}
const videojsOptions = {
@ -208,7 +235,7 @@ export class PeertubePlayerManager {
: undefined, // Undefined so the player knows it has to check the local storage
poster: commonOptions.poster,
autoplay: false,
autoplay,
inactivityTimeout: commonOptions.inactivityTimeout,
playbackRates: [ 0.5, 0.75, 1, 1.25, 1.5, 2 ],
plugins,

View File

@ -52,12 +52,12 @@ class PeerTubePlugin extends Plugin {
this.player.ready(() => {
const playerOptions = this.player.options_
if (this.player.webtorrent) {
if (options.mode === 'webtorrent') {
this.player.webtorrent().on('resolutionChange', (_: any, d: any) => this.handleResolutionChange(d))
this.player.webtorrent().on('autoResolutionChange', (_: any, d: any) => this.trigger('autoResolutionChange', d))
}
if (this.player.p2pMediaLoader) {
if (options.mode === 'p2p-media-loader') {
this.player.p2pMediaLoader().on('resolutionChange', (_: any, d: any) => this.handleResolutionChange(d))
}

View File

@ -4,12 +4,15 @@ import * as videojs from 'video.js'
import { VideoFile } from '../../../../shared/models/videos/video.model'
import { PeerTubePlugin } from './peertube-plugin'
import { WebTorrentPlugin } from './webtorrent-plugin'
import { WebTorrentPlugin } from './webtorrent/webtorrent-plugin'
import { P2pMediaLoaderPlugin } from './p2p-media-loader/p2p-media-loader-plugin'
import { PlayerMode } from './peertube-player-manager'
declare namespace videojs {
interface Player {
peertube (): PeerTubePlugin
webtorrent (): WebTorrentPlugin
p2pMediaLoader (): P2pMediaLoaderPlugin
}
}
@ -33,6 +36,8 @@ type UserWatching = {
}
type PeerTubePluginOptions = {
mode: PlayerMode
autoplay: boolean
videoViewUrl: string
videoDuration: number
@ -54,6 +59,7 @@ type WebtorrentPluginOptions = {
}
type P2PMediaLoaderPluginOptions = {
redundancyBaseUrls: string[]
type: string
src: string
}
@ -91,6 +97,13 @@ type AutoResolutionUpdateData = {
}
type PlayerNetworkInfo = {
http: {
downloadSpeed: number
uploadSpeed: number
downloaded: number
uploaded: number
}
p2p: {
downloadSpeed: number
uploadSpeed: number

View File

@ -112,9 +112,23 @@ function videoFileMinByResolution (files: VideoFile[]) {
return min
}
function getRtcConfig () {
return {
iceServers: [
{
urls: 'stun:stun.stunprotocol.org'
},
{
urls: 'stun:stun.framasoft.org'
}
]
}
}
// ---------------------------------------------------------------------------
export {
getRtcConfig,
toTitleCase,
timeToInt,
buildVideoLink,

View File

@ -75,11 +75,12 @@ class P2pInfoButton extends Button {
}
const p2pStats = data.p2p
const httpStats = data.http
const downloadSpeed = bytes(p2pStats.downloadSpeed)
const uploadSpeed = bytes(p2pStats.uploadSpeed)
const totalDownloaded = bytes(p2pStats.downloaded)
const totalUploaded = bytes(p2pStats.uploaded)
const downloadSpeed = bytes(p2pStats.downloadSpeed + httpStats.downloadSpeed)
const uploadSpeed = bytes(p2pStats.uploadSpeed + httpStats.uploadSpeed)
const totalDownloaded = bytes(p2pStats.downloaded + httpStats.downloaded)
const totalUploaded = bytes(p2pStats.uploaded + httpStats.uploaded)
const numPeers = p2pStats.numPeers
subDivWebtorrent.title = this.player_.localize('Total downloaded: ') + totalDownloaded.join(' ') + '\n' +
@ -92,7 +93,7 @@ class P2pInfoButton extends Button {
uploadSpeedUnit.textContent = ' ' + uploadSpeed[ 1 ]
peersNumber.textContent = numPeers
peersText.textContent = ' ' + this.player_.localize('peers')
peersText.textContent = ' ' + (numPeers > 1 ? this.player_.localize('peers') : this.player_.localize('peer'))
subDivHttp.className = 'vjs-peertube-hidden'
subDivWebtorrent.className = 'vjs-peertube-displayed'

View File

@ -3,18 +3,18 @@
import * as videojs from 'video.js'
import * as WebTorrent from 'webtorrent'
import { VideoFile } from '../../../../shared/models/videos/video.model'
import { renderVideo } from './webtorrent/video-renderer'
import { LoadedQualityData, PlayerNetworkInfo, VideoJSComponentInterface, WebtorrentPluginOptions } from './peertube-videojs-typings'
import { videoFileMaxByResolution, videoFileMinByResolution } from './utils'
import { PeertubeChunkStore } from './webtorrent/peertube-chunk-store'
import { VideoFile } from '../../../../../shared/models/videos/video.model'
import { renderVideo } from './video-renderer'
import { LoadedQualityData, PlayerNetworkInfo, VideoJSComponentInterface, WebtorrentPluginOptions } from '../peertube-videojs-typings'
import { getRtcConfig, videoFileMaxByResolution, videoFileMinByResolution } from '../utils'
import { PeertubeChunkStore } from './peertube-chunk-store'
import {
getAverageBandwidthInStore,
getStoredMute,
getStoredVolume,
getStoredWebTorrentEnabled,
saveAverageBandwidth
} from './peertube-player-local-storage'
} from '../peertube-player-local-storage'
const CacheChunkStore = require('cache-chunk-store')
@ -44,16 +44,7 @@ class WebTorrentPlugin extends Plugin {
private readonly webtorrent = new WebTorrent({
tracker: {
rtcConfig: {
iceServers: [
{
urls: 'stun:stun.stunprotocol.org'
},
{
urls: 'stun:stun.framasoft.org'
}
]
}
rtcConfig: getRtcConfig()
},
dht: false
})
@ -472,6 +463,12 @@ class WebTorrentPlugin extends Plugin {
if (this.webtorrent.downloadSpeed !== 0) this.downloadSpeeds.push(this.webtorrent.downloadSpeed)
return this.player.trigger('p2pInfo', {
http: {
downloadSpeed: 0,
uploadSpeed: 0,
downloaded: 0,
uploaded: 0
},
p2p: {
downloadSpeed: this.torrent.downloadSpeed,
numPeers: this.torrent.numPeers,

View File

@ -23,7 +23,13 @@ import { peertubeTranslate, ResultList, VideoDetails } from '../../../../shared'
import { PeerTubeResolution } from '../player/definitions'
import { VideoJSCaption } from '../../assets/player/peertube-videojs-typings'
import { VideoCaption } from '../../../../shared/models/videos/caption/video-caption.model'
import { PeertubePlayerManager, PeertubePlayerManagerOptions, PlayerMode } from '../../assets/player/peertube-player-manager'
import {
P2PMediaLoaderOptions,
PeertubePlayerManager,
PeertubePlayerManagerOptions,
PlayerMode
} from '../../assets/player/peertube-player-manager'
import { VideoStreamingPlaylistType } from '../../../../shared/models/videos/video-streaming-playlist.type'
/**
* Embed API exposes control of the embed player to the outside world via
@ -319,13 +325,16 @@ class PeerTubeEmbed {
}
if (this.mode === 'p2p-media-loader') {
const hlsPlaylist = videoInfo.streamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS)
Object.assign(options, {
p2pMediaLoader: {
// playlistUrl: 'https://akamai-axtest.akamaized.net/routes/lapd-v1-acceptance/www_c4/Manifest.m3u8'
// playlistUrl: 'https://d2zihajmogu5jn.cloudfront.net/bipbop-advanced/bipbop_16x9_variant.m3u8'
// trackerAnnounce: [ window.location.origin.replace(/^http/, 'ws') + '/tracker/socket' ],
playlistUrl: 'https://cdn.theoplayer.com/video/elephants-dream/playlist.m3u8'
}
playlistUrl: hlsPlaylist.playlistUrl,
segmentsSha256Url: hlsPlaylist.segmentsSha256Url,
redundancyBaseUrls: hlsPlaylist.redundancies.map(r => r.baseUrl),
trackerAnnounce: videoInfo.trackerUrls,
videoFiles: videoInfo.files
} as P2PMediaLoaderOptions
})
} else {
Object.assign(options, {

View File

@ -2641,6 +2641,13 @@ debug@^3.1.0, debug@^3.2.5:
dependencies:
ms "^2.1.1"
debug@^4.1.1:
version "4.1.1"
resolved "https://registry.yarnpkg.com/debug/-/debug-4.1.1.tgz#3b72260255109c6b589cee050f1d516139664791"
integrity sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==
dependencies:
ms "^2.1.1"
decamelize@^1.1.1, decamelize@^1.1.2, decamelize@^1.2.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290"
@ -6131,7 +6138,7 @@ m3u8-parser@4.2.0:
resolved "https://registry.yarnpkg.com/m3u8-parser/-/m3u8-parser-4.2.0.tgz#c8e0785fd17f741f4408b49466889274a9e36447"
integrity sha512-LVHw0U6IPJjwk9i9f7Xe26NqaUHTNlIt4SSWoEfYFROeVKHN6MIjOhbRheI3dg8Jbq5WCuMFQ0QU3EgZpmzFPg==
m3u8-parser@^4.2.0:
m3u8-parser@^4.3.0:
version "4.3.0"
resolved "https://registry.yarnpkg.com/m3u8-parser/-/m3u8-parser-4.3.0.tgz#4b4e988f87b6d8b2401d209a1d17798285a9da04"
integrity sha512-bVbjuBMoVIgFL1vpXVIxjeaoB5TPDJRb0m5qiTdM738SGqv/LAmsnVVPlKjM4fulm/rr1XZsKM+owHm+zvqxYA==
@ -7244,25 +7251,25 @@ p-try@^2.0.0:
resolved "https://registry.yarnpkg.com/p-try/-/p-try-2.0.0.tgz#85080bb87c64688fa47996fe8f7dfbe8211760b1"
integrity sha512-hMp0onDKIajHfIkdRk3P4CdCmErkYAxxDtP3Wx/4nZ3aGlau2VKh3mZpcuFkH27WQkL/3WBCPOktzA9ZOAnMQQ==
p2p-media-loader-core@^0.3.0:
version "0.3.0"
resolved "https://registry.yarnpkg.com/p2p-media-loader-core/-/p2p-media-loader-core-0.3.0.tgz#75687d7d7bee835d5c6c2f17d346add2dbe43b83"
integrity sha512-WKB9ONdA0kDQHXr6nixIL8t0UZuTD9Pqi/BIuaTiPUGDwYXUS/Mf5YynLAUupniLkIaDYD7/jmSLWqpZUDsAyw==
p2p-media-loader-core@^0.4.0:
version "0.4.0"
resolved "https://registry.yarnpkg.com/p2p-media-loader-core/-/p2p-media-loader-core-0.4.0.tgz#767d56785545bc9c0d8c1a04eb7b67a33e40d0c8"
integrity sha512-llcFqEDs19o916g2OSIPHPjZweO5caHUm/7P18Qu+qb3swYQYSPNwMLoHnpXROHiH5I+00K8w5enz31oUwiCgA==
dependencies:
bittorrent-tracker "^9.10.1"
debug "^4.1.0"
debug "^4.1.1"
events "^3.0.0"
get-browser-rtc "^1.0.2"
sha.js "^2.4.11"
p2p-media-loader-hlsjs@^0.3.0:
version "0.3.0"
resolved "https://registry.yarnpkg.com/p2p-media-loader-hlsjs/-/p2p-media-loader-hlsjs-0.3.0.tgz#4ee15d4d1a23aa0322a5be2bc6c329b6c913028d"
integrity sha512-U7PzMG5X7CVQ15OtMPRQjW68Msu0fuw8Pp0PRznX5uK0p26tSYMT/ZYCNeYCoDg3wGgJHM+327ed3M7TRJ4lcw==
p2p-media-loader-hlsjs@^0.4.0:
version "0.4.0"
resolved "https://registry.yarnpkg.com/p2p-media-loader-hlsjs/-/p2p-media-loader-hlsjs-0.4.0.tgz#1b90c88580503d4c3d8017c813abe41803b613ed"
integrity sha512-IWRs/aGasKD//+dtQkYWAjD/cQx3LMaLkMn0EzLhLpeBj4SLNjlbwOPlbx36M4i39X04Y3WZe9YUeIciId3G5Q==
dependencies:
events "^3.0.0"
m3u8-parser "^4.2.0"
p2p-media-loader-core "^0.3.0"
m3u8-parser "^4.3.0"
p2p-media-loader-core "^0.4.0"
package-json-versionify@^1.0.2:
version "1.0.4"

View File

@ -48,6 +48,7 @@ storage:
tmp: 'storage/tmp/' # Used to download data (imports etc), store uploaded files before processing...
avatars: 'storage/avatars/'
videos: 'storage/videos/'
playlists: 'storage/playlists/'
redundancy: 'storage/redundancy/'
logs: 'storage/logs/'
previews: 'storage/previews/'
@ -138,6 +139,14 @@ transcoding:
480p: false
720p: false
1080p: false
# /!\ EXPERIMENTAL /!\
# Generate HLS playlist/segments. Better playback than with WebTorrent:
# * Resolution change is smoother
# * Faster playback in particular with long videos
# * More stable playback (less bugs/infinite loading)
# /!\ Multiply videos storage by two /!\
hls:
enabled: false
import:
# Add ability for your users to import remote videos (from YouTube, torrent...)

View File

@ -49,6 +49,7 @@ storage:
tmp: '/var/www/peertube/storage/tmp/' # Used to download data (imports etc), store uploaded files before processing...
avatars: '/var/www/peertube/storage/avatars/'
videos: '/var/www/peertube/storage/videos/'
playlists: '/var/www/peertube/storage/playlists/'
redundancy: '/var/www/peertube/storage/videos/'
logs: '/var/www/peertube/storage/logs/'
previews: '/var/www/peertube/storage/previews/'
@ -151,6 +152,14 @@ transcoding:
480p: false
720p: false
1080p: false
# /!\ EXPERIMENTAL /!\
# Generate HLS playlist/segments. Better playback than with WebTorrent:
# * Resolution change is smoother
# * Faster playback in particular with long videos
# * More stable playback (less bugs/infinite loading)
# /!\ Multiply videos storage by two /!\
hls:
enabled: false
import:
# Add ability for your users to import remote videos (from YouTube, torrent...)

View File

@ -13,6 +13,7 @@ storage:
tmp: 'test1/tmp/'
avatars: 'test1/avatars/'
videos: 'test1/videos/'
playlists: 'test1/playlists/'
redundancy: 'test1/redundancy/'
logs: 'test1/logs/'
previews: 'test1/previews/'

View File

@ -13,6 +13,7 @@ storage:
tmp: 'test2/tmp/'
avatars: 'test2/avatars/'
videos: 'test2/videos/'
playlists: 'test2/playlists/'
redundancy: 'test2/redundancy/'
logs: 'test2/logs/'
previews: 'test2/previews/'

View File

@ -13,6 +13,7 @@ storage:
tmp: 'test3/tmp/'
avatars: 'test3/avatars/'
videos: 'test3/videos/'
playlists: 'test3/playlists/'
redundancy: 'test3/redundancy/'
logs: 'test3/logs/'
previews: 'test3/previews/'

View File

@ -13,6 +13,7 @@ storage:
tmp: 'test4/tmp/'
avatars: 'test4/avatars/'
videos: 'test4/videos/'
playlists: 'test4/playlists/'
redundancy: 'test4/redundancy/'
logs: 'test4/logs/'
previews: 'test4/previews/'

View File

@ -13,6 +13,7 @@ storage:
tmp: 'test5/tmp/'
avatars: 'test5/avatars/'
videos: 'test5/videos/'
playlists: 'test5/playlists/'
redundancy: 'test5/redundancy/'
logs: 'test5/logs/'
previews: 'test5/previews/'

View File

@ -13,6 +13,7 @@ storage:
tmp: 'test6/tmp/'
avatars: 'test6/avatars/'
videos: 'test6/videos/'
playlists: 'test6/playlists/'
redundancy: 'test6/redundancy/'
logs: 'test6/logs/'
previews: 'test6/previews/'

View File

@ -62,6 +62,8 @@ transcoding:
480p: true
720p: true
1080p: true
hls:
enabled: true
import:
videos:

View File

@ -117,6 +117,7 @@
"fluent-ffmpeg": "^2.1.0",
"fs-extra": "^7.0.0",
"helmet": "^3.12.1",
"hlsdownloader": "https://github.com/Chocobozzz/hlsdownloader#build",
"http-signature": "^1.2.0",
"ip-anonymize": "^0.0.6",
"ipaddr.js": "1.8.1",

View File

@ -23,12 +23,15 @@ const playerKeys = {
'Speed': 'Speed',
'Subtitles/CC': 'Subtitles/CC',
'peers': 'peers',
'peer': 'peer',
'Go to the video page': 'Go to the video page',
'Settings': 'Settings',
'Uses P2P, others may know you are watching this video.': 'Uses P2P, others may know you are watching this video.',
'Copy the video URL': 'Copy the video URL',
'Copy the video URL at the current time': 'Copy the video URL at the current time',
'Copy embed code': 'Copy embed code'
'Copy embed code': 'Copy embed code',
'Total downloaded: ': 'Total downloaded: ',
'Total uploaded: ': 'Total uploaded: '
}
const playerTranslations = {
target: join(__dirname, '../../../client/src/locale/source/player_en_US.xml'),

View File

@ -13,6 +13,7 @@ import { VideoCommentModel } from '../server/models/video/video-comment'
import { getServerActor } from '../server/helpers/utils'
import { AccountModel } from '../server/models/account/account'
import { VideoChannelModel } from '../server/models/video/video-channel'
import { VideoStreamingPlaylistModel } from '../server/models/video/video-streaming-playlist'
run()
.then(() => process.exit(0))
@ -109,11 +110,9 @@ async function run () {
console.log('Updating video and torrent files.')
const videos = await VideoModel.list()
const videos = await VideoModel.listLocal()
for (const video of videos) {
if (video.isOwned() === false) continue
console.log('Updated video ' + video.uuid)
console.log('Updating video ' + video.uuid)
video.url = getVideoActivityPubUrl(video)
await video.save()
@ -122,5 +121,12 @@ async function run () {
console.log('Updating torrent file %s of video %s.', file.resolution, video.uuid)
await video.createTorrentAndSetInfoHash(file)
}
for (const playlist of video.VideoStreamingPlaylists) {
playlist.playlistUrl = CONFIG.WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid)
playlist.segmentsSha256Url = CONFIG.WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid)
await playlist.save()
}
}
}

View File

@ -37,7 +37,7 @@ import {
getVideoSharesActivityPubUrl
} from '../../lib/activitypub'
import { VideoCaptionModel } from '../../models/video/video-caption'
import { videoRedundancyGetValidator } from '../../middlewares/validators/redundancy'
import { videoFileRedundancyGetValidator, videoPlaylistRedundancyGetValidator } from '../../middlewares/validators/redundancy'
import { getServerActor } from '../../helpers/utils'
import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
@ -66,11 +66,11 @@ activityPubClientRouter.get('/accounts?/:name/dislikes/:videoId',
activityPubClientRouter.get('/videos/watch/:id',
executeIfActivityPub(asyncMiddleware(cacheRoute(ROUTE_CACHE_LIFETIME.ACTIVITY_PUB.VIDEOS))),
executeIfActivityPub(asyncMiddleware(videosGetValidator)),
executeIfActivityPub(asyncMiddleware(videosCustomGetValidator('only-video-with-rights'))),
executeIfActivityPub(asyncMiddleware(videoController))
)
activityPubClientRouter.get('/videos/watch/:id/activity',
executeIfActivityPub(asyncMiddleware(videosGetValidator)),
executeIfActivityPub(asyncMiddleware(videosCustomGetValidator('only-video-with-rights'))),
executeIfActivityPub(asyncMiddleware(videoController))
)
activityPubClientRouter.get('/videos/watch/:id/announces',
@ -116,7 +116,11 @@ activityPubClientRouter.get('/video-channels/:name/following',
)
activityPubClientRouter.get('/redundancy/videos/:videoId/:resolution([0-9]+)(-:fps([0-9]+))?',
executeIfActivityPub(asyncMiddleware(videoRedundancyGetValidator)),
executeIfActivityPub(asyncMiddleware(videoFileRedundancyGetValidator)),
executeIfActivityPub(asyncMiddleware(videoRedundancyController))
)
activityPubClientRouter.get('/redundancy/video-playlists/:streamingPlaylistType/:videoId',
executeIfActivityPub(asyncMiddleware(videoPlaylistRedundancyGetValidator)),
executeIfActivityPub(asyncMiddleware(videoRedundancyController))
)
@ -163,7 +167,8 @@ function getAccountVideoRate (rateType: VideoRateType) {
}
async function videoController (req: express.Request, res: express.Response) {
const video: VideoModel = res.locals.video
// We need more attributes
const video: VideoModel = await VideoModel.loadForGetAPI(res.locals.video.id)
if (video.url.startsWith(CONFIG.WEBSERVER.URL) === false) return res.redirect(video.url)

View File

@ -1,5 +1,5 @@
import * as express from 'express'
import { omit, snakeCase } from 'lodash'
import { snakeCase } from 'lodash'
import { ServerConfig, UserRight } from '../../../shared'
import { About } from '../../../shared/models/server/about.model'
import { CustomConfig } from '../../../shared/models/server/custom-config.model'
@ -78,6 +78,9 @@ async function getConfig (req: express.Request, res: express.Response) {
requiresEmailVerification: CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION
},
transcoding: {
hls: {
enabled: CONFIG.TRANSCODING.HLS.ENABLED
},
enabledResolutions
},
import: {
@ -246,6 +249,9 @@ function customConfig (): CustomConfig {
'480p': CONFIG.TRANSCODING.RESOLUTIONS[ '480p' ],
'720p': CONFIG.TRANSCODING.RESOLUTIONS[ '720p' ],
'1080p': CONFIG.TRANSCODING.RESOLUTIONS[ '1080p' ]
},
hls: {
enabled: CONFIG.TRANSCODING.HLS.ENABLED
}
},
import: {

View File

@ -37,6 +37,7 @@ import {
setDefaultPagination,
setDefaultSort,
videosAddValidator,
videosCustomGetValidator,
videosGetValidator,
videosRemoveValidator,
videosSortValidator,
@ -123,9 +124,9 @@ videosRouter.get('/:id/description',
)
videosRouter.get('/:id',
optionalAuthenticate,
asyncMiddleware(videosGetValidator),
asyncMiddleware(videosCustomGetValidator('only-video-with-rights')),
asyncMiddleware(checkVideoFollowConstraints),
getVideo
asyncMiddleware(getVideo)
)
videosRouter.post('/:id/views',
asyncMiddleware(videosGetValidator),
@ -395,15 +396,17 @@ async function updateVideo (req: express.Request, res: express.Response) {
return res.type('json').status(204).end()
}
function getVideo (req: express.Request, res: express.Response) {
const videoInstance = res.locals.video
async function getVideo (req: express.Request, res: express.Response) {
// We need more attributes
const userId: number = res.locals.oauth ? res.locals.oauth.token.User.id : null
const video: VideoModel = await VideoModel.loadForGetAPI(res.locals.video.id, undefined, userId)
if (videoInstance.isOutdated()) {
JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video', url: videoInstance.url } })
.catch(err => logger.error('Cannot create AP refresher job for video %s.', videoInstance.url, { err }))
if (video.isOutdated()) {
JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video', url: video.url } })
.catch(err => logger.error('Cannot create AP refresher job for video %s.', video.url, { err }))
}
return res.json(videoInstance.toFormattedDetailsJSON())
return res.json(video.toFormattedDetailsJSON())
}
async function viewVideo (req: express.Request, res: express.Response) {

View File

@ -1,6 +1,6 @@
import * as cors from 'cors'
import * as express from 'express'
import { CONFIG, ROUTE_CACHE_LIFETIME, STATIC_DOWNLOAD_PATHS, STATIC_MAX_AGE, STATIC_PATHS } from '../initializers'
import { CONFIG, HLS_PLAYLIST_DIRECTORY, ROUTE_CACHE_LIFETIME, STATIC_DOWNLOAD_PATHS, STATIC_MAX_AGE, STATIC_PATHS } from '../initializers'
import { VideosPreviewCache } from '../lib/cache'
import { cacheRoute } from '../middlewares/cache'
import { asyncMiddleware, videosGetValidator } from '../middlewares'
@ -51,6 +51,13 @@ staticRouter.use(
asyncMiddleware(downloadVideoFile)
)
// HLS
staticRouter.use(
STATIC_PATHS.PLAYLISTS.HLS,
cors(),
express.static(HLS_PLAYLIST_DIRECTORY, { fallthrough: false }) // 404 if the file does not exist
)
// Thumbnails path for express
const thumbnailsPhysicalPath = CONFIG.STORAGE.THUMBNAILS_DIR
staticRouter.use(

View File

@ -7,6 +7,7 @@ import { Server as WebSocketServer } from 'ws'
import { CONFIG, TRACKER_RATE_LIMITS } from '../initializers/constants'
import { VideoFileModel } from '../models/video/video-file'
import { parse } from 'url'
import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
const TrackerServer = bitTorrentTracker.Server
@ -21,7 +22,7 @@ const trackerServer = new TrackerServer({
udp: false,
ws: false,
dht: false,
filter: function (infoHash, params, cb) {
filter: async function (infoHash, params, cb) {
let ip: string
if (params.type === 'ws') {
@ -32,19 +33,25 @@ const trackerServer = new TrackerServer({
const key = ip + '-' + infoHash
peersIps[ip] = peersIps[ip] ? peersIps[ip] + 1 : 1
peersIpInfoHash[key] = peersIpInfoHash[key] ? peersIpInfoHash[key] + 1 : 1
peersIps[ ip ] = peersIps[ ip ] ? peersIps[ ip ] + 1 : 1
peersIpInfoHash[ key ] = peersIpInfoHash[ key ] ? peersIpInfoHash[ key ] + 1 : 1
if (peersIpInfoHash[key] > TRACKER_RATE_LIMITS.ANNOUNCES_PER_IP_PER_INFOHASH) {
if (peersIpInfoHash[ key ] > TRACKER_RATE_LIMITS.ANNOUNCES_PER_IP_PER_INFOHASH) {
return cb(new Error(`Too many requests (${peersIpInfoHash[ key ]} of ip ${ip} for torrent ${infoHash}`))
}
VideoFileModel.isInfohashExists(infoHash)
.then(exists => {
if (exists === false) return cb(new Error(`Unknown infoHash ${infoHash}`))
try {
const videoFileExists = await VideoFileModel.doesInfohashExist(infoHash)
if (videoFileExists === true) return cb()
return cb()
})
const playlistExists = await VideoStreamingPlaylistModel.doesInfohashExist(infoHash)
if (playlistExists === true) return cb()
return cb(new Error(`Unknown infoHash ${infoHash}`))
} catch (err) {
logger.error('Error in tracker filter.', { err })
return cb(err)
}
}
})

View File

@ -15,7 +15,7 @@ function activityPubContextify <T> (data: T) {
'https://w3id.org/security/v1',
{
RsaSignature2017: 'https://w3id.org/security#RsaSignature2017',
pt: 'https://joinpeertube.org/ns',
pt: 'https://joinpeertube.org/ns#',
sc: 'http://schema.org#',
Hashtag: 'as:Hashtag',
uuid: 'sc:identifier',
@ -32,7 +32,8 @@ function activityPubContextify <T> (data: T) {
waitTranscoding: 'sc:Boolean',
expires: 'sc:expires',
support: 'sc:Text',
CacheFile: 'pt:CacheFile'
CacheFile: 'pt:CacheFile',
Infohash: 'pt:Infohash'
},
{
likes: {

View File

@ -193,10 +193,14 @@ function peertubeTruncate (str: string, maxLength: number) {
return truncate(str, options)
}
function sha256 (str: string, encoding: HexBase64Latin1Encoding = 'hex') {
function sha256 (str: string | Buffer, encoding: HexBase64Latin1Encoding = 'hex') {
return createHash('sha256').update(str).digest(encoding)
}
function sha1 (str: string | Buffer, encoding: HexBase64Latin1Encoding = 'hex') {
return createHash('sha1').update(str).digest(encoding)
}
function promisify0<A> (func: (cb: (err: any, result: A) => void) => void): () => Promise<A> {
return function promisified (): Promise<A> {
return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => {
@ -262,7 +266,9 @@ export {
sanitizeHost,
buildPath,
peertubeTruncate,
sha256,
sha1,
promisify0,
promisify1,

View File

@ -8,9 +8,19 @@ function isCacheFileObjectValid (object: CacheFileObject) {
object.type === 'CacheFile' &&
isDateValid(object.expires) &&
isActivityPubUrlValid(object.object) &&
isRemoteVideoUrlValid(object.url)
(isRemoteVideoUrlValid(object.url) || isPlaylistRedundancyUrlValid(object.url))
}
// ---------------------------------------------------------------------------
export {
isCacheFileObjectValid
}
// ---------------------------------------------------------------------------
function isPlaylistRedundancyUrlValid (url: any) {
return url.type === 'Link' &&
(url.mediaType || url.mimeType) === 'application/x-mpegURL' &&
isActivityPubUrlValid(url.href)
}

View File

@ -1,7 +1,7 @@
import * as validator from 'validator'
import { ACTIVITY_PUB, CONSTRAINTS_FIELDS } from '../../../initializers'
import { peertubeTruncate } from '../../core-utils'
import { exists, isBooleanValid, isDateValid, isUUIDValid } from '../misc'
import { exists, isArray, isBooleanValid, isDateValid, isUUIDValid } from '../misc'
import {
isVideoDurationValid,
isVideoNameValid,
@ -12,7 +12,6 @@ import {
} from '../videos'
import { isActivityPubUrlValid, isBaseActivityValid, setValidAttributedTo } from './misc'
import { VideoState } from '../../../../shared/models/videos'
import { isVideoAbuseReasonValid } from '../video-abuses'
function sanitizeAndCheckVideoTorrentUpdateActivity (activity: any) {
return isBaseActivityValid(activity, 'Update') &&
@ -81,6 +80,11 @@ function isRemoteVideoUrlValid (url: any) {
ACTIVITY_PUB.URL_MIME_TYPES.MAGNET.indexOf(url.mediaType || url.mimeType) !== -1 &&
validator.isLength(url.href, { min: 5 }) &&
validator.isInt(url.height + '', { min: 0 })
) ||
(
(url.mediaType || url.mimeType) === 'application/x-mpegURL' &&
isActivityPubUrlValid(url.href) &&
isArray(url.tag)
)
}

View File

@ -13,6 +13,10 @@ function isNotEmptyIntArray (value: any) {
return Array.isArray(value) && value.every(v => validator.isInt('' + v)) && value.length !== 0
}
function isArrayOf (value: any, validator: (value: any) => boolean) {
return isArray(value) && value.every(v => validator(v))
}
function isDateValid (value: string) {
return exists(value) && validator.isISO8601(value)
}
@ -82,6 +86,7 @@ function isFileValid (
export {
exists,
isArrayOf,
isNotEmptyIntArray,
isArray,
isIdValid,

View File

@ -1,5 +1,5 @@
import * as ffmpeg from 'fluent-ffmpeg'
import { join } from 'path'
import { dirname, join } from 'path'
import { getTargetBitrate, VideoResolution } from '../../shared/models/videos'
import { CONFIG, FFMPEG_NICE, VIDEO_TRANSCODING_FPS } from '../initializers/constants'
import { processImage } from './image-utils'
@ -29,12 +29,21 @@ function computeResolutionsToTranscode (videoFileHeight: number) {
return resolutionsEnabled
}
async function getVideoFileResolution (path: string) {
async function getVideoFileSize (path: string) {
const videoStream = await getVideoFileStream(path)
return {
videoFileResolution: Math.min(videoStream.height, videoStream.width),
isPortraitMode: videoStream.height > videoStream.width
width: videoStream.width,
height: videoStream.height
}
}
async function getVideoFileResolution (path: string) {
const size = await getVideoFileSize(path)
return {
videoFileResolution: Math.min(size.height, size.width),
isPortraitMode: size.height > size.width
}
}
@ -110,8 +119,10 @@ async function generateImageFromVideoFile (fromPath: string, folder: string, ima
type TranscodeOptions = {
inputPath: string
outputPath: string
resolution?: VideoResolution
resolution: VideoResolution
isPortraitMode?: boolean
generateHlsPlaylist?: boolean
}
function transcode (options: TranscodeOptions) {
@ -150,6 +161,16 @@ function transcode (options: TranscodeOptions) {
command = command.withFPS(fps)
}
if (options.generateHlsPlaylist) {
const segmentFilename = `${dirname(options.outputPath)}/${options.resolution}_%03d.ts`
command = command.outputOption('-hls_time 4')
.outputOption('-hls_list_size 0')
.outputOption('-hls_playlist_type vod')
.outputOption('-hls_segment_filename ' + segmentFilename)
.outputOption('-f hls')
}
command
.on('error', (err, stdout, stderr) => {
logger.error('Error in transcoding job.', { stdout, stderr })
@ -166,6 +187,7 @@ function transcode (options: TranscodeOptions) {
// ---------------------------------------------------------------------------
export {
getVideoFileSize,
getVideoFileResolution,
getDurationFromVideoFile,
generateImageFromVideoFile,

View File

@ -1,10 +1,12 @@
import { VideoModel } from '../models/video/video'
type VideoFetchType = 'all' | 'only-video' | 'id' | 'none'
type VideoFetchType = 'all' | 'only-video' | 'only-video-with-rights' | 'id' | 'none'
function fetchVideo (id: number | string, fetchType: VideoFetchType, userId?: number) {
if (fetchType === 'all') return VideoModel.loadAndPopulateAccountAndServerAndTags(id, undefined, userId)
if (fetchType === 'only-video-with-rights') return VideoModel.loadWithRights(id)
if (fetchType === 'only-video') return VideoModel.load(id)
if (fetchType === 'id' || fetchType === 'none') return VideoModel.loadOnlyId(id)

View File

@ -12,7 +12,7 @@ function checkMissedConfig () {
'database.hostname', 'database.port', 'database.suffix', 'database.username', 'database.password', 'database.pool.max',
'smtp.hostname', 'smtp.port', 'smtp.username', 'smtp.password', 'smtp.tls', 'smtp.from_address',
'storage.avatars', 'storage.videos', 'storage.logs', 'storage.previews', 'storage.thumbnails', 'storage.torrents', 'storage.cache',
'storage.redundancy', 'storage.tmp',
'storage.redundancy', 'storage.tmp', 'storage.playlists',
'log.level',
'user.video_quota', 'user.video_quota_daily',
'cache.previews.size', 'admin.email', 'contact_form.enabled',

View File

@ -16,7 +16,7 @@ let config: IConfig = require('config')
// ---------------------------------------------------------------------------
const LAST_MIGRATION_VERSION = 325
const LAST_MIGRATION_VERSION = 330
// ---------------------------------------------------------------------------
@ -192,6 +192,7 @@ const CONFIG = {
AVATARS_DIR: buildPath(config.get<string>('storage.avatars')),
LOG_DIR: buildPath(config.get<string>('storage.logs')),
VIDEOS_DIR: buildPath(config.get<string>('storage.videos')),
PLAYLISTS_DIR: buildPath(config.get<string>('storage.playlists')),
REDUNDANCY_DIR: buildPath(config.get<string>('storage.redundancy')),
THUMBNAILS_DIR: buildPath(config.get<string>('storage.thumbnails')),
PREVIEWS_DIR: buildPath(config.get<string>('storage.previews')),
@ -259,6 +260,9 @@ const CONFIG = {
get '480p' () { return config.get<boolean>('transcoding.resolutions.480p') },
get '720p' () { return config.get<boolean>('transcoding.resolutions.720p') },
get '1080p' () { return config.get<boolean>('transcoding.resolutions.1080p') }
},
HLS: {
get ENABLED () { return config.get<boolean>('transcoding.hls.enabled') }
}
},
IMPORT: {
@ -590,6 +594,9 @@ const STATIC_PATHS = {
TORRENTS: '/static/torrents/',
WEBSEED: '/static/webseed/',
REDUNDANCY: '/static/redundancy/',
PLAYLISTS: {
HLS: '/static/playlists/hls'
},
AVATARS: '/static/avatars/',
VIDEO_CAPTIONS: '/static/video-captions/'
}
@ -632,6 +639,9 @@ const CACHE = {
}
}
const HLS_PLAYLIST_DIRECTORY = join(CONFIG.STORAGE.PLAYLISTS_DIR, 'hls')
const HLS_REDUNDANCY_DIRECTORY = join(CONFIG.STORAGE.REDUNDANCY_DIR, 'hls')
const MEMOIZE_TTL = {
OVERVIEWS_SAMPLE: 1000 * 3600 * 4 // 4 hours
}
@ -709,6 +719,7 @@ updateWebserverUrls()
export {
API_VERSION,
HLS_REDUNDANCY_DIRECTORY,
AVATARS_SIZE,
ACCEPT_HEADERS,
BCRYPT_SALT_SIZE,
@ -733,6 +744,7 @@ export {
PRIVATE_RSA_KEY_SIZE,
ROUTE_CACHE_LIFETIME,
SORTABLE_COLUMNS,
HLS_PLAYLIST_DIRECTORY,
FEEDS,
JOB_TTL,
NSFW_POLICY_TYPES,

View File

@ -33,6 +33,7 @@ import { AccountBlocklistModel } from '../models/account/account-blocklist'
import { ServerBlocklistModel } from '../models/server/server-blocklist'
import { UserNotificationModel } from '../models/account/user-notification'
import { UserNotificationSettingModel } from '../models/account/user-notification-setting'
import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string
@ -99,7 +100,8 @@ async function initDatabaseModels (silent: boolean) {
AccountBlocklistModel,
ServerBlocklistModel,
UserNotificationModel,
UserNotificationSettingModel
UserNotificationSettingModel,
VideoStreamingPlaylistModel
])
// Check extensions exist in the database

View File

@ -6,7 +6,7 @@ import { UserModel } from '../models/account/user'
import { ApplicationModel } from '../models/application/application'
import { OAuthClientModel } from '../models/oauth/oauth-client'
import { applicationExist, clientsExist, usersExist } from './checker-after-init'
import { CACHE, CONFIG, LAST_MIGRATION_VERSION } from './constants'
import { CACHE, CONFIG, HLS_PLAYLIST_DIRECTORY, LAST_MIGRATION_VERSION } from './constants'
import { sequelizeTypescript } from './database'
import { remove, ensureDir } from 'fs-extra'
@ -73,6 +73,9 @@ function createDirectoriesIfNotExist () {
tasks.push(ensureDir(dir))
}
// Playlist directories
tasks.push(ensureDir(HLS_PLAYLIST_DIRECTORY))
return Promise.all(tasks)
}

View File

@ -0,0 +1,51 @@
import * as Sequelize from 'sequelize'
async function up (utils: {
transaction: Sequelize.Transaction,
queryInterface: Sequelize.QueryInterface,
sequelize: Sequelize.Sequelize
}): Promise<void> {
{
const query = `
CREATE TABLE IF NOT EXISTS "videoStreamingPlaylist"
(
"id" SERIAL,
"type" INTEGER NOT NULL,
"playlistUrl" VARCHAR(2000) NOT NULL,
"p2pMediaLoaderInfohashes" VARCHAR(255)[] NOT NULL,
"segmentsSha256Url" VARCHAR(255) NOT NULL,
"videoId" INTEGER NOT NULL REFERENCES "video" ("id") ON DELETE CASCADE ON UPDATE CASCADE,
"createdAt" TIMESTAMP WITH TIME ZONE NOT NULL,
"updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL,
PRIMARY KEY ("id")
);`
await utils.sequelize.query(query)
}
{
const data = {
type: Sequelize.INTEGER,
allowNull: true,
defaultValue: null
}
await utils.queryInterface.changeColumn('videoRedundancy', 'videoFileId', data)
}
{
const query = 'ALTER TABLE "videoRedundancy" ADD COLUMN "videoStreamingPlaylistId" INTEGER NULL ' +
'REFERENCES "videoStreamingPlaylist" ("id") ON DELETE CASCADE ON UPDATE CASCADE'
await utils.sequelize.query(query)
}
}
function down (options) {
throw new Error('Not implemented.')
}
export {
up,
down
}

View File

@ -1,11 +1,28 @@
import { CacheFileObject } from '../../../shared/index'
import { ActivityPlaylistUrlObject, ActivityVideoUrlObject, CacheFileObject } from '../../../shared/index'
import { VideoModel } from '../../models/video/video'
import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
import { Transaction } from 'sequelize'
import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: VideoModel, byActor: { id?: number }) {
const url = cacheFileObject.url
if (cacheFileObject.url.mediaType === 'application/x-mpegURL') {
const url = cacheFileObject.url
const playlist = video.VideoStreamingPlaylists.find(t => t.type === VideoStreamingPlaylistType.HLS)
if (!playlist) throw new Error('Cannot find HLS playlist of video ' + video.url)
return {
expiresOn: new Date(cacheFileObject.expires),
url: cacheFileObject.id,
fileUrl: url.href,
strategy: null,
videoStreamingPlaylistId: playlist.id,
actorId: byActor.id
}
}
const url = cacheFileObject.url
const videoFile = video.VideoFiles.find(f => {
return f.resolution === url.height && f.fps === url.fps
})
@ -15,7 +32,7 @@ function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject
return {
expiresOn: new Date(cacheFileObject.expires),
url: cacheFileObject.id,
fileUrl: cacheFileObject.url.href,
fileUrl: url.href,
strategy: null,
videoFileId: videoFile.id,
actorId: byActor.id

View File

@ -1,6 +1,6 @@
import { Transaction } from 'sequelize'
import { ActivityAudience, ActivityCreate } from '../../../../shared/models/activitypub'
import { VideoPrivacy } from '../../../../shared/models/videos'
import { Video, VideoPrivacy } from '../../../../shared/models/videos'
import { ActorModel } from '../../../models/activitypub/actor'
import { VideoModel } from '../../../models/video/video'
import { VideoAbuseModel } from '../../../models/video/video-abuse'
@ -39,17 +39,14 @@ async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel,
return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
}
async function sendCreateCacheFile (byActor: ActorModel, fileRedundancy: VideoRedundancyModel) {
async function sendCreateCacheFile (byActor: ActorModel, video: VideoModel, fileRedundancy: VideoRedundancyModel) {
logger.info('Creating job to send file cache of %s.', fileRedundancy.url)
const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(fileRedundancy.VideoFile.Video.id)
const redundancyObject = fileRedundancy.toActivityPubObject()
return sendVideoRelatedCreateActivity({
byActor,
video,
url: fileRedundancy.url,
object: redundancyObject
object: fileRedundancy.toActivityPubObject()
})
}

View File

@ -73,7 +73,8 @@ async function sendUndoDislike (byActor: ActorModel, video: VideoModel, t: Trans
async function sendUndoCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel, t: Transaction) {
logger.info('Creating job to undo cache file %s.', redundancyModel.url)
const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id)
const videoId = redundancyModel.getVideo().id
const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId)
const createActivity = buildCreateActivity(redundancyModel.url, byActor, redundancyModel.toActivityPubObject())
return sendUndoVideoRelatedActivity({ byActor, video, url: redundancyModel.url, activity: createActivity, transaction: t })

View File

@ -61,7 +61,7 @@ async function sendUpdateActor (accountOrChannel: AccountModel | VideoChannelMod
async function sendUpdateCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel) {
logger.info('Creating job to update cache file %s.', redundancyModel.url)
const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id)
const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.getVideo().id)
const activityBuilder = (audience: ActivityAudience) => {
const redundancyObject = redundancyModel.toActivityPubObject()

View File

@ -5,6 +5,8 @@ import { VideoModel } from '../../models/video/video'
import { VideoAbuseModel } from '../../models/video/video-abuse'
import { VideoCommentModel } from '../../models/video/video-comment'
import { VideoFileModel } from '../../models/video/video-file'
import { VideoStreamingPlaylist } from '../../../shared/models/videos/video-streaming-playlist.model'
import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
function getVideoActivityPubUrl (video: VideoModel) {
return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid
@ -16,6 +18,10 @@ function getVideoCacheFileActivityPubUrl (videoFile: VideoFileModel) {
return `${CONFIG.WEBSERVER.URL}/redundancy/videos/${videoFile.Video.uuid}/${videoFile.resolution}${suffixFPS}`
}
function getVideoCacheStreamingPlaylistActivityPubUrl (video: VideoModel, playlist: VideoStreamingPlaylistModel) {
return `${CONFIG.WEBSERVER.URL}/redundancy/video-playlists/${playlist.getStringType()}/${video.uuid}`
}
function getVideoCommentActivityPubUrl (video: VideoModel, videoComment: VideoCommentModel) {
return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid + '/comments/' + videoComment.id
}
@ -92,6 +98,7 @@ function getUndoActivityPubUrl (originalUrl: string) {
export {
getVideoActivityPubUrl,
getVideoCacheStreamingPlaylistActivityPubUrl,
getVideoChannelActivityPubUrl,
getAccountActivityPubUrl,
getVideoAbuseActivityPubUrl,

View File

@ -2,7 +2,14 @@ import * as Bluebird from 'bluebird'
import * as sequelize from 'sequelize'
import * as magnetUtil from 'magnet-uri'
import * as request from 'request'
import { ActivityIconObject, ActivityUrlObject, ActivityVideoUrlObject, VideoState } from '../../../shared/index'
import {
ActivityIconObject,
ActivityPlaylistSegmentHashesObject,
ActivityPlaylistUrlObject,
ActivityUrlObject,
ActivityVideoUrlObject,
VideoState
} from '../../../shared/index'
import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
import { VideoPrivacy } from '../../../shared/models/videos'
import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos'
@ -30,6 +37,9 @@ import { AccountModel } from '../../models/account/account'
import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video'
import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
import { Notifier } from '../notifier'
import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model'
async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) {
// If the video is not private and published, we federate it
@ -263,6 +273,25 @@ async function updateVideoFromAP (options: {
options.video.VideoFiles = await Promise.all(upsertTasks)
}
{
const streamingPlaylistAttributes = streamingPlaylistActivityUrlToDBAttributes(options.video, options.videoObject)
const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a))
// Remove video files that do not exist anymore
const destroyTasks = options.video.VideoStreamingPlaylists
.filter(f => !newStreamingPlaylists.find(newPlaylist => newPlaylist.hasSameUniqueKeysThan(f)))
.map(f => f.destroy(sequelizeOptions))
await Promise.all(destroyTasks)
// Update or add other one
const upsertTasks = streamingPlaylistAttributes.map(a => {
return VideoStreamingPlaylistModel.upsert<VideoStreamingPlaylistModel>(a, { returning: true, transaction: t })
.then(([ streamingPlaylist ]) => streamingPlaylist)
})
options.video.VideoStreamingPlaylists = await Promise.all(upsertTasks)
}
{
// Update Tags
const tags = options.videoObject.tag.map(tag => tag.name)
@ -367,13 +396,25 @@ export {
// ---------------------------------------------------------------------------
function isActivityVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject {
function isAPVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject {
const mimeTypes = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT)
const urlMediaType = url.mediaType || url.mimeType
return mimeTypes.indexOf(urlMediaType) !== -1 && urlMediaType.startsWith('video/')
}
function isAPStreamingPlaylistUrlObject (url: ActivityUrlObject): url is ActivityPlaylistUrlObject {
const urlMediaType = url.mediaType || url.mimeType
return urlMediaType === 'application/x-mpegURL'
}
function isAPPlaylistSegmentHashesUrlObject (tag: any): tag is ActivityPlaylistSegmentHashesObject {
const urlMediaType = tag.mediaType || tag.mimeType
return tag.name === 'sha256' && tag.type === 'Link' && urlMediaType === 'application/json'
}
async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) {
logger.debug('Adding remote video %s.', videoObject.id)
@ -394,8 +435,14 @@ async function createVideo (videoObject: VideoTorrentObject, channelActor: Actor
const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t }))
await Promise.all(videoFilePromises)
const videoStreamingPlaylists = streamingPlaylistActivityUrlToDBAttributes(videoCreated, videoObject)
const playlistPromises = videoStreamingPlaylists.map(p => VideoStreamingPlaylistModel.create(p, { transaction: t }))
await Promise.all(playlistPromises)
// Process tags
const tags = videoObject.tag.map(t => t.name)
const tags = videoObject.tag
.filter(t => t.type === 'Hashtag')
.map(t => t.name)
const tagInstances = await TagModel.findOrCreateTags(tags, t)
await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
@ -473,13 +520,13 @@ async function videoActivityObjectToDBAttributes (
}
function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject) {
const fileUrls = videoObject.url.filter(u => isActivityVideoUrlObject(u)) as ActivityVideoUrlObject[]
const fileUrls = videoObject.url.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[]
if (fileUrls.length === 0) {
throw new Error('Cannot find video files for ' + video.url)
}
const attributes: VideoFileModel[] = []
const attributes: FilteredModelAttributes<VideoFileModel>[] = []
for (const fileUrl of fileUrls) {
// Fetch associated magnet uri
const magnet = videoObject.url.find(u => {
@ -502,7 +549,45 @@ function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: Vid
size: fileUrl.size,
videoId: video.id,
fps: fileUrl.fps || -1
} as VideoFileModel
}
attributes.push(attribute)
}
return attributes
}
function streamingPlaylistActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject) {
const playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u)) as ActivityPlaylistUrlObject[]
if (playlistUrls.length === 0) return []
const attributes: FilteredModelAttributes<VideoStreamingPlaylistModel>[] = []
for (const playlistUrlObject of playlistUrls) {
const p2pMediaLoaderInfohashes = playlistUrlObject.tag
.filter(t => t.type === 'Infohash')
.map(t => t.name)
if (p2pMediaLoaderInfohashes.length === 0) {
logger.warn('No infohashes found in AP playlist object.', { playlistUrl: playlistUrlObject })
continue
}
const segmentsSha256UrlObject = playlistUrlObject.tag
.find(t => {
return isAPPlaylistSegmentHashesUrlObject(t)
}) as ActivityPlaylistSegmentHashesObject
if (!segmentsSha256UrlObject) {
logger.warn('No segment sha256 URL found in AP playlist object.', { playlistUrl: playlistUrlObject })
continue
}
const attribute = {
type: VideoStreamingPlaylistType.HLS,
playlistUrl: playlistUrlObject.href,
segmentsSha256Url: segmentsSha256UrlObject.href,
p2pMediaLoaderInfohashes,
videoId: video.id
}
attributes.push(attribute)
}

110
server/lib/hls.ts Normal file
View File

@ -0,0 +1,110 @@
import { VideoModel } from '../models/video/video'
import { basename, dirname, join } from 'path'
import { HLS_PLAYLIST_DIRECTORY, CONFIG } from '../initializers'
import { outputJSON, pathExists, readdir, readFile, remove, writeFile, move } from 'fs-extra'
import { getVideoFileSize } from '../helpers/ffmpeg-utils'
import { sha256 } from '../helpers/core-utils'
import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
import HLSDownloader from 'hlsdownloader'
import { logger } from '../helpers/logger'
import { parse } from 'url'
async function updateMasterHLSPlaylist (video: VideoModel) {
const directory = join(HLS_PLAYLIST_DIRECTORY, video.uuid)
const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ]
const masterPlaylistPath = join(directory, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename())
for (const file of video.VideoFiles) {
// If we did not generated a playlist for this resolution, skip
const filePlaylistPath = join(directory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution))
if (await pathExists(filePlaylistPath) === false) continue
const videoFilePath = video.getVideoFilePath(file)
const size = await getVideoFileSize(videoFilePath)
const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file)
const resolution = `RESOLUTION=${size.width}x${size.height}`
let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}`
if (file.fps) line += ',FRAME-RATE=' + file.fps
masterPlaylists.push(line)
masterPlaylists.push(VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution))
}
await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n')
}
async function updateSha256Segments (video: VideoModel) {
const directory = join(HLS_PLAYLIST_DIRECTORY, video.uuid)
const files = await readdir(directory)
const json: { [filename: string]: string} = {}
for (const file of files) {
if (file.endsWith('.ts') === false) continue
const buffer = await readFile(join(directory, file))
const filename = basename(file)
json[filename] = sha256(buffer)
}
const outputPath = join(directory, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename())
await outputJSON(outputPath, json)
}
function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number) {
let timer
logger.info('Importing HLS playlist %s', playlistUrl)
const params = {
playlistURL: playlistUrl,
destination: CONFIG.STORAGE.TMP_DIR
}
const downloader = new HLSDownloader(params)
const hlsDestinationDir = join(CONFIG.STORAGE.TMP_DIR, dirname(parse(playlistUrl).pathname))
return new Promise<string>(async (res, rej) => {
downloader.startDownload(err => {
clearTimeout(timer)
if (err) {
deleteTmpDirectory(hlsDestinationDir)
return rej(err)
}
move(hlsDestinationDir, destinationDir, { overwrite: true })
.then(() => res())
.catch(err => {
deleteTmpDirectory(hlsDestinationDir)
return rej(err)
})
})
timer = setTimeout(() => {
deleteTmpDirectory(hlsDestinationDir)
return rej(new Error('HLS download timeout.'))
}, timeout)
function deleteTmpDirectory (directory: string) {
remove(directory)
.catch(err => logger.error('Cannot delete path on HLS download error.', { err }))
}
})
}
// ---------------------------------------------------------------------------
export {
updateMasterHLSPlaylist,
updateSha256Segments,
downloadPlaylistSegments
}
// ---------------------------------------------------------------------------

View File

@ -5,17 +5,18 @@ import { VideoModel } from '../../../models/video/video'
import { JobQueue } from '../job-queue'
import { federateVideoIfNeeded } from '../../activitypub'
import { retryTransactionWrapper } from '../../../helpers/database-utils'
import { sequelizeTypescript } from '../../../initializers'
import { sequelizeTypescript, CONFIG } from '../../../initializers'
import * as Bluebird from 'bluebird'
import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg-utils'
import { importVideoFile, optimizeVideofile, transcodeOriginalVideofile } from '../../video-transcoding'
import { generateHlsPlaylist, importVideoFile, optimizeVideofile, transcodeOriginalVideofile } from '../../video-transcoding'
import { Notifier } from '../../notifier'
export type VideoFilePayload = {
videoUUID: string
isNewVideo?: boolean
resolution?: VideoResolution
isNewVideo?: boolean
isPortraitMode?: boolean
generateHlsPlaylist?: boolean
}
export type VideoFileImportPayload = {
@ -51,21 +52,38 @@ async function processVideoFile (job: Bull.Job) {
return undefined
}
// Transcoding in other resolution
if (payload.resolution) {
if (payload.generateHlsPlaylist) {
await generateHlsPlaylist(video, payload.resolution, payload.isPortraitMode || false)
await retryTransactionWrapper(onHlsPlaylistGenerationSuccess, video)
} else if (payload.resolution) { // Transcoding in other resolution
await transcodeOriginalVideofile(video, payload.resolution, payload.isPortraitMode || false)
await retryTransactionWrapper(onVideoFileTranscoderOrImportSuccess, video)
await retryTransactionWrapper(onVideoFileTranscoderOrImportSuccess, video, payload)
} else {
await optimizeVideofile(video)
await retryTransactionWrapper(onVideoFileOptimizerSuccess, video, payload.isNewVideo)
await retryTransactionWrapper(onVideoFileOptimizerSuccess, video, payload)
}
return video
}
async function onVideoFileTranscoderOrImportSuccess (video: VideoModel) {
async function onHlsPlaylistGenerationSuccess (video: VideoModel) {
if (video === undefined) return undefined
await sequelizeTypescript.transaction(async t => {
// Maybe the video changed in database, refresh it
let videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t)
// Video does not exist anymore
if (!videoDatabase) return undefined
// If the video was not published, we consider it is a new one for other instances
await federateVideoIfNeeded(videoDatabase, false, t)
})
}
async function onVideoFileTranscoderOrImportSuccess (video: VideoModel, payload?: VideoFilePayload) {
if (video === undefined) return undefined
const { videoDatabase, videoPublished } = await sequelizeTypescript.transaction(async t => {
@ -96,9 +114,11 @@ async function onVideoFileTranscoderOrImportSuccess (video: VideoModel) {
Notifier.Instance.notifyOnNewVideo(videoDatabase)
Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase)
}
await createHlsJobIfEnabled(payload)
}
async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: boolean) {
async function onVideoFileOptimizerSuccess (videoArg: VideoModel, payload: VideoFilePayload) {
if (videoArg === undefined) return undefined
// Outside the transaction (IO on disk)
@ -145,7 +165,7 @@ async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: bo
logger.info('No transcoding jobs created for video %s (no resolutions).', videoDatabase.uuid, { privacy: videoDatabase.privacy })
}
await federateVideoIfNeeded(videoDatabase, isNewVideo, t)
await federateVideoIfNeeded(videoDatabase, payload.isNewVideo, t)
return { videoDatabase, videoPublished }
})
@ -155,6 +175,8 @@ async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: bo
if (isNewVideo) Notifier.Instance.notifyOnNewVideo(videoDatabase)
if (videoPublished) Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase)
}
await createHlsJobIfEnabled(Object.assign({}, payload, { resolution: videoDatabase.getOriginalFile().resolution }))
}
// ---------------------------------------------------------------------------
@ -163,3 +185,20 @@ export {
processVideoFile,
processVideoFileImport
}
// ---------------------------------------------------------------------------
function createHlsJobIfEnabled (payload?: VideoFilePayload) {
// Generate HLS playlist?
if (payload && CONFIG.TRANSCODING.HLS.ENABLED) {
const hlsTranscodingPayload = {
videoUUID: payload.videoUUID,
resolution: payload.resolution,
isPortraitMode: payload.isPortraitMode,
generateHlsPlaylist: true
}
return JobQueue.Instance.createJob({ type: 'video-file', payload: hlsTranscodingPayload })
}
}

View File

@ -1,5 +1,5 @@
import { AbstractScheduler } from './abstract-scheduler'
import { CONFIG, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers'
import { CONFIG, HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers'
import { logger } from '../../helpers/logger'
import { VideosRedundancy } from '../../../shared/models/redundancy'
import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
@ -9,9 +9,19 @@ import { join } from 'path'
import { move } from 'fs-extra'
import { getServerActor } from '../../helpers/utils'
import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send'
import { getVideoCacheFileActivityPubUrl } from '../activitypub/url'
import { getVideoCacheFileActivityPubUrl, getVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url'
import { removeVideoRedundancy } from '../redundancy'
import { getOrCreateVideoAndAccountAndChannel } from '../activitypub'
import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
import { VideoModel } from '../../models/video/video'
import { downloadPlaylistSegments } from '../hls'
type CandidateToDuplicate = {
redundancy: VideosRedundancy,
video: VideoModel,
files: VideoFileModel[],
streamingPlaylists: VideoStreamingPlaylistModel[]
}
export class VideosRedundancyScheduler extends AbstractScheduler {
@ -24,28 +34,32 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
}
protected async internalExecute () {
for (const obj of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) {
logger.info('Running redundancy scheduler for strategy %s.', obj.strategy)
for (const redundancyConfig of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) {
logger.info('Running redundancy scheduler for strategy %s.', redundancyConfig.strategy)
try {
const videoToDuplicate = await this.findVideoToDuplicate(obj)
const videoToDuplicate = await this.findVideoToDuplicate(redundancyConfig)
if (!videoToDuplicate) continue
const videoFiles = videoToDuplicate.VideoFiles
videoFiles.forEach(f => f.Video = videoToDuplicate)
const candidateToDuplicate = {
video: videoToDuplicate,
redundancy: redundancyConfig,
files: videoToDuplicate.VideoFiles,
streamingPlaylists: videoToDuplicate.VideoStreamingPlaylists
}
await this.purgeCacheIfNeeded(obj, videoFiles)
await this.purgeCacheIfNeeded(candidateToDuplicate)
if (await this.isTooHeavy(obj, videoFiles)) {
if (await this.isTooHeavy(candidateToDuplicate)) {
logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url)
continue
}
logger.info('Will duplicate video %s in redundancy scheduler "%s".', videoToDuplicate.url, obj.strategy)
logger.info('Will duplicate video %s in redundancy scheduler "%s".', videoToDuplicate.url, redundancyConfig.strategy)
await this.createVideoRedundancy(obj, videoFiles)
await this.createVideoRedundancies(candidateToDuplicate)
} catch (err) {
logger.error('Cannot run videos redundancy %s.', obj.strategy, { err })
logger.error('Cannot run videos redundancy %s.', redundancyConfig.strategy, { err })
}
}
@ -63,25 +77,35 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
for (const redundancyModel of expired) {
try {
await this.extendsOrDeleteRedundancy(redundancyModel)
const redundancyConfig = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy)
const candidate = {
redundancy: redundancyConfig,
video: null,
files: [],
streamingPlaylists: []
}
// If the administrator disabled the redundancy or decreased the cache size, remove this redundancy instead of extending it
if (!redundancyConfig || await this.isTooHeavy(candidate)) {
logger.info('Destroying redundancy %s because the cache size %s is too heavy.', redundancyModel.url, redundancyModel.strategy)
await removeVideoRedundancy(redundancyModel)
} else {
await this.extendsRedundancy(redundancyModel)
}
} catch (err) {
logger.error('Cannot extend expiration of %s video from our redundancy system.', this.buildEntryLogId(redundancyModel))
logger.error(
'Cannot extend or remove expiration of %s video from our redundancy system.', this.buildEntryLogId(redundancyModel),
{ err }
)
}
}
}
private async extendsOrDeleteRedundancy (redundancyModel: VideoRedundancyModel) {
// Refresh the video, maybe it was deleted
const video = await this.loadAndRefreshVideo(redundancyModel.VideoFile.Video.url)
if (!video) {
logger.info('Destroying existing redundancy %s, because the associated video does not exist anymore.', redundancyModel.url)
await redundancyModel.destroy()
return
}
private async extendsRedundancy (redundancyModel: VideoRedundancyModel) {
const redundancy = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy)
// Redundancy strategy disabled, remove our redundancy instead of extending expiration
if (!redundancy) await removeVideoRedundancy(redundancyModel)
await this.extendsExpirationOf(redundancyModel, redundancy.minLifetime)
}
@ -112,49 +136,93 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
}
}
private async createVideoRedundancy (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) {
const serverActor = await getServerActor()
private async createVideoRedundancies (data: CandidateToDuplicate) {
const video = await this.loadAndRefreshVideo(data.video.url)
for (const file of filesToDuplicate) {
const video = await this.loadAndRefreshVideo(file.Video.url)
if (!video) {
logger.info('Video %s we want to duplicate does not existing anymore, skipping.', data.video.url)
return
}
for (const file of data.files) {
const existingRedundancy = await VideoRedundancyModel.loadLocalByFileId(file.id)
if (existingRedundancy) {
await this.extendsOrDeleteRedundancy(existingRedundancy)
await this.extendsRedundancy(existingRedundancy)
continue
}
if (!video) {
logger.info('Video %s we want to duplicate does not existing anymore, skipping.', file.Video.url)
continue
}
logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy)
const { baseUrlHttp, baseUrlWs } = video.getBaseUrls()
const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs)
const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT)
const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, video.getVideoFilename(file))
await move(tmpPath, destPath)
const createdModel = await VideoRedundancyModel.create({
expiresOn: this.buildNewExpiration(redundancy.minLifetime),
url: getVideoCacheFileActivityPubUrl(file),
fileUrl: video.getVideoRedundancyUrl(file, CONFIG.WEBSERVER.URL),
strategy: redundancy.strategy,
videoFileId: file.id,
actorId: serverActor.id
})
createdModel.VideoFile = file
await sendCreateCacheFile(serverActor, createdModel)
logger.info('Duplicated %s - %d -> %s.', video.url, file.resolution, createdModel.url)
await this.createVideoFileRedundancy(data.redundancy, video, file)
}
for (const streamingPlaylist of data.streamingPlaylists) {
const existingRedundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(streamingPlaylist.id)
if (existingRedundancy) {
await this.extendsRedundancy(existingRedundancy)
continue
}
await this.createStreamingPlaylistRedundancy(data.redundancy, video, streamingPlaylist)
}
}
private async createVideoFileRedundancy (redundancy: VideosRedundancy, video: VideoModel, file: VideoFileModel) {
file.Video = video
const serverActor = await getServerActor()
logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy)
const { baseUrlHttp, baseUrlWs } = video.getBaseUrls()
const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs)
const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT)
const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, video.getVideoFilename(file))
await move(tmpPath, destPath)
const createdModel = await VideoRedundancyModel.create({
expiresOn: this.buildNewExpiration(redundancy.minLifetime),
url: getVideoCacheFileActivityPubUrl(file),
fileUrl: video.getVideoRedundancyUrl(file, CONFIG.WEBSERVER.URL),
strategy: redundancy.strategy,
videoFileId: file.id,
actorId: serverActor.id
})
createdModel.VideoFile = file
await sendCreateCacheFile(serverActor, video, createdModel)
logger.info('Duplicated %s - %d -> %s.', video.url, file.resolution, createdModel.url)
}
private async createStreamingPlaylistRedundancy (redundancy: VideosRedundancy, video: VideoModel, playlist: VideoStreamingPlaylistModel) {
playlist.Video = video
const serverActor = await getServerActor()
logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, redundancy.strategy)
const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid)
await downloadPlaylistSegments(playlist.playlistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT)
const createdModel = await VideoRedundancyModel.create({
expiresOn: this.buildNewExpiration(redundancy.minLifetime),
url: getVideoCacheStreamingPlaylistActivityPubUrl(video, playlist),
fileUrl: playlist.getVideoRedundancyUrl(CONFIG.WEBSERVER.URL),
strategy: redundancy.strategy,
videoStreamingPlaylistId: playlist.id,
actorId: serverActor.id
})
createdModel.VideoStreamingPlaylist = playlist
await sendCreateCacheFile(serverActor, video, createdModel)
logger.info('Duplicated playlist %s -> %s.', playlist.playlistUrl, createdModel.url)
}
private async extendsExpirationOf (redundancy: VideoRedundancyModel, expiresAfterMs: number) {
@ -168,8 +236,9 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
await sendUpdateCacheFile(serverActor, redundancy)
}
private async purgeCacheIfNeeded (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) {
while (this.isTooHeavy(redundancy, filesToDuplicate)) {
private async purgeCacheIfNeeded (candidateToDuplicate: CandidateToDuplicate) {
while (this.isTooHeavy(candidateToDuplicate)) {
const redundancy = candidateToDuplicate.redundancy
const toDelete = await VideoRedundancyModel.loadOldestLocalThatAlreadyExpired(redundancy.strategy, redundancy.minLifetime)
if (!toDelete) return
@ -177,11 +246,11 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
}
}
private async isTooHeavy (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) {
const maxSize = redundancy.size
private async isTooHeavy (candidateToDuplicate: CandidateToDuplicate) {
const maxSize = candidateToDuplicate.redundancy.size
const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(redundancy.strategy)
const totalWillDuplicate = totalDuplicated + this.getTotalFileSizes(filesToDuplicate)
const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(candidateToDuplicate.redundancy.strategy)
const totalWillDuplicate = totalDuplicated + this.getTotalFileSizes(candidateToDuplicate.files, candidateToDuplicate.streamingPlaylists)
return totalWillDuplicate > maxSize
}
@ -191,13 +260,15 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
}
private buildEntryLogId (object: VideoRedundancyModel) {
return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}`
if (object.VideoFile) return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}`
return `${object.VideoStreamingPlaylist.playlistUrl}`
}
private getTotalFileSizes (files: VideoFileModel[]) {
private getTotalFileSizes (files: VideoFileModel[], playlists: VideoStreamingPlaylistModel[]) {
const fileReducer = (previous: number, current: VideoFileModel) => previous + current.size
return files.reduce(fileReducer, 0)
return files.reduce(fileReducer, 0) * playlists.length
}
private async loadAndRefreshVideo (videoUrl: string) {

View File

@ -1,11 +1,14 @@
import { CONFIG } from '../initializers'
import { CONFIG, HLS_PLAYLIST_DIRECTORY } from '../initializers'
import { extname, join } from 'path'
import { getVideoFileFPS, getVideoFileResolution, transcode } from '../helpers/ffmpeg-utils'
import { copy, remove, move, stat } from 'fs-extra'
import { copy, ensureDir, move, remove, stat } from 'fs-extra'
import { logger } from '../helpers/logger'
import { VideoResolution } from '../../shared/models/videos'
import { VideoFileModel } from '../models/video/video-file'
import { VideoModel } from '../models/video/video'
import { updateMasterHLSPlaylist, updateSha256Segments } from './hls'
import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
import { VideoStreamingPlaylistType } from '../../shared/models/videos/video-streaming-playlist.type'
async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFileModel) {
const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR
@ -17,7 +20,8 @@ async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFi
const transcodeOptions = {
inputPath: videoInputPath,
outputPath: videoTranscodedPath
outputPath: videoTranscodedPath,
resolution: inputVideoFile.resolution
}
// Could be very long!
@ -47,7 +51,7 @@ async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFi
}
}
async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoResolution, isPortraitMode: boolean) {
async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoResolution, isPortrait: boolean) {
const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR
const extname = '.mp4'
@ -60,13 +64,13 @@ async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoR
size: 0,
videoId: video.id
})
const videoOutputPath = join(videosDirectory, video.getVideoFilename(newVideoFile))
const videoOutputPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(newVideoFile))
const transcodeOptions = {
inputPath: videoInputPath,
outputPath: videoOutputPath,
resolution,
isPortraitMode
isPortraitMode: isPortrait
}
await transcode(transcodeOptions)
@ -84,6 +88,38 @@ async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoR
video.VideoFiles.push(newVideoFile)
}
async function generateHlsPlaylist (video: VideoModel, resolution: VideoResolution, isPortraitMode: boolean) {
const baseHlsDirectory = join(HLS_PLAYLIST_DIRECTORY, video.uuid)
await ensureDir(join(HLS_PLAYLIST_DIRECTORY, video.uuid))
const videoInputPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(video.getOriginalFile()))
const outputPath = join(baseHlsDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(resolution))
const transcodeOptions = {
inputPath: videoInputPath,
outputPath,
resolution,
isPortraitMode,
generateHlsPlaylist: true
}
await transcode(transcodeOptions)
await updateMasterHLSPlaylist(video)
await updateSha256Segments(video)
const playlistUrl = CONFIG.WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid)
await VideoStreamingPlaylistModel.upsert({
videoId: video.id,
playlistUrl,
segmentsSha256Url: CONFIG.WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid),
p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrl, video.VideoFiles),
type: VideoStreamingPlaylistType.HLS
})
}
async function importVideoFile (video: VideoModel, inputFilePath: string) {
const { videoFileResolution } = await getVideoFileResolution(inputFilePath)
const { size } = await stat(inputFilePath)
@ -125,6 +161,7 @@ async function importVideoFile (video: VideoModel, inputFilePath: string) {
}
export {
generateHlsPlaylist,
optimizeVideofile,
transcodeOriginalVideofile,
importVideoFile

View File

@ -13,7 +13,7 @@ import { ActorFollowModel } from '../../models/activitypub/actor-follow'
import { SERVER_ACTOR_NAME } from '../../initializers'
import { ServerModel } from '../../models/server/server'
const videoRedundancyGetValidator = [
const videoFileRedundancyGetValidator = [
param('videoId').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid video id'),
param('resolution')
.customSanitizer(toIntOrNull)
@ -24,7 +24,7 @@ const videoRedundancyGetValidator = [
.custom(exists).withMessage('Should have a valid fps'),
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
logger.debug('Checking videoRedundancyGetValidator parameters', { parameters: req.params })
logger.debug('Checking videoFileRedundancyGetValidator parameters', { parameters: req.params })
if (areValidationErrors(req, res)) return
if (!await isVideoExist(req.params.videoId, res)) return
@ -38,7 +38,31 @@ const videoRedundancyGetValidator = [
res.locals.videoFile = videoFile
const videoRedundancy = await VideoRedundancyModel.loadLocalByFileId(videoFile.id)
if (!videoRedundancy)return res.status(404).json({ error: 'Video redundancy not found.' })
if (!videoRedundancy) return res.status(404).json({ error: 'Video redundancy not found.' })
res.locals.videoRedundancy = videoRedundancy
return next()
}
]
const videoPlaylistRedundancyGetValidator = [
param('videoId').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid video id'),
param('streamingPlaylistType').custom(exists).withMessage('Should have a valid streaming playlist type'),
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
logger.debug('Checking videoPlaylistRedundancyGetValidator parameters', { parameters: req.params })
if (areValidationErrors(req, res)) return
if (!await isVideoExist(req.params.videoId, res)) return
const video: VideoModel = res.locals.video
const videoStreamingPlaylist = video.VideoStreamingPlaylists.find(p => p === req.params.streamingPlaylistType)
if (!videoStreamingPlaylist) return res.status(404).json({ error: 'Video playlist not found.' })
res.locals.videoStreamingPlaylist = videoStreamingPlaylist
const videoRedundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(videoStreamingPlaylist.id)
if (!videoRedundancy) return res.status(404).json({ error: 'Video redundancy not found.' })
res.locals.videoRedundancy = videoRedundancy
return next()
@ -75,6 +99,7 @@ const updateServerRedundancyValidator = [
// ---------------------------------------------------------------------------
export {
videoRedundancyGetValidator,
videoFileRedundancyGetValidator,
videoPlaylistRedundancyGetValidator,
updateServerRedundancyValidator
}

View File

@ -28,6 +28,7 @@ import { sample } from 'lodash'
import { isTestInstance } from '../../helpers/core-utils'
import * as Bluebird from 'bluebird'
import * as Sequelize from 'sequelize'
import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist'
export enum ScopeNames {
WITH_VIDEO = 'WITH_VIDEO'
@ -38,7 +39,17 @@ export enum ScopeNames {
include: [
{
model: () => VideoFileModel,
required: true,
required: false,
include: [
{
model: () => VideoModel,
required: true
}
]
},
{
model: () => VideoStreamingPlaylistModel,
required: false,
include: [
{
model: () => VideoModel,
@ -97,12 +108,24 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
@BelongsTo(() => VideoFileModel, {
foreignKey: {
allowNull: false
allowNull: true
},
onDelete: 'cascade'
})
VideoFile: VideoFileModel
@ForeignKey(() => VideoStreamingPlaylistModel)
@Column
videoStreamingPlaylistId: number
@BelongsTo(() => VideoStreamingPlaylistModel, {
foreignKey: {
allowNull: true
},
onDelete: 'cascade'
})
VideoStreamingPlaylist: VideoStreamingPlaylistModel
@ForeignKey(() => ActorModel)
@Column
actorId: number
@ -119,13 +142,25 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
static async removeFile (instance: VideoRedundancyModel) {
if (!instance.isOwned()) return
const videoFile = await VideoFileModel.loadWithVideo(instance.videoFileId)
if (instance.videoFileId) {
const videoFile = await VideoFileModel.loadWithVideo(instance.videoFileId)
const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}`
logger.info('Removing duplicated video file %s.', logIdentifier)
const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}`
logger.info('Removing duplicated video file %s.', logIdentifier)
videoFile.Video.removeFile(videoFile, true)
.catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
videoFile.Video.removeFile(videoFile, true)
.catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
}
if (instance.videoStreamingPlaylistId) {
const videoStreamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(instance.videoStreamingPlaylistId)
const videoUUID = videoStreamingPlaylist.Video.uuid
logger.info('Removing duplicated video streaming playlist %s.', videoUUID)
videoStreamingPlaylist.Video.removeStreamingPlaylist(true)
.catch(err => logger.error('Cannot delete video streaming playlist files of %s.', videoUUID, { err }))
}
return undefined
}
@ -143,6 +178,19 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
}
static async loadLocalByStreamingPlaylistId (videoStreamingPlaylistId: number) {
const actor = await getServerActor()
const query = {
where: {
actorId: actor.id,
videoStreamingPlaylistId
}
}
return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
}
static loadByUrl (url: string, transaction?: Sequelize.Transaction) {
const query = {
where: {
@ -191,7 +239,7 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
const ids = rows.map(r => r.id)
const id = sample(ids)
return VideoModel.loadWithFile(id, undefined, !isTestInstance())
return VideoModel.loadWithFiles(id, undefined, !isTestInstance())
}
static async findMostViewToDuplicate (randomizedFactor: number) {
@ -333,6 +381,27 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
static async listLocalOfServer (serverId: number) {
const actor = await getServerActor()
const buildVideoInclude = () => ({
model: VideoModel,
required: true,
include: [
{
attributes: [],
model: VideoChannelModel.unscoped(),
required: true,
include: [
{
attributes: [],
model: ActorModel.unscoped(),
required: true,
where: {
serverId
}
}
]
}
]
})
const query = {
where: {
@ -341,30 +410,13 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
include: [
{
model: VideoFileModel,
required: true,
include: [
{
model: VideoModel,
required: true,
include: [
{
attributes: [],
model: VideoChannelModel.unscoped(),
required: true,
include: [
{
attributes: [],
model: ActorModel.unscoped(),
required: true,
where: {
serverId
}
}
]
}
]
}
]
required: false,
include: [ buildVideoInclude() ]
},
{
model: VideoStreamingPlaylistModel,
required: false,
include: [ buildVideoInclude() ]
}
]
}
@ -403,11 +455,32 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
}))
}
getVideo () {
if (this.VideoFile) return this.VideoFile.Video
return this.VideoStreamingPlaylist.Video
}
isOwned () {
return !!this.strategy
}
toActivityPubObject (): CacheFileObject {
if (this.VideoStreamingPlaylist) {
return {
id: this.url,
type: 'CacheFile' as 'CacheFile',
object: this.VideoStreamingPlaylist.Video.url,
expires: this.expiresOn.toISOString(),
url: {
type: 'Link',
mimeType: 'application/x-mpegURL',
mediaType: 'application/x-mpegURL',
href: this.fileUrl
}
}
}
return {
id: this.url,
type: 'CacheFile' as 'CacheFile',
@ -431,7 +504,7 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
const notIn = Sequelize.literal(
'(' +
`SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id}` +
`SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "videoFileId" IS NOT NULL` +
')'
)

View File

@ -62,7 +62,7 @@ export class VideoFileModel extends Model<VideoFileModel> {
extname: string
@AllowNull(false)
@Is('VideoFileSize', value => throwIfNotValid(value, isVideoFileInfoHashValid, 'info hash'))
@Is('VideoFileInfohash', value => throwIfNotValid(value, isVideoFileInfoHashValid, 'info hash'))
@Column
infoHash: string
@ -86,14 +86,14 @@ export class VideoFileModel extends Model<VideoFileModel> {
@HasMany(() => VideoRedundancyModel, {
foreignKey: {
allowNull: false
allowNull: true
},
onDelete: 'CASCADE',
hooks: true
})
RedundancyVideos: VideoRedundancyModel[]
static isInfohashExists (infoHash: string) {
static doesInfohashExist (infoHash: string) {
const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1'
const options = {
type: Sequelize.QueryTypes.SELECT,

View File

@ -1,7 +1,12 @@
import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos'
import { VideoModel } from './video'
import { VideoFileModel } from './video-file'
import { ActivityUrlObject, VideoTorrentObject } from '../../../shared/models/activitypub/objects'
import {
ActivityPlaylistInfohashesObject,
ActivityPlaylistSegmentHashesObject,
ActivityUrlObject,
VideoTorrentObject
} from '../../../shared/models/activitypub/objects'
import { CONFIG, MIMETYPES, THUMBNAILS_SIZE } from '../../initializers'
import { VideoCaptionModel } from './video-caption'
import {
@ -11,6 +16,8 @@ import {
getVideoSharesActivityPubUrl
} from '../../lib/activitypub'
import { isArray } from '../../helpers/custom-validators/misc'
import { VideoStreamingPlaylist } from '../../../shared/models/videos/video-streaming-playlist.model'
import { VideoStreamingPlaylistModel } from './video-streaming-playlist'
export type VideoFormattingJSONOptions = {
completeDescription?: boolean
@ -120,7 +127,12 @@ function videoModelToFormattedDetailsJSON (video: VideoModel): VideoDetails {
}
})
const { baseUrlHttp, baseUrlWs } = video.getBaseUrls()
const tags = video.Tags ? video.Tags.map(t => t.name) : []
const streamingPlaylists = streamingPlaylistsModelToFormattedJSON(video, video.VideoStreamingPlaylists)
const detailsJson = {
support: video.support,
descriptionPath: video.getDescriptionAPIPath(),
@ -133,7 +145,11 @@ function videoModelToFormattedDetailsJSON (video: VideoModel): VideoDetails {
id: video.state,
label: VideoModel.getStateLabel(video.state)
},
files: []
trackerUrls: video.getTrackerUrls(baseUrlHttp, baseUrlWs),
files: [],
streamingPlaylists
}
// Format and sort video files
@ -142,6 +158,25 @@ function videoModelToFormattedDetailsJSON (video: VideoModel): VideoDetails {
return Object.assign(formattedJson, detailsJson)
}
function streamingPlaylistsModelToFormattedJSON (video: VideoModel, playlists: VideoStreamingPlaylistModel[]): VideoStreamingPlaylist[] {
if (isArray(playlists) === false) return []
return playlists
.map(playlist => {
const redundancies = isArray(playlist.RedundancyVideos)
? playlist.RedundancyVideos.map(r => ({ baseUrl: r.fileUrl }))
: []
return {
id: playlist.id,
type: playlist.type,
playlistUrl: playlist.playlistUrl,
segmentsSha256Url: playlist.segmentsSha256Url,
redundancies
} as VideoStreamingPlaylist
})
}
function videoFilesModelToFormattedJSON (video: VideoModel, videoFiles: VideoFileModel[]): VideoFile[] {
const { baseUrlHttp, baseUrlWs } = video.getBaseUrls()
@ -232,6 +267,28 @@ function videoModelToActivityPubObject (video: VideoModel): VideoTorrentObject {
})
}
for (const playlist of (video.VideoStreamingPlaylists || [])) {
let tag: (ActivityPlaylistSegmentHashesObject | ActivityPlaylistInfohashesObject)[]
tag = playlist.p2pMediaLoaderInfohashes
.map(i => ({ type: 'Infohash' as 'Infohash', name: i }))
tag.push({
type: 'Link',
name: 'sha256',
mimeType: 'application/json' as 'application/json',
mediaType: 'application/json' as 'application/json',
href: playlist.segmentsSha256Url
})
url.push({
type: 'Link',
mimeType: 'application/x-mpegURL' as 'application/x-mpegURL',
mediaType: 'application/x-mpegURL' as 'application/x-mpegURL',
href: playlist.playlistUrl,
tag
})
}
// Add video url too
url.push({
type: 'Link',

View File

@ -0,0 +1,154 @@
import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, HasMany, Is, Model, Table, UpdatedAt } from 'sequelize-typescript'
import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
import { throwIfNotValid } from '../utils'
import { VideoModel } from './video'
import * as Sequelize from 'sequelize'
import { VideoRedundancyModel } from '../redundancy/video-redundancy'
import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
import { CONSTRAINTS_FIELDS, STATIC_PATHS } from '../../initializers'
import { VideoFileModel } from './video-file'
import { join } from 'path'
import { sha1 } from '../../helpers/core-utils'
import { isArrayOf } from '../../helpers/custom-validators/misc'
@Table({
tableName: 'videoStreamingPlaylist',
indexes: [
{
fields: [ 'videoId' ]
},
{
fields: [ 'videoId', 'type' ],
unique: true
},
{
fields: [ 'p2pMediaLoaderInfohashes' ],
using: 'gin'
}
]
})
export class VideoStreamingPlaylistModel extends Model<VideoStreamingPlaylistModel> {
@CreatedAt
createdAt: Date
@UpdatedAt
updatedAt: Date
@AllowNull(false)
@Column
type: VideoStreamingPlaylistType
@AllowNull(false)
@Is('PlaylistUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'playlist url'))
@Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS.URL.max))
playlistUrl: string
@AllowNull(false)
@Is('VideoStreamingPlaylistInfoHashes', value => throwIfNotValid(value, v => isArrayOf(v, isVideoFileInfoHashValid), 'info hashes'))
@Column(DataType.ARRAY(DataType.STRING))
p2pMediaLoaderInfohashes: string[]
@AllowNull(false)
@Is('VideoStreamingSegmentsSha256Url', value => throwIfNotValid(value, isActivityPubUrlValid, 'segments sha256 url'))
@Column
segmentsSha256Url: string
@ForeignKey(() => VideoModel)
@Column
videoId: number
@BelongsTo(() => VideoModel, {
foreignKey: {
allowNull: false
},
onDelete: 'CASCADE'
})
Video: VideoModel
@HasMany(() => VideoRedundancyModel, {
foreignKey: {
allowNull: false
},
onDelete: 'CASCADE',
hooks: true
})
RedundancyVideos: VideoRedundancyModel[]
static doesInfohashExist (infoHash: string) {
const query = 'SELECT 1 FROM "videoStreamingPlaylist" WHERE $infoHash = ANY("p2pMediaLoaderInfohashes") LIMIT 1'
const options = {
type: Sequelize.QueryTypes.SELECT,
bind: { infoHash },
raw: true
}
return VideoModel.sequelize.query(query, options)
.then(results => {
return results.length === 1
})
}
static buildP2PMediaLoaderInfoHashes (playlistUrl: string, videoFiles: VideoFileModel[]) {
const hashes: string[] = []
// https://github.com/Novage/p2p-media-loader/blob/master/p2p-media-loader-core/lib/p2p-media-manager.ts#L97
for (let i = 0; i < videoFiles.length; i++) {
hashes.push(sha1(`1${playlistUrl}+V${i}`))
}
return hashes
}
static loadWithVideo (id: number) {
const options = {
include: [
{
model: VideoModel.unscoped(),
required: true
}
]
}
return VideoStreamingPlaylistModel.findById(id, options)
}
static getHlsPlaylistFilename (resolution: number) {
return resolution + '.m3u8'
}
static getMasterHlsPlaylistFilename () {
return 'master.m3u8'
}
static getHlsSha256SegmentsFilename () {
return 'segments-sha256.json'
}
static getHlsMasterPlaylistStaticPath (videoUUID: string) {
return join(STATIC_PATHS.PLAYLISTS.HLS, videoUUID, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename())
}
static getHlsPlaylistStaticPath (videoUUID: string, resolution: number) {
return join(STATIC_PATHS.PLAYLISTS.HLS, videoUUID, VideoStreamingPlaylistModel.getHlsPlaylistFilename(resolution))
}
static getHlsSha256SegmentsStaticPath (videoUUID: string) {
return join(STATIC_PATHS.PLAYLISTS.HLS, videoUUID, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename())
}
getStringType () {
if (this.type === VideoStreamingPlaylistType.HLS) return 'hls'
return 'unknown'
}
getVideoRedundancyUrl (baseUrlHttp: string) {
return baseUrlHttp + STATIC_PATHS.REDUNDANCY + this.getStringType() + '/' + this.Video.uuid
}
hasSameUniqueKeysThan (other: VideoStreamingPlaylistModel) {
return this.type === other.type &&
this.videoId === other.videoId
}
}

View File

@ -52,7 +52,7 @@ import {
ACTIVITY_PUB,
API_VERSION,
CONFIG,
CONSTRAINTS_FIELDS,
CONSTRAINTS_FIELDS, HLS_PLAYLIST_DIRECTORY, HLS_REDUNDANCY_DIRECTORY,
PREVIEWS_SIZE,
REMOTE_SCHEME,
STATIC_DOWNLOAD_PATHS,
@ -95,6 +95,7 @@ import * as validator from 'validator'
import { UserVideoHistoryModel } from '../account/user-video-history'
import { UserModel } from '../account/user'
import { VideoImportModel } from './video-import'
import { VideoStreamingPlaylistModel } from './video-streaming-playlist'
// FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation
const indexes: Sequelize.DefineIndexesOptions[] = [
@ -159,7 +160,9 @@ export enum ScopeNames {
WITH_FILES = 'WITH_FILES',
WITH_SCHEDULED_UPDATE = 'WITH_SCHEDULED_UPDATE',
WITH_BLACKLISTED = 'WITH_BLACKLISTED',
WITH_USER_HISTORY = 'WITH_USER_HISTORY'
WITH_USER_HISTORY = 'WITH_USER_HISTORY',
WITH_STREAMING_PLAYLISTS = 'WITH_STREAMING_PLAYLISTS',
WITH_USER_ID = 'WITH_USER_ID'
}
type ForAPIOptions = {
@ -463,6 +466,22 @@ type AvailableForListIDsOptions = {
return query
},
[ ScopeNames.WITH_USER_ID ]: {
include: [
{
attributes: [ 'accountId' ],
model: () => VideoChannelModel.unscoped(),
required: true,
include: [
{
attributes: [ 'userId' ],
model: () => AccountModel.unscoped(),
required: true
}
]
}
]
},
[ ScopeNames.WITH_ACCOUNT_DETAILS ]: {
include: [
{
@ -527,22 +546,55 @@ type AvailableForListIDsOptions = {
}
]
},
[ ScopeNames.WITH_FILES ]: {
include: [
{
model: () => VideoFileModel.unscoped(),
// FIXME: typings
[ 'separate' as any ]: true, // We may have multiple files, having multiple redundancies so let's separate this join
required: false,
include: [
{
attributes: [ 'fileUrl' ],
model: () => VideoRedundancyModel.unscoped(),
required: false
}
]
}
]
[ ScopeNames.WITH_FILES ]: (withRedundancies = false) => {
let subInclude: any[] = []
if (withRedundancies === true) {
subInclude = [
{
attributes: [ 'fileUrl' ],
model: VideoRedundancyModel.unscoped(),
required: false
}
]
}
return {
include: [
{
model: VideoFileModel.unscoped(),
// FIXME: typings
[ 'separate' as any ]: true, // We may have multiple files, having multiple redundancies so let's separate this join
required: false,
include: subInclude
}
]
}
},
[ ScopeNames.WITH_STREAMING_PLAYLISTS ]: (withRedundancies = false) => {
let subInclude: any[] = []
if (withRedundancies === true) {
subInclude = [
{
attributes: [ 'fileUrl' ],
model: VideoRedundancyModel.unscoped(),
required: false
}
]
}
return {
include: [
{
model: VideoStreamingPlaylistModel.unscoped(),
// FIXME: typings
[ 'separate' as any ]: true, // We may have multiple streaming playlists, having multiple redundancies so let's separate this join
required: false,
include: subInclude
}
]
}
},
[ ScopeNames.WITH_SCHEDULED_UPDATE ]: {
include: [
@ -722,6 +774,16 @@ export class VideoModel extends Model<VideoModel> {
})
VideoFiles: VideoFileModel[]
@HasMany(() => VideoStreamingPlaylistModel, {
foreignKey: {
name: 'videoId',
allowNull: false
},
hooks: true,
onDelete: 'cascade'
})
VideoStreamingPlaylists: VideoStreamingPlaylistModel[]
@HasMany(() => VideoShareModel, {
foreignKey: {
name: 'videoId',
@ -847,6 +909,9 @@ export class VideoModel extends Model<VideoModel> {
tasks.push(instance.removeFile(file))
tasks.push(instance.removeTorrent(file))
})
// Remove playlists file
tasks.push(instance.removeStreamingPlaylist())
}
// Do not wait video deletion because we could be in a transaction
@ -858,10 +923,6 @@ export class VideoModel extends Model<VideoModel> {
return undefined
}
static list () {
return VideoModel.scope(ScopeNames.WITH_FILES).findAll()
}
static listLocal () {
const query = {
where: {
@ -869,7 +930,7 @@ export class VideoModel extends Model<VideoModel> {
}
}
return VideoModel.scope(ScopeNames.WITH_FILES).findAll(query)
return VideoModel.scope([ ScopeNames.WITH_FILES, ScopeNames.WITH_STREAMING_PLAYLISTS ]).findAll(query)
}
static listAllAndSharedByActorForOutbox (actorId: number, start: number, count: number) {
@ -1200,6 +1261,16 @@ export class VideoModel extends Model<VideoModel> {
return VideoModel.findOne(options)
}
static loadWithRights (id: number | string, t?: Sequelize.Transaction) {
const where = VideoModel.buildWhereIdOrUUID(id)
const options = {
where,
transaction: t
}
return VideoModel.scope([ ScopeNames.WITH_BLACKLISTED, ScopeNames.WITH_USER_ID ]).findOne(options)
}
static loadOnlyId (id: number | string, t?: Sequelize.Transaction) {
const where = VideoModel.buildWhereIdOrUUID(id)
@ -1212,8 +1283,8 @@ export class VideoModel extends Model<VideoModel> {
return VideoModel.findOne(options)
}
static loadWithFile (id: number, t?: Sequelize.Transaction, logging?: boolean) {
return VideoModel.scope(ScopeNames.WITH_FILES)
static loadWithFiles (id: number, t?: Sequelize.Transaction, logging?: boolean) {
return VideoModel.scope([ ScopeNames.WITH_FILES, ScopeNames.WITH_STREAMING_PLAYLISTS ])
.findById(id, { transaction: t, logging })
}
@ -1224,9 +1295,7 @@ export class VideoModel extends Model<VideoModel> {
}
}
return VideoModel
.scope([ ScopeNames.WITH_FILES ])
.findOne(options)
return VideoModel.findOne(options)
}
static loadByUrl (url: string, transaction?: Sequelize.Transaction) {
@ -1248,7 +1317,11 @@ export class VideoModel extends Model<VideoModel> {
transaction
}
return VideoModel.scope([ ScopeNames.WITH_ACCOUNT_DETAILS, ScopeNames.WITH_FILES ]).findOne(query)
return VideoModel.scope([
ScopeNames.WITH_ACCOUNT_DETAILS,
ScopeNames.WITH_FILES,
ScopeNames.WITH_STREAMING_PLAYLISTS
]).findOne(query)
}
static loadAndPopulateAccountAndServerAndTags (id: number | string, t?: Sequelize.Transaction, userId?: number) {
@ -1263,9 +1336,37 @@ export class VideoModel extends Model<VideoModel> {
const scopes = [
ScopeNames.WITH_TAGS,
ScopeNames.WITH_BLACKLISTED,
ScopeNames.WITH_FILES,
ScopeNames.WITH_ACCOUNT_DETAILS,
ScopeNames.WITH_SCHEDULED_UPDATE
ScopeNames.WITH_SCHEDULED_UPDATE,
ScopeNames.WITH_FILES,
ScopeNames.WITH_STREAMING_PLAYLISTS
]
if (userId) {
scopes.push({ method: [ ScopeNames.WITH_USER_HISTORY, userId ] } as any) // FIXME: typings
}
return VideoModel
.scope(scopes)
.findOne(options)
}
static loadForGetAPI (id: number | string, t?: Sequelize.Transaction, userId?: number) {
const where = VideoModel.buildWhereIdOrUUID(id)
const options = {
order: [ [ 'Tags', 'name', 'ASC' ] ],
where,
transaction: t
}
const scopes = [
ScopeNames.WITH_TAGS,
ScopeNames.WITH_BLACKLISTED,
ScopeNames.WITH_ACCOUNT_DETAILS,
ScopeNames.WITH_SCHEDULED_UPDATE,
{ method: [ ScopeNames.WITH_FILES, true ] } as any, // FIXME: typings
{ method: [ ScopeNames.WITH_STREAMING_PLAYLISTS, true ] } as any // FIXME: typings
]
if (userId) {
@ -1612,6 +1713,14 @@ export class VideoModel extends Model<VideoModel> {
.catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err }))
}
removeStreamingPlaylist (isRedundancy = false) {
const baseDir = isRedundancy ? HLS_REDUNDANCY_DIRECTORY : HLS_PLAYLIST_DIRECTORY
const filePath = join(baseDir, this.uuid)
return remove(filePath)
.catch(err => logger.warn('Cannot delete playlist directory %s.', filePath, { err }))
}
isOutdated () {
if (this.isOwned()) return false
@ -1646,7 +1755,7 @@ export class VideoModel extends Model<VideoModel> {
generateMagnetUri (videoFile: VideoFileModel, baseUrlHttp: string, baseUrlWs: string) {
const xs = this.getTorrentUrl(videoFile, baseUrlHttp)
const announce = [ baseUrlWs + '/tracker/socket', baseUrlHttp + '/tracker/announce' ]
const announce = this.getTrackerUrls(baseUrlHttp, baseUrlWs)
let urlList = [ this.getVideoFileUrl(videoFile, baseUrlHttp) ]
const redundancies = videoFile.RedundancyVideos
@ -1663,6 +1772,10 @@ export class VideoModel extends Model<VideoModel> {
return magnetUtil.encode(magnetHash)
}
getTrackerUrls (baseUrlHttp: string, baseUrlWs: string) {
return [ baseUrlWs + '/tracker/socket', baseUrlHttp + '/tracker/announce' ]
}
getThumbnailUrl (baseUrlHttp: string) {
return baseUrlHttp + STATIC_PATHS.THUMBNAILS + this.getThumbnailName()
}
@ -1686,4 +1799,8 @@ export class VideoModel extends Model<VideoModel> {
getVideoFileDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) {
return baseUrlHttp + STATIC_DOWNLOAD_PATHS.VIDEOS + this.getVideoFilename(videoFile)
}
getBandwidthBits (videoFile: VideoFileModel) {
return Math.ceil((videoFile.size * 8) / this.duration)
}
}

View File

@ -65,6 +65,9 @@ describe('Test config API validators', function () {
'480p': true,
'720p': false,
'1080p': false
},
hls: {
enabled: false
}
},
import: {

View File

@ -17,7 +17,7 @@ import {
viewVideo,
wait,
waitUntilLog,
checkVideoFilesWereRemoved, removeVideo, getVideoWithToken
checkVideoFilesWereRemoved, removeVideo, getVideoWithToken, reRunServer
} from '../../../../shared/utils'
import { waitJobs } from '../../../../shared/utils/server/jobs'
@ -48,6 +48,11 @@ function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: numbe
async function runServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) {
const config = {
transcoding: {
hls: {
enabled: true
}
},
redundancy: {
videos: {
check_interval: '5 seconds',
@ -85,7 +90,7 @@ async function runServers (strategy: VideoRedundancyStrategy, additionalParams:
await waitJobs(servers)
}
async function check1WebSeed (strategy: VideoRedundancyStrategy, videoUUID?: string) {
async function check1WebSeed (videoUUID?: string) {
if (!videoUUID) videoUUID = video1Server2UUID
const webseeds = [
@ -93,14 +98,100 @@ async function check1WebSeed (strategy: VideoRedundancyStrategy, videoUUID?: str
]
for (const server of servers) {
{
// With token to avoid issues with video follow constraints
const res = await getVideoWithToken(server.url, server.accessToken, videoUUID)
// With token to avoid issues with video follow constraints
const res = await getVideoWithToken(server.url, server.accessToken, videoUUID)
const video: VideoDetails = res.body
for (const f of video.files) {
checkMagnetWebseeds(f, webseeds, server)
}
const video: VideoDetails = res.body
for (const f of video.files) {
checkMagnetWebseeds(f, webseeds, server)
}
}
}
async function check2Webseeds (videoUUID?: string) {
if (!videoUUID) videoUUID = video1Server2UUID
const webseeds = [
'http://localhost:9001/static/redundancy/' + videoUUID,
'http://localhost:9002/static/webseed/' + videoUUID
]
for (const server of servers) {
const res = await getVideo(server.url, videoUUID)
const video: VideoDetails = res.body
for (const file of video.files) {
checkMagnetWebseeds(file, webseeds, server)
await makeGetRequest({
url: servers[0].url,
statusCodeExpected: 200,
path: '/static/redundancy/' + `${videoUUID}-${file.resolution.id}.mp4`,
contentType: null
})
await makeGetRequest({
url: servers[1].url,
statusCodeExpected: 200,
path: `/static/webseed/${videoUUID}-${file.resolution.id}.mp4`,
contentType: null
})
}
}
for (const directory of [ 'test1/redundancy', 'test2/videos' ]) {
const files = await readdir(join(root(), directory))
expect(files).to.have.length.at.least(4)
for (const resolution of [ 240, 360, 480, 720 ]) {
expect(files.find(f => f === `${videoUUID}-${resolution}.mp4`)).to.not.be.undefined
}
}
}
async function check0PlaylistRedundancies (videoUUID?: string) {
if (!videoUUID) videoUUID = video1Server2UUID
for (const server of servers) {
// With token to avoid issues with video follow constraints
const res = await getVideoWithToken(server.url, server.accessToken, videoUUID)
const video: VideoDetails = res.body
expect(video.streamingPlaylists).to.be.an('array')
expect(video.streamingPlaylists).to.have.lengthOf(1)
expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(0)
}
}
async function check1PlaylistRedundancies (videoUUID?: string) {
if (!videoUUID) videoUUID = video1Server2UUID
for (const server of servers) {
const res = await getVideo(server.url, videoUUID)
const video: VideoDetails = res.body
expect(video.streamingPlaylists).to.have.lengthOf(1)
expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(1)
const redundancy = video.streamingPlaylists[0].redundancies[0]
expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID)
}
await makeGetRequest({
url: servers[0].url,
statusCodeExpected: 200,
path: `/static/redundancy/hls/${videoUUID}/360_000.ts`,
contentType: null
})
for (const directory of [ 'test1/redundancy/hls', 'test2/playlists/hls' ]) {
const files = await readdir(join(root(), directory, videoUUID))
expect(files).to.have.length.at.least(4)
for (const resolution of [ 240, 360, 480, 720 ]) {
expect(files.find(f => f === `${resolution}_000.ts`)).to.not.be.undefined
expect(files.find(f => f === `${resolution}_001.ts`)).to.not.be.undefined
}
}
}
@ -133,47 +224,6 @@ async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) {
expect(stat.totalVideos).to.equal(0)
}
async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: string) {
if (!videoUUID) videoUUID = video1Server2UUID
const webseeds = [
'http://localhost:9001/static/redundancy/' + videoUUID,
'http://localhost:9002/static/webseed/' + videoUUID
]
for (const server of servers) {
const res = await getVideo(server.url, videoUUID)
const video: VideoDetails = res.body
for (const file of video.files) {
checkMagnetWebseeds(file, webseeds, server)
await makeGetRequest({
url: servers[0].url,
statusCodeExpected: 200,
path: '/static/redundancy/' + `${videoUUID}-${file.resolution.id}.mp4`,
contentType: null
})
await makeGetRequest({
url: servers[1].url,
statusCodeExpected: 200,
path: '/static/webseed/' + `${videoUUID}-${file.resolution.id}.mp4`,
contentType: null
})
}
}
for (const directory of [ 'test1/redundancy', 'test2/videos' ]) {
const files = await readdir(join(root(), directory))
expect(files).to.have.length.at.least(4)
for (const resolution of [ 240, 360, 480, 720 ]) {
expect(files.find(f => f === `${videoUUID}-${resolution}.mp4`)).to.not.be.undefined
}
}
}
async function enableRedundancyOnServer1 () {
await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true)
@ -220,7 +270,8 @@ describe('Test videos redundancy', function () {
})
it('Should have 1 webseed on the first video', async function () {
await check1WebSeed(strategy)
await check1WebSeed()
await check0PlaylistRedundancies()
await checkStatsWith1Webseed(strategy)
})
@ -229,27 +280,29 @@ describe('Test videos redundancy', function () {
})
it('Should have 2 webseeds on the first video', async function () {
this.timeout(40000)
this.timeout(80000)
await waitJobs(servers)
await waitUntilLog(servers[0], 'Duplicated ', 4)
await waitUntilLog(servers[0], 'Duplicated ', 5)
await waitJobs(servers)
await check2Webseeds(strategy)
await check2Webseeds()
await check1PlaylistRedundancies()
await checkStatsWith2Webseed(strategy)
})
it('Should undo redundancy on server 1 and remove duplicated videos', async function () {
this.timeout(40000)
this.timeout(80000)
await disableRedundancyOnServer1()
await waitJobs(servers)
await wait(5000)
await check1WebSeed(strategy)
await check1WebSeed()
await check0PlaylistRedundancies()
await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ])
await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos', join('playlists', 'hls') ])
})
after(function () {
@ -267,7 +320,8 @@ describe('Test videos redundancy', function () {
})
it('Should have 1 webseed on the first video', async function () {
await check1WebSeed(strategy)
await check1WebSeed()
await check0PlaylistRedundancies()
await checkStatsWith1Webseed(strategy)
})
@ -276,25 +330,27 @@ describe('Test videos redundancy', function () {
})
it('Should have 2 webseeds on the first video', async function () {
this.timeout(40000)
this.timeout(80000)
await waitJobs(servers)
await waitUntilLog(servers[0], 'Duplicated ', 4)
await waitUntilLog(servers[0], 'Duplicated ', 5)
await waitJobs(servers)
await check2Webseeds(strategy)
await check2Webseeds()
await check1PlaylistRedundancies()
await checkStatsWith2Webseed(strategy)
})
it('Should unfollow on server 1 and remove duplicated videos', async function () {
this.timeout(40000)
this.timeout(80000)
await unfollow(servers[0].url, servers[0].accessToken, servers[1])
await waitJobs(servers)
await wait(5000)
await check1WebSeed(strategy)
await check1WebSeed()
await check0PlaylistRedundancies()
await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ])
})
@ -314,7 +370,8 @@ describe('Test videos redundancy', function () {
})
it('Should have 1 webseed on the first video', async function () {
await check1WebSeed(strategy)
await check1WebSeed()
await check0PlaylistRedundancies()
await checkStatsWith1Webseed(strategy)
})
@ -323,18 +380,19 @@ describe('Test videos redundancy', function () {
})
it('Should still have 1 webseed on the first video', async function () {
this.timeout(40000)
this.timeout(80000)
await waitJobs(servers)
await wait(15000)
await waitJobs(servers)
await check1WebSeed(strategy)
await check1WebSeed()
await check0PlaylistRedundancies()
await checkStatsWith1Webseed(strategy)
})
it('Should view 2 times the first video to have > min_views config', async function () {
this.timeout(40000)
this.timeout(80000)
await viewVideo(servers[ 0 ].url, video1Server2UUID)
await viewVideo(servers[ 2 ].url, video1Server2UUID)
@ -344,13 +402,14 @@ describe('Test videos redundancy', function () {
})
it('Should have 2 webseeds on the first video', async function () {
this.timeout(40000)
this.timeout(80000)
await waitJobs(servers)
await waitUntilLog(servers[0], 'Duplicated ', 4)
await waitUntilLog(servers[0], 'Duplicated ', 5)
await waitJobs(servers)
await check2Webseeds(strategy)
await check2Webseeds()
await check1PlaylistRedundancies()
await checkStatsWith2Webseed(strategy)
})
@ -405,7 +464,7 @@ describe('Test videos redundancy', function () {
})
it('Should still have 2 webseeds after 10 seconds', async function () {
this.timeout(40000)
this.timeout(80000)
await wait(10000)
@ -420,7 +479,7 @@ describe('Test videos redundancy', function () {
})
it('Should stop server 1 and expire video redundancy', async function () {
this.timeout(40000)
this.timeout(80000)
killallServers([ servers[0] ])
@ -446,10 +505,11 @@ describe('Test videos redundancy', function () {
await enableRedundancyOnServer1()
await waitJobs(servers)
await waitUntilLog(servers[0], 'Duplicated ', 4)
await waitUntilLog(servers[0], 'Duplicated ', 5)
await waitJobs(servers)
await check2Webseeds(strategy)
await check2Webseeds()
await check1PlaylistRedundancies()
await checkStatsWith2Webseed(strategy)
const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' })
@ -467,8 +527,10 @@ describe('Test videos redundancy', function () {
await wait(1000)
try {
await check1WebSeed(strategy, video1Server2UUID)
await check2Webseeds(strategy, video2Server2UUID)
await check1WebSeed(video1Server2UUID)
await check0PlaylistRedundancies(video1Server2UUID)
await check2Webseeds(video2Server2UUID)
await check1PlaylistRedundancies(video2Server2UUID)
checked = true
} catch {
@ -477,6 +539,26 @@ describe('Test videos redundancy', function () {
}
})
it('Should disable strategy and remove redundancies', async function () {
this.timeout(80000)
await waitJobs(servers)
killallServers([ servers[ 0 ] ])
await reRunServer(servers[ 0 ], {
redundancy: {
videos: {
check_interval: '1 second',
strategies: []
}
}
})
await waitJobs(servers)
await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ join('redundancy', 'hls') ])
})
after(function () {
return cleanServers()
})

View File

@ -57,6 +57,8 @@ function checkInitialConfig (data: CustomConfig) {
expect(data.transcoding.resolutions['480p']).to.be.true
expect(data.transcoding.resolutions['720p']).to.be.true
expect(data.transcoding.resolutions['1080p']).to.be.true
expect(data.transcoding.hls.enabled).to.be.true
expect(data.import.videos.http.enabled).to.be.true
expect(data.import.videos.torrent.enabled).to.be.true
}
@ -95,6 +97,7 @@ function checkUpdatedConfig (data: CustomConfig) {
expect(data.transcoding.resolutions['480p']).to.be.true
expect(data.transcoding.resolutions['720p']).to.be.false
expect(data.transcoding.resolutions['1080p']).to.be.false
expect(data.transcoding.hls.enabled).to.be.false
expect(data.import.videos.http.enabled).to.be.false
expect(data.import.videos.torrent.enabled).to.be.false
@ -205,6 +208,9 @@ describe('Test config', function () {
'480p': true,
'720p': false,
'1080p': false
},
hls: {
enabled: false
}
},
import: {

View File

@ -8,6 +8,7 @@ import './video-change-ownership'
import './video-channels'
import './video-comments'
import './video-description'
import './video-hls'
import './video-imports'
import './video-nsfw'
import './video-privacy'

View File

@ -0,0 +1,145 @@
/* tslint:disable:no-unused-expression */
import * as chai from 'chai'
import 'mocha'
import {
checkDirectoryIsEmpty,
checkTmpIsEmpty,
doubleFollow,
flushAndRunMultipleServers,
flushTests,
getPlaylist,
getSegment,
getSegmentSha256,
getVideo,
killallServers,
removeVideo,
ServerInfo,
setAccessTokensToServers,
updateVideo,
uploadVideo,
waitJobs
} from '../../../../shared/utils'
import { VideoDetails } from '../../../../shared/models/videos'
import { VideoStreamingPlaylistType } from '../../../../shared/models/videos/video-streaming-playlist.type'
import { sha256 } from '../../../helpers/core-utils'
import { join } from 'path'
const expect = chai.expect
async function checkHlsPlaylist (servers: ServerInfo[], videoUUID: string) {
const resolutions = [ 240, 360, 480, 720 ]
for (const server of servers) {
const res = await getVideo(server.url, videoUUID)
const videoDetails: VideoDetails = res.body
expect(videoDetails.streamingPlaylists).to.have.lengthOf(1)
const hlsPlaylist = videoDetails.streamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS)
expect(hlsPlaylist).to.not.be.undefined
{
const res2 = await getPlaylist(hlsPlaylist.playlistUrl)
const masterPlaylist = res2.text
expect(masterPlaylist).to.contain('#EXT-X-STREAM-INF:BANDWIDTH=55472,RESOLUTION=640x360,FRAME-RATE=25')
for (const resolution of resolutions) {
expect(masterPlaylist).to.contain(`${resolution}.m3u8`)
}
}
{
for (const resolution of resolutions) {
const res2 = await getPlaylist(`http://localhost:9001/static/playlists/hls/${videoUUID}/${resolution}.m3u8`)
const subPlaylist = res2.text
expect(subPlaylist).to.contain(resolution + '_000.ts')
}
}
{
for (const resolution of resolutions) {
const res2 = await getSegment(`http://localhost:9001/static/playlists/hls/${videoUUID}/${resolution}_000.ts`)
const resSha = await getSegmentSha256(hlsPlaylist.segmentsSha256Url)
const sha256Server = resSha.body[ resolution + '_000.ts' ]
expect(sha256(res2.body)).to.equal(sha256Server)
}
}
}
}
describe('Test HLS videos', function () {
let servers: ServerInfo[] = []
let videoUUID = ''
before(async function () {
this.timeout(120000)
servers = await flushAndRunMultipleServers(2, { transcoding: { enabled: true, hls: { enabled: true } } })
// Get the access tokens
await setAccessTokensToServers(servers)
// Server 1 and server 2 follow each other
await doubleFollow(servers[0], servers[1])
})
it('Should upload a video and transcode it to HLS', async function () {
this.timeout(120000)
{
const res = await uploadVideo(servers[ 0 ].url, servers[ 0 ].accessToken, { name: 'video 1', fixture: 'video_short.webm' })
videoUUID = res.body.video.uuid
}
await waitJobs(servers)
await checkHlsPlaylist(servers, videoUUID)
})
it('Should update the video', async function () {
await updateVideo(servers[0].url, servers[0].accessToken, videoUUID, { name: 'video 1 updated' })
await waitJobs(servers)
await checkHlsPlaylist(servers, videoUUID)
})
it('Should delete the video', async function () {
await removeVideo(servers[0].url, servers[0].accessToken, videoUUID)
await waitJobs(servers)
for (const server of servers) {
await getVideo(server.url, videoUUID, 404)
}
})
it('Should have the playlists/segment deleted from the disk', async function () {
for (const server of servers) {
await checkDirectoryIsEmpty(server, 'videos')
await checkDirectoryIsEmpty(server, join('playlists', 'hls'))
}
})
it('Should have an empty tmp directory', async function () {
for (const server of servers) {
await checkTmpIsEmpty(server)
}
})
after(async function () {
killallServers(servers)
// Keep the logs if the test failed
if (this['ok']) {
await flushTests()
}
})
})

View File

@ -86,6 +86,13 @@ describe('Test update host scripts', function () {
const { body } = await makeActivityPubGetRequest(server.url, '/videos/watch/' + video.uuid)
expect(body.id).to.equal('http://localhost:9002/videos/watch/' + video.uuid)
const res = await getVideo(server.url, video.uuid)
const videoDetails: VideoDetails = res.body
expect(videoDetails.trackerUrls[0]).to.include(server.host)
expect(videoDetails.streamingPlaylists[0].playlistUrl).to.include(server.host)
expect(videoDetails.streamingPlaylists[0].segmentsSha256Url).to.include(server.host)
}
})
@ -100,7 +107,7 @@ describe('Test update host scripts', function () {
}
})
it('Should have update accounts url', async function () {
it('Should have updated accounts url', async function () {
const res = await getAccountsList(server.url)
expect(res.body.total).to.equal(3)
@ -112,7 +119,7 @@ describe('Test update host scripts', function () {
}
})
it('Should update torrent hosts', async function () {
it('Should have updated torrent hosts', async function () {
this.timeout(30000)
const res = await getVideosList(server.url)

View File

@ -1,9 +1,9 @@
import { ActivityVideoUrlObject } from './common-objects'
import { ActivityVideoUrlObject, ActivityPlaylistUrlObject } from './common-objects'
export interface CacheFileObject {
id: string
type: 'CacheFile',
object: string
expires: string
url: ActivityVideoUrlObject
url: ActivityVideoUrlObject | ActivityPlaylistUrlObject
}

View File

@ -28,25 +28,47 @@ export type ActivityVideoUrlObject = {
fps: number
}
export type ActivityUrlObject =
ActivityVideoUrlObject
|
{
type: 'Link'
// TODO: remove mimeType (backward compatibility, introduced in v1.1.0)
mimeType?: 'application/x-bittorrent' | 'application/x-bittorrent;x-scheme-handler/magnet'
mediaType: 'application/x-bittorrent' | 'application/x-bittorrent;x-scheme-handler/magnet'
href: string
height: number
}
|
{
type: 'Link'
// TODO: remove mimeType (backward compatibility, introduced in v1.1.0)
mimeType?: 'text/html'
mediaType: 'text/html'
href: string
}
export type ActivityPlaylistSegmentHashesObject = {
type: 'Link'
name: 'sha256'
// TODO: remove mimeType (backward compatibility, introduced in v1.1.0)
mimeType?: 'application/json'
mediaType: 'application/json'
href: string
}
export type ActivityPlaylistInfohashesObject = {
type: 'Infohash'
name: string
}
export type ActivityPlaylistUrlObject = {
type: 'Link'
// TODO: remove mimeType (backward compatibility, introduced in v1.1.0)
mimeType?: 'application/x-mpegURL'
mediaType: 'application/x-mpegURL'
href: string
tag?: (ActivityPlaylistSegmentHashesObject | ActivityPlaylistInfohashesObject)[]
}
export type ActivityBitTorrentUrlObject = {
type: 'Link'
// TODO: remove mimeType (backward compatibility, introduced in v1.1.0)
mimeType?: 'application/x-bittorrent' | 'application/x-bittorrent;x-scheme-handler/magnet'
mediaType: 'application/x-bittorrent' | 'application/x-bittorrent;x-scheme-handler/magnet'
href: string
height: number
}
export type ActivityHtmlUrlObject = {
type: 'Link'
// TODO: remove mimeType (backward compatibility, introduced in v1.1.0)
mimeType?: 'text/html'
mediaType: 'text/html'
href: string
}
export type ActivityUrlObject = ActivityVideoUrlObject | ActivityPlaylistUrlObject | ActivityBitTorrentUrlObject | ActivityHtmlUrlObject
export interface ActivityPubAttributedTo {
type: 'Group' | 'Person'

View File

@ -61,6 +61,9 @@ export interface CustomConfig {
'720p': boolean
'1080p': boolean
}
hls: {
enabled: boolean
}
}
import: {

View File

@ -25,11 +25,15 @@ export interface ServerConfig {
signup: {
allowed: boolean,
allowedForCurrentIP: boolean,
allowedForCurrentIP: boolean
requiresEmailVerification: boolean
}
transcoding: {
hls: {
enabled: boolean
}
enabledResolutions: number[]
}
@ -48,7 +52,7 @@ export interface ServerConfig {
file: {
size: {
max: number
},
}
extensions: string[]
}
}

View File

@ -0,0 +1,12 @@
import { VideoStreamingPlaylistType } from './video-streaming-playlist.type'
export class VideoStreamingPlaylist {
id: number
type: VideoStreamingPlaylistType
playlistUrl: string
segmentsSha256Url: string
redundancies: {
baseUrl: string
}[]
}

View File

@ -0,0 +1,3 @@
export enum VideoStreamingPlaylistType {
HLS = 1
}

View File

@ -5,6 +5,7 @@ import { VideoChannel } from './channel/video-channel.model'
import { VideoPrivacy } from './video-privacy.enum'
import { VideoScheduleUpdate } from './video-schedule-update.model'
import { VideoConstant } from './video-constant.model'
import { VideoStreamingPlaylist } from './video-streaming-playlist.model'
export interface VideoFile {
magnetUri: string
@ -86,4 +87,8 @@ export interface VideoDetails extends Video {
// Not optional in details (unlike in Video)
waitTranscoding: boolean
state: VideoConstant<VideoState>
trackerUrls: string[]
streamingPlaylists: VideoStreamingPlaylist[]
}

View File

@ -17,6 +17,8 @@ export * from './users/users'
export * from './videos/video-abuses'
export * from './videos/video-blacklist'
export * from './videos/video-channels'
export * from './videos/video-comments'
export * from './videos/video-playlists'
export * from './videos/videos'
export * from './videos/video-change-ownership'
export * from './feeds/feeds'

View File

@ -1,10 +1,17 @@
import * as request from 'supertest'
import { buildAbsoluteFixturePath, root } from '../miscs/miscs'
import { isAbsolute, join } from 'path'
import { parse } from 'url'
function makeRawRequest (url: string, statusCodeExpected?: number) {
const { host, protocol, pathname } = parse(url)
return makeGetRequest({ url: `${protocol}//${host}`, path: pathname, statusCodeExpected })
}
function makeGetRequest (options: {
url: string,
path: string,
path?: string,
query?: any,
token?: string,
statusCodeExpected?: number,
@ -13,8 +20,7 @@ function makeGetRequest (options: {
if (!options.statusCodeExpected) options.statusCodeExpected = 400
if (options.contentType === undefined) options.contentType = 'application/json'
const req = request(options.url)
.get(options.path)
const req = request(options.url).get(options.path)
if (options.contentType) req.set('Accept', options.contentType)
if (options.token) req.set('Authorization', 'Bearer ' + options.token)
@ -164,5 +170,6 @@ export {
makePostBodyRequest,
makePutBodyRequest,
makeDeleteRequest,
makeRawRequest,
updateAvatarRequest
}

View File

@ -97,6 +97,9 @@ function updateCustomSubConfig (url: string, token: string, newConfig: any) {
'480p': true,
'720p': false,
'1080p': false
},
hls: {
enabled: false
}
},
import: {

View File

@ -166,9 +166,13 @@ async function reRunServer (server: ServerInfo, configOverride?: any) {
}
async function checkTmpIsEmpty (server: ServerInfo) {
return checkDirectoryIsEmpty(server, 'tmp')
}
async function checkDirectoryIsEmpty (server: ServerInfo, directory: string) {
const testDirectory = 'test' + server.serverNumber
const directoryPath = join(root(), testDirectory, 'tmp')
const directoryPath = join(root(), testDirectory, directory)
const directoryExists = existsSync(directoryPath)
expect(directoryExists).to.be.true
@ -199,6 +203,7 @@ async function waitUntilLog (server: ServerInfo, str: string, count = 1) {
// ---------------------------------------------------------------------------
export {
checkDirectoryIsEmpty,
checkTmpIsEmpty,
ServerInfo,
flushAndRunMultipleServers,

View File

@ -0,0 +1,21 @@
import { makeRawRequest } from '../requests/requests'
function getPlaylist (url: string, statusCodeExpected = 200) {
return makeRawRequest(url, statusCodeExpected)
}
function getSegment (url: string, statusCodeExpected = 200) {
return makeRawRequest(url, statusCodeExpected)
}
function getSegmentSha256 (url: string, statusCodeExpected = 200) {
return makeRawRequest(url, statusCodeExpected)
}
// ---------------------------------------------------------------------------
export {
getPlaylist,
getSegment,
getSegmentSha256
}

View File

@ -271,7 +271,16 @@ function removeVideo (url: string, token: string, id: number | string, expectedS
async function checkVideoFilesWereRemoved (
videoUUID: string,
serverNumber: number,
directories = [ 'redundancy', 'videos', 'thumbnails', 'torrents', 'previews', 'captions' ]
directories = [
'redundancy',
'videos',
'thumbnails',
'torrents',
'previews',
'captions',
join('playlists', 'hls'),
join('redundancy', 'hls')
]
) {
const testDirectory = 'test' + serverNumber
@ -279,7 +288,7 @@ async function checkVideoFilesWereRemoved (
const directoryPath = join(root(), testDirectory, directory)
const directoryExists = existsSync(directoryPath)
expect(directoryExists).to.be.true
if (!directoryExists) continue
const files = await readdir(directoryPath)
for (const file of files) {

View File

@ -2,6 +2,14 @@
# yarn lockfile v1
"@babel/polyfill@^7.2.5":
version "7.2.5"
resolved "https://registry.yarnpkg.com/@babel/polyfill/-/polyfill-7.2.5.tgz#6c54b964f71ad27edddc567d065e57e87ed7fa7d"
integrity sha512-8Y/t3MWThtMLYr0YNC/Q76tqN1w30+b0uQMeFUYauG2UGTR19zyUtFrAzT23zNtBxPp+LbE5E/nwV/q/r3y6ug==
dependencies:
core-js "^2.5.7"
regenerator-runtime "^0.12.0"
"@iamstarkov/listr-update-renderer@0.4.1":
version "0.4.1"
resolved "https://registry.yarnpkg.com/@iamstarkov/listr-update-renderer/-/listr-update-renderer-0.4.1.tgz#d7c48092a2dcf90fd672b6c8b458649cb350c77e"
@ -3585,6 +3593,17 @@ hide-powered-by@1.0.0:
resolved "https://registry.yarnpkg.com/hide-powered-by/-/hide-powered-by-1.0.0.tgz#4a85ad65881f62857fc70af7174a1184dccce32b"
integrity sha1-SoWtZYgfYoV/xwr3F0oRhNzM4ys=
"hlsdownloader@https://github.com/Chocobozzz/hlsdownloader#build":
version "0.0.0-semantic-release"
resolved "https://github.com/Chocobozzz/hlsdownloader#e19f9d803dcfe7ec25fd734b4743184f19a9b0cc"
dependencies:
"@babel/polyfill" "^7.2.5"
async "^2.6.1"
minimist "^1.2.0"
mkdirp "^0.5.1"
request "^2.88.0"
request-promise "^4.2.2"
hosted-git-info@^2.1.4, hosted-git-info@^2.6.0, hosted-git-info@^2.7.1:
version "2.7.1"
resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.7.1.tgz#97f236977bd6e125408930ff6de3eec6281ec047"
@ -4851,7 +4870,7 @@ lodash@=3.10.1:
resolved "https://registry.yarnpkg.com/lodash/-/lodash-3.10.1.tgz#5bf45e8e49ba4189e17d482789dfd15bd140b7b6"
integrity sha1-W/Rejkm6QYnhfUgnid/RW9FAt7Y=
lodash@^4.0.0, lodash@^4.17.1, lodash@^4.17.10, lodash@^4.17.11, lodash@^4.17.5, lodash@^4.3.0, lodash@^4.8.2, lodash@~4.17.10:
lodash@^4.0.0, lodash@^4.13.1, lodash@^4.17.1, lodash@^4.17.10, lodash@^4.17.11, lodash@^4.17.5, lodash@^4.3.0, lodash@^4.8.2, lodash@~4.17.10:
version "4.17.11"
resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.11.tgz#b39ea6229ef607ecd89e2c8df12536891cac9b8d"
integrity sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==
@ -6632,6 +6651,11 @@ psl@^1.1.24:
resolved "https://registry.yarnpkg.com/psl/-/psl-1.1.29.tgz#60f580d360170bb722a797cc704411e6da850c67"
integrity sha512-AeUmQ0oLN02flVHXWh9sSJF7mcdFq0ppid/JkErufc3hGIV/AMa8Fo9VgDo/cT2jFdOWoFvHp90qqBH54W+gjQ==
psl@^1.1.28:
version "1.1.31"
resolved "https://registry.yarnpkg.com/psl/-/psl-1.1.31.tgz#e9aa86d0101b5b105cbe93ac6b784cd547276184"
integrity sha512-/6pt4+C+T+wZUieKR620OpzN/LlnNKuWjy1iFLQ/UG35JqHlR/89MP1d96dUfkf6Dne3TuLQzOYEYshJ+Hx8mw==
pstree.remy@^1.1.2:
version "1.1.2"
resolved "https://registry.yarnpkg.com/pstree.remy/-/pstree.remy-1.1.2.tgz#4448bbeb4b2af1fed242afc8dc7416a6f504951a"
@ -6675,7 +6699,7 @@ punycode@^1.4.1:
resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.4.1.tgz#c0d5a63b2718800ad8e1eb0fa5269c84dd41845e"
integrity sha1-wNWmOycYgArY4esPpSachN1BhF4=
punycode@^2.1.0:
punycode@^2.1.0, punycode@^2.1.1:
version "2.1.1"
resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec"
integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==
@ -6958,6 +6982,11 @@ reflect-metadata@^0.1.12:
resolved "https://registry.yarnpkg.com/reflect-metadata/-/reflect-metadata-0.1.12.tgz#311bf0c6b63cd782f228a81abe146a2bfa9c56f2"
integrity sha512-n+IyV+nGz3+0q3/Yf1ra12KpCyi001bi4XFxSjbiWWjfqb52iTTtpGXmCCAOWWIAn9KEuFZKGqBERHmrtScZ3A==
regenerator-runtime@^0.12.0:
version "0.12.1"
resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.12.1.tgz#fa1a71544764c036f8c49b13a08b2594c9f8a0de"
integrity sha512-odxIc1/vDlo4iZcfXqRYFj0vpXFNoGdKMAUieAlFYO6m/nl5e9KR/beGf41z4a1FI+aQgtjhuaSlDxQ0hmkrHg==
regex-not@^1.0.0, regex-not@^1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/regex-not/-/regex-not-1.0.2.tgz#1f4ece27e00b0b65e0247a6810e6a85d83a5752c"
@ -7007,6 +7036,23 @@ repeat-string@^1.6.1:
resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637"
integrity sha1-jcrkcOHIirwtYA//Sndihtp15jc=
request-promise-core@1.1.1:
version "1.1.1"
resolved "https://registry.yarnpkg.com/request-promise-core/-/request-promise-core-1.1.1.tgz#3eee00b2c5aa83239cfb04c5700da36f81cd08b6"
integrity sha1-Pu4AssWqgyOc+wTFcA2jb4HNCLY=
dependencies:
lodash "^4.13.1"
request-promise@^4.2.2:
version "4.2.2"
resolved "https://registry.yarnpkg.com/request-promise/-/request-promise-4.2.2.tgz#d1ea46d654a6ee4f8ee6a4fea1018c22911904b4"
integrity sha1-0epG1lSm7k+O5qT+oQGMIpEZBLQ=
dependencies:
bluebird "^3.5.0"
request-promise-core "1.1.1"
stealthy-require "^1.1.0"
tough-cookie ">=2.3.3"
request@^2.74.0, request@^2.81.0, request@^2.83.0, request@^2.87.0, request@^2.88.0:
version "2.88.0"
resolved "https://registry.yarnpkg.com/request/-/request-2.88.0.tgz#9c2fca4f7d35b592efe57c7f0a55e81052124fef"
@ -7924,6 +7970,11 @@ statuses@~1.4.0:
resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.4.0.tgz#bb73d446da2796106efcc1b601a253d6c46bd087"
integrity sha512-zhSCtt8v2NDrRlPQpCNtw/heZLtfUDqxBM1udqikb/Hbk52LK4nQSwr10u77iopCW5LsyHpuXS0GnEc48mLeew==
stealthy-require@^1.1.0:
version "1.1.1"
resolved "https://registry.yarnpkg.com/stealthy-require/-/stealthy-require-1.1.1.tgz#35b09875b4ff49f26a777e509b3090a3226bf24b"
integrity sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks=
stream-each@^1.1.0:
version "1.2.3"
resolved "https://registry.yarnpkg.com/stream-each/-/stream-each-1.2.3.tgz#ebe27a0c389b04fbcc233642952e10731afa9bae"
@ -8416,6 +8467,15 @@ touch@^3.1.0:
dependencies:
nopt "~1.0.10"
tough-cookie@>=2.3.3:
version "3.0.1"
resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-3.0.1.tgz#9df4f57e739c26930a018184887f4adb7dca73b2"
integrity sha512-yQyJ0u4pZsv9D4clxO69OEjLWYw+jbgspjTue4lTQZLfV0c5l1VmK2y1JK8E9ahdpltPOaAThPcp5nKPUgSnsg==
dependencies:
ip-regex "^2.1.0"
psl "^1.1.28"
punycode "^2.1.1"
tough-cookie@~2.4.3:
version "2.4.3"
resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.4.3.tgz#53f36da3f47783b0925afa06ff9f3b165280f781"