Use random names for VOD HLS playlists

scripts/optimize-old-videos.ts

@@ -19,13 +19,13 @@ run()
     process.exit(-1)
   })
 
-let currentVideoId = null
+let currentVideoId: string
-let currentFile = null
+let currentFilePath: string
 
 process.on('SIGINT', async function () {
   console.log('Cleaning up temp files')
-  await remove(`${currentFile}_backup`)
+  await remove(`${currentFilePath}_backup`)
-  await remove(`${dirname(currentFile)}/${currentVideoId}-transcoded.mp4`)
+  await remove(`${dirname(currentFilePath)}/${currentVideoId}-transcoded.mp4`)
   process.exit(0)
 })
 
@@ -40,12 +40,12 @@ async function run () {
     currentVideoId = video.id
 
     for (const file of video.VideoFiles) {
-      currentFile = getVideoFilePath(video, file)
+      currentFilePath = getVideoFilePath(video, file)
 
       const [ videoBitrate, fps, resolution ] = await Promise.all([
-        getVideoFileBitrate(currentFile),
+        getVideoFileBitrate(currentFilePath),
-        getVideoFileFPS(currentFile),
+        getVideoFileFPS(currentFilePath),
-        getVideoFileResolution(currentFile)
+        getVideoFileResolution(currentFilePath)
       ])
 
       const maxBitrate = getMaxBitrate(resolution.videoFileResolution, fps, VIDEO_TRANSCODING_FPS)
@@ -53,25 +53,27 @@ async function run () {
       if (isMaxBitrateExceeded) {
         console.log(
           'Optimizing video file %s with bitrate %s kbps (max: %s kbps)',
-          basename(currentFile), videoBitrate / 1000, maxBitrate / 1000
+          basename(currentFilePath), videoBitrate / 1000, maxBitrate / 1000
         )
 
-        const backupFile = `${currentFile}_backup`
+        const backupFile = `${currentFilePath}_backup`
-        await copy(currentFile, backupFile)
+        await copy(currentFilePath, backupFile)
 
         await optimizeOriginalVideofile(video, file)
+        // Update file path, the video filename changed
+        currentFilePath = getVideoFilePath(video, file)
 
         const originalDuration = await getDurationFromVideoFile(backupFile)
-        const newDuration = await getDurationFromVideoFile(currentFile)
+        const newDuration = await getDurationFromVideoFile(currentFilePath)
 
         if (originalDuration === newDuration) {
-          console.log('Finished optimizing %s', basename(currentFile))
+          console.log('Finished optimizing %s', basename(currentFilePath))
           await remove(backupFile)
           continue
         }
 
-        console.log('Failed to optimize %s, restoring original', basename(currentFile))
+        console.log('Failed to optimize %s, restoring original', basename(currentFilePath))
-        await move(backupFile, currentFile, { overwrite: true })
+        await move(backupFile, currentFilePath, { overwrite: true })
         await createTorrentAndSetInfoHash(video, file)
         await file.save()
       }

scripts/prune-storage.ts

@@ -2,11 +2,11 @@ import { registerTSPaths } from '../server/helpers/register-ts-paths'
 registerTSPaths()
 
 import * as prompt from 'prompt'
-import { join } from 'path'
+import { join, basename } from 'path'
 import { CONFIG } from '../server/initializers/config'
 import { VideoModel } from '../server/models/video/video'
 import { initDatabaseModels } from '../server/initializers/database'
-import { readdir, remove } from 'fs-extra'
+import { readdir, remove, stat } from 'fs-extra'
 import { VideoRedundancyModel } from '../server/models/redundancy/video-redundancy'
 import * as Bluebird from 'bluebird'
 import { getUUIDFromFilename } from '../server/helpers/utils'
@@ -14,6 +14,7 @@ import { ThumbnailModel } from '../server/models/video/thumbnail'
 import { ActorImageModel } from '../server/models/actor/actor-image'
 import { uniq, values } from 'lodash'
 import { ThumbnailType } from '@shared/models'
+import { VideoFileModel } from '@server/models/video/video-file'
 
 run()
   .then(() => process.exit(0))
@@ -37,8 +38,8 @@ async function run () {
   console.log('Detecting files to remove, it could take a while...')
 
   toDelete = toDelete.concat(
-    await pruneDirectory(CONFIG.STORAGE.VIDEOS_DIR, doesVideoExist(true)),
+    await pruneDirectory(CONFIG.STORAGE.VIDEOS_DIR, doesWebTorrentFileExist()),
-    await pruneDirectory(CONFIG.STORAGE.TORRENTS_DIR, doesVideoExist(true)),
+    await pruneDirectory(CONFIG.STORAGE.TORRENTS_DIR, doesTorrentFileExist()),
 
     await pruneDirectory(CONFIG.STORAGE.REDUNDANCY_DIR, doesRedundancyExist),
 
@@ -78,26 +79,27 @@ async function pruneDirectory (directory: string, existFun: ExistFun) {
 
   const toDelete: string[] = []
   await Bluebird.map(files, async file => {
-    if (await existFun(file) !== true) {
+    const filePath = join(directory, file)
-      toDelete.push(join(directory, file))
+
+    if (await existFun(filePath) !== true) {
+      toDelete.push(filePath)
     }
   }, { concurrency: 20 })
 
   return toDelete
 }
 
-function doesVideoExist (keepOnlyOwned: boolean) {
+function doesWebTorrentFileExist () {
-  return async (file: string) => {
+  return (filePath: string) => VideoFileModel.doesOwnedWebTorrentVideoFileExist(basename(filePath))
-    const uuid = getUUIDFromFilename(file)
+}
-    const video = await VideoModel.load(uuid)
 
-    return video && (keepOnlyOwned === false || video.isOwned())
+function doesTorrentFileExist () {
-  }
+  return (filePath: string) => VideoFileModel.doesOwnedTorrentFileExist(basename(filePath))
 }
 
 function doesThumbnailExist (keepOnlyOwned: boolean, type: ThumbnailType) {
-  return async (file: string) => {
+  return async (filePath: string) => {
-    const thumbnail = await ThumbnailModel.loadByFilename(file, type)
+    const thumbnail = await ThumbnailModel.loadByFilename(basename(filePath), type)
     if (!thumbnail) return false
 
     if (keepOnlyOwned) {
@@ -109,21 +111,20 @@ function doesThumbnailExist (keepOnlyOwned: boolean, type: ThumbnailType) {
   }
 }
 
-async function doesActorImageExist (file: string) {
+async function doesActorImageExist (filePath: string) {
-  const image = await ActorImageModel.loadByName(file)
+  const image = await ActorImageModel.loadByName(basename(filePath))
 
   return !!image
 }
 
-async function doesRedundancyExist (file: string) {
+async function doesRedundancyExist (filePath: string) {
-  const uuid = getUUIDFromFilename(file)
+  const isPlaylist = (await stat(filePath)).isDirectory()
-  const video = await VideoModel.loadWithFiles(uuid)
-
-  if (!video) return false
-
-  const isPlaylist = file.includes('.') === false
 
   if (isPlaylist) {
+    const uuid = getUUIDFromFilename(filePath)
+    const video = await VideoModel.loadWithFiles(uuid)
+    if (!video) return false
+
     const p = video.getHLSPlaylist()
     if (!p) return false
 
@@ -131,19 +132,10 @@ async function doesRedundancyExist (file: string) {
     return !!redundancy
   }
 
-  const resolution = parseInt(file.split('-')[5], 10)
+  const file = await VideoFileModel.loadByFilename(basename(filePath))
-  if (isNaN(resolution)) {
+  if (!file) return false
-    console.error('Cannot prune %s because we cannot guess guess the resolution.', file)
-    return true
-  }
 
-  const videoFile = video.getWebTorrentFile(resolution)
+  const redundancy = await VideoRedundancyModel.loadLocalByFileId(file.id)
-  if (!videoFile) {
-    console.error('Cannot find webtorrent file of video %s - %d', video.url, resolution)
-    return true
-  }
-
-  const redundancy = await VideoRedundancyModel.loadLocalByFileId(videoFile.id)
   return !!redundancy
 }
 

scripts/update-host.ts

@@ -16,7 +16,6 @@ import { VideoShareModel } from '../server/models/video/video-share'
 import { VideoCommentModel } from '../server/models/video/video-comment'
 import { AccountModel } from '../server/models/account/account'
 import { VideoChannelModel } from '../server/models/video/video-channel'
-import { VideoStreamingPlaylistModel } from '../server/models/video/video-streaming-playlist'
 import { initDatabaseModels } from '../server/initializers/database'
 import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
 import { getServerActor } from '@server/models/application/application'
@@ -128,13 +127,17 @@ async function run () {
     for (const file of video.VideoFiles) {
       console.log('Updating torrent file %s of video %s.', file.resolution, video.uuid)
       await createTorrentAndSetInfoHash(video, file)
+
+      await file.save()
     }
 
-    for (const playlist of video.VideoStreamingPlaylists) {
+    const playlist = video.getHLSPlaylist()
-      playlist.playlistUrl = WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid)
+    for (const file of (playlist?.VideoFiles || [])) {
-      playlist.segmentsSha256Url = WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid, video.isLive)
+      console.log('Updating fragmented torrent file %s of video %s.', file.resolution, video.uuid)
 
-      await playlist.save()
+      await createTorrentAndSetInfoHash(video, file)
+
+      await file.save()
     }
   }
 }

server/controllers/api/videos/upload.ts

@@ -209,10 +209,12 @@ async function addVideo (options: {
   })
 
   createTorrentFederate(video, videoFile)
+    .then(() => {
+      if (video.state !== VideoState.TO_TRANSCODE) return
 
-  if (video.state === VideoState.TO_TRANSCODE) {
+      return addOptimizeOrMergeAudioJob(videoCreated, videoFile, user)
-    await addOptimizeOrMergeAudioJob(videoCreated, videoFile, user)
+    })
-  }
+    .catch(err => logger.error('Cannot add optimize/merge audio job for %s.', videoCreated.uuid, { err, ...lTags(videoCreated.uuid) }))
 
   Hooks.runAction('action:api.video.uploaded', { video: videoCreated })
 
@@ -259,9 +261,9 @@ async function createTorrentAndSetInfoHashAsync (video: MVideo, fileArg: MVideoF
   return refreshedFile.save()
 }
 
-function createTorrentFederate (video: MVideoFullLight, videoFile: MVideoFile): void {
+function createTorrentFederate (video: MVideoFullLight, videoFile: MVideoFile) {
   // Create the torrent file in async way because it could be long
-  createTorrentAndSetInfoHashAsync(video, videoFile)
+  return createTorrentAndSetInfoHashAsync(video, videoFile)
     .catch(err => logger.error('Cannot create torrent file for video %s', video.url, { err, ...lTags(video.uuid) }))
     .then(() => VideoModel.loadAndPopulateAccountAndServerAndTags(video.id))
     .then(refreshedVideo => {

server/helpers/database-utils.ts

@@ -1,6 +1,6 @@
 import * as retry from 'async/retry'
 import * as Bluebird from 'bluebird'
-import { QueryTypes, Transaction } from 'sequelize'
+import { BindOrReplacements, QueryTypes, Transaction } from 'sequelize'
 import { Model } from 'sequelize-typescript'
 import { sequelizeTypescript } from '@server/initializers/database'
 import { logger } from './logger'
@@ -84,13 +84,15 @@ function resetSequelizeInstance (instance: Model<any>, savedFields: object) {
   })
 }
 
-function deleteNonExistingModels <T extends { hasSameUniqueKeysThan (other: T): boolean } & Pick<Model, 'destroy'>> (
+function filterNonExistingModels <T extends { hasSameUniqueKeysThan (other: T): boolean }> (
   fromDatabase: T[],
-  newModels: T[],
+  newModels: T[]
-  t: Transaction
 ) {
   return fromDatabase.filter(f => !newModels.find(newModel => newModel.hasSameUniqueKeysThan(f)))
-              .map(f => f.destroy({ transaction: t }))
+}
+
+function deleteAllModels <T extends Pick<Model, 'destroy'>> (models: T[], transaction: Transaction) {
+  return Promise.all(models.map(f => f.destroy({ transaction })))
 }
 
 // Sequelize always skip the update if we only update updatedAt field
@@ -121,13 +123,28 @@ function afterCommitIfTransaction (t: Transaction, fn: Function) {
 
 // ---------------------------------------------------------------------------
 
+function doesExist (query: string, bind?: BindOrReplacements) {
+  const options = {
+    type: QueryTypes.SELECT as QueryTypes.SELECT,
+    bind,
+    raw: true
+  }
+
+  return sequelizeTypescript.query(query, options)
+            .then(results => results.length === 1)
+}
+
+// ---------------------------------------------------------------------------
+
 export {
   resetSequelizeInstance,
   retryTransactionWrapper,
   transactionRetryer,
   updateInstanceWithAnother,
   afterCommitIfTransaction,
-  deleteNonExistingModels,
+  filterNonExistingModels,
+  deleteAllModels,
   setAsUpdated,
-  runInReadCommittedTransaction
+  runInReadCommittedTransaction,
+  doesExist
 }

server/helpers/ffmpeg-utils.ts

@@ -212,14 +212,17 @@ async function transcode (options: TranscodeOptions) {
 
 async function getLiveTranscodingCommand (options: {
   rtmpUrl: string
+
   outPath: string
+  masterPlaylistName: string
+
   resolutions: number[]
   fps: number
 
   availableEncoders: AvailableEncoders
   profile: string
 }) {
-  const { rtmpUrl, outPath, resolutions, fps, availableEncoders, profile } = options
+  const { rtmpUrl, outPath, resolutions, fps, availableEncoders, profile, masterPlaylistName } = options
   const input = rtmpUrl
 
   const command = getFFmpeg(input, 'live')
@@ -301,14 +304,14 @@ async function getLiveTranscodingCommand (options: {
 
   command.complexFilter(complexFilter)
 
-  addDefaultLiveHLSParams(command, outPath)
+  addDefaultLiveHLSParams(command, outPath, masterPlaylistName)
 
   command.outputOption('-var_stream_map', varStreamMap.join(' '))
 
   return command
 }
 
-function getLiveMuxingCommand (rtmpUrl: string, outPath: string) {
+function getLiveMuxingCommand (rtmpUrl: string, outPath: string, masterPlaylistName: string) {
   const command = getFFmpeg(rtmpUrl, 'live')
 
   command.outputOption('-c:v copy')
@@ -316,7 +319,7 @@ function getLiveMuxingCommand (rtmpUrl: string, outPath: string) {
   command.outputOption('-map 0:a?')
   command.outputOption('-map 0:v?')
 
-  addDefaultLiveHLSParams(command, outPath)
+  addDefaultLiveHLSParams(command, outPath, masterPlaylistName)
 
   return command
 }
@@ -371,12 +374,12 @@ function addDefaultEncoderParams (options: {
   }
 }
 
-function addDefaultLiveHLSParams (command: ffmpeg.FfmpegCommand, outPath: string) {
+function addDefaultLiveHLSParams (command: ffmpeg.FfmpegCommand, outPath: string, masterPlaylistName: string) {
   command.outputOption('-hls_time ' + VIDEO_LIVE.SEGMENT_TIME_SECONDS)
   command.outputOption('-hls_list_size ' + VIDEO_LIVE.SEGMENTS_LIST_SIZE)
   command.outputOption('-hls_flags delete_segments+independent_segments')
   command.outputOption(`-hls_segment_filename ${join(outPath, '%v-%06d.ts')}`)
-  command.outputOption('-master_pl_name master.m3u8')
+  command.outputOption('-master_pl_name ' + masterPlaylistName)
   command.outputOption(`-f hls`)
 
   command.output(join(outPath, '%v.m3u8'))

server/helpers/webtorrent.ts

@@ -103,6 +103,11 @@ async function createTorrentAndSetInfoHash (
 
   await writeFile(torrentPath, torrent)
 
+  // Remove old torrent file if it existed
+  if (videoFile.hasTorrent()) {
+    await remove(join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename))
+  }
+
   const parsedTorrent = parseTorrent(torrent)
   videoFile.infoHash = parsedTorrent.infoHash
   videoFile.torrentFilename = torrentFilename

server/initializers/constants.ts

@@ -24,7 +24,7 @@ import { CONFIG, registerConfigChangedHandler } from './config'
 
 // ---------------------------------------------------------------------------
 
-const LAST_MIGRATION_VERSION = 650
+const LAST_MIGRATION_VERSION = 655
 
 // ---------------------------------------------------------------------------
 

server/initializers/migrations/0655-streaming-playlist-filenames.ts

@@ -0,0 +1,66 @@
+import * as Sequelize from 'sequelize'
+
+async function up (utils: {
+  transaction: Sequelize.Transaction
+  queryInterface: Sequelize.QueryInterface
+  sequelize: Sequelize.Sequelize
+  db: any
+}): Promise<void> {
+  {
+    for (const column of [ 'playlistUrl', 'segmentsSha256Url' ]) {
+      const data = {
+        type: Sequelize.STRING,
+        allowNull: true,
+        defaultValue: null
+      }
+
+      await utils.queryInterface.changeColumn('videoStreamingPlaylist', column, data)
+    }
+  }
+
+  {
+    await utils.sequelize.query(
+      `UPDATE "videoStreamingPlaylist" SET "playlistUrl" = NULL, "segmentsSha256Url" = NULL ` +
+      `WHERE "videoId" IN (SELECT id FROM video WHERE remote IS FALSE)`
+    )
+  }
+
+  {
+    for (const column of [ 'playlistFilename', 'segmentsSha256Filename' ]) {
+      const data = {
+        type: Sequelize.STRING,
+        allowNull: true,
+        defaultValue: null
+      }
+
+      await utils.queryInterface.addColumn('videoStreamingPlaylist', column, data)
+    }
+  }
+
+  {
+    await utils.sequelize.query(
+      `UPDATE "videoStreamingPlaylist" SET "playlistFilename" = 'master.m3u8', "segmentsSha256Filename" = 'segments-sha256.json'`
+    )
+  }
+
+  {
+    for (const column of [ 'playlistFilename', 'segmentsSha256Filename' ]) {
+      const data = {
+        type: Sequelize.STRING,
+        allowNull: false,
+        defaultValue: null
+      }
+
+      await utils.queryInterface.changeColumn('videoStreamingPlaylist', column, data)
+    }
+  }
+}
+
+function down (options) {
+  throw new Error('Not implemented.')
+}
+
+export {
+  up,
+  down
+}

server/lib/activitypub/videos/shared/abstract-builder.ts

@@ -1,6 +1,6 @@
 import { Transaction } from 'sequelize/types'
 import { checkUrlsSameHost } from '@server/helpers/activitypub'
-import { deleteNonExistingModels } from '@server/helpers/database-utils'
+import { deleteAllModels, filterNonExistingModels } from '@server/helpers/database-utils'
 import { logger, LoggerTagsFn } from '@server/helpers/logger'
 import { updatePlaceholderThumbnail, updateVideoMiniatureFromUrl } from '@server/lib/thumbnail'
 import { setVideoTags } from '@server/lib/video'
@@ -111,8 +111,7 @@ export abstract class APVideoAbstractBuilder {
     const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a))
 
     // Remove video files that do not exist anymore
-    const destroyTasks = deleteNonExistingModels(video.VideoFiles || [], newVideoFiles, t)
+    await deleteAllModels(filterNonExistingModels(video.VideoFiles || [], newVideoFiles), t)
-    await Promise.all(destroyTasks)
 
     // Update or add other one
     const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'video', t))
@@ -124,13 +123,11 @@ export abstract class APVideoAbstractBuilder {
     const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a))
 
     // Remove video playlists that do not exist anymore
-    const destroyTasks = deleteNonExistingModels(video.VideoStreamingPlaylists || [], newStreamingPlaylists, t)
+    await deleteAllModels(filterNonExistingModels(video.VideoStreamingPlaylists || [], newStreamingPlaylists), t)
-    await Promise.all(destroyTasks)
 
     video.VideoStreamingPlaylists = []
 
     for (const playlistAttributes of streamingPlaylistAttributes) {
-
       const streamingPlaylistModel = await this.insertOrReplaceStreamingPlaylist(playlistAttributes, t)
       streamingPlaylistModel.Video = video
 
@@ -163,8 +160,7 @@ export abstract class APVideoAbstractBuilder {
 
     const newVideoFiles: MVideoFile[] = getFileAttributesFromUrl(playlistModel, tagObjects).map(a => new VideoFileModel(a))
 
-    const destroyTasks = deleteNonExistingModels(oldStreamingPlaylistFiles, newVideoFiles, t)
+    await deleteAllModels(filterNonExistingModels(oldStreamingPlaylistFiles, newVideoFiles), t)
-    await Promise.all(destroyTasks)
 
     // Update or add other one
     const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'streaming-playlist', t))

server/lib/activitypub/videos/shared/object-to-model-attributes.ts

@@ -7,10 +7,11 @@ import { logger } from '@server/helpers/logger'
 import { getExtFromMimetype } from '@server/helpers/video'
 import { ACTIVITY_PUB, MIMETYPES, P2P_MEDIA_LOADER_PEER_VERSION, PREVIEWS_SIZE, THUMBNAILS_SIZE } from '@server/initializers/constants'
 import { generateTorrentFileName } from '@server/lib/video-paths'
+import { VideoCaptionModel } from '@server/models/video/video-caption'
 import { VideoFileModel } from '@server/models/video/video-file'
 import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
 import { FilteredModelAttributes } from '@server/types'
-import { MChannelId, MStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoId } from '@server/types/models'
+import { isStreamingPlaylist, MChannelId, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoId } from '@server/types/models'
 import {
   ActivityHashTagObject,
   ActivityMagnetUrlObject,
@@ -23,7 +24,6 @@ import {
   VideoPrivacy,
   VideoStreamingPlaylistType
 } from '@shared/models'
-import { VideoCaptionModel } from '@server/models/video/video-caption'
 
 function getThumbnailFromIcons (videoObject: VideoObject) {
   let validIcons = videoObject.icon.filter(i => i.width > THUMBNAILS_SIZE.minWidth)
@@ -80,8 +80,8 @@ function getFileAttributesFromUrl (
 
     const extname = getExtFromMimetype(MIMETYPES.VIDEO.MIMETYPE_EXT, fileUrl.mediaType)
     const resolution = fileUrl.height
-    const videoId = (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? null : videoOrPlaylist.id
+    const videoId = isStreamingPlaylist(videoOrPlaylist) ? null : videoOrPlaylist.id
-    const videoStreamingPlaylistId = (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? videoOrPlaylist.id : null
+    const videoStreamingPlaylistId = isStreamingPlaylist(videoOrPlaylist) ? videoOrPlaylist.id : null
 
     const attribute = {
       extname,
@@ -130,8 +130,13 @@ function getStreamingPlaylistAttributesFromObject (video: MVideoId, videoObject:
 
     const attribute = {
       type: VideoStreamingPlaylistType.HLS,
+
+      playlistFilename: basename(playlistUrlObject.href),
       playlistUrl: playlistUrlObject.href,
+
+      segmentsSha256Filename: basename(segmentsSha256UrlObject.href),
       segmentsSha256Url: segmentsSha256UrlObject.href,
+
       p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrlObject.href, files),
       p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION,
       videoId: video.id,

server/lib/hls.ts

@@ -1,7 +1,7 @@
 import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, writeFile } from 'fs-extra'
 import { flatten, uniq } from 'lodash'
 import { basename, dirname, join } from 'path'
-import { MVideoWithFile } from '@server/types/models'
+import { MStreamingPlaylistFilesVideo, MVideoWithFile } from '@server/types/models'
 import { sha256 } from '../helpers/core-utils'
 import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamSize } from '../helpers/ffprobe-utils'
 import { logger } from '../helpers/logger'
@@ -12,7 +12,7 @@ import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION } from
 import { sequelizeTypescript } from '../initializers/database'
 import { VideoFileModel } from '../models/video/video-file'
 import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
-import { getVideoFilePath } from './video-paths'
+import { getHlsResolutionPlaylistFilename, getVideoFilePath } from './video-paths'
 
 async function updateStreamingPlaylistsInfohashesIfNeeded () {
   const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion()
@@ -22,27 +22,29 @@ async function updateStreamingPlaylistsInfohashesIfNeeded () {
     await sequelizeTypescript.transaction(async t => {
       const videoFiles = await VideoFileModel.listByStreamingPlaylist(playlist.id, t)
 
-      playlist.p2pMediaLoaderInfohashes = VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlist.playlistUrl, videoFiles)
+      playlist.assignP2PMediaLoaderInfoHashes(playlist.Video, videoFiles)
       playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
+
       await playlist.save({ transaction: t })
     })
   }
 }
 
-async function updateMasterHLSPlaylist (video: MVideoWithFile) {
+async function updateMasterHLSPlaylist (video: MVideoWithFile, playlist: MStreamingPlaylistFilesVideo) {
   const directory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)
-  const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ]
-  const masterPlaylistPath = join(directory, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename())
-  const streamingPlaylist = video.getHLSPlaylist()
 
-  for (const file of streamingPlaylist.VideoFiles) {
+  const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ]
-    const playlistFilename = VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution)
+
+  const masterPlaylistPath = join(directory, playlist.playlistFilename)
+
+  for (const file of playlist.VideoFiles) {
+    const playlistFilename = getHlsResolutionPlaylistFilename(file.filename)
 
     // If we did not generated a playlist for this resolution, skip
     const filePlaylistPath = join(directory, playlistFilename)
     if (await pathExists(filePlaylistPath) === false) continue
 
-    const videoFilePath = getVideoFilePath(streamingPlaylist, file)
+    const videoFilePath = getVideoFilePath(playlist, file)
 
     const size = await getVideoStreamSize(videoFilePath)
 
@@ -66,23 +68,22 @@ async function updateMasterHLSPlaylist (video: MVideoWithFile) {
   await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n')
 }
 
-async function updateSha256VODSegments (video: MVideoWithFile) {
+async function updateSha256VODSegments (video: MVideoWithFile, playlist: MStreamingPlaylistFilesVideo) {
   const json: { [filename: string]: { [range: string]: string } } = {}
 
   const playlistDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)
-  const hlsPlaylist = video.getHLSPlaylist()
 
   // For all the resolutions available for this video
-  for (const file of hlsPlaylist.VideoFiles) {
+  for (const file of playlist.VideoFiles) {
     const rangeHashes: { [range: string]: string } = {}
 
-    const videoPath = getVideoFilePath(hlsPlaylist, file)
+    const videoPath = getVideoFilePath(playlist, file)
-    const playlistPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution))
+    const resolutionPlaylistPath = join(playlistDirectory, getHlsResolutionPlaylistFilename(file.filename))
 
     // Maybe the playlist is not generated for this resolution yet
-    if (!await pathExists(playlistPath)) continue
+    if (!await pathExists(resolutionPlaylistPath)) continue
 
-    const playlistContent = await readFile(playlistPath)
+    const playlistContent = await readFile(resolutionPlaylistPath)
     const ranges = getRangesFromPlaylist(playlistContent.toString())
 
     const fd = await open(videoPath, 'r')
@@ -98,7 +99,7 @@ async function updateSha256VODSegments (video: MVideoWithFile) {
     json[videoFilename] = rangeHashes
   }
 
-  const outputPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename())
+  const outputPath = join(playlistDirectory, playlist.segmentsSha256Filename)
   await outputJSON(outputPath, json)
 }
 

server/lib/job-queue/handlers/video-file-import.ts

@@ -61,8 +61,7 @@ async function updateVideoFile (video: MVideoFullLight, inputFilePath: string) {
 
   if (currentVideoFile) {
     // Remove old file and old torrent
-    await video.removeFile(currentVideoFile)
+    await video.removeFileAndTorrent(currentVideoFile)
-    await currentVideoFile.removeTorrent()
     // Remove the old video file from the array
     video.VideoFiles = video.VideoFiles.filter(f => f !== currentVideoFile)
 

server/lib/job-queue/handlers/video-live-ending.ts

@@ -7,12 +7,12 @@ import { buildConcatenatedName, cleanupLive, LiveSegmentShaStore } from '@server
 import { generateVideoMiniature } from '@server/lib/thumbnail'
 import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/video-transcoding'
 import { publishAndFederateIfNeeded } from '@server/lib/video'
-import { getHLSDirectory } from '@server/lib/video-paths'
+import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHLSDirectory } from '@server/lib/video-paths'
 import { VideoModel } from '@server/models/video/video'
 import { VideoFileModel } from '@server/models/video/video-file'
 import { VideoLiveModel } from '@server/models/video/video-live'
 import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
-import { MVideo, MVideoLive } from '@server/types/models'
+import { MStreamingPlaylist, MVideo, MVideoLive } from '@server/types/models'
 import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models'
 import { logger } from '../../../helpers/logger'
 
@@ -43,7 +43,7 @@ async function processVideoLiveEnding (job: Bull.Job) {
     return cleanupLive(video, streamingPlaylist)
   }
 
-  return saveLive(video, live)
+  return saveLive(video, live, streamingPlaylist)
 }
 
 // ---------------------------------------------------------------------------
@@ -54,14 +54,14 @@ export {
 
 // ---------------------------------------------------------------------------
 
-async function saveLive (video: MVideo, live: MVideoLive) {
+async function saveLive (video: MVideo, live: MVideoLive, streamingPlaylist: MStreamingPlaylist) {
   const hlsDirectory = getHLSDirectory(video, false)
   const replayDirectory = join(hlsDirectory, VIDEO_LIVE.REPLAY_DIRECTORY)
 
   const rootFiles = await readdir(hlsDirectory)
 
   const playlistFiles = rootFiles.filter(file => {
-    return file.endsWith('.m3u8') && file !== 'master.m3u8'
+    return file.endsWith('.m3u8') && file !== streamingPlaylist.playlistFilename
   })
 
   await cleanupLiveFiles(hlsDirectory)
@@ -80,7 +80,12 @@ async function saveLive (video: MVideo, live: MVideoLive) {
 
   const hlsPlaylist = videoWithFiles.getHLSPlaylist()
   await VideoFileModel.removeHLSFilesOfVideoId(hlsPlaylist.id)
+
+  // Reset playlist
   hlsPlaylist.VideoFiles = []
+  hlsPlaylist.playlistFilename = generateHLSMasterPlaylistFilename()
+  hlsPlaylist.segmentsSha256Filename = generateHlsSha256SegmentsFilename()
+  await hlsPlaylist.save()
 
   let durationDone = false
 

server/lib/job-queue/handlers/video-transcoding.ts

@@ -125,8 +125,7 @@ async function onHlsPlaylistGeneration (video: MVideoFullLight, user: MUser, pay
   if (payload.isMaxQuality && CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false) {
     // Remove webtorrent files if not enabled
     for (const file of video.VideoFiles) {
-      await video.removeFile(file)
+      await video.removeFileAndTorrent(file)
-      await file.removeTorrent()
       await file.destroy()
     }
 

server/lib/live/live-manager.ts

@@ -4,16 +4,17 @@ import { isTestInstance } from '@server/helpers/core-utils'
 import { computeResolutionsToTranscode, getVideoFileFPS, getVideoFileResolution } from '@server/helpers/ffprobe-utils'
 import { logger, loggerTagsFactory } from '@server/helpers/logger'
 import { CONFIG, registerConfigChangedHandler } from '@server/initializers/config'
-import { P2P_MEDIA_LOADER_PEER_VERSION, VIDEO_LIVE, VIEW_LIFETIME, WEBSERVER } from '@server/initializers/constants'
+import { P2P_MEDIA_LOADER_PEER_VERSION, VIDEO_LIVE, VIEW_LIFETIME } from '@server/initializers/constants'
 import { UserModel } from '@server/models/user/user'
 import { VideoModel } from '@server/models/video/video'
 import { VideoLiveModel } from '@server/models/video/video-live'
 import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
-import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoLiveVideo } from '@server/types/models'
+import { MStreamingPlaylistVideo, MVideo, MVideoLiveVideo } from '@server/types/models'
 import { VideoState, VideoStreamingPlaylistType } from '@shared/models'
 import { federateVideoIfNeeded } from '../activitypub/videos'
 import { JobQueue } from '../job-queue'
 import { PeerTubeSocket } from '../peertube-socket'
+import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename } from '../video-paths'
 import { LiveQuotaStore } from './live-quota-store'
 import { LiveSegmentShaStore } from './live-segment-sha-store'
 import { cleanupLive } from './live-utils'
@@ -392,19 +393,18 @@ class LiveManager {
     return resolutionsEnabled.concat([ originResolution ])
   }
 
-  private async createLivePlaylist (video: MVideo, allResolutions: number[]) {
+  private async createLivePlaylist (video: MVideo, allResolutions: number[]): Promise<MStreamingPlaylistVideo> {
-    const playlistUrl = WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid)
+    const playlist = await VideoStreamingPlaylistModel.loadOrGenerate(video)
-    const [ videoStreamingPlaylist ] = await VideoStreamingPlaylistModel.upsert({
-      videoId: video.id,
-      playlistUrl,
-      segmentsSha256Url: WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid, video.isLive),
-      p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrl, allResolutions),
-      p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION,
 
-      type: VideoStreamingPlaylistType.HLS
+    playlist.playlistFilename = generateHLSMasterPlaylistFilename(true)
-    }, { returning: true }) as [ MStreamingPlaylist, boolean ]
+    playlist.segmentsSha256Filename = generateHlsSha256SegmentsFilename(true)
 
-    return Object.assign(videoStreamingPlaylist, { Video: video })
+    playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
+    playlist.type = VideoStreamingPlaylistType.HLS
+
+    playlist.assignP2PMediaLoaderInfoHashes(video, allResolutions)
+
+    return playlist.save()
   }
 
   static get Instance () {

server/lib/live/shared/muxing-session.ts

@@ -112,13 +112,16 @@ class MuxingSession extends EventEmitter {
     this.ffmpegCommand = CONFIG.LIVE.TRANSCODING.ENABLED
       ? await getLiveTranscodingCommand({
         rtmpUrl: this.rtmpUrl,
+
         outPath,
+        masterPlaylistName: this.streamingPlaylist.playlistFilename,
+
         resolutions: this.allResolutions,
         fps: this.fps,
         availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
         profile: CONFIG.LIVE.TRANSCODING.PROFILE
       })
-      : getLiveMuxingCommand(this.rtmpUrl, outPath)
+      : getLiveMuxingCommand(this.rtmpUrl, outPath, this.streamingPlaylist.playlistFilename)
 
     logger.info('Running live muxing/transcoding for %s.', this.videoUUID, this.lTags)
 
@@ -182,7 +185,7 @@ class MuxingSession extends EventEmitter {
   }
 
   private watchMasterFile (outPath: string) {
-    this.masterWatcher = chokidar.watch(outPath + '/master.m3u8')
+    this.masterWatcher = chokidar.watch(outPath + '/' + this.streamingPlaylist.playlistFilename)
 
     this.masterWatcher.on('add', async () => {
       this.emit('master-playlist-created', { videoId: this.videoId })

server/lib/schedulers/videos-redundancy-scheduler.ts

@@ -267,7 +267,8 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
     logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, strategy)
 
     const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid)
-    await downloadPlaylistSegments(playlist.playlistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT)
+    const masterPlaylistUrl = playlist.getMasterPlaylistUrl(video)
+    await downloadPlaylistSegments(masterPlaylistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT)
 
     const createdModel: MVideoRedundancyStreamingPlaylistVideo = await VideoRedundancyModel.create({
       expiresOn,
@@ -282,7 +283,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
 
     await sendCreateCacheFile(serverActor, video, createdModel)
 
-    logger.info('Duplicated playlist %s -> %s.', playlist.playlistUrl, createdModel.url)
+    logger.info('Duplicated playlist %s -> %s.', masterPlaylistUrl, createdModel.url)
   }
 
   private async extendsExpirationOf (redundancy: MVideoRedundancyVideo, expiresAfterMs: number) {
@@ -330,7 +331,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
   private buildEntryLogId (object: MVideoRedundancyFileVideo | MVideoRedundancyStreamingPlaylistVideo) {
     if (isMVideoRedundancyFileVideo(object)) return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}`
 
-    return `${object.VideoStreamingPlaylist.playlistUrl}`
+    return `${object.VideoStreamingPlaylist.getMasterPlaylistUrl(object.VideoStreamingPlaylist.Video)}`
   }
 
   private getTotalFileSizes (files: MVideoFile[], playlists: MStreamingPlaylistFiles[]) {

server/lib/transcoding/video-transcoding.ts

@@ -10,11 +10,18 @@ import { transcode, TranscodeOptions, TranscodeOptionsType } from '../../helpers
 import { canDoQuickTranscode, getDurationFromVideoFile, getMetadataFromFile, getVideoFileFPS } from '../../helpers/ffprobe-utils'
 import { logger } from '../../helpers/logger'
 import { CONFIG } from '../../initializers/config'
-import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION, WEBSERVER } from '../../initializers/constants'
+import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION } from '../../initializers/constants'
 import { VideoFileModel } from '../../models/video/video-file'
 import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
 import { updateMasterHLSPlaylist, updateSha256VODSegments } from '../hls'
-import { generateHLSVideoFilename, generateWebTorrentVideoFilename, getVideoFilePath } from '../video-paths'
+import {
+  generateHLSMasterPlaylistFilename,
+  generateHlsSha256SegmentsFilename,
+  generateHLSVideoFilename,
+  generateWebTorrentVideoFilename,
+  getHlsResolutionPlaylistFilename,
+  getVideoFilePath
+} from '../video-paths'
 import { VideoTranscodingProfilesManager } from './video-transcoding-profiles'
 
 /**
@@ -272,14 +279,14 @@ async function generateHlsPlaylistCommon (options: {
   await ensureDir(videoTranscodedBasePath)
 
   const videoFilename = generateHLSVideoFilename(resolution)
-  const playlistFilename = VideoStreamingPlaylistModel.getHlsPlaylistFilename(resolution)
+  const resolutionPlaylistFilename = getHlsResolutionPlaylistFilename(videoFilename)
-  const playlistFileTranscodePath = join(videoTranscodedBasePath, playlistFilename)
+  const resolutionPlaylistFileTranscodePath = join(videoTranscodedBasePath, resolutionPlaylistFilename)
 
   const transcodeOptions = {
     type,
 
     inputPath,
-    outputPath: playlistFileTranscodePath,
+    outputPath: resolutionPlaylistFileTranscodePath,
 
     availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
     profile: CONFIG.TRANSCODING.PROFILE,
@@ -299,19 +306,23 @@ async function generateHlsPlaylistCommon (options: {
 
   await transcode(transcodeOptions)
 
-  const playlistUrl = WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid)
-
   // Create or update the playlist
-  const [ videoStreamingPlaylist ] = await VideoStreamingPlaylistModel.upsert({
+  const playlist = await VideoStreamingPlaylistModel.loadOrGenerate(video)
-    videoId: video.id,
-    playlistUrl,
-    segmentsSha256Url: WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid, video.isLive),
-    p2pMediaLoaderInfohashes: [],
-    p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION,
 
-    type: VideoStreamingPlaylistType.HLS
+  if (!playlist.playlistFilename) {
-  }, { returning: true }) as [ MStreamingPlaylistFilesVideo, boolean ]
+    playlist.playlistFilename = generateHLSMasterPlaylistFilename(video.isLive)
-  videoStreamingPlaylist.Video = video
+  }
+
+  if (!playlist.segmentsSha256Filename) {
+    playlist.segmentsSha256Filename = generateHlsSha256SegmentsFilename(video.isLive)
+  }
+
+  playlist.p2pMediaLoaderInfohashes = []
+  playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
+
+  playlist.type = VideoStreamingPlaylistType.HLS
+
+  await playlist.save()
 
   // Build the new playlist file
   const extname = extnameUtil(videoFilename)
@@ -321,18 +332,18 @@ async function generateHlsPlaylistCommon (options: {
     size: 0,
     filename: videoFilename,
     fps: -1,
-    videoStreamingPlaylistId: videoStreamingPlaylist.id
+    videoStreamingPlaylistId: playlist.id
   })
 
-  const videoFilePath = getVideoFilePath(videoStreamingPlaylist, newVideoFile)
+  const videoFilePath = getVideoFilePath(playlist, newVideoFile)
 
   // Move files from tmp transcoded directory to the appropriate place
   const baseHlsDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)
   await ensureDir(baseHlsDirectory)
 
   // Move playlist file
-  const playlistPath = join(baseHlsDirectory, playlistFilename)
+  const resolutionPlaylistPath = join(baseHlsDirectory, resolutionPlaylistFilename)
-  await move(playlistFileTranscodePath, playlistPath, { overwrite: true })
+  await move(resolutionPlaylistFileTranscodePath, resolutionPlaylistPath, { overwrite: true })
   // Move video file
   await move(join(videoTranscodedBasePath, videoFilename), videoFilePath, { overwrite: true })
 
@@ -342,20 +353,20 @@ async function generateHlsPlaylistCommon (options: {
   newVideoFile.fps = await getVideoFileFPS(videoFilePath)
   newVideoFile.metadata = await getMetadataFromFile(videoFilePath)
 
-  await createTorrentAndSetInfoHash(videoStreamingPlaylist, newVideoFile)
+  await createTorrentAndSetInfoHash(playlist, newVideoFile)
 
   await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined)
-  videoStreamingPlaylist.VideoFiles = await videoStreamingPlaylist.$get('VideoFiles')
 
-  videoStreamingPlaylist.p2pMediaLoaderInfohashes = VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(
+  const playlistWithFiles = playlist as MStreamingPlaylistFilesVideo
-    playlistUrl, videoStreamingPlaylist.VideoFiles
+  playlistWithFiles.VideoFiles = await playlist.$get('VideoFiles')
-  )
+  playlist.assignP2PMediaLoaderInfoHashes(video, playlistWithFiles.VideoFiles)
-  await videoStreamingPlaylist.save()
 
-  video.setHLSPlaylist(videoStreamingPlaylist)
+  await playlist.save()
 
-  await updateMasterHLSPlaylist(video)
+  video.setHLSPlaylist(playlist)
-  await updateSha256VODSegments(video)
 
-  return playlistPath
+  await updateMasterHLSPlaylist(video, playlistWithFiles)
+  await updateSha256VODSegments(video, playlistWithFiles)
+
+  return resolutionPlaylistPath
 }

server/lib/video-paths.ts

@@ -4,19 +4,16 @@ import { CONFIG } from '@server/initializers/config'
 import { HLS_REDUNDANCY_DIRECTORY, HLS_STREAMING_PLAYLIST_DIRECTORY, STATIC_PATHS, WEBSERVER } from '@server/initializers/constants'
 import { isStreamingPlaylist, MStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models'
 import { buildUUID } from '@server/helpers/uuid'
+import { removeFragmentedMP4Ext } from '@shared/core-utils'
 
 // ################## Video file name ##################
 
 function generateWebTorrentVideoFilename (resolution: number, extname: string) {
-  const uuid = buildUUID()
+  return buildUUID() + '-' + resolution + extname
-
-  return uuid + '-' + resolution + extname
 }
 
 function generateHLSVideoFilename (resolution: number) {
-  const uuid = buildUUID()
+  return `${buildUUID()}-${resolution}-fragmented.mp4`
-
-  return `${uuid}-${resolution}-fragmented.mp4`
 }
 
 function getVideoFilePath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile, isRedundancy = false) {
@@ -54,6 +51,23 @@ function getHLSDirectory (video: MVideoUUID, isRedundancy = false) {
   return join(baseDir, video.uuid)
 }
 
+function getHlsResolutionPlaylistFilename (videoFilename: string) {
+  // Video file name already contain resolution
+  return removeFragmentedMP4Ext(videoFilename) + '.m3u8'
+}
+
+function generateHLSMasterPlaylistFilename (isLive = false) {
+  if (isLive) return 'master.m3u8'
+
+  return buildUUID() + '-master.m3u8'
+}
+
+function generateHlsSha256SegmentsFilename (isLive = false) {
+  if (isLive) return 'segments-sha256.json'
+
+  return buildUUID() + '-segments-sha256.json'
+}
+
 // ################## Torrents ##################
 
 function generateTorrentFileName (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, resolution: number) {
@@ -91,6 +105,9 @@ export {
   getTorrentFilePath,
 
   getHLSDirectory,
+  generateHLSMasterPlaylistFilename,
+  generateHlsSha256SegmentsFilename,
+  getHlsResolutionPlaylistFilename,
 
   getLocalVideoFileMetadataUrl,
 

server/lib/video.ts

@@ -5,7 +5,7 @@ import { sequelizeTypescript } from '@server/initializers/database'
 import { TagModel } from '@server/models/video/tag'
 import { VideoModel } from '@server/models/video/video'
 import { FilteredModelAttributes } from '@server/types'
-import { MThumbnail, MUserId, MVideo, MVideoFile, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models'
+import { MThumbnail, MUserId, MVideoFile, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models'
 import { ThumbnailType, VideoCreate, VideoPrivacy, VideoTranscodingPayload } from '@shared/models'
 import { federateVideoIfNeeded } from './activitypub/videos'
 import { JobQueue } from './job-queue/job-queue'
@@ -105,7 +105,7 @@ async function publishAndFederateIfNeeded (video: MVideoUUID, wasLive = false) {
   }
 }
 
-async function addOptimizeOrMergeAudioJob (video: MVideo, videoFile: MVideoFile, user: MUserId) {
+async function addOptimizeOrMergeAudioJob (video: MVideoUUID, videoFile: MVideoFile, user: MUserId) {
   let dataInput: VideoTranscodingPayload
 
   if (videoFile.isAudio()) {

server/models/actor/actor-follow.ts

@@ -19,8 +19,8 @@ import {
   UpdatedAt
 } from 'sequelize-typescript'
 import { isActivityPubUrlValid } from '@server/helpers/custom-validators/activitypub/misc'
+import { doesExist } from '@server/helpers/database-utils'
 import { getServerActor } from '@server/models/application/application'
-import { VideoModel } from '@server/models/video/video'
 import {
   MActorFollowActorsDefault,
   MActorFollowActorsDefaultSubscription,
@@ -166,14 +166,8 @@ export class ActorFollowModel extends Model<Partial<AttributesOnly<ActorFollowMo
 
   static isFollowedBy (actorId: number, followerActorId: number) {
     const query = 'SELECT 1 FROM "actorFollow" WHERE "actorId" = $followerActorId AND "targetActorId" = $actorId LIMIT 1'
-    const options = {
-      type: QueryTypes.SELECT as QueryTypes.SELECT,
-      bind: { actorId, followerActorId },
-      raw: true
-    }
 
-    return VideoModel.sequelize.query(query, options)
+    return doesExist(query, { actorId, followerActorId })
-                     .then(results => results.length === 1)
   }
 
   static loadByActorAndTarget (actorId: number, targetActorId: number, t?: Transaction): Promise<MActorFollowActorsDefault> {

server/models/redundancy/video-redundancy.ts

@@ -160,8 +160,8 @@ export class VideoRedundancyModel extends Model<Partial<AttributesOnly<VideoRedu
       const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}`
       logger.info('Removing duplicated video file %s.', logIdentifier)
 
-      videoFile.Video.removeFile(videoFile, true)
+      videoFile.Video.removeFileAndTorrent(videoFile, true)
-               .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
+        .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
     }
 
     if (instance.videoStreamingPlaylistId) {

server/models/video/formatter/video-format-utils.ts

@@ -182,8 +182,8 @@ function streamingPlaylistsModelToFormattedJSON (
       return {
         id: playlist.id,
         type: playlist.type,
-        playlistUrl: playlist.playlistUrl,
+        playlistUrl: playlist.getMasterPlaylistUrl(video),
-        segmentsSha256Url: playlist.segmentsSha256Url,
+        segmentsSha256Url: playlist.getSha256SegmentsUrl(video),
         redundancies,
         files
       }
@@ -331,7 +331,7 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject {
       type: 'Link',
       name: 'sha256',
       mediaType: 'application/json' as 'application/json',
-      href: playlist.segmentsSha256Url
+      href: playlist.getSha256SegmentsUrl(video)
     })
 
     addVideoFilesInAPAcc(tag, video, playlist.VideoFiles || [])
@@ -339,7 +339,7 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject {
     url.push({
       type: 'Link',
       mediaType: 'application/x-mpegURL' as 'application/x-mpegURL',
-      href: playlist.playlistUrl,
+      href: playlist.getMasterPlaylistUrl(video),
       tag
     })
   }

server/models/video/sql/shared/video-tables.ts

@@ -92,12 +92,13 @@ export class VideoTables {
   }
 
   getStreamingPlaylistAttributes () {
-    let playlistKeys = [ 'id', 'playlistUrl', 'type' ]
+    let playlistKeys = [ 'id', 'playlistUrl', 'playlistFilename', 'type' ]
 
     if (this.mode === 'get') {
       playlistKeys = playlistKeys.concat([
         'p2pMediaLoaderInfohashes',
         'p2pMediaLoaderPeerVersion',
+        'segmentsSha256Filename',
         'segmentsSha256Url',
         'videoId',
         'createdAt',

server/models/video/video-file.ts

@@ -1,7 +1,7 @@
 import { remove } from 'fs-extra'
 import * as memoizee from 'memoizee'
 import { join } from 'path'
-import { FindOptions, Op, QueryTypes, Transaction } from 'sequelize'
+import { FindOptions, Op, Transaction } from 'sequelize'
 import {
   AllowNull,
   BelongsTo,
@@ -21,6 +21,7 @@ import {
 import { Where } from 'sequelize/types/lib/utils'
 import validator from 'validator'
 import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub'
+import { doesExist } from '@server/helpers/database-utils'
 import { logger } from '@server/helpers/logger'
 import { extractVideo } from '@server/helpers/video'
 import { getTorrentFilePath } from '@server/lib/video-paths'
@@ -250,14 +251,8 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
 
   static doesInfohashExist (infoHash: string) {
     const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1'
-    const options = {
-      type: QueryTypes.SELECT as QueryTypes.SELECT,
-      bind: { infoHash },
-      raw: true
-    }
 
-    return VideoModel.sequelize.query(query, options)
+    return doesExist(query, { infoHash })
-              .then(results => results.length === 1)
   }
 
   static async doesVideoExistForVideoFile (id: number, videoIdOrUUID: number | string) {
@@ -266,6 +261,33 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
     return !!videoFile
   }
 
+  static async doesOwnedTorrentFileExist (filename: string) {
+    const query = 'SELECT 1 FROM "videoFile" ' +
+                  'LEFT JOIN "video" "webtorrent" ON "webtorrent"."id" = "videoFile"."videoId" AND "webtorrent"."remote" IS FALSE ' +
+                  'LEFT JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."id" = "videoFile"."videoStreamingPlaylistId" ' +
+                  'LEFT JOIN "video" "hlsVideo" ON "hlsVideo"."id" = "videoStreamingPlaylist"."videoId" AND "hlsVideo"."remote" IS FALSE ' +
+                  'WHERE "torrentFilename" = $filename AND ("hlsVideo"."id" IS NOT NULL OR "webtorrent"."id" IS NOT NULL) LIMIT 1'
+
+    return doesExist(query, { filename })
+  }
+
+  static async doesOwnedWebTorrentVideoFileExist (filename: string) {
+    const query = 'SELECT 1 FROM "videoFile" INNER JOIN "video" ON "video"."id" = "videoFile"."videoId" AND "video"."remote" IS FALSE ' +
+                  'WHERE "filename" = $filename LIMIT 1'
+
+    return doesExist(query, { filename })
+  }
+
+  static loadByFilename (filename: string) {
+    const query = {
+      where: {
+        filename
+      }
+    }
+
+    return VideoFileModel.findOne(query)
+  }
+
   static loadWithVideoOrPlaylistByTorrentFilename (filename: string) {
     const query = {
       where: {
@@ -443,10 +465,9 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
   }
 
   getFileDownloadUrl (video: MVideoWithHost) {
-    const basePath = this.isHLS()
+    const path = this.isHLS()
-      ? STATIC_DOWNLOAD_PATHS.HLS_VIDEOS
+      ? join(STATIC_DOWNLOAD_PATHS.HLS_VIDEOS, `${video.uuid}-${this.resolution}-fragmented${this.extname}`)
-      : STATIC_DOWNLOAD_PATHS.VIDEOS
+      : join(STATIC_DOWNLOAD_PATHS.VIDEOS, `${video.uuid}-${this.resolution}${this.extname}`)
-    const path = join(basePath, this.filename)
 
     if (video.isOwned()) return WEBSERVER.URL + path
 

server/models/video/video-streaming-playlist.ts

@@ -1,19 +1,27 @@
 import * as memoizee from 'memoizee'
 import { join } from 'path'
-import { Op, QueryTypes } from 'sequelize'
+import { Op } from 'sequelize'
 import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, HasMany, Is, Model, Table, UpdatedAt } from 'sequelize-typescript'
+import { doesExist } from '@server/helpers/database-utils'
 import { VideoFileModel } from '@server/models/video/video-file'
-import { MStreamingPlaylist } from '@server/types/models'
+import { MStreamingPlaylist, MVideo } from '@server/types/models'
+import { AttributesOnly } from '@shared/core-utils'
 import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
 import { sha1 } from '../../helpers/core-utils'
 import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
 import { isArrayOf } from '../../helpers/custom-validators/misc'
 import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
-import { CONSTRAINTS_FIELDS, MEMOIZE_LENGTH, MEMOIZE_TTL, P2P_MEDIA_LOADER_PEER_VERSION, STATIC_PATHS } from '../../initializers/constants'
+import {
+  CONSTRAINTS_FIELDS,
+  MEMOIZE_LENGTH,
+  MEMOIZE_TTL,
+  P2P_MEDIA_LOADER_PEER_VERSION,
+  STATIC_PATHS,
+  WEBSERVER
+} from '../../initializers/constants'
 import { VideoRedundancyModel } from '../redundancy/video-redundancy'
 import { throwIfNotValid } from '../utils'
 import { VideoModel } from './video'
-import { AttributesOnly } from '@shared/core-utils'
 
 @Table({
   tableName: 'videoStreamingPlaylist',
@@ -43,7 +51,11 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
   type: VideoStreamingPlaylistType
 
   @AllowNull(false)
-  @Is('PlaylistUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'playlist url'))
+  @Column
+  playlistFilename: string
+
+  @AllowNull(true)
+  @Is('PlaylistUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'playlist url', true))
   @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS.URL.max))
   playlistUrl: string
 
@@ -57,7 +69,11 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
   p2pMediaLoaderPeerVersion: number
 
   @AllowNull(false)
-  @Is('VideoStreamingSegmentsSha256Url', value => throwIfNotValid(value, isActivityPubUrlValid, 'segments sha256 url'))
+  @Column
+  segmentsSha256Filename: string
+
+  @AllowNull(true)
+  @Is('VideoStreamingSegmentsSha256Url', value => throwIfNotValid(value, isActivityPubUrlValid, 'segments sha256 url', true))
   @Column
   segmentsSha256Url: string
 
@@ -98,14 +114,8 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
 
   static doesInfohashExist (infoHash: string) {
     const query = 'SELECT 1 FROM "videoStreamingPlaylist" WHERE $infoHash = ANY("p2pMediaLoaderInfohashes") LIMIT 1'
-    const options = {
-      type: QueryTypes.SELECT as QueryTypes.SELECT,
-      bind: { infoHash },
-      raw: true
-    }
 
-    return VideoModel.sequelize.query<object>(query, options)
+    return doesExist(query, { infoHash })
-              .then(results => results.length === 1)
   }
 
   static buildP2PMediaLoaderInfoHashes (playlistUrl: string, files: unknown[]) {
@@ -125,7 +135,13 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
         p2pMediaLoaderPeerVersion: {
           [Op.ne]: P2P_MEDIA_LOADER_PEER_VERSION
         }
-      }
+      },
+      include: [
+        {
+          model: VideoModel.unscoped(),
+          required: true
+        }
+      ]
     }
 
     return VideoStreamingPlaylistModel.findAll(query)
@@ -144,7 +160,7 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
     return VideoStreamingPlaylistModel.findByPk(id, options)
   }
 
-  static loadHLSPlaylistByVideo (videoId: number) {
+  static loadHLSPlaylistByVideo (videoId: number): Promise<MStreamingPlaylist> {
     const options = {
       where: {
         type: VideoStreamingPlaylistType.HLS,
@@ -155,30 +171,29 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
     return VideoStreamingPlaylistModel.findOne(options)
   }
 
-  static getHlsPlaylistFilename (resolution: number) {
+  static async loadOrGenerate (video: MVideo) {
-    return resolution + '.m3u8'
+    let playlist = await VideoStreamingPlaylistModel.loadHLSPlaylistByVideo(video.id)
+    if (!playlist) playlist = new VideoStreamingPlaylistModel()
+
+    return Object.assign(playlist, { videoId: video.id, Video: video })
   }
 
-  static getMasterHlsPlaylistFilename () {
+  assignP2PMediaLoaderInfoHashes (video: MVideo, files: unknown[]) {
-    return 'master.m3u8'
+    const masterPlaylistUrl = this.getMasterPlaylistUrl(video)
+
+    this.p2pMediaLoaderInfohashes = VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(masterPlaylistUrl, files)
   }
 
-  static getHlsSha256SegmentsFilename () {
+  getMasterPlaylistUrl (video: MVideo) {
-    return 'segments-sha256.json'
+    if (video.isOwned()) return WEBSERVER.URL + this.getMasterPlaylistStaticPath(video.uuid)
+
+    return this.playlistUrl
   }
 
-  static getHlsMasterPlaylistStaticPath (videoUUID: string) {
+  getSha256SegmentsUrl (video: MVideo) {
-    return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename())
+    if (video.isOwned()) return WEBSERVER.URL + this.getSha256SegmentsStaticPath(video.uuid, video.isLive)
-  }
 
-  static getHlsPlaylistStaticPath (videoUUID: string, resolution: number) {
+    return this.segmentsSha256Url
-    return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, VideoStreamingPlaylistModel.getHlsPlaylistFilename(resolution))
-  }
-
-  static getHlsSha256SegmentsStaticPath (videoUUID: string, isLive: boolean) {
-    if (isLive) return join('/live', 'segments-sha256', videoUUID)
-
-    return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename())
   }
 
   getStringType () {
@@ -195,4 +210,14 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
     return this.type === other.type &&
       this.videoId === other.videoId
   }
+
+  private getMasterPlaylistStaticPath (videoUUID: string) {
+    return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, this.playlistFilename)
+  }
+
+  private getSha256SegmentsStaticPath (videoUUID: string, isLive: boolean) {
+    if (isLive) return join('/live', 'segments-sha256', videoUUID)
+
+    return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, this.segmentsSha256Filename)
+  }
 }

server/models/video/video.ts

@@ -762,8 +762,7 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
 
       // Remove physical files and torrents
       instance.VideoFiles.forEach(file => {
-        tasks.push(instance.removeFile(file))
+        tasks.push(instance.removeFileAndTorrent(file))
-        tasks.push(file.removeTorrent())
       })
 
       // Remove playlists file
@@ -1670,10 +1669,13 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
                                        .concat(toAdd)
   }
 
-  removeFile (videoFile: MVideoFile, isRedundancy = false) {
+  removeFileAndTorrent (videoFile: MVideoFile, isRedundancy = false) {
     const filePath = getVideoFilePath(this, videoFile, isRedundancy)
-    return remove(filePath)
+
-      .catch(err => logger.warn('Cannot delete file %s.', filePath, { err }))
+    const promises: Promise<any>[] = [ remove(filePath) ]
+    if (!isRedundancy) promises.push(videoFile.removeTorrent())
+
+    return Promise.all(promises)
   }
 
   async removeStreamingPlaylistFiles (streamingPlaylist: MStreamingPlaylist, isRedundancy = false) {

server/tests/api/live/live-constraints.ts

@@ -4,7 +4,7 @@ import 'mocha'
 import * as chai from 'chai'
 import { VideoPrivacy } from '@shared/models'
 import {
-  checkLiveCleanup,
+  checkLiveCleanupAfterSave,
   cleanupTests,
   ConfigCommand,
   createMultipleServers,
@@ -43,7 +43,7 @@ describe('Test live constraints', function () {
       expect(video.duration).to.be.greaterThan(0)
     }
 
-    await checkLiveCleanup(servers[0], videoId, resolutions)
+    await checkLiveCleanupAfterSave(servers[0], videoId, resolutions)
   }
 
   async function waitUntilLivePublishedOnAllServers (videoId: string) {

server/tests/api/live/live-save-replay.ts

@@ -4,7 +4,7 @@ import 'mocha'
 import * as chai from 'chai'
 import { FfmpegCommand } from 'fluent-ffmpeg'
 import {
-  checkLiveCleanup,
+  checkLiveCleanupAfterSave,
   cleanupTests,
   ConfigCommand,
   createMultipleServers,
@@ -150,7 +150,7 @@ describe('Save replay setting', function () {
       await checkVideoState(liveVideoUUID, VideoState.LIVE_ENDED)
 
       // No resolutions saved since we did not save replay
-      await checkLiveCleanup(servers[0], liveVideoUUID, [])
+      await checkLiveCleanupAfterSave(servers[0], liveVideoUUID, [])
     })
 
     it('Should correctly terminate the stream on blacklist and delete the live', async function () {
@@ -179,7 +179,7 @@ describe('Save replay setting', function () {
 
       await wait(5000)
       await waitJobs(servers)
-      await checkLiveCleanup(servers[0], liveVideoUUID, [])
+      await checkLiveCleanupAfterSave(servers[0], liveVideoUUID, [])
     })
 
     it('Should correctly terminate the stream on delete and delete the video', async function () {
@@ -203,7 +203,7 @@ describe('Save replay setting', function () {
       await waitJobs(servers)
 
       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404)
-      await checkLiveCleanup(servers[0], liveVideoUUID, [])
+      await checkLiveCleanupAfterSave(servers[0], liveVideoUUID, [])
     })
   })
 
@@ -259,7 +259,7 @@ describe('Save replay setting', function () {
     })
 
     it('Should have cleaned up the live files', async function () {
-      await checkLiveCleanup(servers[0], liveVideoUUID, [ 720 ])
+      await checkLiveCleanupAfterSave(servers[0], liveVideoUUID, [ 720 ])
     })
 
     it('Should correctly terminate the stream on blacklist and blacklist the saved replay video', async function () {
@@ -287,7 +287,7 @@ describe('Save replay setting', function () {
 
       await wait(5000)
       await waitJobs(servers)
-      await checkLiveCleanup(servers[0], liveVideoUUID, [ 720 ])
+      await checkLiveCleanupAfterSave(servers[0], liveVideoUUID, [ 720 ])
     })
 
     it('Should correctly terminate the stream on delete and delete the video', async function () {
@@ -310,7 +310,7 @@ describe('Save replay setting', function () {
       await waitJobs(servers)
 
       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404)
-      await checkLiveCleanup(servers[0], liveVideoUUID, [])
+      await checkLiveCleanupAfterSave(servers[0], liveVideoUUID, [])
     })
   })
 

server/tests/api/live/live.ts

@@ -2,10 +2,10 @@
 
 import 'mocha'
 import * as chai from 'chai'
-import { join } from 'path'
+import { basename, join } from 'path'
 import { ffprobePromise, getVideoStreamFromFile } from '@server/helpers/ffprobe-utils'
 import {
-  checkLiveCleanup,
+  checkLiveCleanupAfterSave,
   checkLiveSegmentHash,
   checkResolutionsInMasterPlaylist,
   cleanupTests,
@@ -506,6 +506,10 @@ describe('Test live', function () {
         await makeRawRequest(hlsPlaylist.playlistUrl, HttpStatusCode.OK_200)
         await makeRawRequest(hlsPlaylist.segmentsSha256Url, HttpStatusCode.OK_200)
 
+        // We should have generated random filenames
+        expect(basename(hlsPlaylist.playlistUrl)).to.not.equal('master.m3u8')
+        expect(basename(hlsPlaylist.segmentsSha256Url)).to.not.equal('segments-sha256.json')
+
         expect(hlsPlaylist.files).to.have.lengthOf(resolutions.length)
 
         for (const resolution of resolutions) {
@@ -520,7 +524,9 @@ describe('Test live', function () {
             expect(file.fps).to.be.approximately(30, 2)
           }
 
-          const filename = `${video.uuid}-${resolution}-fragmented.mp4`
+          const filename = basename(file.fileUrl)
+          expect(filename).to.not.contain(video.uuid)
+
           const segmentPath = servers[0].servers.buildDirectory(join('streaming-playlists', 'hls', video.uuid, filename))
 
           const probe = await ffprobePromise(segmentPath)
@@ -537,7 +543,7 @@ describe('Test live', function () {
     it('Should correctly have cleaned up the live files', async function () {
       this.timeout(30000)
 
-      await checkLiveCleanup(servers[0], liveVideoId, [ 240, 360, 720 ])
+      await checkLiveCleanupAfterSave(servers[0], liveVideoId, [ 240, 360, 720 ])
     })
   })
 

server/tests/api/users/users-multiple-servers.ts

@@ -58,10 +58,10 @@ describe('Test users with multiple servers', function () {
       const { uuid } = await servers[0].videos.upload({ token: userAccessToken })
       videoUUID = uuid
 
+      await waitJobs(servers)
+
       await saveVideoInServers(servers, videoUUID)
     }
-
-    await waitJobs(servers)
   })
 
   it('Should be able to update my display name', async function () {

server/tests/api/videos/resumable-upload.ts

@@ -170,8 +170,13 @@ describe('Test resumable upload', function () {
 
       const size = 1000
 
+      // Content length check seems to have changed in v16
+      const expectedStatus = process.version.startsWith('v16')
+        ? HttpStatusCode.CONFLICT_409
+        : HttpStatusCode.BAD_REQUEST_400
+
       const contentRangeBuilder = (start: number) => `bytes ${start}-${start + size - 1}/${size}`
-      await sendChunks({ pathUploadId: uploadId, expectedStatus: HttpStatusCode.CONFLICT_409, contentRangeBuilder, contentLength: size })
+      await sendChunks({ pathUploadId: uploadId, expectedStatus, contentRangeBuilder, contentLength: size })
       await checkFileSize(uploadId, 0)
     })
   })

server/tests/api/videos/video-hls.ts

@@ -2,7 +2,8 @@
 
 import 'mocha'
 import * as chai from 'chai'
-import { join } from 'path'
+import { basename, join } from 'path'
+import { removeFragmentedMP4Ext, uuidRegex } from '@shared/core-utils'
 import {
   checkDirectoryIsEmpty,
   checkResolutionsInMasterPlaylist,
@@ -19,8 +20,6 @@ import {
 } from '@shared/extra-utils'
 import { HttpStatusCode, VideoStreamingPlaylistType } from '@shared/models'
 import { DEFAULT_AUDIO_RESOLUTION } from '../../../initializers/constants'
-import { uuidRegex } from '@shared/core-utils'
-import { basename } from 'path/posix'
 
 const expect = chai.expect
 
@@ -78,11 +77,13 @@ async function checkHlsPlaylist (servers: PeerTubeServer[], videoUUID: string, h
     // Check resolution playlists
     {
       for (const resolution of resolutions) {
+        const file = hlsFiles.find(f => f.resolution.id === resolution)
+        const playlistName = removeFragmentedMP4Ext(basename(file.fileUrl)) + '.m3u8'
+
         const subPlaylist = await server.streamingPlaylists.get({
-          url: `${baseUrl}/static/streaming-playlists/hls/${videoUUID}/${resolution}.m3u8`
+          url: `${baseUrl}/static/streaming-playlists/hls/${videoUUID}/${playlistName}`
         })
 
-        const file = hlsFiles.find(f => f.resolution.id === resolution)
         expect(subPlaylist).to.match(new RegExp(`${uuidRegex}-${resolution}-fragmented.mp4`))
         expect(subPlaylist).to.contain(basename(file.fileUrl))
       }

server/tests/cli/optimize-old-videos.ts

@@ -2,7 +2,6 @@
 
 import 'mocha'
 import * as chai from 'chai'
-import { join } from 'path'
 import {
   cleanupTests,
   createMultipleServers,
@@ -86,7 +85,7 @@ describe('Test optimize old videos', function () {
 
         expect(file.size).to.be.below(8000000)
 
-        const path = servers[0].servers.buildDirectory(join('videos', video.uuid + '-' + file.resolution.id + '.mp4'))
+        const path = servers[0].servers.buildWebTorrentFilePath(file.fileUrl)
         const bitrate = await getVideoFileBitrate(path)
         const fps = await getVideoFileFPS(path)
         const resolution = await getVideoFileResolution(path)

server/tests/cli/prune-storage.ts

@@ -36,7 +36,7 @@ async function assertNotExists (server: PeerTubeServer, directory: string, subst
   }
 }
 
-async function assertCountAreOkay (servers: PeerTubeServer[]) {
+async function assertCountAreOkay (servers: PeerTubeServer[], videoServer2UUID: string) {
   for (const server of servers) {
     const videosCount = await countFiles(server, 'videos')
     expect(videosCount).to.equal(8)
@@ -53,12 +53,21 @@ async function assertCountAreOkay (servers: PeerTubeServer[]) {
     const avatarsCount = await countFiles(server, 'avatars')
     expect(avatarsCount).to.equal(2)
   }
+
+  // When we'll prune HLS directories too
+  // const hlsRootCount = await countFiles(servers[1], 'streaming-playlists/hls/')
+  // expect(hlsRootCount).to.equal(2)
+
+  // const hlsCount = await countFiles(servers[1], 'streaming-playlists/hls/' + videoServer2UUID)
+  // expect(hlsCount).to.equal(10)
 }
 
 describe('Test prune storage scripts', function () {
   let servers: PeerTubeServer[]
   const badNames: { [directory: string]: string[] } = {}
 
+  let videoServer2UUID: string
+
   before(async function () {
     this.timeout(120000)
 
@@ -68,7 +77,9 @@ describe('Test prune storage scripts', function () {
 
     for (const server of servers) {
       await server.videos.upload({ attributes: { name: 'video 1' } })
-      await server.videos.upload({ attributes: { name: 'video 2' } })
+
+      const { uuid } = await server.videos.upload({ attributes: { name: 'video 2' } })
+      if (server.serverNumber === 2) videoServer2UUID = uuid
 
       await server.users.updateMyAvatar({ fixture: 'avatar.png' })
 
@@ -112,7 +123,7 @@ describe('Test prune storage scripts', function () {
   })
 
   it('Should have the files on the disk', async function () {
-    await assertCountAreOkay(servers)
+    await assertCountAreOkay(servers, videoServer2UUID)
   })
 
   it('Should create some dirty files', async function () {
@@ -176,6 +187,28 @@ describe('Test prune storage scripts', function () {
 
         badNames['avatars'] = [ n1, n2 ]
       }
+
+      // When we'll prune HLS directories too
+      // {
+      //   const directory = join('streaming-playlists', 'hls')
+      //   const base = servers[1].servers.buildDirectory(directory)
+
+      //   const n1 = buildUUID()
+      //   await createFile(join(base, n1))
+      //   badNames[directory] = [ n1 ]
+      // }
+
+      // {
+      //   const directory = join('streaming-playlists', 'hls', videoServer2UUID)
+      //   const base = servers[1].servers.buildDirectory(directory)
+      //   const n1 = buildUUID() + '-240-fragmented-.mp4'
+      //   const n2 = buildUUID() + '-master.m3u8'
+
+      //   await createFile(join(base, n1))
+      //   await createFile(join(base, n2))
+
+      //   badNames[directory] = [ n1, n2 ]
+      // }
     }
   })
 
@@ -187,7 +220,7 @@ describe('Test prune storage scripts', function () {
   })
 
   it('Should have removed files', async function () {
-    await assertCountAreOkay(servers)
+    await assertCountAreOkay(servers, videoServer2UUID)
 
     for (const directory of Object.keys(badNames)) {
       for (const name of badNames[directory]) {

server/tests/cli/update-host.ts

@@ -108,21 +108,22 @@ describe('Test update host scripts', function () {
 
     for (const video of data) {
       const videoDetails = await server.videos.get({ id: video.id })
+      const files = videoDetails.files.concat(videoDetails.streamingPlaylists[0].files)
 
-      expect(videoDetails.files).to.have.lengthOf(4)
+      expect(files).to.have.lengthOf(8)
 
-      for (const file of videoDetails.files) {
+      for (const file of files) {
         expect(file.magnetUri).to.contain('localhost%3A9002%2Ftracker%2Fsocket')
-        expect(file.magnetUri).to.contain('localhost%3A9002%2Fstatic%2Fwebseed%2F')
+        expect(file.magnetUri).to.contain('localhost%3A9002%2Fstatic%2F')
 
-        const torrent = await parseTorrentVideo(server, videoDetails.uuid, file.resolution.id)
+        const torrent = await parseTorrentVideo(server, file)
         const announceWS = torrent.announce.find(a => a === 'ws://localhost:9002/tracker/socket')
         expect(announceWS).to.not.be.undefined
 
         const announceHttp = torrent.announce.find(a => a === 'http://localhost:9002/tracker/announce')
         expect(announceHttp).to.not.be.undefined
 
-        expect(torrent.urlList[0]).to.contain('http://localhost:9002/static/webseed')
+        expect(torrent.urlList[0]).to.contain('http://localhost:9002/static/')
       }
     }
   })

server/tests/plugins/plugin-transcoding.ts

@@ -2,7 +2,6 @@
 
 import 'mocha'
 import { expect } from 'chai'
-import { join } from 'path'
 import { getAudioStream, getVideoFileFPS, getVideoStreamFromFile } from '@server/helpers/ffprobe-utils'
 import {
   cleanupTests,
@@ -247,7 +246,9 @@ describe('Test transcoding plugins', function () {
       const videoUUID = (await server.videos.quickUpload({ name: 'video', fixture: 'video_very_short_240p.mp4' })).uuid
       await waitJobs([ server ])
 
-      const path = server.servers.buildDirectory(join('videos', videoUUID + '-240.mp4'))
+      const video = await server.videos.get({ id: videoUUID })
+
+      const path = server.servers.buildWebTorrentFilePath(video.files[0].fileUrl)
       const audioProbe = await getAudioStream(path)
       expect(audioProbe.audioStream.codec_name).to.equal('opus')
 

server/types/models/video/video-streaming-playlist.ts

@@ -39,5 +39,5 @@ export type MStreamingPlaylistRedundanciesOpt =
   PickWithOpt<VideoStreamingPlaylistModel, 'RedundancyVideos', MVideoRedundancyFileUrl[]>
 
 export function isStreamingPlaylist (value: MVideo | MStreamingPlaylistVideo): value is MStreamingPlaylistVideo {
-  return !!(value as MStreamingPlaylist).playlistUrl
+  return !!(value as MStreamingPlaylist).videoId
 }

shared/core-utils/miscs/regexp.ts

@@ -1 +1,5 @@
 export const uuidRegex = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'
+
+export function removeFragmentedMP4Ext (path: string) {
+  return path.replace(/-fragmented.mp4$/i, '')
+}

shared/extra-utils/miscs/webtorrent.ts

@@ -1,7 +1,8 @@
 import { readFile } from 'fs-extra'
 import * as parseTorrent from 'parse-torrent'
-import { join } from 'path'
+import { basename, join } from 'path'
 import * as WebTorrent from 'webtorrent'
+import { VideoFile } from '@shared/models'
 import { PeerTubeServer } from '../server'
 
 let webtorrent: WebTorrent.Instance
@@ -15,8 +16,8 @@ function webtorrentAdd (torrent: string, refreshWebTorrent = false) {
   return new Promise<WebTorrent.Torrent>(res => webtorrent.add(torrent, res))
 }
 
-async function parseTorrentVideo (server: PeerTubeServer, videoUUID: string, resolution: number) {
+async function parseTorrentVideo (server: PeerTubeServer, file: VideoFile) {
-  const torrentName = videoUUID + '-' + resolution + '.torrent'
+  const torrentName = basename(file.torrentUrl)
   const torrentPath = server.servers.buildDirectory(join('torrents', torrentName))
 
   const data = await readFile(torrentPath)

shared/extra-utils/server/servers-command.ts

@@ -1,7 +1,6 @@
 import { exec } from 'child_process'
 import { copy, ensureDir, readFile, remove } from 'fs-extra'
-import { join } from 'path'
+import { basename, join } from 'path'
-import { basename } from 'path/posix'
 import { root } from '@server/helpers/core-utils'
 import { HttpStatusCode } from '@shared/models'
 import { getFileSize, isGithubCI, wait } from '../miscs'

shared/extra-utils/videos/live.ts

@@ -76,7 +76,7 @@ async function waitUntilLivePublishedOnAllServers (servers: PeerTubeServer[], vi
   }
 }
 
-async function checkLiveCleanup (server: PeerTubeServer, videoUUID: string, resolutions: number[] = []) {
+async function checkLiveCleanupAfterSave (server: PeerTubeServer, videoUUID: string, resolutions: number[] = []) {
   const basePath = server.servers.buildDirectory('streaming-playlists')
   const hlsPath = join(basePath, 'hls', videoUUID)
 
@@ -93,12 +93,18 @@ async function checkLiveCleanup (server: PeerTubeServer, videoUUID: string, reso
   expect(files).to.have.lengthOf(resolutions.length * 2 + 2)
 
   for (const resolution of resolutions) {
-    expect(files).to.contain(`${videoUUID}-${resolution}-fragmented.mp4`)
+    const fragmentedFile = files.find(f => f.endsWith(`-${resolution}-fragmented.mp4`))
-    expect(files).to.contain(`${resolution}.m3u8`)
+    expect(fragmentedFile).to.exist
+
+    const playlistFile = files.find(f => f.endsWith(`${resolution}.m3u8`))
+    expect(playlistFile).to.exist
   }
 
-  expect(files).to.contain('master.m3u8')
+  const masterPlaylistFile = files.find(f => f.endsWith('-master.m3u8'))
-  expect(files).to.contain('segments-sha256.json')
+  expect(masterPlaylistFile).to.exist
+
+  const shaFile = files.find(f => f.endsWith('-segments-sha256.json'))
+  expect(shaFile).to.exist
 }
 
 export {
@@ -107,5 +113,5 @@ export {
   testFfmpegStreamError,
   stopFfmpeg,
   waitUntilLivePublishedOnAllServers,
-  checkLiveCleanup
+  checkLiveCleanupAfterSave
 }

shared/extra-utils/videos/streaming-playlists.ts

@@ -1,6 +1,7 @@
 import { expect } from 'chai'
 import { basename } from 'path'
 import { sha256 } from '@server/helpers/core-utils'
+import { removeFragmentedMP4Ext } from '@shared/core-utils'
 import { HttpStatusCode, VideoStreamingPlaylist } from '@shared/models'
 import { PeerTubeServer } from '../server'
 
@@ -15,11 +16,11 @@ async function checkSegmentHash (options: {
   const { server, baseUrlPlaylist, baseUrlSegment, videoUUID, resolution, hlsPlaylist } = options
   const command = server.streamingPlaylists
 
-  const playlist = await command.get({ url: `${baseUrlPlaylist}/${videoUUID}/${resolution}.m3u8` })
-
   const file = hlsPlaylist.files.find(f => f.resolution.id === resolution)
   const videoName = basename(file.fileUrl)
 
+  const playlist = await command.get({ url: `${baseUrlPlaylist}/${videoUUID}/${removeFragmentedMP4Ext(videoName)}.m3u8` })
+
   const matches = /#EXT-X-BYTERANGE:(\d+)@(\d+)/.exec(playlist)
 
   const length = parseInt(matches[1], 10)