Support CLI move of original video file

packages/server-commands/src/requests/requests.ts

@@ -166,9 +166,10 @@ export function makePutBodyRequest (options: {
 
 // ---------------------------------------------------------------------------
 
-export async function getRedirectionUrl (url: string) {
+export async function getRedirectionUrl (url: string, token?: string) {
   const res = await makeRawRequest({
     url,
+    token,
     redirects: 0,
     expectedStatus: HttpStatusCode.FOUND_302
   })

packages/tests/src/cli/create-move-video-storage-job.ts

@@ -1,23 +1,26 @@
 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
 
-import { join } from 'path'
+import { getAllFiles } from '@peertube/peertube-core-utils'
-import { areMockObjectStorageTestsDisabled } from '@peertube/peertube-node-utils'
 import { HttpStatusCode, VideoDetails } from '@peertube/peertube-models'
+import { areMockObjectStorageTestsDisabled } from '@peertube/peertube-node-utils'
 import {
+  ObjectStorageCommand,
+  PeerTubeServer,
   cleanupTests,
   createMultipleServers,
   doubleFollow,
+  getRedirectionUrl,
   makeRawRequest,
-  ObjectStorageCommand,
-  PeerTubeServer,
   setAccessTokensToServers,
   waitJobs
 } from '@peertube/peertube-server-commands'
-import { expectStartWith } from '../shared/checks.js'
 import { checkDirectoryIsEmpty } from '@tests/shared/directories.js'
-import { getAllFiles } from '@peertube/peertube-core-utils'
+import { join } from 'path'
+import { expectStartWith } from '../shared/checks.js'
 
 async function checkFiles (origin: PeerTubeServer, video: VideoDetails, objectStorage?: ObjectStorageCommand) {
+
+  // Web videos
   for (const file of video.files) {
     const start = objectStorage
       ? objectStorage.getMockWebVideosBaseUrl()
@@ -28,6 +31,8 @@ async function checkFiles (origin: PeerTubeServer, video: VideoDetails, objectSt
     await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
   }
 
+  // Playlists
+  {
     const start = objectStorage
       ? objectStorage.getMockPlaylistBaseUrl()
       : origin.url
@@ -43,6 +48,22 @@ async function checkFiles (origin: PeerTubeServer, video: VideoDetails, objectSt
     }
   }
 
+  // Original file
+  {
+    const source = await origin.videos.getSource({ id: video.uuid })
+
+    if (objectStorage) {
+      await makeRawRequest({ url: source.fileDownloadUrl, token: origin.accessToken, expectedStatus: HttpStatusCode.FOUND_302 })
+
+      const redirected = await getRedirectionUrl(source.fileDownloadUrl, origin.accessToken)
+      expectStartWith(redirected, objectStorage.getMockOriginalFileBaseUrl())
+    } else {
+      await makeRawRequest({ url: source.fileDownloadUrl, token: origin.accessToken, expectedStatus: HttpStatusCode.OK_200 })
+      expectStartWith(source.fileDownloadUrl, origin.url)
+    }
+  }
+}
+
 describe('Test create move video storage job', function () {
   if (areMockObjectStorageTestsDisabled()) return
 
@@ -61,10 +82,10 @@ describe('Test create move video storage job', function () {
 
     await objectStorage.prepareDefaultMockBuckets()
 
-    await servers[0].config.enableTranscoding()
+    await servers[0].config.enableMinimumTranscoding({ keepOriginal: true })
 
     for (let i = 0; i < 3; i++) {
-      const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video' + i } })
+      const { uuid } = await servers[0].videos.quickUpload({ name: 'video' + i })
       uuids.push(uuid)
     }
 

server/core/lib/job-queue/handlers/move-to-file-system.ts

@@ -1,20 +1,23 @@
-import { Job } from 'bullmq'
+import { FileStorage, MoveStoragePayload, VideoStateType } from '@peertube/peertube-models'
-import { join } from 'path'
-import { MoveStoragePayload, VideoStateType, FileStorage } from '@peertube/peertube-models'
 import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
 import { updateTorrentMetadata } from '@server/helpers/webtorrent.js'
 import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants.js'
 import {
   makeHLSFileAvailable,
+  makeOriginalFileAvailable,
   makeWebVideoFileAvailable,
   removeHLSFileObjectStorageByFilename,
   removeHLSObjectStorage,
+  removeOriginalFileObjectStorage,
   removeWebVideoObjectStorage
 } from '@server/lib/object-storage/index.js'
 import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths.js'
 import { VideoPathManager } from '@server/lib/video-path-manager.js'
 import { moveToFailedMoveToFileSystemState, moveToNextState } from '@server/lib/video-state.js'
 import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models/index.js'
+import { MVideoSource } from '@server/types/models/video/video-source.js'
+import { Job } from 'bullmq'
+import { join } from 'path'
 import { moveToJob, onMoveToStorageFailure } from './shared/move-video.js'
 
 const lTagsBase = loggerTagsFactory('move-file-system')
@@ -30,6 +33,8 @@ export async function processMoveToFileSystem (job: Job) {
 
     moveWebVideoFiles,
     moveHLSFiles,
+    moveVideoSourceFile,
+
     doAfterLastMove: video => doAfterLastMove({ video, previousVideoState: payload.previousVideoState, isNewVideo: payload.isNewVideo }),
     moveToFailedState: moveToFailedMoveToFileSystemState
   })
@@ -50,12 +55,31 @@ export async function onMoveToFileSystemFailure (job: Job, err: any) {
 // Private
 // ---------------------------------------------------------------------------
 
+async function moveVideoSourceFile (source: MVideoSource) {
+  if (source.storage === FileStorage.FILE_SYSTEM) return
+
+  await makeOriginalFileAvailable(
+    source.keptOriginalFilename,
+    VideoPathManager.Instance.getFSOriginalVideoFilePath(source.keptOriginalFilename)
+  )
+
+  const oldFileUrl = source.fileUrl
+
+  source.fileUrl = null
+  source.storage = FileStorage.FILE_SYSTEM
+  await source.save()
+
+  logger.debug('Removing original video file %s because it\'s now on file system', oldFileUrl, lTagsBase())
+
+  await removeOriginalFileObjectStorage(source)
+}
+
 async function moveWebVideoFiles (video: MVideoWithAllFiles) {
   for (const file of video.VideoFiles) {
     if (file.storage === FileStorage.FILE_SYSTEM) continue
 
     await makeWebVideoFileAvailable(file.filename, VideoPathManager.Instance.getFSVideoFileOutputPath(video, file))
-    await onFileMoved({
+    await onVideoFileMoved({
       videoOrPlaylist: video,
       file,
       objetStorageRemover: () => removeWebVideoObjectStorage(file)
@@ -75,7 +99,7 @@ async function moveHLSFiles (video: MVideoWithAllFiles) {
       await makeHLSFileAvailable(playlistWithVideo, playlistFilename, join(getHLSDirectory(video), playlistFilename))
       await makeHLSFileAvailable(playlistWithVideo, file.filename, join(getHLSDirectory(video), file.filename))
 
-      await onFileMoved({
+      await onVideoFileMoved({
         videoOrPlaylist: playlistWithVideo,
         file,
         objetStorageRemover: async () => {
@@ -87,7 +111,7 @@ async function moveHLSFiles (video: MVideoWithAllFiles) {
   }
 }
 
-async function onFileMoved (options: {
+async function onVideoFileMoved (options: {
   videoOrPlaylist: MVideo | MStreamingPlaylistVideo
   file: MVideoFile
   objetStorageRemover: () => Promise<any>

server/core/lib/job-queue/handlers/move-to-object-storage.ts

@@ -1,15 +1,16 @@
-import { Job } from 'bullmq'
+import { FileStorage, MoveStoragePayload, VideoStateType } from '@peertube/peertube-models'
-import { remove } from 'fs-extra/esm'
-import { join } from 'path'
-import { MoveStoragePayload, VideoStateType, FileStorage } from '@peertube/peertube-models'
 import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
 import { updateTorrentMetadata } from '@server/helpers/webtorrent.js'
 import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants.js'
-import { storeHLSFileFromFilename, storeWebVideoFile } from '@server/lib/object-storage/index.js'
+import { storeHLSFileFromFilename, storeOriginalVideoFile, storeWebVideoFile } from '@server/lib/object-storage/index.js'
 import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths.js'
 import { VideoPathManager } from '@server/lib/video-path-manager.js'
 import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state.js'
 import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models/index.js'
+import { MVideoSource } from '@server/types/models/video/video-source.js'
+import { Job } from 'bullmq'
+import { remove } from 'fs-extra/esm'
+import { join } from 'path'
 import { moveToJob, onMoveToStorageFailure } from './shared/move-video.js'
 
 const lTagsBase = loggerTagsFactory('move-object-storage')
@@ -25,6 +26,7 @@ export async function processMoveToObjectStorage (job: Job) {
 
     moveWebVideoFiles,
     moveHLSFiles,
+    moveVideoSourceFile,
     doAfterLastMove: video => doAfterLastMove({ video, previousVideoState: payload.previousVideoState, isNewVideo: payload.isNewVideo }),
     moveToFailedState: moveToFailedMoveToObjectStorageState
   })
@@ -43,6 +45,21 @@ export async function onMoveToObjectStorageFailure (job: Job, err: any) {
 
 // ---------------------------------------------------------------------------
 
+async function moveVideoSourceFile (source: MVideoSource) {
+  if (source.storage !== FileStorage.FILE_SYSTEM) return
+
+  const sourcePath = VideoPathManager.Instance.getFSOriginalVideoFilePath(source.keptOriginalFilename)
+  const fileUrl = await storeOriginalVideoFile(sourcePath, source.keptOriginalFilename)
+
+  source.storage = FileStorage.OBJECT_STORAGE
+  source.fileUrl = fileUrl
+  await source.save()
+
+  logger.debug('Removing original video file ' + sourcePath + ' because it\'s now on object storage', lTagsBase())
+
+  await remove(sourcePath)
+}
+
 async function moveWebVideoFiles (video: MVideoWithAllFiles) {
   for (const file of video.VideoFiles) {
     if (file.storage !== FileStorage.FILE_SYSTEM) continue
@@ -50,7 +67,7 @@ async function moveWebVideoFiles (video: MVideoWithAllFiles) {
     const fileUrl = await storeWebVideoFile(video, file)
 
     const oldPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, file)
-    await onFileMoved({ videoOrPlaylist: video, file, fileUrl, oldPath })
+    await onVideoFileMoved({ videoOrPlaylist: video, file, fileUrl, oldPath })
   }
 }
 
@@ -70,12 +87,12 @@ async function moveHLSFiles (video: MVideoWithAllFiles) {
 
       const oldPath = join(getHLSDirectory(video), file.filename)
 
-      await onFileMoved({ videoOrPlaylist: Object.assign(playlist, { Video: video }), file, fileUrl, oldPath })
+      await onVideoFileMoved({ videoOrPlaylist: Object.assign(playlist, { Video: video }), file, fileUrl, oldPath })
     }
   }
 }
 
-async function onFileMoved (options: {
+async function onVideoFileMoved (options: {
   videoOrPlaylist: MVideo | MStreamingPlaylistVideo
   file: MVideoFile
   fileUrl: string

server/core/lib/job-queue/handlers/shared/move-video.ts

@@ -1,8 +1,10 @@
 import { LoggerTags, logger, loggerTagsFactory } from '@server/helpers/logger.js'
 import { VideoPathManager } from '@server/lib/video-path-manager.js'
 import { VideoJobInfoModel } from '@server/models/video/video-job-info.js'
+import { VideoSourceModel } from '@server/models/video/video-source.js'
 import { VideoModel } from '@server/models/video/video.js'
 import { MVideoWithAllFiles } from '@server/types/models/index.js'
+import { MVideoSource } from '@server/types/models/video/video-source.js'
 
 export async function moveToJob (options: {
   jobId: string
@@ -11,10 +13,20 @@ export async function moveToJob (options: {
 
   moveWebVideoFiles: (video: MVideoWithAllFiles) => Promise<void>
   moveHLSFiles: (video: MVideoWithAllFiles) => Promise<void>
+  moveVideoSourceFile: (source: MVideoSource) => Promise<void>
   moveToFailedState: (video: MVideoWithAllFiles) => Promise<void>
   doAfterLastMove: (video: MVideoWithAllFiles) => Promise<void>
 }) {
-  const { jobId, loggerTags, videoUUID, moveHLSFiles, moveWebVideoFiles, moveToFailedState, doAfterLastMove } = options
+  const {
+    jobId,
+    loggerTags,
+    videoUUID,
+    moveVideoSourceFile,
+    moveHLSFiles,
+    moveWebVideoFiles,
+    moveToFailedState,
+    doAfterLastMove
+  } = options
 
   const lTagsBase = loggerTagsFactory(...loggerTags)
 
@@ -31,6 +43,13 @@ export async function moveToJob (options: {
   const lTags = lTagsBase(video.uuid, video.url)
 
   try {
+    const source = await VideoSourceModel.loadLatest(video.id)
+    if (source) {
+      logger.debug(`Moving video source ${source.keptOriginalFilename} file of video ${video.uuid}`, lTags)
+
+      await moveVideoSourceFile(source)
+    }
+
     if (video.VideoFiles) {
       logger.debug('Moving %d web video files for video %s.', video.VideoFiles.length, video.uuid, lTags)
 

server/core/lib/object-storage/videos.ts

@@ -159,6 +159,20 @@ export async function makeWebVideoFileAvailable (filename: string, destination:
   return destination
 }
 
+export async function makeOriginalFileAvailable (keptOriginalFilename: string, destination: string) {
+  const key = generateOriginalVideoObjectStorageKey(keptOriginalFilename)
+
+  logger.info('Fetching Original Video file %s from object storage to %s.', key, destination, lTags())
+
+  await makeAvailable({
+    key,
+    destination,
+    bucketInfo: CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES
+  })
+
+  return destination
+}
+
 // ---------------------------------------------------------------------------
 
 export function getWebVideoFileReadStream (options: {