Fix transcoding failure when importing a video

server/controllers/api/videos/import.ts

@@ -3,7 +3,7 @@ import { move, readFile } from 'fs-extra'
 import { decode } from 'magnet-uri'
 import parseTorrent, { Instance } from 'parse-torrent'
 import { join } from 'path'
-import { buildYoutubeDLImport, buildVideoFromImport, insertFromImportIntoDB, YoutubeDlImportError } from '@server/lib/video-import'
+import { buildYoutubeDLImport, buildVideoFromImport, insertFromImportIntoDB, YoutubeDlImportError } from '@server/lib/video-pre-import'
 import { MThumbnail, MVideoThumbnail } from '@server/types/models'
 import { HttpStatusCode, ServerErrorCode, ThumbnailType, VideoImportCreate, VideoImportPayload, VideoImportState } from '@shared/models'
 import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger'

server/lib/job-queue/handlers/video-import.ts

@@ -12,7 +12,8 @@ import { buildMoveToObjectStorageJob, buildOptimizeOrMergeAudioJob } from '@serv
 import { VideoPathManager } from '@server/lib/video-path-manager'
 import { buildNextVideoState } from '@server/lib/video-state'
 import { ThumbnailModel } from '@server/models/video/thumbnail'
-import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import'
+import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models'
+import { MVideoImport, MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import'
 import { getLowercaseExtension } from '@shared/core-utils'
 import { isAudioFile } from '@shared/extra-utils'
 import {
@@ -36,7 +37,6 @@ import { sequelizeTypescript } from '../../../initializers/database'
 import { VideoModel } from '../../../models/video/video'
 import { VideoFileModel } from '../../../models/video/video-file'
 import { VideoImportModel } from '../../../models/video/video-import'
-import { MThumbnail } from '../../../types/models/video/thumbnail'
 import { federateVideoIfNeeded } from '../../activitypub/videos'
 import { Notifier } from '../../notifier'
 import { generateVideoMiniature } from '../../thumbnail'
@@ -178,38 +178,26 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
     }
 
     // Video is accepted, resuming preparation
-    const videoWithFiles = Object.assign(videoImport.Video, { VideoFiles: [ videoFile ], VideoStreamingPlaylists: [] })
-    // To clean files if the import fails
-    const videoImportWithFiles: MVideoImportDefaultFiles = Object.assign(videoImport, { Video: videoWithFiles })
+    const videoFileLockReleaser = await VideoPathManager.Instance.lockFiles(videoImport.Video.uuid)
+
+    try {
+      const videoImportWithFiles = await refreshVideoImportFromDB(videoImport, videoFile)
 
       // Move file
       const videoDestFile = VideoPathManager.Instance.getFSVideoFileOutputPath(videoImportWithFiles.Video, videoFile)
       await move(tempVideoPath, videoDestFile)
+
       tempVideoPath = null // This path is not used anymore
 
-    // Generate miniature if the import did not created it
-    let thumbnailModel: MThumbnail
-    let thumbnailSave: object
-    if (!videoImportWithFiles.Video.getMiniature()) {
-      thumbnailModel = await generateVideoMiniature({
-        video: videoImportWithFiles.Video,
-        videoFile,
-        type: ThumbnailType.MINIATURE
-      })
-      thumbnailSave = thumbnailModel.toJSON()
-    }
+      let {
+        miniatureModel: thumbnailModel,
+        miniatureJSONSave: thumbnailSave
+      } = await generateMiniature(videoImportWithFiles, videoFile, ThumbnailType.MINIATURE)
 
-    // Generate preview if the import did not created it
-    let previewModel: MThumbnail
-    let previewSave: object
-    if (!videoImportWithFiles.Video.getPreview()) {
-      previewModel = await generateVideoMiniature({
-        video: videoImportWithFiles.Video,
-        videoFile,
-        type: ThumbnailType.PREVIEW
-      })
-      previewSave = previewModel.toJSON()
-    }
+      let {
+        miniatureModel: previewModel,
+        miniatureJSONSave: previewSave
+      } = await generateMiniature(videoImportWithFiles, videoFile, ThumbnailType.PREVIEW)
 
       // Create torrent
       await createTorrentAndSetInfoHash(videoImportWithFiles.Video, videoFile)
@@ -218,13 +206,11 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
 
       const { videoImportUpdated, video } = await retryTransactionWrapper(() => {
         return sequelizeTypescript.transaction(async t => {
-        const videoImportToUpdate = videoImportWithFiles as MVideoImportVideo
-
           // Refresh video
-        const video = await VideoModel.load(videoImportToUpdate.videoId, t)
-        if (!video) throw new Error('Video linked to import ' + videoImportToUpdate.videoId + ' does not exist anymore.')
+          const video = await VideoModel.load(videoImportWithFiles.videoId, t)
+          if (!video) throw new Error('Video linked to import ' + videoImportWithFiles.videoId + ' does not exist anymore.')
 
-        const videoFileCreated = await videoFile.save({ transaction: t })
+          await videoFile.save({ transaction: t })
 
           // Update video DB object
           video.duration = duration
@@ -239,11 +225,8 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
           await federateVideoIfNeeded(videoForFederation, true, t)
 
           // Update video import object
-        videoImportToUpdate.state = VideoImportState.SUCCESS
-        const videoImportUpdated = await videoImportToUpdate.save({ transaction: t }) as MVideoImportVideo
-        videoImportUpdated.Video = video
-
-        videoImportToUpdate.Video = Object.assign(video, { VideoFiles: [ videoFileCreated ] })
+          videoImportWithFiles.state = VideoImportState.SUCCESS
+          const videoImportUpdated = await videoImportWithFiles.save({ transaction: t }) as MVideoImport
 
           logger.info('Video %s imported.', video.uuid)
 
@@ -259,7 +242,60 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
         })
       })
 
-    Notifier.Instance.notifyOnFinishedVideoImport({ videoImport: videoImportUpdated, success: true })
+      await afterImportSuccess({ videoImport: videoImportUpdated, video, videoFile, user: videoImport.User })
+    } finally {
+      videoFileLockReleaser()
+    }
+  } catch (err) {
+    await onImportError(err, tempVideoPath, videoImport)
+
+    throw err
+  }
+}
+
+async function refreshVideoImportFromDB (videoImport: MVideoImportDefault, videoFile: MVideoFile): Promise<MVideoImportDefaultFiles> {
+  // Refresh video, privacy may have changed
+  const video = await videoImport.Video.reload()
+  const videoWithFiles = Object.assign(video, { VideoFiles: [ videoFile ], VideoStreamingPlaylists: [] })
+
+  return Object.assign(videoImport, { Video: videoWithFiles })
+}
+
+async function generateMiniature (videoImportWithFiles: MVideoImportDefaultFiles, videoFile: MVideoFile, thumbnailType: ThumbnailType) {
+  // Generate miniature if the import did not created it
+  const needsMiniature = thumbnailType === ThumbnailType.MINIATURE
+    ? !videoImportWithFiles.Video.getMiniature()
+    : !videoImportWithFiles.Video.getPreview()
+
+  if (!needsMiniature) {
+    return {
+      miniatureModel: null,
+      miniatureJSONSave: null
+    }
+  }
+
+  const miniatureModel = await generateVideoMiniature({
+    video: videoImportWithFiles.Video,
+    videoFile,
+    type: ThumbnailType.MINIATURE
+  })
+  const miniatureJSONSave = miniatureModel.toJSON()
+
+  return {
+    miniatureModel,
+    miniatureJSONSave
+  }
+}
+
+async function afterImportSuccess (options: {
+  videoImport: MVideoImport
+  video: MVideoFullLight
+  videoFile: MVideoFile
+  user: MUserId
+}) {
+  const { video, videoFile, videoImport, user } = options
+
+  Notifier.Instance.notifyOnFinishedVideoImport({ videoImport: Object.assign(videoImport, { Video: video }), success: true })
 
   if (video.isBlacklisted()) {
     const videoBlacklist = Object.assign(video.VideoBlacklist, { Video: video })
@@ -271,18 +307,19 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
 
   if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
     await JobQueue.Instance.createJob(
-        await buildMoveToObjectStorageJob({ video: videoImportUpdated.Video, previousVideoState: VideoState.TO_IMPORT })
+      await buildMoveToObjectStorageJob({ video, previousVideoState: VideoState.TO_IMPORT })
     )
+    return
   }
 
-    // Create transcoding jobs?
-    if (video.state === VideoState.TO_TRANSCODE) {
+  if (video.state === VideoState.TO_TRANSCODE) { // Create transcoding jobs?
     await JobQueue.Instance.createJob(
-        await buildOptimizeOrMergeAudioJob({ video: videoImportUpdated.Video, videoFile, user: videoImport.User })
+      await buildOptimizeOrMergeAudioJob({ video, videoFile, user })
     )
   }
+}
 
-  } catch (err) {
+async function onImportError (err: Error, tempVideoPath: string, videoImport: MVideoImportVideo) {
   try {
     if (tempVideoPath) await remove(tempVideoPath)
   } catch (errUnlink) {
@@ -296,7 +333,4 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
   await videoImport.save()
 
   Notifier.Instance.notifyOnFinishedVideoImport({ videoImport, success: false })
-
-    throw err
-  }
 }

server/lib/sync-channel.ts

@@ -1,7 +1,7 @@
 import { logger } from '@server/helpers/logger'
 import { YoutubeDLWrapper } from '@server/helpers/youtube-dl'
 import { CONFIG } from '@server/initializers/config'
-import { buildYoutubeDLImport } from '@server/lib/video-import'
+import { buildYoutubeDLImport } from '@server/lib/video-pre-import'
 import { UserModel } from '@server/models/user/user'
 import { VideoImportModel } from '@server/models/video/video-import'
 import { MChannel, MChannelAccountDefault, MChannelSync } from '@server/types/models'

server/lib/transcoding/transcoding.ts

@@ -46,6 +46,7 @@ async function optimizeOriginalVideofile (options: {
   const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
   const newExtname = '.mp4'
 
+  // Will be released by our transcodeVOD function once ffmpeg is ran
   const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
 
   try {