Fix transcoding failure when importing a video

server/controllers/api/videos/import.ts

@@ -3,7 +3,7 @@ import { move, readFile } from 'fs-extra'
 import { decode } from 'magnet-uri'
 import parseTorrent, { Instance } from 'parse-torrent'
 import { join } from 'path'
-import { buildYoutubeDLImport, buildVideoFromImport, insertFromImportIntoDB, YoutubeDlImportError } from '@server/lib/video-import'
+import { buildYoutubeDLImport, buildVideoFromImport, insertFromImportIntoDB, YoutubeDlImportError } from '@server/lib/video-pre-import'
 import { MThumbnail, MVideoThumbnail } from '@server/types/models'
 import { HttpStatusCode, ServerErrorCode, ThumbnailType, VideoImportCreate, VideoImportPayload, VideoImportState } from '@shared/models'
 import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger'

server/lib/job-queue/handlers/video-import.ts

@@ -12,7 +12,8 @@ import { buildMoveToObjectStorageJob, buildOptimizeOrMergeAudioJob } from '@serv
 import { VideoPathManager } from '@server/lib/video-path-manager'
 import { buildNextVideoState } from '@server/lib/video-state'
 import { ThumbnailModel } from '@server/models/video/thumbnail'
-import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import'
+import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models'
+import { MVideoImport, MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import'
 import { getLowercaseExtension } from '@shared/core-utils'
 import { isAudioFile } from '@shared/extra-utils'
 import {
@@ -36,7 +37,6 @@ import { sequelizeTypescript } from '../../../initializers/database'
 import { VideoModel } from '../../../models/video/video'
 import { VideoFileModel } from '../../../models/video/video-file'
 import { VideoImportModel } from '../../../models/video/video-import'
-import { MThumbnail } from '../../../types/models/video/thumbnail'
 import { federateVideoIfNeeded } from '../../activitypub/videos'
 import { Notifier } from '../../notifier'
 import { generateVideoMiniature } from '../../thumbnail'
@@ -178,125 +178,159 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
     }
 
     // Video is accepted, resuming preparation
-    const videoWithFiles = Object.assign(videoImport.Video, { VideoFiles: [ videoFile ], VideoStreamingPlaylists: [] })
+    const videoFileLockReleaser = await VideoPathManager.Instance.lockFiles(videoImport.Video.uuid)
-    // To clean files if the import fails
-    const videoImportWithFiles: MVideoImportDefaultFiles = Object.assign(videoImport, { Video: videoWithFiles })
 
-    // Move file
-    const videoDestFile = VideoPathManager.Instance.getFSVideoFileOutputPath(videoImportWithFiles.Video, videoFile)
-    await move(tempVideoPath, videoDestFile)
-    tempVideoPath = null // This path is not used anymore
-
-    // Generate miniature if the import did not created it
-    let thumbnailModel: MThumbnail
-    let thumbnailSave: object
-    if (!videoImportWithFiles.Video.getMiniature()) {
-      thumbnailModel = await generateVideoMiniature({
-        video: videoImportWithFiles.Video,
-        videoFile,
-        type: ThumbnailType.MINIATURE
-      })
-      thumbnailSave = thumbnailModel.toJSON()
-    }
-
-    // Generate preview if the import did not created it
-    let previewModel: MThumbnail
-    let previewSave: object
-    if (!videoImportWithFiles.Video.getPreview()) {
-      previewModel = await generateVideoMiniature({
-        video: videoImportWithFiles.Video,
-        videoFile,
-        type: ThumbnailType.PREVIEW
-      })
-      previewSave = previewModel.toJSON()
-    }
-
-    // Create torrent
-    await createTorrentAndSetInfoHash(videoImportWithFiles.Video, videoFile)
-
-    const videoFileSave = videoFile.toJSON()
-
-    const { videoImportUpdated, video } = await retryTransactionWrapper(() => {
-      return sequelizeTypescript.transaction(async t => {
-        const videoImportToUpdate = videoImportWithFiles as MVideoImportVideo
-
-        // Refresh video
-        const video = await VideoModel.load(videoImportToUpdate.videoId, t)
-        if (!video) throw new Error('Video linked to import ' + videoImportToUpdate.videoId + ' does not exist anymore.')
-
-        const videoFileCreated = await videoFile.save({ transaction: t })
-
-        // Update video DB object
-        video.duration = duration
-        video.state = buildNextVideoState(video.state)
-        await video.save({ transaction: t })
-
-        if (thumbnailModel) await video.addAndSaveThumbnail(thumbnailModel, t)
-        if (previewModel) await video.addAndSaveThumbnail(previewModel, t)
-
-        // Now we can federate the video (reload from database, we need more attributes)
-        const videoForFederation = await VideoModel.loadFull(video.uuid, t)
-        await federateVideoIfNeeded(videoForFederation, true, t)
-
-        // Update video import object
-        videoImportToUpdate.state = VideoImportState.SUCCESS
-        const videoImportUpdated = await videoImportToUpdate.save({ transaction: t }) as MVideoImportVideo
-        videoImportUpdated.Video = video
-
-        videoImportToUpdate.Video = Object.assign(video, { VideoFiles: [ videoFileCreated ] })
-
-        logger.info('Video %s imported.', video.uuid)
-
-        return { videoImportUpdated, video: videoForFederation }
-      }).catch(err => {
-        // Reset fields
-        if (thumbnailModel) thumbnailModel = new ThumbnailModel(thumbnailSave)
-        if (previewModel) previewModel = new ThumbnailModel(previewSave)
-
-        videoFile = new VideoFileModel(videoFileSave)
-
-        throw err
-      })
-    })
-
-    Notifier.Instance.notifyOnFinishedVideoImport({ videoImport: videoImportUpdated, success: true })
-
-    if (video.isBlacklisted()) {
-      const videoBlacklist = Object.assign(video.VideoBlacklist, { Video: video })
-
-      Notifier.Instance.notifyOnVideoAutoBlacklist(videoBlacklist)
-    } else {
-      Notifier.Instance.notifyOnNewVideoIfNeeded(video)
-    }
-
-    if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
-      await JobQueue.Instance.createJob(
-        await buildMoveToObjectStorageJob({ video: videoImportUpdated.Video, previousVideoState: VideoState.TO_IMPORT })
-      )
-    }
-
-    // Create transcoding jobs?
-    if (video.state === VideoState.TO_TRANSCODE) {
-      await JobQueue.Instance.createJob(
-        await buildOptimizeOrMergeAudioJob({ video: videoImportUpdated.Video, videoFile, user: videoImport.User })
-      )
-    }
-
-  } catch (err) {
     try {
-      if (tempVideoPath) await remove(tempVideoPath)
+      const videoImportWithFiles = await refreshVideoImportFromDB(videoImport, videoFile)
-    } catch (errUnlink) {
-      logger.warn('Cannot cleanup files after a video import error.', { err: errUnlink })
-    }
 
-    videoImport.error = err.message
+      // Move file
-    if (videoImport.state !== VideoImportState.REJECTED) {
+      const videoDestFile = VideoPathManager.Instance.getFSVideoFileOutputPath(videoImportWithFiles.Video, videoFile)
-      videoImport.state = VideoImportState.FAILED
+      await move(tempVideoPath, videoDestFile)
-    }
-    await videoImport.save()
 
-    Notifier.Instance.notifyOnFinishedVideoImport({ videoImport, success: false })
+      tempVideoPath = null // This path is not used anymore
+
+      let {
+        miniatureModel: thumbnailModel,
+        miniatureJSONSave: thumbnailSave
+      } = await generateMiniature(videoImportWithFiles, videoFile, ThumbnailType.MINIATURE)
+
+      let {
+        miniatureModel: previewModel,
+        miniatureJSONSave: previewSave
+      } = await generateMiniature(videoImportWithFiles, videoFile, ThumbnailType.PREVIEW)
+
+      // Create torrent
+      await createTorrentAndSetInfoHash(videoImportWithFiles.Video, videoFile)
+
+      const videoFileSave = videoFile.toJSON()
+
+      const { videoImportUpdated, video } = await retryTransactionWrapper(() => {
+        return sequelizeTypescript.transaction(async t => {
+          // Refresh video
+          const video = await VideoModel.load(videoImportWithFiles.videoId, t)
+          if (!video) throw new Error('Video linked to import ' + videoImportWithFiles.videoId + ' does not exist anymore.')
+
+          await videoFile.save({ transaction: t })
+
+          // Update video DB object
+          video.duration = duration
+          video.state = buildNextVideoState(video.state)
+          await video.save({ transaction: t })
+
+          if (thumbnailModel) await video.addAndSaveThumbnail(thumbnailModel, t)
+          if (previewModel) await video.addAndSaveThumbnail(previewModel, t)
+
+          // Now we can federate the video (reload from database, we need more attributes)
+          const videoForFederation = await VideoModel.loadFull(video.uuid, t)
+          await federateVideoIfNeeded(videoForFederation, true, t)
+
+          // Update video import object
+          videoImportWithFiles.state = VideoImportState.SUCCESS
+          const videoImportUpdated = await videoImportWithFiles.save({ transaction: t }) as MVideoImport
+
+          logger.info('Video %s imported.', video.uuid)
+
+          return { videoImportUpdated, video: videoForFederation }
+        }).catch(err => {
+          // Reset fields
+          if (thumbnailModel) thumbnailModel = new ThumbnailModel(thumbnailSave)
+          if (previewModel) previewModel = new ThumbnailModel(previewSave)
+
+          videoFile = new VideoFileModel(videoFileSave)
+
+          throw err
+        })
+      })
+
+      await afterImportSuccess({ videoImport: videoImportUpdated, video, videoFile, user: videoImport.User })
+    } finally {
+      videoFileLockReleaser()
+    }
+  } catch (err) {
+    await onImportError(err, tempVideoPath, videoImport)
 
     throw err
   }
 }
+
+async function refreshVideoImportFromDB (videoImport: MVideoImportDefault, videoFile: MVideoFile): Promise<MVideoImportDefaultFiles> {
+  // Refresh video, privacy may have changed
+  const video = await videoImport.Video.reload()
+  const videoWithFiles = Object.assign(video, { VideoFiles: [ videoFile ], VideoStreamingPlaylists: [] })
+
+  return Object.assign(videoImport, { Video: videoWithFiles })
+}
+
+async function generateMiniature (videoImportWithFiles: MVideoImportDefaultFiles, videoFile: MVideoFile, thumbnailType: ThumbnailType) {
+  // Generate miniature if the import did not created it
+  const needsMiniature = thumbnailType === ThumbnailType.MINIATURE
+    ? !videoImportWithFiles.Video.getMiniature()
+    : !videoImportWithFiles.Video.getPreview()
+
+  if (!needsMiniature) {
+    return {
+      miniatureModel: null,
+      miniatureJSONSave: null
+    }
+  }
+
+  const miniatureModel = await generateVideoMiniature({
+    video: videoImportWithFiles.Video,
+    videoFile,
+    type: ThumbnailType.MINIATURE
+  })
+  const miniatureJSONSave = miniatureModel.toJSON()
+
+  return {
+    miniatureModel,
+    miniatureJSONSave
+  }
+}
+
+async function afterImportSuccess (options: {
+  videoImport: MVideoImport
+  video: MVideoFullLight
+  videoFile: MVideoFile
+  user: MUserId
+}) {
+  const { video, videoFile, videoImport, user } = options
+
+  Notifier.Instance.notifyOnFinishedVideoImport({ videoImport: Object.assign(videoImport, { Video: video }), success: true })
+
+  if (video.isBlacklisted()) {
+    const videoBlacklist = Object.assign(video.VideoBlacklist, { Video: video })
+
+    Notifier.Instance.notifyOnVideoAutoBlacklist(videoBlacklist)
+  } else {
+    Notifier.Instance.notifyOnNewVideoIfNeeded(video)
+  }
+
+  if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
+    await JobQueue.Instance.createJob(
+      await buildMoveToObjectStorageJob({ video, previousVideoState: VideoState.TO_IMPORT })
+    )
+    return
+  }
+
+  if (video.state === VideoState.TO_TRANSCODE) { // Create transcoding jobs?
+    await JobQueue.Instance.createJob(
+      await buildOptimizeOrMergeAudioJob({ video, videoFile, user })
+    )
+  }
+}
+
+async function onImportError (err: Error, tempVideoPath: string, videoImport: MVideoImportVideo) {
+  try {
+    if (tempVideoPath) await remove(tempVideoPath)
+  } catch (errUnlink) {
+    logger.warn('Cannot cleanup files after a video import error.', { err: errUnlink })
+  }
+
+  videoImport.error = err.message
+  if (videoImport.state !== VideoImportState.REJECTED) {
+    videoImport.state = VideoImportState.FAILED
+  }
+  await videoImport.save()
+
+  Notifier.Instance.notifyOnFinishedVideoImport({ videoImport, success: false })
+}

server/lib/sync-channel.ts

@@ -1,7 +1,7 @@
 import { logger } from '@server/helpers/logger'
 import { YoutubeDLWrapper } from '@server/helpers/youtube-dl'
 import { CONFIG } from '@server/initializers/config'
-import { buildYoutubeDLImport } from '@server/lib/video-import'
+import { buildYoutubeDLImport } from '@server/lib/video-pre-import'
 import { UserModel } from '@server/models/user/user'
 import { VideoImportModel } from '@server/models/video/video-import'
 import { MChannel, MChannelAccountDefault, MChannelSync } from '@server/types/models'

server/lib/transcoding/transcoding.ts

@@ -46,6 +46,7 @@ async function optimizeOriginalVideofile (options: {
   const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
   const newExtname = '.mp4'
 
+  // Will be released by our transcodeVOD function once ffmpeg is ran
   const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
 
   try {