Refactor caption creation

server/core/controllers/api/video-channel.ts

@@ -21,7 +21,7 @@ import { sequelizeTypescript } from '../../initializers/database.js'
 import { sendUpdateActor } from '../../lib/activitypub/send/index.js'
 import { JobQueue } from '../../lib/job-queue/index.js'
 import { deleteLocalActorImageFile, updateLocalActorImageFiles } from '../../lib/local-actor.js'
-import { createLocalVideoChannel, federateAllVideosOfChannel } from '../../lib/video-channel.js'
+import { createLocalVideoChannelWithoutKeys, federateAllVideosOfChannel } from '../../lib/video-channel.js'
 import {
   apiRateLimiter,
   asyncMiddleware,
@@ -77,7 +77,7 @@ videoChannelRouter.get('/',
 videoChannelRouter.post('/',
   authenticate,
   asyncMiddleware(videoChannelsAddValidator),
-  asyncRetryTransactionMiddleware(addVideoChannel)
+  asyncRetryTransactionMiddleware(createVideoChannel)
 )
 
 videoChannelRouter.post('/:nameWithHost/avatar/pick',
@@ -262,17 +262,19 @@ async function deleteVideoChannelBanner (req: express.Request, res: express.Resp
   return res.status(HttpStatusCode.NO_CONTENT_204).end()
 }
 
-async function addVideoChannel (req: express.Request, res: express.Response) {
+async function createVideoChannel (req: express.Request, res: express.Response) {
   const videoChannelInfo: VideoChannelCreate = req.body
 
   const videoChannelCreated = await sequelizeTypescript.transaction(async t => {
     const account = await AccountModel.load(res.locals.oauth.token.User.Account.id, t)
 
-    return createLocalVideoChannel(videoChannelInfo, account, t)
+    return createLocalVideoChannelWithoutKeys(videoChannelInfo, account, t)
   })
 
-  const payload = { actorId: videoChannelCreated.actorId }
+  await JobQueue.Instance.createJob({
-  await JobQueue.Instance.createJob({ type: 'actor-keys', payload })
+    type: 'actor-keys',
+    payload: { actorId: videoChannelCreated.actorId }
+  })
 
   auditLogger.create(getAuditIdFromRes(res), new VideoChannelAuditView(videoChannelCreated.toFormattedJSON()))
   logger.info('Video channel %s created.', videoChannelCreated.Actor.url)

server/core/controllers/api/video-playlist.ts

@@ -78,7 +78,7 @@ videoPlaylistRouter.post('/',
   authenticate,
   reqThumbnailFile,
   asyncMiddleware(videoPlaylistsAddValidator),
-  asyncRetryTransactionMiddleware(addVideoPlaylist)
+  asyncRetryTransactionMiddleware(createVideoPlaylist)
 )
 
 videoPlaylistRouter.put('/:playlistId',
@@ -159,7 +159,7 @@ function getVideoPlaylist (req: express.Request, res: express.Response) {
   return res.json(videoPlaylist.toFormattedJSON())
 }
 
-async function addVideoPlaylist (req: express.Request, res: express.Response) {
+async function createVideoPlaylist (req: express.Request, res: express.Response) {
   const videoPlaylistInfo: VideoPlaylistCreate = req.body
   const user = res.locals.oauth.token.User
 

server/core/controllers/api/videos/captions.ts

@@ -1,8 +1,6 @@
 import express from 'express'
 import { HttpStatusCode } from '@peertube/peertube-models'
 import { Hooks } from '@server/lib/plugins/hooks.js'
-import { MVideoCaption } from '@server/types/models/index.js'
-import { moveAndProcessCaptionFile } from '../../../helpers/captions-utils.js'
 import { createReqFiles } from '../../../helpers/express-utils.js'
 import { logger } from '../../../helpers/logger.js'
 import { getFormattedObjects } from '../../../helpers/utils.js'
@@ -12,6 +10,7 @@ import { federateVideoIfNeeded } from '../../../lib/activitypub/videos/index.js'
 import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate } from '../../../middlewares/index.js'
 import { addVideoCaptionValidator, deleteVideoCaptionValidator, listVideoCaptionsValidator } from '../../../middlewares/validators/index.js'
 import { VideoCaptionModel } from '../../../models/video/video-caption.js'
+import { createLocalCaption } from '@server/lib/video-captions.js'
 
 const reqVideoCaptionAdd = createReqFiles([ 'captionfile' ], MIMETYPES.VIDEO_CAPTIONS.MIMETYPE_EXT)
 
@@ -25,7 +24,7 @@ videoCaptionsRouter.put('/:videoId/captions/:captionLanguage',
   authenticate,
   reqVideoCaptionAdd,
   asyncMiddleware(addVideoCaptionValidator),
-  asyncRetryTransactionMiddleware(addVideoCaption)
+  asyncRetryTransactionMiddleware(createVideoCaption)
 )
 videoCaptionsRouter.delete('/:videoId/captions/:captionLanguage',
   authenticate,
@@ -47,25 +46,15 @@ async function listVideoCaptions (req: express.Request, res: express.Response) {
   return res.json(getFormattedObjects(data, data.length))
 }
 
-async function addVideoCaption (req: express.Request, res: express.Response) {
+async function createVideoCaption (req: express.Request, res: express.Response) {
   const videoCaptionPhysicalFile = req.files['captionfile'][0]
   const video = res.locals.videoAll
 
   const captionLanguage = req.params.captionLanguage
 
-  const videoCaption = new VideoCaptionModel({
+  const videoCaption = await createLocalCaption({ video, language: captionLanguage, path: videoCaptionPhysicalFile })
-    videoId: video.id,
-    filename: VideoCaptionModel.generateCaptionName(captionLanguage),
-    language: captionLanguage
-  }) as MVideoCaption
-
-  // Move physical file
-  await moveAndProcessCaptionFile(videoCaptionPhysicalFile, videoCaption)
 
   await sequelizeTypescript.transaction(async t => {
-    await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t)
-
-    // Update video update
     await federateVideoIfNeeded(video, false, t)
   })
 

server/core/lib/user-import-export/importers/channels-importer.ts

@@ -3,7 +3,7 @@ import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
 import { pick } from '@peertube/peertube-core-utils'
 import { AbstractUserImporter } from './abstract-user-importer.js'
 import { sequelizeTypescript } from '@server/initializers/database.js'
-import { createLocalVideoChannel } from '@server/lib/video-channel.js'
+import { createLocalVideoChannelWithoutKeys } from '@server/lib/video-channel.js'
 import { JobQueue } from '@server/lib/job-queue/job-queue.js'
 import { updateLocalActorImageFiles } from '@server/lib/local-actor.js'
 import { VideoChannelModel } from '@server/models/video/video-channel.js'
@@ -43,7 +43,7 @@ export class ChannelsImporter extends AbstractUserImporter <ChannelExportJSON, C
       logger.info(`Do not import channel ${existingChannel.name} that already exists on this PeerTube instance`, lTags())
     } else {
       const videoChannelCreated = await sequelizeTypescript.transaction(async t => {
-        return createLocalVideoChannel(pick(channelImportData, [ 'displayName', 'name', 'description', 'support' ]), account, t)
+        return createLocalVideoChannelWithoutKeys(pick(channelImportData, [ 'displayName', 'name', 'description', 'support' ]), account, t)
       })
 
       await JobQueue.Instance.createJob({ type: 'actor-keys', payload: { actorId: videoChannelCreated.actorId } })

server/core/lib/user-import-export/importers/videos-importer.ts

@@ -5,12 +5,9 @@ import { buildNextVideoState } from '@server/lib/video-state.js'
 import { VideoModel } from '@server/models/video/video.js'
 import { pick } from '@peertube/peertube-core-utils'
 import { buildUUID, getFileSize } from '@peertube/peertube-node-utils'
-import { MChannelId, MVideoCaption, MVideoFullLight } from '@server/types/models/index.js'
+import { MChannelId, MVideoFullLight } from '@server/types/models/index.js'
 import { ffprobePromise, getVideoStreamDuration } from '@peertube/peertube-ffmpeg'
-import { sequelizeTypescript } from '@server/initializers/database.js'
 import { VideoChannelModel } from '@server/models/video/video-channel.js'
-import { VideoCaptionModel } from '@server/models/video/video-caption.js'
-import { moveAndProcessCaptionFile } from '@server/helpers/captions-utils.js'
 import { AbstractUserImporter } from './abstract-user-importer.js'
 import { isUserQuotaValid } from '@server/lib/user.js'
 import {
@@ -38,6 +35,7 @@ import { parse } from 'path'
 import { isLocalVideoFileAccepted } from '@server/lib/moderation.js'
 import { LocalVideoCreator, ThumbnailOptions } from '@server/lib/local-video-creator.js'
 import { isVideoChapterTimecodeValid, isVideoChapterTitleValid } from '@server/helpers/custom-validators/video-chapters.js'
+import { createLocalCaption } from '@server/lib/video-captions.js'
 
 const lTags = loggerTagsFactory('user-import')
 
@@ -243,17 +241,7 @@ export class VideosImporter extends AbstractUserImporter <VideoExportJSON, Impor
 
       if (!await this.isFileValidOrLog(absoluteFilePath, CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.FILE_SIZE.max)) continue
 
-      const videoCaption = new VideoCaptionModel({
+      await createLocalCaption({ video, language: captionImport.language, path: absoluteFilePath })
-        videoId: video.id,
-        filename: VideoCaptionModel.generateCaptionName(captionImport.language),
-        language: captionImport.language
-      }) as MVideoCaption
-
-      await moveAndProcessCaptionFile({ path: absoluteFilePath }, videoCaption)
-
-      await sequelizeTypescript.transaction(async (t) => {
-        await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t)
-      })
 
       captionPaths.push(absoluteFilePath)
     }

server/core/lib/user.ts

@@ -24,7 +24,7 @@ import { Emailer } from './emailer.js'
 import { LiveQuotaStore } from './live/live-quota-store.js'
 import { buildActorInstance, findAvailableLocalActorName } from './local-actor.js'
 import { Redis } from './redis.js'
-import { createLocalVideoChannel } from './video-channel.js'
+import { createLocalVideoChannelWithoutKeys } from './video-channel.js'
 import { createWatchLaterPlaylist } from './video-playlist.js'
 
 type ChannelNames = { name: string, displayName: string }
@@ -107,7 +107,7 @@ async function createUserAccountAndChannelAndPlaylist (parameters: {
     userCreated.Account = accountCreated
 
     const channelAttributes = await buildChannelAttributes({ user: userCreated, transaction: t, channelNames })
-    const videoChannel = await createLocalVideoChannel(channelAttributes, accountCreated, t)
+    const videoChannel = await createLocalVideoChannelWithoutKeys(channelAttributes, accountCreated, t)
 
     const videoPlaylist = await createWatchLaterPlaylist(accountCreated, t)
 

server/core/lib/video-captions.ts

@@ -0,0 +1,26 @@
+import { moveAndProcessCaptionFile } from '@server/helpers/captions-utils.js'
+import { sequelizeTypescript } from '@server/initializers/database.js'
+import { VideoCaptionModel } from '@server/models/video/video-caption.js'
+import { MVideo, MVideoCaption } from '@server/types/models/index.js'
+
+export async function createLocalCaption (options: {
+  video: MVideo
+  path: string
+  language: string
+}) {
+  const { language, path, video } = options
+
+  const videoCaption = new VideoCaptionModel({
+    videoId: video.id,
+    filename: VideoCaptionModel.generateCaptionName(language),
+    language
+  }) as MVideoCaption
+
+  await moveAndProcessCaptionFile({ path }, videoCaption)
+
+  await sequelizeTypescript.transaction(async t => {
+    await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t)
+  })
+
+  return videoCaption
+}

server/core/lib/video-channel.ts

@@ -7,7 +7,7 @@ import { getLocalVideoChannelActivityPubUrl } from './activitypub/url.js'
 import { federateVideoIfNeeded } from './activitypub/videos/index.js'
 import { buildActorInstance } from './local-actor.js'
 
-async function createLocalVideoChannel (videoChannelInfo: VideoChannelCreate, account: MAccountId, t: Sequelize.Transaction) {
+async function createLocalVideoChannelWithoutKeys (videoChannelInfo: VideoChannelCreate, account: MAccountId, t: Sequelize.Transaction) {
   const url = getLocalVideoChannelActivityPubUrl(videoChannelInfo.name)
   const actorInstance = buildActorInstance('Group', url, videoChannelInfo.name)
 
@@ -45,6 +45,6 @@ async function federateAllVideosOfChannel (videoChannel: MChannelId) {
 // ---------------------------------------------------------------------------
 
 export {
-  createLocalVideoChannel,
+  createLocalVideoChannelWithoutKeys,
   federateAllVideosOfChannel
 }

server/core/lib/video-pre-import.ts

@@ -8,7 +8,6 @@ import {
   VideoPrivacy,
   VideoState
 } from '@peertube/peertube-models'
-import { moveAndProcessCaptionFile } from '@server/helpers/captions-utils.js'
 import { isVTTFileValid } from '@server/helpers/custom-validators/video-captions.js'
 import { isVideoFileExtnameValid } from '@server/helpers/custom-validators/videos.js'
 import { isResolvingToUnicastOnly } from '@server/helpers/dns.js'
@@ -20,7 +19,6 @@ import { Hooks } from '@server/lib/plugins/hooks.js'
 import { ServerConfigManager } from '@server/lib/server-config-manager.js'
 import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist.js'
 import { setVideoTags } from '@server/lib/video.js'
-import { VideoCaptionModel } from '@server/models/video/video-caption.js'
 import { VideoImportModel } from '@server/models/video/video-import.js'
 import { VideoPasswordModel } from '@server/models/video/video-password.js'
 import { VideoModel } from '@server/models/video/video.js'
@@ -30,9 +28,8 @@ import {
   MChannelSync,
   MThumbnail,
   MUser,
-  MVideoAccountDefault,
+  MVideo,
-  MVideoCaption,
+  MVideoAccountDefault, MVideoImportFormattable,
-  MVideoImportFormattable,
   MVideoTag,
   MVideoThumbnail,
   MVideoWithBlacklistLight
@@ -40,6 +37,7 @@ import {
 import { getLocalVideoActivityPubUrl } from './activitypub/url.js'
 import { updateLocalVideoMiniatureFromExisting, updateLocalVideoMiniatureFromUrl } from './thumbnail.js'
 import { replaceChapters, replaceChaptersFromDescriptionIfNeeded } from './video-chapters.js'
+import { createLocalCaption } from './video-captions.js'
 
 class YoutubeDlImportError extends Error {
   code: YoutubeDlImportError.CODE
@@ -252,7 +250,7 @@ async function buildYoutubeDLImport (options: {
   })
 
   // Get video subtitles
-  await processYoutubeSubtitles(youtubeDL, targetUrl, video.id)
+  await processYoutubeSubtitles(youtubeDL, targetUrl, video)
 
   let fileExt = `.${youtubeDLInfo.ext}`
   if (!isVideoFileExtnameValid(fileExt)) fileExt = '.mp4'
@@ -308,7 +306,7 @@ async function forgeThumbnail ({ inputPath, video, downloadUrl, type }: {
   return null
 }
 
-async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl: string, videoId: number) {
+async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl: string, video: MVideo) {
   try {
     const subtitles = await youtubeDL.getSubtitles()
 
@@ -321,18 +319,7 @@ async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl:
         continue
       }
 
-      const videoCaption = new VideoCaptionModel({
+      await createLocalCaption({ language: subtitle.language, path: subtitle.path, video })
-        videoId,
-        language: subtitle.language,
-        filename: VideoCaptionModel.generateCaptionName(subtitle.language)
-      }) as MVideoCaption
-
-      // Move physical file
-      await moveAndProcessCaptionFile(subtitle, videoCaption)
-
-      await sequelizeTypescript.transaction(async t => {
-        await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t)
-      })
 
       logger.info('Added %s youtube-dl subtitle', subtitle.path)
     }

server/core/models/video/sql/video/videos-id-list-query-builder.ts

@@ -60,7 +60,10 @@ export type BuildVideosListQueryOptions = {
   trendingAlgorithm?: string // best, hot, or any other algorithm implemented
   trendingDays?: number
 
+  // Used to include user history information, exclude blocked videos, include internal videos, adapt hot algorithm...
   user?: MUserAccountId
+
+  // Only list videos watched by this user
   historyOfUser?: MUserId
 
   startDate?: string // ISO 8601