Add server plugin filter hooks for import with torrent and url (#2621)

* Add server plugin filter hooks for import with torrent and url

* WIP: pre and post-import filter hooks

* Rebased

* Cleanup filters to accept imports

Co-authored-by: Chocobozzz <me@florianbigard.com>

server/initializers/constants.ts

@@ -372,7 +372,8 @@ const VIDEO_STATES = {
 const VIDEO_IMPORT_STATES = {
   [VideoImportState.FAILED]: 'Failed',
   [VideoImportState.PENDING]: 'Pending',
-  [VideoImportState.SUCCESS]: 'Success'
+  [VideoImportState.SUCCESS]: 'Success',
+  [VideoImportState.REJECTED]: 'Rejected'
 }
 
 const VIDEO_ABUSE_STATES = {

server/lib/job-queue/handlers/video-import.ts

@@ -1,27 +1,36 @@
 import * as Bull from 'bull'
-import { logger } from '../../../helpers/logger'
-import { downloadYoutubeDLVideo } from '../../../helpers/youtube-dl'
-import { VideoImportModel } from '../../../models/video/video-import'
-import { VideoImportState } from '../../../../shared/models/videos'
-import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils'
-import { extname } from 'path'
-import { VideoFileModel } from '../../../models/video/video-file'
-import { VIDEO_IMPORT_TIMEOUT } from '../../../initializers/constants'
-import { VideoImportPayload, VideoImportTorrentPayload, VideoImportYoutubeDLPayload, VideoState } from '../../../../shared'
-import { federateVideoIfNeeded } from '../../activitypub/videos'
-import { VideoModel } from '../../../models/video/video'
-import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent'
-import { getSecureTorrentName } from '../../../helpers/utils'
 import { move, remove, stat } from 'fs-extra'
-import { Notifier } from '../../notifier'
-import { CONFIG } from '../../../initializers/config'
-import { sequelizeTypescript } from '../../../initializers/database'
-import { generateVideoMiniature } from '../../thumbnail'
-import { ThumbnailType } from '../../../../shared/models/videos/thumbnail.type'
-import { MThumbnail } from '../../../typings/models/video/thumbnail'
-import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/typings/models/video/video-import'
-import { getVideoFilePath } from '@server/lib/video-paths'
+import { extname } from 'path'
 import { addOptimizeOrMergeAudioJob } from '@server/helpers/video'
+import { isPostImportVideoAccepted } from '@server/lib/moderation'
+import { Hooks } from '@server/lib/plugins/hooks'
+import { getVideoFilePath } from '@server/lib/video-paths'
+import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/typings/models/video/video-import'
+import {
+  VideoImportPayload,
+  VideoImportTorrentPayload,
+  VideoImportTorrentPayloadType,
+  VideoImportYoutubeDLPayload,
+  VideoImportYoutubeDLPayloadType,
+  VideoState
+} from '../../../../shared'
+import { VideoImportState } from '../../../../shared/models/videos'
+import { ThumbnailType } from '../../../../shared/models/videos/thumbnail.type'
+import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils'
+import { logger } from '../../../helpers/logger'
+import { getSecureTorrentName } from '../../../helpers/utils'
+import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent'
+import { downloadYoutubeDLVideo } from '../../../helpers/youtube-dl'
+import { CONFIG } from '../../../initializers/config'
+import { VIDEO_IMPORT_TIMEOUT } from '../../../initializers/constants'
+import { sequelizeTypescript } from '../../../initializers/database'
+import { VideoModel } from '../../../models/video/video'
+import { VideoFileModel } from '../../../models/video/video-file'
+import { VideoImportModel } from '../../../models/video/video-import'
+import { MThumbnail } from '../../../typings/models/video/thumbnail'
+import { federateVideoIfNeeded } from '../../activitypub/videos'
+import { Notifier } from '../../notifier'
+import { generateVideoMiniature } from '../../thumbnail'
 
 async function processVideoImport (job: Bull.Job) {
   const payload = job.data as VideoImportPayload
@@ -44,6 +53,7 @@ async function processTorrentImport (job: Bull.Job, payload: VideoImportTorrentP
   const videoImport = await getVideoImportOrDie(payload.videoImportId)
 
   const options = {
+    type: payload.type,
     videoImportId: payload.videoImportId,
 
     generateThumbnail: true,
@@ -61,6 +71,7 @@ async function processYoutubeDLImport (job: Bull.Job, payload: VideoImportYoutub
 
   const videoImport = await getVideoImportOrDie(payload.videoImportId)
   const options = {
+    type: payload.type,
     videoImportId: videoImport.id,
 
     generateThumbnail: payload.generateThumbnail,
@@ -80,6 +91,7 @@ async function getVideoImportOrDie (videoImportId: number) {
 }
 
 type ProcessFileOptions = {
+  type: VideoImportYoutubeDLPayloadType | VideoImportTorrentPayloadType
   videoImportId: number
 
   generateThumbnail: boolean
@@ -105,7 +117,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
     const fps = await getVideoFileFPS(tempVideoPath)
     const duration = await getDurationFromVideoFile(tempVideoPath)
 
-    // Create video file object in database
+    // Prepare video file object for creation in database
     const videoFileData = {
       extname: extname(tempVideoPath),
       resolution: videoFileResolution,
@@ -115,6 +127,30 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
     }
     videoFile = new VideoFileModel(videoFileData)
 
+    const hookName = options.type === 'youtube-dl'
+      ? 'filter:api.video.post-import-url.accept.result'
+      : 'filter:api.video.post-import-torrent.accept.result'
+
+    // Check we accept this video
+    const acceptParameters = {
+      videoImport,
+      video: videoImport.Video,
+      videoFilePath: tempVideoPath,
+      videoFile,
+      user: videoImport.User
+    }
+    const acceptedResult = await Hooks.wrapFun(isPostImportVideoAccepted, acceptParameters, hookName)
+
+    if (acceptedResult.accepted !== true) {
+      logger.info('Refused imported video.', { acceptedResult, acceptParameters })
+
+      videoImport.state = VideoImportState.REJECTED
+      await videoImport.save()
+
+      throw new Error(acceptedResult.errorMessage)
+    }
+
+    // Video is accepted, resuming preparation
     const videoWithFiles = Object.assign(videoImport.Video, { VideoFiles: [ videoFile ], VideoStreamingPlaylists: [] })
     // To clean files if the import fails
     const videoImportWithFiles: MVideoImportDefaultFiles = Object.assign(videoImport, { Video: videoWithFiles })
@@ -194,7 +230,9 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
     }
 
     videoImport.error = err.message
-    videoImport.state = VideoImportState.FAILED
+    if (videoImport.state !== VideoImportState.REJECTED) {
+      videoImport.state = VideoImportState.FAILED
+    }
     await videoImport.save()
 
     Notifier.Instance.notifyOnFinishedVideoImport(videoImport, false)

server/lib/moderation.ts

@@ -1,12 +1,15 @@
 import { VideoModel } from '../models/video/video'
 import { VideoCommentModel } from '../models/video/video-comment'
 import { VideoCommentCreate } from '../../shared/models/videos/video-comment.model'
-import { VideoCreate } from '../../shared/models/videos'
+import { VideoCreate, VideoImportCreate } from '../../shared/models/videos'
 import { UserModel } from '../models/account/user'
 import { VideoTorrentObject } from '../../shared/models/activitypub/objects'
 import { ActivityCreate } from '../../shared/models/activitypub'
 import { ActorModel } from '../models/activitypub/actor'
 import { VideoCommentObject } from '../../shared/models/activitypub/objects/video-comment-object'
+import { VideoFileModel } from '@server/models/video/video-file'
+import { PathLike } from 'fs-extra'
+import { MUser } from '@server/typings/models'
 
 export type AcceptResult = {
   accepted: boolean
@@ -55,10 +58,27 @@ function isRemoteVideoCommentAccepted (_object: {
   return { accepted: true }
 }
 
+function isPreImportVideoAccepted (object: {
+  videoImportBody: VideoImportCreate
+  user: MUser
+}): AcceptResult {
+  return { accepted: true }
+}
+
+function isPostImportVideoAccepted (object: {
+  videoFilePath: PathLike
+  videoFile: VideoFileModel
+  user: MUser
+}): AcceptResult {
+  return { accepted: true }
+}
+
 export {
   isLocalVideoAccepted,
   isLocalVideoThreadAccepted,
   isRemoteVideoAccepted,
   isRemoteVideoCommentAccepted,
-  isLocalVideoCommentReplyAccepted
+  isLocalVideoCommentReplyAccepted,
+  isPreImportVideoAccepted,
+  isPostImportVideoAccepted
 }

server/middlewares/validators/videos/video-imports.ts

@@ -1,15 +1,18 @@
 import * as express from 'express'
 import { body } from 'express-validator'
+import { isPreImportVideoAccepted } from '@server/lib/moderation'
+import { Hooks } from '@server/lib/plugins/hooks'
+import { VideoImportCreate } from '@shared/models/videos/import/video-import-create.model'
 import { isIdValid, toIntOrNull } from '../../../helpers/custom-validators/misc'
-import { logger } from '../../../helpers/logger'
-import { areValidationErrors } from '../utils'
-import { getCommonVideoEditAttributes } from './videos'
 import { isVideoImportTargetUrlValid, isVideoImportTorrentFile } from '../../../helpers/custom-validators/video-imports'
-import { cleanUpReqFiles } from '../../../helpers/express-utils'
 import { isVideoMagnetUriValid, isVideoNameValid } from '../../../helpers/custom-validators/videos'
+import { cleanUpReqFiles } from '../../../helpers/express-utils'
+import { logger } from '../../../helpers/logger'
+import { doesVideoChannelOfAccountExist } from '../../../helpers/middlewares'
 import { CONFIG } from '../../../initializers/config'
 import { CONSTRAINTS_FIELDS } from '../../../initializers/constants'
-import { doesVideoChannelOfAccountExist } from '../../../helpers/middlewares'
+import { areValidationErrors } from '../utils'
+import { getCommonVideoEditAttributes } from './videos'
 
 const videoImportAddValidator = getCommonVideoEditAttributes().concat([
   body('channelId')
@@ -64,6 +67,8 @@ const videoImportAddValidator = getCommonVideoEditAttributes().concat([
         .end()
     }
 
+    if (!await isImportAccepted(req, res)) return cleanUpReqFiles(req)
+
     return next()
   }
 ])
@@ -75,3 +80,31 @@ export {
 }
 
 // ---------------------------------------------------------------------------
+
+async function isImportAccepted (req: express.Request, res: express.Response) {
+  const body: VideoImportCreate = req.body
+  const hookName = body.targetUrl
+    ? 'filter:api.video.pre-import-url.accept.result'
+    : 'filter:api.video.pre-import-torrent.accept.result'
+
+  // Check we accept this video
+  const acceptParameters = {
+    videoImportBody: body,
+    user: res.locals.oauth.token.User
+  }
+  const acceptedResult = await Hooks.wrapFun(
+    isPreImportVideoAccepted,
+    acceptParameters,
+    hookName
+  )
+
+  if (!acceptedResult || acceptedResult.accepted !== true) {
+    logger.info('Refused to import video.', { acceptedResult, acceptParameters })
+    res.status(403)
+       .json({ error: acceptedResult.errorMessage || 'Refused to import video' })
+
+    return false
+  }
+
+  return true
+}

server/tests/fixtures/peertube-plugin-test/main.js

@@ -50,7 +50,47 @@ async function register ({ registerHook, registerSetting, settingsManager, stora
     target: 'filter:api.video.upload.accept.result',
     handler: ({ accepted }, { videoBody }) => {
       if (!accepted) return { accepted: false }
-      if (videoBody.name.indexOf('bad word') !== -1) return { accepted: false, errorMessage: 'bad word '}
+      if (videoBody.name.indexOf('bad word') !== -1) return { accepted: false, errorMessage: 'bad word' }
+
+      return { accepted: true }
+    }
+  })
+
+  registerHook({
+    target: 'filter:api.video.pre-import-url.accept.result',
+    handler: ({ accepted }, { videoImportBody }) => {
+      if (!accepted) return { accepted: false }
+      if (videoImportBody.targetUrl.includes('bad')) return { accepted: false, errorMessage: 'bad target url' }
+
+      return { accepted: true }
+    }
+  })
+
+  registerHook({
+    target: 'filter:api.video.pre-import-torrent.accept.result',
+    handler: ({ accepted }, { videoImportBody }) => {
+      if (!accepted) return { accepted: false }
+      if (videoImportBody.name.includes('bad torrent')) return { accepted: false, errorMessage: 'bad torrent' }
+
+      return { accepted: true }
+    }
+  })
+
+  registerHook({
+    target: 'filter:api.video.post-import-url.accept.result',
+    handler: ({ accepted }, { video }) => {
+      if (!accepted) return { accepted: false }
+      if (video.name.includes('bad word')) return { accepted: false, errorMessage: 'bad word' }
+
+      return { accepted: true }
+    }
+  })
+
+  registerHook({
+    target: 'filter:api.video.post-import-torrent.accept.result',
+    handler: ({ accepted }, { video }) => {
+      if (!accepted) return { accepted: false }
+      if (video.name.includes('bad word')) return { accepted: false, errorMessage: 'bad word' }
 
       return { accepted: true }
     }

server/tests/plugins/filter-hooks.ts

@@ -1,8 +1,8 @@
 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
 
-import * as chai from 'chai'
 import 'mocha'
-import { cleanupTests, flushAndRunMultipleServers, ServerInfo } from '../../../shared/extra-utils/server/servers'
+import * as chai from 'chai'
+import { ServerConfig } from '@shared/models'
 import {
   addVideoCommentReply,
   addVideoCommentThread,
@@ -23,10 +23,10 @@ import {
   uploadVideo,
   waitJobs
 } from '../../../shared/extra-utils'
+import { cleanupTests, flushAndRunMultipleServers, ServerInfo } from '../../../shared/extra-utils/server/servers'
+import { getMyVideoImports, getYoutubeVideoUrl, importVideo } from '../../../shared/extra-utils/videos/video-imports'
+import { VideoDetails, VideoImport, VideoImportState, VideoPrivacy } from '../../../shared/models/videos'
 import { VideoCommentThreadTree } from '../../../shared/models/videos/video-comment.model'
-import { VideoDetails } from '../../../shared/models/videos'
-import { getYoutubeVideoUrl, importVideo } from '../../../shared/extra-utils/videos/video-imports'
-import { ServerConfig } from '@shared/models'
 
 const expect = chai.expect
 
@@ -87,6 +87,84 @@ describe('Test plugin filter hooks', function () {
     await uploadVideo(servers[0].url, servers[0].accessToken, { name: 'video with bad word' }, 403)
   })
 
+  it('Should run filter:api.video.pre-import-url.accept.result', async function () {
+    const baseAttributes = {
+      name: 'normal title',
+      privacy: VideoPrivacy.PUBLIC,
+      channelId: servers[0].videoChannel.id,
+      targetUrl: getYoutubeVideoUrl() + 'bad'
+    }
+    await importVideo(servers[0].url, servers[0].accessToken, baseAttributes, 403)
+  })
+
+  it('Should run filter:api.video.pre-import-torrent.accept.result', async function () {
+    const baseAttributes = {
+      name: 'bad torrent',
+      privacy: VideoPrivacy.PUBLIC,
+      channelId: servers[0].videoChannel.id,
+      torrentfile: 'video-720p.torrent' as any
+    }
+    await importVideo(servers[0].url, servers[0].accessToken, baseAttributes, 403)
+  })
+
+  it('Should run filter:api.video.post-import-url.accept.result', async function () {
+    this.timeout(60000)
+
+    let videoImportId: number
+
+    {
+      const baseAttributes = {
+        name: 'title with bad word',
+        privacy: VideoPrivacy.PUBLIC,
+        channelId: servers[0].videoChannel.id,
+        targetUrl: getYoutubeVideoUrl()
+      }
+      const res = await importVideo(servers[0].url, servers[0].accessToken, baseAttributes)
+      videoImportId = res.body.id
+    }
+
+    await waitJobs(servers)
+
+    {
+      const res = await getMyVideoImports(servers[0].url, servers[0].accessToken)
+      const videoImports = res.body.data as VideoImport[]
+
+      const videoImport = videoImports.find(i => i.id === videoImportId)
+
+      expect(videoImport.state.id).to.equal(VideoImportState.REJECTED)
+      expect(videoImport.state.label).to.equal('Rejected')
+    }
+  })
+
+  it('Should run filter:api.video.post-import-torrent.accept.result', async function () {
+    this.timeout(60000)
+
+    let videoImportId: number
+
+    {
+      const baseAttributes = {
+        name: 'title with bad word',
+        privacy: VideoPrivacy.PUBLIC,
+        channelId: servers[0].videoChannel.id,
+        torrentfile: 'video-720p.torrent' as any
+      }
+      const res = await importVideo(servers[0].url, servers[0].accessToken, baseAttributes)
+      videoImportId = res.body.id
+    }
+
+    await waitJobs(servers)
+
+    {
+      const res = await getMyVideoImports(servers[0].url, servers[0].accessToken)
+      const videoImports = res.body.data as VideoImport[]
+
+      const videoImport = videoImports.find(i => i.id === videoImportId)
+
+      expect(videoImport.state.id).to.equal(VideoImportState.REJECTED)
+      expect(videoImport.state.label).to.equal('Rejected')
+    }
+  })
+
   it('Should run filter:api.video-thread.create.accept.result', async function () {
     await addVideoCommentThread(servers[0].url, servers[0].accessToken, videoUUID, 'comment with bad word', 403)
   })

shared/extra-utils/videos/video-imports.ts

@@ -15,7 +15,7 @@ function getBadVideoUrl () {
   return 'https://download.cpy.re/peertube/bad_video.mp4'
 }
 
-function importVideo (url: string, token: string, attributes: VideoImportCreate) {
+function importVideo (url: string, token: string, attributes: VideoImportCreate & { torrentfile?: string }, statusCodeExpected = 200) {
   const path = '/api/v1/videos/imports'
 
   let attaches: any = {}
@@ -27,7 +27,7 @@ function importVideo (url: string, token: string, attributes: VideoImportCreate)
     token,
     attaches,
     fields: attributes,
-    statusCodeExpected: 200
+    statusCodeExpected
   })
 }
 

shared/models/plugins/server-hook.model.ts

@@ -9,9 +9,13 @@ export const serverFilterHookObject = {
   // Used to get detailed video information (video watch page for example)
   'filter:api.video.get.result': true,
 
-  // Filter the result of the accept upload function
+  // Filter the result of the accept upload, import via torrent or url functions
   // If this function returns false then the upload is aborted with an error
   'filter:api.video.upload.accept.result': true,
+  'filter:api.video.pre-import-url.accept.result': true,
+  'filter:api.video.pre-import-torrent.accept.result': true,
+  'filter:api.video.post-import-url.accept.result': true,
+  'filter:api.video.post-import-torrent.accept.result': true,
   // Filter the result of the accept comment (thread or reply) functions
   // If the functions return false then the user cannot post its comment
   'filter:api.video-thread.create.accept.result': true,

shared/models/server/job.model.ts

@@ -70,8 +70,11 @@ export type VideoFileImportPayload = {
   filePath: string
 }
 
+export type VideoImportTorrentPayloadType = 'magnet-uri' | 'torrent-file'
+export type VideoImportYoutubeDLPayloadType = 'youtube-dl'
+
 export type VideoImportYoutubeDLPayload = {
-  type: 'youtube-dl'
+  type: VideoImportYoutubeDLPayloadType
   videoImportId: number
 
   generateThumbnail: boolean
@@ -80,7 +83,7 @@ export type VideoImportYoutubeDLPayload = {
   fileExt?: string
 }
 export type VideoImportTorrentPayload = {
-  type: 'magnet-uri' | 'torrent-file'
+  type: VideoImportTorrentPayloadType
   videoImportId: number
 }
 export type VideoImportPayload = VideoImportYoutubeDLPayload | VideoImportTorrentPayload

shared/models/videos/import/video-import-state.enum.ts

@@ -1,5 +1,6 @@
 export enum VideoImportState {
   PENDING = 1,
   SUCCESS = 2,
-  FAILED = 3
+  FAILED = 3,
+  REJECTED = 4
 }