Refractor videos AP functions

server/controllers/api/search.ts

@@ -139,7 +139,7 @@ async function searchVideoURI (url: string, res: express.Response) {
         refreshVideo: false
       }
 
-      const result = await getOrCreateVideoAndAccountAndChannel(url, syncParam)
+      const result = await getOrCreateVideoAndAccountAndChannel({ videoObject: url, syncParam })
       video = result ? result.video : undefined
     } catch (err) {
       logger.info('Cannot search remote video %s.', url, { err })

server/helpers/custom-validators/videos.ts

@@ -18,6 +18,7 @@ import { exists, isArray, isFileValid } from './misc'
 import { VideoChannelModel } from '../../models/video/video-channel'
 import { UserModel } from '../../models/account/user'
 import * as magnetUtil from 'magnet-uri'
+import { fetchVideo, VideoFetchType } from '../video'
 
 const VIDEOS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEOS
 
@@ -152,17 +153,8 @@ function checkUserCanManageVideo (user: UserModel, video: VideoModel, right: Use
   return true
 }
 
-export type VideoFetchType = 'all' | 'only-video' | 'id' | 'none'
 async function isVideoExist (id: string, res: Response, fetchType: VideoFetchType = 'all') {
-  let video: VideoModel | null
+  const video = await fetchVideo(id, fetchType)
-
-  if (fetchType === 'all') {
-    video = await VideoModel.loadAndPopulateAccountAndServerAndTags(id)
-  } else if (fetchType === 'only-video') {
-    video = await VideoModel.load(id)
-  } else if (fetchType === 'id' || fetchType === 'none') {
-    video = await VideoModel.loadOnlyId(id)
-  }
 
   if (video === null) {
     res.status(404)

server/helpers/video.ts

@@ -0,0 +1,25 @@
+import { VideoModel } from '../models/video/video'
+
+type VideoFetchType = 'all' | 'only-video' | 'id' | 'none'
+
+function fetchVideo (id: number | string, fetchType: VideoFetchType) {
+  if (fetchType === 'all') return VideoModel.loadAndPopulateAccountAndServerAndTags(id)
+
+  if (fetchType === 'only-video') return VideoModel.load(id)
+
+  if (fetchType === 'id' || fetchType === 'none') return VideoModel.loadOnlyId(id)
+}
+
+type VideoFetchByUrlType = 'all' | 'only-video'
+function fetchVideoByUrl (url: string, fetchType: VideoFetchByUrlType) {
+  if (fetchType === 'all') return VideoModel.loadByUrlAndPopulateAccount(url)
+
+  if (fetchType === 'only-video') return VideoModel.loadByUrl(url)
+}
+
+export {
+  VideoFetchType,
+  VideoFetchByUrlType,
+  fetchVideo,
+  fetchVideoByUrl
+}

server/lib/activitypub/process/process-announce.ts

@@ -25,7 +25,7 @@ export {
 async function processVideoShare (actorAnnouncer: ActorModel, activity: ActivityAnnounce) {
   const objectUri = typeof activity.object === 'string' ? activity.object : activity.object.id
 
-  const { video } = await getOrCreateVideoAndAccountAndChannel(objectUri)
+  const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: objectUri })
 
   return sequelizeTypescript.transaction(async t => {
     // Add share entry

server/lib/activitypub/process/process-create.ts

@@ -48,7 +48,7 @@ export {
 async function processCreateVideo (activity: ActivityCreate) {
   const videoToCreateData = activity.object as VideoTorrentObject
 
-  const { video } = await getOrCreateVideoAndAccountAndChannel(videoToCreateData)
+  const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoToCreateData })
 
   return video
 }
@@ -59,7 +59,7 @@ async function processCreateDislike (byActor: ActorModel, activity: ActivityCrea
 
   if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url)
 
-  const { video } = await getOrCreateVideoAndAccountAndChannel(dislike.object)
+  const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislike.object })
 
   return sequelizeTypescript.transaction(async t => {
     const rate = {
@@ -86,7 +86,7 @@ async function processCreateDislike (byActor: ActorModel, activity: ActivityCrea
 async function processCreateView (byActor: ActorModel, activity: ActivityCreate) {
   const view = activity.object as ViewObject
 
-  const { video } = await getOrCreateVideoAndAccountAndChannel(view.object)
+  const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: view.object })
 
   const actor = await ActorModel.loadByUrl(view.actor)
   if (!actor) throw new Error('Unknown actor ' + view.actor)
@@ -103,7 +103,7 @@ async function processCreateView (byActor: ActorModel, activity: ActivityCreate)
 async function processCacheFile (byActor: ActorModel, activity: ActivityCreate) {
   const cacheFile = activity.object as CacheFileObject
 
-  const { video } = await getOrCreateVideoAndAccountAndChannel(cacheFile.object)
+  const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFile.object })
 
   await createCacheFile(cacheFile, video, byActor)
 
@@ -120,7 +120,7 @@ async function processCreateVideoAbuse (actor: ActorModel, videoAbuseToCreateDat
   const account = actor.Account
   if (!account) throw new Error('Cannot create dislike with the non account actor ' + actor.url)
 
-  const { video } = await getOrCreateVideoAndAccountAndChannel(videoAbuseToCreateData.object)
+  const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoAbuseToCreateData.object })
 
   return sequelizeTypescript.transaction(async t => {
     const videoAbuseData = {

server/lib/activitypub/process/process-like.ts

@@ -27,7 +27,7 @@ async function processLikeVideo (byActor: ActorModel, activity: ActivityLike) {
   const byAccount = byActor.Account
   if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url)
 
-  const { video } = await getOrCreateVideoAndAccountAndChannel(videoUrl)
+  const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoUrl })
 
   return sequelizeTypescript.transaction(async t => {
     const rate = {

server/lib/activitypub/process/process-undo.ts

@@ -54,7 +54,7 @@ export {
 async function processUndoLike (actorUrl: string, activity: ActivityUndo) {
   const likeActivity = activity.object as ActivityLike
 
-  const { video } = await getOrCreateVideoAndAccountAndChannel(likeActivity.object)
+  const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: likeActivity.object })
 
   return sequelizeTypescript.transaction(async t => {
     const byAccount = await AccountModel.loadByUrl(actorUrl, t)
@@ -78,7 +78,7 @@ async function processUndoLike (actorUrl: string, activity: ActivityUndo) {
 async function processUndoDislike (actorUrl: string, activity: ActivityUndo) {
   const dislike = activity.object.object as DislikeObject
 
-  const { video } = await getOrCreateVideoAndAccountAndChannel(dislike.object)
+  const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislike.object })
 
   return sequelizeTypescript.transaction(async t => {
     const byAccount = await AccountModel.loadByUrl(actorUrl, t)
@@ -102,7 +102,7 @@ async function processUndoDislike (actorUrl: string, activity: ActivityUndo) {
 async function processUndoCacheFile (actorUrl: string, activity: ActivityUndo) {
   const cacheFileObject = activity.object.object as CacheFileObject
 
-  const { video } = await getOrCreateVideoAndAccountAndChannel(cacheFileObject.object)
+  const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFileObject.object })
 
   return sequelizeTypescript.transaction(async t => {
     const byActor = await ActorModel.loadByUrl(actorUrl)

server/lib/activitypub/process/process-update.ts

@@ -48,7 +48,7 @@ async function processUpdateVideo (actor: ActorModel, activity: ActivityUpdate)
     return undefined
   }
 
-  const { video } = await getOrCreateVideoAndAccountAndChannel(videoObject.id)
+  const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoObject.id })
   const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject)
 
   return updateVideoFromAP(video, videoObject, actor.Account, channelActor.VideoChannel, activity.to)
@@ -64,7 +64,7 @@ async function processUpdateCacheFile (byActor: ActorModel, activity: ActivityUp
 
   const redundancyModel = await VideoRedundancyModel.loadByUrl(cacheFileObject.id)
   if (!redundancyModel) {
-    const { video } = await getOrCreateVideoAndAccountAndChannel(cacheFileObject.id)
+    const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFileObject.id })
     return createCacheFile(cacheFileObject, video, byActor)
   }
 

server/lib/activitypub/video-comments.ts

@@ -94,7 +94,7 @@ async function resolveThread (url: string, comments: VideoCommentModel[] = []) {
   try {
     // Maybe it's a reply to a video?
     // If yes, it's done: we resolved all the thread
-    const { video } = await getOrCreateVideoAndAccountAndChannel(url)
+    const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: url })
 
     if (comments.length !== 0) {
       const firstReply = comments[ comments.length - 1 ]

server/lib/activitypub/videos.ts

@@ -3,7 +3,7 @@ import * as sequelize from 'sequelize'
 import * as magnetUtil from 'magnet-uri'
 import { join } from 'path'
 import * as request from 'request'
-import { ActivityIconObject, ActivityVideoUrlObject, VideoState, ActivityUrlObject } from '../../../shared/index'
+import { ActivityIconObject, ActivityUrlObject, ActivityVideoUrlObject, VideoState } from '../../../shared/index'
 import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
 import { VideoPrivacy } from '../../../shared/models/videos'
 import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos'
@@ -28,6 +28,7 @@ import { ActivitypubHttpFetcherPayload } from '../job-queue/handlers/activitypub
 import { createRates } from './video-rates'
 import { addVideoShares, shareVideoByServerAndChannel } from './share'
 import { AccountModel } from '../../models/account/account'
+import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video'
 
 async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) {
   // If the video is not private and published, we federate it
@@ -50,13 +51,24 @@ async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, tr
   }
 }
 
-function fetchRemoteVideoStaticFile (video: VideoModel, path: string, reject: Function) {
+async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.RequestResponse, videoObject: VideoTorrentObject }> {
-  const host = video.VideoChannel.Account.Actor.Server.host
+  const options = {
+    uri: videoUrl,
+    method: 'GET',
+    json: true,
+    activityPub: true
+  }
 
-  // We need to provide a callback, if no we could have an uncaught exception
+  logger.info('Fetching remote video %s.', videoUrl)
-  return request.get(REMOTE_SCHEME.HTTP + '://' + host + path, err => {
+
-    if (err) reject(err)
+  const { response, body } = await doRequest(options)
-  })
+
+  if (sanitizeAndCheckVideoTorrentObject(body) === false) {
+    logger.debug('Remote video JSON is not valid.', { body })
+    return { response, videoObject: undefined }
+  }
+
+  return { response, videoObject: body }
 }
 
 async function fetchRemoteVideoDescription (video: VideoModel) {
@@ -71,6 +83,15 @@ async function fetchRemoteVideoDescription (video: VideoModel) {
   return body.description ? body.description : ''
 }
 
+function fetchRemoteVideoStaticFile (video: VideoModel, path: string, reject: Function) {
+  const host = video.VideoChannel.Account.Actor.Server.host
+
+  // We need to provide a callback, if no we could have an uncaught exception
+  return request.get(REMOTE_SCHEME.HTTP + '://' + host + path, err => {
+    if (err) reject(err)
+  })
+}
+
 function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject) {
   const thumbnailName = video.getThumbnailName()
   const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName)
@@ -82,94 +103,6 @@ function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject)
   return doRequestAndSaveToFile(options, thumbnailPath)
 }
 
-async function videoActivityObjectToDBAttributes (
-  videoChannel: VideoChannelModel,
-  videoObject: VideoTorrentObject,
-  to: string[] = []
-) {
-  const privacy = to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 ? VideoPrivacy.PUBLIC : VideoPrivacy.UNLISTED
-  const duration = videoObject.duration.replace(/[^\d]+/, '')
-
-  let language: string | undefined
-  if (videoObject.language) {
-    language = videoObject.language.identifier
-  }
-
-  let category: number | undefined
-  if (videoObject.category) {
-    category = parseInt(videoObject.category.identifier, 10)
-  }
-
-  let licence: number | undefined
-  if (videoObject.licence) {
-    licence = parseInt(videoObject.licence.identifier, 10)
-  }
-
-  const description = videoObject.content || null
-  const support = videoObject.support || null
-
-  return {
-    name: videoObject.name,
-    uuid: videoObject.uuid,
-    url: videoObject.id,
-    category,
-    licence,
-    language,
-    description,
-    support,
-    nsfw: videoObject.sensitive,
-    commentsEnabled: videoObject.commentsEnabled,
-    waitTranscoding: videoObject.waitTranscoding,
-    state: videoObject.state,
-    channelId: videoChannel.id,
-    duration: parseInt(duration, 10),
-    createdAt: new Date(videoObject.published),
-    publishedAt: new Date(videoObject.published),
-    // FIXME: updatedAt does not seems to be considered by Sequelize
-    updatedAt: new Date(videoObject.updated),
-    views: videoObject.views,
-    likes: 0,
-    dislikes: 0,
-    remote: true,
-    privacy
-  }
-}
-
-function videoFileActivityUrlToDBAttributes (videoCreated: VideoModel, videoObject: VideoTorrentObject) {
-  const fileUrls = videoObject.url.filter(u => isActivityVideoUrlObject(u)) as ActivityVideoUrlObject[]
-
-  if (fileUrls.length === 0) {
-    throw new Error('Cannot find video files for ' + videoCreated.url)
-  }
-
-  const attributes: VideoFileModel[] = []
-  for (const fileUrl of fileUrls) {
-    // Fetch associated magnet uri
-    const magnet = videoObject.url.find(u => {
-      return u.mimeType === 'application/x-bittorrent;x-scheme-handler/magnet' && u.height === fileUrl.height
-    })
-
-    if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href)
-
-    const parsed = magnetUtil.decode(magnet.href)
-    if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) {
-      throw new Error('Cannot parse magnet URI ' + magnet.href)
-    }
-
-    const attribute = {
-      extname: VIDEO_MIMETYPE_EXT[ fileUrl.mimeType ],
-      infoHash: parsed.infoHash,
-      resolution: fileUrl.height,
-      size: fileUrl.size,
-      videoId: videoCreated.id,
-      fps: fileUrl.fps
-    } as VideoFileModel
-    attributes.push(attribute)
-  }
-
-  return attributes
-}
-
 function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) {
   const channel = videoObject.attributedTo.find(a => a.type === 'Group')
   if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url)
@@ -177,51 +110,6 @@ function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject
   return getOrCreateActorAndServerAndModel(channel.id)
 }
 
-async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) {
-  logger.debug('Adding remote video %s.', videoObject.id)
-
-  const videoCreated: VideoModel = await sequelizeTypescript.transaction(async t => {
-    const sequelizeOptions = { transaction: t }
-
-    const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to)
-    const video = VideoModel.build(videoData)
-
-    const videoCreated = await video.save(sequelizeOptions)
-
-    // Process files
-    const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject)
-    if (videoFileAttributes.length === 0) {
-      throw new Error('Cannot find valid files for video %s ' + videoObject.url)
-    }
-
-    const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t }))
-    await Promise.all(videoFilePromises)
-
-    // Process tags
-    const tags = videoObject.tag.map(t => t.name)
-    const tagInstances = await TagModel.findOrCreateTags(tags, t)
-    await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
-
-    // Process captions
-    const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
-      return VideoCaptionModel.insertOrReplaceLanguage(videoCreated.id, c.identifier, t)
-    })
-    await Promise.all(videoCaptionsPromises)
-
-    logger.info('Remote video with uuid %s inserted.', videoObject.uuid)
-
-    videoCreated.VideoChannel = channelActor.VideoChannel
-    return videoCreated
-  })
-
-  const p = generateThumbnailFromUrl(videoCreated, videoObject.icon)
-    .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err }))
-
-  if (waitThumbnail === true) await p
-
-  return videoCreated
-}
-
 type SyncParam = {
   likes: boolean
   dislikes: boolean
@@ -230,28 +118,7 @@ type SyncParam = {
   thumbnail: boolean
   refreshVideo: boolean
 }
-async function getOrCreateVideoAndAccountAndChannel (
+async function syncVideoExternalAttributes (video: VideoModel, fetchedVideo: VideoTorrentObject, syncParam: SyncParam) {
-  videoObject: VideoTorrentObject | string,
-  syncParam: SyncParam = { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false }
-) {
-  const videoUrl = typeof videoObject === 'string' ? videoObject : videoObject.id
-
-  let videoFromDatabase = await VideoModel.loadByUrlAndPopulateAccount(videoUrl)
-  if (videoFromDatabase) {
-    const p = retryTransactionWrapper(refreshVideoIfNeeded, videoFromDatabase)
-    if (syncParam.refreshVideo === true) videoFromDatabase = await p
-
-    return { video: videoFromDatabase }
-  }
-
-  const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl)
-  if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl)
-
-  const channelActor = await getOrCreateVideoChannelFromVideoObject(fetchedVideo)
-  const video = await retryTransactionWrapper(createVideo, fetchedVideo, channelActor, syncParam.thumbnail)
-
-  // Process outside the transaction because we could fetch remote data
-
   logger.info('Adding likes/dislikes/shares/comments of video %s.', video.uuid)
 
   const jobPayloads: ActivitypubHttpFetcherPayload[] = []
@@ -285,56 +152,39 @@ async function getOrCreateVideoAndAccountAndChannel (
   }
 
   await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }))
+}
+
+async function getOrCreateVideoAndAccountAndChannel (options: {
+  videoObject: VideoTorrentObject | string,
+  syncParam?: SyncParam,
+  fetchType?: VideoFetchByUrlType
+}) {
+  // Default params
+  const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false }
+  const fetchType = options.fetchType || 'all'
+
+  // Get video url
+  const videoUrl = typeof options.videoObject === 'string' ? options.videoObject : options.videoObject.id
+
+  let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType)
+  if (videoFromDatabase) {
+    const p = retryTransactionWrapper(refreshVideoIfNeeded, videoFromDatabase, fetchType, syncParam)
+    if (syncParam.refreshVideo === true) videoFromDatabase = await p
+
+    return { video: videoFromDatabase }
+  }
+
+  const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl)
+  if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl)
+
+  const channelActor = await getOrCreateVideoChannelFromVideoObject(fetchedVideo)
+  const video = await retryTransactionWrapper(createVideo, fetchedVideo, channelActor, syncParam.thumbnail)
+
+  await syncVideoExternalAttributes(video, fetchedVideo, syncParam)
 
   return { video }
 }
 
-async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.RequestResponse, videoObject: VideoTorrentObject }> {
-  const options = {
-    uri: videoUrl,
-    method: 'GET',
-    json: true,
-    activityPub: true
-  }
-
-  logger.info('Fetching remote video %s.', videoUrl)
-
-  const { response, body } = await doRequest(options)
-
-  if (sanitizeAndCheckVideoTorrentObject(body) === false) {
-    logger.debug('Remote video JSON is not valid.', { body })
-    return { response, videoObject: undefined }
-  }
-
-  return { response, videoObject: body }
-}
-
-async function refreshVideoIfNeeded (video: VideoModel): Promise<VideoModel> {
-  if (!video.isOutdated()) return video
-
-  try {
-    const { response, videoObject } = await fetchRemoteVideo(video.url)
-    if (response.statusCode === 404) {
-      // Video does not exist anymore
-      await video.destroy()
-      return undefined
-    }
-
-    if (videoObject === undefined) {
-      logger.warn('Cannot refresh remote video: invalid body.')
-      return video
-    }
-
-    const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject)
-    const account = await AccountModel.load(channelActor.VideoChannel.accountId)
-
-    return updateVideoFromAP(video, videoObject, account, channelActor.VideoChannel)
-  } catch (err) {
-    logger.warn('Cannot refresh video.', { err })
-    return video
-  }
-}
-
 async function updateVideoFromAP (
   video: VideoModel,
   videoObject: VideoTorrentObject,
@@ -433,12 +283,7 @@ export {
   fetchRemoteVideoStaticFile,
   fetchRemoteVideoDescription,
   generateThumbnailFromUrl,
-  videoActivityObjectToDBAttributes,
+  getOrCreateVideoChannelFromVideoObject
-  videoFileActivityUrlToDBAttributes,
-  createVideo,
-  getOrCreateVideoChannelFromVideoObject,
-  addVideoShares,
-  createRates
 }
 
 // ---------------------------------------------------------------------------
@@ -448,3 +293,166 @@ function isActivityVideoUrlObject (url: ActivityUrlObject): url is ActivityVideo
 
   return mimeTypes.indexOf(url.mimeType) !== -1 && url.mimeType.startsWith('video/')
 }
+
+async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) {
+  logger.debug('Adding remote video %s.', videoObject.id)
+
+  const videoCreated: VideoModel = await sequelizeTypescript.transaction(async t => {
+    const sequelizeOptions = { transaction: t }
+
+    const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to)
+    const video = VideoModel.build(videoData)
+
+    const videoCreated = await video.save(sequelizeOptions)
+
+    // Process files
+    const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject)
+    if (videoFileAttributes.length === 0) {
+      throw new Error('Cannot find valid files for video %s ' + videoObject.url)
+    }
+
+    const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t }))
+    await Promise.all(videoFilePromises)
+
+    // Process tags
+    const tags = videoObject.tag.map(t => t.name)
+    const tagInstances = await TagModel.findOrCreateTags(tags, t)
+    await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
+
+    // Process captions
+    const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
+      return VideoCaptionModel.insertOrReplaceLanguage(videoCreated.id, c.identifier, t)
+    })
+    await Promise.all(videoCaptionsPromises)
+
+    logger.info('Remote video with uuid %s inserted.', videoObject.uuid)
+
+    videoCreated.VideoChannel = channelActor.VideoChannel
+    return videoCreated
+  })
+
+  const p = generateThumbnailFromUrl(videoCreated, videoObject.icon)
+    .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err }))
+
+  if (waitThumbnail === true) await p
+
+  return videoCreated
+}
+
+async function refreshVideoIfNeeded (videoArg: VideoModel, fetchedType: VideoFetchByUrlType, syncParam: SyncParam): Promise<VideoModel> {
+  // We need more attributes if the argument video was fetched with not enough joints
+  const video = fetchedType === 'all' ? videoArg : await VideoModel.loadByUrlAndPopulateAccount(videoArg.url)
+
+  if (!video.isOutdated()) return video
+
+  try {
+    const { response, videoObject } = await fetchRemoteVideo(video.url)
+    if (response.statusCode === 404) {
+      // Video does not exist anymore
+      await video.destroy()
+      return undefined
+    }
+
+    if (videoObject === undefined) {
+      logger.warn('Cannot refresh remote video: invalid body.')
+      return video
+    }
+
+    const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject)
+    const account = await AccountModel.load(channelActor.VideoChannel.accountId)
+
+    await updateVideoFromAP(video, videoObject, account, channelActor.VideoChannel)
+    await syncVideoExternalAttributes(video, videoObject, syncParam)
+  } catch (err) {
+    logger.warn('Cannot refresh video.', { err })
+    return video
+  }
+}
+
+async function videoActivityObjectToDBAttributes (
+  videoChannel: VideoChannelModel,
+  videoObject: VideoTorrentObject,
+  to: string[] = []
+) {
+  const privacy = to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 ? VideoPrivacy.PUBLIC : VideoPrivacy.UNLISTED
+  const duration = videoObject.duration.replace(/[^\d]+/, '')
+
+  let language: string | undefined
+  if (videoObject.language) {
+    language = videoObject.language.identifier
+  }
+
+  let category: number | undefined
+  if (videoObject.category) {
+    category = parseInt(videoObject.category.identifier, 10)
+  }
+
+  let licence: number | undefined
+  if (videoObject.licence) {
+    licence = parseInt(videoObject.licence.identifier, 10)
+  }
+
+  const description = videoObject.content || null
+  const support = videoObject.support || null
+
+  return {
+    name: videoObject.name,
+    uuid: videoObject.uuid,
+    url: videoObject.id,
+    category,
+    licence,
+    language,
+    description,
+    support,
+    nsfw: videoObject.sensitive,
+    commentsEnabled: videoObject.commentsEnabled,
+    waitTranscoding: videoObject.waitTranscoding,
+    state: videoObject.state,
+    channelId: videoChannel.id,
+    duration: parseInt(duration, 10),
+    createdAt: new Date(videoObject.published),
+    publishedAt: new Date(videoObject.published),
+    // FIXME: updatedAt does not seems to be considered by Sequelize
+    updatedAt: new Date(videoObject.updated),
+    views: videoObject.views,
+    likes: 0,
+    dislikes: 0,
+    remote: true,
+    privacy
+  }
+}
+
+function videoFileActivityUrlToDBAttributes (videoCreated: VideoModel, videoObject: VideoTorrentObject) {
+  const fileUrls = videoObject.url.filter(u => isActivityVideoUrlObject(u)) as ActivityVideoUrlObject[]
+
+  if (fileUrls.length === 0) {
+    throw new Error('Cannot find video files for ' + videoCreated.url)
+  }
+
+  const attributes: VideoFileModel[] = []
+  for (const fileUrl of fileUrls) {
+    // Fetch associated magnet uri
+    const magnet = videoObject.url.find(u => {
+      return u.mimeType === 'application/x-bittorrent;x-scheme-handler/magnet' && u.height === fileUrl.height
+    })
+
+    if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href)
+
+    const parsed = magnetUtil.decode(magnet.href)
+    if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) {
+      throw new Error('Cannot parse magnet URI ' + magnet.href)
+    }
+
+    const attribute = {
+      extname: VIDEO_MIMETYPE_EXT[ fileUrl.mimeType ],
+      infoHash: parsed.infoHash,
+      resolution: fileUrl.height,
+      size: fileUrl.size,
+      videoId: videoCreated.id,
+      fps: fileUrl.fps
+    } as VideoFileModel
+    attributes.push(attribute)
+  }
+
+  return attributes
+}

server/lib/job-queue/handlers/activitypub-http-fetcher.ts

@@ -1,10 +1,10 @@
 import * as Bull from 'bull'
 import { logger } from '../../../helpers/logger'
 import { processActivities } from '../../activitypub/process'
-import { VideoModel } from '../../../models/video/video'
-import { addVideoShares, createRates } from '../../activitypub/videos'
 import { addVideoComments } from '../../activitypub/video-comments'
 import { crawlCollectionPage } from '../../activitypub/crawl'
+import { VideoModel } from '../../../models/video/video'
+import { addVideoShares, createRates } from '../../activitypub'
 
 type FetchType = 'activity' | 'video-likes' | 'video-dislikes' | 'video-shares' | 'video-comments'
 

server/middlewares/validators/videos.ts

@@ -26,8 +26,7 @@ import {
   isVideoPrivacyValid,
   isVideoRatingTypeValid,
   isVideoSupportValid,
-  isVideoTagsValid,
+  isVideoTagsValid
-  VideoFetchType
 } from '../../helpers/custom-validators/videos'
 import { getDurationFromVideoFile } from '../../helpers/ffmpeg-utils'
 import { logger } from '../../helpers/logger'
@@ -42,6 +41,7 @@ import { checkUserCanTerminateOwnershipChange, doesChangeVideoOwnershipExist } f
 import { VideoChangeOwnershipAccept } from '../../../shared/models/videos/video-change-ownership-accept.model'
 import { VideoChangeOwnershipModel } from '../../models/video/video-change-ownership'
 import { AccountModel } from '../../models/account/account'
+import { VideoFetchType } from '../../helpers/video'
 
 const videosAddValidator = getCommonVideoAttributes().concat([
   body('videofile')

server/models/video/video.ts

@@ -1103,14 +1103,24 @@ export class VideoModel extends Model<VideoModel> {
       .findOne(options)
   }
 
-  static loadByUrlAndPopulateAccount (url: string, t?: Sequelize.Transaction) {
+  static loadByUrl (url: string, transaction?: Sequelize.Transaction) {
     const query: IFindOptions<VideoModel> = {
       where: {
         url
-      }
+      },
+      transaction
     }
 
-    if (t !== undefined) query.transaction = t
+    return VideoModel.findOne(query)
+  }
+
+  static loadByUrlAndPopulateAccount (url: string, transaction?: Sequelize.Transaction) {
+    const query: IFindOptions<VideoModel> = {
+      where: {
+        url
+      },
+      transaction
+    }
 
     return VideoModel.scope([ ScopeNames.WITH_ACCOUNT_DETAILS, ScopeNames.WITH_FILES ]).findOne(query)
   }