Federate video views

server/controllers/activitypub/outbox.ts

@@ -1,14 +1,14 @@
 import * as express from 'express'
 import { Activity, ActivityAdd } from '../../../shared/models/activitypub/activity'
-import { activityPubCollectionPagination, activityPubContextify } from '../../helpers/activitypub'
+import { activityPubCollectionPagination } from '../../helpers/activitypub'
+import { pageToStartAndCount } from '../../helpers/core-utils'
 import { database as db } from '../../initializers'
+import { ACTIVITY_PUB } from '../../initializers/constants'
 import { addActivityData } from '../../lib/activitypub/send/send-add'
 import { getAnnounceActivityPubUrl } from '../../lib/activitypub/url'
 import { announceActivityData } from '../../lib/index'
 import { asyncMiddleware, localAccountValidator } from '../../middlewares'
 import { AccountInstance } from '../../models/account/account-interface'
-import { pageToStartAndCount } from '../../helpers/core-utils'
-import { ACTIVITY_PUB } from '../../initializers/constants'
 
 const outboxRouter = express.Router()
 
@@ -36,14 +36,18 @@ async function outboxController (req: express.Request, res: express.Response, ne
 
   for (const video of data.data) {
     const videoObject = video.toActivityPubObject()
-    let addActivity: ActivityAdd = await addActivityData(video.url, account, video, video.VideoChannel.url, videoObject)
 
     // This is a shared video
-    if (video.VideoShare !== undefined) {
+    if (video.VideoShares !== undefined && video.VideoShares.length !== 0) {
+      const addActivity = await addActivityData(video.url, video.VideoChannel.Account, video, video.VideoChannel.url, videoObject)
+
       const url = getAnnounceActivityPubUrl(video.url, account)
       const announceActivity = await announceActivityData(url, account, addActivity)
+
       activities.push(announceActivity)
     } else {
+      const addActivity = await addActivityData(video.url, account, video, video.VideoChannel.url, videoObject)
+
       activities.push(addActivity)
     }
   }

server/controllers/api/server/follows.ts

@@ -148,10 +148,17 @@ async function removeFollow (req: express.Request, res: express.Response, next:
   const follow: AccountFollowInstance = res.locals.follow
 
   await db.sequelize.transaction(async t => {
-    await sendUndoFollow(follow, t)
+    if (follow.state === 'accepted') await sendUndoFollow(follow, t)
+
     await follow.destroy({ transaction: t })
   })
 
+  // Destroy the account that will destroy video channels, videos and video files too
+  // This could be long so don't wait this task
+  const following = follow.AccountFollowing
+  following.destroy()
+    .catch(err => logger.error('Cannot destroy account that we do not follow anymore %s.', following.url, err))
+
   return res.status(204).end()
 }
 

server/controllers/api/videos/index.ts

@@ -11,10 +11,15 @@ import {
   resetSequelizeInstance,
   retryTransactionWrapper
 } from '../../../helpers'
+import { getServerAccount } from '../../../helpers/utils'
 import { CONFIG, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES, VIDEO_MIMETYPE_EXT, VIDEO_PRIVACIES } from '../../../initializers'
 import { database as db } from '../../../initializers/database'
 import { sendAddVideo } from '../../../lib/activitypub/send/send-add'
 import { sendUpdateVideo } from '../../../lib/activitypub/send/send-update'
+import { shareVideoByServer } from '../../../lib/activitypub/share'
+import { getVideoActivityPubUrl } from '../../../lib/activitypub/url'
+import { fetchRemoteVideoDescription } from '../../../lib/activitypub/videos'
+import { sendCreateViewToVideoFollowers } from '../../../lib/index'
 import { transcodingJobScheduler } from '../../../lib/jobs/transcoding-job-scheduler/transcoding-job-scheduler'
 import {
   asyncMiddleware,
@@ -35,9 +40,7 @@ import { abuseVideoRouter } from './abuse'
 import { blacklistRouter } from './blacklist'
 import { videoChannelRouter } from './channel'
 import { rateVideoRouter } from './rate'
-import { getVideoActivityPubUrl } from '../../../lib/activitypub/url'
+import { sendCreateViewToOrigin } from '../../../lib/activitypub/send/send-create'
-import { shareVideoByServer } from '../../../lib/activitypub/share'
-import { fetchRemoteVideoDescription } from '../../../lib/activitypub/videos'
 
 const videosRouter = express.Router()
 
@@ -311,17 +314,18 @@ async function updateVideo (req: express.Request, res: express.Response) {
 async function getVideo (req: express.Request, res: express.Response) {
   const videoInstance = res.locals.video
 
+  const baseIncrementPromise = videoInstance.increment('views')
+    .then(() => getServerAccount())
+
   if (videoInstance.isOwned()) {
     // The increment is done directly in the database, not using the instance value
-    // FIXME: make a real view system
+    baseIncrementPromise
-    // For example, only add a view when a user watch a video during 30s etc
+      .then(serverAccount => sendCreateViewToVideoFollowers(serverAccount, videoInstance, undefined))
-    videoInstance.increment('views')
+      .catch(err => logger.error('Cannot add view to video/send view to followers for %s.', videoInstance.uuid, err))
-      .then(() => {
-        // TODO: send to followers a notification
-      })
-      .catch(err => logger.error('Cannot add view to video %s.', videoInstance.uuid, err))
   } else {
-    // TODO: send view event to followers
+    baseIncrementPromise
+      .then(serverAccount => sendCreateViewToOrigin(serverAccount, videoInstance, undefined))
+      .catch(err => logger.error('Cannot send view to origin server for %s.', videoInstance.uuid, err))
   }
 
   // Do not wait the view system

server/helpers/custom-validators/activitypub/activity.ts

@@ -11,6 +11,7 @@ import {
   isVideoTorrentDeleteActivityValid,
   isVideoTorrentUpdateActivityValid
 } from './videos'
+import { isViewActivityValid } from './view'
 
 function isRootActivityValid (activity: any) {
   return Array.isArray(activity['@context']) &&
@@ -55,7 +56,8 @@ export {
 
 function checkCreateActivity (activity: any) {
   return isVideoChannelCreateActivityValid(activity) ||
-    isVideoFlagValid(activity)
+    isVideoFlagValid(activity) ||
+    isViewActivityValid(activity)
 }
 
 function checkAddActivity (activity: any) {

server/helpers/custom-validators/activitypub/index.ts

@@ -5,3 +5,4 @@ export * from './signature'
 export * from './undo'
 export * from './video-channels'
 export * from './videos'
+export * from './view'

server/helpers/custom-validators/activitypub/videos.ts

@@ -52,7 +52,7 @@ function isVideoTorrentObjectValid (video: any) {
     setValidRemoteTags(video) &&
     isRemoteIdentifierValid(video.category) &&
     isRemoteIdentifierValid(video.licence) &&
-    isRemoteIdentifierValid(video.language) &&
+    (!video.language || isRemoteIdentifierValid(video.language)) &&
     isVideoViewsValid(video.views) &&
     isVideoNSFWValid(video.nsfw) &&
     isDateValid(video.published) &&

server/helpers/custom-validators/activitypub/view.ts

@@ -0,0 +1,13 @@
+import { isActivityPubUrlValid, isBaseActivityValid } from './misc'
+
+function isViewActivityValid (activity: any) {
+  return isBaseActivityValid(activity, 'Create') &&
+    activity.object.type === 'View' &&
+    isActivityPubUrlValid(activity.object.actor) &&
+    isActivityPubUrlValid(activity.object.object)
+}
+// ---------------------------------------------------------------------------
+
+export {
+  isViewActivityValid
+}

server/lib/activitypub/process/misc.ts

@@ -33,13 +33,18 @@ async function videoActivityObjectToDBAttributes (
   else if (cc.indexOf(ACTIVITY_PUB.PUBLIC) !== -1) privacy = VideoPrivacy.UNLISTED
 
   const duration = videoObject.duration.replace(/[^\d]+/, '')
+  let language = null
+  if (videoObject.language) {
+    language = parseInt(videoObject.language.identifier, 10)
+  }
+
   const videoData: VideoAttributes = {
     name: videoObject.name,
     uuid: videoObject.uuid,
     url: videoObject.id,
     category: parseInt(videoObject.category.identifier, 10),
     licence: parseInt(videoObject.licence.identifier, 10),
-    language: parseInt(videoObject.language.identifier, 10),
+    language,
     nsfw: videoObject.nsfw,
     description: videoObject.content,
     channelId: videoChannel.id,

server/lib/activitypub/process/process-create.ts

@@ -1,9 +1,11 @@
 import { ActivityCreate, VideoChannelObject } from '../../../../shared'
 import { VideoAbuseObject } from '../../../../shared/models/activitypub/objects/video-abuse-object'
+import { ViewObject } from '../../../../shared/models/activitypub/objects/view-object'
 import { logger, retryTransactionWrapper } from '../../../helpers'
 import { database as db } from '../../../initializers'
 import { AccountInstance } from '../../../models/account/account-interface'
 import { getOrCreateAccountAndServer } from '../account'
+import { sendCreateViewToVideoFollowers } from '../send/send-create'
 import { getVideoChannelActivityPubUrl } from '../url'
 import { videoChannelActivityObjectToDBAttributes } from './misc'
 
@@ -12,7 +14,9 @@ async function processCreateActivity (activity: ActivityCreate) {
   const activityType = activityObject.type
   const account = await getOrCreateAccountAndServer(activity.actor)
 
-  if (activityType === 'VideoChannel') {
+  if (activityType === 'View') {
+    return processCreateView(activityObject as ViewObject)
+  } else if (activityType === 'VideoChannel') {
     return processCreateVideoChannel(account, activityObject as VideoChannelObject)
   } else if (activityType === 'Flag') {
     return processCreateVideoAbuse(account, activityObject as VideoAbuseObject)
@@ -30,6 +34,19 @@ export {
 
 // ---------------------------------------------------------------------------
 
+async function processCreateView (view: ViewObject) {
+  const video = await db.Video.loadByUrlAndPopulateAccount(view.object)
+
+  if (!video) throw new Error('Unknown video ' + view.object)
+
+  const account = await db.Account.loadByUrl(view.actor)
+  if (!account) throw new Error('Unknown account ' + view.actor)
+
+  await video.increment('views')
+
+  if (video.isOwned()) await sendCreateViewToVideoFollowers(account, video, undefined)
+}
+
 function processCreateVideoChannel (account: AccountInstance, videoChannelToCreateData: VideoChannelObject) {
   const options = {
     arguments: [ account, videoChannelToCreateData ],

server/lib/activitypub/process/process-follow.ts

@@ -49,6 +49,12 @@ async function follow (account: AccountInstance, targetAccountURL: string) {
       },
       transaction: t
     })
+
+    if (accountFollow.state !== 'accepted') {
+      accountFollow.state = 'accepted'
+      await accountFollow.save({ transaction: t })
+    }
+
     accountFollow.AccountFollower = account
     accountFollow.AccountFollowing = targetAccount
 

server/lib/activitypub/send/misc.ts

@@ -4,16 +4,26 @@ import { ACTIVITY_PUB, database as db } from '../../../initializers'
 import { AccountInstance } from '../../../models/account/account-interface'
 import { activitypubHttpJobScheduler } from '../../jobs/activitypub-http-job-scheduler/activitypub-http-job-scheduler'
 
-async function broadcastToFollowers (data: any, byAccount: AccountInstance, toAccountFollowers: AccountInstance[], t: Transaction) {
+async function broadcastToFollowers (
+  data: any,
+  byAccount: AccountInstance,
+  toAccountFollowers: AccountInstance[],
+  t: Transaction,
+  followersException: AccountInstance[] = []
+) {
   const toAccountFollowerIds = toAccountFollowers.map(a => a.id)
+
   const result = await db.AccountFollow.listAcceptedFollowerSharedInboxUrls(toAccountFollowerIds)
   if (result.data.length === 0) {
     logger.info('Not broadcast because of 0 followers for %s.', toAccountFollowerIds.join(', '))
     return undefined
   }
 
+  const followersSharedInboxException = followersException.map(f => f.sharedInboxUrl)
+  const uris = result.data.filter(sharedInbox => followersSharedInboxException.indexOf(sharedInbox) === -1)
+
   const jobPayload = {
-    uris: result.data,
+    uris,
     signatureAccountId: byAccount.id,
     body: data
   }

server/lib/activitypub/send/send-create.ts

@@ -3,7 +3,9 @@ import { ActivityCreate } from '../../../../shared/models/activitypub/activity'
 import { AccountInstance, VideoChannelInstance, VideoInstance } from '../../../models'
 import { VideoAbuseInstance } from '../../../models/video/video-abuse-interface'
 import { broadcastToFollowers, getAudience, unicastTo } from './misc'
-import { getVideoAbuseActivityPubUrl } from '../url'
+import { getVideoAbuseActivityPubUrl, getVideoViewActivityPubUrl } from '../url'
+import { getServerAccount } from '../../../helpers/utils'
+import { database as db } from '../../../initializers'
 
 async function sendCreateVideoChannel (videoChannel: VideoChannelInstance, t: Transaction) {
   const byAccount = videoChannel.Account
@@ -16,21 +18,53 @@ async function sendCreateVideoChannel (videoChannel: VideoChannelInstance, t: Tr
 
 async function sendVideoAbuse (byAccount: AccountInstance, videoAbuse: VideoAbuseInstance, video: VideoInstance, t: Transaction) {
   const url = getVideoAbuseActivityPubUrl(videoAbuse)
-  const data = await createActivityData(url, byAccount, videoAbuse.toActivityPubObject())
+
+  const audience = { to: [ video.VideoChannel.Account.url ], cc: [] }
+  const data = await createActivityData(url, byAccount, videoAbuse.toActivityPubObject(), audience)
 
   return unicastTo(data, byAccount, video.VideoChannel.Account.sharedInboxUrl, t)
 }
 
-// async function sendCreateView ()
+async function sendCreateViewToOrigin (byAccount: AccountInstance, video: VideoInstance, t: Transaction) {
+  const url = getVideoViewActivityPubUrl(byAccount, video)
+  const viewActivity = createViewActivityData(byAccount, video)
+
+  const audience = { to: [ video.VideoChannel.Account.url ], cc: [ video.VideoChannel.Account.url + '/followers' ] }
+  const data = await createActivityData(url, byAccount, viewActivity, audience)
+
+  return unicastTo(data, byAccount, video.VideoChannel.Account.sharedInboxUrl, t)
+}
+
+async function sendCreateViewToVideoFollowers (byAccount: AccountInstance, video: VideoInstance, t: Transaction) {
+  const url = getVideoViewActivityPubUrl(byAccount, video)
+  const viewActivity = createViewActivityData(byAccount, video)
+
+  const audience = { to: [ video.VideoChannel.Account.url + '/followers' ], cc: [] }
+  const data = await createActivityData(url, byAccount, viewActivity, audience)
+
+  const serverAccount = await getServerAccount()
+  const accountsToForwardView = await db.VideoShare.loadAccountsByShare(video.id)
+  accountsToForwardView.push(video.VideoChannel.Account)
+
+  // Don't forward view to server that sent it to us
+  const index = accountsToForwardView.findIndex(a => a.id === byAccount.id)
+  if (index) accountsToForwardView.splice(index, 1)
+
+  const followersException = [ byAccount ]
+  return broadcastToFollowers(data, serverAccount, accountsToForwardView, t, followersException)
+}
+
+async function createActivityData (url: string, byAccount: AccountInstance, object: any, audience?: { to: string[], cc: string[] }) {
+  if (!audience) {
+    audience = await getAudience(byAccount)
+  }
 
-async function createActivityData (url: string, byAccount: AccountInstance, object: any) {
-  const { to, cc } = await getAudience(byAccount)
   const activity: ActivityCreate = {
     type: 'Create',
     id: url,
     actor: byAccount.url,
-    to,
+    to: audience.to,
-    cc,
+    cc: audience.cc,
     object
   }
 
@@ -42,5 +76,19 @@ async function createActivityData (url: string, byAccount: AccountInstance, obje
 export {
   sendCreateVideoChannel,
   sendVideoAbuse,
-  createActivityData
+  createActivityData,
+  sendCreateViewToOrigin,
+  sendCreateViewToVideoFollowers
+}
+
+// ---------------------------------------------------------------------------
+
+function createViewActivityData (byAccount: AccountInstance, video: VideoInstance) {
+  const obj = {
+    type: 'View',
+    actor: byAccount.url,
+    object: video.url
+  }
+
+  return obj
 }

server/lib/activitypub/url.ts

@@ -21,6 +21,10 @@ function getVideoAbuseActivityPubUrl (videoAbuse: VideoAbuseInstance) {
   return CONFIG.WEBSERVER.URL + '/admin/video-abuses/' + videoAbuse.id
 }
 
+function getVideoViewActivityPubUrl (byAccount: AccountInstance, video: VideoInstance) {
+  return video.url + '#views/' + byAccount.uuid + '/' + new Date().toISOString()
+}
+
 function getAccountFollowActivityPubUrl (accountFollow: AccountFollowInstance) {
   const me = accountFollow.AccountFollower
   const following = accountFollow.AccountFollowing
@@ -56,5 +60,6 @@ export {
   getAccountFollowAcceptActivityPubUrl,
   getAnnounceActivityPubUrl,
   getUpdateActivityPubUrl,
-  getUndoActivityPubUrl
+  getUndoActivityPubUrl,
+  getVideoViewActivityPubUrl
 }

server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-fetcher-handler.ts

@@ -1,10 +1,8 @@
 import { logger } from '../../../helpers'
-import { buildSignedActivity } from '../../../helpers/activitypub'
 import { doRequest } from '../../../helpers/requests'
-import { database as db } from '../../../initializers'
-import { ActivityPubHttpPayload } from './activitypub-http-job-scheduler'
-import { processActivities } from '../../activitypub/process/process'
 import { ACTIVITY_PUB } from '../../../initializers/constants'
+import { processActivities } from '../../activitypub/process/process'
+import { ActivityPubHttpPayload } from './activitypub-http-job-scheduler'
 
 async function process (payload: ActivityPubHttpPayload, jobId: number) {
   logger.info('Processing ActivityPub fetcher in job %d.', jobId)

server/models/video/video-interface.ts

@@ -122,7 +122,7 @@ export interface VideoAttributes {
   VideoChannel?: VideoChannelInstance
   Tags?: TagInstance[]
   VideoFiles?: VideoFileInstance[]
-  VideoShare?: VideoShareInstance
+  VideoShares?: VideoShareInstance[]
 }
 
 export interface VideoInstance extends VideoClass, VideoAttributes, Sequelize.Instance<VideoAttributes> {

server/models/video/video.ts

@@ -567,6 +567,14 @@ toActivityPubObject = function (this: VideoInstance) {
     name: t.name
   }))
 
+  let language
+  if (this.language) {
+    language = {
+      identifier: this.language + '',
+      name: this.getLanguageLabel()
+    }
+  }
+
   const url = []
   for (const file of this.VideoFiles) {
     url.push({
@@ -608,10 +616,7 @@ toActivityPubObject = function (this: VideoInstance) {
       identifier: this.licence + '',
       name: this.getLicenceLabel()
     },
-    language: {
+    language,
-      identifier: this.language + '',
-      name: this.getLanguageLabel()
-    },
     views: this.views,
     nsfw: this.nsfw,
     published: this.createdAt.toISOString(),
@@ -816,7 +821,19 @@ listAllAndSharedByAccountForOutbox = function (accountId: number, start: number,
     include: [
       {
         model: Video['sequelize'].models.VideoShare,
-        required: false
+        required: false,
+        where: {
+          [Sequelize.Op.and]: [
+            {
+              id: {
+                [Sequelize.Op.not]: null
+              }
+            },
+            {
+              accountId
+            }
+          ]
+        }
       },
       {
         model: Video['sequelize'].models.VideoChannel,

shared/models/activitypub/activity.ts

@@ -1,6 +1,7 @@
-import { VideoChannelObject, VideoTorrentObject } from './objects'
 import { ActivityPubSignature } from './activitypub-signature'
+import { VideoChannelObject, VideoTorrentObject } from './objects'
 import { VideoAbuseObject } from './objects/video-abuse-object'
+import { ViewObject } from './objects/view-object'
 
 export type Activity = ActivityCreate | ActivityAdd | ActivityUpdate |
   ActivityDelete | ActivityFollow | ActivityAccept | ActivityAnnounce |
@@ -20,7 +21,7 @@ export interface BaseActivity {
 
 export interface ActivityCreate extends BaseActivity {
   type: 'Create'
-  object: VideoChannelObject | VideoAbuseObject
+  object: VideoChannelObject | VideoAbuseObject | ViewObject
 }
 
 export interface ActivityAdd extends BaseActivity {

shared/models/activitypub/objects/index.ts

@@ -2,3 +2,4 @@ export * from './common-objects'
 export * from './video-abuse-object'
 export * from './video-channel-object'
 export * from './video-torrent-object'
+export * from './view-object'

shared/models/activitypub/objects/view-object.ts

@@ -0,0 +1,5 @@
+export interface ViewObject {
+  type: 'View',
+  actor: string
+  object: string
+}