Add scores to follows and remove bad ones

client/src/app/+admin/follows/followers-list/followers-list.component.html

@@ -3,8 +3,8 @@
     sortField="createdAt" (onLazyLoad)="loadLazy($event)"
 >
   <p-column field="id" header="ID" [style]="{ width: '60px' }"></p-column>
+  <p-column field="score" header="Score"></p-column>
   <p-column field="follower.host" header="Host"></p-column>
-  <p-column field="follower.score" header="Score"></p-column>
   <p-column field="state" header="State"></p-column>
   <p-column field="createdAt" header="Created date" [sortable]="true"></p-column>
 </p-dataTable>

client/src/app/shared/misc/utils.ts

@@ -31,7 +31,7 @@ function populateAsyncUserVideoChannels (authService: AuthService, channel: any[
           const videoChannels = user.videoChannels
           if (Array.isArray(videoChannels) === false) return
 
-          videoChannels.forEach(c => channel.push({ id: c.id, label: c.name }))
+          videoChannels.forEach(c => channel.push({ id: c.id, label: c.displayName }))
 
           return res()
         }

client/src/app/videos/+video-edit/video-add.component.html

@@ -44,7 +44,6 @@
       [validationMessages]="validationMessages" [videoPrivacies]="videoPrivacies" [userVideoChannels]="userVideoChannels"
     ></my-video-edit>
 
-
     <div class="submit-container">
       <div *ngIf="videoUploaded === false" class="message-submit">Publish will be available when upload is finished</div>
 

client/src/app/videos/+video-watch/video-watch.component.html

@@ -85,7 +85,7 @@
       </div>
 
       <div class="video-info-channel">
-        {{ video.channel.name }}
+        {{ video.channel.displayName }}
         <!-- Here will be the subscribe button -->
       </div>
 

server.ts

@@ -56,6 +56,7 @@ import { installApplication } from './server/initializers'
 import { activitypubHttpJobScheduler, transcodingJobScheduler } from './server/lib/jobs'
 import { VideosPreviewCache } from './server/lib/cache'
 import { apiRouter, clientsRouter, staticRouter, servicesRouter, webfingerRouter, activityPubRouter } from './server/controllers'
+import { BadActorFollowScheduler } from './server/lib/schedulers/bad-actor-follow-scheduler'
 
 // ----------- Command line -----------
 
@@ -168,6 +169,8 @@ function onDatabaseInitDone () {
       // ----------- Make the server listening -----------
       server.listen(port, () => {
         VideosPreviewCache.Instance.init(CONFIG.CACHE.PREVIEWS.SIZE)
+        BadActorFollowScheduler.Instance.enable()
+
         activitypubHttpJobScheduler.activate()
         transcodingJobScheduler.activate()
 

server/initializers/constants.ts

@@ -9,7 +9,7 @@ import { isTestInstance, root, sanitizeHost, sanitizeUrl } from '../helpers/core
 
 // ---------------------------------------------------------------------------
 
-const LAST_MIGRATION_VERSION = 165
+const LAST_MIGRATION_VERSION = 170
 
 // ---------------------------------------------------------------------------
 
@@ -40,12 +40,12 @@ const OAUTH_LIFETIME = {
 
 // ---------------------------------------------------------------------------
 
-// Number of points we add/remove from a friend after a successful/bad request
+// Number of points we add/remove after a successful/bad request
-const SERVERS_SCORE = {
+const ACTOR_FOLLOW_SCORE = {
   PENALTY: -10,
   BONUS: 10,
-  BASE: 100,
+  BASE: 1000,
-  MAX: 1000
+  MAX: 10000
 }
 
 const FOLLOW_STATES: { [ id: string ]: FollowState } = {
@@ -76,6 +76,9 @@ const JOBS_FETCH_LIMIT_PER_CYCLE = {
 // 1 minutes
 let JOBS_FETCHING_INTERVAL = 60000
 
+// 1 hour
+let SCHEDULER_INTERVAL = 60000 * 60
+
 // ---------------------------------------------------------------------------
 
 const CONFIG = {
@@ -346,7 +349,7 @@ const OPENGRAPH_AND_OEMBED_COMMENT = '<!-- open graph and oembed tags -->'
 
 // Special constants for a test instance
 if (isTestInstance() === true) {
-  SERVERS_SCORE.BASE = 20
+  ACTOR_FOLLOW_SCORE.BASE = 20
   JOBS_FETCHING_INTERVAL = 1000
   REMOTE_SCHEME.HTTP = 'http'
   REMOTE_SCHEME.WS = 'ws'
@@ -354,6 +357,7 @@ if (isTestInstance() === true) {
   ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE = 2
   ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL = 60 // 1 minute
   CONSTRAINTS_FIELDS.ACTORS.AVATAR.FILE_SIZE.max = 100 * 1024 // 100KB
+  SCHEDULER_INTERVAL = 10000
 }
 
 CONFIG.WEBSERVER.URL = sanitizeUrl(CONFIG.WEBSERVER.SCHEME + '://' + CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT)
@@ -378,7 +382,7 @@ export {
   OAUTH_LIFETIME,
   OPENGRAPH_AND_OEMBED_COMMENT,
   PAGINATION_COUNT_DEFAULT,
-  SERVERS_SCORE,
+  ACTOR_FOLLOW_SCORE,
   PREVIEWS_SIZE,
   REMOTE_SCHEME,
   FOLLOW_STATES,
@@ -396,5 +400,6 @@ export {
   VIDEO_LICENCES,
   VIDEO_RATE_TYPES,
   VIDEO_MIMETYPE_EXT,
-  AVATAR_MIMETYPE_EXT
+  AVATAR_MIMETYPE_EXT,
+  SCHEDULER_INTERVAL
 }

server/initializers/migrations/0170-actor-follow-score.ts

@@ -0,0 +1,28 @@
+import * as Sequelize from 'sequelize'
+import { ACTOR_FOLLOW_SCORE } from '../index'
+
+async function up (utils: {
+  transaction: Sequelize.Transaction,
+  queryInterface: Sequelize.QueryInterface,
+  sequelize: Sequelize.Sequelize
+}): Promise<void> {
+  await utils.queryInterface.removeColumn('server', 'score')
+
+  const data = {
+    type: Sequelize.INTEGER,
+    allowNull: false,
+    defaultValue: ACTOR_FOLLOW_SCORE.BASE
+  }
+
+  await utils.queryInterface.addColumn('actorFollow', 'score', data)
+
+}
+
+function down (options) {
+  throw new Error('Not implemented.')
+}
+
+export {
+  up,
+  down
+}

server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-broadcast-handler.ts

@@ -1,5 +1,6 @@
 import { logger } from '../../../helpers/logger'
 import { doRequest } from '../../../helpers/requests'
+import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
 import { ActivityPubHttpPayload, buildSignedRequestOptions, computeBody, maybeRetryRequestLater } from './activitypub-http-job-scheduler'
 
 async function process (payload: ActivityPubHttpPayload, jobId: number) {
@@ -15,15 +16,22 @@ async function process (payload: ActivityPubHttpPayload, jobId: number) {
     httpSignature: httpSignatureOptions
   }
 
+  const badUrls: string[] = []
+  const goodUrls: string[] = []
+
   for (const uri of payload.uris) {
     options.uri = uri
 
     try {
       await doRequest(options)
+      goodUrls.push(uri)
     } catch (err) {
-      await maybeRetryRequestLater(err, payload, uri)
+      const isRetryingLater = await maybeRetryRequestLater(err, payload, uri)
+      if (isRetryingLater === false) badUrls.push(uri)
     }
   }
+
+  return ActorFollowModel.updateActorFollowsScoreAndRemoveBadOnes(goodUrls, badUrls, undefined)
 }
 
 function onError (err: Error, jobId: number) {

server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-job-scheduler.ts

@@ -4,6 +4,7 @@ import { logger } from '../../../helpers/logger'
 import { getServerActor } from '../../../helpers/utils'
 import { ACTIVITY_PUB } from '../../../initializers'
 import { ActorModel } from '../../../models/activitypub/actor'
+import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
 import { JobHandler, JobScheduler } from '../job-scheduler'
 
 import * as activitypubHttpBroadcastHandler from './activitypub-http-broadcast-handler'
@@ -26,7 +27,7 @@ const jobCategory: JobCategory = 'activitypub-http'
 
 const activitypubHttpJobScheduler = new JobScheduler(jobCategory, jobHandlers)
 
-function maybeRetryRequestLater (err: Error, payload: ActivityPubHttpPayload, uri: string) {
+async function maybeRetryRequestLater (err: Error, payload: ActivityPubHttpPayload, uri: string) {
   logger.warn('Cannot make request to %s.', uri, err)
 
   let attemptNumber = payload.attemptNumber || 1
@@ -39,8 +40,12 @@ function maybeRetryRequestLater (err: Error, payload: ActivityPubHttpPayload, ur
       uris: [ uri ],
       attemptNumber
     })
-    return activitypubHttpJobScheduler.createJob(undefined, 'activitypubHttpUnicastHandler', newPayload)
+    await activitypubHttpJobScheduler.createJob(undefined, 'activitypubHttpUnicastHandler', newPayload)
+
+    return true
   }
+
+  return false
 }
 
 async function computeBody (payload: ActivityPubHttpPayload) {

server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-unicast-handler.ts

@@ -1,5 +1,6 @@
 import { logger } from '../../../helpers/logger'
 import { doRequest } from '../../../helpers/requests'
+import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
 import { ActivityPubHttpPayload, buildSignedRequestOptions, computeBody, maybeRetryRequestLater } from './activitypub-http-job-scheduler'
 
 async function process (payload: ActivityPubHttpPayload, jobId: number) {
@@ -18,8 +19,13 @@ async function process (payload: ActivityPubHttpPayload, jobId: number) {
 
   try {
     await doRequest(options)
+    await ActorFollowModel.updateActorFollowsScoreAndRemoveBadOnes([ uri ], [], undefined)
   } catch (err) {
-    await maybeRetryRequestLater(err, payload, uri)
+    const isRetryingLater = await maybeRetryRequestLater(err, payload, uri)
+    if (isRetryingLater === false) {
+      await ActorFollowModel.updateActorFollowsScoreAndRemoveBadOnes([], [ uri ], undefined)
+    }
+
     throw err
   }
 }

server/lib/schedulers/abstract-scheduler.ts

@@ -0,0 +1,16 @@
+import { SCHEDULER_INTERVAL } from '../../initializers'
+
+export abstract class AbstractScheduler {
+
+  private interval: NodeJS.Timer
+
+  enable () {
+    this.interval = setInterval(() => this.execute(), SCHEDULER_INTERVAL)
+  }
+
+  disable () {
+    clearInterval(this.interval)
+  }
+
+  protected abstract execute ()
+}

server/lib/schedulers/bad-actor-follow-scheduler.ts

@@ -0,0 +1,24 @@
+import { logger } from '../../helpers/logger'
+import { ActorFollowModel } from '../../models/activitypub/actor-follow'
+import { AbstractScheduler } from './abstract-scheduler'
+
+export class BadActorFollowScheduler extends AbstractScheduler {
+
+  private static instance: AbstractScheduler
+
+  private constructor () {
+    super()
+  }
+
+  async execute () {
+    try {
+      await ActorFollowModel.removeBadActorFollows()
+    } catch (err) {
+      logger.error('Error in bad actor follows scheduler.', err)
+    }
+  }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+}

server/models/account/account.ts

@@ -179,7 +179,6 @@ export class AccountModel extends Model<AccountModel> {
     const actor = this.Actor.toFormattedJSON()
     const account = {
       id: this.id,
-      name: this.Actor.preferredUsername,
       displayName: this.name,
       createdAt: this.createdAt,
       updatedAt: this.updatedAt

server/models/activitypub/actor-follow.ts

@@ -1,8 +1,14 @@
 import * as Bluebird from 'bluebird'
 import { values } from 'lodash'
 import * as Sequelize from 'sequelize'
-import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript'
+import {
+  AllowNull, BelongsTo, Column, CreatedAt, DataType, Default, ForeignKey, IsInt, Max, Model, Table,
+  UpdatedAt
+} from 'sequelize-typescript'
 import { FollowState } from '../../../shared/models/actors'
+import { AccountFollow } from '../../../shared/models/actors/follow.model'
+import { logger } from '../../helpers/logger'
+import { ACTOR_FOLLOW_SCORE } from '../../initializers'
 import { FOLLOW_STATES } from '../../initializers/constants'
 import { ServerModel } from '../server/server'
 import { getSort } from '../utils'
@@ -20,6 +26,9 @@ import { ActorModel } from './actor'
     {
       fields: [ 'actorId', 'targetActorId' ],
       unique: true
+    },
+    {
+      fields: [ 'score' ]
     }
   ]
 })
@@ -29,6 +38,13 @@ export class ActorFollowModel extends Model<ActorFollowModel> {
   @Column(DataType.ENUM(values(FOLLOW_STATES)))
   state: FollowState
 
+  @AllowNull(false)
+  @Default(ACTOR_FOLLOW_SCORE.BASE)
+  @IsInt
+  @Max(ACTOR_FOLLOW_SCORE.MAX)
+  @Column
+  score: number
+
   @CreatedAt
   createdAt: Date
 
@@ -63,6 +79,34 @@ export class ActorFollowModel extends Model<ActorFollowModel> {
   })
   ActorFollowing: ActorModel
 
+  // Remove actor follows with a score of 0 (too many requests where they were unreachable)
+  static async removeBadActorFollows () {
+    const actorFollows = await ActorFollowModel.listBadActorFollows()
+
+    const actorFollowsRemovePromises = actorFollows.map(actorFollow => actorFollow.destroy())
+    await Promise.all(actorFollowsRemovePromises)
+
+    const numberOfActorFollowsRemoved = actorFollows.length
+
+    if (numberOfActorFollowsRemoved) logger.info('Removed bad %d actor follows.', numberOfActorFollowsRemoved)
+  }
+
+  static updateActorFollowsScoreAndRemoveBadOnes (goodInboxes: string[], badInboxes: string[], t: Sequelize.Transaction) {
+    if (goodInboxes.length === 0 && badInboxes.length === 0) return
+
+    logger.info('Updating %d good actor follows and %d bad actor follows scores.', goodInboxes.length, badInboxes.length)
+
+    if (goodInboxes.length !== 0) {
+      ActorFollowModel.incrementScores(goodInboxes, ACTOR_FOLLOW_SCORE.BONUS, t)
+        .catch(err => logger.error('Cannot increment scores of good actor follows.', err))
+    }
+
+    if (badInboxes.length !== 0) {
+      ActorFollowModel.incrementScores(badInboxes, ACTOR_FOLLOW_SCORE.PENALTY, t)
+        .catch(err => logger.error('Cannot decrement scores of bad actor follows.', err))
+    }
+  }
+
   static loadByActorAndTarget (actorId: number, targetActorId: number, t?: Sequelize.Transaction) {
     const query = {
       where: {
@@ -260,7 +304,37 @@ export class ActorFollowModel extends Model<ActorFollowModel> {
     }
   }
 
-  toFormattedJSON () {
+  private static incrementScores (inboxUrls: string[], value: number, t: Sequelize.Transaction) {
+    const inboxUrlsString = inboxUrls.map(url => `'${url}'`).join(',')
+
+    const query = 'UPDATE "actorFollow" SET "score" = "score" +' + value + ' ' +
+      'WHERE id IN (' +
+        'SELECT "actorFollow"."id" FROM "actorFollow" ' +
+        'INNER JOIN "actor" ON "actor"."id" = "actorFollow"."actorId" ' +
+        'WHERE "actor"."inboxUrl" IN (' + inboxUrlsString + ') OR "actor"."sharedInboxUrl" IN (' + inboxUrlsString + ')' +
+      ')'
+
+    const options = {
+      type: Sequelize.QueryTypes.BULKUPDATE,
+      transaction: t
+    }
+
+    return ActorFollowModel.sequelize.query(query, options)
+  }
+
+  private static listBadActorFollows () {
+    const query = {
+      where: {
+        score: {
+          [Sequelize.Op.lte]: 0
+        }
+      }
+    }
+
+    return ActorFollowModel.findAll(query)
+  }
+
+  toFormattedJSON (): AccountFollow {
     const follower = this.ActorFollower.toFormattedJSON()
     const following = this.ActorFollowing.toFormattedJSON()
 
@@ -268,6 +342,7 @@ export class ActorFollowModel extends Model<ActorFollowModel> {
       id: this.id,
       follower,
       following,
+      score: this.score,
       state: this.state,
       createdAt: this.createdAt,
       updatedAt: this.updatedAt

server/models/activitypub/actor.ts

@@ -204,7 +204,7 @@ export class ActorModel extends Model<ActorModel> {
   VideoChannel: VideoChannelModel
 
   static load (id: number) {
-    return ActorModel.scope(ScopeNames.FULL).findById(id)
+    return ActorModel.unscoped().findById(id)
   }
 
   static listByFollowersUrls (followersUrls: string[], transaction?: Sequelize.Transaction) {
@@ -267,20 +267,17 @@ export class ActorModel extends Model<ActorModel> {
       avatar = this.Avatar.toFormattedJSON()
     }
 
-    let score: number
-    if (this.Server) {
-      score = this.Server.score
-    }
-
     return {
       id: this.id,
       url: this.url,
       uuid: this.uuid,
+      name: this.preferredUsername,
       host: this.getHost(),
-      score,
       followingCount: this.followingCount,
       followersCount: this.followersCount,
-      avatar
+      avatar,
+      createdAt: this.createdAt,
+      updatedAt: this.updatedAt
     }
   }
 

server/models/server/server.ts

@@ -1,8 +1,5 @@
-import * as Sequelize from 'sequelize'
+import { AllowNull, Column, CreatedAt, Is, Model, Table, UpdatedAt } from 'sequelize-typescript'
-import { AllowNull, Column, CreatedAt, Default, Is, IsInt, Max, Model, Table, UpdatedAt } from 'sequelize-typescript'
 import { isHostValid } from '../../helpers/custom-validators/servers'
-import { logger } from '../../helpers/logger'
-import { SERVERS_SCORE } from '../../initializers'
 import { throwIfNotValid } from '../utils'
 
 @Table({
@@ -11,9 +8,6 @@ import { throwIfNotValid } from '../utils'
     {
       fields: [ 'host' ],
       unique: true
-    },
-    {
-      fields: [ 'score' ]
     }
   ]
 })
@@ -24,86 +18,9 @@ export class ServerModel extends Model<ServerModel> {
   @Column
   host: string
 
-  @AllowNull(false)
-  @Default(SERVERS_SCORE.BASE)
-  @IsInt
-  @Max(SERVERS_SCORE.MAX)
-  @Column
-  score: number
-
   @CreatedAt
   createdAt: Date
 
   @UpdatedAt
   updatedAt: Date
-
-  static updateServersScoreAndRemoveBadOnes (goodServers: number[], badServers: number[]) {
-    logger.info('Updating %d good servers and %d bad servers scores.', goodServers.length, badServers.length)
-
-    if (goodServers.length !== 0) {
-      ServerModel.incrementScores(goodServers, SERVERS_SCORE.BONUS)
-        .catch(err => {
-          logger.error('Cannot increment scores of good servers.', err)
-        })
-    }
-
-    if (badServers.length !== 0) {
-      ServerModel.incrementScores(badServers, SERVERS_SCORE.PENALTY)
-        .then(() => ServerModel.removeBadServers())
-        .catch(err => {
-          if (err) logger.error('Cannot decrement scores of bad servers.', err)
-        })
-
-    }
-  }
-
-  // Remove servers with a score of 0 (too many requests where they were unreachable)
-  private static async removeBadServers () {
-    try {
-      const servers = await ServerModel.listBadServers()
-
-      const serversRemovePromises = servers.map(server => server.destroy())
-      await Promise.all(serversRemovePromises)
-
-      const numberOfServersRemoved = servers.length
-
-      if (numberOfServersRemoved) {
-        logger.info('Removed %d servers.', numberOfServersRemoved)
-      } else {
-        logger.info('No need to remove bad servers.')
-      }
-    } catch (err) {
-      logger.error('Cannot remove bad servers.', err)
-    }
-  }
-
-  private static incrementScores (ids: number[], value: number) {
-    const update = {
-      score: Sequelize.literal('score +' + value)
-    }
-
-    const options = {
-      where: {
-        id: {
-          [Sequelize.Op.in]: ids
-        }
-      },
-      // In this case score is a literal and not an integer so we do not validate it
-      validate: false
-    }
-
-    return ServerModel.update(update, options)
-  }
-
-  private static listBadServers () {
-    const query = {
-      where: {
-        score: {
-          [Sequelize.Op.lte]: 0
-        }
-      }
-    }
-
-    return ServerModel.findAll(query)
-  }
 }

server/models/video/video-channel.ts

@@ -228,7 +228,7 @@ export class VideoChannelModel extends Model<VideoChannelModel> {
     const actor = this.Actor.toFormattedJSON()
     const account = {
       id: this.id,
-      name: this.name,
+      displayName: this.name,
       description: this.description,
       isLocal: this.Actor.isOwned(),
       createdAt: this.createdAt,

shared/models/actors/account.model.ts

@@ -1,15 +1,5 @@
-import { Avatar } from '../avatars/avatar.model'
+import { Actor } from './actor.model'
 
-export interface Account {
+export interface Account extends Actor {
-  id: number
-  uuid: string
-  url: string
-  name: string
   displayName: string
-  host: string
-  followingCount: number
-  followersCount: number
-  createdAt: Date
-  updatedAt: Date
-  avatar: Avatar
 }

shared/models/actors/actor.model.ts

@@ -0,0 +1,14 @@
+import { Avatar } from '../avatars/avatar.model'
+
+export interface Actor {
+  id: number
+  uuid: string
+  url: string
+  name: string
+  host: string
+  followingCount: number
+  followersCount: number
+  createdAt: Date
+  updatedAt: Date
+  avatar: Avatar
+}

shared/models/actors/follow.model.ts

@@ -1,11 +1,12 @@
-import { Account } from './account.model'
+import { Actor } from './actor.model'
 
 export type FollowState = 'pending' | 'accepted'
 
 export interface AccountFollow {
   id: number
-  follower: Account
+  follower: Actor
-  following: Account
+  following: Actor
+  score: number
   state: FollowState
   createdAt: Date
   updatedAt: Date

shared/models/videos/video-channel.model.ts

@@ -1,13 +1,10 @@
+import { Actor } from '../actors/actor.model'
 import { Video } from './video.model'
 
-export interface VideoChannel {
+export interface VideoChannel extends Actor {
-  id: number
+  displayName: string
-  name: string
-  url: string
   description: string
   isLocal: boolean
-  createdAt: Date | string
-  updatedAt: Date | string
   owner?: {
     name: string
     uuid: string