Merge branch 'develop' into cli-wrapper

README.md

@@ -24,8 +24,8 @@ directly in the web browser with <a href="https://github.com/feross/webtorrent">
     <img src="https://david-dm.org/Chocobozzz/PeerTube/dev-status.svg?path=client" alt="devDependency Status" />
   </a>
   
-  <a href="https://www.browserstack.com/automate/public-build/VXBPc0szNjUvRUNsREJQRFF6RkEvSjJBclZ4VUJBUm1hcS9RZGpUbitRST0tLWFWbjNEdVN6eEZpYTk4dGVpMkVlQWc9PQ==--644e755052bf7fe2346eb6e868be8e706718a17c%">
+  <a href="https://www.browserstack.com/automate/public-build/cWJhRDFJbS9qeUhzYW04MnlIVjlQQ0x3aE5POXBaV1lycGo5VlQxK3JqZz0tLTNUWW5ySEVvS1N4UnBhYlhsdXVCeVE9PQ==--db09e291d36a582af8b2929d62a625ed660cdf1d">
-    <img src='https://www.browserstack.com/automate/badge.svg?badge_key=VXBPc0szNjUvRUNsREJQRFF6RkEvSjJBclZ4VUJBUm1hcS9RZGpUbitRST0tLWFWbjNEdVN6eEZpYTk4dGVpMkVlQWc9PQ==--644e755052bf7fe2346eb6e868be8e706718a17c%'/>
+    <img src='https://www.browserstack.com/automate/badge.svg?badge_key=cWJhRDFJbS9qeUhzYW04MnlIVjlQQ0x3aE5POXBaV1lycGo5VlQxK3JqZz0tLTNUWW5ySEVvS1N4UnBhYlhsdXVCeVE9PQ==--db09e291d36a582af8b2929d62a625ed660cdf1d'/>
   </a>
 </p>
 
@@ -97,11 +97,10 @@ BitTorrent) inside the web browser, as of today.
 ## Dependencies
 
   * nginx
-  * PostgreSQL
+  * **PostgreSQL >= 9.6**
   * **Redis >= 2.8.18**
   * **NodeJS >= 8.x**
   * yarn
-  * OpenSSL (cli)
   * **FFmpeg >= 3.x**
 
 ## Run in production

SECURITY.md

@@ -30,7 +30,7 @@ To encourage vulnerability research and to avoid any confusion between good-fait
 - Avoid violating the privacy of others, disrupting our systems, destroying data, and/or harming user experience.
 - Use only the Official Channels to discuss vulnerability information with us.
 - Keep the details of any discovered vulnerabilities confidential until they are fixed, according to the Disclosure Terms in this policy.
-- Perform testing only on in-scope systems, and respect systems and activities which are out-of-scope.
+- Perform testing only on in-scope systems, and respect systems and activities which are out-of-scope. Systems currently considered in-scope are the official demonstration/test servers provided by the PeerTube development team.
 - If a vulnerability provides unintended access to data: Limit the amount of data you access to the minimum required for effectively demonstrating a Proof of Concept; and cease testing and submit a report immediately if you encounter any user data during testing, such as Personally Identifiable Information (PII), Personal Healthcare Information (PHI), credit card data, or proprietary information.
 - You should only interact with test accounts you own or with explicit permission from the account holder.
 - Do not engage in extortion.

client/angular.json

@@ -24,7 +24,7 @@
             },
             "assets": [
               "src/assets/images",
-              "src/manifest.json"
+              "src/manifest.webmanifest"
             ],
             "styles": [
               "src/sass/application.scss"
@@ -105,7 +105,7 @@
             ],
             "assets": [
               "src/assets/images",
-              "src/manifest.json"
+              "src/manifest.webmanifest"
             ]
           }
         },

client/e2e/src/po/video-watch.po.ts

@@ -26,8 +26,11 @@ export class VideoWatchPage {
                   .then((texts: any) => texts.map(t => t.trim()))
   }
 
-  waitWatchVideoName (videoName: string, isSafari: boolean) {
+  waitWatchVideoName (videoName: string, isMobileDevice: boolean, isSafari: boolean) {
-    const elem = element(by.css('.video-info .video-info-name'))
+    // On mobile we display the first node, on desktop the second
+    const index = isMobileDevice ? 0 : 1
+
+    const elem = element.all(by.css('.video-info .video-info-name')).get(index)
 
     if (isSafari) return browser.sleep(5000)
 

client/e2e/src/videos.e2e-spec.ts

@@ -12,7 +12,7 @@ describe('Videos workflow', () => {
   let isSafari = false
 
   beforeEach(async () => {
-    browser.waitForAngularEnabled(false)
+    await browser.waitForAngularEnabled(false)
 
     videoWatchPage = new VideoWatchPage()
     pageUploadPage = new VideoUploadPage()
@@ -62,7 +62,7 @@ describe('Videos workflow', () => {
     if (isMobileDevice || isSafari) videoNameToExcept = await videoWatchPage.clickOnFirstVideo()
     else await videoWatchPage.clickOnVideo(videoName)
 
-    return videoWatchPage.waitWatchVideoName(videoNameToExcept, isSafari)
+    return videoWatchPage.waitWatchVideoName(videoNameToExcept, isMobileDevice, isSafari)
   })
 
   it('Should play the video', async () => {

client/src/app/+admin/moderation/moderation.routes.ts

@@ -15,6 +15,16 @@ export const ModerationRoutes: Routes = [
         redirectTo: 'video-abuses/list',
         pathMatch: 'full'
       },
+      {
+        path: 'video-abuses',
+        redirectTo: 'video-abuses/list',
+        pathMatch: 'full'
+      },
+      {
+        path: 'video-blacklist',
+        redirectTo: 'video-blacklist/list',
+        pathMatch: 'full'
+      },
       {
         path: 'video-abuses/list',
         component: VideoAbuseListComponent,

client/src/app/+admin/users/user-list/user-list.component.ts

@@ -105,7 +105,8 @@ export class UserListComponent extends RestTable implements OnInit {
       return
     }
 
-    const res = await this.confirmService.confirm(this.i18n('Do you really want to delete this user?'), this.i18n('Delete'))
+    const message = this.i18n('If you remove this user, you will not be able to create another with the same username!')
+    const res = await this.confirmService.confirm(message, this.i18n('Delete'))
     if (res === false) return
 
     this.userService.removeUser(user).subscribe(

client/src/app/menu/menu.component.ts

@@ -19,8 +19,10 @@ export class MenuComponent implements OnInit {
   private routesPerRight = {
     [UserRight.MANAGE_USERS]: '/admin/users',
     [UserRight.MANAGE_SERVER_FOLLOW]: '/admin/friends',
-    [UserRight.MANAGE_VIDEO_ABUSES]: '/admin/video-abuses',
+    [UserRight.MANAGE_VIDEO_ABUSES]: '/admin/moderation/video-abuses',
-    [UserRight.MANAGE_VIDEO_BLACKLIST]: '/admin/video-blacklist'
+    [UserRight.MANAGE_VIDEO_BLACKLIST]: '/admin/moderation/video-blacklist',
+    [UserRight.MANAGE_JOBS]: '/admin/jobs',
+    [UserRight.MANAGE_CONFIGURATION]: '/admin/config'
   }
 
   constructor (
@@ -67,7 +69,9 @@ export class MenuComponent implements OnInit {
       UserRight.MANAGE_USERS,
       UserRight.MANAGE_SERVER_FOLLOW,
       UserRight.MANAGE_VIDEO_ABUSES,
-      UserRight.MANAGE_VIDEO_BLACKLIST
+      UserRight.MANAGE_VIDEO_BLACKLIST,
+      UserRight.MANAGE_JOBS,
+      UserRight.MANAGE_CONFIGURATION
     ]
 
     for (const adminRight of adminRights) {

client/src/app/shared/overview/overview.service.ts

@@ -56,6 +56,8 @@ export class OverviewService {
       }
     }
 
+    if (observables.length === 0) return of(videosOverviewResult)
+
     return forkJoin(observables)
       .pipe(
         // Translate categories

client/src/app/shared/video/abstract-video-list.html

@@ -7,12 +7,12 @@
   <div class="no-results" i18n *ngIf="pagination.totalItems === 0">No results.</div>
   <div
     myInfiniteScroller
-    [pageHeight]="pageHeight"
+    [pageHeight]="pageHeight" [firstLoadedPage]="firstLoadedPage"
     (nearOfTop)="onNearOfTop()" (nearOfBottom)="onNearOfBottom()" (pageChanged)="onPageChanged($event)"
     class="videos" #videosElement
   >
-    <div *ngFor="let videos of videoPages" class="videos-page">
+    <div *ngFor="let videos of videoPages; trackBy: pageByVideoId" class="videos-page">
-      <my-video-miniature *ngFor="let video of videos" [video]="video" [user]="user" [ownerDisplayType]="ownerDisplayType"></my-video-miniature>
+      <my-video-miniature *ngFor="let video of videos; trackBy: videoById" [video]="video" [user]="user" [ownerDisplayType]="ownerDisplayType"></my-video-miniature>
     </div>
   </div>
 </div>

client/src/app/shared/video/abstract-video-list.ts

@@ -36,9 +36,10 @@ export abstract class AbstractVideoList implements OnInit, OnDestroy {
   videoHeight: number
   videoPages: Video[][] = []
   ownerDisplayType: OwnerDisplayType = 'account'
+  firstLoadedPage: number
 
   protected baseVideoWidth = 215
-  protected baseVideoHeight = 230
+  protected baseVideoHeight = 205
 
   protected abstract notificationsService: NotificationsService
   protected abstract authService: AuthService
@@ -80,6 +81,15 @@ export abstract class AbstractVideoList implements OnInit, OnDestroy {
     if (this.resizeSubscription) this.resizeSubscription.unsubscribe()
   }
 
+  pageByVideoId (index: number, page: Video[]) {
+    // Video are unique in all pages
+    return page[0].id
+  }
+
+  videoById (index: number, video: Video) {
+    return video.id
+  }
+
   onNearOfTop () {
     this.previousPage()
   }
@@ -100,7 +110,11 @@ export abstract class AbstractVideoList implements OnInit, OnDestroy {
     this.loadMoreVideos(this.pagination.currentPage)
   }
 
-  loadMoreVideos (page: number) {
+  loadMoreVideos (page: number, loadOnTop = false) {
+    this.adjustVideoPageHeight()
+
+    const currentY = window.scrollY
+
     if (this.loadedPages[page] !== undefined) return
     if (this.loadingPage[page] === true) return
 
@@ -111,6 +125,8 @@ export abstract class AbstractVideoList implements OnInit, OnDestroy {
       ({ videos, totalVideos }) => {
         this.loadingPage[page] = false
 
+        if (this.firstLoadedPage === undefined || this.firstLoadedPage > page) this.firstLoadedPage = page
+
         // Paging is too high, return to the first one
         if (this.pagination.currentPage > 1 && totalVideos <= ((this.pagination.currentPage - 1) * this.pagination.itemsPerPage)) {
           this.pagination.currentPage = 1
@@ -125,8 +141,17 @@ export abstract class AbstractVideoList implements OnInit, OnDestroy {
         // Initialize infinite scroller now we loaded the first page
         if (Object.keys(this.loadedPages).length === 1) {
           // Wait elements creation
-          setTimeout(() => this.infiniteScroller.initialize(), 500)
+          setTimeout(() => {
+            this.infiniteScroller.initialize()
+
+            // At our first load, we did not load the first page
+            // Load the previous page so the user can move on the top (and browser previous pages)
+            if (this.pagination.currentPage > 1) this.loadMoreVideos(this.pagination.currentPage - 1, true)
+          }, 500)
         }
+
+        // Insert elements on the top but keep the scroll in the previous position
+        if (loadOnTop) setTimeout(() => { window.scrollTo(0, currentY + this.pageHeight) }, 0)
       },
       error => {
         this.loadingPage[page] = false
@@ -150,7 +175,7 @@ export abstract class AbstractVideoList implements OnInit, OnDestroy {
     const min = this.minPageLoaded()
 
     if (min > 1) {
-      this.loadMoreVideos(min - 1)
+      this.loadMoreVideos(min - 1, true)
     }
   }
 
@@ -189,6 +214,13 @@ export abstract class AbstractVideoList implements OnInit, OnDestroy {
     this.videoPages = Object.values(this.loadedPages)
   }
 
+  protected adjustVideoPageHeight () {
+    const numberOfPagesLoaded = Object.keys(this.loadedPages).length
+    if (!numberOfPagesLoaded) return
+
+    this.pageHeight = this.videosElement.nativeElement.offsetHeight / numberOfPagesLoaded
+  }
+
   protected buildVideoHeight () {
     // Same ratios than base width/height
     return this.videosElement.nativeElement.offsetWidth * (this.baseVideoHeight / this.baseVideoWidth)

client/src/app/shared/video/infinite-scroller.directive.ts

@@ -6,10 +6,9 @@ import { fromEvent, Subscription } from 'rxjs'
   selector: '[myInfiniteScroller]'
 })
 export class InfiniteScrollerDirective implements OnInit, OnDestroy {
-  private static PAGE_VIEW_TOP_MARGIN = 500
-
   @Input() containerHeight: number
   @Input() pageHeight: number
+  @Input() firstLoadedPage = 1
   @Input() percentLimit = 70
   @Input() autoInit = false
 
@@ -23,6 +22,7 @@ export class InfiniteScrollerDirective implements OnInit, OnDestroy {
   private scrollDownSub: Subscription
   private scrollUpSub: Subscription
   private pageChangeSub: Subscription
+  private middleScreen: number
 
   constructor () {
     this.decimalLimit = this.percentLimit / 100
@@ -39,6 +39,8 @@ export class InfiniteScrollerDirective implements OnInit, OnDestroy {
   }
 
   initialize () {
+    this.middleScreen = window.innerHeight / 2
+
     // Emit the last value
     const throttleOptions = { leading: true, trailing: true }
 
@@ -92,6 +94,11 @@ export class InfiniteScrollerDirective implements OnInit, OnDestroy {
   }
 
   private calculateCurrentPage (current: number) {
-    return Math.max(1, Math.round((current + InfiniteScrollerDirective.PAGE_VIEW_TOP_MARGIN) / this.pageHeight))
+    const scrollY = current + this.middleScreen
+
+    const page = Math.max(1, Math.ceil(scrollY / this.pageHeight))
+
+    // Offset page
+    return page + (this.firstLoadedPage - 1)
   }
 }

client/src/app/shared/video/video-miniature.component.html

@@ -1,11 +1,11 @@
 <div class="video-miniature">
-  <my-video-thumbnail [video]="video" [nsfw]="isVideoBlur()"></my-video-thumbnail>
+  <my-video-thumbnail [video]="video" [nsfw]="isVideoBlur"></my-video-thumbnail>
 
   <div class="video-miniature-information">
     <a
       tabindex="-1"
       class="video-miniature-name"
-      [routerLink]="[ '/videos/watch', video.uuid ]" [attr.title]="video.name" [ngClass]="{ 'blur-filter': isVideoBlur() }"
+      [routerLink]="[ '/videos/watch', video.uuid ]" [attr.title]="video.name" [ngClass]="{ 'blur-filter': isVideoBlur }"
     >
       {{ video.name }}
     </a>

client/src/app/shared/video/video-miniature.component.ts

@@ -1,4 +1,4 @@
-import { Component, Input, OnInit } from '@angular/core'
+import { ChangeDetectionStrategy, Component, Input, OnInit } from '@angular/core'
 import { User } from '../users'
 import { Video } from './video.model'
 import { ServerService } from '@app/core'
@@ -8,13 +8,16 @@ export type OwnerDisplayType = 'account' | 'videoChannel' | 'auto'
 @Component({
   selector: 'my-video-miniature',
   styleUrls: [ './video-miniature.component.scss' ],
-  templateUrl: './video-miniature.component.html'
+  templateUrl: './video-miniature.component.html',
+  changeDetection: ChangeDetectionStrategy.OnPush
 })
 export class VideoMiniatureComponent implements OnInit {
   @Input() user: User
   @Input() video: Video
   @Input() ownerDisplayType: OwnerDisplayType = 'account'
 
+  isVideoBlur: boolean
+
   private ownerDisplayTypeChosen: 'account' | 'videoChannel'
 
   constructor (private serverService: ServerService) { }
@@ -35,10 +38,8 @@ export class VideoMiniatureComponent implements OnInit {
     } else {
       this.ownerDisplayTypeChosen = 'videoChannel'
     }
-  }
 
-  isVideoBlur () {
+    this.isVideoBlur = this.video.isVideoNSFWForUser(this.user, this.serverService.getConfig())
-    return this.video.isVideoNSFWForUser(this.user, this.serverService.getConfig())
   }
 
   displayOwnerAccount () {

client/src/app/videos/+video-edit/video-add-components/video-upload.component.html

@@ -22,7 +22,7 @@
       <div class="peertube-select-container">
         <select id="first-step-privacy" [(ngModel)]="firstStepPrivacyId">
           <option *ngFor="let privacy of videoPrivacies" [value]="privacy.id">{{ privacy.label }}</option>
-          <option [value]="SPECIAL_SCHEDULED_PRIVACY">Scheduled</option>
+          <option i18n [value]="SPECIAL_SCHEDULED_PRIVACY">Scheduled</option>
         </select>
       </div>
     </div>

client/src/app/videos/+video-watch/comment/video-comment-add.component.scss

@@ -39,3 +39,9 @@ form {
     @include orange-button
   }
 }
+
+@media screen and (max-width: 450px) {
+  textarea, .submit-comment button {
+    font-size: 14px !important;
+  }
+}

client/src/app/videos/+video-watch/comment/video-comment.component.scss

@@ -35,6 +35,7 @@
       .comment-account {
         @include disable-default-a-behaviour;
 
+        word-break: break-all;
         color: var(--mainForegroundColor);
         font-weight: $font-bold;
       }
@@ -102,3 +103,9 @@
     img { margin-right: 10px; }
   }
 }
+
+@media screen and (max-width: 450px) {
+  .root-comment {
+    font-size: 14px;
+  }
+}

client/src/app/videos/+video-watch/comment/video-comments.component.scss

@@ -31,4 +31,10 @@ my-help {
   .view-replies {
     margin-left: 46px;
   }
-}
+}
+
+@media screen and (max-width: 450px) {
+  .view-replies {
+    font-size: 14px;
+  }
+}

client/src/app/videos/+video-watch/video-watch.component.scss

@@ -81,6 +81,7 @@
     flex-grow: 1;
     // Set min width for flex item
     min-width: 1px;
+    max-width: 100%;
 
     .video-info-first-row {
       display: flex;
@@ -472,6 +473,7 @@ my-video-comments {
     margin: 20px 0 0 0;
 
     .video-info {
+      padding: 0;
 
       .video-info-first-row {
 
@@ -484,6 +486,8 @@ my-video-comments {
   }
 
   /deep/ .other-videos {
+    padding-left: 0 !important;
+
     /deep/ .video-miniature  {
       flex-direction: column;
     }
@@ -499,7 +503,27 @@ my-video-comments {
 }
 
 @media screen and (max-width: 450px) {
-  .video-bottom  .action-button .icon-text {
+  .video-bottom {
-    display: none !important;
+    .action-button .icon-text {
+      display: none !important;
+    }
+
+    .video-info .video-info-first-row {
+      .video-info-name {
+        font-size: 18px;
+      }
+
+      .video-info-date-views {
+        font-size: 14px;
+      }
+
+      .video-actions-rates {
+        margin-top: 10px;
+      }
+    }
+
+    .video-info-description {
+      font-size: 14px !important;
+    }
   }
 }

client/src/app/videos/+video-watch/video-watch.component.ts

@@ -1,4 +1,4 @@
-import { catchError, subscribeOn } from 'rxjs/operators'
+import { catchError } from 'rxjs/operators'
 import { ChangeDetectorRef, Component, ElementRef, Inject, LOCALE_ID, NgZone, OnDestroy, OnInit, ViewChild } from '@angular/core'
 import { ActivatedRoute, Router } from '@angular/router'
 import { RedirectService } from '@app/core/routing/redirect.service'

client/src/app/videos/recommendations/recent-videos-recommendation.service.ts

@@ -25,8 +25,8 @@ export class RecentVideosRecommendationService implements RecommendationService
   getRecommendations (recommendation: RecommendationInfo): Observable<Video[]> {
     return this.fetchPage(1, recommendation)
       .pipe(
-        map(vids => {
+        map(videos => {
-          const otherVideos = vids.filter(v => v.uuid !== recommendation.uuid)
+          const otherVideos = videos.filter(v => v.uuid !== recommendation.uuid)
           return otherVideos.slice(0, this.pageSize)
         })
       )

client/src/app/videos/recommendations/recommended-videos.store.ts

@@ -3,8 +3,8 @@ import { Observable, ReplaySubject } from 'rxjs'
 import { Video } from '@app/shared/video/video.model'
 import { RecommendationInfo } from '@app/shared/video/recommendation-info.model'
 import { RecentVideosRecommendationService } from '@app/videos/recommendations/recent-videos-recommendation.service'
-import { RecommendationService, UUID } from '@app/videos/recommendations/recommendations.service'
+import { RecommendationService } from '@app/videos/recommendations/recommendations.service'
-import { map, switchMap, take } from 'rxjs/operators'
+import { map, shareReplay, switchMap, take } from 'rxjs/operators'
 
 /**
  * This store is intended to provide data for the RecommendedVideosComponent.
@@ -19,9 +19,13 @@ export class RecommendedVideosStore {
     @Inject(RecentVideosRecommendationService) private recommendations: RecommendationService
   ) {
     this.recommendations$ = this.requestsForLoad$$.pipe(
-      switchMap(requestedRecommendation => recommendations.getRecommendations(requestedRecommendation)
+      switchMap(requestedRecommendation => {
-        .pipe(take(1))
+        return recommendations.getRecommendations(requestedRecommendation)
-      ))
+                              .pipe(take(1))
+      }),
+      shareReplay()
+    )
+
     this.hasRecommendations$ = this.recommendations$.pipe(
       map(otherVideos => otherVideos.length > 0)
     )

client/src/app/videos/video-list/video-overview.component.html

@@ -12,7 +12,7 @@
 
   <div class="section" *ngFor="let object of overview.tags">
     <div class="section-title" i18n>
-      <a routerLink="/search" [queryParams]="{ tagOneOf: [ object.tag ] }">{{ object.tag }}</a>
+      <a routerLink="/search" [queryParams]="{ tagsOneOf: [ object.tag ] }">{{ object.tag }}</a>
     </div>
 
     <my-video-miniature *ngFor="let video of object.videos" [video]="video" [user]="user"></my-video-miniature>

client/src/assets/player/peertube-videojs-plugin.ts

@@ -4,7 +4,7 @@ import { VideoFile } from '../../../../shared/models/videos/video.model'
 import { renderVideo } from './video-renderer'
 import './settings-menu-button'
 import { PeertubePluginOptions, VideoJSCaption, VideoJSComponentInterface, videojsUntyped } from './peertube-videojs-typings'
-import { isMobile, videoFileMaxByResolution, videoFileMinByResolution, timeToInt } from './utils'
+import { isMobile, timeToInt, videoFileMaxByResolution, videoFileMinByResolution } from './utils'
 import * as CacheChunkStore from 'cache-chunk-store'
 import { PeertubeChunkStore } from './peertube-chunk-store'
 import {
@@ -83,11 +83,6 @@ class PeerTubePlugin extends Plugin {
     this.videoCaptions = options.videoCaptions
 
     this.savePlayerSrcFunction = this.player.src
-    // Hack to "simulate" src link in video.js >= 6
-    // Without this, we can't play the video after pausing it
-    // https://github.com/videojs/video.js/blob/master/src/js/player.js#L1633
-    this.player.src = () => true
-
     this.playerElement = options.playerElement
 
     if (this.autoplay === true) this.player.addClass('vjs-has-autoplay')
@@ -104,9 +99,7 @@ class PeerTubePlugin extends Plugin {
 
       this.player.one('play', () => {
         // Don't run immediately scheduler, wait some seconds the TCP connections are made
-        this.runAutoQualitySchedulerTimer = setTimeout(() => {
+        this.runAutoQualitySchedulerTimer = setTimeout(() => this.runAutoQualityScheduler(), this.CONSTANTS.AUTO_QUALITY_SCHEDULER)
-          this.runAutoQualityScheduler()
-        }, this.CONSTANTS.AUTO_QUALITY_SCHEDULER)
       })
     })
 
@@ -167,6 +160,9 @@ class PeerTubePlugin extends Plugin {
     // Do not display error to user because we will have multiple fallback
     this.disableErrorDisplay()
 
+    // Hack to "simulate" src link in video.js >= 6
+    // Without this, we can't play the video after pausing it
+    // https://github.com/videojs/video.js/blob/master/src/js/player.js#L1633
     this.player.src = () => true
     const oldPlaybackRate = this.player.playbackRate()
 
@@ -181,102 +177,6 @@ class PeerTubePlugin extends Plugin {
     this.trigger('videoFileUpdate')
   }
 
-  addTorrent (
-    magnetOrTorrentUrl: string,
-    previousVideoFile: VideoFile,
-    options: {
-      forcePlay?: boolean,
-      seek?: number,
-      delay?: number
-    },
-    done: Function
-  ) {
-    console.log('Adding ' + magnetOrTorrentUrl + '.')
-
-    const oldTorrent = this.torrent
-    const torrentOptions = {
-      store: (chunkLength, storeOpts) => new CacheChunkStore(new PeertubeChunkStore(chunkLength, storeOpts), {
-        max: 100
-      })
-    }
-
-    this.torrent = this.webtorrent.add(magnetOrTorrentUrl, torrentOptions, torrent => {
-      console.log('Added ' + magnetOrTorrentUrl + '.')
-
-      if (oldTorrent) {
-        // Pause the old torrent
-        oldTorrent.pause()
-        // Pause does not remove actual peers (in particular the webseed peer)
-        oldTorrent.removePeer(oldTorrent['ws'])
-
-        // We use a fake renderer so we download correct pieces of the next file
-        if (options.delay) {
-          const fakeVideoElem = document.createElement('video')
-          renderVideo(torrent.files[0], fakeVideoElem, { autoplay: false, controls: false }, (err, renderer) => {
-            this.fakeRenderer = renderer
-
-            if (err) console.error('Cannot render new torrent in fake video element.', err)
-
-            // Load the future file at the correct time
-            fakeVideoElem.currentTime = this.player.currentTime() + (options.delay / 2000)
-          })
-        }
-      }
-
-      // Render the video in a few seconds? (on resolution change for example, we wait some seconds of the new video resolution)
-      this.addTorrentDelay = setTimeout(() => {
-        this.destroyFakeRenderer()
-
-        const paused = this.player.paused()
-
-        this.flushVideoFile(previousVideoFile)
-
-        const renderVideoOptions = { autoplay: false, controls: true }
-        renderVideo(torrent.files[0], this.playerElement, renderVideoOptions,(err, renderer) => {
-          this.renderer = renderer
-
-          if (err) return this.fallbackToHttp(done)
-
-          return this.tryToPlay(err => {
-            if (err) return done(err)
-
-            if (options.seek) this.seek(options.seek)
-            if (options.forcePlay === false && paused === true) this.player.pause()
-
-            return done(err)
-          })
-        })
-      }, options.delay || 0)
-    })
-
-    this.torrent.on('error', err => console.error(err))
-
-    this.torrent.on('warning', (err: any) => {
-      // We don't support HTTP tracker but we don't care -> we use the web socket tracker
-      if (err.message.indexOf('Unsupported tracker protocol') !== -1) return
-
-      // Users don't care about issues with WebRTC, but developers do so log it in the console
-      if (err.message.indexOf('Ice connection failed') !== -1) {
-        console.log(err)
-        return
-      }
-
-      // Magnet hash is not up to date with the torrent file, add directly the torrent file
-      if (err.message.indexOf('incorrect info hash') !== -1) {
-        console.error('Incorrect info hash detected, falling back to torrent file.')
-        const newOptions = { forcePlay: true, seek: options.seek }
-        return this.addTorrent(this.torrent['xs'], previousVideoFile, newOptions, done)
-      }
-
-      // Remote instance is down
-      if (err.message.indexOf('from xs param') !== -1) {
-        this.handleError(err)
-      }
-
-      console.warn(err)
-    })
-  }
-
   updateResolution (resolutionId: number, delay = 0) {
     // Remember player state
     const currentTime = this.player.currentTime()
@@ -336,6 +236,91 @@ class PeerTubePlugin extends Plugin {
     return this.torrent
   }
 
+  private addTorrent (
+    magnetOrTorrentUrl: string,
+    previousVideoFile: VideoFile,
+    options: {
+      forcePlay?: boolean,
+      seek?: number,
+      delay?: number
+    },
+    done: Function
+  ) {
+    console.log('Adding ' + magnetOrTorrentUrl + '.')
+
+    const oldTorrent = this.torrent
+    const torrentOptions = {
+      store: (chunkLength, storeOpts) => new CacheChunkStore(new PeertubeChunkStore(chunkLength, storeOpts), {
+        max: 100
+      })
+    }
+
+    this.torrent = this.webtorrent.add(magnetOrTorrentUrl, torrentOptions, torrent => {
+      console.log('Added ' + magnetOrTorrentUrl + '.')
+
+      if (oldTorrent) {
+        // Pause the old torrent
+        this.stopTorrent(oldTorrent)
+
+        // We use a fake renderer so we download correct pieces of the next file
+        if (options.delay) this.renderFileInFakeElement(torrent.files[ 0 ], options.delay)
+      }
+
+      // Render the video in a few seconds? (on resolution change for example, we wait some seconds of the new video resolution)
+      this.addTorrentDelay = setTimeout(() => {
+        // We don't need the fake renderer anymore
+        this.destroyFakeRenderer()
+
+        const paused = this.player.paused()
+
+        this.flushVideoFile(previousVideoFile)
+
+        const renderVideoOptions = { autoplay: false, controls: true }
+        renderVideo(torrent.files[ 0 ], this.playerElement, renderVideoOptions, (err, renderer) => {
+          this.renderer = renderer
+
+          if (err) return this.fallbackToHttp(done)
+
+          return this.tryToPlay(err => {
+            if (err) return done(err)
+
+            if (options.seek) this.seek(options.seek)
+            if (options.forcePlay === false && paused === true) this.player.pause()
+
+            return done(err)
+          })
+        })
+      }, options.delay || 0)
+    })
+
+    this.torrent.on('error', err => console.error(err))
+
+    this.torrent.on('warning', (err: any) => {
+      // We don't support HTTP tracker but we don't care -> we use the web socket tracker
+      if (err.message.indexOf('Unsupported tracker protocol') !== -1) return
+
+      // Users don't care about issues with WebRTC, but developers do so log it in the console
+      if (err.message.indexOf('Ice connection failed') !== -1) {
+        console.log(err)
+        return
+      }
+
+      // Magnet hash is not up to date with the torrent file, add directly the torrent file
+      if (err.message.indexOf('incorrect info hash') !== -1) {
+        console.error('Incorrect info hash detected, falling back to torrent file.')
+        const newOptions = { forcePlay: true, seek: options.seek }
+        return this.addTorrent(this.torrent[ 'xs' ], previousVideoFile, newOptions, done)
+      }
+
+      // Remote instance is down
+      if (err.message.indexOf('from xs param') !== -1) {
+        this.handleError(err)
+      }
+
+      console.warn(err)
+    })
+  }
+
   private tryToPlay (done?: Function) {
     if (!done) done = function () { /* empty */ }
 
@@ -435,22 +420,22 @@ class PeerTubePlugin extends Plugin {
     if (this.autoplay === true) {
       this.player.posterImage.hide()
 
+      return this.updateVideoFile(undefined, { forcePlay: true, seek: this.startTime })
+    }
+
+    // Don't try on iOS that does not support MediaSource
+    if (this.isIOS()) {
+      this.currentVideoFile = this.pickAverageVideoFile()
+      return this.fallbackToHttp(undefined, false)
+    }
+
+    // Proxy first play
+    const oldPlay = this.player.play.bind(this.player)
+    this.player.play = () => {
+      this.player.addClass('vjs-has-big-play-button-clicked')
+      this.player.play = oldPlay
+
       this.updateVideoFile(undefined, { forcePlay: true, seek: this.startTime })
-    } else {
-      // Don't try on iOS that does not support MediaSource
-      if (this.isIOS()) {
-        this.currentVideoFile = this.pickAverageVideoFile()
-        return this.fallbackToHttp(undefined, false)
-      }
-
-      // Proxy first play
-      const oldPlay = this.player.play.bind(this.player)
-      this.player.play = () => {
-        this.player.addClass('vjs-has-big-play-button-clicked')
-        this.player.play = oldPlay
-
-        this.updateVideoFile(undefined, { forcePlay: true, seek: this.startTime })
-      }
     }
   }
 
@@ -607,6 +592,24 @@ class PeerTubePlugin extends Plugin {
     return this.videoFiles[Math.floor(this.videoFiles.length / 2)]
   }
 
+  private stopTorrent (torrent: WebTorrent.Torrent) {
+    torrent.pause()
+    // Pause does not remove actual peers (in particular the webseed peer)
+    torrent.removePeer(torrent[ 'ws' ])
+  }
+
+  private renderFileInFakeElement (file: WebTorrent.TorrentFile, delay: number) {
+    const fakeVideoElem = document.createElement('video')
+    renderVideo(file, fakeVideoElem, { autoplay: false, controls: false }, (err, renderer) => {
+      this.fakeRenderer = renderer
+
+      if (err) console.error('Cannot render new torrent in fake video element.', err)
+
+      // Load the future file at the correct time (in delay MS - 2 seconds)
+      fakeVideoElem.currentTime = this.player.currentTime() + (delay - 2000)
+    })
+  }
+
   private destroyFakeRenderer () {
     if (this.fakeRenderer) {
       if (this.fakeRenderer.destroy) {

client/src/assets/player/settings-menu-item.ts

@@ -38,8 +38,11 @@ class SettingsMenuItem extends MenuItem {
     this.eventHandlers()
 
     player.ready(() => {
-      this.build()
+      // Voodoo magic for IOS
-      this.reset()
+      setTimeout(() => {
+        this.build()
+        this.reset()
+      }, 0)
     })
   }
 

client/src/hmr.ts

@@ -1,11 +1,19 @@
 import { NgModuleRef, ApplicationRef } from '@angular/core'
 import { createNewHosts } from '@angularclass/hmr'
+import { enableDebugTools } from '@angular/platform-browser'
 
 export const hmrBootstrap = (module: any, bootstrap: () => Promise<NgModuleRef<any>>) => {
   let ngModule: NgModuleRef<any>
   module.hot.accept()
   bootstrap()
-    .then(mod => ngModule = mod)
+    .then(mod => {
+      ngModule = mod
+
+      const applicationRef = ngModule.injector.get(ApplicationRef);
+      const componentRef = applicationRef.components[ 0 ]
+      // allows to run `ng.profiler.timeChangeDetection();`
+      enableDebugTools(componentRef)
+    })
   module.hot.dispose(() => {
     const appRef: ApplicationRef = ngModule.injector.get(ApplicationRef)
     const elements = appRef.components.map(c => c.location.nativeElement)

client/src/index.html

@@ -7,7 +7,7 @@
     <meta name="theme-color" content="#fff" />
 
     <!-- Web Manifest file -->
-    <link rel="manifest" href="/manifest.json">
+    <link rel="manifest" href="/manifest.webmanifest">
 
     <!-- /!\ The following comment is used by the server to prerender some tags /!\ -->
 

client/src/manifest.webmanifest

@@ -24,7 +24,7 @@
       "src": "/client/assets/images/icons/icon-96x96.png",
       "sizes": "96x96",
       "type": "image/png"
-    },	
+    },
 	{
       "src": "/client/assets/images/icons/icon-144x144.png",
       "sizes": "144x144",

client/src/sass/application.scss

@@ -9,7 +9,7 @@ $icon-font-path: '../../node_modules/@neos21/bootstrap3-glyphicons/assets/fonts/
 @import '~video.js/dist/video-js.css';
 
 $assets-path: '../assets/';
-@import './player/player';
+@import './player/index';
 @import './loading-bar';
 
 @import './primeng-custom';

client/src/sass/include/_mixins.scss

@@ -53,7 +53,6 @@
   -ms-hyphens: auto;
   -moz-hyphens: auto;
   hyphens: auto;
-  text-align: justify;
 }
 
 @mixin peertube-input-text($width) {

client/src/sass/player/peertube-skin.scss

@@ -406,6 +406,7 @@
 
       width: 37px;
       margin-right: 1px;
+      cursor: pointer;
 
       .vjs-icon-placeholder {
         transition: transform 0.2s ease;
@@ -504,10 +505,6 @@
       }
     }
 
-    .vjs-playback-rate {
-      display: none;
-    }
-
     .vjs-peertube {
       padding: 0 !important;
 

client/src/standalone/videos/embed.scss

@@ -4,7 +4,7 @@
 @import '~videojs-dock/dist/videojs-dock.css';
 
 $assets-path: '../../assets/';
-@import '../../sass/player/player';
+@import '../../sass/player/index';
 
 [hidden] {
   display: none !important;

config/default.yaml

@@ -71,9 +71,18 @@ trending:
 # Once you have defined your strategies, choose which instances you want to cache in admin -> manage follows -> following
 redundancy:
   videos:
-#    -
+    check_interval: '1 hour' # How often you want to check new videos to cache
-#      size: '10GB'
+    strategies:
-#      strategy: 'most-views' # Cache videos that have the most views
+#      -
+#        size: '10GB'
+#        strategy: 'most-views' # Cache videos that have the most views
+#      -
+#        size: '10GB'
+#        strategy: 'trending' # Cache trending videos
+#      -
+#        size: '10GB'
+#        strategy: 'recently-added' # Cache recently added videos
+#        minViews: 10 # Having at least x views
 
 cache:
   previews:
@@ -135,7 +144,7 @@ instance:
   # Robot.txt rules. To disallow robots to crawl your instance and disallow indexation of your site, add '/' to "Disallow:'
   robots: |
     User-agent: *
-    Disallow: ''
+    Disallow:
   # Security.txt rules. To discourage researchers from testing your instance and disable security.txt integration, set this to an empty string.
   securitytxt:
     "# If you would like to report a security issue\n# you may report it to:\nContact: https://github.com/Chocobozzz/PeerTube/blob/develop/SECURITY.md\nContact: mailto:"

config/production.yaml.example

@@ -72,9 +72,18 @@ trending:
 # Once you have defined your strategies, choose which instances you want to cache in admin -> manage follows -> following
 redundancy:
   videos:
-#    -
+    check_interval: '1 hour' # How often you want to check new videos to cache
-#      size: '10GB'
+    strategies:
-#      strategy: 'most-views' # Cache videos that have the most views
+#      -
+#        size: '10GB'
+#        strategy: 'most-views' # Cache videos that have the most views
+#      -
+#        size: '10GB'
+#        strategy: 'trending' # Cache trending videos
+#      -
+#        size: '10GB'
+#        strategy: 'recently-added' # Cache recently added videos
+#        minViews: 10 # Having at least x views
 
 ###############################################################################
 #
@@ -149,7 +158,7 @@ instance:
   # Robot.txt rules. To disallow robots to crawl your instance and disallow indexation of your site, add '/' to "Disallow:'
   robots: |
     User-agent: *
-    Disallow: ''
+    Disallow:
   # Security.txt rules. To discourage researchers from testing your instance and disable security.txt integration, set this to an empty string.
   securitytxt:
     "# If you would like to report a security issue\n# you may report it to:\nContact: https://github.com/Chocobozzz/PeerTube/blob/develop/SECURITY.md\nContact: mailto:"

config/test.yaml

@@ -23,9 +23,18 @@ log:
 
 redundancy:
   videos:
-    -
+    check_interval: '5 seconds'
-      size: '100KB'
+    strategies:
-      strategy: 'most-views'
+      -
+        size: '10MB'
+        strategy: 'most-views'
+      -
+        size: '10MB'
+        strategy: 'trending'
+      -
+        size: '10MB'
+        strategy: 'recently-added'
+        minViews: 1
 
 cache:
   previews:

package.json

@@ -73,7 +73,7 @@
   },
   "lint-staged": {
     "*.scss": [
-      "sass-lint -c .sass-lint.yml",
+      "sass-lint -c client/.sass-lint.yml",
       "git add"
     ]
   },
@@ -116,6 +116,7 @@
     "jsonld-signatures": "https://github.com/Chocobozzz/jsonld-signatures#rsa2017",
     "lodash": "^4.17.10",
     "magnet-uri": "^5.1.4",
+    "memoizee": "^0.4.14",
     "morgan": "^1.5.3",
     "multer": "^1.1.0",
     "netrc-parser": "^3.1.6",
@@ -165,6 +166,7 @@
     "@types/lodash": "^4.14.64",
     "@types/magnet-uri": "^5.1.1",
     "@types/maildev": "^0.0.1",
+    "@types/memoizee": "^0.4.2",
     "@types/mkdirp": "^0.5.1",
     "@types/mocha": "^5.0.0",
     "@types/morgan": "^1.7.32",

scripts/clean/server/test.sh

@@ -2,15 +2,28 @@
 
 set -eu
 
-for i in $(seq 1 6); do
+recreateDB () {
-  dbname="peertube_test$i"
+  dbname="peertube_test$1"
 
   dropdb --if-exists "$dbname"
-  rm -rf "./test$i"
+
-  rm -f "./config/local-test.json"
-  rm -f "./config/local-test-$i.json"
   createdb -O peertube "$dbname"
-  psql -c "CREATE EXTENSION pg_trgm;" "$dbname"
+  psql -c "CREATE EXTENSION pg_trgm;" "$dbname" &
-  psql -c "CREATE EXTENSION unaccent;" "$dbname"
+  psql -c "CREATE EXTENSION unaccent;" "$dbname" &
-  redis-cli KEYS "bull-localhost:900$i*" | grep -v empty | xargs --no-run-if-empty redis-cli DEL
+}
+
+removeFiles () {
+  rm -rf "./test$1" "./config/local-test.json" "./config/local-test-$1.json"
+}
+
+dropRedis () {
+  redis-cli KEYS "bull-localhost:900$1*" | grep -v empty | xargs --no-run-if-empty redis-cli DEL
+}
+
+for i in $(seq 1 6); do
+  recreateDB "$i" &
+  dropRedis "$i" &
+  removeFiles "$i" &
 done
+
+wait

scripts/create-import-video-file-job.ts

@@ -25,7 +25,7 @@ run()
 async function run () {
   await initDatabaseModels(true)
 
-  const video = await VideoModel.loadByUUID(program['video'])
+  const video = await VideoModel.loadByUUIDWithFile(program['video'])
   if (!video) throw new Error('Video not found.')
   if (video.isOwned() === false) throw new Error('Cannot import files of a non owned video.')
 

scripts/create-transcoding-job.ts

@@ -28,7 +28,7 @@ run()
 async function run () {
   await initDatabaseModels(true)
 
-  const video = await VideoModel.loadByUUID(program['video'])
+  const video = await VideoModel.loadByUUIDWithFile(program['video'])
   if (!video) throw new Error('Video not found.')
 
   const dataInput = {

scripts/prune-storage.ts

@@ -56,7 +56,7 @@ async function pruneDirectory (directory: string) {
     const uuid = getUUIDFromFilename(file)
     let video: VideoModel
 
-    if (uuid) video = await VideoModel.loadByUUID(uuid)
+    if (uuid) video = await VideoModel.loadByUUIDWithFile(uuid)
 
     if (!uuid || !video) toDelete.push(join(directory, file))
   }

server/controllers/activitypub/client.ts

@@ -6,7 +6,13 @@ import { CONFIG, ROUTE_CACHE_LIFETIME } from '../../initializers'
 import { buildAnnounceWithVideoAudience } from '../../lib/activitypub/send'
 import { audiencify, getAudience } from '../../lib/activitypub/audience'
 import { buildCreateActivity } from '../../lib/activitypub/send/send-create'
-import { asyncMiddleware, executeIfActivityPub, localAccountValidator, localVideoChannelValidator } from '../../middlewares'
+import {
+  asyncMiddleware,
+  executeIfActivityPub,
+  localAccountValidator,
+  localVideoChannelValidator,
+  videosCustomGetValidator
+} from '../../middlewares'
 import { videosGetValidator, videosShareValidator } from '../../middlewares/validators'
 import { videoCommentGetValidator } from '../../middlewares/validators/video-comments'
 import { AccountModel } from '../../models/account/account'
@@ -54,7 +60,7 @@ activityPubClientRouter.get('/videos/watch/:id/activity',
   executeIfActivityPub(asyncMiddleware(videoController))
 )
 activityPubClientRouter.get('/videos/watch/:id/announces',
-  executeIfActivityPub(asyncMiddleware(videosGetValidator)),
+  executeIfActivityPub(asyncMiddleware(videosCustomGetValidator('only-video'))),
   executeIfActivityPub(asyncMiddleware(videoAnnouncesController))
 )
 activityPubClientRouter.get('/videos/watch/:id/announces/:accountId',
@@ -62,15 +68,15 @@ activityPubClientRouter.get('/videos/watch/:id/announces/:accountId',
   executeIfActivityPub(asyncMiddleware(videoAnnounceController))
 )
 activityPubClientRouter.get('/videos/watch/:id/likes',
-  executeIfActivityPub(asyncMiddleware(videosGetValidator)),
+  executeIfActivityPub(asyncMiddleware(videosCustomGetValidator('only-video'))),
   executeIfActivityPub(asyncMiddleware(videoLikesController))
 )
 activityPubClientRouter.get('/videos/watch/:id/dislikes',
-  executeIfActivityPub(asyncMiddleware(videosGetValidator)),
+  executeIfActivityPub(asyncMiddleware(videosCustomGetValidator('only-video'))),
   executeIfActivityPub(asyncMiddleware(videoDislikesController))
 )
 activityPubClientRouter.get('/videos/watch/:id/comments',
-  executeIfActivityPub(asyncMiddleware(videosGetValidator)),
+  executeIfActivityPub(asyncMiddleware(videosCustomGetValidator('only-video'))),
   executeIfActivityPub(asyncMiddleware(videoCommentsController))
 )
 activityPubClientRouter.get('/videos/watch/:videoId/comments/:commentId',

server/controllers/activitypub/inbox.ts

@@ -7,6 +7,8 @@ import { asyncMiddleware, checkSignature, localAccountValidator, localVideoChann
 import { activityPubValidator } from '../../middlewares/validators/activitypub/activity'
 import { VideoChannelModel } from '../../models/video/video-channel'
 import { AccountModel } from '../../models/account/account'
+import { queue } from 'async'
+import { ActorModel } from '../../models/activitypub/actor'
 
 const inboxRouter = express.Router()
 
@@ -14,7 +16,7 @@ inboxRouter.post('/inbox',
   signatureValidator,
   asyncMiddleware(checkSignature),
   asyncMiddleware(activityPubValidator),
-  asyncMiddleware(inboxController)
+  inboxController
 )
 
 inboxRouter.post('/accounts/:name/inbox',
@@ -22,14 +24,14 @@ inboxRouter.post('/accounts/:name/inbox',
   asyncMiddleware(checkSignature),
   asyncMiddleware(localAccountValidator),
   asyncMiddleware(activityPubValidator),
-  asyncMiddleware(inboxController)
+  inboxController
 )
 inboxRouter.post('/video-channels/:name/inbox',
   signatureValidator,
   asyncMiddleware(checkSignature),
   asyncMiddleware(localVideoChannelValidator),
   asyncMiddleware(activityPubValidator),
-  asyncMiddleware(inboxController)
+  inboxController
 )
 
 // ---------------------------------------------------------------------------
@@ -40,7 +42,12 @@ export {
 
 // ---------------------------------------------------------------------------
 
-async function inboxController (req: express.Request, res: express.Response, next: express.NextFunction) {
+const inboxQueue = queue<{ activities: Activity[], signatureActor?: ActorModel, inboxActor?: ActorModel }, Error>((task, cb) => {
+  processActivities(task.activities, task.signatureActor, task.inboxActor)
+    .then(() => cb())
+})
+
+function inboxController (req: express.Request, res: express.Response, next: express.NextFunction) {
   const rootActivity: RootActivity = req.body
   let activities: Activity[] = []
 
@@ -66,7 +73,11 @@ async function inboxController (req: express.Request, res: express.Response, nex
 
   logger.info('Receiving inbox requests for %d activities by %s.', activities.length, res.locals.signature.actor.url)
 
-  await processActivities(activities, res.locals.signature.actor, accountOrChannel ? accountOrChannel.Actor : undefined)
+  inboxQueue.push({
+    activities,
+    signatureActor: res.locals.signature.actor,
+    inboxActor: accountOrChannel ? accountOrChannel.Actor : undefined
+  })
 
-  res.status(204).end()
+  return res.status(204).end()
 }

server/controllers/api/config.ts

@@ -8,7 +8,7 @@ import { CONFIG, CONSTRAINTS_FIELDS, reloadConfig } from '../../initializers'
 import { asyncMiddleware, authenticate, ensureUserHasRight } from '../../middlewares'
 import { customConfigUpdateValidator } from '../../middlewares/validators/config'
 import { ClientHtml } from '../../lib/client-html'
-import { auditLoggerFactory, CustomConfigAuditView } from '../../helpers/audit-logger'
+import { auditLoggerFactory, CustomConfigAuditView, getAuditIdFromRes } from '../../helpers/audit-logger'
 import { remove, writeJSON } from 'fs-extra'
 
 const packageJSON = require('../../../../package.json')
@@ -134,10 +134,7 @@ async function getCustomConfig (req: express.Request, res: express.Response, nex
 async function deleteCustomConfig (req: express.Request, res: express.Response, next: express.NextFunction) {
   await remove(CONFIG.CUSTOM_FILE)
 
-  auditLogger.delete(
+  auditLogger.delete(getAuditIdFromRes(res), new CustomConfigAuditView(customConfig()))
-    res.locals.oauth.token.User.Account.Actor.getIdentifier(),
-    new CustomConfigAuditView(customConfig())
-  )
 
   reloadConfig()
   ClientHtml.invalidCache()
@@ -183,7 +180,7 @@ async function updateCustomConfig (req: express.Request, res: express.Response,
   const data = customConfig()
 
   auditLogger.update(
-    res.locals.oauth.token.User.Account.Actor.getIdentifier(),
+    getAuditIdFromRes(res),
     new CustomConfigAuditView(data),
     oldCustomConfigAuditKeys
   )

server/controllers/api/overviews.ts

@@ -4,8 +4,9 @@ import { VideoModel } from '../../models/video/video'
 import { asyncMiddleware } from '../../middlewares'
 import { TagModel } from '../../models/video/tag'
 import { VideosOverview } from '../../../shared/models/overviews'
-import { OVERVIEWS, ROUTE_CACHE_LIFETIME } from '../../initializers'
+import { MEMOIZE_TTL, OVERVIEWS, ROUTE_CACHE_LIFETIME } from '../../initializers'
 import { cacheRoute } from '../../middlewares/cache'
+import * as memoizee from 'memoizee'
 
 const overviewsRouter = express.Router()
 
@@ -20,13 +21,30 @@ export { overviewsRouter }
 
 // ---------------------------------------------------------------------------
 
+const buildSamples = memoizee(async function () {
+  const [ categories, channels, tags ] = await Promise.all([
+    VideoModel.getRandomFieldSamples('category', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT),
+    VideoModel.getRandomFieldSamples('channelId', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD ,OVERVIEWS.VIDEOS.SAMPLES_COUNT),
+    TagModel.getRandomSamples(OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT)
+  ])
+
+  return { categories, channels, tags }
+}, { maxAge: MEMOIZE_TTL.OVERVIEWS_SAMPLE })
+
 // This endpoint could be quite long, but we cache it
 async function getVideosOverview (req: express.Request, res: express.Response) {
   const attributes = await buildSamples()
+
+  const [ categories, channels, tags ] = await Promise.all([
+    Promise.all(attributes.categories.map(c => getVideosByCategory(c, res))),
+    Promise.all(attributes.channels.map(c => getVideosByChannel(c, res))),
+    Promise.all(attributes.tags.map(t => getVideosByTag(t, res)))
+  ])
+
   const result: VideosOverview = {
-    categories: await Promise.all(attributes.categories.map(c => getVideosByCategory(c, res))),
+    categories,
-    channels: await Promise.all(attributes.channels.map(c => getVideosByChannel(c, res))),
+    channels,
-    tags: await Promise.all(attributes.tags.map(t => getVideosByTag(t, res)))
+    tags
   }
 
   // Cleanup our object
@@ -37,16 +55,6 @@ async function getVideosOverview (req: express.Request, res: express.Response) {
   return res.json(result)
 }
 
-async function buildSamples () {
-  const [ categories, channels, tags ] = await Promise.all([
-    VideoModel.getRandomFieldSamples('category', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT),
-    VideoModel.getRandomFieldSamples('channelId', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD ,OVERVIEWS.VIDEOS.SAMPLES_COUNT),
-    TagModel.getRandomSamples(OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT)
-  ])
-
-  return { categories, channels, tags }
-}
-
 async function getVideosByTag (tag: string, res: express.Response) {
   const videos = await getVideos(res, { tagsOneOf: [ tag ] })
 
@@ -84,14 +92,16 @@ async function getVideos (
   res: express.Response,
   where: { videoChannelId?: number, tagsOneOf?: string[], categoryOneOf?: number[] }
 ) {
-  const { data } = await VideoModel.listForApi(Object.assign({
+  const query = Object.assign({
     start: 0,
     count: 10,
     sort: '-createdAt',
     includeLocalVideos: true,
     nsfw: buildNSFWFilter(res),
     withFiles: false
-  }, where))
+  }, where)
+
+  const { data } = await VideoModel.listForApi(query, false)
 
   return data.map(d => d.toFormattedJSON())
 }

server/controllers/api/search.ts

@@ -56,6 +56,9 @@ function searchVideoChannels (req: express.Request, res: express.Response) {
   const isURISearch = search.startsWith('http://') || search.startsWith('https://')
 
   const parts = search.split('@')
+
+  // Handle strings like @toto@example.com
+  if (parts.length === 3 && parts[0].length === 0) parts.shift()
   const isWebfingerSearch = parts.length === 2 && parts.every(p => p.indexOf(' ') === -1)
 
   if (isURISearch || isWebfingerSearch) return searchVideoChannelURI(search, isWebfingerSearch, res)
@@ -86,7 +89,7 @@ async function searchVideoChannelURI (search: string, isWebfingerSearch: boolean
 
   if (isUserAbleToSearchRemoteURI(res)) {
     try {
-      const actor = await getOrCreateActorAndServerAndModel(uri, true, true)
+      const actor = await getOrCreateActorAndServerAndModel(uri, 'all', true, true)
       videoChannel = actor.VideoChannel
     } catch (err) {
       logger.info('Cannot search remote video channel %s.', uri, { err })
@@ -136,7 +139,7 @@ async function searchVideoURI (url: string, res: express.Response) {
         refreshVideo: false
       }
 
-      const result = await getOrCreateVideoAndAccountAndChannel(url, syncParam)
+      const result = await getOrCreateVideoAndAccountAndChannel({ videoObject: url, syncParam })
       video = result ? result.video : undefined
     } catch (err) {
       logger.info('Cannot search remote video %s.', url, { err })

server/controllers/api/server/stats.ts

@@ -5,10 +5,14 @@ import { UserModel } from '../../../models/account/user'
 import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
 import { VideoModel } from '../../../models/video/video'
 import { VideoCommentModel } from '../../../models/video/video-comment'
+import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
+import { CONFIG, ROUTE_CACHE_LIFETIME } from '../../../initializers/constants'
+import { cacheRoute } from '../../../middlewares/cache'
 
 const statsRouter = express.Router()
 
 statsRouter.get('/stats',
+  asyncMiddleware(cacheRoute(ROUTE_CACHE_LIFETIME.STATS)),
   asyncMiddleware(getStats)
 )
 
@@ -18,6 +22,13 @@ async function getStats (req: express.Request, res: express.Response, next: expr
   const { totalUsers } = await UserModel.getStats()
   const { totalInstanceFollowers, totalInstanceFollowing } = await ActorFollowModel.getStats()
 
+  const videosRedundancyStats = await Promise.all(
+    CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.map(r => {
+      return VideoRedundancyModel.getStats(r.strategy)
+        .then(stats => Object.assign(stats, { strategy: r.strategy, totalSize: r.size }))
+    })
+  )
+
   const data: ServerStats = {
     totalLocalVideos,
     totalLocalVideoViews,
@@ -26,7 +37,8 @@ async function getStats (req: express.Request, res: express.Response, next: expr
     totalVideoComments,
     totalUsers,
     totalInstanceFollowers,
-    totalInstanceFollowing
+    totalInstanceFollowing,
+    videosRedundancy: videosRedundancyStats
   }
 
   return res.json(data).end()

server/controllers/api/users/index.ts

@@ -27,13 +27,17 @@ import {
   usersUpdateValidator
 } from '../../../middlewares'
 import {
-  usersAskResetPasswordValidator, usersBlockingValidator, usersResetPasswordValidator,
+  usersAskResetPasswordValidator,
-  usersAskSendVerifyEmailValidator, usersVerifyEmailValidator
+  usersAskSendVerifyEmailValidator,
+  usersBlockingValidator,
+  usersResetPasswordValidator,
+  usersVerifyEmailValidator
 } from '../../../middlewares/validators'
 import { UserModel } from '../../../models/account/user'
 import { OAuthTokenModel } from '../../../models/oauth/oauth-token'
-import { auditLoggerFactory, UserAuditView } from '../../../helpers/audit-logger'
+import { auditLoggerFactory, getAuditIdFromRes, UserAuditView } from '../../../helpers/audit-logger'
 import { meRouter } from './me'
+import { deleteUserToken } from '../../../lib/oauth-model'
 
 const auditLogger = auditLoggerFactory('users')
 
@@ -166,7 +170,7 @@ async function createUser (req: express.Request, res: express.Response) {
 
   const { user, account } = await createUserAccountAndChannel(userToCreate)
 
-  auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new UserAuditView(user.toFormattedJSON()))
+  auditLogger.create(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON()))
   logger.info('User %s with its channel and account created.', body.username)
 
   return res.json({
@@ -245,7 +249,7 @@ async function removeUser (req: express.Request, res: express.Response, next: ex
 
   await user.destroy()
 
-  auditLogger.delete(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new UserAuditView(user.toFormattedJSON()))
+  auditLogger.delete(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON()))
 
   return res.sendStatus(204)
 }
@@ -264,15 +268,9 @@ async function updateUser (req: express.Request, res: express.Response, next: ex
   const user = await userToUpdate.save()
 
   // Destroy user token to refresh rights
-  if (roleChanged) {
+  if (roleChanged) await deleteUserToken(userToUpdate.id)
-    await OAuthTokenModel.deleteUserToken(userToUpdate.id)
-  }
 
-  auditLogger.update(
+  auditLogger.update(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON()), oldUserAuditView)
-    res.locals.oauth.token.User.Account.Actor.getIdentifier(),
-    new UserAuditView(user.toFormattedJSON()),
-    oldUserAuditView
-  )
 
   // Don't need to send this update to followers, these attributes are not propagated
 
@@ -333,16 +331,12 @@ async function changeUserBlock (res: express.Response, user: UserModel, block: b
   user.blockedReason = reason || null
 
   await sequelizeTypescript.transaction(async t => {
-    await OAuthTokenModel.deleteUserToken(user.id, t)
+    await deleteUserToken(user.id, t)
 
     await user.save({ transaction: t })
   })
 
   await Emailer.Instance.addUserBlockJob(user, block, reason)
 
-  auditLogger.update(
+  auditLogger.update(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON()), oldUserAuditView)
-    res.locals.oauth.token.User.Account.Actor.getIdentifier(),
-    new UserAuditView(user.toFormattedJSON()),
-    oldUserAuditView
-  )
 }

server/controllers/api/users/me.ts

@@ -5,7 +5,8 @@ import { getFormattedObjects } from '../../../helpers/utils'
 import { CONFIG, IMAGE_MIMETYPE_EXT, sequelizeTypescript } from '../../../initializers'
 import { sendUpdateActor } from '../../../lib/activitypub/send'
 import {
-  asyncMiddleware, asyncRetryTransactionMiddleware,
+  asyncMiddleware,
+  asyncRetryTransactionMiddleware,
   authenticate,
   commonVideosFiltersValidator,
   paginationValidator,
@@ -17,11 +18,11 @@ import {
   usersVideoRatingValidator
 } from '../../../middlewares'
 import {
+  areSubscriptionsExistValidator,
   deleteMeValidator,
   userSubscriptionsSortValidator,
   videoImportsSortValidator,
-  videosSortValidator,
+  videosSortValidator
-  areSubscriptionsExistValidator
 } from '../../../middlewares/validators'
 import { AccountVideoRateModel } from '../../../models/account/account-video-rate'
 import { UserModel } from '../../../models/account/user'
@@ -31,12 +32,13 @@ import { buildNSFWFilter, createReqFiles } from '../../../helpers/express-utils'
 import { UserVideoQuota } from '../../../../shared/models/users/user-video-quota.model'
 import { updateAvatarValidator } from '../../../middlewares/validators/avatar'
 import { updateActorAvatarFile } from '../../../lib/avatar'
-import { auditLoggerFactory, UserAuditView } from '../../../helpers/audit-logger'
+import { auditLoggerFactory, getAuditIdFromRes, UserAuditView } from '../../../helpers/audit-logger'
 import { VideoImportModel } from '../../../models/video/video-import'
 import { VideoFilter } from '../../../../shared/models/videos/video-query.type'
 import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
 import { JobQueue } from '../../../lib/job-queue'
 import { logger } from '../../../helpers/logger'
+import { AccountModel } from '../../../models/account/account'
 
 const auditLogger = auditLoggerFactory('users-me')
 
@@ -293,7 +295,7 @@ async function getUserVideoQuotaUsed (req: express.Request, res: express.Respons
 }
 
 async function getUserVideoRating (req: express.Request, res: express.Response, next: express.NextFunction) {
-  const videoId = +req.params.videoId
+  const videoId = res.locals.video.id
   const accountId = +res.locals.oauth.token.User.Account.id
 
   const ratingObj = await AccountVideoRateModel.load(accountId, videoId, null)
@@ -311,7 +313,7 @@ async function deleteMe (req: express.Request, res: express.Response) {
 
   await user.destroy()
 
-  auditLogger.delete(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new UserAuditView(user.toFormattedJSON()))
+  auditLogger.delete(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON()))
 
   return res.sendStatus(204)
 }
@@ -328,19 +330,17 @@ async function updateMe (req: express.Request, res: express.Response, next: expr
   if (body.autoPlayVideo !== undefined) user.autoPlayVideo = body.autoPlayVideo
 
   await sequelizeTypescript.transaction(async t => {
+    const userAccount = await AccountModel.load(user.Account.id)
+
     await user.save({ transaction: t })
 
-    if (body.displayName !== undefined) user.Account.name = body.displayName
+    if (body.displayName !== undefined) userAccount.name = body.displayName
-    if (body.description !== undefined) user.Account.description = body.description
+    if (body.description !== undefined) userAccount.description = body.description
-    await user.Account.save({ transaction: t })
+    await userAccount.save({ transaction: t })
 
-    await sendUpdateActor(user.Account, t)
+    await sendUpdateActor(userAccount, t)
 
-    auditLogger.update(
+    auditLogger.update(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON()), oldUserAuditView)
-      res.locals.oauth.token.User.Account.Actor.getIdentifier(),
-      new UserAuditView(user.toFormattedJSON()),
-      oldUserAuditView
-    )
   })
 
   return res.sendStatus(204)
@@ -350,15 +350,12 @@ async function updateMyAvatar (req: express.Request, res: express.Response, next
   const avatarPhysicalFile = req.files[ 'avatarfile' ][ 0 ]
   const user: UserModel = res.locals.oauth.token.user
   const oldUserAuditView = new UserAuditView(user.toFormattedJSON())
-  const account = user.Account
 
-  const avatar = await updateActorAvatarFile(avatarPhysicalFile, account.Actor, account)
+  const userAccount = await AccountModel.load(user.Account.id)
 
-  auditLogger.update(
+  const avatar = await updateActorAvatarFile(avatarPhysicalFile, userAccount)
-    res.locals.oauth.token.User.Account.Actor.getIdentifier(),
+
-    new UserAuditView(user.toFormattedJSON()),
+  auditLogger.update(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON()), oldUserAuditView)
-    oldUserAuditView
-  )
 
   return res.json({ avatar: avatar.toFormattedJSON() })
 }

server/controllers/api/video-channel.ts

@@ -27,8 +27,9 @@ import { logger } from '../../helpers/logger'
 import { VideoModel } from '../../models/video/video'
 import { updateAvatarValidator } from '../../middlewares/validators/avatar'
 import { updateActorAvatarFile } from '../../lib/avatar'
-import { auditLoggerFactory, VideoChannelAuditView } from '../../helpers/audit-logger'
+import { auditLoggerFactory, getAuditIdFromRes, VideoChannelAuditView } from '../../helpers/audit-logger'
 import { resetSequelizeInstance } from '../../helpers/database-utils'
+import { UserModel } from '../../models/account/user'
 
 const auditLogger = auditLoggerFactory('channels')
 const reqAvatarFile = createReqFiles([ 'avatarfile' ], IMAGE_MIMETYPE_EXT, { avatarfile: CONFIG.STORAGE.AVATARS_DIR })
@@ -55,7 +56,7 @@ videoChannelRouter.post('/:nameWithHost/avatar/pick',
   // Check the rights
   asyncMiddleware(videoChannelsUpdateValidator),
   updateAvatarValidator,
-  asyncMiddleware(updateVideoChannelAvatar)
+  asyncRetryTransactionMiddleware(updateVideoChannelAvatar)
 )
 
 videoChannelRouter.put('/:nameWithHost',
@@ -106,13 +107,9 @@ async function updateVideoChannelAvatar (req: express.Request, res: express.Resp
   const videoChannel = res.locals.videoChannel as VideoChannelModel
   const oldVideoChannelAuditKeys = new VideoChannelAuditView(videoChannel.toFormattedJSON())
 
-  const avatar = await updateActorAvatarFile(avatarPhysicalFile, videoChannel.Actor, videoChannel)
+  const avatar = await updateActorAvatarFile(avatarPhysicalFile, videoChannel)
 
-  auditLogger.update(
+  auditLogger.update(getAuditIdFromRes(res), new VideoChannelAuditView(videoChannel.toFormattedJSON()), oldVideoChannelAuditKeys)
-    res.locals.oauth.token.User.Account.Actor.getIdentifier(),
-    new VideoChannelAuditView(videoChannel.toFormattedJSON()),
-    oldVideoChannelAuditKeys
-  )
 
   return res
     .json({
@@ -123,19 +120,17 @@ async function updateVideoChannelAvatar (req: express.Request, res: express.Resp
 
 async function addVideoChannel (req: express.Request, res: express.Response) {
   const videoChannelInfo: VideoChannelCreate = req.body
-  const account: AccountModel = res.locals.oauth.token.User.Account
 
   const videoChannelCreated: VideoChannelModel = await sequelizeTypescript.transaction(async t => {
+    const account = await AccountModel.load((res.locals.oauth.token.User as UserModel).Account.id, t)
+
     return createVideoChannel(videoChannelInfo, account, t)
   })
 
   setAsyncActorKeys(videoChannelCreated.Actor)
     .catch(err => logger.error('Cannot set async actor keys for account %s.', videoChannelCreated.Actor.uuid, { err }))
 
-  auditLogger.create(
+  auditLogger.create(getAuditIdFromRes(res), new VideoChannelAuditView(videoChannelCreated.toFormattedJSON()))
-    res.locals.oauth.token.User.Account.Actor.getIdentifier(),
-    new VideoChannelAuditView(videoChannelCreated.toFormattedJSON())
-  )
   logger.info('Video channel with uuid %s created.', videoChannelCreated.Actor.uuid)
 
   return res.json({
@@ -166,7 +161,7 @@ async function updateVideoChannel (req: express.Request, res: express.Response)
       await sendUpdateActor(videoChannelInstanceUpdated, t)
 
       auditLogger.update(
-        res.locals.oauth.token.User.Account.Actor.getIdentifier(),
+        getAuditIdFromRes(res),
         new VideoChannelAuditView(videoChannelInstanceUpdated.toFormattedJSON()),
         oldVideoChannelAuditKeys
       )
@@ -192,10 +187,7 @@ async function removeVideoChannel (req: express.Request, res: express.Response)
   await sequelizeTypescript.transaction(async t => {
     await videoChannelInstance.destroy({ transaction: t })
 
-    auditLogger.delete(
+    auditLogger.delete(getAuditIdFromRes(res), new VideoChannelAuditView(videoChannelInstance.toFormattedJSON()))
-      res.locals.oauth.token.User.Account.Actor.getIdentifier(),
-      new VideoChannelAuditView(videoChannelInstance.toFormattedJSON())
-    )
     logger.info('Video channel with name %s and uuid %s deleted.', videoChannelInstance.name, videoChannelInstance.Actor.uuid)
   })
 

server/controllers/api/videos/abuse.ts

@@ -21,6 +21,7 @@ import { AccountModel } from '../../../models/account/account'
 import { VideoModel } from '../../../models/video/video'
 import { VideoAbuseModel } from '../../../models/video/video-abuse'
 import { auditLoggerFactory, VideoAbuseAuditView } from '../../../helpers/audit-logger'
+import { UserModel } from '../../../models/account/user'
 
 const auditLogger = auditLoggerFactory('abuse')
 const abuseVideoRouter = express.Router()
@@ -95,17 +96,18 @@ async function deleteVideoAbuse (req: express.Request, res: express.Response) {
 
 async function reportVideoAbuse (req: express.Request, res: express.Response) {
   const videoInstance = res.locals.video as VideoModel
-  const reporterAccount = res.locals.oauth.token.User.Account as AccountModel
   const body: VideoAbuseCreate = req.body
 
-  const abuseToCreate = {
-    reporterAccountId: reporterAccount.id,
-    reason: body.reason,
-    videoId: videoInstance.id,
-    state: VideoAbuseState.PENDING
-  }
-
   const videoAbuse: VideoAbuseModel = await sequelizeTypescript.transaction(async t => {
+    const reporterAccount = await AccountModel.load((res.locals.oauth.token.User as UserModel).Account.id, t)
+
+    const abuseToCreate = {
+      reporterAccountId: reporterAccount.id,
+      reason: body.reason,
+      videoId: videoInstance.id,
+      state: VideoAbuseState.PENDING
+    }
+
     const videoAbuseInstance = await VideoAbuseModel.create(abuseToCreate, { transaction: t })
     videoAbuseInstance.Video = videoInstance
     videoAbuseInstance.Account = reporterAccount
@@ -121,7 +123,6 @@ async function reportVideoAbuse (req: express.Request, res: express.Response) {
   })
 
   logger.info('Abuse report for video %s created.', videoInstance.name)
-  return res.json({
+
-    videoAbuse: videoAbuse.toFormattedJSON()
+  return res.json({ videoAbuse: videoAbuse.toFormattedJSON() }).end()
-  }).end()
 }

server/controllers/api/videos/comment.ts

@@ -23,7 +23,9 @@ import {
 } from '../../../middlewares/validators/video-comments'
 import { VideoModel } from '../../../models/video/video'
 import { VideoCommentModel } from '../../../models/video/video-comment'
-import { auditLoggerFactory, CommentAuditView } from '../../../helpers/audit-logger'
+import { auditLoggerFactory, CommentAuditView, getAuditIdFromRes } from '../../../helpers/audit-logger'
+import { AccountModel } from '../../../models/account/account'
+import { UserModel } from '../../../models/account/user'
 
 const auditLogger = auditLoggerFactory('comments')
 const videoCommentRouter = express.Router()
@@ -86,7 +88,7 @@ async function listVideoThreadComments (req: express.Request, res: express.Respo
   let resultList: ResultList<VideoCommentModel>
 
   if (video.commentsEnabled === true) {
-    resultList = await VideoCommentModel.listThreadCommentsForApi(res.locals.video.id, res.locals.videoCommentThread.id)
+    resultList = await VideoCommentModel.listThreadCommentsForApi(video.id, res.locals.videoCommentThread.id)
   } else {
     resultList = {
       total: 0,
@@ -101,15 +103,17 @@ async function addVideoCommentThread (req: express.Request, res: express.Respons
   const videoCommentInfo: VideoCommentCreate = req.body
 
   const comment = await sequelizeTypescript.transaction(async t => {
+    const account = await AccountModel.load((res.locals.oauth.token.User as UserModel).Account.id, t)
+
     return createVideoComment({
       text: videoCommentInfo.text,
       inReplyToComment: null,
       video: res.locals.video,
-      account: res.locals.oauth.token.User.Account
+      account
     }, t)
   })
 
-  auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new CommentAuditView(comment.toFormattedJSON()))
+  auditLogger.create(getAuditIdFromRes(res), new CommentAuditView(comment.toFormattedJSON()))
 
   return res.json({
     comment: comment.toFormattedJSON()
@@ -120,19 +124,19 @@ async function addVideoCommentReply (req: express.Request, res: express.Response
   const videoCommentInfo: VideoCommentCreate = req.body
 
   const comment = await sequelizeTypescript.transaction(async t => {
+    const account = await AccountModel.load((res.locals.oauth.token.User as UserModel).Account.id, t)
+
     return createVideoComment({
       text: videoCommentInfo.text,
       inReplyToComment: res.locals.videoComment,
       video: res.locals.video,
-      account: res.locals.oauth.token.User.Account
+      account
     }, t)
   })
 
-  auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new CommentAuditView(comment.toFormattedJSON()))
+  auditLogger.create(getAuditIdFromRes(res), new CommentAuditView(comment.toFormattedJSON()))
 
-  return res.json({
+  return res.json({ comment: comment.toFormattedJSON() }).end()
-    comment: comment.toFormattedJSON()
-  }).end()
 }
 
 async function removeVideoComment (req: express.Request, res: express.Response) {
@@ -143,7 +147,7 @@ async function removeVideoComment (req: express.Request, res: express.Response)
   })
 
   auditLogger.delete(
-    res.locals.oauth.token.User.Account.Actor.getIdentifier(),
+    getAuditIdFromRes(res),
     new CommentAuditView(videoCommentInstance.toFormattedJSON())
   )
   logger.info('Video comment %d deleted.', videoCommentInstance.id)

server/controllers/api/videos/import.ts

@@ -1,7 +1,7 @@
 import * as express from 'express'
 import * as magnetUtil from 'magnet-uri'
 import 'multer'
-import { auditLoggerFactory, VideoImportAuditView } from '../../../helpers/audit-logger'
+import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger'
 import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoImportAddValidator } from '../../../middlewares'
 import {
   CONFIG,
@@ -114,7 +114,7 @@ async function addTorrentImport (req: express.Request, res: express.Response, to
   }
   await JobQueue.Instance.createJob({ type: 'video-import', payload })
 
-  auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new VideoImportAuditView(videoImport.toFormattedJSON()))
+  auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
 
   return res.json(videoImport.toFormattedJSON()).end()
 }
@@ -158,7 +158,7 @@ async function addYoutubeDLImport (req: express.Request, res: express.Response)
   }
   await JobQueue.Instance.createJob({ type: 'video-import', payload })
 
-  auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new VideoImportAuditView(videoImport.toFormattedJSON()))
+  auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
 
   return res.json(videoImport.toFormattedJSON()).end()
 }

server/controllers/api/videos/index.ts

@@ -4,7 +4,7 @@ import { VideoCreate, VideoPrivacy, VideoState, VideoUpdate } from '../../../../
 import { getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils'
 import { processImage } from '../../../helpers/image-utils'
 import { logger } from '../../../helpers/logger'
-import { auditLoggerFactory, VideoAuditView } from '../../../helpers/audit-logger'
+import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger'
 import { getFormattedObjects, getServerActor } from '../../../helpers/utils'
 import {
   CONFIG,
@@ -253,7 +253,7 @@ async function addVideo (req: express.Request, res: express.Response) {
 
     await federateVideoIfNeeded(video, true, t)
 
-    auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new VideoAuditView(videoCreated.toFormattedDetailsJSON()))
+    auditLogger.create(getAuditIdFromRes(res), new VideoAuditView(videoCreated.toFormattedDetailsJSON()))
     logger.info('Video with name %s and uuid %s created.', videoInfo.name, videoCreated.uuid)
 
     return videoCreated
@@ -354,7 +354,7 @@ async function updateVideo (req: express.Request, res: express.Response) {
       await federateVideoIfNeeded(videoInstanceUpdated, isNewVideo, t)
 
       auditLogger.update(
-        res.locals.oauth.token.User.Account.Actor.getIdentifier(),
+        getAuditIdFromRes(res),
         new VideoAuditView(videoInstanceUpdated.toFormattedDetailsJSON()),
         oldVideoAuditView
       )
@@ -393,9 +393,9 @@ async function viewVideo (req: express.Request, res: express.Response) {
     Redis.Instance.setIPVideoView(ip, videoInstance.uuid)
   ])
 
-  const serverAccount = await getServerActor()
+  const serverActor = await getServerActor()
 
-  await sendCreateView(serverAccount, videoInstance, undefined)
+  await sendCreateView(serverActor, videoInstance, undefined)
 
   return res.status(204).end()
 }
@@ -439,7 +439,7 @@ async function removeVideo (req: express.Request, res: express.Response) {
     await videoInstance.destroy({ transaction: t })
   })
 
-  auditLogger.delete(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new VideoAuditView(videoInstance.toFormattedDetailsJSON()))
+  auditLogger.delete(getAuditIdFromRes(res), new VideoAuditView(videoInstance.toFormattedDetailsJSON()))
   logger.info('Video with name %s and uuid %s deleted.', videoInstance.name, videoInstance.uuid)
 
   return res.type('json').status(204).end()

server/controllers/api/videos/ownership.ts

@@ -19,6 +19,7 @@ import { VideoChannelModel } from '../../../models/video/video-channel'
 import { getFormattedObjects } from '../../../helpers/utils'
 import { changeVideoChannelShare } from '../../../lib/activitypub'
 import { sendUpdateVideo } from '../../../lib/activitypub/send'
+import { UserModel } from '../../../models/account/user'
 
 const ownershipVideoRouter = express.Router()
 
@@ -58,26 +59,25 @@ export {
 
 async function giveVideoOwnership (req: express.Request, res: express.Response) {
   const videoInstance = res.locals.video as VideoModel
-  const initiatorAccount = res.locals.oauth.token.User.Account as AccountModel
+  const initiatorAccountId = (res.locals.oauth.token.User as UserModel).Account.id
   const nextOwner = res.locals.nextOwner as AccountModel
 
   await sequelizeTypescript.transaction(t => {
     return VideoChangeOwnershipModel.findOrCreate({
       where: {
-        initiatorAccountId: initiatorAccount.id,
+        initiatorAccountId,
         nextOwnerAccountId: nextOwner.id,
         videoId: videoInstance.id,
         status: VideoChangeOwnershipStatus.WAITING
       },
       defaults: {
-        initiatorAccountId: initiatorAccount.id,
+        initiatorAccountId,
         nextOwnerAccountId: nextOwner.id,
         videoId: videoInstance.id,
         status: VideoChangeOwnershipStatus.WAITING
       },
       transaction: t
     })
-
   })
 
   logger.info('Ownership change for video %s created.', videoInstance.name)
@@ -85,9 +85,10 @@ async function giveVideoOwnership (req: express.Request, res: express.Response)
 }
 
 async function listVideoOwnership (req: express.Request, res: express.Response) {
-  const currentAccount = res.locals.oauth.token.User.Account as AccountModel
+  const currentAccountId = (res.locals.oauth.token.User as UserModel).Account.id
+
   const resultList = await VideoChangeOwnershipModel.listForApi(
-    currentAccount.id,
+    currentAccountId,
     req.query.start || 0,
     req.query.count || 10,
     req.query.sort || 'createdAt'

server/controllers/api/videos/rate.ts

@@ -28,10 +28,11 @@ async function rateVideo (req: express.Request, res: express.Response) {
   const body: UserVideoRateUpdate = req.body
   const rateType = body.rating
   const videoInstance: VideoModel = res.locals.video
-  const accountInstance: AccountModel = res.locals.oauth.token.User.Account
 
   await sequelizeTypescript.transaction(async t => {
     const sequelizeOptions = { transaction: t }
+
+    const accountInstance = await AccountModel.load(res.locals.oauth.token.User.Account.id, t)
     const previousRate = await AccountVideoRateModel.load(accountInstance.id, videoInstance.id, t)
 
     let likesToIncrement = 0
@@ -47,10 +48,10 @@ async function rateVideo (req: express.Request, res: express.Response) {
       else if (previousRate.type === VIDEO_RATE_TYPES.DISLIKE) dislikesToIncrement--
 
       if (rateType === 'none') { // Destroy previous rate
-        await previousRate.destroy({ transaction: t })
+        await previousRate.destroy(sequelizeOptions)
       } else { // Update previous rate
         previousRate.type = rateType
-        await previousRate.save({ transaction: t })
+        await previousRate.save(sequelizeOptions)
       }
     } else if (rateType !== 'none') { // There was not a previous rate, insert a new one if there is a rate
       const query = {
@@ -70,9 +71,9 @@ async function rateVideo (req: express.Request, res: express.Response) {
     await videoInstance.increment(incrementQuery, sequelizeOptions)
 
     await sendVideoRateChange(accountInstance, videoInstance, likesToIncrement, dislikesToIncrement, t)
-  })
 
-  logger.info('Account video rate for video %s of account %s updated.', videoInstance.name, accountInstance.name)
+    logger.info('Account video rate for video %s of account %s updated.', videoInstance.name, accountInstance.name)
+  })
 
   return res.type('json').status(204).end()
 }

server/controllers/client.ts

@@ -35,7 +35,7 @@ clientsRouter.use('' +
 // Static HTML/CSS/JS client files
 
 const staticClientFiles = [
-  'manifest.json',
+  'manifest.webmanifest',
   'ngsw-worker.js',
   'ngsw.json'
 ]

server/helpers/actor.ts

@@ -0,0 +1,13 @@
+import { ActorModel } from '../models/activitypub/actor'
+
+type ActorFetchByUrlType = 'all' | 'actor-and-association-ids'
+function fetchActorByUrl (url: string, fetchType: ActorFetchByUrlType) {
+  if (fetchType === 'all') return ActorModel.loadByUrlAndPopulateAccountAndChannel(url)
+
+  if (fetchType === 'actor-and-association-ids') return ActorModel.loadByUrl(url)
+}
+
+export {
+  ActorFetchByUrlType,
+  fetchActorByUrl
+}

server/helpers/audit-logger.ts

@@ -1,4 +1,5 @@
 import * as path from 'path'
+import * as express from 'express'
 import { diff } from 'deep-object-diff'
 import { chain } from 'lodash'
 import * as flatten from 'flat'
@@ -8,6 +9,11 @@ import { jsonLoggerFormat, labelFormatter } from './logger'
 import { VideoDetails, User, VideoChannel, VideoAbuse, VideoImport } from '../../shared'
 import { VideoComment } from '../../shared/models/videos/video-comment.model'
 import { CustomConfig } from '../../shared/models/server/custom-config.model'
+import { UserModel } from '../models/account/user'
+
+function getAuditIdFromRes (res: express.Response) {
+  return (res.locals.oauth.token.User as UserModel).username
+}
 
 enum AUDIT_TYPE {
   CREATE = 'create',
@@ -255,6 +261,8 @@ class CustomConfigAuditView extends EntityAuditView {
 }
 
 export {
+  getAuditIdFromRes,
+
   auditLoggerFactory,
   VideoImportAuditView,
   VideoChannelAuditView,

server/helpers/custom-validators/activitypub/videos.ts

@@ -171,5 +171,3 @@ function setRemoteVideoTruncatedContent (video: any) {
 
   return true
 }
-
-

server/helpers/custom-validators/video-ownership.ts

@@ -31,7 +31,7 @@ export function checkUserCanTerminateOwnershipChange (
   videoChangeOwnership: VideoChangeOwnershipModel,
   res: Response
 ): boolean {
-  if (videoChangeOwnership.NextOwner.userId === user.Account.userId) {
+  if (videoChangeOwnership.NextOwner.userId === user.id) {
     return true
   }
 

server/helpers/custom-validators/videos.ts

@@ -18,6 +18,7 @@ import { exists, isArray, isFileValid } from './misc'
 import { VideoChannelModel } from '../../models/video/video-channel'
 import { UserModel } from '../../models/account/user'
 import * as magnetUtil from 'magnet-uri'
+import { fetchVideo, VideoFetchType } from '../video'
 
 const VIDEOS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEOS
 
@@ -152,14 +153,8 @@ function checkUserCanManageVideo (user: UserModel, video: VideoModel, right: Use
   return true
 }
 
-async function isVideoExist (id: string, res: Response) {
+async function isVideoExist (id: string, res: Response, fetchType: VideoFetchType = 'all') {
-  let video: VideoModel | null
+  const video = await fetchVideo(id, fetchType)
-
-  if (validator.isInt(id)) {
-    video = await VideoModel.loadAndPopulateAccountAndServerAndTags(+id)
-  } else { // UUID
-    video = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(id)
-  }
 
   if (video === null) {
     res.status(404)
@@ -169,7 +164,7 @@ async function isVideoExist (id: string, res: Response) {
     return false
   }
 
-  res.locals.video = video
+  if (fetchType !== 'none') res.locals.video = video
   return true
 }
 

server/helpers/utils.ts

@@ -1,12 +1,12 @@
 import { ResultList } from '../../shared'
 import { CONFIG } from '../initializers'
-import { ActorModel } from '../models/activitypub/actor'
 import { ApplicationModel } from '../models/application/application'
 import { pseudoRandomBytesPromise, sha256 } from './core-utils'
 import { logger } from './logger'
 import { join } from 'path'
 import { Instance as ParseTorrent } from 'parse-torrent'
 import { remove } from 'fs-extra'
+import * as memoizee from 'memoizee'
 
 function deleteFileAsync (path: string) {
   remove(path)
@@ -36,24 +36,12 @@ function getFormattedObjects<U, T extends FormattableToJSON> (objects: T[], obje
   } as ResultList<U>
 }
 
-async function getServerActor () {
+const getServerActor = memoizee(async function () {
-  if (getServerActor.serverActor === undefined) {
+  const application = await ApplicationModel.load()
-    const application = await ApplicationModel.load()
+  if (!application) throw Error('Could not load Application from database.')
-    if (!application) throw Error('Could not load Application from database.')
 
-    getServerActor.serverActor = application.Account.Actor
+  return application.Account.Actor
-  }
+})
-
-  if (!getServerActor.serverActor) {
-    logger.error('Cannot load server actor.')
-    process.exit(0)
-  }
-
-  return Promise.resolve(getServerActor.serverActor)
-}
-namespace getServerActor {
-  export let serverActor: ActorModel
-}
 
 function generateVideoTmpPath (target: string | ParseTorrent) {
   const id = typeof target === 'string' ? target : target.infoHash

server/helpers/video.ts

@@ -0,0 +1,25 @@
+import { VideoModel } from '../models/video/video'
+
+type VideoFetchType = 'all' | 'only-video' | 'id' | 'none'
+
+function fetchVideo (id: number | string, fetchType: VideoFetchType) {
+  if (fetchType === 'all') return VideoModel.loadAndPopulateAccountAndServerAndTags(id)
+
+  if (fetchType === 'only-video') return VideoModel.load(id)
+
+  if (fetchType === 'id' || fetchType === 'none') return VideoModel.loadOnlyId(id)
+}
+
+type VideoFetchByUrlType = 'all' | 'only-video'
+function fetchVideoByUrl (url: string, fetchType: VideoFetchByUrlType) {
+  if (fetchType === 'all') return VideoModel.loadByUrlAndPopulateAccount(url)
+
+  if (fetchType === 'only-video') return VideoModel.loadByUrl(url)
+}
+
+export {
+  VideoFetchType,
+  VideoFetchByUrlType,
+  fetchVideo,
+  fetchVideoByUrl
+}

server/helpers/webfinger.ts

@@ -12,7 +12,10 @@ const webfinger = new WebFinger({
   request_timeout: 3000
 })
 
-async function loadActorUrlOrGetFromWebfinger (uri: string) {
+async function loadActorUrlOrGetFromWebfinger (uriArg: string) {
+  // Handle strings like @toto@example.com
+  const uri = uriArg.startsWith('@') ? uriArg.slice(1) : uriArg
+
   const [ name, host ] = uri.split('@')
   let actor: ActorModel
 

server/helpers/webtorrent.ts

@@ -24,7 +24,7 @@ function downloadWebTorrentVideo (target: { magnetUri: string, torrentName?: str
         if (timer) clearTimeout(timer)
 
         return safeWebtorrentDestroy(webtorrent, torrentId, file.name, target.torrentName)
-          .then(() => rej(new Error('The number of files is not equal to 1 for ' + torrentId)))
+          .then(() => rej(new Error('Cannot import torrent ' + torrentId + ': there are multiple files in it')))
       }
 
       file = torrent.files[ 0 ]

server/helpers/youtube-dl.ts

@@ -2,7 +2,11 @@ import { truncate } from 'lodash'
 import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES } from '../initializers'
 import { logger } from './logger'
 import { generateVideoTmpPath } from './utils'
-import { YoutubeDlUpdateScheduler } from '../lib/schedulers/youtube-dl-update-scheduler'
+import { join } from 'path'
+import { root } from './core-utils'
+import { ensureDir, writeFile } from 'fs-extra'
+import * as request from 'request'
+import { createWriteStream } from 'fs'
 
 export type YoutubeDLInfo = {
   name?: string
@@ -40,7 +44,7 @@ function downloadYoutubeDLVideo (url: string) {
 
   return new Promise<string>(async (res, rej) => {
     const youtubeDL = await safeGetYoutubeDL()
-    youtubeDL.exec(url, options, async (err, output) => {
+    youtubeDL.exec(url, options, err => {
       if (err) return rej(err)
 
       return res(path)
@@ -48,6 +52,64 @@ function downloadYoutubeDLVideo (url: string) {
   })
 }
 
+// Thanks: https://github.com/przemyslawpluta/node-youtube-dl/blob/master/lib/downloader.js
+// We rewrote it to avoid sync calls
+async function updateYoutubeDLBinary () {
+  logger.info('Updating youtubeDL binary.')
+
+  const binDirectory = join(root(), 'node_modules', 'youtube-dl', 'bin')
+  const bin = join(binDirectory, 'youtube-dl')
+  const detailsPath = join(binDirectory, 'details')
+  const url = 'https://yt-dl.org/downloads/latest/youtube-dl'
+
+  await ensureDir(binDirectory)
+
+  return new Promise(res => {
+    request.get(url, { followRedirect: false }, (err, result) => {
+      if (err) {
+        logger.error('Cannot update youtube-dl.', { err })
+        return res()
+      }
+
+      if (result.statusCode !== 302) {
+        logger.error('youtube-dl update error: did not get redirect for the latest version link. Status %d', result.statusCode)
+        return res()
+      }
+
+      const url = result.headers.location
+      const downloadFile = request.get(url)
+      const newVersion = /yt-dl\.org\/downloads\/(\d{4}\.\d\d\.\d\d(\.\d)?)\/youtube-dl/.exec(url)[ 1 ]
+
+      downloadFile.on('response', result => {
+        if (result.statusCode !== 200) {
+          logger.error('Cannot update youtube-dl: new version response is not 200, it\'s %d.', result.statusCode)
+          return res()
+        }
+
+        downloadFile.pipe(createWriteStream(bin, { mode: 493 }))
+      })
+
+      downloadFile.on('error', err => {
+        logger.error('youtube-dl update error.', { err })
+        return res()
+      })
+
+      downloadFile.on('end', () => {
+        const details = JSON.stringify({ version: newVersion, path: bin, exec: 'youtube-dl' })
+        writeFile(detailsPath, details, { encoding: 'utf8' }, err => {
+          if (err) {
+            logger.error('youtube-dl update error: cannot write details.', { err })
+            return res()
+          }
+
+          logger.info('youtube-dl updated to version %s.', newVersion)
+          return res()
+        })
+      })
+    })
+  })
+}
+
 async function safeGetYoutubeDL () {
   let youtubeDL
 
@@ -55,7 +117,7 @@ async function safeGetYoutubeDL () {
     youtubeDL = require('youtube-dl')
   } catch (e) {
     // Download binary
-    await YoutubeDlUpdateScheduler.Instance.execute()
+    await updateYoutubeDLBinary()
     youtubeDL = require('youtube-dl')
   }
 
@@ -65,6 +127,7 @@ async function safeGetYoutubeDL () {
 // ---------------------------------------------------------------------------
 
 export {
+  updateYoutubeDLBinary,
   downloadYoutubeDLVideo,
   getYoutubeDLInfo,
   safeGetYoutubeDL

server/initializers/checker.ts

@@ -7,7 +7,7 @@ import { parse } from 'url'
 import { CONFIG } from './constants'
 import { logger } from '../helpers/logger'
 import { getServerActor } from '../helpers/utils'
-import { VideosRedundancy } from '../../shared/models/redundancy'
+import { RecentlyAddedStrategy, VideosRedundancy } from '../../shared/models/redundancy'
 import { isArray } from '../helpers/custom-validators/misc'
 import { uniq } from 'lodash'
 
@@ -34,21 +34,28 @@ async function checkActivityPubUrls () {
 function checkConfig () {
   const defaultNSFWPolicy = config.get<string>('instance.default_nsfw_policy')
 
+  // NSFW policy
   if ([ 'do_not_list', 'blur', 'display' ].indexOf(defaultNSFWPolicy) === -1) {
     return 'NSFW policy setting should be "do_not_list" or "blur" or "display" instead of ' + defaultNSFWPolicy
   }
 
-  const redundancyVideos = config.get<VideosRedundancy[]>('redundancy.videos')
+  // Redundancies
+  const redundancyVideos = config.get<VideosRedundancy[]>('redundancy.videos.strategies')
   if (isArray(redundancyVideos)) {
     for (const r of redundancyVideos) {
-      if ([ 'most-views' ].indexOf(r.strategy) === -1) {
+      if ([ 'most-views', 'trending', 'recently-added' ].indexOf(r.strategy) === -1) {
         return 'Redundancy video entries should have "most-views" strategy instead of ' + r.strategy
       }
     }
 
     const filtered = uniq(redundancyVideos.map(r => r.strategy))
     if (filtered.length !== redundancyVideos.length) {
-      return 'Redundancy video entries should have uniq strategies'
+      return 'Redundancy video entries should have unique strategies'
+    }
+
+    const recentlyAddedStrategy = redundancyVideos.find(r => r.strategy === 'recently-added') as RecentlyAddedStrategy
+    if (recentlyAddedStrategy && isNaN(recentlyAddedStrategy.minViews)) {
+      return 'Min views in recently added strategy is not a number'
     }
   }
 
@@ -68,6 +75,7 @@ function checkMissedConfig () {
     'cache.previews.size', 'admin.email',
     'signup.enabled', 'signup.limit', 'signup.requires_email_verification',
     'signup.filters.cidr.whitelist', 'signup.filters.cidr.blacklist',
+    'redundancy.videos.strategies', 'redundancy.videos.check_interval',
     'transcoding.enabled', 'transcoding.threads',
     'import.videos.http.enabled', 'import.videos.torrent.enabled',
     'trending.videos.interval_days',

server/initializers/constants.ts

@@ -1,11 +1,11 @@
 import { IConfig } from 'config'
 import { dirname, join } from 'path'
-import { JobType, VideoRateType, VideoRedundancyStrategy, VideoState, VideosRedundancy } from '../../shared/models'
+import { JobType, VideoRateType, VideoState, VideosRedundancy } from '../../shared/models'
 import { ActivityPubActorType } from '../../shared/models/activitypub'
 import { FollowState } from '../../shared/models/actors'
 import { VideoAbuseState, VideoImportState, VideoPrivacy } from '../../shared/models/videos'
 // Do not use barrels, remain constants as independent as possible
-import { buildPath, isTestInstance, root, sanitizeHost, sanitizeUrl } from '../helpers/core-utils'
+import { buildPath, isTestInstance, parseDuration, root, sanitizeHost, sanitizeUrl } from '../helpers/core-utils'
 import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type'
 import { invert } from 'lodash'
 import { CronRepeatOptions, EveryRepeatOptions } from 'bull'
@@ -66,7 +66,8 @@ const ROUTE_CACHE_LIFETIME = {
   },
   ACTIVITY_PUB: {
     VIDEOS: '1 second' // 1 second, cache concurrent requests after a broadcast for example
-  }
+  },
+  STATS: '4 hours'
 }
 
 // ---------------------------------------------------------------------------
@@ -138,8 +139,7 @@ let SCHEDULER_INTERVALS_MS = {
   badActorFollow: 60000 * 60, // 1 hour
   removeOldJobs: 60000 * 60, // 1 hour
   updateVideos: 60000, // 1 minute
-  youtubeDLUpdate: 60000 * 60 * 24, // 1 day
+  youtubeDLUpdate: 60000 * 60 * 24 // 1 day
-  videosRedundancy: 60000 * 2 // 2 hours
 }
 
 // ---------------------------------------------------------------------------
@@ -211,7 +211,10 @@ const CONFIG = {
     }
   },
   REDUNDANCY: {
-    VIDEOS: buildVideosRedundancy(config.get<any[]>('redundancy.videos'))
+    VIDEOS: {
+      CHECK_INTERVAL: parseDuration(config.get<string>('redundancy.videos.check_interval')),
+      STRATEGIES: buildVideosRedundancy(config.get<any[]>('redundancy.videos.strategies'))
+    }
   },
   ADMIN: {
     get EMAIL () { return config.get<string>('admin.email') }
@@ -592,6 +595,10 @@ const CACHE = {
   }
 }
 
+const MEMOIZE_TTL = {
+  OVERVIEWS_SAMPLE: 1000 * 3600 * 4 // 4 hours
+}
+
 const REDUNDANCY = {
   VIDEOS: {
     EXPIRES_AFTER_MS: 48 * 3600 * 1000, // 2 days
@@ -644,7 +651,6 @@ if (isTestInstance() === true) {
   SCHEDULER_INTERVALS_MS.badActorFollow = 10000
   SCHEDULER_INTERVALS_MS.removeOldJobs = 10000
   SCHEDULER_INTERVALS_MS.updateVideos = 5000
-  SCHEDULER_INTERVALS_MS.videosRedundancy = 5000
   REPEAT_JOBS['videos-views'] = { every: 5000 }
 
   REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR = 1
@@ -654,6 +660,8 @@ if (isTestInstance() === true) {
   JOB_ATTEMPTS['email'] = 1
 
   CACHE.VIDEO_CAPTIONS.MAX_AGE = 3000
+  MEMOIZE_TTL.OVERVIEWS_SAMPLE = 1
+  ROUTE_CACHE_LIFETIME.OVERVIEWS.VIDEOS = '0ms'
 }
 
 updateWebserverConfig()
@@ -708,6 +716,7 @@ export {
   VIDEO_ABUSE_STATES,
   JOB_REQUEST_TIMEOUT,
   USER_PASSWORD_RESET_LIFETIME,
+  MEMOIZE_TTL,
   USER_EMAIL_VERIFY_LIFETIME,
   IMAGE_MIMETYPE_EXT,
   OVERVIEWS,
@@ -741,15 +750,10 @@ function updateWebserverConfig () {
   CONFIG.WEBSERVER.HOST = sanitizeHost(CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT, REMOTE_SCHEME.HTTP)
 }
 
-function buildVideosRedundancy (objs: { strategy: VideoRedundancyStrategy, size: string }[]): VideosRedundancy[] {
+function buildVideosRedundancy (objs: VideosRedundancy[]): VideosRedundancy[] {
   if (!objs) return []
 
-  return objs.map(obj => {
+  return objs.map(obj => Object.assign(obj, { size: bytes.parse(obj.size) }))
-    return {
-      strategy: obj.strategy,
-      size: bytes.parse(obj.size)
-    }
-  })
 }
 
 function buildLanguages () {

server/lib/activitypub/actor.ts

@@ -21,6 +21,7 @@ import { ServerModel } from '../../models/server/server'
 import { VideoChannelModel } from '../../models/video/video-channel'
 import { JobQueue } from '../job-queue'
 import { getServerActor } from '../../helpers/utils'
+import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor'
 
 // Set account keys, this could be long so process after the account creation and do not block the client
 function setAsyncActorKeys (actor: ActorModel) {
@@ -38,13 +39,14 @@ function setAsyncActorKeys (actor: ActorModel) {
 
 async function getOrCreateActorAndServerAndModel (
   activityActor: string | ActivityPubActor,
+  fetchType: ActorFetchByUrlType = 'actor-and-association-ids',
   recurseIfNeeded = true,
   updateCollections = false
 ) {
   const actorUrl = getActorUrl(activityActor)
   let created = false
 
-  let actor = await ActorModel.loadByUrl(actorUrl)
+  let actor = await fetchActorByUrl(actorUrl, fetchType)
   // Orphan actor (not associated to an account of channel) so recreate it
   if (actor && (!actor.Account && !actor.VideoChannel)) {
     await actor.destroy()
@@ -65,7 +67,7 @@ async function getOrCreateActorAndServerAndModel (
 
       try {
         // Assert we don't recurse another time
-        ownerActor = await getOrCreateActorAndServerAndModel(accountAttributedTo.id, false)
+        ownerActor = await getOrCreateActorAndServerAndModel(accountAttributedTo.id, 'all', false)
       } catch (err) {
         logger.error('Cannot get or create account attributed to video channel ' + actor.url)
         throw new Error(err)
@@ -79,7 +81,7 @@ async function getOrCreateActorAndServerAndModel (
   if (actor.Account) actor.Account.Actor = actor
   if (actor.VideoChannel) actor.VideoChannel.Actor = actor
 
-  const { actor: actorRefreshed, refreshed } = await retryTransactionWrapper(refreshActorIfNeeded, actor)
+  const { actor: actorRefreshed, refreshed } = await retryTransactionWrapper(refreshActorIfNeeded, actor, fetchType)
   if (!actorRefreshed) throw new Error('Actor ' + actorRefreshed.url + ' does not exist anymore.')
 
   if ((created === true || refreshed === true) && updateCollections === true) {
@@ -370,8 +372,14 @@ async function saveVideoChannel (actor: ActorModel, result: FetchRemoteActorResu
   return videoChannelCreated
 }
 
-async function refreshActorIfNeeded (actor: ActorModel): Promise<{ actor: ActorModel, refreshed: boolean }> {
+async function refreshActorIfNeeded (
-  if (!actor.isOutdated()) return { actor, refreshed: false }
+  actorArg: ActorModel,
+  fetchedType: ActorFetchByUrlType
+): Promise<{ actor: ActorModel, refreshed: boolean }> {
+  if (!actorArg.isOutdated()) return { actor: actorArg, refreshed: false }
+
+  // We need more attributes
+  const actor = fetchedType === 'all' ? actorArg : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url)
 
   try {
     const actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost())

server/lib/activitypub/audience.ts

@@ -6,7 +6,7 @@ import { VideoModel } from '../../models/video/video'
 import { VideoCommentModel } from '../../models/video/video-comment'
 import { VideoShareModel } from '../../models/video/video-share'
 
-function getVideoAudience (video: VideoModel, actorsInvolvedInVideo: ActorModel[]) {
+function getRemoteVideoAudience (video: VideoModel, actorsInvolvedInVideo: ActorModel[]): ActivityAudience {
   return {
     to: [ video.VideoChannel.Account.Actor.url ],
     cc: actorsInvolvedInVideo.map(a => a.followersUrl)
@@ -18,7 +18,7 @@ function getVideoCommentAudience (
   threadParentComments: VideoCommentModel[],
   actorsInvolvedInVideo: ActorModel[],
   isOrigin = false
-) {
+): ActivityAudience {
   const to = [ ACTIVITY_PUB.PUBLIC ]
   const cc: string[] = []
 
@@ -41,7 +41,7 @@ function getVideoCommentAudience (
   }
 }
 
-function getObjectFollowersAudience (actorsInvolvedInObject: ActorModel[]) {
+function getAudienceFromFollowersOf (actorsInvolvedInObject: ActorModel[]): ActivityAudience {
   return {
     to: [ ACTIVITY_PUB.PUBLIC ].concat(actorsInvolvedInObject.map(a => a.followersUrl)),
     cc: []
@@ -83,9 +83,9 @@ function audiencify<T> (object: T, audience: ActivityAudience) {
 export {
   buildAudience,
   getAudience,
-  getVideoAudience,
+  getRemoteVideoAudience,
   getActorsInvolvedInVideo,
-  getObjectFollowersAudience,
+  getAudienceFromFollowersOf,
   audiencify,
   getVideoCommentAudience
 }

server/lib/activitypub/cache-file.ts

@@ -1,10 +1,9 @@
 import { CacheFileObject } from '../../../shared/index'
 import { VideoModel } from '../../models/video/video'
-import { ActorModel } from '../../models/activitypub/actor'
 import { sequelizeTypescript } from '../../initializers'
 import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
 
-function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: VideoModel, byActor: ActorModel) {
+function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: VideoModel, byActor: { id?: number }) {
   const url = cacheFileObject.url
 
   const videoFile = video.VideoFiles.find(f => {
@@ -23,7 +22,7 @@ function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject
   }
 }
 
-function createCacheFile (cacheFileObject: CacheFileObject, video: VideoModel, byActor: ActorModel) {
+function createCacheFile (cacheFileObject: CacheFileObject, video: VideoModel, byActor: { id?: number }) {
   return sequelizeTypescript.transaction(async t => {
     const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor)
 
@@ -31,7 +30,11 @@ function createCacheFile (cacheFileObject: CacheFileObject, video: VideoModel, b
   })
 }
 
-function updateCacheFile (cacheFileObject: CacheFileObject, redundancyModel: VideoRedundancyModel, byActor: ActorModel) {
+function updateCacheFile (cacheFileObject: CacheFileObject, redundancyModel: VideoRedundancyModel, byActor: { id?: number }) {
+  if (redundancyModel.actorId !== byActor.id) {
+    throw new Error('Cannot update redundancy ' + redundancyModel.url + ' of another actor.')
+  }
+
   const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, redundancyModel.VideoFile.Video, byActor)
 
   redundancyModel.set('expires', attributes.expiresOn)

server/lib/activitypub/process/process-accept.ts

@@ -1,15 +1,11 @@
 import { ActivityAccept } from '../../../../shared/models/activitypub'
-import { getActorUrl } from '../../../helpers/activitypub'
 import { ActorModel } from '../../../models/activitypub/actor'
 import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
 import { addFetchOutboxJob } from '../actor'
 
-async function processAcceptActivity (activity: ActivityAccept, inboxActor?: ActorModel) {
+async function processAcceptActivity (activity: ActivityAccept, targetActor: ActorModel, inboxActor?: ActorModel) {
   if (inboxActor === undefined) throw new Error('Need to accept on explicit inbox.')
 
-  const actorUrl = getActorUrl(activity.actor)
-  const targetActor = await ActorModel.loadByUrl(actorUrl)
-
   return processAccept(inboxActor, targetActor)
 }
 

server/lib/activitypub/process/process-announce.ts

@@ -2,15 +2,11 @@ import { ActivityAnnounce } from '../../../../shared/models/activitypub'
 import { retryTransactionWrapper } from '../../../helpers/database-utils'
 import { sequelizeTypescript } from '../../../initializers'
 import { ActorModel } from '../../../models/activitypub/actor'
-import { VideoModel } from '../../../models/video/video'
 import { VideoShareModel } from '../../../models/video/video-share'
-import { getOrCreateActorAndServerAndModel } from '../actor'
 import { forwardVideoRelatedActivity } from '../send/utils'
 import { getOrCreateVideoAndAccountAndChannel } from '../videos'
 
-async function processAnnounceActivity (activity: ActivityAnnounce) {
+async function processAnnounceActivity (activity: ActivityAnnounce, actorAnnouncer: ActorModel) {
-  const actorAnnouncer = await getOrCreateActorAndServerAndModel(activity.actor)
-
   return retryTransactionWrapper(processVideoShare, actorAnnouncer, activity)
 }
 
@@ -25,7 +21,7 @@ export {
 async function processVideoShare (actorAnnouncer: ActorModel, activity: ActivityAnnounce) {
   const objectUri = typeof activity.object === 'string' ? activity.object : activity.object.id
 
-  const { video } = await getOrCreateVideoAndAccountAndChannel(objectUri)
+  const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: objectUri })
 
   return sequelizeTypescript.transaction(async t => {
     // Add share entry

server/lib/activitypub/process/process-create.ts

@@ -7,30 +7,28 @@ import { sequelizeTypescript } from '../../../initializers'
 import { AccountVideoRateModel } from '../../../models/account/account-video-rate'
 import { ActorModel } from '../../../models/activitypub/actor'
 import { VideoAbuseModel } from '../../../models/video/video-abuse'
-import { getOrCreateActorAndServerAndModel } from '../actor'
 import { addVideoComment, resolveThread } from '../video-comments'
 import { getOrCreateVideoAndAccountAndChannel } from '../videos'
 import { forwardActivity, forwardVideoRelatedActivity } from '../send/utils'
 import { Redis } from '../../redis'
 import { createCacheFile } from '../cache-file'
 
-async function processCreateActivity (activity: ActivityCreate) {
+async function processCreateActivity (activity: ActivityCreate, byActor: ActorModel) {
   const activityObject = activity.object
   const activityType = activityObject.type
-  const actor = await getOrCreateActorAndServerAndModel(activity.actor)
 
   if (activityType === 'View') {
-    return processCreateView(actor, activity)
+    return processCreateView(byActor, activity)
   } else if (activityType === 'Dislike') {
-    return retryTransactionWrapper(processCreateDislike, actor, activity)
+    return retryTransactionWrapper(processCreateDislike, byActor, activity)
   } else if (activityType === 'Video') {
     return processCreateVideo(activity)
   } else if (activityType === 'Flag') {
-    return retryTransactionWrapper(processCreateVideoAbuse, actor, activityObject as VideoAbuseObject)
+    return retryTransactionWrapper(processCreateVideoAbuse, byActor, activityObject as VideoAbuseObject)
   } else if (activityType === 'Note') {
-    return retryTransactionWrapper(processCreateVideoComment, actor, activity)
+    return retryTransactionWrapper(processCreateVideoComment, byActor, activity)
   } else if (activityType === 'CacheFile') {
-    return retryTransactionWrapper(processCacheFile, actor, activity)
+    return retryTransactionWrapper(processCacheFile, byActor, activity)
   }
 
   logger.warn('Unknown activity object type %s when creating activity.', activityType, { activity: activity.id })
@@ -48,7 +46,7 @@ export {
 async function processCreateVideo (activity: ActivityCreate) {
   const videoToCreateData = activity.object as VideoTorrentObject
 
-  const { video } = await getOrCreateVideoAndAccountAndChannel(videoToCreateData)
+  const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoToCreateData })
 
   return video
 }
@@ -59,7 +57,7 @@ async function processCreateDislike (byActor: ActorModel, activity: ActivityCrea
 
   if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url)
 
-  const { video } = await getOrCreateVideoAndAccountAndChannel(dislike.object)
+  const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislike.object })
 
   return sequelizeTypescript.transaction(async t => {
     const rate = {
@@ -86,10 +84,14 @@ async function processCreateDislike (byActor: ActorModel, activity: ActivityCrea
 async function processCreateView (byActor: ActorModel, activity: ActivityCreate) {
   const view = activity.object as ViewObject
 
-  const { video } = await getOrCreateVideoAndAccountAndChannel(view.object)
+  const options = {
+    videoObject: view.object,
+    fetchType: 'only-video' as 'only-video'
+  }
+  const { video } = await getOrCreateVideoAndAccountAndChannel(options)
 
-  const actor = await ActorModel.loadByUrl(view.actor)
+  const actorExists = await ActorModel.isActorUrlExist(view.actor)
-  if (!actor) throw new Error('Unknown actor ' + view.actor)
+  if (actorExists === false) throw new Error('Unknown actor ' + view.actor)
 
   await Redis.Instance.addVideoView(video.id)
 
@@ -103,7 +105,7 @@ async function processCreateView (byActor: ActorModel, activity: ActivityCreate)
 async function processCacheFile (byActor: ActorModel, activity: ActivityCreate) {
   const cacheFile = activity.object as CacheFileObject
 
-  const { video } = await getOrCreateVideoAndAccountAndChannel(cacheFile.object)
+  const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFile.object })
 
   await createCacheFile(cacheFile, video, byActor)
 
@@ -114,13 +116,13 @@ async function processCacheFile (byActor: ActorModel, activity: ActivityCreate)
   }
 }
 
-async function processCreateVideoAbuse (actor: ActorModel, videoAbuseToCreateData: VideoAbuseObject) {
+async function processCreateVideoAbuse (byActor: ActorModel, videoAbuseToCreateData: VideoAbuseObject) {
   logger.debug('Reporting remote abuse for video %s.', videoAbuseToCreateData.object)
 
-  const account = actor.Account
+  const account = byActor.Account
-  if (!account) throw new Error('Cannot create dislike with the non account actor ' + actor.url)
+  if (!account) throw new Error('Cannot create dislike with the non account actor ' + byActor.url)
 
-  const { video } = await getOrCreateVideoAndAccountAndChannel(videoAbuseToCreateData.object)
+  const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoAbuseToCreateData.object })
 
   return sequelizeTypescript.transaction(async t => {
     const videoAbuseData = {

server/lib/activitypub/process/process-delete.ts

@@ -7,41 +7,41 @@ import { ActorModel } from '../../../models/activitypub/actor'
 import { VideoModel } from '../../../models/video/video'
 import { VideoChannelModel } from '../../../models/video/video-channel'
 import { VideoCommentModel } from '../../../models/video/video-comment'
-import { getOrCreateActorAndServerAndModel } from '../actor'
 import { forwardActivity } from '../send/utils'
 
-async function processDeleteActivity (activity: ActivityDelete) {
+async function processDeleteActivity (activity: ActivityDelete, byActor: ActorModel) {
   const objectUrl = typeof activity.object === 'string' ? activity.object : activity.object.id
 
   if (activity.actor === objectUrl) {
-    let actor = await ActorModel.loadByUrl(activity.actor)
+    // We need more attributes (all the account and channel)
-    if (!actor) return undefined
+    const byActorFull = await ActorModel.loadByUrlAndPopulateAccountAndChannel(byActor.url)
 
-    if (actor.type === 'Person') {
+    if (byActorFull.type === 'Person') {
-      if (!actor.Account) throw new Error('Actor ' + actor.url + ' is a person but we cannot find it in database.')
+      if (!byActorFull.Account) throw new Error('Actor ' + byActorFull.url + ' is a person but we cannot find it in database.')
 
-      actor.Account.Actor = await actor.Account.$get('Actor') as ActorModel
+      byActorFull.Account.Actor = await byActorFull.Account.$get('Actor') as ActorModel
-      return retryTransactionWrapper(processDeleteAccount, actor.Account)
+      return retryTransactionWrapper(processDeleteAccount, byActorFull.Account)
-    } else if (actor.type === 'Group') {
+    } else if (byActorFull.type === 'Group') {
-      if (!actor.VideoChannel) throw new Error('Actor ' + actor.url + ' is a group but we cannot find it in database.')
+      if (!byActorFull.VideoChannel) throw new Error('Actor ' + byActorFull.url + ' is a group but we cannot find it in database.')
 
-      actor.VideoChannel.Actor = await actor.VideoChannel.$get('Actor') as ActorModel
+      byActorFull.VideoChannel.Actor = await byActorFull.VideoChannel.$get('Actor') as ActorModel
-      return retryTransactionWrapper(processDeleteVideoChannel, actor.VideoChannel)
+      return retryTransactionWrapper(processDeleteVideoChannel, byActorFull.VideoChannel)
     }
   }
 
-  const actor = await getOrCreateActorAndServerAndModel(activity.actor)
   {
     const videoCommentInstance = await VideoCommentModel.loadByUrlAndPopulateAccount(objectUrl)
     if (videoCommentInstance) {
-      return retryTransactionWrapper(processDeleteVideoComment, actor, videoCommentInstance, activity)
+      return retryTransactionWrapper(processDeleteVideoComment, byActor, videoCommentInstance, activity)
     }
   }
 
   {
     const videoInstance = await VideoModel.loadByUrlAndPopulateAccount(objectUrl)
     if (videoInstance) {
-      return retryTransactionWrapper(processDeleteVideo, actor, videoInstance)
+      if (videoInstance.isOwned()) throw new Error(`Remote instance cannot delete owned video ${videoInstance.url}.`)
+
+      return retryTransactionWrapper(processDeleteVideo, byActor, videoInstance)
     }
   }
 
@@ -94,6 +94,10 @@ function processDeleteVideoComment (byActor: ActorModel, videoComment: VideoComm
   logger.debug('Removing remote video comment "%s".', videoComment.url)
 
   return sequelizeTypescript.transaction(async t => {
+    if (videoComment.Account.id !== byActor.Account.id) {
+      throw new Error('Account ' + byActor.url + ' does not own video comment ' + videoComment.url)
+    }
+
     await videoComment.destroy({ transaction: t })
 
     if (videoComment.Video.isOwned()) {

server/lib/activitypub/process/process-follow.ts

@@ -4,14 +4,12 @@ import { logger } from '../../../helpers/logger'
 import { sequelizeTypescript } from '../../../initializers'
 import { ActorModel } from '../../../models/activitypub/actor'
 import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
-import { getOrCreateActorAndServerAndModel } from '../actor'
 import { sendAccept } from '../send'
 
-async function processFollowActivity (activity: ActivityFollow) {
+async function processFollowActivity (activity: ActivityFollow, byActor: ActorModel) {
   const activityObject = activity.object
-  const actor = await getOrCreateActorAndServerAndModel(activity.actor)
 
-  return retryTransactionWrapper(processFollow, actor, activityObject)
+  return retryTransactionWrapper(processFollow, byActor, activityObject)
 }
 
 // ---------------------------------------------------------------------------
@@ -24,7 +22,7 @@ export {
 
 async function processFollow (actor: ActorModel, targetActorURL: string) {
   await sequelizeTypescript.transaction(async t => {
-    const targetActor = await ActorModel.loadByUrl(targetActorURL, t)
+    const targetActor = await ActorModel.loadByUrlAndPopulateAccountAndChannel(targetActorURL, t)
 
     if (!targetActor) throw new Error('Unknown actor')
     if (targetActor.isOwned() === false) throw new Error('This is not a local actor.')

server/lib/activitypub/process/process-like.ts

@@ -3,14 +3,11 @@ import { retryTransactionWrapper } from '../../../helpers/database-utils'
 import { sequelizeTypescript } from '../../../initializers'
 import { AccountVideoRateModel } from '../../../models/account/account-video-rate'
 import { ActorModel } from '../../../models/activitypub/actor'
-import { getOrCreateActorAndServerAndModel } from '../actor'
 import { forwardVideoRelatedActivity } from '../send/utils'
 import { getOrCreateVideoAndAccountAndChannel } from '../videos'
 
-async function processLikeActivity (activity: ActivityLike) {
+async function processLikeActivity (activity: ActivityLike, byActor: ActorModel) {
-  const actor = await getOrCreateActorAndServerAndModel(activity.actor)
+  return retryTransactionWrapper(processLikeVideo, byActor, activity)
-
-  return retryTransactionWrapper(processLikeVideo, actor, activity)
 }
 
 // ---------------------------------------------------------------------------
@@ -27,7 +24,7 @@ async function processLikeVideo (byActor: ActorModel, activity: ActivityLike) {
   const byAccount = byActor.Account
   if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url)
 
-  const { video } = await getOrCreateVideoAndAccountAndChannel(videoUrl)
+  const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoUrl })
 
   return sequelizeTypescript.transaction(async t => {
     const rate = {

server/lib/activitypub/process/process-reject.ts

@@ -1,15 +1,11 @@
 import { ActivityReject } from '../../../../shared/models/activitypub/activity'
-import { getActorUrl } from '../../../helpers/activitypub'
 import { sequelizeTypescript } from '../../../initializers'
 import { ActorModel } from '../../../models/activitypub/actor'
 import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
 
-async function processRejectActivity (activity: ActivityReject, inboxActor?: ActorModel) {
+async function processRejectActivity (activity: ActivityReject, targetActor: ActorModel, inboxActor?: ActorModel) {
   if (inboxActor === undefined) throw new Error('Need to reject on explicit inbox.')
 
-  const actorUrl = getActorUrl(activity.actor)
-  const targetActor = await ActorModel.loadByUrl(actorUrl)
-
   return processReject(inboxActor, targetActor)
 }
 
@@ -21,11 +17,11 @@ export {
 
 // ---------------------------------------------------------------------------
 
-async function processReject (actor: ActorModel, targetActor: ActorModel) {
+async function processReject (follower: ActorModel, targetActor: ActorModel) {
   return sequelizeTypescript.transaction(async t => {
-    const actorFollow = await ActorFollowModel.loadByActorAndTarget(actor.id, targetActor.id, t)
+    const actorFollow = await ActorFollowModel.loadByActorAndTarget(follower.id, targetActor.id, t)
 
-    if (!actorFollow) throw new Error(`'Unknown actor follow ${actor.id} -> ${targetActor.id}.`)
+    if (!actorFollow) throw new Error(`'Unknown actor follow ${follower.id} -> ${targetActor.id}.`)
 
     await actorFollow.destroy({ transaction: t })
 

server/lib/activitypub/process/process-undo.ts

@@ -1,10 +1,8 @@
 import { ActivityAnnounce, ActivityFollow, ActivityLike, ActivityUndo, CacheFileObject } from '../../../../shared/models/activitypub'
 import { DislikeObject } from '../../../../shared/models/activitypub/objects'
-import { getActorUrl } from '../../../helpers/activitypub'
 import { retryTransactionWrapper } from '../../../helpers/database-utils'
 import { logger } from '../../../helpers/logger'
 import { sequelizeTypescript } from '../../../initializers'
-import { AccountModel } from '../../../models/account/account'
 import { AccountVideoRateModel } from '../../../models/account/account-video-rate'
 import { ActorModel } from '../../../models/activitypub/actor'
 import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
@@ -13,29 +11,27 @@ import { getOrCreateVideoAndAccountAndChannel } from '../videos'
 import { VideoShareModel } from '../../../models/video/video-share'
 import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
 
-async function processUndoActivity (activity: ActivityUndo) {
+async function processUndoActivity (activity: ActivityUndo, byActor: ActorModel) {
   const activityToUndo = activity.object
 
-  const actorUrl = getActorUrl(activity.actor)
-
   if (activityToUndo.type === 'Like') {
-    return retryTransactionWrapper(processUndoLike, actorUrl, activity)
+    return retryTransactionWrapper(processUndoLike, byActor, activity)
   }
 
   if (activityToUndo.type === 'Create') {
     if (activityToUndo.object.type === 'Dislike') {
-      return retryTransactionWrapper(processUndoDislike, actorUrl, activity)
+      return retryTransactionWrapper(processUndoDislike, byActor, activity)
     } else if (activityToUndo.object.type === 'CacheFile') {
-      return retryTransactionWrapper(processUndoCacheFile, actorUrl, activity)
+      return retryTransactionWrapper(processUndoCacheFile, byActor, activity)
     }
   }
 
   if (activityToUndo.type === 'Follow') {
-    return retryTransactionWrapper(processUndoFollow, actorUrl, activityToUndo)
+    return retryTransactionWrapper(processUndoFollow, byActor, activityToUndo)
   }
 
   if (activityToUndo.type === 'Announce') {
-    return retryTransactionWrapper(processUndoAnnounce, actorUrl, activityToUndo)
+    return retryTransactionWrapper(processUndoAnnounce, byActor, activityToUndo)
   }
 
   logger.warn('Unknown activity object type %s -> %s when undo activity.', activityToUndo.type, { activity: activity.id })
@@ -51,66 +47,63 @@ export {
 
 // ---------------------------------------------------------------------------
 
-async function processUndoLike (actorUrl: string, activity: ActivityUndo) {
+async function processUndoLike (byActor: ActorModel, activity: ActivityUndo) {
   const likeActivity = activity.object as ActivityLike
 
-  const { video } = await getOrCreateVideoAndAccountAndChannel(likeActivity.object)
+  const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: likeActivity.object })
 
   return sequelizeTypescript.transaction(async t => {
-    const byAccount = await AccountModel.loadByUrl(actorUrl, t)
+    if (!byActor.Account) throw new Error('Unknown account ' + byActor.url)
-    if (!byAccount) throw new Error('Unknown account ' + actorUrl)
 
-    const rate = await AccountVideoRateModel.load(byAccount.id, video.id, t)
+    const rate = await AccountVideoRateModel.load(byActor.Account.id, video.id, t)
-    if (!rate) throw new Error(`Unknown rate by account ${byAccount.id} for video ${video.id}.`)
+    if (!rate) throw new Error(`Unknown rate by account ${byActor.Account.id} for video ${video.id}.`)
 
     await rate.destroy({ transaction: t })
     await video.decrement('likes', { transaction: t })
 
     if (video.isOwned()) {
       // Don't resend the activity to the sender
-      const exceptions = [ byAccount.Actor ]
+      const exceptions = [ byActor ]
 
       await forwardVideoRelatedActivity(activity, t, exceptions, video)
     }
   })
 }
 
-async function processUndoDislike (actorUrl: string, activity: ActivityUndo) {
+async function processUndoDislike (byActor: ActorModel, activity: ActivityUndo) {
   const dislike = activity.object.object as DislikeObject
 
-  const { video } = await getOrCreateVideoAndAccountAndChannel(dislike.object)
+  const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislike.object })
 
   return sequelizeTypescript.transaction(async t => {
-    const byAccount = await AccountModel.loadByUrl(actorUrl, t)
+    if (!byActor.Account) throw new Error('Unknown account ' + byActor.url)
-    if (!byAccount) throw new Error('Unknown account ' + actorUrl)
 
-    const rate = await AccountVideoRateModel.load(byAccount.id, video.id, t)
+    const rate = await AccountVideoRateModel.load(byActor.Account.id, video.id, t)
-    if (!rate) throw new Error(`Unknown rate by account ${byAccount.id} for video ${video.id}.`)
+    if (!rate) throw new Error(`Unknown rate by account ${byActor.Account.id} for video ${video.id}.`)
 
     await rate.destroy({ transaction: t })
     await video.decrement('dislikes', { transaction: t })
 
     if (video.isOwned()) {
       // Don't resend the activity to the sender
-      const exceptions = [ byAccount.Actor ]
+      const exceptions = [ byActor ]
 
       await forwardVideoRelatedActivity(activity, t, exceptions, video)
     }
   })
 }
 
-async function processUndoCacheFile (actorUrl: string, activity: ActivityUndo) {
+async function processUndoCacheFile (byActor: ActorModel, activity: ActivityUndo) {
   const cacheFileObject = activity.object.object as CacheFileObject
 
-  const { video } = await getOrCreateVideoAndAccountAndChannel(cacheFileObject.object)
+  const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFileObject.object })
 
   return sequelizeTypescript.transaction(async t => {
-    const byActor = await ActorModel.loadByUrl(actorUrl)
-    if (!byActor) throw new Error('Unknown actor ' + actorUrl)
-
     const cacheFile = await VideoRedundancyModel.loadByUrl(cacheFileObject.id)
     if (!cacheFile) throw new Error('Unknown video cache ' + cacheFile.url)
 
+    if (cacheFile.actorId !== byActor.id) throw new Error('Cannot delete redundancy ' + cacheFile.url + ' of another actor.')
+
     await cacheFile.destroy()
 
     if (video.isOwned()) {
@@ -122,10 +115,9 @@ async function processUndoCacheFile (actorUrl: string, activity: ActivityUndo) {
   })
 }
 
-function processUndoFollow (actorUrl: string, followActivity: ActivityFollow) {
+function processUndoFollow (follower: ActorModel, followActivity: ActivityFollow) {
   return sequelizeTypescript.transaction(async t => {
-    const follower = await ActorModel.loadByUrl(actorUrl, t)
+    const following = await ActorModel.loadByUrlAndPopulateAccountAndChannel(followActivity.object, t)
-    const following = await ActorModel.loadByUrl(followActivity.object, t)
     const actorFollow = await ActorFollowModel.loadByActorAndTarget(follower.id, following.id, t)
 
     if (!actorFollow) throw new Error(`'Unknown actor follow ${follower.id} -> ${following.id}.`)
@@ -136,11 +128,8 @@ function processUndoFollow (actorUrl: string, followActivity: ActivityFollow) {
   })
 }
 
-function processUndoAnnounce (actorUrl: string, announceActivity: ActivityAnnounce) {
+function processUndoAnnounce (byActor: ActorModel, announceActivity: ActivityAnnounce) {
   return sequelizeTypescript.transaction(async t => {
-    const byActor = await ActorModel.loadByUrl(actorUrl, t)
-    if (!byActor) throw new Error('Unknown actor ' + actorUrl)
-
     const share = await VideoShareModel.loadByUrl(announceActivity.id, t)
     if (!share) throw new Error(`Unknown video share ${announceActivity.id}.`)
 

server/lib/activitypub/process/process-update.ts

@@ -6,27 +6,30 @@ import { sequelizeTypescript } from '../../../initializers'
 import { AccountModel } from '../../../models/account/account'
 import { ActorModel } from '../../../models/activitypub/actor'
 import { VideoChannelModel } from '../../../models/video/video-channel'
-import { fetchAvatarIfExists, getOrCreateActorAndServerAndModel, updateActorAvatarInstance, updateActorInstance } from '../actor'
+import { fetchAvatarIfExists, updateActorAvatarInstance, updateActorInstance } from '../actor'
-import { getOrCreateVideoAndAccountAndChannel, updateVideoFromAP, getOrCreateVideoChannelFromVideoObject } from '../videos'
+import { getOrCreateVideoAndAccountAndChannel, getOrCreateVideoChannelFromVideoObject, updateVideoFromAP } from '../videos'
 import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-validators/activitypub/videos'
 import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file'
 import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
 import { createCacheFile, updateCacheFile } from '../cache-file'
 
-async function processUpdateActivity (activity: ActivityUpdate) {
+async function processUpdateActivity (activity: ActivityUpdate, byActor: ActorModel) {
-  const actor = await getOrCreateActorAndServerAndModel(activity.actor)
   const objectType = activity.object.type
 
   if (objectType === 'Video') {
-    return retryTransactionWrapper(processUpdateVideo, actor, activity)
+    return retryTransactionWrapper(processUpdateVideo, byActor, activity)
   }
 
   if (objectType === 'Person' || objectType === 'Application' || objectType === 'Group') {
-    return retryTransactionWrapper(processUpdateActor, actor, activity)
+    // We need more attributes
+    const byActorFull = await ActorModel.loadByUrlAndPopulateAccountAndChannel(byActor.url)
+    return retryTransactionWrapper(processUpdateActor, byActorFull, activity)
   }
 
   if (objectType === 'CacheFile') {
-    return retryTransactionWrapper(processUpdateCacheFile, actor, activity)
+    // We need more attributes
+    const byActorFull = await ActorModel.loadByUrlAndPopulateAccountAndChannel(byActor.url)
+    return retryTransactionWrapper(processUpdateCacheFile, byActorFull, activity)
   }
 
   return undefined
@@ -48,10 +51,18 @@ async function processUpdateVideo (actor: ActorModel, activity: ActivityUpdate)
     return undefined
   }
 
-  const { video } = await getOrCreateVideoAndAccountAndChannel(videoObject.id)
+  const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoObject.id })
   const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject)
 
-  return updateVideoFromAP(video, videoObject, actor.Account, channelActor.VideoChannel, activity.to)
+  const updateOptions = {
+    video,
+    videoObject,
+    account: actor.Account,
+    channel: channelActor.VideoChannel,
+    updateViews: true,
+    overrideTo: activity.to
+  }
+  return updateVideoFromAP(updateOptions)
 }
 
 async function processUpdateCacheFile (byActor: ActorModel, activity: ActivityUpdate) {
@@ -64,7 +75,7 @@ async function processUpdateCacheFile (byActor: ActorModel, activity: ActivityUp
 
   const redundancyModel = await VideoRedundancyModel.loadByUrl(cacheFileObject.id)
   if (!redundancyModel) {
-    const { video } = await getOrCreateVideoAndAccountAndChannel(cacheFileObject.id)
+    const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFileObject.id })
     return createCacheFile(cacheFileObject, video, byActor)
   }
 

server/lib/activitypub/process/process.ts

@@ -11,8 +11,9 @@ import { processLikeActivity } from './process-like'
 import { processRejectActivity } from './process-reject'
 import { processUndoActivity } from './process-undo'
 import { processUpdateActivity } from './process-update'
+import { getOrCreateActorAndServerAndModel } from '../actor'
 
-const processActivity: { [ P in ActivityType ]: (activity: Activity, inboxActor?: ActorModel) => Promise<any> } = {
+const processActivity: { [ P in ActivityType ]: (activity: Activity, byActor: ActorModel, inboxActor?: ActorModel) => Promise<any> } = {
   Create: processCreateActivity,
   Update: processUpdateActivity,
   Delete: processDeleteActivity,
@@ -25,7 +26,14 @@ const processActivity: { [ P in ActivityType ]: (activity: Activity, inboxActor?
 }
 
 async function processActivities (activities: Activity[], signatureActor?: ActorModel, inboxActor?: ActorModel) {
+  const actorsCache: { [ url: string ]: ActorModel } = {}
+
   for (const activity of activities) {
+    if (!signatureActor && [ 'Create', 'Announce', 'Like' ].indexOf(activity.type) === -1) {
+      logger.error('Cannot process activity %s (type: %s) without the actor signature.', activity.id, activity.type)
+      continue
+    }
+
     const actorUrl = getActorUrl(activity.actor)
 
     // When we fetch remote data, we don't have signature
@@ -34,6 +42,9 @@ async function processActivities (activities: Activity[], signatureActor?: Actor
       continue
     }
 
+    const byActor = signatureActor || actorsCache[actorUrl] || await getOrCreateActorAndServerAndModel(actorUrl)
+    actorsCache[actorUrl] = byActor
+
     const activityProcessor = processActivity[activity.type]
     if (activityProcessor === undefined) {
       logger.warn('Unknown activity type %s.', activity.type, { activityId: activity.id })
@@ -41,7 +52,7 @@ async function processActivities (activities: Activity[], signatureActor?: Actor
     }
 
     try {
-      await activityProcessor(activity, inboxActor)
+      await activityProcessor(activity, byActor, inboxActor)
     } catch (err) {
       logger.warn('Cannot process activity %s.', activity.type, { err })
     }

server/lib/activitypub/send/send-announce.ts

@@ -4,14 +4,14 @@ import { ActorModel } from '../../../models/activitypub/actor'
 import { VideoModel } from '../../../models/video/video'
 import { VideoShareModel } from '../../../models/video/video-share'
 import { broadcastToFollowers } from './utils'
-import { audiencify, getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience } from '../audience'
+import { audiencify, getActorsInvolvedInVideo, getAudience, getAudienceFromFollowersOf } from '../audience'
 import { logger } from '../../../helpers/logger'
 
 async function buildAnnounceWithVideoAudience (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) {
   const announcedObject = video.url
 
   const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t)
-  const audience = getObjectFollowersAudience(actorsInvolvedInVideo)
+  const audience = getAudienceFromFollowersOf(actorsInvolvedInVideo)
 
   const activity = buildAnnounceActivity(videoShare.url, byActor, announcedObject, audience)
 

server/lib/activitypub/send/send-create.ts

@@ -1,21 +1,13 @@
 import { Transaction } from 'sequelize'
 import { ActivityAudience, ActivityCreate } from '../../../../shared/models/activitypub'
 import { VideoPrivacy } from '../../../../shared/models/videos'
-import { getServerActor } from '../../../helpers/utils'
 import { ActorModel } from '../../../models/activitypub/actor'
 import { VideoModel } from '../../../models/video/video'
 import { VideoAbuseModel } from '../../../models/video/video-abuse'
 import { VideoCommentModel } from '../../../models/video/video-comment'
 import { getVideoAbuseActivityPubUrl, getVideoDislikeActivityPubUrl, getVideoViewActivityPubUrl } from '../url'
-import { broadcastToActors, broadcastToFollowers, unicastTo } from './utils'
+import { broadcastToActors, broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils'
-import {
+import { audiencify, getActorsInvolvedInVideo, getAudience, getAudienceFromFollowersOf, getVideoCommentAudience } from '../audience'
-  audiencify,
-  getActorsInvolvedInVideo,
-  getAudience,
-  getObjectFollowersAudience,
-  getVideoAudience,
-  getVideoCommentAudience
-} from '../audience'
 import { logger } from '../../../helpers/logger'
 import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
 
@@ -40,6 +32,7 @@ async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel,
 
   logger.info('Creating job to send video abuse %s.', url)
 
+  // Custom audience, we only send the abuse to the origin instance
   const audience = { to: [ video.VideoChannel.Account.Actor.url ], cc: [] }
   const createActivity = buildCreateActivity(url, byActor, videoAbuse.toActivityPubObject(), audience)
 
@@ -49,15 +42,15 @@ async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel,
 async function sendCreateCacheFile (byActor: ActorModel, fileRedundancy: VideoRedundancyModel) {
   logger.info('Creating job to send file cache of %s.', fileRedundancy.url)
 
+  const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(fileRedundancy.VideoFile.Video.id)
   const redundancyObject = fileRedundancy.toActivityPubObject()
 
-  const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(fileRedundancy.VideoFile.Video.id)
+  return sendVideoRelatedCreateActivity({
-  const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, undefined)
+    byActor,
-
+    video,
-  const audience = getVideoAudience(video, actorsInvolvedInVideo)
+    url: fileRedundancy.url,
-  const createActivity = buildCreateActivity(fileRedundancy.url, byActor, redundancyObject, audience)
+    object: redundancyObject
-
+  })
-  return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
 }
 
 async function sendCreateVideoComment (comment: VideoCommentModel, t: Transaction) {
@@ -70,6 +63,7 @@ async function sendCreateVideoComment (comment: VideoCommentModel, t: Transactio
   const commentObject = comment.toActivityPubObject(threadParentComments)
 
   const actorsInvolvedInComment = await getActorsInvolvedInVideo(comment.Video, t)
+  // Add the actor that commented too
   actorsInvolvedInComment.push(byActor)
 
   const parentsCommentActors = threadParentComments.map(c => c.Account.Actor)
@@ -78,7 +72,7 @@ async function sendCreateVideoComment (comment: VideoCommentModel, t: Transactio
   if (isOrigin) {
     audience = getVideoCommentAudience(comment, threadParentComments, actorsInvolvedInComment, isOrigin)
   } else {
-    audience = getObjectFollowersAudience(actorsInvolvedInComment.concat(parentsCommentActors))
+    audience = getAudienceFromFollowersOf(actorsInvolvedInComment.concat(parentsCommentActors))
   }
 
   const createActivity = buildCreateActivity(comment.url, byActor, commentObject, audience)
@@ -103,24 +97,14 @@ async function sendCreateView (byActor: ActorModel, video: VideoModel, t: Transa
   const url = getVideoViewActivityPubUrl(byActor, video)
   const viewActivity = buildViewActivity(byActor, video)
 
-  const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t)
+  return sendVideoRelatedCreateActivity({
-
+    // Use the server actor to send the view
-  // Send to origin
+    byActor,
-  if (video.isOwned() === false) {
+    video,
-    const audience = getVideoAudience(video, actorsInvolvedInVideo)
+    url,
-    const createActivity = buildCreateActivity(url, byActor, viewActivity, audience)
+    object: viewActivity,
-
+    transaction: t
-    return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
+  })
-  }
-
-  // Send to followers
-  const audience = getObjectFollowersAudience(actorsInvolvedInVideo)
-  const createActivity = buildCreateActivity(url, byActor, viewActivity, audience)
-
-  // Use the server actor to send the view
-  const serverActor = await getServerActor()
-  const actorsException = [ byActor ]
-  return broadcastToFollowers(createActivity, serverActor, actorsInvolvedInVideo, t, actorsException)
 }
 
 async function sendCreateDislike (byActor: ActorModel, video: VideoModel, t: Transaction) {
@@ -129,22 +113,13 @@ async function sendCreateDislike (byActor: ActorModel, video: VideoModel, t: Tra
   const url = getVideoDislikeActivityPubUrl(byActor, video)
   const dislikeActivity = buildDislikeActivity(byActor, video)
 
-  const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t)
+  return sendVideoRelatedCreateActivity({
-
+    byActor,
-  // Send to origin
+    video,
-  if (video.isOwned() === false) {
+    url,
-    const audience = getVideoAudience(video, actorsInvolvedInVideo)
+    object: dislikeActivity,
-    const createActivity = buildCreateActivity(url, byActor, dislikeActivity, audience)
+    transaction: t
-
+  })
-    return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
-  }
-
-  // Send to followers
-  const audience = getObjectFollowersAudience(actorsInvolvedInVideo)
-  const createActivity = buildCreateActivity(url, byActor, dislikeActivity, audience)
-
-  const actorsException = [ byActor ]
-  return broadcastToFollowers(createActivity, byActor, actorsInvolvedInVideo, t, actorsException)
 }
 
 function buildCreateActivity (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityCreate {
@@ -189,3 +164,19 @@ export {
   sendCreateVideoComment,
   sendCreateCacheFile
 }
+
+// ---------------------------------------------------------------------------
+
+async function sendVideoRelatedCreateActivity (options: {
+  byActor: ActorModel,
+  video: VideoModel,
+  url: string,
+  object: any,
+  transaction?: Transaction
+}) {
+  const activityBuilder = (audience: ActivityAudience) => {
+    return buildCreateActivity(options.url, options.byActor, options.object, audience)
+  }
+
+  return sendVideoRelatedActivity(activityBuilder, options)
+}

server/lib/activitypub/send/send-delete.ts

@@ -5,21 +5,22 @@ import { VideoModel } from '../../../models/video/video'
 import { VideoCommentModel } from '../../../models/video/video-comment'
 import { VideoShareModel } from '../../../models/video/video-share'
 import { getDeleteActivityPubUrl } from '../url'
-import { broadcastToActors, broadcastToFollowers, unicastTo } from './utils'
+import { broadcastToActors, broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils'
 import { audiencify, getActorsInvolvedInVideo, getVideoCommentAudience } from '../audience'
 import { logger } from '../../../helpers/logger'
 
-async function sendDeleteVideo (video: VideoModel, t: Transaction) {
+async function sendDeleteVideo (video: VideoModel, transaction: Transaction) {
   logger.info('Creating job to broadcast delete of video %s.', video.url)
 
-  const url = getDeleteActivityPubUrl(video.url)
   const byActor = video.VideoChannel.Account.Actor
 
-  const activity = buildDeleteActivity(url, video.url, byActor)
+  const activityBuilder = (audience: ActivityAudience) => {
+    const url = getDeleteActivityPubUrl(video.url)
 
-  const actorsInvolved = await getActorsInvolvedInVideo(video, t)
+    return buildDeleteActivity(url, video.url, byActor, audience)
+  }
 
-  return broadcastToFollowers(activity, byActor, actorsInvolved, t)
+  return sendVideoRelatedActivity(activityBuilder, { byActor, video, transaction })
 }
 
 async function sendDeleteActor (byActor: ActorModel, t: Transaction) {

server/lib/activitypub/send/send-like.ts

@@ -3,31 +3,20 @@ import { ActivityAudience, ActivityLike } from '../../../../shared/models/activi
 import { ActorModel } from '../../../models/activitypub/actor'
 import { VideoModel } from '../../../models/video/video'
 import { getVideoLikeActivityPubUrl } from '../url'
-import { broadcastToFollowers, unicastTo } from './utils'
+import { sendVideoRelatedActivity } from './utils'
-import { audiencify, getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience, getVideoAudience } from '../audience'
+import { audiencify, getAudience } from '../audience'
 import { logger } from '../../../helpers/logger'
 
 async function sendLike (byActor: ActorModel, video: VideoModel, t: Transaction) {
   logger.info('Creating job to like %s.', video.url)
 
-  const url = getVideoLikeActivityPubUrl(byActor, video)
+  const activityBuilder = (audience: ActivityAudience) => {
+    const url = getVideoLikeActivityPubUrl(byActor, video)
 
-  const accountsInvolvedInVideo = await getActorsInvolvedInVideo(video, t)
+    return buildLikeActivity(url, byActor, video, audience)
-
-  // Send to origin
-  if (video.isOwned() === false) {
-    const audience = getVideoAudience(video, accountsInvolvedInVideo)
-    const data = buildLikeActivity(url, byActor, video, audience)
-
-    return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
   }
 
-  // Send to followers
+  return sendVideoRelatedActivity(activityBuilder, { byActor, video, transaction: t })
-  const audience = getObjectFollowersAudience(accountsInvolvedInVideo)
-  const activity = buildLikeActivity(url, byActor, video, audience)
-
-  const followersException = [ byActor ]
-  return broadcastToFollowers(activity, byActor, accountsInvolvedInVideo, t, followersException)
 }
 
 function buildLikeActivity (url: string, byActor: ActorModel, video: VideoModel, audience?: ActivityAudience): ActivityLike {

server/lib/activitypub/send/send-undo.ts

@@ -11,8 +11,8 @@ import { ActorModel } from '../../../models/activitypub/actor'
 import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
 import { VideoModel } from '../../../models/video/video'
 import { getActorFollowActivityPubUrl, getUndoActivityPubUrl, getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from '../url'
-import { broadcastToFollowers, unicastTo } from './utils'
+import { broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils'
-import { audiencify, getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience, getVideoAudience } from '../audience'
+import { audiencify, getAudience } from '../audience'
 import { buildCreateActivity, buildDislikeActivity } from './send-create'
 import { buildFollowActivity } from './send-follow'
 import { buildLikeActivity } from './send-like'
@@ -39,53 +39,6 @@ async function sendUndoFollow (actorFollow: ActorFollowModel, t: Transaction) {
   return unicastTo(undoActivity, me, following.inboxUrl)
 }
 
-async function sendUndoLike (byActor: ActorModel, video: VideoModel, t: Transaction) {
-  logger.info('Creating job to undo a like of video %s.', video.url)
-
-  const likeUrl = getVideoLikeActivityPubUrl(byActor, video)
-  const undoUrl = getUndoActivityPubUrl(likeUrl)
-
-  const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t)
-  const likeActivity = buildLikeActivity(likeUrl, byActor, video)
-
-  // Send to origin
-  if (video.isOwned() === false) {
-    const audience = getVideoAudience(video, actorsInvolvedInVideo)
-    const undoActivity = undoActivityData(undoUrl, byActor, likeActivity, audience)
-
-    return unicastTo(undoActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
-  }
-
-  const audience = getObjectFollowersAudience(actorsInvolvedInVideo)
-  const undoActivity = undoActivityData(undoUrl, byActor, likeActivity, audience)
-
-  const followersException = [ byActor ]
-  return broadcastToFollowers(undoActivity, byActor, actorsInvolvedInVideo, t, followersException)
-}
-
-async function sendUndoDislike (byActor: ActorModel, video: VideoModel, t: Transaction) {
-  logger.info('Creating job to undo a dislike of video %s.', video.url)
-
-  const dislikeUrl = getVideoDislikeActivityPubUrl(byActor, video)
-  const undoUrl = getUndoActivityPubUrl(dislikeUrl)
-
-  const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t)
-  const dislikeActivity = buildDislikeActivity(byActor, video)
-  const createDislikeActivity = buildCreateActivity(dislikeUrl, byActor, dislikeActivity)
-
-  if (video.isOwned() === false) {
-    const audience = getVideoAudience(video, actorsInvolvedInVideo)
-    const undoActivity = undoActivityData(undoUrl, byActor, createDislikeActivity, audience)
-
-    return unicastTo(undoActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
-  }
-
-  const undoActivity = undoActivityData(undoUrl, byActor, createDislikeActivity)
-
-  const followersException = [ byActor ]
-  return broadcastToFollowers(undoActivity, byActor, actorsInvolvedInVideo, t, followersException)
-}
-
 async function sendUndoAnnounce (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) {
   logger.info('Creating job to undo announce %s.', videoShare.url)
 
@@ -98,20 +51,32 @@ async function sendUndoAnnounce (byActor: ActorModel, videoShare: VideoShareMode
   return broadcastToFollowers(undoActivity, byActor, actorsInvolvedInVideo, t, followersException)
 }
 
+async function sendUndoLike (byActor: ActorModel, video: VideoModel, t: Transaction) {
+  logger.info('Creating job to undo a like of video %s.', video.url)
+
+  const likeUrl = getVideoLikeActivityPubUrl(byActor, video)
+  const likeActivity = buildLikeActivity(likeUrl, byActor, video)
+
+  return sendUndoVideoRelatedActivity({ byActor, video, url: likeUrl, activity: likeActivity, transaction: t })
+}
+
+async function sendUndoDislike (byActor: ActorModel, video: VideoModel, t: Transaction) {
+  logger.info('Creating job to undo a dislike of video %s.', video.url)
+
+  const dislikeUrl = getVideoDislikeActivityPubUrl(byActor, video)
+  const dislikeActivity = buildDislikeActivity(byActor, video)
+  const createDislikeActivity = buildCreateActivity(dislikeUrl, byActor, dislikeActivity)
+
+  return sendUndoVideoRelatedActivity({ byActor, video, url: dislikeUrl, activity: createDislikeActivity, transaction: t })
+}
+
 async function sendUndoCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel, t: Transaction) {
   logger.info('Creating job to undo cache file %s.', redundancyModel.url)
 
-  const undoUrl = getUndoActivityPubUrl(redundancyModel.url)
-
   const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id)
-  const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t)
-
-  const audience = getVideoAudience(video, actorsInvolvedInVideo)
   const createActivity = buildCreateActivity(redundancyModel.url, byActor, redundancyModel.toActivityPubObject())
 
-  const undoActivity = undoActivityData(undoUrl, byActor, createActivity, audience)
+  return sendUndoVideoRelatedActivity({ byActor, video, url: redundancyModel.url, activity: createActivity, transaction: t })
-
-  return unicastTo(undoActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
 }
 
 // ---------------------------------------------------------------------------
@@ -144,3 +109,19 @@ function undoActivityData (
     audience
   )
 }
+
+async function sendUndoVideoRelatedActivity (options: {
+  byActor: ActorModel,
+  video: VideoModel,
+  url: string,
+  activity: ActivityFollow | ActivityLike | ActivityCreate | ActivityAnnounce,
+  transaction: Transaction
+}) {
+  const activityBuilder = (audience: ActivityAudience) => {
+    const undoUrl = getUndoActivityPubUrl(options.url)
+
+    return undoActivityData(undoUrl, options.byActor, options.activity, audience)
+  }
+
+  return sendVideoRelatedActivity(activityBuilder, options)
+}

server/lib/activitypub/send/send-update.ts

@@ -7,8 +7,8 @@ import { VideoModel } from '../../../models/video/video'
 import { VideoChannelModel } from '../../../models/video/video-channel'
 import { VideoShareModel } from '../../../models/video/video-share'
 import { getUpdateActivityPubUrl } from '../url'
-import { broadcastToFollowers, unicastTo } from './utils'
+import { broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils'
-import { audiencify, getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience } from '../audience'
+import { audiencify, getActorsInvolvedInVideo, getAudience, getAudienceFromFollowersOf } from '../audience'
 import { logger } from '../../../helpers/logger'
 import { VideoCaptionModel } from '../../../models/video/video-caption'
 import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
@@ -61,16 +61,16 @@ async function sendUpdateActor (accountOrChannel: AccountModel | VideoChannelMod
 async function sendUpdateCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel) {
   logger.info('Creating job to update cache file %s.', redundancyModel.url)
 
-  const url = getUpdateActivityPubUrl(redundancyModel.url, redundancyModel.updatedAt.toISOString())
   const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id)
 
-  const redundancyObject = redundancyModel.toActivityPubObject()
+  const activityBuilder = (audience: ActivityAudience) => {
+    const redundancyObject = redundancyModel.toActivityPubObject()
+    const url = getUpdateActivityPubUrl(redundancyModel.url, redundancyModel.updatedAt.toISOString())
 
-  const accountsInvolvedInVideo = await getActorsInvolvedInVideo(video, undefined)
+    return buildUpdateActivity(url, byActor, redundancyObject, audience)
-  const audience = getObjectFollowersAudience(accountsInvolvedInVideo)
+  }
 
-  const updateActivity = buildUpdateActivity(url, byActor, redundancyObject, audience)
+  return sendVideoRelatedActivity(activityBuilder, { byActor, video })
-  return unicastTo(updateActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
 }
 
 // ---------------------------------------------------------------------------

server/lib/activitypub/send/utils.ts

@@ -1,13 +1,36 @@
 import { Transaction } from 'sequelize'
-import { Activity } from '../../../../shared/models/activitypub'
+import { Activity, ActivityAudience } from '../../../../shared/models/activitypub'
 import { logger } from '../../../helpers/logger'
 import { ActorModel } from '../../../models/activitypub/actor'
 import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
 import { JobQueue } from '../../job-queue'
 import { VideoModel } from '../../../models/video/video'
-import { getActorsInvolvedInVideo } from '../audience'
+import { getActorsInvolvedInVideo, getAudienceFromFollowersOf, getRemoteVideoAudience } from '../audience'
 import { getServerActor } from '../../../helpers/utils'
 
+async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAudience) => Activity, options: {
+  byActor: ActorModel,
+  video: VideoModel,
+  transaction?: Transaction
+}) {
+  const actorsInvolvedInVideo = await getActorsInvolvedInVideo(options.video, options.transaction)
+
+  // Send to origin
+  if (options.video.isOwned() === false) {
+    const audience = getRemoteVideoAudience(options.video, actorsInvolvedInVideo)
+    const activity = activityBuilder(audience)
+
+    return unicastTo(activity, options.byActor, options.video.VideoChannel.Account.Actor.sharedInboxUrl)
+  }
+
+  // Send to followers
+  const audience = getAudienceFromFollowersOf(actorsInvolvedInVideo)
+  const activity = activityBuilder(audience)
+
+  const actorsException = [ options.byActor ]
+  return broadcastToFollowers(activity, options.byActor, actorsInvolvedInVideo, options.transaction, actorsException)
+}
+
 async function forwardVideoRelatedActivity (
   activity: Activity,
   t: Transaction,
@@ -110,7 +133,8 @@ export {
   unicastTo,
   forwardActivity,
   broadcastToActors,
-  forwardVideoRelatedActivity
+  forwardVideoRelatedActivity,
+  sendVideoRelatedActivity
 }
 
 // ---------------------------------------------------------------------------

server/lib/activitypub/video-comments.ts

@@ -94,7 +94,7 @@ async function resolveThread (url: string, comments: VideoCommentModel[] = []) {
   try {
     // Maybe it's a reply to a video?
     // If yes, it's done: we resolved all the thread
-    const { video } = await getOrCreateVideoAndAccountAndChannel(url)
+    const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: url })
 
     if (comments.length !== 0) {
       const firstReply = comments[ comments.length - 1 ]

server/lib/activitypub/videos.ts

@@ -3,7 +3,7 @@ import * as sequelize from 'sequelize'
 import * as magnetUtil from 'magnet-uri'
 import { join } from 'path'
 import * as request from 'request'
-import { ActivityIconObject, ActivityVideoUrlObject, VideoState, ActivityUrlObject } from '../../../shared/index'
+import { ActivityIconObject, ActivityUrlObject, ActivityVideoUrlObject, VideoState } from '../../../shared/index'
 import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
 import { VideoPrivacy } from '../../../shared/models/videos'
 import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos'
@@ -28,6 +28,7 @@ import { ActivitypubHttpFetcherPayload } from '../job-queue/handlers/activitypub
 import { createRates } from './video-rates'
 import { addVideoShares, shareVideoByServerAndChannel } from './share'
 import { AccountModel } from '../../models/account/account'
+import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video'
 
 async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) {
   // If the video is not private and published, we federate it
@@ -50,18 +51,29 @@ async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, tr
   }
 }
 
-function fetchRemoteVideoStaticFile (video: VideoModel, path: string, reject: Function) {
+async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.RequestResponse, videoObject: VideoTorrentObject }> {
-  const host = video.VideoChannel.Account.Actor.Server.host
+  const options = {
+    uri: videoUrl,
+    method: 'GET',
+    json: true,
+    activityPub: true
+  }
 
-  // We need to provide a callback, if no we could have an uncaught exception
+  logger.info('Fetching remote video %s.', videoUrl)
-  return request.get(REMOTE_SCHEME.HTTP + '://' + host + path, err => {
+
-    if (err) reject(err)
+  const { response, body } = await doRequest(options)
-  })
+
+  if (sanitizeAndCheckVideoTorrentObject(body) === false) {
+    logger.debug('Remote video JSON is not valid.', { body })
+    return { response, videoObject: undefined }
+  }
+
+  return { response, videoObject: body }
 }
 
 async function fetchRemoteVideoDescription (video: VideoModel) {
   const host = video.VideoChannel.Account.Actor.Server.host
-  const path = video.getDescriptionPath()
+  const path = video.getDescriptionAPIPath()
   const options = {
     uri: REMOTE_SCHEME.HTTP + '://' + host + path,
     json: true
@@ -71,6 +83,15 @@ async function fetchRemoteVideoDescription (video: VideoModel) {
   return body.description ? body.description : ''
 }
 
+function fetchRemoteVideoStaticFile (video: VideoModel, path: string, reject: Function) {
+  const host = video.VideoChannel.Account.Actor.Server.host
+
+  // We need to provide a callback, if no we could have an uncaught exception
+  return request.get(REMOTE_SCHEME.HTTP + '://' + host + path, err => {
+    if (err) reject(err)
+  })
+}
+
 function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject) {
   const thumbnailName = video.getThumbnailName()
   const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName)
@@ -82,6 +103,293 @@ function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject)
   return doRequestAndSaveToFile(options, thumbnailPath)
 }
 
+function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) {
+  const channel = videoObject.attributedTo.find(a => a.type === 'Group')
+  if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url)
+
+  return getOrCreateActorAndServerAndModel(channel.id, 'all')
+}
+
+type SyncParam = {
+  likes: boolean
+  dislikes: boolean
+  shares: boolean
+  comments: boolean
+  thumbnail: boolean
+  refreshVideo: boolean
+}
+async function syncVideoExternalAttributes (video: VideoModel, fetchedVideo: VideoTorrentObject, syncParam: SyncParam) {
+  logger.info('Adding likes/dislikes/shares/comments of video %s.', video.uuid)
+
+  const jobPayloads: ActivitypubHttpFetcherPayload[] = []
+
+  if (syncParam.likes === true) {
+    await crawlCollectionPage<string>(fetchedVideo.likes, items => createRates(items, video, 'like'))
+      .catch(err => logger.error('Cannot add likes of video %s.', video.uuid, { err }))
+  } else {
+    jobPayloads.push({ uri: fetchedVideo.likes, videoId: video.id, type: 'video-likes' as 'video-likes' })
+  }
+
+  if (syncParam.dislikes === true) {
+    await crawlCollectionPage<string>(fetchedVideo.dislikes, items => createRates(items, video, 'dislike'))
+      .catch(err => logger.error('Cannot add dislikes of video %s.', video.uuid, { err }))
+  } else {
+    jobPayloads.push({ uri: fetchedVideo.dislikes, videoId: video.id, type: 'video-dislikes' as 'video-dislikes' })
+  }
+
+  if (syncParam.shares === true) {
+    await crawlCollectionPage<string>(fetchedVideo.shares, items => addVideoShares(items, video))
+      .catch(err => logger.error('Cannot add shares of video %s.', video.uuid, { err }))
+  } else {
+    jobPayloads.push({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' as 'video-shares' })
+  }
+
+  if (syncParam.comments === true) {
+    await crawlCollectionPage<string>(fetchedVideo.comments, items => addVideoComments(items, video))
+      .catch(err => logger.error('Cannot add comments of video %s.', video.uuid, { err }))
+  } else {
+    jobPayloads.push({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' as 'video-shares' })
+  }
+
+  await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }))
+}
+
+async function getOrCreateVideoAndAccountAndChannel (options: {
+  videoObject: VideoTorrentObject | string,
+  syncParam?: SyncParam,
+  fetchType?: VideoFetchByUrlType,
+  refreshViews?: boolean
+}) {
+  // Default params
+  const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false }
+  const fetchType = options.fetchType || 'all'
+  const refreshViews = options.refreshViews || false
+
+  // Get video url
+  const videoUrl = typeof options.videoObject === 'string' ? options.videoObject : options.videoObject.id
+
+  let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType)
+  if (videoFromDatabase) {
+    const refreshOptions = {
+      video: videoFromDatabase,
+      fetchedType: fetchType,
+      syncParam,
+      refreshViews
+    }
+    const p = retryTransactionWrapper(refreshVideoIfNeeded, refreshOptions)
+    if (syncParam.refreshVideo === true) videoFromDatabase = await p
+
+    return { video: videoFromDatabase }
+  }
+
+  const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl)
+  if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl)
+
+  const channelActor = await getOrCreateVideoChannelFromVideoObject(fetchedVideo)
+  const video = await retryTransactionWrapper(createVideo, fetchedVideo, channelActor, syncParam.thumbnail)
+
+  await syncVideoExternalAttributes(video, fetchedVideo, syncParam)
+
+  return { video }
+}
+
+async function updateVideoFromAP (options: {
+  video: VideoModel,
+  videoObject: VideoTorrentObject,
+  account: AccountModel,
+  channel: VideoChannelModel,
+  updateViews: boolean,
+  overrideTo?: string[]
+}) {
+  logger.debug('Updating remote video "%s".', options.videoObject.uuid)
+  let videoFieldsSave: any
+
+  try {
+    const updatedVideo: VideoModel = await sequelizeTypescript.transaction(async t => {
+      const sequelizeOptions = {
+        transaction: t
+      }
+
+      videoFieldsSave = options.video.toJSON()
+
+      // Check actor has the right to update the video
+      const videoChannel = options.video.VideoChannel
+      if (videoChannel.Account.id !== options.account.id) {
+        throw new Error('Account ' + options.account.Actor.url + ' does not own video channel ' + videoChannel.Actor.url)
+      }
+
+      const to = options.overrideTo ? options.overrideTo : options.videoObject.to
+      const videoData = await videoActivityObjectToDBAttributes(options.channel, options.videoObject, to)
+      options.video.set('name', videoData.name)
+      options.video.set('uuid', videoData.uuid)
+      options.video.set('url', videoData.url)
+      options.video.set('category', videoData.category)
+      options.video.set('licence', videoData.licence)
+      options.video.set('language', videoData.language)
+      options.video.set('description', videoData.description)
+      options.video.set('support', videoData.support)
+      options.video.set('nsfw', videoData.nsfw)
+      options.video.set('commentsEnabled', videoData.commentsEnabled)
+      options.video.set('waitTranscoding', videoData.waitTranscoding)
+      options.video.set('state', videoData.state)
+      options.video.set('duration', videoData.duration)
+      options.video.set('createdAt', videoData.createdAt)
+      options.video.set('publishedAt', videoData.publishedAt)
+      options.video.set('privacy', videoData.privacy)
+      options.video.set('channelId', videoData.channelId)
+
+      if (options.updateViews === true) options.video.set('views', videoData.views)
+      await options.video.save(sequelizeOptions)
+
+      // Don't block on request
+      generateThumbnailFromUrl(options.video, options.videoObject.icon)
+        .catch(err => logger.warn('Cannot generate thumbnail of %s.', options.videoObject.id, { err }))
+
+      // Remove old video files
+      const videoFileDestroyTasks: Bluebird<void>[] = []
+      for (const videoFile of options.video.VideoFiles) {
+        videoFileDestroyTasks.push(videoFile.destroy(sequelizeOptions))
+      }
+      await Promise.all(videoFileDestroyTasks)
+
+      const videoFileAttributes = videoFileActivityUrlToDBAttributes(options.video, options.videoObject)
+      const tasks = videoFileAttributes.map(f => VideoFileModel.create(f, sequelizeOptions))
+      await Promise.all(tasks)
+
+      // Update Tags
+      const tags = options.videoObject.tag.map(tag => tag.name)
+      const tagInstances = await TagModel.findOrCreateTags(tags, t)
+      await options.video.$set('Tags', tagInstances, sequelizeOptions)
+
+      // Update captions
+      await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(options.video.id, t)
+
+      const videoCaptionsPromises = options.videoObject.subtitleLanguage.map(c => {
+        return VideoCaptionModel.insertOrReplaceLanguage(options.video.id, c.identifier, t)
+      })
+      await Promise.all(videoCaptionsPromises)
+    })
+
+    logger.info('Remote video with uuid %s updated', options.videoObject.uuid)
+
+    return updatedVideo
+  } catch (err) {
+    if (options.video !== undefined && videoFieldsSave !== undefined) {
+      resetSequelizeInstance(options.video, videoFieldsSave)
+    }
+
+    // This is just a debug because we will retry the insert
+    logger.debug('Cannot update the remote video.', { err })
+    throw err
+  }
+}
+
+export {
+  updateVideoFromAP,
+  federateVideoIfNeeded,
+  fetchRemoteVideo,
+  getOrCreateVideoAndAccountAndChannel,
+  fetchRemoteVideoStaticFile,
+  fetchRemoteVideoDescription,
+  generateThumbnailFromUrl,
+  getOrCreateVideoChannelFromVideoObject
+}
+
+// ---------------------------------------------------------------------------
+
+function isActivityVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject {
+  const mimeTypes = Object.keys(VIDEO_MIMETYPE_EXT)
+
+  return mimeTypes.indexOf(url.mimeType) !== -1 && url.mimeType.startsWith('video/')
+}
+
+async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) {
+  logger.debug('Adding remote video %s.', videoObject.id)
+
+  const videoCreated: VideoModel = await sequelizeTypescript.transaction(async t => {
+    const sequelizeOptions = { transaction: t }
+
+    const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to)
+    const video = VideoModel.build(videoData)
+
+    const videoCreated = await video.save(sequelizeOptions)
+
+    // Process files
+    const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject)
+    if (videoFileAttributes.length === 0) {
+      throw new Error('Cannot find valid files for video %s ' + videoObject.url)
+    }
+
+    const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t }))
+    await Promise.all(videoFilePromises)
+
+    // Process tags
+    const tags = videoObject.tag.map(t => t.name)
+    const tagInstances = await TagModel.findOrCreateTags(tags, t)
+    await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
+
+    // Process captions
+    const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
+      return VideoCaptionModel.insertOrReplaceLanguage(videoCreated.id, c.identifier, t)
+    })
+    await Promise.all(videoCaptionsPromises)
+
+    logger.info('Remote video with uuid %s inserted.', videoObject.uuid)
+
+    videoCreated.VideoChannel = channelActor.VideoChannel
+    return videoCreated
+  })
+
+  const p = generateThumbnailFromUrl(videoCreated, videoObject.icon)
+    .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err }))
+
+  if (waitThumbnail === true) await p
+
+  return videoCreated
+}
+
+async function refreshVideoIfNeeded (options: {
+  video: VideoModel,
+  fetchedType: VideoFetchByUrlType,
+  syncParam: SyncParam,
+  refreshViews: boolean
+}): Promise<VideoModel> {
+  if (!options.video.isOutdated()) return options.video
+
+  // We need more attributes if the argument video was fetched with not enough joints
+  const video = options.fetchedType === 'all' ? options.video : await VideoModel.loadByUrlAndPopulateAccount(options.video.url)
+
+  try {
+    const { response, videoObject } = await fetchRemoteVideo(video.url)
+    if (response.statusCode === 404) {
+      // Video does not exist anymore
+      await video.destroy()
+      return undefined
+    }
+
+    if (videoObject === undefined) {
+      logger.warn('Cannot refresh remote video: invalid body.')
+      return video
+    }
+
+    const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject)
+    const account = await AccountModel.load(channelActor.VideoChannel.accountId)
+
+    const updateOptions = {
+      video,
+      videoObject,
+      account,
+      channel: channelActor.VideoChannel,
+      updateViews: options.refreshViews
+    }
+    await updateVideoFromAP(updateOptions)
+    await syncVideoExternalAttributes(video, videoObject, options.syncParam)
+  } catch (err) {
+    logger.warn('Cannot refresh video.', { err })
+    return video
+  }
+}
+
 async function videoActivityObjectToDBAttributes (
   videoChannel: VideoChannelModel,
   videoObject: VideoTorrentObject,
@@ -169,282 +477,3 @@ function videoFileActivityUrlToDBAttributes (videoCreated: VideoModel, videoObje
 
   return attributes
 }
-
-function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) {
-  const channel = videoObject.attributedTo.find(a => a.type === 'Group')
-  if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url)
-
-  return getOrCreateActorAndServerAndModel(channel.id)
-}
-
-async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) {
-  logger.debug('Adding remote video %s.', videoObject.id)
-
-  const videoCreated: VideoModel = await sequelizeTypescript.transaction(async t => {
-    const sequelizeOptions = { transaction: t }
-
-    const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to)
-    const video = VideoModel.build(videoData)
-
-    const videoCreated = await video.save(sequelizeOptions)
-
-    // Process files
-    const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject)
-    if (videoFileAttributes.length === 0) {
-      throw new Error('Cannot find valid files for video %s ' + videoObject.url)
-    }
-
-    const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t }))
-    await Promise.all(videoFilePromises)
-
-    // Process tags
-    const tags = videoObject.tag.map(t => t.name)
-    const tagInstances = await TagModel.findOrCreateTags(tags, t)
-    await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
-
-    // Process captions
-    const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
-      return VideoCaptionModel.insertOrReplaceLanguage(videoCreated.id, c.identifier, t)
-    })
-    await Promise.all(videoCaptionsPromises)
-
-    logger.info('Remote video with uuid %s inserted.', videoObject.uuid)
-
-    videoCreated.VideoChannel = channelActor.VideoChannel
-    return videoCreated
-  })
-
-  const p = generateThumbnailFromUrl(videoCreated, videoObject.icon)
-    .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err }))
-
-  if (waitThumbnail === true) await p
-
-  return videoCreated
-}
-
-type SyncParam = {
-  likes: boolean
-  dislikes: boolean
-  shares: boolean
-  comments: boolean
-  thumbnail: boolean
-  refreshVideo: boolean
-}
-async function getOrCreateVideoAndAccountAndChannel (
-  videoObject: VideoTorrentObject | string,
-  syncParam: SyncParam = { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false }
-) {
-  const videoUrl = typeof videoObject === 'string' ? videoObject : videoObject.id
-
-  let videoFromDatabase = await VideoModel.loadByUrlAndPopulateAccount(videoUrl)
-  if (videoFromDatabase) {
-    const p = retryTransactionWrapper(refreshVideoIfNeeded, videoFromDatabase)
-    if (syncParam.refreshVideo === true) videoFromDatabase = await p
-
-    return { video: videoFromDatabase }
-  }
-
-  const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl)
-  if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl)
-
-  const channelActor = await getOrCreateVideoChannelFromVideoObject(fetchedVideo)
-  const video = await retryTransactionWrapper(createVideo, fetchedVideo, channelActor, syncParam.thumbnail)
-
-  // Process outside the transaction because we could fetch remote data
-
-  logger.info('Adding likes/dislikes/shares/comments of video %s.', video.uuid)
-
-  const jobPayloads: ActivitypubHttpFetcherPayload[] = []
-
-  if (syncParam.likes === true) {
-    await crawlCollectionPage<string>(fetchedVideo.likes, items => createRates(items, video, 'like'))
-      .catch(err => logger.error('Cannot add likes of video %s.', video.uuid, { err }))
-  } else {
-    jobPayloads.push({ uri: fetchedVideo.likes, videoId: video.id, type: 'video-likes' as 'video-likes' })
-  }
-
-  if (syncParam.dislikes === true) {
-    await crawlCollectionPage<string>(fetchedVideo.dislikes, items => createRates(items, video, 'dislike'))
-      .catch(err => logger.error('Cannot add dislikes of video %s.', video.uuid, { err }))
-  } else {
-    jobPayloads.push({ uri: fetchedVideo.dislikes, videoId: video.id, type: 'video-dislikes' as 'video-dislikes' })
-  }
-
-  if (syncParam.shares === true) {
-    await crawlCollectionPage<string>(fetchedVideo.shares, items => addVideoShares(items, video))
-      .catch(err => logger.error('Cannot add shares of video %s.', video.uuid, { err }))
-  } else {
-    jobPayloads.push({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' as 'video-shares' })
-  }
-
-  if (syncParam.comments === true) {
-    await crawlCollectionPage<string>(fetchedVideo.comments, items => addVideoComments(items, video))
-      .catch(err => logger.error('Cannot add comments of video %s.', video.uuid, { err }))
-  } else {
-    jobPayloads.push({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' as 'video-shares' })
-  }
-
-  await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }))
-
-  return { video }
-}
-
-async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.RequestResponse, videoObject: VideoTorrentObject }> {
-  const options = {
-    uri: videoUrl,
-    method: 'GET',
-    json: true,
-    activityPub: true
-  }
-
-  logger.info('Fetching remote video %s.', videoUrl)
-
-  const { response, body } = await doRequest(options)
-
-  if (sanitizeAndCheckVideoTorrentObject(body) === false) {
-    logger.debug('Remote video JSON is not valid.', { body })
-    return { response, videoObject: undefined }
-  }
-
-  return { response, videoObject: body }
-}
-
-async function refreshVideoIfNeeded (video: VideoModel): Promise<VideoModel> {
-  if (!video.isOutdated()) return video
-
-  try {
-    const { response, videoObject } = await fetchRemoteVideo(video.url)
-    if (response.statusCode === 404) {
-      // Video does not exist anymore
-      await video.destroy()
-      return undefined
-    }
-
-    if (videoObject === undefined) {
-      logger.warn('Cannot refresh remote video: invalid body.')
-      return video
-    }
-
-    const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject)
-    const account = await AccountModel.load(channelActor.VideoChannel.accountId)
-
-    return updateVideoFromAP(video, videoObject, account, channelActor.VideoChannel)
-  } catch (err) {
-    logger.warn('Cannot refresh video.', { err })
-    return video
-  }
-}
-
-async function updateVideoFromAP (
-  video: VideoModel,
-  videoObject: VideoTorrentObject,
-  account: AccountModel,
-  channel: VideoChannelModel,
-  overrideTo?: string[]
-) {
-  logger.debug('Updating remote video "%s".', videoObject.uuid)
-  let videoFieldsSave: any
-
-  try {
-    const updatedVideo: VideoModel = await sequelizeTypescript.transaction(async t => {
-      const sequelizeOptions = {
-        transaction: t
-      }
-
-      videoFieldsSave = video.toJSON()
-
-      // Check actor has the right to update the video
-      const videoChannel = video.VideoChannel
-      if (videoChannel.Account.id !== account.id) {
-        throw new Error('Account ' + account.Actor.url + ' does not own video channel ' + videoChannel.Actor.url)
-      }
-
-      const to = overrideTo ? overrideTo : videoObject.to
-      const videoData = await videoActivityObjectToDBAttributes(channel, videoObject, to)
-      video.set('name', videoData.name)
-      video.set('uuid', videoData.uuid)
-      video.set('url', videoData.url)
-      video.set('category', videoData.category)
-      video.set('licence', videoData.licence)
-      video.set('language', videoData.language)
-      video.set('description', videoData.description)
-      video.set('support', videoData.support)
-      video.set('nsfw', videoData.nsfw)
-      video.set('commentsEnabled', videoData.commentsEnabled)
-      video.set('waitTranscoding', videoData.waitTranscoding)
-      video.set('state', videoData.state)
-      video.set('duration', videoData.duration)
-      video.set('createdAt', videoData.createdAt)
-      video.set('publishedAt', videoData.publishedAt)
-      video.set('views', videoData.views)
-      video.set('privacy', videoData.privacy)
-      video.set('channelId', videoData.channelId)
-
-      await video.save(sequelizeOptions)
-
-      // Don't block on request
-      generateThumbnailFromUrl(video, videoObject.icon)
-        .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err }))
-
-      // Remove old video files
-      const videoFileDestroyTasks: Bluebird<void>[] = []
-      for (const videoFile of video.VideoFiles) {
-        videoFileDestroyTasks.push(videoFile.destroy(sequelizeOptions))
-      }
-      await Promise.all(videoFileDestroyTasks)
-
-      const videoFileAttributes = videoFileActivityUrlToDBAttributes(video, videoObject)
-      const tasks = videoFileAttributes.map(f => VideoFileModel.create(f, sequelizeOptions))
-      await Promise.all(tasks)
-
-      // Update Tags
-      const tags = videoObject.tag.map(tag => tag.name)
-      const tagInstances = await TagModel.findOrCreateTags(tags, t)
-      await video.$set('Tags', tagInstances, sequelizeOptions)
-
-      // Update captions
-      await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(video.id, t)
-
-      const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
-        return VideoCaptionModel.insertOrReplaceLanguage(video.id, c.identifier, t)
-      })
-      await Promise.all(videoCaptionsPromises)
-    })
-
-    logger.info('Remote video with uuid %s updated', videoObject.uuid)
-
-    return updatedVideo
-  } catch (err) {
-    if (video !== undefined && videoFieldsSave !== undefined) {
-      resetSequelizeInstance(video, videoFieldsSave)
-    }
-
-    // This is just a debug because we will retry the insert
-    logger.debug('Cannot update the remote video.', { err })
-    throw err
-  }
-}
-
-export {
-  updateVideoFromAP,
-  federateVideoIfNeeded,
-  fetchRemoteVideo,
-  getOrCreateVideoAndAccountAndChannel,
-  fetchRemoteVideoStaticFile,
-  fetchRemoteVideoDescription,
-  generateThumbnailFromUrl,
-  videoActivityObjectToDBAttributes,
-  videoFileActivityUrlToDBAttributes,
-  createVideo,
-  getOrCreateVideoChannelFromVideoObject,
-  addVideoShares,
-  createRates
-}
-
-// ---------------------------------------------------------------------------
-
-function isActivityVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject {
-  const mimeTypes = Object.keys(VIDEO_MIMETYPE_EXT)
-
-  return mimeTypes.indexOf(url.mimeType) !== -1 && url.mimeType.startsWith('video/')
-}

server/lib/avatar.ts

@@ -3,23 +3,18 @@ import { sendUpdateActor } from './activitypub/send'
 import { AVATARS_SIZE, CONFIG, sequelizeTypescript } from '../initializers'
 import { updateActorAvatarInstance } from './activitypub'
 import { processImage } from '../helpers/image-utils'
-import { ActorModel } from '../models/activitypub/actor'
 import { AccountModel } from '../models/account/account'
 import { VideoChannelModel } from '../models/video/video-channel'
 import { extname, join } from 'path'
 
-async function updateActorAvatarFile (
+async function updateActorAvatarFile (avatarPhysicalFile: Express.Multer.File, accountOrChannel: AccountModel | VideoChannelModel) {
-  avatarPhysicalFile: Express.Multer.File,
-  actor: ActorModel,
-  accountOrChannel: AccountModel | VideoChannelModel
-) {
   const extension = extname(avatarPhysicalFile.filename)
-  const avatarName = actor.uuid + extension
+  const avatarName = accountOrChannel.Actor.uuid + extension
   const destination = join(CONFIG.STORAGE.AVATARS_DIR, avatarName)
   await processImage(avatarPhysicalFile, destination, AVATARS_SIZE)
 
   return sequelizeTypescript.transaction(async t => {
-    const updatedActor = await updateActorAvatarInstance(actor, avatarName, t)
+    const updatedActor = await updateActorAvatarInstance(accountOrChannel.Actor, avatarName, t)
     await updatedActor.save({ transaction: t })
 
     await sendUpdateActor(accountOrChannel, t)

server/lib/cache/videos-caption-cache.ts

@@ -38,7 +38,7 @@ class VideosCaptionCache extends AbstractVideoStaticFileCache <GetPathParam> {
     if (videoCaption.isOwned()) throw new Error('Cannot load remote caption of owned video.')
 
     // Used to fetch the path
-    const video = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(videoId)
+    const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId)
     if (!video) return undefined
 
     const remoteStaticPath = videoCaption.getCaptionStaticPath()

server/lib/cache/videos-preview-cache.ts

@@ -16,7 +16,7 @@ class VideosPreviewCache extends AbstractVideoStaticFileCache <string> {
   }
 
   async getFilePath (videoUUID: string) {
-    const video = await VideoModel.loadByUUID(videoUUID)
+    const video = await VideoModel.loadByUUIDWithFile(videoUUID)
     if (!video) return undefined
 
     if (video.isOwned()) return join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName())
@@ -25,7 +25,7 @@ class VideosPreviewCache extends AbstractVideoStaticFileCache <string> {
   }
 
   protected async loadRemoteFile (key: string) {
-    const video = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(key)
+    const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(key)
     if (!video) return undefined
 
     if (video.isOwned()) throw new Error('Cannot load remote preview of owned video.')

server/lib/client-html.ts

@@ -8,6 +8,7 @@ import { VideoModel } from '../models/video/video'
 import * as validator from 'validator'
 import { VideoPrivacy } from '../../shared/models/videos'
 import { readFile } from 'fs-extra'
+import { getActivityStreamDuration } from '../models/video/video-format-utils'
 
 export class ClientHtml {
 
@@ -38,10 +39,8 @@ export class ClientHtml {
     let videoPromise: Bluebird<VideoModel>
 
     // Let Angular application handle errors
-    if (validator.isUUID(videoId, 4)) {
+    if (validator.isInt(videoId) || validator.isUUID(videoId, 4)) {
-      videoPromise = VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(videoId)
+      videoPromise = VideoModel.loadAndPopulateAccountAndServerAndTags(videoId)
-    } else if (validator.isInt(videoId)) {
-      videoPromise = VideoModel.loadAndPopulateAccountAndServerAndTags(+videoId)
     } else {
       return ClientHtml.getIndexHTML(req, res)
     }
@@ -150,7 +149,7 @@ export class ClientHtml {
       description: videoDescriptionEscaped,
       thumbnailUrl: previewUrl,
       uploadDate: video.createdAt.toISOString(),
-      duration: video.getActivityStreamDuration(),
+      duration: getActivityStreamDuration(video.duration),
       contentUrl: videoUrl,
       embedUrl: embedUrl,
       interactionCount: video.views

server/lib/job-queue/handlers/activitypub-http-fetcher.ts

@@ -1,10 +1,10 @@
 import * as Bull from 'bull'
 import { logger } from '../../../helpers/logger'
 import { processActivities } from '../../activitypub/process'
-import { VideoModel } from '../../../models/video/video'
-import { addVideoShares, createRates } from '../../activitypub/videos'
 import { addVideoComments } from '../../activitypub/video-comments'
 import { crawlCollectionPage } from '../../activitypub/crawl'
+import { VideoModel } from '../../../models/video/video'
+import { addVideoShares, createRates } from '../../activitypub'
 
 type FetchType = 'activity' | 'video-likes' | 'video-dislikes' | 'video-shares' | 'video-comments'
 

server/lib/job-queue/handlers/video-file.ts

@@ -8,6 +8,7 @@ import { retryTransactionWrapper } from '../../../helpers/database-utils'
 import { sequelizeTypescript } from '../../../initializers'
 import * as Bluebird from 'bluebird'
 import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg-utils'
+import { importVideoFile, transcodeOriginalVideofile, optimizeOriginalVideofile } from '../../video-transcoding'
 
 export type VideoFilePayload = {
   videoUUID: string
@@ -25,14 +26,14 @@ async function processVideoFileImport (job: Bull.Job) {
   const payload = job.data as VideoFileImportPayload
   logger.info('Processing video file import in job %d.', job.id)
 
-  const video = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(payload.videoUUID)
+  const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(payload.videoUUID)
   // No video, maybe deleted?
   if (!video) {
     logger.info('Do not process job %d, video does not exist.', job.id)
     return undefined
   }
 
-  await video.importVideoFile(payload.filePath)
+  await importVideoFile(video, payload.filePath)
 
   await onVideoFileTranscoderOrImportSuccess(video)
   return video
@@ -42,7 +43,7 @@ async function processVideoFile (job: Bull.Job) {
   const payload = job.data as VideoFilePayload
   logger.info('Processing video file in job %d.', job.id)
 
-  const video = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(payload.videoUUID)
+  const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(payload.videoUUID)
   // No video, maybe deleted?
   if (!video) {
     logger.info('Do not process job %d, video does not exist.', job.id)
@@ -51,11 +52,11 @@ async function processVideoFile (job: Bull.Job) {
 
   // Transcoding in other resolution
   if (payload.resolution) {
-    await video.transcodeOriginalVideofile(payload.resolution, payload.isPortraitMode || false)
+    await transcodeOriginalVideofile(video, payload.resolution, payload.isPortraitMode || false)
 
     await retryTransactionWrapper(onVideoFileTranscoderOrImportSuccess, video)
   } else {
-    await video.optimizeOriginalVideofile()
+    await optimizeOriginalVideofile(video)
 
     await retryTransactionWrapper(onVideoFileOptimizerSuccess, video, payload.isNewVideo)
   }
@@ -68,7 +69,7 @@ async function onVideoFileTranscoderOrImportSuccess (video: VideoModel) {
 
   return sequelizeTypescript.transaction(async t => {
     // Maybe the video changed in database, refresh it
-    let videoDatabase = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(video.uuid, t)
+    let videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t)
     // Video does not exist anymore
     if (!videoDatabase) return undefined
 
@@ -98,7 +99,7 @@ async function onVideoFileOptimizerSuccess (video: VideoModel, isNewVideo: boole
 
   return sequelizeTypescript.transaction(async t => {
     // Maybe the video changed in database, refresh it
-    const videoDatabase = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(video.uuid, t)
+    const videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t)
     // Video does not exist anymore
     if (!videoDatabase) return undefined
 

server/lib/job-queue/handlers/video-import.ts

@@ -183,7 +183,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: Vide
       const videoUpdated = await video.save({ transaction: t })
 
       // Now we can federate the video (reload from database, we need more attributes)
-      const videoForFederation = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(video.uuid, t)
+      const videoForFederation = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t)
       await federateVideoIfNeeded(videoForFederation, true, t)
 
       // Update video import object

server/lib/oauth-model.ts

@@ -4,15 +4,50 @@ import { UserModel } from '../models/account/user'
 import { OAuthClientModel } from '../models/oauth/oauth-client'
 import { OAuthTokenModel } from '../models/oauth/oauth-token'
 import { CONFIG } from '../initializers/constants'
+import { Transaction } from 'sequelize'
 
 type TokenInfo = { accessToken: string, refreshToken: string, accessTokenExpiresAt: Date, refreshTokenExpiresAt: Date }
+const accessTokenCache: { [ accessToken: string ]: OAuthTokenModel } = {}
+const userHavingToken: { [ userId: number ]: string } = {}
 
 // ---------------------------------------------------------------------------
 
+function deleteUserToken (userId: number, t?: Transaction) {
+  clearCacheByUserId(userId)
+
+  return OAuthTokenModel.deleteUserToken(userId, t)
+}
+
+function clearCacheByUserId (userId: number) {
+  const token = userHavingToken[userId]
+  if (token !== undefined) {
+    accessTokenCache[ token ] = undefined
+    userHavingToken[ userId ] = undefined
+  }
+}
+
+function clearCacheByToken (token: string) {
+  const tokenModel = accessTokenCache[ token ]
+  if (tokenModel !== undefined) {
+    userHavingToken[tokenModel.userId] = undefined
+    accessTokenCache[ token ] = undefined
+  }
+}
+
 function getAccessToken (bearerToken: string) {
   logger.debug('Getting access token (bearerToken: ' + bearerToken + ').')
 
+  if (accessTokenCache[bearerToken] !== undefined) return accessTokenCache[bearerToken]
+
   return OAuthTokenModel.getByTokenAndPopulateUser(bearerToken)
+    .then(tokenModel => {
+      if (tokenModel) {
+        accessTokenCache[ bearerToken ] = tokenModel
+        userHavingToken[ tokenModel.userId ] = tokenModel.accessToken
+      }
+
+      return tokenModel
+    })
 }
 
 function getClient (clientId: string, clientSecret: string) {
@@ -48,6 +83,8 @@ async function getUser (usernameOrEmail: string, password: string) {
 async function revokeToken (tokenInfo: TokenInfo) {
   const token = await OAuthTokenModel.getByRefreshTokenAndPopulateUser(tokenInfo.refreshToken)
   if (token) {
+    clearCacheByToken(token.accessToken)
+
     token.destroy()
          .catch(err => logger.error('Cannot destroy token when revoking token.', { err }))
   }
@@ -85,6 +122,9 @@ async function saveToken (token: TokenInfo, client: OAuthClientModel, user: User
 
 // See https://github.com/oauthjs/node-oauth2-server/wiki/Model-specification for the model specifications
 export {
+  deleteUserToken,
+  clearCacheByUserId,
+  clearCacheByToken,
   getAccessToken,
   getClient,
   getRefreshToken,

server/lib/schedulers/videos-redundancy-scheduler.ts

@@ -1,10 +1,9 @@
 import { AbstractScheduler } from './abstract-scheduler'
 import { CONFIG, JOB_TTL, REDUNDANCY, SCHEDULER_INTERVALS_MS } from '../../initializers'
 import { logger } from '../../helpers/logger'
-import { VideoRedundancyStrategy } from '../../../shared/models/redundancy'
+import { VideoRedundancyStrategy, VideosRedundancy } from '../../../shared/models/redundancy'
 import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
 import { VideoFileModel } from '../../models/video/video-file'
-import { sortBy } from 'lodash'
 import { downloadWebTorrentVideo } from '../../helpers/webtorrent'
 import { join } from 'path'
 import { rename } from 'fs-extra'
@@ -12,7 +11,6 @@ import { getServerActor } from '../../helpers/utils'
 import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send'
 import { VideoModel } from '../../models/video/video'
 import { getVideoCacheFileActivityPubUrl } from '../activitypub/url'
-import { removeVideoRedundancy } from '../redundancy'
 import { isTestInstance } from '../../helpers/core-utils'
 
 export class VideosRedundancyScheduler extends AbstractScheduler {
@@ -20,7 +18,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
   private static instance: AbstractScheduler
   private executing = false
 
-  protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.videosRedundancy
+  protected schedulerIntervalMs = CONFIG.REDUNDANCY.VIDEOS.CHECK_INTERVAL
 
   private constructor () {
     super()
@@ -31,17 +29,15 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
 
     this.executing = true
 
-    for (const obj of CONFIG.REDUNDANCY.VIDEOS) {
+    for (const obj of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) {
-
       try {
-        const videoToDuplicate = await this.findVideoToDuplicate(obj.strategy)
+        const videoToDuplicate = await this.findVideoToDuplicate(obj)
         if (!videoToDuplicate) continue
 
         const videoFiles = videoToDuplicate.VideoFiles
         videoFiles.forEach(f => f.Video = videoToDuplicate)
 
-        const videosRedundancy = await VideoRedundancyModel.getVideoFiles(obj.strategy)
+        if (await this.isTooHeavy(obj.strategy, videoFiles, obj.size)) {
-        if (this.isTooHeavy(videosRedundancy, videoFiles, obj.size)) {
           if (!isTestInstance()) logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url)
           continue
         }
@@ -54,6 +50,16 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
       }
     }
 
+    await this.removeExpired()
+
+    this.executing = false
+  }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+
+  private async removeExpired () {
     const expired = await VideoRedundancyModel.listAllExpired()
 
     for (const m of expired) {
@@ -65,16 +71,21 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
         logger.error('Cannot remove %s video from our redundancy system.', this.buildEntryLogId(m))
       }
     }
-
-    this.executing = false
   }
 
-  static get Instance () {
+  private findVideoToDuplicate (cache: VideosRedundancy) {
-    return this.instance || (this.instance = new this())
+    if (cache.strategy === 'most-views') {
-  }
+      return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR)
+    }
 
-  private findVideoToDuplicate (strategy: VideoRedundancyStrategy) {
+    if (cache.strategy === 'trending') {
-    if (strategy === 'most-views') return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR)
+      return VideoRedundancyModel.findTrendingToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR)
+    }
+
+    if (cache.strategy === 'recently-added') {
+      const minViews = cache.minViews
+      return VideoRedundancyModel.findRecentlyAddedToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR, minViews)
+    }
   }
 
   private async createVideoRedundancy (strategy: VideoRedundancyStrategy, filesToDuplicate: VideoFileModel[]) {
@@ -120,27 +131,10 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
     }
   }
 
-  // Unused, but could be useful in the future, with a custom strategy
+  private async isTooHeavy (strategy: VideoRedundancyStrategy, filesToDuplicate: VideoFileModel[], maxSizeArg: number) {
-  private async purgeVideosIfNeeded (videosRedundancy: VideoRedundancyModel[], filesToDuplicate: VideoFileModel[], maxSize: number) {
-    const sortedVideosRedundancy = sortBy(videosRedundancy, 'createdAt')
-
-    while (this.isTooHeavy(sortedVideosRedundancy, filesToDuplicate, maxSize)) {
-      const toDelete = sortedVideosRedundancy.shift()
-
-      const videoFile = toDelete.VideoFile
-      logger.info('Purging video %s (resolution %d) from our redundancy system.', videoFile.Video.url, videoFile.resolution)
-
-      await removeVideoRedundancy(toDelete, undefined)
-    }
-
-    return sortedVideosRedundancy
-  }
-
-  private isTooHeavy (videosRedundancy: VideoRedundancyModel[], filesToDuplicate: VideoFileModel[], maxSizeArg: number) {
     const maxSize = maxSizeArg - this.getTotalFileSizes(filesToDuplicate)
 
-    const redundancyReducer = (previous: number, current: VideoRedundancyModel) => previous + current.VideoFile.size
+    const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(strategy)
-    const totalDuplicated = videosRedundancy.reduce(redundancyReducer, 0)
 
     return totalDuplicated > maxSize
   }