Speed up overviews route

package.json

@@ -111,6 +111,7 @@
     "jsonld-signatures": "https://github.com/Chocobozzz/jsonld-signatures#rsa2017",
     "lodash": "^4.17.10",
     "magnet-uri": "^5.1.4",
+    "memoizee": "^0.4.14",
     "morgan": "^1.5.3",
     "multer": "^1.1.0",
     "nodemailer": "^4.4.2",
@@ -158,6 +159,7 @@
     "@types/lodash": "^4.14.64",
     "@types/magnet-uri": "^5.1.1",
     "@types/maildev": "^0.0.1",
+    "@types/memoizee": "^0.4.2",
     "@types/mkdirp": "^0.5.1",
     "@types/mocha": "^5.0.0",
     "@types/morgan": "^1.7.32",

server/controllers/api/overviews.ts

@@ -4,8 +4,9 @@ import { VideoModel } from '../../models/video/video'
 import { asyncMiddleware } from '../../middlewares'
 import { TagModel } from '../../models/video/tag'
 import { VideosOverview } from '../../../shared/models/overviews'
-import { OVERVIEWS, ROUTE_CACHE_LIFETIME } from '../../initializers'
+import { MEMOIZE_TTL, OVERVIEWS, ROUTE_CACHE_LIFETIME } from '../../initializers'
 import { cacheRoute } from '../../middlewares/cache'
+import * as memoizee from 'memoizee'
 
 const overviewsRouter = express.Router()
 
@@ -23,10 +24,17 @@ export { overviewsRouter }
 // This endpoint could be quite long, but we cache it
 async function getVideosOverview (req: express.Request, res: express.Response) {
   const attributes = await buildSamples()
+
+  const [ categories, channels, tags ] = await Promise.all([
+    Promise.all(attributes.categories.map(c => getVideosByCategory(c, res))),
+    Promise.all(attributes.channels.map(c => getVideosByChannel(c, res))),
+    Promise.all(attributes.tags.map(t => getVideosByTag(t, res)))
+  ])
+
   const result: VideosOverview = {
-    categories: await Promise.all(attributes.categories.map(c => getVideosByCategory(c, res))),
+    categories,
-    channels: await Promise.all(attributes.channels.map(c => getVideosByChannel(c, res))),
+    channels,
-    tags: await Promise.all(attributes.tags.map(t => getVideosByTag(t, res)))
+    tags
   }
 
   // Cleanup our object
@@ -37,7 +45,7 @@ async function getVideosOverview (req: express.Request, res: express.Response) {
   return res.json(result)
 }
 
-async function buildSamples () {
+const buildSamples = memoizee(async function () {
   const [ categories, channels, tags ] = await Promise.all([
     VideoModel.getRandomFieldSamples('category', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT),
     VideoModel.getRandomFieldSamples('channelId', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD ,OVERVIEWS.VIDEOS.SAMPLES_COUNT),
@@ -45,7 +53,7 @@ async function buildSamples () {
   ])
 
   return { categories, channels, tags }
-}
+}, { maxAge: MEMOIZE_TTL.OVERVIEWS_SAMPLE })
 
 async function getVideosByTag (tag: string, res: express.Response) {
   const videos = await getVideos(res, { tagsOneOf: [ tag ] })
@@ -84,14 +92,16 @@ async function getVideos (
   res: express.Response,
   where: { videoChannelId?: number, tagsOneOf?: string[], categoryOneOf?: number[] }
 ) {
-  const { data } = await VideoModel.listForApi(Object.assign({
+  const query = Object.assign({
     start: 0,
     count: 10,
     sort: '-createdAt',
     includeLocalVideos: true,
     nsfw: buildNSFWFilter(res),
     withFiles: false
-  }, where))
+  }, where)
+
+  const { data } = await VideoModel.listForApi(query, false)
 
   return data.map(d => d.toFormattedJSON())
 }

server/helpers/utils.ts

@@ -1,12 +1,12 @@
 import { ResultList } from '../../shared'
 import { CONFIG } from '../initializers'
-import { ActorModel } from '../models/activitypub/actor'
 import { ApplicationModel } from '../models/application/application'
 import { pseudoRandomBytesPromise, sha256 } from './core-utils'
 import { logger } from './logger'
 import { join } from 'path'
 import { Instance as ParseTorrent } from 'parse-torrent'
 import { remove } from 'fs-extra'
+import * as memoizee from 'memoizee'
 
 function deleteFileAsync (path: string) {
   remove(path)
@@ -36,24 +36,12 @@ function getFormattedObjects<U, T extends FormattableToJSON> (objects: T[], obje
   } as ResultList<U>
 }
 
-async function getServerActor () {
+const getServerActor = memoizee(async function () {
-  if (getServerActor.serverActor === undefined) {
+  const application = await ApplicationModel.load()
-    const application = await ApplicationModel.load()
+  if (!application) throw Error('Could not load Application from database.')
-    if (!application) throw Error('Could not load Application from database.')
 
-    getServerActor.serverActor = application.Account.Actor
+  return application.Account.Actor
-  }
+})
-
-  if (!getServerActor.serverActor) {
-    logger.error('Cannot load server actor.')
-    process.exit(0)
-  }
-
-  return Promise.resolve(getServerActor.serverActor)
-}
-namespace getServerActor {
-  export let serverActor: ActorModel
-}
 
 function generateVideoTmpPath (target: string | ParseTorrent) {
   const id = typeof target === 'string' ? target : target.infoHash

server/initializers/constants.ts

@@ -592,6 +592,10 @@ const CACHE = {
   }
 }
 
+const MEMOIZE_TTL = {
+  OVERVIEWS_SAMPLE: 1000 * 3600 * 4 // 4 hours
+}
+
 const REDUNDANCY = {
   VIDEOS: {
     EXPIRES_AFTER_MS: 48 * 3600 * 1000, // 2 days
@@ -708,6 +712,7 @@ export {
   VIDEO_ABUSE_STATES,
   JOB_REQUEST_TIMEOUT,
   USER_PASSWORD_RESET_LIFETIME,
+  MEMOIZE_TTL,
   USER_EMAIL_VERIFY_LIFETIME,
   IMAGE_MIMETYPE_EXT,
   OVERVIEWS,

server/lib/schedulers/videos-redundancy-scheduler.ts

@@ -81,7 +81,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
     }
 
     if (cache.strategy === 'recently-added') {
-      const minViews = (cache as RecentlyAddedStrategy).minViews
+      const minViews = cache.minViews
       return VideoRedundancyModel.findRecentlyAddedToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR, minViews)
     }
   }

server/models/video/video.ts

@@ -929,7 +929,7 @@ export class VideoModel extends Model<VideoModel> {
     videoChannelId?: number,
     actorId?: number
     trendingDays?: number
-  }) {
+  }, countVideos = true) {
     const query: IFindOptions<VideoModel> = {
       offset: options.start,
       limit: options.count,
@@ -962,7 +962,7 @@ export class VideoModel extends Model<VideoModel> {
       trendingDays
     }
 
-    return VideoModel.getAvailableForApi(query, queryOptions)
+    return VideoModel.getAvailableForApi(query, queryOptions, countVideos)
   }
 
   static async searchAndPopulateAccountAndServer (options: {
@@ -1164,7 +1164,14 @@ export class VideoModel extends Model<VideoModel> {
   }
 
   // threshold corresponds to how many video the field should have to be returned
-  static getRandomFieldSamples (field: 'category' | 'channelId', threshold: number, count: number) {
+  static async getRandomFieldSamples (field: 'category' | 'channelId', threshold: number, count: number) {
+    const actorId = (await getServerActor()).id
+
+    const scopeOptions = {
+      actorId,
+      includeLocalVideos: true
+    }
+
     const query: IFindOptions<VideoModel> = {
       attributes: [ field ],
       limit: count,
@@ -1172,17 +1179,11 @@ export class VideoModel extends Model<VideoModel> {
       having: Sequelize.where(Sequelize.fn('COUNT', Sequelize.col(field)), {
         [ Sequelize.Op.gte ]: threshold
       }) as any, // FIXME: typings
-      where: {
-        [ field ]: {
-          [ Sequelize.Op.not ]: null
-        },
-        privacy: VideoPrivacy.PUBLIC,
-        state: VideoState.PUBLISHED
-      },
       order: [ this.sequelize.random() ]
     }
 
-    return VideoModel.findAll(query)
+    return VideoModel.scope({ method: [ ScopeNames.AVAILABLE_FOR_LIST_IDS, scopeOptions ] })
+                     .findAll(query)
                      .then(rows => rows.map(r => r[ field ]))
   }
 
@@ -1210,7 +1211,7 @@ export class VideoModel extends Model<VideoModel> {
     return {}
   }
 
-  private static async getAvailableForApi (query: IFindOptions<VideoModel>, options: AvailableForListIDsOptions) {
+  private static async getAvailableForApi (query: IFindOptions<VideoModel>, options: AvailableForListIDsOptions, countVideos = true) {
     const idsScope = {
       method: [
         ScopeNames.AVAILABLE_FOR_LIST_IDS, options
@@ -1227,7 +1228,7 @@ export class VideoModel extends Model<VideoModel> {
     }
 
     const [ count, rowsId ] = await Promise.all([
-      VideoModel.scope(countScope).count(countQuery),
+      countVideos ? VideoModel.scope(countScope).count(countQuery) : Promise.resolve(undefined),
       VideoModel.scope(idsScope).findAll(query)
     ])
     const ids = rowsId.map(r => r.id)

yarn.lock

@@ -160,6 +160,10 @@
   dependencies:
     "@types/node" "*"
 
+"@types/memoizee@^0.4.2":
+  version "0.4.2"
+  resolved "https://registry.yarnpkg.com/@types/memoizee/-/memoizee-0.4.2.tgz#a500158999a8144a9b46cf9a9fb49b15f1853573"
+
 "@types/mime@*":
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/@types/mime/-/mime-2.0.0.tgz#5a7306e367c539b9f6543499de8dd519fac37a8b"
@@ -2058,7 +2062,7 @@ error@^7.0.0:
     string-template "~0.2.1"
     xtend "~4.0.0"
 
-es5-ext@^0.10.14, es5-ext@^0.10.35, es5-ext@^0.10.9, es5-ext@~0.10.14:
+es5-ext@^0.10.14, es5-ext@^0.10.35, es5-ext@^0.10.45, es5-ext@^0.10.9, es5-ext@~0.10.14, es5-ext@~0.10.2:
   version "0.10.46"
   resolved "https://registry.yarnpkg.com/es5-ext/-/es5-ext-0.10.46.tgz#efd99f67c5a7ec789baa3daa7f79870388f7f572"
   dependencies:
@@ -2110,7 +2114,7 @@ es6-symbol@3.1.1, es6-symbol@^3.1.1, es6-symbol@~3.1.1:
     d "1"
     es5-ext "~0.10.14"
 
-es6-weak-map@^2.0.1:
+es6-weak-map@^2.0.1, es6-weak-map@^2.0.2:
   version "2.0.2"
   resolved "https://registry.yarnpkg.com/es6-weak-map/-/es6-weak-map-2.0.2.tgz#5e3ab32251ffd1538a1f8e5ffa1357772f92d96f"
   dependencies:
@@ -2223,7 +2227,7 @@ etag@~1.8.1:
   version "1.8.1"
   resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887"
 
-event-emitter@~0.3.5:
+event-emitter@^0.3.5, event-emitter@~0.3.5:
   version "0.3.5"
   resolved "https://registry.yarnpkg.com/event-emitter/-/event-emitter-0.3.5.tgz#df8c69eef1647923c7157b9ce83840610b02cc39"
   dependencies:
@@ -3757,7 +3761,7 @@ is-plain-object@^2.0.1, is-plain-object@^2.0.3, is-plain-object@^2.0.4:
   dependencies:
     isobject "^3.0.1"
 
-is-promise@^2.1.0:
+is-promise@^2.1, is-promise@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/is-promise/-/is-promise-2.1.0.tgz#79a2a9ece7f096e80f36d2b2f3bc16c1ff4bf3fa"
 
@@ -4490,6 +4494,12 @@ lru-cache@4.1.x, lru-cache@^4.0.1:
     pseudomap "^1.0.2"
     yallist "^2.1.2"
 
+lru-queue@0.1:
+  version "0.1.0"
+  resolved "https://registry.yarnpkg.com/lru-queue/-/lru-queue-0.1.0.tgz#2738bd9f0d3cf4f84490c5736c48699ac632cda3"
+  dependencies:
+    es5-ext "~0.10.2"
+
 lru@^3.0.0, lru@^3.1.0:
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/lru/-/lru-3.1.0.tgz#ea7fb8546d83733396a13091d76cfeb4c06837d5"
@@ -4594,6 +4604,19 @@ mem@^1.1.0:
   dependencies:
     mimic-fn "^1.0.0"
 
+memoizee@^0.4.14:
+  version "0.4.14"
+  resolved "https://registry.yarnpkg.com/memoizee/-/memoizee-0.4.14.tgz#07a00f204699f9a95c2d9e77218271c7cd610d57"
+  dependencies:
+    d "1"
+    es5-ext "^0.10.45"
+    es6-weak-map "^2.0.2"
+    event-emitter "^0.3.5"
+    is-promise "^2.1"
+    lru-queue "0.1"
+    next-tick "1"
+    timers-ext "^0.1.5"
+
 memory-chunk-store@^1.2.0:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/memory-chunk-store/-/memory-chunk-store-1.3.0.tgz#ae99e7e3b58b52db43d49d94722930d39459d0c4"
@@ -7201,6 +7224,13 @@ timed-out@^4.0.0:
   version "4.0.1"
   resolved "https://registry.yarnpkg.com/timed-out/-/timed-out-4.0.1.tgz#f32eacac5a175bea25d7fab565ab3ed8741ef56f"
 
+timers-ext@^0.1.5:
+  version "0.1.5"
+  resolved "https://registry.yarnpkg.com/timers-ext/-/timers-ext-0.1.5.tgz#77147dd4e76b660c2abb8785db96574cbbd12922"
+  dependencies:
+    es5-ext "~0.10.14"
+    next-tick "1"
+
 tiny-lr@^1.1.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/tiny-lr/-/tiny-lr-1.1.1.tgz#9fa547412f238fedb068ee295af8b682c98b2aab"