Implement pagination for overviews endpoint

server/controllers/api/overviews.ts

@@ -1,17 +1,18 @@
 import * as express from 'express'
 import { buildNSFWFilter } from '../../helpers/express-utils'
 import { VideoModel } from '../../models/video/video'
-import { asyncMiddleware } from '../../middlewares'
+import { asyncMiddleware, optionalAuthenticate, videosOverviewValidator } from '../../middlewares'
 import { TagModel } from '../../models/video/tag'
-import { VideosOverview } from '../../../shared/models/overviews'
+import { CategoryOverview, ChannelOverview, TagOverview, VideosOverview } from '../../../shared/models/overviews'
-import { MEMOIZE_TTL, OVERVIEWS, ROUTE_CACHE_LIFETIME } from '../../initializers/constants'
+import { MEMOIZE_TTL, OVERVIEWS } from '../../initializers/constants'
-import { cacheRoute } from '../../middlewares/cache'
 import * as memoizee from 'memoizee'
+import { logger } from '@server/helpers/logger'
 
 const overviewsRouter = express.Router()
 
 overviewsRouter.get('/videos',
-  asyncMiddleware(cacheRoute()(ROUTE_CACHE_LIFETIME.OVERVIEWS.VIDEOS)),
+  videosOverviewValidator,
+  optionalAuthenticate,
   asyncMiddleware(getVideosOverview)
 )
 
@@ -28,17 +29,28 @@ const buildSamples = memoizee(async function () {
     TagModel.getRandomSamples(OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT)
   ])
 
-  return { categories, channels, tags }
+  const result = { categories, channels, tags }
+
+  logger.debug('Building samples for overview endpoint.', { result })
+
+  return result
 }, { maxAge: MEMOIZE_TTL.OVERVIEWS_SAMPLE })
 
 // This endpoint could be quite long, but we cache it
 async function getVideosOverview (req: express.Request, res: express.Response) {
   const attributes = await buildSamples()
 
-  const [ categories, channels, tags ] = await Promise.all([
+  const page = req.query.page || 1
-    Promise.all(attributes.categories.map(c => getVideosByCategory(c, res))),
+  const index = page - 1
-    Promise.all(attributes.channels.map(c => getVideosByChannel(c, res))),
+
-    Promise.all(attributes.tags.map(t => getVideosByTag(t, res)))
+  const categories: CategoryOverview[] = []
+  const channels: ChannelOverview[] = []
+  const tags: TagOverview[] = []
+
+  await Promise.all([
+    getVideosByCategory(attributes.categories, index, res, categories),
+    getVideosByChannel(attributes.channels, index, res, channels),
+    getVideosByTag(attributes.tags, index, res, tags)
   ])
 
   const result: VideosOverview = {
@@ -47,45 +59,49 @@ async function getVideosOverview (req: express.Request, res: express.Response) {
     tags
   }
 
-  // Cleanup our object
-  for (const key of Object.keys(result)) {
-    result[key] = result[key].filter(v => v !== undefined)
-  }
-
   return res.json(result)
 }
 
-async function getVideosByTag (tag: string, res: express.Response) {
+async function getVideosByTag (tagsSample: string[], index: number, res: express.Response, acc: TagOverview[]) {
+  if (tagsSample.length <= index) return
+
+  const tag = tagsSample[index]
   const videos = await getVideos(res, { tagsOneOf: [ tag ] })
 
-  if (videos.length === 0) return undefined
+  if (videos.length === 0) return
 
-  return {
+  acc.push({
     tag,
     videos
-  }
+  })
 }
 
-async function getVideosByCategory (category: number, res: express.Response) {
+async function getVideosByCategory (categoriesSample: number[], index: number, res: express.Response, acc: CategoryOverview[]) {
+  if (categoriesSample.length <= index) return
+
+  const category = categoriesSample[index]
   const videos = await getVideos(res, { categoryOneOf: [ category ] })
 
-  if (videos.length === 0) return undefined
+  if (videos.length === 0) return
 
-  return {
+  acc.push({
     category: videos[0].category,
     videos
-  }
+  })
 }
 
-async function getVideosByChannel (channelId: number, res: express.Response) {
+async function getVideosByChannel (channelsSample: number[], index: number, res: express.Response, acc: ChannelOverview[]) {
+  if (channelsSample.length <= index) return
+
+  const channelId = channelsSample[index]
   const videos = await getVideos(res, { videoChannelId: channelId })
 
-  if (videos.length === 0) return undefined
+  if (videos.length === 0) return
 
-  return {
+  acc.push({
     channel: videos[0].channel,
     videos
-  }
+  })
 }
 
 async function getVideos (

server/initializers/constants.ts

@@ -90,9 +90,6 @@ const ROUTE_CACHE_LIFETIME = {
   SECURITYTXT: '2 hours',
   NODEINFO: '10 minutes',
   DNT_POLICY: '1 week',
-  OVERVIEWS: {
-    VIDEOS: '1 hour'
-  },
   ACTIVITY_PUB: {
     VIDEOS: '1 second' // 1 second, cache concurrent requests after a broadcast for example
   },
@@ -446,7 +443,7 @@ MIMETYPES.IMAGE.EXT_MIMETYPE = invert(MIMETYPES.IMAGE.MIMETYPE_EXT)
 const OVERVIEWS = {
   VIDEOS: {
     SAMPLE_THRESHOLD: 6,
-    SAMPLES_COUNT: 2
+    SAMPLES_COUNT: 20
   }
 }
 
@@ -687,8 +684,8 @@ if (isTestInstance() === true) {
   JOB_ATTEMPTS['email'] = 1
 
   FILES_CACHE.VIDEO_CAPTIONS.MAX_AGE = 3000
-  MEMOIZE_TTL.OVERVIEWS_SAMPLE = 1
+  MEMOIZE_TTL.OVERVIEWS_SAMPLE = 3000
-  ROUTE_CACHE_LIFETIME.OVERVIEWS.VIDEOS = '0ms'
+  OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD = 2
 }
 
 updateWebserverUrls()

server/middlewares/validators/videos/videos.ts

@@ -29,7 +29,7 @@ import {
 } from '../../../helpers/custom-validators/videos'
 import { getDurationFromVideoFile } from '../../../helpers/ffmpeg-utils'
 import { logger } from '../../../helpers/logger'
-import { CONSTRAINTS_FIELDS } from '../../../initializers/constants'
+import { CONSTRAINTS_FIELDS, OVERVIEWS } from '../../../initializers/constants'
 import { authenticatePromiseIfNeeded } from '../../oauth'
 import { areValidationErrors } from '../utils'
 import { cleanUpReqFiles } from '../../../helpers/express-utils'
@@ -301,6 +301,19 @@ const videosAcceptChangeOwnershipValidator = [
   }
 ]
 
+const videosOverviewValidator = [
+  query('page')
+    .optional()
+    .isInt({ min: 1, max: OVERVIEWS.VIDEOS.SAMPLES_COUNT })
+    .withMessage('Should have a valid pagination'),
+
+  (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    if (areValidationErrors(req, res)) return
+
+    return next()
+  }
+]
+
 function getCommonVideoEditAttributes () {
   return [
     body('thumbnailfile')
@@ -442,7 +455,9 @@ export {
 
   getCommonVideoEditAttributes,
 
-  commonVideosFiltersValidator
+  commonVideosFiltersValidator,
+
+  videosOverviewValidator
 }
 
 // ---------------------------------------------------------------------------

server/tests/api/check-params/index.ts

@@ -23,3 +23,4 @@ import './video-playlists'
 import './videos'
 import './videos-filter'
 import './videos-history'
+import './videos-overviews'

server/tests/api/check-params/videos-overviews.ts

@@ -0,0 +1,33 @@
+/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
+
+import 'mocha'
+import { cleanupTests, flushAndRunServer, ServerInfo } from '../../../../shared/extra-utils'
+import { getVideosOverview } from '@shared/extra-utils/overviews/overviews'
+
+describe('Test videos overview', function () {
+  let server: ServerInfo
+
+  // ---------------------------------------------------------------
+
+  before(async function () {
+    this.timeout(30000)
+
+    server = await flushAndRunServer(1)
+  })
+
+  describe('When getting videos overview', function () {
+
+    it('Should fail with a bad pagination', async function () {
+      await getVideosOverview(server.url, 0, 400)
+      await getVideosOverview(server.url, 100, 400)
+    })
+
+    it('Should succeed with a good pagination', async function () {
+      await getVideosOverview(server.url, 1)
+    })
+  })
+
+  after(async function () {
+    await cleanupTests([ server ])
+  })
+})

server/tests/api/videos/video-nsfw.ts

@@ -19,12 +19,20 @@ import {
   updateCustomConfig,
   updateMyUser
 } from '../../../../shared/extra-utils'
-import { ServerConfig } from '../../../../shared/models'
+import { ServerConfig, VideosOverview } from '../../../../shared/models'
 import { CustomConfig } from '../../../../shared/models/server/custom-config.model'
 import { User } from '../../../../shared/models/users'
+import { getVideosOverview, getVideosOverviewWithToken } from '@shared/extra-utils/overviews/overviews'
 
 const expect = chai.expect
 
+function createOverviewRes (res: any) {
+  const overview = res.body as VideosOverview
+
+  const videos = overview.categories[0].videos
+  return { body: { data: videos, total: videos.length } }
+}
+
 describe('Test video NSFW policy', function () {
   let server: ServerInfo
   let userAccessToken: string
@@ -36,22 +44,38 @@ describe('Test video NSFW policy', function () {
         const user: User = res.body
         const videoChannelName = user.videoChannels[0].name
         const accountName = user.account.name + '@' + user.account.host
+        const hasQuery = Object.keys(query).length !== 0
+        let promises: Promise<any>[]
 
         if (token) {
-          return Promise.all([
+          promises = [
             getVideosListWithToken(server.url, token, query),
             searchVideoWithToken(server.url, 'n', token, query),
             getAccountVideos(server.url, token, accountName, 0, 5, undefined, query),
             getVideoChannelVideos(server.url, token, videoChannelName, 0, 5, undefined, query)
-          ])
+          ]
+
+          // Overviews do not support video filters
+          if (!hasQuery) {
+            promises.push(getVideosOverviewWithToken(server.url, 1, token).then(res => createOverviewRes(res)))
+          }
+
+          return Promise.all(promises)
         }
 
-        return Promise.all([
+        promises = [
           getVideosList(server.url),
           searchVideo(server.url, 'n'),
           getAccountVideos(server.url, undefined, accountName, 0, 5),
           getVideoChannelVideos(server.url, undefined, videoChannelName, 0, 5)
-        ])
+        ]
+
+        // Overviews do not support video filters
+        if (!hasQuery) {
+          promises.push(getVideosOverview(server.url, 1).then(res => createOverviewRes(res)))
+        }
+
+        return Promise.all(promises)
       })
   }
 
@@ -63,12 +87,12 @@ describe('Test video NSFW policy', function () {
     await setAccessTokensToServers([ server ])
 
     {
-      const attributes = { name: 'nsfw', nsfw: true }
+      const attributes = { name: 'nsfw', nsfw: true, category: 1 }
       await uploadVideo(server.url, server.accessToken, attributes)
     }
 
     {
-      const attributes = { name: 'normal', nsfw: false }
+      const attributes = { name: 'normal', nsfw: false, category: 1 }
       await uploadVideo(server.url, server.accessToken, attributes)
     }
 

server/tests/api/videos/videos-overview.ts

@@ -2,7 +2,7 @@
 
 import * as chai from 'chai'
 import 'mocha'
-import { cleanupTests, flushAndRunServer, ServerInfo, setAccessTokensToServers, uploadVideo } from '../../../../shared/extra-utils'
+import { cleanupTests, flushAndRunServer, ServerInfo, setAccessTokensToServers, uploadVideo, wait } from '../../../../shared/extra-utils'
 import { getVideosOverview } from '../../../../shared/extra-utils/overviews/overviews'
 import { VideosOverview } from '../../../../shared/models/overviews'
 
@@ -20,7 +20,7 @@ describe('Test a videos overview', function () {
   })
 
   it('Should send empty overview', async function () {
-    const res = await getVideosOverview(server.url)
+    const res = await getVideosOverview(server.url, 1)
 
     const overview: VideosOverview = res.body
     expect(overview.tags).to.have.lengthOf(0)
@@ -31,15 +31,15 @@ describe('Test a videos overview', function () {
   it('Should upload 5 videos in a specific category, tag and channel but not include them in overview', async function () {
     this.timeout(15000)
 
-    for (let i = 0; i < 5; i++) {
+    await wait(3000)
-      await uploadVideo(server.url, server.accessToken, {
-        name: 'video ' + i,
-        category: 3,
-        tags: [ 'coucou1', 'coucou2' ]
-      })
-    }
 
-    const res = await getVideosOverview(server.url)
+    await uploadVideo(server.url, server.accessToken, {
+      name: 'video 0',
+      category: 3,
+      tags: [ 'coucou1', 'coucou2' ]
+    })
+
+    const res = await getVideosOverview(server.url, 1)
 
     const overview: VideosOverview = res.body
     expect(overview.tags).to.have.lengthOf(0)
@@ -48,27 +48,55 @@ describe('Test a videos overview', function () {
   })
 
   it('Should upload another video and include all videos in the overview', async function () {
-    await uploadVideo(server.url, server.accessToken, {
+    this.timeout(15000)
-      name: 'video 5',
-      category: 3,
-      tags: [ 'coucou1', 'coucou2' ]
-    })
 
-    const res = await getVideosOverview(server.url)
+    for (let i = 1; i < 6; i++) {
+      await uploadVideo(server.url, server.accessToken, {
+        name: 'video ' + i,
+        category: 3,
+        tags: [ 'coucou1', 'coucou2' ]
+      })
+    }
 
-    const overview: VideosOverview = res.body
+    await wait(3000)
-    expect(overview.tags).to.have.lengthOf(2)
+
-    expect(overview.categories).to.have.lengthOf(1)
+    {
-    expect(overview.channels).to.have.lengthOf(1)
+      const res = await getVideosOverview(server.url, 1)
+
+      const overview: VideosOverview = res.body
+      expect(overview.tags).to.have.lengthOf(1)
+      expect(overview.categories).to.have.lengthOf(1)
+      expect(overview.channels).to.have.lengthOf(1)
+    }
+
+    {
+      const res = await getVideosOverview(server.url, 2)
+
+      const overview: VideosOverview = res.body
+      expect(overview.tags).to.have.lengthOf(1)
+      expect(overview.categories).to.have.lengthOf(0)
+      expect(overview.channels).to.have.lengthOf(0)
+    }
   })
 
   it('Should have the correct overview', async function () {
-    const res = await getVideosOverview(server.url)
+    const res1 = await getVideosOverview(server.url, 1)
+    const res2 = await getVideosOverview(server.url, 2)
 
-    const overview: VideosOverview = res.body
+    const overview1: VideosOverview = res1.body
+    const overview2: VideosOverview = res2.body
 
-    for (const attr of [ 'tags', 'categories', 'channels' ]) {
+    const tmp = [
-      const obj = overview[attr][0]
+      overview1.tags,
+      overview1.categories,
+      overview1.channels,
+      overview2.tags
+    ]
+
+    for (const arr of tmp) {
+      expect(arr).to.have.lengthOf(1)
+
+      const obj = arr[0]
 
       expect(obj.videos).to.have.lengthOf(6)
       expect(obj.videos[0].name).to.equal('video 5')
@@ -79,12 +107,13 @@ describe('Test a videos overview', function () {
       expect(obj.videos[5].name).to.equal('video 0')
     }
 
-    expect(overview.tags.find(t => t.tag === 'coucou1')).to.not.be.undefined
+    const tags = [ overview1.tags[0].tag, overview2.tags[0].tag ]
-    expect(overview.tags.find(t => t.tag === 'coucou2')).to.not.be.undefined
+    expect(tags.find(t => t === 'coucou1')).to.not.be.undefined
+    expect(tags.find(t => t === 'coucou2')).to.not.be.undefined
 
-    expect(overview.categories[0].category.id).to.equal(3)
+    expect(overview1.categories[0].category.id).to.equal(3)
 
-    expect(overview.channels[0].channel.name).to.equal('root_channel')
+    expect(overview1.channels[0].channel.name).to.equal('root_channel')
   })
 
   after(async function () {

shared/extra-utils/overviews/overviews.ts

@@ -1,18 +1,33 @@
 import { makeGetRequest } from '../requests/requests'
 
-function getVideosOverview (url: string, useCache = false) {
+function getVideosOverview (url: string, page: number, statusCodeExpected = 200) {
   const path = '/api/v1/overviews/videos'
 
-  const query = {
+  const query = { page }
-    t: useCache ? undefined : new Date().getTime()
-  }
 
   return makeGetRequest({
     url,
     path,
     query,
-    statusCodeExpected: 200
+    statusCodeExpected
   })
 }
 
-export { getVideosOverview }
+function getVideosOverviewWithToken (url: string, page: number, token: string, statusCodeExpected = 200) {
+  const path = '/api/v1/overviews/videos'
+
+  const query = { page }
+
+  return makeGetRequest({
+    url,
+    path,
+    query,
+    token,
+    statusCodeExpected
+  })
+}
+
+export {
+  getVideosOverview,
+  getVideosOverviewWithToken
+}

shared/models/overviews/videos-overview.ts

@@ -1,18 +1,24 @@
 import { Video, VideoChannelSummary, VideoConstant } from '../videos'
 
-export interface VideosOverview {
+export interface ChannelOverview {
-  channels: {
+  channel: VideoChannelSummary
-    channel: VideoChannelSummary
+  videos: Video[]
-    videos: Video[]
+}
-  }[]
+
-
+export interface CategoryOverview {
-  categories: {
+  category: VideoConstant<number>
-    category: VideoConstant<number>
+  videos: Video[]
-    videos: Video[]
+}
-  }[]
+
-
+export interface TagOverview {
-  tags: {
+  tag: string
-    tag: string
+  videos: Video[]
-    videos: Video[]
+}
-  }[]
+
+export interface VideosOverview {
+  channels: ChannelOverview[]
+
+  categories: CategoryOverview[]
+
+  tags: TagOverview[]
 }