Limit user tokens cache

server.ts

@@ -28,7 +28,7 @@ import { checkMissedConfig, checkFFmpeg } from './server/initializers/checker-be
 
 // Do not use barrels because we don't want to load all modules here (we need to initialize database first)
 import { logger } from './server/helpers/logger'
-import { API_VERSION, CONFIG, CACHE } from './server/initializers/constants'
+import { API_VERSION, CONFIG, FILES_CACHE } from './server/initializers/constants'
 
 const missed = checkMissedConfig()
 if (missed.length !== 0) {
@@ -82,7 +82,7 @@ migrate()
 import { installApplication } from './server/initializers'
 import { Emailer } from './server/lib/emailer'
 import { JobQueue } from './server/lib/job-queue'
-import { VideosPreviewCache, VideosCaptionCache } from './server/lib/cache'
+import { VideosPreviewCache, VideosCaptionCache } from './server/lib/files-cache'
 import {
   activityPubRouter,
   apiRouter,
@@ -218,8 +218,8 @@ async function startApplication () {
   ])
 
   // Caches initializations
-  VideosPreviewCache.Instance.init(CONFIG.CACHE.PREVIEWS.SIZE, CACHE.PREVIEWS.MAX_AGE)
+  VideosPreviewCache.Instance.init(CONFIG.CACHE.PREVIEWS.SIZE, FILES_CACHE.PREVIEWS.MAX_AGE)
-  VideosCaptionCache.Instance.init(CONFIG.CACHE.VIDEO_CAPTIONS.SIZE, CACHE.VIDEO_CAPTIONS.MAX_AGE)
+  VideosCaptionCache.Instance.init(CONFIG.CACHE.VIDEO_CAPTIONS.SIZE, FILES_CACHE.VIDEO_CAPTIONS.MAX_AGE)
 
   // Enable Schedulers
   ActorFollowScheduler.Instance.enable()

server/controllers/static.ts

@@ -8,11 +8,10 @@ import {
   STATIC_MAX_AGE,
   STATIC_PATHS
 } from '../initializers'
-import { VideosPreviewCache } from '../lib/cache'
+import { VideosCaptionCache, VideosPreviewCache } from '../lib/files-cache'
 import { cacheRoute } from '../middlewares/cache'
 import { asyncMiddleware, videosGetValidator } from '../middlewares'
 import { VideoModel } from '../models/video/video'
-import { VideosCaptionCache } from '../lib/cache/videos-caption-cache'
 import { UserModel } from '../models/account/user'
 import { VideoCommentModel } from '../models/video/video-comment'
 import { HttpNodeinfoDiasporaSoftwareNsSchema20 } from '../../shared/models/nodeinfo'

server/initializers/constants.ts

@@ -660,7 +660,7 @@ const EMBED_SIZE = {
 }
 
 // Sub folders of cache directory
-const CACHE = {
+const FILES_CACHE = {
   PREVIEWS: {
     DIRECTORY: join(CONFIG.STORAGE.CACHE_DIR, 'previews'),
     MAX_AGE: 1000 * 3600 * 3 // 3 hours
@@ -671,6 +671,12 @@ const CACHE = {
   }
 }
 
+const CACHE = {
+  USER_TOKENS: {
+    MAX_SIZE: 10000
+  }
+}
+
 const HLS_STREAMING_PLAYLIST_DIRECTORY = join(CONFIG.STORAGE.STREAMING_PLAYLISTS_DIR, 'hls')
 const HLS_REDUNDANCY_DIRECTORY = join(CONFIG.STORAGE.REDUNDANCY_DIR, 'hls')
 
@@ -741,7 +747,7 @@ if (isTestInstance() === true) {
 
   JOB_ATTEMPTS['email'] = 1
 
-  CACHE.VIDEO_CAPTIONS.MAX_AGE = 3000
+  FILES_CACHE.VIDEO_CAPTIONS.MAX_AGE = 3000
   MEMOIZE_TTL.OVERVIEWS_SAMPLE = 1
   ROUTE_CACHE_LIFETIME.OVERVIEWS.VIDEOS = '0ms'
 
@@ -759,7 +765,7 @@ export {
   ACCEPT_HEADERS,
   BCRYPT_SALT_SIZE,
   TRACKER_RATE_LIMITS,
-  CACHE,
+  FILES_CACHE,
   CONFIG,
   CONSTRAINTS_FIELDS,
   EMBED_SIZE,
@@ -799,6 +805,7 @@ export {
   VIDEO_TRANSCODING_FPS,
   FFMPEG_NICE,
   VIDEO_ABUSE_STATES,
+  CACHE,
   JOB_REQUEST_TIMEOUT,
   USER_PASSWORD_RESET_LIFETIME,
   MEMOIZE_TTL,

server/initializers/installer.ts

@@ -6,7 +6,7 @@ import { UserModel } from '../models/account/user'
 import { ApplicationModel } from '../models/application/application'
 import { OAuthClientModel } from '../models/oauth/oauth-client'
 import { applicationExist, clientsExist, usersExist } from './checker-after-init'
-import { CACHE, CONFIG, HLS_STREAMING_PLAYLIST_DIRECTORY, LAST_MIGRATION_VERSION } from './constants'
+import { FILES_CACHE, CONFIG, HLS_STREAMING_PLAYLIST_DIRECTORY, LAST_MIGRATION_VERSION } from './constants'
 import { sequelizeTypescript } from './database'
 import { remove, ensureDir } from 'fs-extra'
 
@@ -42,8 +42,8 @@ export {
 // ---------------------------------------------------------------------------
 
 function removeCacheAndTmpDirectories () {
-  const cacheDirectories = Object.keys(CACHE)
+  const cacheDirectories = Object.keys(FILES_CACHE)
-    .map(k => CACHE[k].DIRECTORY)
+    .map(k => FILES_CACHE[k].DIRECTORY)
 
   const tasks: Promise<any>[] = []
 
@@ -60,8 +60,8 @@ function removeCacheAndTmpDirectories () {
 
 function createDirectoriesIfNotExist () {
   const storage = CONFIG.STORAGE
-  const cacheDirectories = Object.keys(CACHE)
+  const cacheDirectories = Object.keys(FILES_CACHE)
-                                 .map(k => CACHE[k].DIRECTORY)
+                                 .map(k => FILES_CACHE[k].DIRECTORY)
 
   const tasks: Promise<void>[] = []
   for (const key of Object.keys(storage)) {

server/lib/files-cache/videos-caption-cache.ts

@@ -1,5 +1,5 @@
 import { join } from 'path'
-import { CACHE, CONFIG } from '../../initializers'
+import { FILES_CACHE, CONFIG } from '../../initializers'
 import { VideoModel } from '../../models/video/video'
 import { VideoCaptionModel } from '../../models/video/video-caption'
 import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache'
@@ -42,7 +42,7 @@ class VideosCaptionCache extends AbstractVideoStaticFileCache <GetPathParam> {
     if (!video) return undefined
 
     const remoteStaticPath = videoCaption.getCaptionStaticPath()
-    const destPath = join(CACHE.VIDEO_CAPTIONS.DIRECTORY, videoCaption.getCaptionName())
+    const destPath = join(FILES_CACHE.VIDEO_CAPTIONS.DIRECTORY, videoCaption.getCaptionName())
 
     return this.saveRemoteVideoFileAndReturnPath(video, remoteStaticPath, destPath)
   }

server/lib/files-cache/videos-preview-cache.ts

@@ -1,5 +1,5 @@
 import { join } from 'path'
-import { CACHE, CONFIG, STATIC_PATHS } from '../../initializers'
+import { FILES_CACHE, CONFIG, STATIC_PATHS } from '../../initializers'
 import { VideoModel } from '../../models/video/video'
 import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache'
 
@@ -31,7 +31,7 @@ class VideosPreviewCache extends AbstractVideoStaticFileCache <string> {
     if (video.isOwned()) throw new Error('Cannot load remote preview of owned video.')
 
     const remoteStaticPath = join(STATIC_PATHS.PREVIEWS, video.getPreviewName())
-    const destPath = join(CACHE.PREVIEWS.DIRECTORY, video.getPreviewName())
+    const destPath = join(FILES_CACHE.PREVIEWS.DIRECTORY, video.getPreviewName())
 
     return this.saveRemoteVideoFileAndReturnPath(video, remoteStaticPath, destPath)
   }

server/lib/job-queue/handlers/activitypub-http-broadcast.ts

@@ -2,10 +2,9 @@ import * as Bull from 'bull'
 import * as Bluebird from 'bluebird'
 import { logger } from '../../../helpers/logger'
 import { doRequest } from '../../../helpers/requests'
-import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
 import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils'
 import { BROADCAST_CONCURRENCY, JOB_REQUEST_TIMEOUT } from '../../../initializers'
-import { ActorFollowScoreCache } from '../../cache'
+import { ActorFollowScoreCache } from '../../files-cache'
 
 export type ActivitypubHttpBroadcastPayload = {
   uris: string[]

server/lib/job-queue/handlers/activitypub-http-unicast.ts

@@ -3,7 +3,7 @@ import { logger } from '../../../helpers/logger'
 import { doRequest } from '../../../helpers/requests'
 import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils'
 import { JOB_REQUEST_TIMEOUT } from '../../../initializers'
-import { ActorFollowScoreCache } from '../../cache'
+import { ActorFollowScoreCache } from '../../files-cache'
 
 export type ActivitypubHttpUnicastPayload = {
   uri: string

server/lib/oauth-model.ts

@@ -4,12 +4,12 @@ import { logger } from '../helpers/logger'
 import { UserModel } from '../models/account/user'
 import { OAuthClientModel } from '../models/oauth/oauth-client'
 import { OAuthTokenModel } from '../models/oauth/oauth-token'
-import { CONFIG } from '../initializers/constants'
+import { CONFIG, CACHE } from '../initializers/constants'
 import { Transaction } from 'sequelize'
 
 type TokenInfo = { accessToken: string, refreshToken: string, accessTokenExpiresAt: Date, refreshTokenExpiresAt: Date }
-const accessTokenCache: { [ accessToken: string ]: OAuthTokenModel } = {}
+let accessTokenCache: { [ accessToken: string ]: OAuthTokenModel } = {}
-const userHavingToken: { [ userId: number ]: string } = {}
+let userHavingToken: { [ userId: number ]: string } = {}
 
 // ---------------------------------------------------------------------------
 
@@ -43,6 +43,12 @@ function getAccessToken (bearerToken: string) {
   return OAuthTokenModel.getByTokenAndPopulateUser(bearerToken)
     .then(tokenModel => {
       if (tokenModel) {
+        // Reinit our cache
+        if (Object.keys(accessTokenCache).length > CACHE.USER_TOKENS.MAX_SIZE) {
+          accessTokenCache = {}
+          userHavingToken = {}
+        }
+
         accessTokenCache[ bearerToken ] = tokenModel
         userHavingToken[ tokenModel.userId ] = tokenModel.accessToken
       }

server/lib/schedulers/actor-follow-scheduler.ts

@@ -3,7 +3,7 @@ import { logger } from '../../helpers/logger'
 import { ActorFollowModel } from '../../models/activitypub/actor-follow'
 import { AbstractScheduler } from './abstract-scheduler'
 import { SCHEDULER_INTERVALS_MS } from '../../initializers'
-import { ActorFollowScoreCache } from '../cache'
+import { ActorFollowScoreCache } from '../files-cache'
 
 export class ActorFollowScheduler extends AbstractScheduler {