Support lazy download thumbnails

server.ts

@@ -21,7 +21,7 @@ import { checkMissedConfig, checkFFmpeg, checkNodeVersion } from './server/initi
 
 // Do not use barrels because we don't want to load all modules here (we need to initialize database first)
 import { CONFIG } from './server/initializers/config'
-import { API_VERSION, FILES_CACHE, WEBSERVER, loadLanguages } from './server/initializers/constants'
+import { API_VERSION, WEBSERVER, loadLanguages } from './server/initializers/constants'
 import { logger } from './server/helpers/logger'
 
 const missed = checkMissedConfig()
@@ -101,7 +101,6 @@ loadLanguages()
 import { installApplication } from './server/initializers/installer'
 import { Emailer } from './server/lib/emailer'
 import { JobQueue } from './server/lib/job-queue'
-import { VideosPreviewCache, VideosCaptionCache, VideosStoryboardCache } from './server/lib/files-cache'
 import {
   activityPubRouter,
   apiRouter,
@@ -143,7 +142,6 @@ import { Hooks } from './server/lib/plugins/hooks'
 import { PluginManager } from './server/lib/plugins/plugin-manager'
 import { LiveManager } from './server/lib/live'
 import { HttpStatusCode } from './shared/models/http/http-error-codes'
-import { VideosTorrentCache } from '@server/lib/files-cache/videos-torrent-cache'
 import { ServerConfigManager } from '@server/lib/server-config-manager'
 import { VideoViewsManager } from '@server/lib/views/video-views-manager'
 import { isTestOrDevInstance } from './server/helpers/core-utils'
@@ -312,12 +310,6 @@ async function startApplication () {
     ServerConfigManager.Instance.init()
   ])
 
-  // Caches initializations
-  VideosPreviewCache.Instance.init(CONFIG.CACHE.PREVIEWS.SIZE, FILES_CACHE.PREVIEWS.MAX_AGE)
-  VideosCaptionCache.Instance.init(CONFIG.CACHE.VIDEO_CAPTIONS.SIZE, FILES_CACHE.VIDEO_CAPTIONS.MAX_AGE)
-  VideosTorrentCache.Instance.init(CONFIG.CACHE.TORRENTS.SIZE, FILES_CACHE.TORRENTS.MAX_AGE)
-  VideosStoryboardCache.Instance.init(CONFIG.CACHE.STORYBOARDS.SIZE, FILES_CACHE.STORYBOARDS.MAX_AGE)
-
   // Enable Schedulers
   ActorFollowScheduler.Instance.enable()
   RemoveOldJobsScheduler.Instance.enable()

server/controllers/api/video-playlist.ts

@@ -23,7 +23,7 @@ import { MIMETYPES, VIDEO_PLAYLIST_PRIVACIES } from '../../initializers/constant
 import { sequelizeTypescript } from '../../initializers/database'
 import { sendCreateVideoPlaylist, sendDeleteVideoPlaylist, sendUpdateVideoPlaylist } from '../../lib/activitypub/send'
 import { getLocalVideoPlaylistActivityPubUrl, getLocalVideoPlaylistElementActivityPubUrl } from '../../lib/activitypub/url'
-import { updatePlaylistMiniatureFromExisting } from '../../lib/thumbnail'
+import { updateLocalPlaylistMiniatureFromExisting } from '../../lib/thumbnail'
 import {
   apiRateLimiter,
   asyncMiddleware,
@@ -178,7 +178,7 @@ async function addVideoPlaylist (req: express.Request, res: express.Response) {
 
   const thumbnailField = req.files['thumbnailfile']
   const thumbnailModel = thumbnailField
-    ? await updatePlaylistMiniatureFromExisting({
+    ? await updateLocalPlaylistMiniatureFromExisting({
       inputPath: thumbnailField[0].path,
       playlist: videoPlaylist,
       automaticallyGenerated: false
@@ -220,7 +220,7 @@ async function updateVideoPlaylist (req: express.Request, res: express.Response)
 
   const thumbnailField = req.files['thumbnailfile']
   const thumbnailModel = thumbnailField
-    ? await updatePlaylistMiniatureFromExisting({
+    ? await updateLocalPlaylistMiniatureFromExisting({
       inputPath: thumbnailField[0].path,
       playlist: videoPlaylistInstance,
       automaticallyGenerated: false
@@ -497,7 +497,7 @@ async function generateThumbnailForPlaylist (videoPlaylist: MVideoPlaylistThumbn
   }
 
   const inputPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, videoMiniature.filename)
-  const thumbnailModel = await updatePlaylistMiniatureFromExisting({
+  const thumbnailModel = await updateLocalPlaylistMiniatureFromExisting({
     inputPath,
     playlist: videoPlaylist,
     automaticallyGenerated: true,

server/controllers/api/videos/import.ts

@@ -14,7 +14,7 @@ import { getSecureTorrentName } from '../../../helpers/utils'
 import { CONFIG } from '../../../initializers/config'
 import { MIMETYPES } from '../../../initializers/constants'
 import { JobQueue } from '../../../lib/job-queue/job-queue'
-import { updateVideoMiniatureFromExisting } from '../../../lib/thumbnail'
+import { updateLocalVideoMiniatureFromExisting } from '../../../lib/thumbnail'
 import {
   asyncMiddleware,
   asyncRetryTransactionMiddleware,
@@ -193,7 +193,7 @@ async function processThumbnail (req: express.Request, video: MVideoThumbnail) {
   if (thumbnailField) {
     const thumbnailPhysicalFile = thumbnailField[0]
 
-    return updateVideoMiniatureFromExisting({
+    return updateLocalVideoMiniatureFromExisting({
       inputPath: thumbnailPhysicalFile.path,
       video,
       type: ThumbnailType.MINIATURE,
@@ -209,7 +209,7 @@ async function processPreview (req: express.Request, video: MVideoThumbnail): Pr
   if (previewField) {
     const previewPhysicalFile = previewField[0]
 
-    return updateVideoMiniatureFromExisting({
+    return updateLocalVideoMiniatureFromExisting({
       inputPath: previewPhysicalFile.path,
       video,
       type: ThumbnailType.PREVIEW,

server/controllers/api/videos/live.ts

@@ -21,7 +21,7 @@ import { buildUUID, uuidToShort } from '@shared/extra-utils'
 import { HttpStatusCode, LiveVideoCreate, LiveVideoLatencyMode, LiveVideoUpdate, UserRight, VideoPrivacy, VideoState } from '@shared/models'
 import { logger } from '../../../helpers/logger'
 import { sequelizeTypescript } from '../../../initializers/database'
-import { updateVideoMiniatureFromExisting } from '../../../lib/thumbnail'
+import { updateLocalVideoMiniatureFromExisting } from '../../../lib/thumbnail'
 import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, optionalAuthenticate } from '../../../middlewares'
 import { VideoModel } from '../../../models/video/video'
 import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting'
@@ -166,7 +166,7 @@ async function addLiveVideo (req: express.Request, res: express.Response) {
     video,
     files: req.files,
     fallback: type => {
-      return updateVideoMiniatureFromExisting({
+      return updateLocalVideoMiniatureFromExisting({
         inputPath: ASSETS_PATH.DEFAULT_LIVE_BACKGROUND,
         video,
         type,

server/controllers/api/videos/upload.ts

@@ -21,7 +21,7 @@ import { logger, loggerTagsFactory } from '../../../helpers/logger'
 import { MIMETYPES } from '../../../initializers/constants'
 import { sequelizeTypescript } from '../../../initializers/database'
 import { Hooks } from '../../../lib/plugins/hooks'
-import { generateVideoMiniature } from '../../../lib/thumbnail'
+import { generateLocalVideoMiniature } from '../../../lib/thumbnail'
 import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist'
 import {
   asyncMiddleware,
@@ -153,7 +153,7 @@ async function addVideo (options: {
   const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({
     video,
     files,
-    fallback: type => generateVideoMiniature({ video, videoFile, type })
+    fallback: type => generateLocalVideoMiniature({ video, videoFile, type })
   })
 
   const { videoCreated } = await sequelizeTypescript.transaction(async t => {

server/controllers/download.ts

@@ -1,7 +1,7 @@
 import cors from 'cors'
 import express from 'express'
 import { logger } from '@server/helpers/logger'
-import { VideosTorrentCache } from '@server/lib/files-cache/videos-torrent-cache'
+import { VideoTorrentsSimpleFileCache } from '@server/lib/files-cache'
 import { generateHLSFilePresignedUrl, generateWebVideoPresignedUrl } from '@server/lib/object-storage'
 import { Hooks } from '@server/lib/plugins/hooks'
 import { VideoPathManager } from '@server/lib/video-path-manager'
@@ -43,7 +43,7 @@ export {
 // ---------------------------------------------------------------------------
 
 async function downloadTorrent (req: express.Request, res: express.Response) {
-  const result = await VideosTorrentCache.Instance.getFilePath(req.params.filename)
+  const result = await VideoTorrentsSimpleFileCache.Instance.getFilePath(req.params.filename)
   if (!result) {
     return res.fail({
       status: HttpStatusCode.NOT_FOUND_404,

server/controllers/lazy-static.ts

@@ -1,14 +1,27 @@
 import cors from 'cors'
 import express from 'express'
-import { VideosTorrentCache } from '@server/lib/files-cache/videos-torrent-cache'
+import { CONFIG } from '@server/initializers/config'
-import { MActorImage } from '@server/types/models'
 import { HttpStatusCode } from '../../shared/models/http/http-error-codes'
-import { logger } from '../helpers/logger'
+import { FILES_CACHE, LAZY_STATIC_PATHS, STATIC_MAX_AGE } from '../initializers/constants'
-import { ACTOR_IMAGES_SIZE, LAZY_STATIC_PATHS, STATIC_MAX_AGE } from '../initializers/constants'
+import {
-import { VideosCaptionCache, VideosPreviewCache, VideosStoryboardCache } from '../lib/files-cache'
+  AvatarPermanentFileCache,
-import { actorImagePathUnsafeCache, downloadActorImageFromWorker } from '../lib/local-actor'
+  VideoCaptionsSimpleFileCache,
+  VideoPreviewsSimpleFileCache,
+  VideoStoryboardsSimpleFileCache,
+  VideoTorrentsSimpleFileCache
+} from '../lib/files-cache'
 import { asyncMiddleware, handleStaticError } from '../middlewares'
-import { ActorImageModel } from '../models/actor/actor-image'
+
+// ---------------------------------------------------------------------------
+// Cache initializations
+// ---------------------------------------------------------------------------
+
+VideoPreviewsSimpleFileCache.Instance.init(CONFIG.CACHE.PREVIEWS.SIZE, FILES_CACHE.PREVIEWS.MAX_AGE)
+VideoCaptionsSimpleFileCache.Instance.init(CONFIG.CACHE.VIDEO_CAPTIONS.SIZE, FILES_CACHE.VIDEO_CAPTIONS.MAX_AGE)
+VideoTorrentsSimpleFileCache.Instance.init(CONFIG.CACHE.TORRENTS.SIZE, FILES_CACHE.TORRENTS.MAX_AGE)
+VideoStoryboardsSimpleFileCache.Instance.init(CONFIG.CACHE.STORYBOARDS.SIZE, FILES_CACHE.STORYBOARDS.MAX_AGE)
+
+// ---------------------------------------------------------------------------
 
 const lazyStaticRouter = express.Router()
 
@@ -60,94 +73,37 @@ export {
 
 // ---------------------------------------------------------------------------
 
-async function getActorImage (req: express.Request, res: express.Response, next: express.NextFunction) {
+const avatarPermanentFileCache = new AvatarPermanentFileCache()
+
+function getActorImage (req: express.Request, res: express.Response, next: express.NextFunction) {
   const filename = req.params.filename
 
-  if (actorImagePathUnsafeCache.has(filename)) {
+  return avatarPermanentFileCache.lazyServe({ filename, res, next })
-    return res.sendFile(actorImagePathUnsafeCache.get(filename), { maxAge: STATIC_MAX_AGE.SERVER })
-  }
-
-  const image = await ActorImageModel.loadByName(filename)
-  if (!image) return res.status(HttpStatusCode.NOT_FOUND_404).end()
-
-  if (image.onDisk === false) {
-    if (!image.fileUrl) return res.status(HttpStatusCode.NOT_FOUND_404).end()
-
-    logger.info('Lazy serve remote actor image %s.', image.fileUrl)
-
-    try {
-      await downloadActorImageFromWorker({
-        filename: image.filename,
-        fileUrl: image.fileUrl,
-        size: getActorImageSize(image),
-        type: image.type
-      })
-    } catch (err) {
-      logger.warn('Cannot process remote actor image %s.', image.fileUrl, { err })
-      return res.status(HttpStatusCode.NOT_FOUND_404).end()
-    }
-
-    image.onDisk = true
-    image.save()
-      .catch(err => logger.error('Cannot save new actor image disk state.', { err }))
-  }
-
-  const path = image.getPath()
-
-  actorImagePathUnsafeCache.set(filename, path)
-
-  return res.sendFile(path, { maxAge: STATIC_MAX_AGE.LAZY_SERVER }, (err: any) => {
-    if (!err) return
-
-    // It seems this actor image is not on the disk anymore
-    if (err.status === HttpStatusCode.NOT_FOUND_404 && !image.isOwned()) {
-      logger.error('Cannot lazy serve actor image %s.', filename, { err })
-
-      actorImagePathUnsafeCache.delete(filename)
-
-      image.onDisk = false
-      image.save()
-       .catch(err => logger.error('Cannot save new actor image disk state.', { err }))
-    }
-
-    return next(err)
-  })
-}
-
-function getActorImageSize (image: MActorImage): { width: number, height: number } {
-  if (image.width && image.height) {
-    return {
-      height: image.height,
-      width: image.width
-    }
-  }
-
-  return ACTOR_IMAGES_SIZE[image.type][0]
 }
 
 async function getPreview (req: express.Request, res: express.Response) {
-  const result = await VideosPreviewCache.Instance.getFilePath(req.params.filename)
+  const result = await VideoPreviewsSimpleFileCache.Instance.getFilePath(req.params.filename)
   if (!result) return res.status(HttpStatusCode.NOT_FOUND_404).end()
 
   return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE.LAZY_SERVER })
 }
 
 async function getStoryboard (req: express.Request, res: express.Response) {
-  const result = await VideosStoryboardCache.Instance.getFilePath(req.params.filename)
+  const result = await VideoStoryboardsSimpleFileCache.Instance.getFilePath(req.params.filename)
   if (!result) return res.status(HttpStatusCode.NOT_FOUND_404).end()
 
   return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE.LAZY_SERVER })
 }
 
 async function getVideoCaption (req: express.Request, res: express.Response) {
-  const result = await VideosCaptionCache.Instance.getFilePath(req.params.filename)
+  const result = await VideoCaptionsSimpleFileCache.Instance.getFilePath(req.params.filename)
   if (!result) return res.status(HttpStatusCode.NOT_FOUND_404).end()
 
   return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE.LAZY_SERVER })
 }
 
 async function getTorrent (req: express.Request, res: express.Response) {
-  const result = await VideosTorrentCache.Instance.getFilePath(req.params.filename)
+  const result = await VideoTorrentsSimpleFileCache.Instance.getFilePath(req.params.filename)
   if (!result) return res.status(HttpStatusCode.NOT_FOUND_404).end()
 
   // Torrents still use the old naming convention (video uuid + .torrent)

server/initializers/constants.ts

@@ -854,8 +854,8 @@ const LRU_CACHE = {
   USER_TOKENS: {
     MAX_SIZE: 1000
   },
-  ACTOR_IMAGE_STATIC: {
+  FILENAME_TO_PATH_PERMANENT_FILE_CACHE: {
-    MAX_SIZE: 500
+    MAX_SIZE: 1000
   },
   STATIC_VIDEO_FILES_RIGHTS_CHECK: {
     MAX_SIZE: 5000,

server/lib/activitypub/videos/shared/abstract-builder.ts

@@ -1,7 +1,7 @@
 import { CreationAttributes, Transaction } from 'sequelize/types'
 import { deleteAllModels, filterNonExistingModels } from '@server/helpers/database-utils'
 import { logger, LoggerTagsFn } from '@server/helpers/logger'
-import { updatePlaceholderThumbnail, updateVideoMiniatureFromUrl } from '@server/lib/thumbnail'
+import { updateRemoteThumbnail, updateVideoMiniatureFromUrl } from '@server/lib/thumbnail'
 import { setVideoTags } from '@server/lib/video'
 import { StoryboardModel } from '@server/models/video/storyboard'
 import { VideoCaptionModel } from '@server/models/video/video-caption'
@@ -55,15 +55,15 @@ export abstract class APVideoAbstractBuilder {
   }
 
   protected async setPreview (video: MVideoFullLight, t?: Transaction) {
-    // Don't fetch the preview that could be big, create a placeholder instead
     const previewIcon = getPreviewFromIcons(this.videoObject)
     if (!previewIcon) return
 
-    const previewModel = updatePlaceholderThumbnail({
+    const previewModel = updateRemoteThumbnail({
       fileUrl: previewIcon.url,
       video,
       type: ThumbnailType.PREVIEW,
-      size: previewIcon
+      size: previewIcon,
+      onDisk: false // Don't fetch the preview that could be big, create a placeholder instead
     })
 
     await video.addAndSaveThumbnail(previewModel, t)

server/lib/files-cache/avatar-permanent-file-cache.ts

@@ -0,0 +1,27 @@
+import { ACTOR_IMAGES_SIZE } from '@server/initializers/constants'
+import { ActorImageModel } from '@server/models/actor/actor-image'
+import { MActorImage } from '@server/types/models'
+import { AbstractPermanentFileCache } from './shared'
+import { CONFIG } from '@server/initializers/config'
+
+export class AvatarPermanentFileCache extends AbstractPermanentFileCache<ActorImageModel> {
+
+  constructor () {
+    super(CONFIG.STORAGE.ACTOR_IMAGES)
+  }
+
+  protected loadModel (filename: string) {
+    return ActorImageModel.loadByName(filename)
+  }
+
+  protected getImageSize (image: MActorImage): { width: number, height: number } {
+    if (image.width && image.height) {
+      return {
+        height: image.height,
+        width: image.width
+      }
+    }
+
+    return ACTOR_IMAGES_SIZE[image.type][0]
+  }
+}

server/lib/files-cache/index.ts

@@ -1,4 +1,5 @@
-export * from './videos-caption-cache'
+export * from './avatar-permanent-file-cache'
-export * from './videos-preview-cache'
+export * from './video-captions-simple-file-cache'
-export * from './videos-storyboard-cache'
+export * from './video-previews-simple-file-cache'
-export * from './videos-torrent-cache'
+export * from './video-storyboards-simple-file-cache'
+export * from './video-torrents-simple-file-cache'

server/lib/files-cache/shared/abstract-permanent-file-cache.ts

@@ -0,0 +1,119 @@
+import express from 'express'
+import { LRUCache } from 'lru-cache'
+import { logger } from '@server/helpers/logger'
+import { LRU_CACHE, STATIC_MAX_AGE } from '@server/initializers/constants'
+import { downloadImageFromWorker } from '@server/lib/worker/parent-process'
+import { HttpStatusCode } from '@shared/models'
+import { Model } from 'sequelize'
+
+type ImageModel = {
+  fileUrl: string
+  filename: string
+  onDisk: boolean
+
+  isOwned (): boolean
+  getPath (): string
+
+  save (): Promise<Model>
+}
+
+export abstract class AbstractPermanentFileCache <M extends ImageModel> {
+  // Unsafe because it can return paths that do not exist anymore
+  private readonly filenameToPathUnsafeCache = new LRUCache<string, string>({
+    max: LRU_CACHE.FILENAME_TO_PATH_PERMANENT_FILE_CACHE.MAX_SIZE
+  })
+
+  protected abstract getImageSize (image: M): { width: number, height: number }
+  protected abstract loadModel (filename: string): Promise<M>
+
+  constructor (private readonly directory: string) {
+
+  }
+
+  async lazyServe (options: {
+    filename: string
+    res: express.Response
+    next: express.NextFunction
+  }) {
+    const { filename, res, next } = options
+
+    if (this.filenameToPathUnsafeCache.has(filename)) {
+      return res.sendFile(this.filenameToPathUnsafeCache.get(filename), { maxAge: STATIC_MAX_AGE.SERVER })
+    }
+
+    const image = await this.loadModel(filename)
+    if (!image) return res.status(HttpStatusCode.NOT_FOUND_404).end()
+
+    if (image.onDisk === false) {
+      if (!image.fileUrl) return res.status(HttpStatusCode.NOT_FOUND_404).end()
+
+      try {
+        await this.downloadRemoteFile(image)
+      } catch (err) {
+        logger.warn('Cannot process remote image %s.', image.fileUrl, { err })
+
+        return res.status(HttpStatusCode.NOT_FOUND_404).end()
+      }
+    }
+
+    const path = image.getPath()
+    this.filenameToPathUnsafeCache.set(filename, path)
+
+    return res.sendFile(path, { maxAge: STATIC_MAX_AGE.LAZY_SERVER }, (err: any) => {
+      if (!err) return
+
+      this.onServeError({ err, image, next, filename })
+    })
+  }
+
+  private async downloadRemoteFile (image: M) {
+    logger.info('Download remote image %s lazily.', image.fileUrl)
+
+    await this.downloadImage({
+      filename: image.filename,
+      fileUrl: image.fileUrl,
+      size: this.getImageSize(image)
+    })
+
+    image.onDisk = true
+    image.save()
+      .catch(err => logger.error('Cannot save new image disk state.', { err }))
+  }
+
+  private onServeError (options: {
+    err: any
+    image: M
+    filename: string
+    next: express.NextFunction
+  }) {
+    const { err, image, filename, next } = options
+
+    // It seems this actor image is not on the disk anymore
+    if (err.status === HttpStatusCode.NOT_FOUND_404 && !image.isOwned()) {
+      logger.error('Cannot lazy serve image %s.', filename, { err })
+
+      this.filenameToPathUnsafeCache.delete(filename)
+
+      image.onDisk = false
+      image.save()
+        .catch(err => logger.error('Cannot save new image disk state.', { err }))
+    }
+
+    return next(err)
+  }
+
+  private downloadImage (options: {
+    fileUrl: string
+    filename: string
+    size: { width: number, height: number }
+  }) {
+    const downloaderOptions = {
+      url: options.fileUrl,
+      destDir: this.directory,
+      destName: options.filename,
+      size: options.size
+    }
+
+    return downloadImageFromWorker(downloaderOptions)
+  }
+}

server/lib/files-cache/shared/abstract-simple-file-cache.ts

@@ -1,10 +1,10 @@
 import { remove } from 'fs-extra'
-import { logger } from '../../helpers/logger'
+import { logger } from '../../../helpers/logger'
 import memoizee from 'memoizee'
 
 type GetFilePathResult = { isOwned: boolean, path: string, downloadName?: string } | undefined
 
-export abstract class AbstractVideoStaticFileCache <T> {
+export abstract class AbstractSimpleFileCache <T> {
 
   getFilePath: (params: T) => Promise<GetFilePathResult>
 

server/lib/files-cache/shared/index.ts

@@ -0,0 +1,2 @@
+export * from './abstract-permanent-file-cache'
+export * from './abstract-simple-file-cache'

server/lib/files-cache/video-captions-simple-file-cache.ts

@@ -5,11 +5,11 @@ import { CONFIG } from '../../initializers/config'
 import { FILES_CACHE } from '../../initializers/constants'
 import { VideoModel } from '../../models/video/video'
 import { VideoCaptionModel } from '../../models/video/video-caption'
-import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache'
+import { AbstractSimpleFileCache } from './shared/abstract-simple-file-cache'
 
-class VideosCaptionCache extends AbstractVideoStaticFileCache <string> {
+class VideoCaptionsSimpleFileCache extends AbstractSimpleFileCache <string> {
 
-  private static instance: VideosCaptionCache
+  private static instance: VideoCaptionsSimpleFileCache
 
   private constructor () {
     super()
@@ -23,7 +23,9 @@ class VideosCaptionCache extends AbstractVideoStaticFileCache <string> {
     const videoCaption = await VideoCaptionModel.loadWithVideoByFilename(filename)
     if (!videoCaption) return undefined
 
-    if (videoCaption.isOwned()) return { isOwned: true, path: join(CONFIG.STORAGE.CAPTIONS_DIR, videoCaption.filename) }
+    if (videoCaption.isOwned()) {
+      return { isOwned: true, path: join(CONFIG.STORAGE.CAPTIONS_DIR, videoCaption.filename) }
+    }
 
     return this.loadRemoteFile(filename)
   }
@@ -55,5 +57,5 @@ class VideosCaptionCache extends AbstractVideoStaticFileCache <string> {
 }
 
 export {
-  VideosCaptionCache
+  VideoCaptionsSimpleFileCache
 }

server/lib/files-cache/video-previews-simple-file-cache.ts

@@ -1,15 +1,15 @@
 import { join } from 'path'
 import { FILES_CACHE } from '../../initializers/constants'
 import { VideoModel } from '../../models/video/video'
-import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache'
+import { AbstractSimpleFileCache } from './shared/abstract-simple-file-cache'
 import { doRequestAndSaveToFile } from '@server/helpers/requests'
 import { ThumbnailModel } from '@server/models/video/thumbnail'
 import { ThumbnailType } from '@shared/models'
 import { logger } from '@server/helpers/logger'
 
-class VideosPreviewCache extends AbstractVideoStaticFileCache <string> {
+class VideoPreviewsSimpleFileCache extends AbstractSimpleFileCache <string> {
 
-  private static instance: VideosPreviewCache
+  private static instance: VideoPreviewsSimpleFileCache
 
   private constructor () {
     super()
@@ -54,5 +54,5 @@ class VideosPreviewCache extends AbstractVideoStaticFileCache <string> {
 }
 
 export {
-  VideosPreviewCache
+  VideoPreviewsSimpleFileCache
 }

server/lib/files-cache/video-storyboards-simple-file-cache.ts

@@ -3,11 +3,11 @@ import { logger } from '@server/helpers/logger'
 import { doRequestAndSaveToFile } from '@server/helpers/requests'
 import { StoryboardModel } from '@server/models/video/storyboard'
 import { FILES_CACHE } from '../../initializers/constants'
-import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache'
+import { AbstractSimpleFileCache } from './shared/abstract-simple-file-cache'
 
-class VideosStoryboardCache extends AbstractVideoStaticFileCache <string> {
+class VideoStoryboardsSimpleFileCache extends AbstractSimpleFileCache <string> {
 
-  private static instance: VideosStoryboardCache
+  private static instance: VideoStoryboardsSimpleFileCache
 
   private constructor () {
     super()
@@ -49,5 +49,5 @@ class VideosStoryboardCache extends AbstractVideoStaticFileCache <string> {
 }
 
 export {
-  VideosStoryboardCache
+  VideoStoryboardsSimpleFileCache
 }

server/lib/files-cache/video-torrents-simple-file-cache.ts

@@ -6,11 +6,11 @@ import { MVideo, MVideoFile } from '@server/types/models'
 import { CONFIG } from '../../initializers/config'
 import { FILES_CACHE } from '../../initializers/constants'
 import { VideoModel } from '../../models/video/video'
-import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache'
+import { AbstractSimpleFileCache } from './shared/abstract-simple-file-cache'
 
-class VideosTorrentCache extends AbstractVideoStaticFileCache <string> {
+class VideoTorrentsSimpleFileCache extends AbstractSimpleFileCache <string> {
 
-  private static instance: VideosTorrentCache
+  private static instance: VideoTorrentsSimpleFileCache
 
   private constructor () {
     super()
@@ -66,5 +66,5 @@ class VideosTorrentCache extends AbstractVideoStaticFileCache <string> {
 }
 
 export {
-  VideosTorrentCache
+  VideoTorrentsSimpleFileCache
 }

server/lib/job-queue/handlers/video-import.ts

@@ -39,7 +39,7 @@ import { VideoFileModel } from '../../../models/video/video-file'
 import { VideoImportModel } from '../../../models/video/video-import'
 import { federateVideoIfNeeded } from '../../activitypub/videos'
 import { Notifier } from '../../notifier'
-import { generateVideoMiniature } from '../../thumbnail'
+import { generateLocalVideoMiniature } from '../../thumbnail'
 import { JobQueue } from '../job-queue'
 
 async function processVideoImport (job: Job): Promise<VideoImportPreventExceptionResult> {
@@ -274,7 +274,7 @@ async function generateMiniature (videoImportWithFiles: MVideoImportDefaultFiles
     }
   }
 
-  const miniatureModel = await generateVideoMiniature({
+  const miniatureModel = await generateLocalVideoMiniature({
     video: videoImportWithFiles.Video,
     videoFile,
     type: thumbnailType

server/lib/job-queue/handlers/video-live-ending.ts

@@ -7,7 +7,7 @@ import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url'
 import { federateVideoIfNeeded } from '@server/lib/activitypub/videos'
 import { cleanupAndDestroyPermanentLive, cleanupTMPLiveFiles, cleanupUnsavedNormalLive } from '@server/lib/live'
 import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveReplayBaseDirectory } from '@server/lib/paths'
-import { generateVideoMiniature } from '@server/lib/thumbnail'
+import { generateLocalVideoMiniature } from '@server/lib/thumbnail'
 import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/hls-transcoding'
 import { VideoPathManager } from '@server/lib/video-path-manager'
 import { moveToNextState } from '@server/lib/video-state'
@@ -143,7 +143,7 @@ async function saveReplayToExternalVideo (options: {
   await remove(replayDirectory)
 
   for (const type of [ ThumbnailType.MINIATURE, ThumbnailType.PREVIEW ]) {
-    const image = await generateVideoMiniature({ video: replayVideo, videoFile: replayVideo.getMaxQualityFile(), type })
+    const image = await generateLocalVideoMiniature({ video: replayVideo, videoFile: replayVideo.getMaxQualityFile(), type })
     await replayVideo.addAndSaveThumbnail(image)
   }
 
@@ -198,7 +198,7 @@ async function replaceLiveByReplay (options: {
 
   // Regenerate the thumbnail & preview?
   if (videoWithFiles.getMiniature().automaticallyGenerated === true) {
-    const miniature = await generateVideoMiniature({
+    const miniature = await generateLocalVideoMiniature({
       video: videoWithFiles,
       videoFile: videoWithFiles.getMaxQualityFile(),
       type: ThumbnailType.MINIATURE
@@ -207,7 +207,7 @@ async function replaceLiveByReplay (options: {
   }
 
   if (videoWithFiles.getPreview().automaticallyGenerated === true) {
-    const preview = await generateVideoMiniature({
+    const preview = await generateLocalVideoMiniature({
       video: videoWithFiles,
       videoFile: videoWithFiles.getMaxQualityFile(),
       type: ThumbnailType.PREVIEW

server/lib/local-actor.ts

@@ -1,5 +1,4 @@
 import { remove } from 'fs-extra'
-import { LRUCache } from 'lru-cache'
 import { join } from 'path'
 import { Transaction } from 'sequelize/types'
 import { ActorModel } from '@server/models/actor/actor'
@@ -8,14 +7,14 @@ import { buildUUID } from '@shared/extra-utils'
 import { ActivityPubActorType, ActorImageType } from '@shared/models'
 import { retryTransactionWrapper } from '../helpers/database-utils'
 import { CONFIG } from '../initializers/config'
-import { ACTOR_IMAGES_SIZE, LRU_CACHE, WEBSERVER } from '../initializers/constants'
+import { ACTOR_IMAGES_SIZE, WEBSERVER } from '../initializers/constants'
 import { sequelizeTypescript } from '../initializers/database'
 import { MAccountDefault, MActor, MChannelDefault } from '../types/models'
 import { deleteActorImages, updateActorImages } from './activitypub/actors'
 import { sendUpdateActor } from './activitypub/send'
-import { downloadImageFromWorker, processImageFromWorker } from './worker/parent-process'
+import { processImageFromWorker } from './worker/parent-process'
 
-function buildActorInstance (type: ActivityPubActorType, url: string, preferredUsername: string) {
+export function buildActorInstance (type: ActivityPubActorType, url: string, preferredUsername: string) {
   return new ActorModel({
     type,
     url,
@@ -32,7 +31,7 @@ function buildActorInstance (type: ActivityPubActorType, url: string, preferredU
   }) as MActor
 }
 
-async function updateLocalActorImageFiles (
+export async function updateLocalActorImageFiles (
   accountOrChannel: MAccountDefault | MChannelDefault,
   imagePhysicalFile: Express.Multer.File,
   type: ActorImageType
@@ -73,7 +72,7 @@ async function updateLocalActorImageFiles (
   }))
 }
 
-async function deleteLocalActorImageFile (accountOrChannel: MAccountDefault | MChannelDefault, type: ActorImageType) {
+export async function deleteLocalActorImageFile (accountOrChannel: MAccountDefault | MChannelDefault, type: ActorImageType) {
   return retryTransactionWrapper(() => {
     return sequelizeTypescript.transaction(async t => {
       const updatedActor = await deleteActorImages(accountOrChannel.Actor, type, t)
@@ -88,7 +87,7 @@ async function deleteLocalActorImageFile (accountOrChannel: MAccountDefault | MC
 
 // ---------------------------------------------------------------------------
 
-async function findAvailableLocalActorName (baseActorName: string, transaction?: Transaction) {
+export async function findAvailableLocalActorName (baseActorName: string, transaction?: Transaction) {
   let actor = await ActorModel.loadLocalByName(baseActorName, transaction)
   if (!actor) return baseActorName
 
@@ -101,34 +100,3 @@ async function findAvailableLocalActorName (baseActorName: string, transaction?:
 
   throw new Error('Cannot find available actor local name (too much iterations).')
 }
-
-// ---------------------------------------------------------------------------
-
-function downloadActorImageFromWorker (options: {
-  fileUrl: string
-  filename: string
-  type: ActorImageType
-  size: typeof ACTOR_IMAGES_SIZE[ActorImageType][0]
-}) {
-  const downloaderOptions = {
-    url: options.fileUrl,
-    destDir: CONFIG.STORAGE.ACTOR_IMAGES,
-    destName: options.filename,
-    size: options.size
-  }
-
-  return downloadImageFromWorker(downloaderOptions)
-}
-
-// Unsafe so could returns paths that does not exist anymore
-const actorImagePathUnsafeCache = new LRUCache<string, string>({ max: LRU_CACHE.ACTOR_IMAGE_STATIC.MAX_SIZE })
-
-export {
-  actorImagePathUnsafeCache,
-  updateLocalActorImageFiles,
-  findAvailableLocalActorName,
-  downloadActorImageFromWorker,
-  deleteLocalActorImageFile,
-  downloadImageFromWorker,
-  buildActorInstance
-}

server/lib/thumbnail.ts

@@ -7,13 +7,12 @@ import { ThumbnailModel } from '../models/video/thumbnail'
 import { MVideoFile, MVideoThumbnail, MVideoUUID } from '../types/models'
 import { MThumbnail } from '../types/models/video/thumbnail'
 import { MVideoPlaylistThumbnail } from '../types/models/video/video-playlist'
-import { downloadImageFromWorker } from './local-actor'
 import { VideoPathManager } from './video-path-manager'
-import { processImageFromWorker } from './worker/parent-process'
+import { downloadImageFromWorker, processImageFromWorker } from './worker/parent-process'
 
 type ImageSize = { height?: number, width?: number }
 
-function updatePlaylistMiniatureFromExisting (options: {
+function updateLocalPlaylistMiniatureFromExisting (options: {
   inputPath: string
   playlist: MVideoPlaylistThumbnail
   automaticallyGenerated: boolean
@@ -35,6 +34,7 @@ function updatePlaylistMiniatureFromExisting (options: {
     width,
     type,
     automaticallyGenerated,
+    onDisk: true,
     existingThumbnail
   })
 }
@@ -57,7 +57,7 @@ function updatePlaylistMiniatureFromUrl (options: {
     return downloadImageFromWorker({ url: downloadUrl, destDir: basePath, destName: filename, size: { width, height } })
   }
 
-  return updateThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, fileUrl })
+  return updateThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, fileUrl, onDisk: true })
 }
 
 function updateVideoMiniatureFromUrl (options: {
@@ -89,10 +89,10 @@ function updateVideoMiniatureFromUrl (options: {
     return Promise.resolve()
   }
 
-  return updateThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, fileUrl })
+  return updateThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, fileUrl, onDisk: true })
 }
 
-function updateVideoMiniatureFromExisting (options: {
+function updateLocalVideoMiniatureFromExisting (options: {
   inputPath: string
   video: MVideoThumbnail
   type: ThumbnailType
@@ -115,11 +115,12 @@ function updateVideoMiniatureFromExisting (options: {
     width,
     type,
     automaticallyGenerated,
-    existingThumbnail
+    existingThumbnail,
+    onDisk: true
   })
 }
 
-function generateVideoMiniature (options: {
+function generateLocalVideoMiniature (options: {
   video: MVideoThumbnail
   videoFile: MVideoFile
   type: ThumbnailType
@@ -150,34 +151,36 @@ function generateVideoMiniature (options: {
       width,
       type,
       automaticallyGenerated: true,
+      onDisk: true,
       existingThumbnail
     })
   })
 }
 
-function updatePlaceholderThumbnail (options: {
+function updateRemoteThumbnail (options: {
   fileUrl: string
   video: MVideoThumbnail
   type: ThumbnailType
   size: ImageSize
+  onDisk: boolean
 }) {
-  const { fileUrl, video, type, size } = options
+  const { fileUrl, video, type, size, onDisk } = options
-  const { filename: updatedFilename, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size)
+  const { filename: generatedFilename, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size)
 
   const thumbnailUrlChanged = hasThumbnailUrlChanged(existingThumbnail, fileUrl, video)
 
   const thumbnail = existingThumbnail || new ThumbnailModel()
 
   // Do not change the thumbnail filename if the file did not change
-  const filename = thumbnailUrlChanged
+  if (thumbnailUrlChanged) {
-    ? updatedFilename
+    thumbnail.filename = generatedFilename
-    : existingThumbnail.filename
+  }
 
-  thumbnail.filename = filename
   thumbnail.height = height
   thumbnail.width = width
   thumbnail.type = type
   thumbnail.fileUrl = fileUrl
+  thumbnail.onDisk = onDisk
 
   return thumbnail
 }
@@ -185,14 +188,18 @@ function updatePlaceholderThumbnail (options: {
 // ---------------------------------------------------------------------------
 
 export {
-  generateVideoMiniature,
+  generateLocalVideoMiniature,
   updateVideoMiniatureFromUrl,
-  updateVideoMiniatureFromExisting,
+  updateLocalVideoMiniatureFromExisting,
-  updatePlaceholderThumbnail,
+  updateRemoteThumbnail,
   updatePlaylistMiniatureFromUrl,
-  updatePlaylistMiniatureFromExisting
+  updateLocalPlaylistMiniatureFromExisting
 }
 
+// ---------------------------------------------------------------------------
+// Private
+// ---------------------------------------------------------------------------
+
 function hasThumbnailUrlChanged (existingThumbnail: MThumbnail, downloadUrl: string, video: MVideoUUID) {
   const existingUrl = existingThumbnail
     ? existingThumbnail.fileUrl
@@ -258,6 +265,7 @@ async function updateThumbnailFromFunction (parameters: {
   height: number
   width: number
   type: ThumbnailType
+  onDisk: boolean
   automaticallyGenerated?: boolean
   fileUrl?: string
   existingThumbnail?: MThumbnail
@@ -269,6 +277,7 @@ async function updateThumbnailFromFunction (parameters: {
     height,
     type,
     existingThumbnail,
+    onDisk,
     automaticallyGenerated = null,
     fileUrl = null
   } = parameters
@@ -285,6 +294,7 @@ async function updateThumbnailFromFunction (parameters: {
   thumbnail.type = type
   thumbnail.fileUrl = fileUrl
   thumbnail.automaticallyGenerated = automaticallyGenerated
+  thumbnail.onDisk = onDisk
 
   if (oldFilename) thumbnail.previousThumbnailFilename = oldFilename
 

server/lib/video-pre-import.ts

@@ -29,7 +29,7 @@ import {
 } from '@server/types/models'
 import { ThumbnailType, VideoImportCreate, VideoImportPayload, VideoImportState, VideoPrivacy, VideoState } from '@shared/models'
 import { getLocalVideoActivityPubUrl } from './activitypub/url'
-import { updateVideoMiniatureFromExisting, updateVideoMiniatureFromUrl } from './thumbnail'
+import { updateLocalVideoMiniatureFromExisting, updateVideoMiniatureFromUrl } from './thumbnail'
 import { VideoPasswordModel } from '@server/models/video/video-password'
 
 class YoutubeDlImportError extends Error {
@@ -256,7 +256,7 @@ async function forgeThumbnail ({ inputPath, video, downloadUrl, type }: {
   type: ThumbnailType
 }): Promise<MThumbnail> {
   if (inputPath) {
-    return updateVideoMiniatureFromExisting({
+    return updateLocalVideoMiniatureFromExisting({
       inputPath,
       video,
       type,

server/lib/video.ts

@@ -10,7 +10,7 @@ import { FilteredModelAttributes } from '@server/types'
 import { MThumbnail, MVideoFullLight, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models'
 import { ManageVideoTorrentPayload, ThumbnailType, VideoCreate, VideoPrivacy, VideoState } from '@shared/models'
 import { CreateJobArgument, JobQueue } from './job-queue/job-queue'
-import { updateVideoMiniatureFromExisting } from './thumbnail'
+import { updateLocalVideoMiniatureFromExisting } from './thumbnail'
 import { moveFilesIfPrivacyChanged } from './video-privacy'
 
 function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> {
@@ -55,7 +55,7 @@ async function buildVideoThumbnailsFromReq (options: {
     const fields = files?.[p.fieldName]
 
     if (fields) {
-      return updateVideoMiniatureFromExisting({
+      return updateLocalVideoMiniatureFromExisting({
         inputPath: fields[0].path,
         video,
         type: p.type,

server/models/video/sql/video/shared/video-table-attributes.ts

@@ -60,6 +60,7 @@ export class VideoTableAttributes {
         'height',
         'width',
         'fileUrl',
+        'onDisk',
         'automaticallyGenerated',
         'videoId',
         'videoPlaylistId',

server/models/video/thumbnail.ts

@@ -69,6 +69,10 @@ export class ThumbnailModel extends Model<Partial<AttributesOnly<ThumbnailModel>
   @Column
   automaticallyGenerated: boolean
 
+  @AllowNull(false)
+  @Column
+  onDisk: boolean
+
   @ForeignKey(() => VideoModel)
   @Column
   videoId: number

support/nginx/peertube

@@ -199,28 +199,6 @@ server {
     alias /var/www/peertube/peertube-latest/client/dist/$1;
   }
 
-  # Bypass PeerTube for performance reasons. Optional.
-  location ~ ^/static/(thumbnails|avatars)/ {
-    if ($request_method = 'OPTIONS') {
-      add_header Access-Control-Allow-Origin  '*';
-      add_header Access-Control-Allow-Methods 'GET, OPTIONS';
-      add_header Access-Control-Allow-Headers 'Range,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
-      add_header Access-Control-Max-Age       1728000; # Preflight request can be cached 20 days
-      add_header Content-Type                 'text/plain charset=UTF-8';
-      add_header Content-Length               0;
-      return 204;
-    }
-
-    add_header Access-Control-Allow-Origin    '*';
-    add_header Access-Control-Allow-Methods   'GET, OPTIONS';
-    add_header Access-Control-Allow-Headers   'Range,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
-    add_header Cache-Control                  "public, max-age=7200"; # Cache response 2 hours
-
-    rewrite ^/static/(.*)$ /$1 break;
-
-    try_files $uri @api;
-  }
-
   location ~ ^(/static/(webseed|streaming-playlists)/private/)|^/download {
     # We can't rate limit a try_files directive, so we need to duplicate @api