Add previews cache system between pods

.gitignore

@@ -12,6 +12,7 @@
 /certs/
 /logs/
 /torrents/
+/cache/
 /config/production.yaml
 /ffmpeg/
 /*.sublime-project

config/default.yaml

@@ -22,6 +22,11 @@ storage:
   previews: 'previews/'
   thumbnails: 'thumbnails/'
   torrents: 'torrents/'
+  cache: 'cache/'
+
+cache:
+  previews:
+    size: 1 # Max number of previews you want to cache
 
 admin:
   email: 'admin@example.com'

config/production.yaml.example

@@ -23,6 +23,7 @@ storage:
   previews: 'previews/'
   thumbnails: 'thumbnails/'
   torrents: 'torrents/'
+  cache: 'cache/'
 
 admin:
   email: 'admin@example.com'

config/test-1.yaml

@@ -13,8 +13,10 @@ storage:
   certs: 'test1/certs/'
   videos: 'test1/videos/'
   logs: 'test1/logs/'
+  previews: 'test1/previews/'
   thumbnails: 'test1/thumbnails/'
   torrents: 'test1/torrents/'
+  cache: 'test1/cache/'
 
 admin:
   email: 'admin1@example.com'

config/test-2.yaml

@@ -13,8 +13,10 @@ storage:
   certs: 'test2/certs/'
   videos: 'test2/videos/'
   logs: 'test2/logs/'
+  previews: 'test2/previews/'
   thumbnails: 'test2/thumbnails/'
   torrents: 'test2/torrents/'
+  cache: 'test2/cache/'
 
 admin:
   email: 'admin2@example.com'

config/test-3.yaml

@@ -13,8 +13,10 @@ storage:
   certs: 'test3/certs/'
   videos: 'test3/videos/'
   logs: 'test3/logs/'
+  previews: 'test3/previews/'
   thumbnails: 'test3/thumbnails/'
   torrents: 'test3/torrents/'
+  cache: 'test3/cache/'
 
 admin:
   email: 'admin3@example.com'

config/test-4.yaml

@@ -13,8 +13,10 @@ storage:
   certs: 'test4/certs/'
   videos: 'test4/videos/'
   logs: 'test4/logs/'
+  previews: 'test4/previews/'
   thumbnails: 'test4/thumbnails/'
   torrents: 'test4/torrents/'
+  cache: 'test4/cache/'
 
 admin:
   email: 'admin4@example.com'

config/test-5.yaml

@@ -13,8 +13,10 @@ storage:
   certs: 'test5/certs/'
   videos: 'test5/videos/'
   logs: 'test5/logs/'
+  previews: 'test5/previews/'
   thumbnails: 'test5/thumbnails/'
   torrents: 'test5/torrents/'
+  cache: 'test5/cache/'
 
 admin:
   email: 'admin5@example.com'

config/test-6.yaml

@@ -13,8 +13,10 @@ storage:
   certs: 'test6/certs/'
   videos: 'test6/videos/'
   logs: 'test6/logs/'
+  previews: 'test6/previews/'
   thumbnails: 'test6/thumbnails/'
   torrents: 'test6/torrents/'
+  cache: 'test6/cache/'
 
 admin:
   email: 'admin6@example.com'

package.json

@@ -47,6 +47,7 @@
   },
   "dependencies": {
     "async": "^2.0.0",
+    "async-lru": "^1.1.1",
     "bcrypt": "^1.0.2",
     "bittorrent-tracker": "^9.0.0",
     "bluebird": "^3.5.0",

server.ts

@@ -47,7 +47,7 @@ if (errorMessage !== null) {
 
 // ----------- PeerTube modules -----------
 import { migrate, installApplication } from './server/initializers'
-import { JobScheduler, activateSchedulers } from './server/lib'
+import { JobScheduler, activateSchedulers, VideosPreviewCache } from './server/lib'
 import * as customValidators from './server/helpers/custom-validators'
 import { apiRouter, clientsRouter, staticRouter } from './server/controllers'
 
@@ -147,6 +147,8 @@ function onDatabaseInitDone () {
         // Activate job scheduler
         JobScheduler.Instance.activate()
 
+        VideosPreviewCache.Instance.init(CONFIG.CACHE.PREVIEWS.SIZE)
+
         logger.info('Server listening on port %d', port)
         logger.info('Webserver: %s', CONFIG.WEBSERVER.URL)
       })

server/controllers/static.ts

@@ -6,6 +6,7 @@ import {
   STATIC_MAX_AGE,
   STATIC_PATHS
 } from '../initializers'
+import { VideosPreviewCache } from '../lib'
 
 const staticRouter = express.Router()
 
@@ -38,8 +39,8 @@ staticRouter.use(
 // Video previews path for express
 const previewsPhysicalPath = CONFIG.STORAGE.PREVIEWS_DIR
 staticRouter.use(
-  STATIC_PATHS.PREVIEWS,
-  express.static(previewsPhysicalPath, { maxAge: STATIC_MAX_AGE })
+  STATIC_PATHS.PREVIEWS + ':uuid.jpg',
+  getPreview
 )
 
 // ---------------------------------------------------------------------------
@@ -47,3 +48,14 @@ staticRouter.use(
 export {
   staticRouter
 }
+
+// ---------------------------------------------------------------------------
+
+function getPreview (req: express.Request, res: express.Response, next: express.NextFunction) {
+  VideosPreviewCache.Instance.getPreviewPath(req.params.uuid)
+    .then(path => {
+      if (!path) return res.sendStatus(404)
+
+      return res.sendFile(path, { maxAge: STATIC_MAX_AGE })
+    })
+}

server/helpers/core-utils.ts

@@ -16,6 +16,7 @@ import {
 import * as mkdirp from 'mkdirp'
 import * as bcrypt from 'bcrypt'
 import * as createTorrent from 'create-torrent'
+import * as rimraf from 'rimraf'
 import * as openssl from 'openssl-wrapper'
 import * as Promise from 'bluebird'
 
@@ -83,6 +84,7 @@ const bcryptComparePromise = promisify2<any, string, boolean>(bcrypt.compare)
 const bcryptGenSaltPromise = promisify1<number, string>(bcrypt.genSalt)
 const bcryptHashPromise = promisify2<any, string|number, string>(bcrypt.hash)
 const createTorrentPromise = promisify2<string, any, any>(createTorrent)
+const rimrafPromise = promisify1WithVoid<string>(rimraf)
 
 // ---------------------------------------------------------------------------
 
@@ -105,5 +107,6 @@ export {
   bcryptComparePromise,
   bcryptGenSaltPromise,
   bcryptHashPromise,
-  createTorrentPromise
+  createTorrentPromise,
+  rimrafPromise
 }

server/initializers/constants.ts

@@ -61,7 +61,8 @@ const CONFIG = {
     VIDEOS_DIR: join(root(), config.get<string>('storage.videos')),
     THUMBNAILS_DIR: join(root(), config.get<string>('storage.thumbnails')),
     PREVIEWS_DIR: join(root(), config.get<string>('storage.previews')),
-    TORRENTS_DIR: join(root(), config.get<string>('storage.torrents'))
+    TORRENTS_DIR: join(root(), config.get<string>('storage.torrents')),
+    CACHE_DIR: join(root(), config.get<string>('storage.cache'))
   },
   WEBSERVER: {
     SCHEME: config.get<boolean>('webserver.https') === true ? 'https' : 'http',
@@ -80,6 +81,11 @@ const CONFIG = {
   TRANSCODING: {
     ENABLED: config.get<boolean>('transcoding.enabled'),
     THREADS: config.get<number>('transcoding.threads')
+  },
+  CACHE: {
+    PREVIEWS: {
+      SIZE: config.get<number>('cache.previews.size')
+    }
   }
 }
 CONFIG.WEBSERVER.URL = CONFIG.WEBSERVER.SCHEME + '://' + CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT
@@ -278,6 +284,13 @@ let STATIC_MAX_AGE = '30d'
 const THUMBNAILS_SIZE = '200x110'
 const PREVIEWS_SIZE = '640x480'
 
+// Subfolders of cache directory
+const CACHE = {
+  DIRECTORIES: {
+    PREVIEWS: join(CONFIG.STORAGE.CACHE_DIR, 'previews')
+  }
+}
+
 // ---------------------------------------------------------------------------
 
 const USER_ROLES: { [ id: string ]: UserRole } = {
@@ -307,6 +320,7 @@ if (isTestInstance() === true) {
 export {
   API_VERSION,
   BCRYPT_SALT_SIZE,
+  CACHE,
   CONFIG,
   CONSTRAINTS_FIELDS,
   FRIEND_SCORE,

server/initializers/installer.ts

@@ -4,12 +4,13 @@ import * as passwordGenerator from 'password-generator'
 import * as Promise from 'bluebird'
 
 import { database as db } from './database'
-import { USER_ROLES, CONFIG, LAST_MIGRATION_VERSION } from './constants'
+import { USER_ROLES, CONFIG, LAST_MIGRATION_VERSION, CACHE } from './constants'
 import { clientsExist, usersExist } from './checker'
-import { logger, createCertsIfNotExist, root, mkdirpPromise } from '../helpers'
+import { logger, createCertsIfNotExist, root, mkdirpPromise, rimrafPromise } from '../helpers'
 
 function installApplication () {
   return db.sequelize.sync()
+    .then(() => removeCacheDirectories())
     .then(() => createDirectoriesIfNotExist())
     .then(() => createCertsIfNotExist())
     .then(() => createOAuthClientIfNotExist())
@@ -24,13 +25,34 @@ export {
 
 // ---------------------------------------------------------------------------
 
+function removeCacheDirectories () {
+  const cacheDirectories = CACHE.DIRECTORIES
+
+  const tasks = []
+
+  // Cache directories
+  Object.keys(cacheDirectories).forEach(key => {
+    const dir = cacheDirectories[key]
+    tasks.push(rimrafPromise(dir))
+  })
+
+  return Promise.all(tasks)
+}
+
 function createDirectoriesIfNotExist () {
-  const storages = config.get('storage')
+  const storages = CONFIG.STORAGE
+  const cacheDirectories = CACHE.DIRECTORIES
 
   const tasks = []
   Object.keys(storages).forEach(key => {
     const dir = storages[key]
-    tasks.push(mkdirpPromise(join(root(), dir)))
+    tasks.push(mkdirpPromise(dir))
+  })
+
+  // Cache directories
+  Object.keys(cacheDirectories).forEach(key => {
+    const dir = cacheDirectories[key]
+    tasks.push(mkdirpPromise(dir))
   })
 
   return Promise.all(tasks)

server/lib/cache/index.ts

@@ -0,0 +1 @@
+export * from './videos-preview-cache'

server/lib/cache/videos-preview-cache.ts

@@ -0,0 +1,74 @@
+import * as request from 'request'
+import * as asyncLRU from 'async-lru'
+import { join } from 'path'
+import { createWriteStream } from 'fs'
+import * as Promise from 'bluebird'
+
+import { database as db, CONFIG, CACHE } from '../../initializers'
+import { logger, writeFilePromise, unlinkPromise } from '../../helpers'
+import { VideoInstance } from '../../models'
+import { fetchRemotePreview } from '../../lib'
+
+class VideosPreviewCache {
+
+  private static instance: VideosPreviewCache
+
+  private lru
+
+  private constructor () { }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+
+  init (max: number) {
+    this.lru = new asyncLRU({
+      max,
+      load: (key, cb) => {
+        this.loadPreviews(key)
+          .then(res => cb(null, res))
+          .catch(err => cb(err))
+      }
+    })
+
+    this.lru.on('evict', (obj: { key: string, value: string }) => {
+      unlinkPromise(obj.value).then(() => logger.debug('%s evicted from VideosPreviewCache', obj.value))
+    })
+  }
+
+  getPreviewPath (key: string) {
+    return new Promise<string>((res, rej) => {
+      this.lru.get(key, (err, value) => {
+        err ? rej(err) : res(value)
+      })
+    })
+  }
+
+  private loadPreviews (key: string) {
+    return db.Video.loadByUUIDAndPopulateAuthorAndPodAndTags(key)
+      .then(video => {
+        if (!video) return undefined
+
+        if (video.isOwned()) return join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName())
+
+        return this.saveRemotePreviewAndReturnPath(video)
+      })
+  }
+
+  private saveRemotePreviewAndReturnPath (video: VideoInstance) {
+    const req = fetchRemotePreview(video.Author.Pod, video)
+
+    return new Promise<string>((res, rej) => {
+      const path = join(CACHE.DIRECTORIES.PREVIEWS, video.getPreviewName())
+      const stream = createWriteStream(path)
+
+      req.pipe(stream)
+         .on('finish', () => res(path))
+         .on('error', (err) => rej(err))
+    })
+  }
+}
+
+export {
+  VideosPreviewCache
+}

server/lib/friends.ts

@@ -1,6 +1,7 @@
 import * as request from 'request'
 import * as Sequelize from 'sequelize'
 import * as Promise from 'bluebird'
+import { join } from 'path'
 
 import { database as db } from '../initializers/database'
 import {
@@ -9,7 +10,8 @@ import {
   REQUESTS_IN_PARALLEL,
   REQUEST_ENDPOINTS,
   REQUEST_ENDPOINT_ACTIONS,
-  REMOTE_SCHEME
+  REMOTE_SCHEME,
+  STATIC_PATHS
 } from '../initializers'
 import {
   logger,
@@ -233,6 +235,13 @@ function sendOwnedVideosToPod (podId: number) {
     })
 }
 
+function fetchRemotePreview (pod: PodInstance, video: VideoInstance) {
+  const host = video.Author.Pod.host
+  const path = join(STATIC_PATHS.PREVIEWS, video.getPreviewName())
+
+  return request.get(REMOTE_SCHEME.HTTP + '://' + host + path)
+}
+
 function getRequestScheduler () {
   return requestScheduler
 }
@@ -263,7 +272,8 @@ export {
   sendOwnedVideosToPod,
   getRequestScheduler,
   getRequestVideoQaduScheduler,
-  getRequestVideoEventScheduler
+  getRequestVideoEventScheduler,
+  fetchRemotePreview
 }
 
 // ---------------------------------------------------------------------------

server/lib/index.ts

@@ -1,3 +1,4 @@
+export * from './cache'
 export * from './jobs'
 export * from './request'
 export * from './friends'

server/models/oauth/oauth-token-interface.ts

@@ -35,6 +35,8 @@ export interface OAuthTokenAttributes {
   refreshToken: string
   refreshTokenExpiresAt: Date
 
+  userId?: number
+  oAuthClientId?: number
   User?: UserModel
 }
 

server/models/oauth/oauth-token.ts

@@ -106,10 +106,10 @@ getByRefreshTokenAndPopulateClient = function (refreshToken: string) {
         refreshToken: token.refreshToken,
         refreshTokenExpiresAt: token.refreshTokenExpiresAt,
         client: {
-          id: token['client'].id
+          id: token.oAuthClientId
         },
         user: {
-          id: token['user']
+          id: token.userId
         }
       }
 

server/models/video/video.ts

@@ -451,6 +451,7 @@ toFormatedJSON = function (this: VideoInstance) {
     dislikes: this.dislikes,
     tags: map<TagInstance, string>(this.Tags, 'name'),
     thumbnailPath: join(STATIC_PATHS.THUMBNAILS, this.getThumbnailName()),
+    previewPath: join(STATIC_PATHS.PREVIEWS, this.getPreviewName()),
     createdAt: this.createdAt,
     updatedAt: this.updatedAt
   }

server/tests/api/multiple-pods.js

@@ -747,7 +747,7 @@ describe('Test multiple pods', function () {
           expect(videos[0].name).not.to.equal(toRemove[1].name)
           expect(videos[1].name).not.to.equal(toRemove[1].name)
 
-          videoUUID = videos[0].uuid
+          videoUUID = videos.find(video => video.name === 'my super name for pod 1').uuid
 
           callback()
         })
@@ -781,6 +781,23 @@ describe('Test multiple pods', function () {
         })
       }, done)
     })
+
+    it('Should get the preview from each pod', function (done) {
+      each(servers, function (server, callback) {
+        videosUtils.getVideo(server.url, videoUUID, function (err, res) {
+          if (err) throw err
+
+          const video = res.body
+
+          videosUtils.testVideoImage(server.url, 'video_short1-preview.webm', video.previewPath, function (err, test) {
+            if (err) throw err
+            expect(test).to.equal(true)
+
+            callback()
+          })
+        })
+      }, done)
+    })
   })
 
   after(function (done) {

server/tests/utils/videos.js

@@ -195,7 +195,7 @@ function searchVideoWithSort (url, search, sort, end) {
     .end(end)
 }
 
-function testVideoImage (url, videoName, imagePath, callback) {
+function testVideoImage (url, imageName, imagePath, callback) {
   // Don't test images if the node env is not set
   // Because we need a special ffmpeg version for this test
   if (process.env.NODE_TEST_IMAGE) {
@@ -205,7 +205,7 @@ function testVideoImage (url, videoName, imagePath, callback) {
       .end(function (err, res) {
         if (err) return callback(err)
 
-        fs.readFile(pathUtils.join(__dirname, '..', 'api', 'fixtures', videoName + '.jpg'), function (err, data) {
+        fs.readFile(pathUtils.join(__dirname, '..', 'api', 'fixtures', imageName + '.jpg'), function (err, data) {
           if (err) return callback(err)
 
           callback(null, data.equals(res.body))

shared/models/videos/video.model.ts

@@ -17,6 +17,7 @@ export interface Video {
   podHost: string
   tags: string[]
   thumbnailPath: string
+  previewPath: string
   views: number
   likes: number
   dislikes: number

yarn.lock

@@ -285,6 +285,12 @@ async-each@^1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/async-each/-/async-each-1.0.1.tgz#19d386a1d9edc6e7c1c85d388aedbcc56d33602d"
 
+async-lru@^1.1.1:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/async-lru/-/async-lru-1.1.1.tgz#3edbf7e96484d5c2dd852a8bf9794fc07f5e7274"
+  dependencies:
+    lru "^3.1.0"
+
 async@>=0.2.9, async@^2.0.0:
   version "2.4.1"
   resolved "https://registry.yarnpkg.com/async/-/async-2.4.1.tgz#62a56b279c98a11d0987096a01cc3eeb8eb7bbd7"