From 7eba5e1fa81c8e54cb8fe298a96e8070afa50921 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 4 Feb 2020 15:00:47 +0100 Subject: [PATCH] Add model cache for video When fetching only immutable attributes --- server/controllers/activitypub/client.ts | 20 +++++----- server/helpers/middlewares/videos.ts | 17 +++++++- server/helpers/video.ts | 12 ++++-- .../middlewares/validators/videos/videos.ts | 5 ++- server/models/model-cache.ts | 39 ++++++++++++++++++- server/models/video/video.ts | 30 +++++++++++++- server/typings/express.ts | 3 +- server/typings/models/video/video.ts | 1 + 8 files changed, 106 insertions(+), 21 deletions(-) diff --git a/server/controllers/activitypub/client.ts b/server/controllers/activitypub/client.ts index 2812bfe1e..9a5fd6084 100644 --- a/server/controllers/activitypub/client.ts +++ b/server/controllers/activitypub/client.ts @@ -37,7 +37,7 @@ import { buildDislikeActivity } from '../../lib/activitypub/send/send-dislike' import { videoPlaylistElementAPGetValidator, videoPlaylistsGetValidator } from '../../middlewares/validators/videos/video-playlists' import { VideoPlaylistModel } from '../../models/video/video-playlist' import { VideoPlaylistPrivacy } from '../../../shared/models/videos/playlist/video-playlist-privacy.model' -import { MAccountId, MActorId, MVideo, MVideoAPWithoutCaption } from '@server/typings/models' +import { MAccountId, MActorId, MVideo, MVideoAPWithoutCaption, MVideoId } from '@server/typings/models' const activityPubClientRouter = express.Router() @@ -85,7 +85,7 @@ activityPubClientRouter.get('/videos/watch/:id/activity', ) activityPubClientRouter.get('/videos/watch/:id/announces', executeIfActivityPub, - asyncMiddleware(videosCustomGetValidator('only-video')), + asyncMiddleware(videosCustomGetValidator('only-immutable-attributes')), asyncMiddleware(videoAnnouncesController) ) activityPubClientRouter.get('/videos/watch/:id/announces/:actorId', @@ -95,17 +95,17 @@ activityPubClientRouter.get('/videos/watch/:id/announces/:actorId', ) activityPubClientRouter.get('/videos/watch/:id/likes', executeIfActivityPub, - asyncMiddleware(videosCustomGetValidator('only-video')), + asyncMiddleware(videosCustomGetValidator('only-immutable-attributes')), asyncMiddleware(videoLikesController) ) activityPubClientRouter.get('/videos/watch/:id/dislikes', executeIfActivityPub, - asyncMiddleware(videosCustomGetValidator('only-video')), + asyncMiddleware(videosCustomGetValidator('only-immutable-attributes')), asyncMiddleware(videoDislikesController) ) activityPubClientRouter.get('/videos/watch/:id/comments', executeIfActivityPub, - asyncMiddleware(videosCustomGetValidator('only-video')), + asyncMiddleware(videosCustomGetValidator('only-immutable-attributes')), asyncMiddleware(videoCommentsController) ) activityPubClientRouter.get('/videos/watch/:videoId/comments/:commentId', @@ -238,7 +238,7 @@ async function videoAnnounceController (req: express.Request, res: express.Respo } async function videoAnnouncesController (req: express.Request, res: express.Response) { - const video = res.locals.onlyVideo + const video = res.locals.onlyImmutableVideo const handler = async (start: number, count: number) => { const result = await VideoShareModel.listAndCountByVideoId(video.id, start, count) @@ -253,21 +253,21 @@ async function videoAnnouncesController (req: express.Request, res: express.Resp } async function videoLikesController (req: express.Request, res: express.Response) { - const video = res.locals.onlyVideo + const video = res.locals.onlyImmutableVideo const json = await videoRates(req, 'like', video, getVideoLikesActivityPubUrl(video)) return activityPubResponse(activityPubContextify(json), res) } async function videoDislikesController (req: express.Request, res: express.Response) { - const video = res.locals.onlyVideo + const video = res.locals.onlyImmutableVideo const json = await videoRates(req, 'dislike', video, getVideoDislikesActivityPubUrl(video)) return activityPubResponse(activityPubContextify(json), res) } async function videoCommentsController (req: express.Request, res: express.Response) { - const video = res.locals.onlyVideo + const video = res.locals.onlyImmutableVideo const handler = async (start: number, count: number) => { const result = await VideoCommentModel.listAndCountByVideoId(video.id, start, count) @@ -386,7 +386,7 @@ async function actorPlaylists (req: express.Request, account: MAccountId) { return activityPubCollectionPagination(WEBSERVER.URL + req.path, handler, req.query.page) } -function videoRates (req: express.Request, rateType: VideoRateType, video: MVideo, url: string) { +function videoRates (req: express.Request, rateType: VideoRateType, video: MVideoId, url: string) { const handler = async (start: number, count: number) => { const result = await AccountVideoRateModel.listAndCountAccountUrlsByVideoId(rateType, video.id, start, count) return { diff --git a/server/helpers/middlewares/videos.ts b/server/helpers/middlewares/videos.ts index 74f529804..409f78650 100644 --- a/server/helpers/middlewares/videos.ts +++ b/server/helpers/middlewares/videos.ts @@ -2,7 +2,16 @@ import { Response } from 'express' import { fetchVideo, VideoFetchType } from '../video' import { UserRight } from '../../../shared/models/users' import { VideoChannelModel } from '../../models/video/video-channel' -import { MUser, MUserAccountId, MVideoAccountLight, MVideoFullLight, MVideoThumbnail, MVideoWithRights } from '@server/typings/models' +import { + MUser, + MUserAccountId, + MVideoAccountLight, + MVideoFullLight, + MVideoIdThumbnail, + MVideoImmutable, + MVideoThumbnail, + MVideoWithRights +} from '@server/typings/models' async function doesVideoExist (id: number | string, res: Response, fetchType: VideoFetchType = 'all') { const userId = res.locals.oauth ? res.locals.oauth.token.User.id : undefined @@ -22,8 +31,12 @@ async function doesVideoExist (id: number | string, res: Response, fetchType: Vi res.locals.videoAll = video as MVideoFullLight break + case 'only-immutable-attributes': + res.locals.onlyImmutableVideo = video as MVideoImmutable + break + case 'id': - res.locals.videoId = video + res.locals.videoId = video as MVideoIdThumbnail break case 'only-video': diff --git a/server/helpers/video.ts b/server/helpers/video.ts index 5b9c026b1..907564703 100644 --- a/server/helpers/video.ts +++ b/server/helpers/video.ts @@ -5,13 +5,15 @@ import { MVideoFullLight, MVideoIdThumbnail, MVideoThumbnail, - MVideoWithRights + MVideoWithRights, + MVideoImmutable } from '@server/typings/models' import { Response } from 'express' -type VideoFetchType = 'all' | 'only-video' | 'only-video-with-rights' | 'id' | 'none' +type VideoFetchType = 'all' | 'only-video' | 'only-video-with-rights' | 'id' | 'none' | 'only-immutable-attributes' function fetchVideo (id: number | string, fetchType: 'all', userId?: number): Bluebird +function fetchVideo (id: number | string, fetchType: 'only-immutable-attributes'): Bluebird function fetchVideo (id: number | string, fetchType: 'only-video', userId?: number): Bluebird function fetchVideo (id: number | string, fetchType: 'only-video-with-rights', userId?: number): Bluebird function fetchVideo (id: number | string, fetchType: 'id' | 'none', userId?: number): Bluebird @@ -19,14 +21,16 @@ function fetchVideo ( id: number | string, fetchType: VideoFetchType, userId?: number -): Bluebird +): Bluebird function fetchVideo ( id: number | string, fetchType: VideoFetchType, userId?: number -): Bluebird { +): Bluebird { if (fetchType === 'all') return VideoModel.loadAndPopulateAccountAndServerAndTags(id, undefined, userId) + if (fetchType === 'only-immutable-attributes') return VideoModel.loadImmutableAttributes(id) + if (fetchType === 'only-video-with-rights') return VideoModel.loadWithRights(id) if (fetchType === 'only-video') return VideoModel.load(id) diff --git a/server/middlewares/validators/videos/videos.ts b/server/middlewares/validators/videos/videos.ts index 11dd02706..c14184b35 100644 --- a/server/middlewares/validators/videos/videos.ts +++ b/server/middlewares/validators/videos/videos.ts @@ -147,7 +147,10 @@ async function checkVideoFollowConstraints (req: express.Request, res: express.R }) } -const videosCustomGetValidator = (fetchType: 'all' | 'only-video' | 'only-video-with-rights', authenticateInQuery = false) => { +const videosCustomGetValidator = ( + fetchType: 'all' | 'only-video' | 'only-video-with-rights' | 'only-immutable-attributes', + authenticateInQuery = false +) => { return [ param('id').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid id'), diff --git a/server/models/model-cache.ts b/server/models/model-cache.ts index bfa163b6b..8afe3834f 100644 --- a/server/models/model-cache.ts +++ b/server/models/model-cache.ts @@ -6,6 +6,10 @@ type ModelCacheType = 'local-account-name' | 'local-actor-name' | 'local-actor-url' + | 'video-immutable' + +type DeleteKey = + 'video' class ModelCache { @@ -14,7 +18,14 @@ class ModelCache { private readonly localCache: { [id in ModelCacheType]: Map } = { 'local-account-name': new Map(), 'local-actor-name': new Map(), - 'local-actor-url': new Map() + 'local-actor-url': new Map(), + 'video-immutable': new Map() + } + + private readonly deleteIds: { + [deleteKey in DeleteKey]: Map + } = { + video: new Map() } private constructor () { @@ -29,8 +40,9 @@ class ModelCache { key: string fun: () => Bluebird whitelist?: () => boolean + deleteKey?: DeleteKey }) { - const { cacheType, key, fun, whitelist } = options + const { cacheType, key, fun, whitelist, deleteKey } = options if (whitelist && whitelist() !== true) return fun() @@ -42,11 +54,34 @@ class ModelCache { } return fun().then(m => { + if (!m) return m + if (!whitelist || whitelist()) cache.set(key, m) + if (deleteKey) { + const map = this.deleteIds[deleteKey] + if (!map.has(m.id)) map.set(m.id, []) + + const a = map.get(m.id) + a.push({ cacheType, key }) + } + return m }) } + + invalidateCache (deleteKey: DeleteKey, modelId: number) { + const map = this.deleteIds[deleteKey] + + if (!map.has(modelId)) return + + for (const toDelete of map.get(modelId)) { + logger.debug('Removing %s -> %d of model cache %s -> %s.', deleteKey, modelId, toDelete.cacheType, toDelete.key) + this.localCache[toDelete.cacheType].delete(toDelete.key) + } + + map.delete(modelId) + } } export { diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 1ec8d717e..9e02d163f 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -120,7 +120,7 @@ import { MVideoFormattableDetails, MVideoForUser, MVideoFullLight, - MVideoIdThumbnail, + MVideoIdThumbnail, MVideoImmutable, MVideoThumbnail, MVideoThumbnailBlacklist, MVideoWithAllFiles, @@ -132,6 +132,7 @@ import { MThumbnail } from '../../typings/models/video/thumbnail' import { VideoFile } from '@shared/models/videos/video-file.model' import { getHLSDirectory, getTorrentFileName, getTorrentFilePath, getVideoFilename, getVideoFilePath } from '@server/lib/video-paths' import validator from 'validator' +import { ModelCache } from '@server/models/model-cache' export enum ScopeNames { AVAILABLE_FOR_LIST_IDS = 'AVAILABLE_FOR_LIST_IDS', @@ -1074,6 +1075,11 @@ export class VideoModel extends Model { return undefined } + @BeforeDestroy + static invalidateCache (instance: VideoModel) { + ModelCache.Instance.invalidateCache('video', instance.id) + } + static listLocal (): Bluebird { const query = { where: { @@ -1468,6 +1474,28 @@ export class VideoModel extends Model { ]).findOne(options) } + static loadImmutableAttributes (id: number | string, t?: Transaction): Bluebird { + const fun = () => { + const where = buildWhereIdOrUUID(id) + const options = { + attributes: [ + 'id', 'url', 'uuid' + ], + where, + transaction: t + } + + return VideoModel.unscoped().findOne(options) + } + + return ModelCache.Instance.doCache({ + cacheType: 'video-immutable', + key: '' + id, + deleteKey: 'video', + fun + }) + } + static loadWithRights (id: number | string, t?: Transaction): Bluebird { const where = buildWhereIdOrUUID(id) const options = { diff --git a/server/typings/express.ts b/server/typings/express.ts index 43a9b2c99..f4188bf3d 100644 --- a/server/typings/express.ts +++ b/server/typings/express.ts @@ -21,7 +21,7 @@ import { } from './models' import { MVideoPlaylistFull, MVideoPlaylistFullSummary } from './models/video/video-playlist' import { MVideoImportDefault } from '@server/typings/models/video/video-import' -import { MAccountBlocklist, MActorUrl, MStreamingPlaylist, MVideoFile } from '@server/typings/models' +import { MAccountBlocklist, MActorUrl, MStreamingPlaylist, MVideoFile, MVideoImmutable } from '@server/typings/models' import { MVideoPlaylistElement, MVideoPlaylistElementVideoUrlPlaylistPrivacy } from '@server/typings/models/video/video-playlist-element' import { MAccountVideoRateAccountVideo } from '@server/typings/models/video/video-rate' import { MVideoChangeOwnershipFull } from './models/video/video-change-ownership' @@ -35,6 +35,7 @@ declare module 'express' { locals: { videoAll?: MVideoFullLight + onlyImmutableVideo?: MVideoImmutable onlyVideo?: MVideoThumbnail onlyVideoWithRights?: MVideoWithRights videoId?: MVideoIdThumbnail diff --git a/server/typings/models/video/video.ts b/server/typings/models/video/video.ts index 7eff0a913..3ebb5a762 100644 --- a/server/typings/models/video/video.ts +++ b/server/typings/models/video/video.ts @@ -37,6 +37,7 @@ export type MVideoId = Pick export type MVideoUrl = Pick export type MVideoUUID = Pick +export type MVideoImmutable = Pick export type MVideoIdUrl = MVideoId & MVideoUrl export type MVideoFeed = Pick