Don't refresh videos when processing views
It allows us to use a cache
This commit is contained in:
parent
7eba5e1fa8
commit
943e519390
|
@ -38,14 +38,23 @@ function fetchVideo (
|
||||||
if (fetchType === 'id' || fetchType === 'none') return VideoModel.loadOnlyId(id)
|
if (fetchType === 'id' || fetchType === 'none') return VideoModel.loadOnlyId(id)
|
||||||
}
|
}
|
||||||
|
|
||||||
type VideoFetchByUrlType = 'all' | 'only-video'
|
type VideoFetchByUrlType = 'all' | 'only-video' | 'only-immutable-attributes'
|
||||||
|
|
||||||
function fetchVideoByUrl (url: string, fetchType: 'all'): Bluebird<MVideoAccountLightBlacklistAllFiles>
|
function fetchVideoByUrl (url: string, fetchType: 'all'): Bluebird<MVideoAccountLightBlacklistAllFiles>
|
||||||
|
function fetchVideoByUrl (url: string, fetchType: 'only-immutable-attributes'): Bluebird<MVideoImmutable>
|
||||||
function fetchVideoByUrl (url: string, fetchType: 'only-video'): Bluebird<MVideoThumbnail>
|
function fetchVideoByUrl (url: string, fetchType: 'only-video'): Bluebird<MVideoThumbnail>
|
||||||
function fetchVideoByUrl (url: string, fetchType: VideoFetchByUrlType): Bluebird<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail>
|
function fetchVideoByUrl (
|
||||||
function fetchVideoByUrl (url: string, fetchType: VideoFetchByUrlType): Bluebird<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail> {
|
url: string,
|
||||||
|
fetchType: VideoFetchByUrlType
|
||||||
|
): Bluebird<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail | MVideoImmutable>
|
||||||
|
function fetchVideoByUrl (
|
||||||
|
url: string,
|
||||||
|
fetchType: VideoFetchByUrlType
|
||||||
|
): Bluebird<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail | MVideoImmutable> {
|
||||||
if (fetchType === 'all') return VideoModel.loadByUrlAndPopulateAccount(url)
|
if (fetchType === 'all') return VideoModel.loadByUrlAndPopulateAccount(url)
|
||||||
|
|
||||||
|
if (fetchType === 'only-immutable-attributes') return VideoModel.loadByUrlImmutableAttributes(url)
|
||||||
|
|
||||||
if (fetchType === 'only-video') return VideoModel.loadByUrl(url)
|
if (fetchType === 'only-video') return VideoModel.loadByUrl(url)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -4,7 +4,15 @@ import { ACTIVITY_PUB } from '../../initializers/constants'
|
||||||
import { ActorModel } from '../../models/activitypub/actor'
|
import { ActorModel } from '../../models/activitypub/actor'
|
||||||
import { VideoModel } from '../../models/video/video'
|
import { VideoModel } from '../../models/video/video'
|
||||||
import { VideoShareModel } from '../../models/video/video-share'
|
import { VideoShareModel } from '../../models/video/video-share'
|
||||||
import { MActorFollowersUrl, MActorLight, MCommentOwner, MCommentOwnerVideo, MVideo, MVideoAccountLight } from '../../typings/models'
|
import {
|
||||||
|
MActorFollowersUrl,
|
||||||
|
MActorLight,
|
||||||
|
MCommentOwner,
|
||||||
|
MCommentOwnerVideo,
|
||||||
|
MVideo,
|
||||||
|
MVideoAccountLight,
|
||||||
|
MVideoId
|
||||||
|
} from '../../typings/models'
|
||||||
|
|
||||||
function getRemoteVideoAudience (video: MVideoAccountLight, actorsInvolvedInVideo: MActorFollowersUrl[]): ActivityAudience {
|
function getRemoteVideoAudience (video: MVideoAccountLight, actorsInvolvedInVideo: MActorFollowersUrl[]): ActivityAudience {
|
||||||
return {
|
return {
|
||||||
|
@ -48,7 +56,7 @@ function getAudienceFromFollowersOf (actorsInvolvedInObject: MActorFollowersUrl[
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function getActorsInvolvedInVideo (video: MVideo, t: Transaction) {
|
async function getActorsInvolvedInVideo (video: MVideoId, t: Transaction) {
|
||||||
const actors: MActorLight[] = await VideoShareModel.loadActorsByShare(video.id, t)
|
const actors: MActorLight[] = await VideoShareModel.loadActorsByShare(video.id, t)
|
||||||
|
|
||||||
const videoAll = video as VideoModel
|
const videoAll = video as VideoModel
|
||||||
|
|
|
@ -23,7 +23,8 @@ async function processCreateView (activity: ActivityView | ActivityCreate, byAct
|
||||||
|
|
||||||
const options = {
|
const options = {
|
||||||
videoObject,
|
videoObject,
|
||||||
fetchType: 'only-video' as 'only-video'
|
fetchType: 'only-immutable-attributes' as 'only-immutable-attributes',
|
||||||
|
allowRefresh: false as false
|
||||||
}
|
}
|
||||||
const { video } = await getOrCreateVideoAndAccountAndChannel(options)
|
const { video } = await getOrCreateVideoAndAccountAndChannel(options)
|
||||||
|
|
||||||
|
|
|
@ -7,7 +7,7 @@ import { JobQueue } from '../../job-queue'
|
||||||
import { getActorsInvolvedInVideo, getAudienceFromFollowersOf, getRemoteVideoAudience } from '../audience'
|
import { getActorsInvolvedInVideo, getAudienceFromFollowersOf, getRemoteVideoAudience } from '../audience'
|
||||||
import { getServerActor } from '../../../helpers/utils'
|
import { getServerActor } from '../../../helpers/utils'
|
||||||
import { afterCommitIfTransaction } from '../../../helpers/database-utils'
|
import { afterCommitIfTransaction } from '../../../helpers/database-utils'
|
||||||
import { MActorWithInboxes, MActor, MActorId, MActorLight, MVideo, MVideoAccountLight } from '../../../typings/models'
|
import { MActorWithInboxes, MActor, MActorId, MActorLight, MVideo, MVideoAccountLight, MVideoId } from '../../../typings/models'
|
||||||
import { ContextType } from '@server/helpers/activitypub'
|
import { ContextType } from '@server/helpers/activitypub'
|
||||||
|
|
||||||
async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAudience) => Activity, options: {
|
async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAudience) => Activity, options: {
|
||||||
|
@ -43,7 +43,7 @@ async function forwardVideoRelatedActivity (
|
||||||
activity: Activity,
|
activity: Activity,
|
||||||
t: Transaction,
|
t: Transaction,
|
||||||
followersException: MActorWithInboxes[] = [],
|
followersException: MActorWithInboxes[] = [],
|
||||||
video: MVideo
|
video: MVideoId
|
||||||
) {
|
) {
|
||||||
// Mastodon does not add our announces in audience, so we forward to them manually
|
// Mastodon does not add our announces in audience, so we forward to them manually
|
||||||
const additionalActors = await getActorsInvolvedInVideo(video, t)
|
const additionalActors = await getActorsInvolvedInVideo(video, t)
|
||||||
|
|
|
@ -68,7 +68,7 @@ import {
|
||||||
MVideoAPWithoutCaption,
|
MVideoAPWithoutCaption,
|
||||||
MVideoFile,
|
MVideoFile,
|
||||||
MVideoFullLight,
|
MVideoFullLight,
|
||||||
MVideoId,
|
MVideoId, MVideoImmutable,
|
||||||
MVideoThumbnail
|
MVideoThumbnail
|
||||||
} from '../../typings/models'
|
} from '../../typings/models'
|
||||||
import { MThumbnail } from '../../typings/models/video/thumbnail'
|
import { MThumbnail } from '../../typings/models/video/thumbnail'
|
||||||
|
@ -200,24 +200,41 @@ async function syncVideoExternalAttributes (video: MVideo, fetchedVideo: VideoTo
|
||||||
await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJobWithPromise({ type: 'activitypub-http-fetcher', payload }))
|
await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJobWithPromise({ type: 'activitypub-http-fetcher', payload }))
|
||||||
}
|
}
|
||||||
|
|
||||||
function getOrCreateVideoAndAccountAndChannel (options: {
|
type GetVideoResult <T> = Promise<{
|
||||||
|
video: T
|
||||||
|
created: boolean
|
||||||
|
autoBlacklisted?: boolean
|
||||||
|
}>
|
||||||
|
|
||||||
|
type GetVideoParamAll = {
|
||||||
videoObject: { id: string } | string
|
videoObject: { id: string } | string
|
||||||
syncParam?: SyncParam
|
syncParam?: SyncParam
|
||||||
fetchType?: 'all'
|
fetchType?: 'all'
|
||||||
allowRefresh?: boolean
|
allowRefresh?: boolean
|
||||||
}): Promise<{ video: MVideoAccountLightBlacklistAllFiles, created: boolean, autoBlacklisted?: boolean }>
|
}
|
||||||
function getOrCreateVideoAndAccountAndChannel (options: {
|
|
||||||
|
type GetVideoParamImmutable = {
|
||||||
videoObject: { id: string } | string
|
videoObject: { id: string } | string
|
||||||
syncParam?: SyncParam
|
syncParam?: SyncParam
|
||||||
fetchType?: VideoFetchByUrlType
|
fetchType: 'only-immutable-attributes'
|
||||||
|
allowRefresh: false
|
||||||
|
}
|
||||||
|
|
||||||
|
type GetVideoParamOther = {
|
||||||
|
videoObject: { id: string } | string
|
||||||
|
syncParam?: SyncParam
|
||||||
|
fetchType?: 'all' | 'only-video'
|
||||||
allowRefresh?: boolean
|
allowRefresh?: boolean
|
||||||
}): Promise<{ video: MVideoAccountLightBlacklistAllFiles | MVideoThumbnail, created: boolean, autoBlacklisted?: boolean }>
|
}
|
||||||
async function getOrCreateVideoAndAccountAndChannel (options: {
|
|
||||||
videoObject: { id: string } | string
|
function getOrCreateVideoAndAccountAndChannel (options: GetVideoParamAll): GetVideoResult<MVideoAccountLightBlacklistAllFiles>
|
||||||
syncParam?: SyncParam
|
function getOrCreateVideoAndAccountAndChannel (options: GetVideoParamImmutable): GetVideoResult<MVideoImmutable>
|
||||||
fetchType?: VideoFetchByUrlType
|
function getOrCreateVideoAndAccountAndChannel (
|
||||||
allowRefresh?: boolean // true by default
|
options: GetVideoParamOther
|
||||||
}): Promise<{ video: MVideoAccountLightBlacklistAllFiles | MVideoThumbnail, created: boolean, autoBlacklisted?: boolean }> {
|
): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail>
|
||||||
|
async function getOrCreateVideoAndAccountAndChannel (
|
||||||
|
options: GetVideoParamAll | GetVideoParamImmutable | GetVideoParamOther
|
||||||
|
): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail | MVideoImmutable> {
|
||||||
// Default params
|
// Default params
|
||||||
const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false }
|
const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false }
|
||||||
const fetchType = options.fetchType || 'all'
|
const fetchType = options.fetchType || 'all'
|
||||||
|
@ -225,12 +242,13 @@ async function getOrCreateVideoAndAccountAndChannel (options: {
|
||||||
|
|
||||||
// Get video url
|
// Get video url
|
||||||
const videoUrl = getAPId(options.videoObject)
|
const videoUrl = getAPId(options.videoObject)
|
||||||
|
|
||||||
let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType)
|
let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType)
|
||||||
|
|
||||||
if (videoFromDatabase) {
|
if (videoFromDatabase) {
|
||||||
if (videoFromDatabase.isOutdated() && allowRefresh === true) {
|
// If allowRefresh is true, we could not call this function using 'only-immutable-attributes' fetch type
|
||||||
|
if (allowRefresh === true && (videoFromDatabase as MVideoThumbnail).isOutdated()) {
|
||||||
const refreshOptions = {
|
const refreshOptions = {
|
||||||
video: videoFromDatabase,
|
video: videoFromDatabase as MVideoThumbnail,
|
||||||
fetchedType: fetchType,
|
fetchedType: fetchType,
|
||||||
syncParam
|
syncParam
|
||||||
}
|
}
|
||||||
|
|
|
@ -160,6 +160,9 @@ const videosCustomGetValidator = (
|
||||||
if (areValidationErrors(req, res)) return
|
if (areValidationErrors(req, res)) return
|
||||||
if (!await doesVideoExist(req.params.id, res, fetchType)) return
|
if (!await doesVideoExist(req.params.id, res, fetchType)) return
|
||||||
|
|
||||||
|
// Controllers does not need to check video rights
|
||||||
|
if (fetchType === 'only-immutable-attributes') return next()
|
||||||
|
|
||||||
const video = getVideoWithAttributes(res)
|
const video = getVideoWithAttributes(res)
|
||||||
const videoAll = video as MVideoFullLight
|
const videoAll = video as MVideoFullLight
|
||||||
|
|
||||||
|
|
|
@ -6,7 +6,8 @@ type ModelCacheType =
|
||||||
'local-account-name'
|
'local-account-name'
|
||||||
| 'local-actor-name'
|
| 'local-actor-name'
|
||||||
| 'local-actor-url'
|
| 'local-actor-url'
|
||||||
| 'video-immutable'
|
| 'load-video-immutable-id'
|
||||||
|
| 'load-video-immutable-url'
|
||||||
|
|
||||||
type DeleteKey =
|
type DeleteKey =
|
||||||
'video'
|
'video'
|
||||||
|
@ -19,7 +20,8 @@ class ModelCache {
|
||||||
'local-account-name': new Map(),
|
'local-account-name': new Map(),
|
||||||
'local-actor-name': new Map(),
|
'local-actor-name': new Map(),
|
||||||
'local-actor-url': new Map(),
|
'local-actor-url': new Map(),
|
||||||
'video-immutable': new Map()
|
'load-video-immutable-id': new Map(),
|
||||||
|
'load-video-immutable-url': new Map()
|
||||||
}
|
}
|
||||||
|
|
||||||
private readonly deleteIds: {
|
private readonly deleteIds: {
|
||||||
|
|
|
@ -145,6 +145,7 @@ export enum ScopeNames {
|
||||||
WITH_USER_HISTORY = 'WITH_USER_HISTORY',
|
WITH_USER_HISTORY = 'WITH_USER_HISTORY',
|
||||||
WITH_STREAMING_PLAYLISTS = 'WITH_STREAMING_PLAYLISTS',
|
WITH_STREAMING_PLAYLISTS = 'WITH_STREAMING_PLAYLISTS',
|
||||||
WITH_USER_ID = 'WITH_USER_ID',
|
WITH_USER_ID = 'WITH_USER_ID',
|
||||||
|
WITH_IMMUTABLE_ATTRIBUTES = 'WITH_IMMUTABLE_ATTRIBUTES',
|
||||||
WITH_THUMBNAILS = 'WITH_THUMBNAILS'
|
WITH_THUMBNAILS = 'WITH_THUMBNAILS'
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -188,6 +189,9 @@ export type AvailableForListIDsOptions = {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Scopes(() => ({
|
@Scopes(() => ({
|
||||||
|
[ScopeNames.WITH_IMMUTABLE_ATTRIBUTES]: {
|
||||||
|
attributes: [ 'id', 'url', 'uuid', 'remote' ]
|
||||||
|
},
|
||||||
[ScopeNames.FOR_API]: (options: ForAPIOptions) => {
|
[ScopeNames.FOR_API]: (options: ForAPIOptions) => {
|
||||||
const query: FindOptions = {
|
const query: FindOptions = {
|
||||||
include: [
|
include: [
|
||||||
|
@ -1476,20 +1480,16 @@ export class VideoModel extends Model<VideoModel> {
|
||||||
|
|
||||||
static loadImmutableAttributes (id: number | string, t?: Transaction): Bluebird<MVideoImmutable> {
|
static loadImmutableAttributes (id: number | string, t?: Transaction): Bluebird<MVideoImmutable> {
|
||||||
const fun = () => {
|
const fun = () => {
|
||||||
const where = buildWhereIdOrUUID(id)
|
const query = {
|
||||||
const options = {
|
where: buildWhereIdOrUUID(id),
|
||||||
attributes: [
|
|
||||||
'id', 'url', 'uuid'
|
|
||||||
],
|
|
||||||
where,
|
|
||||||
transaction: t
|
transaction: t
|
||||||
}
|
}
|
||||||
|
|
||||||
return VideoModel.unscoped().findOne(options)
|
return VideoModel.scope(ScopeNames.WITH_IMMUTABLE_ATTRIBUTES).findOne(query)
|
||||||
}
|
}
|
||||||
|
|
||||||
return ModelCache.Instance.doCache({
|
return ModelCache.Instance.doCache({
|
||||||
cacheType: 'video-immutable',
|
cacheType: 'load-video-immutable-id',
|
||||||
key: '' + id,
|
key: '' + id,
|
||||||
deleteKey: 'video',
|
deleteKey: 'video',
|
||||||
fun
|
fun
|
||||||
|
@ -1559,6 +1559,26 @@ export class VideoModel extends Model<VideoModel> {
|
||||||
return VideoModel.scope(ScopeNames.WITH_THUMBNAILS).findOne(query)
|
return VideoModel.scope(ScopeNames.WITH_THUMBNAILS).findOne(query)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static loadByUrlImmutableAttributes (url: string, transaction?: Transaction): Bluebird<MVideoImmutable> {
|
||||||
|
const fun = () => {
|
||||||
|
const query: FindOptions = {
|
||||||
|
where: {
|
||||||
|
url
|
||||||
|
},
|
||||||
|
transaction
|
||||||
|
}
|
||||||
|
|
||||||
|
return VideoModel.scope(ScopeNames.WITH_IMMUTABLE_ATTRIBUTES).findOne(query)
|
||||||
|
}
|
||||||
|
|
||||||
|
return ModelCache.Instance.doCache({
|
||||||
|
cacheType: 'load-video-immutable-url',
|
||||||
|
key: url,
|
||||||
|
deleteKey: 'video',
|
||||||
|
fun
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
static loadByUrlAndPopulateAccount (url: string, transaction?: Transaction): Bluebird<MVideoAccountLightBlacklistAllFiles> {
|
static loadByUrlAndPopulateAccount (url: string, transaction?: Transaction): Bluebird<MVideoAccountLightBlacklistAllFiles> {
|
||||||
const query: FindOptions = {
|
const query: FindOptions = {
|
||||||
where: {
|
where: {
|
||||||
|
|
|
@ -37,7 +37,7 @@ export type MVideoId = Pick<MVideo, 'id'>
|
||||||
export type MVideoUrl = Pick<MVideo, 'url'>
|
export type MVideoUrl = Pick<MVideo, 'url'>
|
||||||
export type MVideoUUID = Pick<MVideo, 'uuid'>
|
export type MVideoUUID = Pick<MVideo, 'uuid'>
|
||||||
|
|
||||||
export type MVideoImmutable = Pick<MVideo, 'id' | 'url' | 'uuid'>
|
export type MVideoImmutable = Pick<MVideo, 'id' | 'url' | 'uuid' | 'remote' | 'isOwned'>
|
||||||
export type MVideoIdUrl = MVideoId & MVideoUrl
|
export type MVideoIdUrl = MVideoId & MVideoUrl
|
||||||
export type MVideoFeed = Pick<MVideo, 'name' | 'uuid'>
|
export type MVideoFeed = Pick<MVideo, 'name' | 'uuid'>
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue