From b36f41ca09e92ecb30d367d91d1089a23d10d585 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 14 Sep 2018 09:57:21 +0200 Subject: [PATCH 01/44] Add trending videos strategy --- config/default.yaml | 3 + config/production.yaml.example | 3 + config/test.yaml | 3 + scripts/clean/server/test.sh | 5 +- server/initializers/checker.ts | 2 +- .../schedulers/videos-redundancy-scheduler.ts | 2 + server/models/redundancy/video-redundancy.ts | 115 ++++++--- server/models/video/video.ts | 32 ++- server/tests/api/server/redundancy.ts | 225 +++++++++++------- server/tests/utils/server/servers.ts | 4 +- .../redundancy/videos-redundancy.model.ts | 2 +- 11 files changed, 254 insertions(+), 142 deletions(-) diff --git a/config/default.yaml b/config/default.yaml index af29a4379..ecb809c6a 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -74,6 +74,9 @@ redundancy: # - # size: '10GB' # strategy: 'most-views' # Cache videos that have the most views +# - +# size: '10GB' +# strategy: 'trending' # Cache trending videos cache: previews: diff --git a/config/production.yaml.example b/config/production.yaml.example index ddd43093f..48d69e987 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example @@ -75,6 +75,9 @@ redundancy: # - # size: '10GB' # strategy: 'most-views' # Cache videos that have the most views +# - +# size: '10GB' +# strategy: 'trending' # Cache trending videos ############################################################################### # diff --git a/config/test.yaml b/config/test.yaml index 0f280eabd..73bc5da98 100644 --- a/config/test.yaml +++ b/config/test.yaml @@ -26,6 +26,9 @@ redundancy: - size: '100KB' strategy: 'most-views' + - + size: '100KB' + strategy: 'trending' cache: previews: diff --git a/scripts/clean/server/test.sh b/scripts/clean/server/test.sh index 3b8fe39ed..5f9a88a2e 100755 --- a/scripts/clean/server/test.sh +++ b/scripts/clean/server/test.sh @@ -6,9 +6,8 @@ for i in $(seq 1 6); do dbname="peertube_test$i" dropdb --if-exists "$dbname" - rm -rf "./test$i" - rm -f "./config/local-test.json" - rm -f "./config/local-test-$i.json" + rm -rf "./test$i" "./config/local-test.json" "./config/local-test-$i.json" + createdb -O peertube "$dbname" psql -c "CREATE EXTENSION pg_trgm;" "$dbname" psql -c "CREATE EXTENSION unaccent;" "$dbname" diff --git a/server/initializers/checker.ts b/server/initializers/checker.ts index 6a2badd35..6048151a3 100644 --- a/server/initializers/checker.ts +++ b/server/initializers/checker.ts @@ -41,7 +41,7 @@ function checkConfig () { const redundancyVideos = config.get('redundancy.videos') if (isArray(redundancyVideos)) { for (const r of redundancyVideos) { - if ([ 'most-views' ].indexOf(r.strategy) === -1) { + if ([ 'most-views', 'trending' ].indexOf(r.strategy) === -1) { return 'Redundancy video entries should have "most-views" strategy instead of ' + r.strategy } } diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index ee9ba1766..c1e619249 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts @@ -75,6 +75,8 @@ export class VideosRedundancyScheduler extends AbstractScheduler { private findVideoToDuplicate (strategy: VideoRedundancyStrategy) { if (strategy === 'most-views') return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) + + if (strategy === 'trending') return VideoRedundancyModel.findTrendingToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) } private async createVideoRedundancy (strategy: VideoRedundancyStrategy, filesToDuplicate: VideoFileModel[]) { diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts index 48ec77206..b13ade0f4 100644 --- a/server/models/redundancy/video-redundancy.ts +++ b/server/models/redundancy/video-redundancy.ts @@ -14,11 +14,10 @@ import { UpdatedAt } from 'sequelize-typescript' import { ActorModel } from '../activitypub/actor' -import { throwIfNotValid } from '../utils' +import { getVideoSort, throwIfNotValid } from '../utils' import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc' -import { CONSTRAINTS_FIELDS, VIDEO_EXT_MIMETYPE } from '../../initializers' +import { CONFIG, CONSTRAINTS_FIELDS, VIDEO_EXT_MIMETYPE } from '../../initializers' import { VideoFileModel } from '../video/video-file' -import { isDateValid } from '../../helpers/custom-validators/misc' import { getServerActor } from '../../helpers/utils' import { VideoModel } from '../video/video' import { VideoRedundancyStrategy } from '../../../shared/models/redundancy' @@ -145,50 +144,51 @@ export class VideoRedundancyModel extends Model { return VideoRedundancyModel.findOne(query) } + static getVideoSample (rows: { id: number }[]) { + const ids = rows.map(r => r.id) + const id = sample(ids) + + return VideoModel.loadWithFile(id, undefined, !isTestInstance()) + } + static async findMostViewToDuplicate (randomizedFactor: number) { // On VideoModel! const query = { + attributes: [ 'id', 'views' ], logging: !isTestInstance(), limit: randomizedFactor, - order: [ [ 'views', 'DESC' ] ], + order: getVideoSort('-views'), include: [ - { - model: VideoFileModel.unscoped(), - required: true, - where: { - id: { - [ Sequelize.Op.notIn ]: await VideoRedundancyModel.buildExcludeIn() - } - } - }, - { - attributes: [], - model: VideoChannelModel.unscoped(), - required: true, - include: [ - { - attributes: [], - model: ActorModel.unscoped(), - required: true, - include: [ - { - attributes: [], - model: ServerModel.unscoped(), - required: true, - where: { - redundancyAllowed: true - } - } - ] - } - ] - } + await VideoRedundancyModel.buildVideoFileForDuplication(), + VideoRedundancyModel.buildServerRedundancyInclude() ] } const rows = await VideoModel.unscoped().findAll(query) - return sample(rows) + return VideoRedundancyModel.getVideoSample(rows as { id: number }[]) + } + + static async findTrendingToDuplicate (randomizedFactor: number) { + // On VideoModel! + const query = { + attributes: [ 'id', 'views' ], + subQuery: false, + logging: !isTestInstance(), + group: 'VideoModel.id', + limit: randomizedFactor, + order: getVideoSort('-trending'), + include: [ + await VideoRedundancyModel.buildVideoFileForDuplication(), + VideoRedundancyModel.buildServerRedundancyInclude(), + + VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS) + ] + } + + const rows = await VideoModel.unscoped().findAll(query) + + return VideoRedundancyModel.getVideoSample(rows as { id: number }[]) } static async getVideoFiles (strategy: VideoRedundancyStrategy) { @@ -211,7 +211,7 @@ export class VideoRedundancyModel extends Model { logging: !isTestInstance(), where: { expiresOn: { - [Sequelize.Op.lt]: new Date() + [ Sequelize.Op.lt ]: new Date() } } } @@ -237,13 +237,50 @@ export class VideoRedundancyModel extends Model { } } - private static async buildExcludeIn () { + // Don't include video files we already duplicated + private static async buildVideoFileForDuplication () { const actor = await getServerActor() - return Sequelize.literal( + const notIn = Sequelize.literal( '(' + `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "expiresOn" >= NOW()` + ')' ) + + return { + attributes: [], + model: VideoFileModel.unscoped(), + required: true, + where: { + id: { + [ Sequelize.Op.notIn ]: notIn + } + } + } + } + + private static buildServerRedundancyInclude () { + return { + attributes: [], + model: VideoChannelModel.unscoped(), + required: true, + include: [ + { + attributes: [], + model: ActorModel.unscoped(), + required: true, + include: [ + { + attributes: [], + model: ServerModel.unscoped(), + required: true, + where: { + redundancyAllowed: true + } + } + ] + } + ] + } } } diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 27c631dcd..ef8be7c86 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -387,16 +387,7 @@ type AvailableForListIDsOptions = { } if (options.trendingDays) { - query.include.push({ - attributes: [], - model: VideoViewModel, - required: false, - where: { - startDate: { - [ Sequelize.Op.gte ]: new Date(new Date().getTime() - (24 * 3600 * 1000) * options.trendingDays) - } - } - }) + query.include.push(VideoModel.buildTrendingQuery(options.trendingDays)) query.subQuery = false } @@ -1071,9 +1062,12 @@ export class VideoModel extends Model { } static load (id: number, t?: Sequelize.Transaction) { - const options = t ? { transaction: t } : undefined + return VideoModel.findById(id, { transaction: t }) + } - return VideoModel.findById(id, options) + static loadWithFile (id: number, t?: Sequelize.Transaction, logging?: boolean) { + return VideoModel.scope(ScopeNames.WITH_FILES) + .findById(id, { transaction: t, logging }) } static loadByUrlAndPopulateAccount (url: string, t?: Sequelize.Transaction) { @@ -1191,6 +1185,20 @@ export class VideoModel extends Model { .then(rows => rows.map(r => r[ field ])) } + static buildTrendingQuery (trendingDays: number) { + return { + attributes: [], + subQuery: false, + model: VideoViewModel, + required: false, + where: { + startDate: { + [ Sequelize.Op.gte ]: new Date(new Date().getTime() - (24 * 3600 * 1000) * trendingDays) + } + } + } + } + private static buildActorWhereWithFilter (filter?: VideoFilter) { if (filter && filter === 'local') { return { diff --git a/server/tests/api/server/redundancy.ts b/server/tests/api/server/redundancy.ts index c0ec75a45..211570d2f 100644 --- a/server/tests/api/server/redundancy.ts +++ b/server/tests/api/server/redundancy.ts @@ -22,9 +22,14 @@ import { updateRedundancy } from '../../utils/server/redundancy' import { ActorFollow } from '../../../../shared/models/actors' import { readdir } from 'fs-extra' import { join } from 'path' +import { VideoRedundancyStrategy } from '../../../../shared/models/redundancy' const expect = chai.expect +let servers: ServerInfo[] = [] +let video1Server2UUID: string +let video2Server2UUID: string + function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[]) { const parsed = magnetUtil.decode(file.magnetUri) @@ -34,107 +39,159 @@ function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: numbe } } +async function runServers (strategy: VideoRedundancyStrategy) { + const config = { + redundancy: { + videos: [ + { + strategy: strategy, + size: '100KB' + } + ] + } + } + servers = await flushAndRunMultipleServers(3, config) + + // Get the access tokens + await setAccessTokensToServers(servers) + + { + const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' }) + video1Server2UUID = res.body.video.uuid + + await viewVideo(servers[ 1 ].url, video1Server2UUID) + } + + { + const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' }) + video2Server2UUID = res.body.video.uuid + } + + await waitJobs(servers) + + // Server 1 and server 2 follow each other + await doubleFollow(servers[ 0 ], servers[ 1 ]) + // Server 1 and server 3 follow each other + await doubleFollow(servers[ 0 ], servers[ 2 ]) + // Server 2 and server 3 follow each other + await doubleFollow(servers[ 1 ], servers[ 2 ]) + + await waitJobs(servers) +} + +async function check1WebSeed () { + const webseeds = [ + 'http://localhost:9002/static/webseed/' + video1Server2UUID + ] + + for (const server of servers) { + const res = await getVideo(server.url, video1Server2UUID) + + const video: VideoDetails = res.body + video.files.forEach(f => checkMagnetWebseeds(f, webseeds)) + } +} + +async function enableRedundancy () { + await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true) + + const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt') + const follows: ActorFollow[] = res.body.data + const server2 = follows.find(f => f.following.host === 'localhost:9002') + const server3 = follows.find(f => f.following.host === 'localhost:9003') + + expect(server3).to.not.be.undefined + expect(server3.following.hostRedundancyAllowed).to.be.false + + expect(server2).to.not.be.undefined + expect(server2.following.hostRedundancyAllowed).to.be.true +} + +async function check2Webseeds () { + await waitJobs(servers) + await wait(15000) + await waitJobs(servers) + + const webseeds = [ + 'http://localhost:9001/static/webseed/' + video1Server2UUID, + 'http://localhost:9002/static/webseed/' + video1Server2UUID + ] + + for (const server of servers) { + const res = await getVideo(server.url, video1Server2UUID) + + const video: VideoDetails = res.body + + for (const file of video.files) { + checkMagnetWebseeds(file, webseeds) + } + } + + const files = await readdir(join(root(), 'test1', 'videos')) + expect(files).to.have.lengthOf(4) + + for (const resolution of [ 240, 360, 480, 720 ]) { + expect(files.find(f => f === `${video1Server2UUID}-${resolution}.mp4`)).to.not.be.undefined + } +} + +async function cleanServers () { + killallServers(servers) +} + describe('Test videos redundancy', function () { - let servers: ServerInfo[] = [] - let video1Server2UUID: string - let video2Server2UUID: string - before(async function () { - this.timeout(120000) + describe('With most-views strategy', function () { - servers = await flushAndRunMultipleServers(3) + before(function () { + this.timeout(120000) - // Get the access tokens - await setAccessTokensToServers(servers) + return runServers('most-views') + }) - { - const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' }) - video1Server2UUID = res.body.video.uuid + it('Should have 1 webseed on the first video', function () { + return check1WebSeed() + }) - await viewVideo(servers[1].url, video1Server2UUID) - } + it('Should enable redundancy on server 1', async function () { + return enableRedundancy() + }) - { - const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' }) - video2Server2UUID = res.body.video.uuid - } + it('Should have 2 webseed on the first video', async function () { + this.timeout(40000) - await waitJobs(servers) + return check2Webseeds() + }) - // Server 1 and server 2 follow each other - await doubleFollow(servers[0], servers[1]) - // Server 1 and server 3 follow each other - await doubleFollow(servers[0], servers[2]) - // Server 2 and server 3 follow each other - await doubleFollow(servers[1], servers[2]) - - await waitJobs(servers) + after(function () { + return cleanServers() + }) }) - it('Should have 1 webseed on the first video', async function () { - const webseeds = [ - 'http://localhost:9002/static/webseed/' + video1Server2UUID - ] + describe('With trending strategy', function () { - for (const server of servers) { - const res = await getVideo(server.url, video1Server2UUID) + before(function () { + this.timeout(120000) - const video: VideoDetails = res.body - video.files.forEach(f => checkMagnetWebseeds(f, webseeds)) - } - }) + return runServers('trending') + }) - it('Should enable redundancy on server 1', async function () { - await updateRedundancy(servers[0].url, servers[0].accessToken, servers[1].host, true) + it('Should have 1 webseed on the first video', function () { + return check1WebSeed() + }) - const res = await getFollowingListPaginationAndSort(servers[0].url, 0, 5, '-createdAt') - const follows: ActorFollow[] = res.body.data - const server2 = follows.find(f => f.following.host === 'localhost:9002') - const server3 = follows.find(f => f.following.host === 'localhost:9003') + it('Should enable redundancy on server 1', async function () { + return enableRedundancy() + }) - expect(server3).to.not.be.undefined - expect(server3.following.hostRedundancyAllowed).to.be.false + it('Should have 2 webseed on the first video', async function () { + this.timeout(40000) - expect(server2).to.not.be.undefined - expect(server2.following.hostRedundancyAllowed).to.be.true - }) + return check2Webseeds() + }) - it('Should have 2 webseed on the first video', async function () { - this.timeout(40000) - - await waitJobs(servers) - await wait(15000) - await waitJobs(servers) - - const webseeds = [ - 'http://localhost:9001/static/webseed/' + video1Server2UUID, - 'http://localhost:9002/static/webseed/' + video1Server2UUID - ] - - for (const server of servers) { - const res = await getVideo(server.url, video1Server2UUID) - - const video: VideoDetails = res.body - - for (const file of video.files) { - checkMagnetWebseeds(file, webseeds) - } - } - - const files = await readdir(join(root(), 'test1', 'videos')) - expect(files).to.have.lengthOf(4) - - for (const resolution of [ 240, 360, 480, 720 ]) { - expect(files.find(f => f === `${video1Server2UUID}-${resolution}.mp4`)).to.not.be.undefined - } - }) - - after(async function () { - killallServers(servers) - - // Keep the logs if the test failed - if (this['ok']) { - await flushTests() - } + after(function () { + return cleanServers() + }) }) }) diff --git a/server/tests/utils/server/servers.ts b/server/tests/utils/server/servers.ts index 1372c03c3..e95be4a16 100644 --- a/server/tests/utils/server/servers.ts +++ b/server/tests/utils/server/servers.ts @@ -35,7 +35,7 @@ interface ServerInfo { } } -function flushAndRunMultipleServers (totalServers) { +function flushAndRunMultipleServers (totalServers: number, configOverride?: Object) { let apps = [] let i = 0 @@ -53,7 +53,7 @@ function flushAndRunMultipleServers (totalServers) { for (let j = 1; j <= totalServers; j++) { // For the virtual buffer setTimeout(() => { - runServer(j).then(app => anotherServerDone(j, app)) + runServer(j, configOverride).then(app => anotherServerDone(j, app)) }, 1000 * (j - 1)) } }) diff --git a/shared/models/redundancy/videos-redundancy.model.ts b/shared/models/redundancy/videos-redundancy.model.ts index eb84964e0..85982e5b3 100644 --- a/shared/models/redundancy/videos-redundancy.model.ts +++ b/shared/models/redundancy/videos-redundancy.model.ts @@ -1,4 +1,4 @@ -export type VideoRedundancyStrategy = 'most-views' +export type VideoRedundancyStrategy = 'most-views' | 'trending' export interface VideosRedundancy { strategy: VideoRedundancyStrategy From d5f044cef285e922244bef7851dbbc12a61ec11f Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 14 Sep 2018 09:58:49 +0200 Subject: [PATCH 02/44] Test to remove delay in tests when running multiple servers --- server/tests/utils/server/servers.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/server/tests/utils/server/servers.ts b/server/tests/utils/server/servers.ts index e95be4a16..26ab4e1bb 100644 --- a/server/tests/utils/server/servers.ts +++ b/server/tests/utils/server/servers.ts @@ -51,10 +51,7 @@ function flushAndRunMultipleServers (totalServers: number, configOverride?: Obje flushTests() .then(() => { for (let j = 1; j <= totalServers; j++) { - // For the virtual buffer - setTimeout(() => { - runServer(j, configOverride).then(app => anotherServerDone(j, app)) - }, 1000 * (j - 1)) + runServer(j, configOverride).then(app => anotherServerDone(j, app)) } }) }) From 780daa7e91336116a5163156f298c9ad875ec1e3 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 14 Sep 2018 10:07:33 +0200 Subject: [PATCH 03/44] Speaup clean script --- scripts/clean/server/test.sh | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/scripts/clean/server/test.sh b/scripts/clean/server/test.sh index 5f9a88a2e..235ff52cc 100755 --- a/scripts/clean/server/test.sh +++ b/scripts/clean/server/test.sh @@ -2,14 +2,28 @@ set -eu -for i in $(seq 1 6); do - dbname="peertube_test$i" +recreateDB () { + dbname="peertube_test$1" dropdb --if-exists "$dbname" - rm -rf "./test$i" "./config/local-test.json" "./config/local-test-$i.json" createdb -O peertube "$dbname" - psql -c "CREATE EXTENSION pg_trgm;" "$dbname" - psql -c "CREATE EXTENSION unaccent;" "$dbname" - redis-cli KEYS "bull-localhost:900$i*" | grep -v empty | xargs --no-run-if-empty redis-cli DEL + psql -c "CREATE EXTENSION pg_trgm;" "$dbname" & + psql -c "CREATE EXTENSION unaccent;" "$dbname" & +} + +removeFiles () { + rm -rf "./test$1" "./config/local-test.json" "./config/local-test-$1.json" +} + +dropRedis () { + redis-cli KEYS "bull-localhost:900$1*" | grep -v empty | xargs --no-run-if-empty redis-cli DEL +} + +for i in $(seq 1 6); do + recreateDB "$i" & + dropRedis "$i" & + removeFiles "$i" & done + +wait From 3f6b6a565dc98a658ec9d8f697252788c0faa46d Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 14 Sep 2018 11:05:38 +0200 Subject: [PATCH 04/44] Add recently added redundancy strategy --- config/default.yaml | 4 ++ config/production.yaml.example | 4 ++ config/test.yaml | 4 ++ .../custom-validators/activitypub/videos.ts | 2 - server/initializers/checker.ts | 13 +++- server/initializers/constants.ts | 11 +--- .../schedulers/videos-redundancy-scheduler.ts | 44 +++++-------- server/models/redundancy/video-redundancy.ts | 59 ++++++++++++----- server/tests/api/server/redundancy.ts | 63 ++++++++++++++++--- .../redundancy/videos-redundancy.model.ts | 19 +++++- 10 files changed, 155 insertions(+), 68 deletions(-) diff --git a/config/default.yaml b/config/default.yaml index ecb809c6a..adac9deeb 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -77,6 +77,10 @@ redundancy: # - # size: '10GB' # strategy: 'trending' # Cache trending videos +# - +# size: '10GB' +# strategy: 'recently-added' # Cache recently added videos +# minViews: 10 # Having at least x views cache: previews: diff --git a/config/production.yaml.example b/config/production.yaml.example index 48d69e987..ca7b936c2 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example @@ -78,6 +78,10 @@ redundancy: # - # size: '10GB' # strategy: 'trending' # Cache trending videos +# - +# size: '10GB' +# strategy: 'recently-added' # Cache recently added videos +# minViews: 10 # Having at least x views ############################################################################### # diff --git a/config/test.yaml b/config/test.yaml index 73bc5da98..517fc7449 100644 --- a/config/test.yaml +++ b/config/test.yaml @@ -29,6 +29,10 @@ redundancy: - size: '100KB' strategy: 'trending' + - + size: '100KB' + strategy: 'recently-added' + minViews: 10 cache: previews: diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts index f76eba474..8772e74cf 100644 --- a/server/helpers/custom-validators/activitypub/videos.ts +++ b/server/helpers/custom-validators/activitypub/videos.ts @@ -171,5 +171,3 @@ function setRemoteVideoTruncatedContent (video: any) { return true } - - diff --git a/server/initializers/checker.ts b/server/initializers/checker.ts index 6048151a3..29f4f3036 100644 --- a/server/initializers/checker.ts +++ b/server/initializers/checker.ts @@ -7,7 +7,7 @@ import { parse } from 'url' import { CONFIG } from './constants' import { logger } from '../helpers/logger' import { getServerActor } from '../helpers/utils' -import { VideosRedundancy } from '../../shared/models/redundancy' +import { RecentlyAddedStrategy, VideosRedundancy } from '../../shared/models/redundancy' import { isArray } from '../helpers/custom-validators/misc' import { uniq } from 'lodash' @@ -34,24 +34,31 @@ async function checkActivityPubUrls () { function checkConfig () { const defaultNSFWPolicy = config.get('instance.default_nsfw_policy') + // NSFW policy if ([ 'do_not_list', 'blur', 'display' ].indexOf(defaultNSFWPolicy) === -1) { return 'NSFW policy setting should be "do_not_list" or "blur" or "display" instead of ' + defaultNSFWPolicy } + // Redundancies const redundancyVideos = config.get('redundancy.videos') if (isArray(redundancyVideos)) { for (const r of redundancyVideos) { - if ([ 'most-views', 'trending' ].indexOf(r.strategy) === -1) { + if ([ 'most-views', 'trending', 'recently-added' ].indexOf(r.strategy) === -1) { return 'Redundancy video entries should have "most-views" strategy instead of ' + r.strategy } } const filtered = uniq(redundancyVideos.map(r => r.strategy)) if (filtered.length !== redundancyVideos.length) { - return 'Redundancy video entries should have uniq strategies' + return 'Redundancy video entries should have unique strategies' } } + const recentlyAddedStrategy = redundancyVideos.find(r => r.strategy === 'recently-added') as RecentlyAddedStrategy + if (recentlyAddedStrategy && isNaN(recentlyAddedStrategy.minViews)) { + return 'Min views in recently added strategy is not a number' + } + return null } diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 6b4afbfd8..5f7bcbd48 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -1,6 +1,6 @@ import { IConfig } from 'config' import { dirname, join } from 'path' -import { JobType, VideoRateType, VideoRedundancyStrategy, VideoState, VideosRedundancy } from '../../shared/models' +import { JobType, VideoRateType, VideoState, VideosRedundancy } from '../../shared/models' import { ActivityPubActorType } from '../../shared/models/activitypub' import { FollowState } from '../../shared/models/actors' import { VideoAbuseState, VideoImportState, VideoPrivacy } from '../../shared/models/videos' @@ -741,15 +741,10 @@ function updateWebserverConfig () { CONFIG.WEBSERVER.HOST = sanitizeHost(CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT, REMOTE_SCHEME.HTTP) } -function buildVideosRedundancy (objs: { strategy: VideoRedundancyStrategy, size: string }[]): VideosRedundancy[] { +function buildVideosRedundancy (objs: VideosRedundancy[]): VideosRedundancy[] { if (!objs) return [] - return objs.map(obj => { - return { - strategy: obj.strategy, - size: bytes.parse(obj.size) - } - }) + return objs.map(obj => Object.assign(obj, { size: bytes.parse(obj.size) })) } function buildLanguages () { diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index c1e619249..8b91d750b 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts @@ -1,7 +1,7 @@ import { AbstractScheduler } from './abstract-scheduler' import { CONFIG, JOB_TTL, REDUNDANCY, SCHEDULER_INTERVALS_MS } from '../../initializers' import { logger } from '../../helpers/logger' -import { VideoRedundancyStrategy } from '../../../shared/models/redundancy' +import { RecentlyAddedStrategy, VideoRedundancyStrategy, VideosRedundancy } from '../../../shared/models/redundancy' import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' import { VideoFileModel } from '../../models/video/video-file' import { sortBy } from 'lodash' @@ -32,16 +32,14 @@ export class VideosRedundancyScheduler extends AbstractScheduler { this.executing = true for (const obj of CONFIG.REDUNDANCY.VIDEOS) { - try { - const videoToDuplicate = await this.findVideoToDuplicate(obj.strategy) + const videoToDuplicate = await this.findVideoToDuplicate(obj) if (!videoToDuplicate) continue const videoFiles = videoToDuplicate.VideoFiles videoFiles.forEach(f => f.Video = videoToDuplicate) - const videosRedundancy = await VideoRedundancyModel.getVideoFiles(obj.strategy) - if (this.isTooHeavy(videosRedundancy, videoFiles, obj.size)) { + if (await this.isTooHeavy(obj.strategy, videoFiles, obj.size)) { if (!isTestInstance()) logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url) continue } @@ -73,10 +71,19 @@ export class VideosRedundancyScheduler extends AbstractScheduler { return this.instance || (this.instance = new this()) } - private findVideoToDuplicate (strategy: VideoRedundancyStrategy) { - if (strategy === 'most-views') return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) + private findVideoToDuplicate (cache: VideosRedundancy) { + if (cache.strategy === 'most-views') { + return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) + } - if (strategy === 'trending') return VideoRedundancyModel.findTrendingToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) + if (cache.strategy === 'trending') { + return VideoRedundancyModel.findTrendingToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) + } + + if (cache.strategy === 'recently-added') { + const minViews = (cache as RecentlyAddedStrategy).minViews + return VideoRedundancyModel.findRecentlyAddedToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR, minViews) + } } private async createVideoRedundancy (strategy: VideoRedundancyStrategy, filesToDuplicate: VideoFileModel[]) { @@ -122,27 +129,10 @@ export class VideosRedundancyScheduler extends AbstractScheduler { } } - // Unused, but could be useful in the future, with a custom strategy - private async purgeVideosIfNeeded (videosRedundancy: VideoRedundancyModel[], filesToDuplicate: VideoFileModel[], maxSize: number) { - const sortedVideosRedundancy = sortBy(videosRedundancy, 'createdAt') - - while (this.isTooHeavy(sortedVideosRedundancy, filesToDuplicate, maxSize)) { - const toDelete = sortedVideosRedundancy.shift() - - const videoFile = toDelete.VideoFile - logger.info('Purging video %s (resolution %d) from our redundancy system.', videoFile.Video.url, videoFile.resolution) - - await removeVideoRedundancy(toDelete, undefined) - } - - return sortedVideosRedundancy - } - - private isTooHeavy (videosRedundancy: VideoRedundancyModel[], filesToDuplicate: VideoFileModel[], maxSizeArg: number) { + private async isTooHeavy (strategy: VideoRedundancyStrategy, filesToDuplicate: VideoFileModel[], maxSizeArg: number) { const maxSize = maxSizeArg - this.getTotalFileSizes(filesToDuplicate) - const redundancyReducer = (previous: number, current: VideoRedundancyModel) => previous + current.VideoFile.size - const totalDuplicated = videosRedundancy.reduce(redundancyReducer, 0) + const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(strategy) return totalDuplicated > maxSize } diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts index b13ade0f4..b7454c617 100644 --- a/server/models/redundancy/video-redundancy.ts +++ b/server/models/redundancy/video-redundancy.ts @@ -27,6 +27,7 @@ import { VideoChannelModel } from '../video/video-channel' import { ServerModel } from '../server/server' import { sample } from 'lodash' import { isTestInstance } from '../../helpers/core-utils' +import * as Bluebird from 'bluebird' export enum ScopeNames { WITH_VIDEO = 'WITH_VIDEO' @@ -144,7 +145,8 @@ export class VideoRedundancyModel extends Model { return VideoRedundancyModel.findOne(query) } - static getVideoSample (rows: { id: number }[]) { + static async getVideoSample (p: Bluebird) { + const rows = await p const ids = rows.map(r => r.id) const id = sample(ids) @@ -164,9 +166,7 @@ export class VideoRedundancyModel extends Model { ] } - const rows = await VideoModel.unscoped().findAll(query) - - return VideoRedundancyModel.getVideoSample(rows as { id: number }[]) + return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query)) } static async findTrendingToDuplicate (randomizedFactor: number) { @@ -186,24 +186,49 @@ export class VideoRedundancyModel extends Model { ] } - const rows = await VideoModel.unscoped().findAll(query) - - return VideoRedundancyModel.getVideoSample(rows as { id: number }[]) + return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query)) } - static async getVideoFiles (strategy: VideoRedundancyStrategy) { - const actor = await getServerActor() - - const queryVideoFiles = { - logging: !isTestInstance(), + static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) { + // On VideoModel! + const query = { + attributes: [ 'id', 'publishedAt' ], + // logging: !isTestInstance(), + limit: randomizedFactor, + order: getVideoSort('-publishedAt'), where: { - actorId: actor.id, - strategy - } + views: { + [ Sequelize.Op.gte ]: minViews + } + }, + include: [ + await VideoRedundancyModel.buildVideoFileForDuplication(), + VideoRedundancyModel.buildServerRedundancyInclude() + ] } - return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO) - .findAll(queryVideoFiles) + return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query)) + } + + static async getTotalDuplicated (strategy: VideoRedundancyStrategy) { + const actor = await getServerActor() + + const options = { + logging: !isTestInstance(), + include: [ + { + attributes: [], + model: VideoRedundancyModel, + required: true, + where: { + actorId: actor.id, + strategy + } + } + ] + } + + return VideoFileModel.sum('size', options) } static listAllExpired () { diff --git a/server/tests/api/server/redundancy.ts b/server/tests/api/server/redundancy.ts index 211570d2f..6574e8ea9 100644 --- a/server/tests/api/server/redundancy.ts +++ b/server/tests/api/server/redundancy.ts @@ -14,7 +14,7 @@ import { setAccessTokensToServers, uploadVideo, wait, - root, viewVideo + root, viewVideo, immutableAssign } from '../../utils' import { waitJobs } from '../../utils/server/jobs' import * as magnetUtil from 'magnet-uri' @@ -39,14 +39,14 @@ function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: numbe } } -async function runServers (strategy: VideoRedundancyStrategy) { +async function runServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) { const config = { redundancy: { videos: [ - { + immutableAssign({ strategy: strategy, size: '100KB' - } + }, additionalParams) ] } } @@ -153,11 +153,11 @@ describe('Test videos redundancy', function () { return check1WebSeed() }) - it('Should enable redundancy on server 1', async function () { + it('Should enable redundancy on server 1', function () { return enableRedundancy() }) - it('Should have 2 webseed on the first video', async function () { + it('Should have 2 webseed on the first video', function () { this.timeout(40000) return check2Webseeds() @@ -180,11 +180,58 @@ describe('Test videos redundancy', function () { return check1WebSeed() }) - it('Should enable redundancy on server 1', async function () { + it('Should enable redundancy on server 1', function () { return enableRedundancy() }) - it('Should have 2 webseed on the first video', async function () { + it('Should have 2 webseed on the first video', function () { + this.timeout(40000) + + return check2Webseeds() + }) + + after(function () { + return cleanServers() + }) + }) + + describe('With recently added strategy', function () { + + before(function () { + this.timeout(120000) + + return runServers('recently-added', { minViews: 3 }) + }) + + it('Should have 1 webseed on the first video', function () { + return check1WebSeed() + }) + + it('Should enable redundancy on server 1', function () { + return enableRedundancy() + }) + + it('Should still have 1 webseed on the first video', async function () { + this.timeout(40000) + + await waitJobs(servers) + await wait(15000) + await waitJobs(servers) + + return check1WebSeed() + }) + + it('Should view 2 times the first video', async function () { + this.timeout(40000) + + await viewVideo(servers[ 0 ].url, video1Server2UUID) + await viewVideo(servers[ 2 ].url, video1Server2UUID) + + await wait(10000) + await waitJobs(servers) + }) + + it('Should have 2 webseed on the first video', function () { this.timeout(40000) return check2Webseeds() diff --git a/shared/models/redundancy/videos-redundancy.model.ts b/shared/models/redundancy/videos-redundancy.model.ts index 85982e5b3..436394c1e 100644 --- a/shared/models/redundancy/videos-redundancy.model.ts +++ b/shared/models/redundancy/videos-redundancy.model.ts @@ -1,6 +1,19 @@ -export type VideoRedundancyStrategy = 'most-views' | 'trending' +export type VideoRedundancyStrategy = 'most-views' | 'trending' | 'recently-added' -export interface VideosRedundancy { - strategy: VideoRedundancyStrategy +export type MostViewsRedundancyStrategy = { + strategy: 'most-views' size: number } + +export type TrendingRedundancyStrategy = { + strategy: 'trending' + size: number +} + +export type RecentlyAddedStrategy = { + strategy: 'recently-added' + size: number + minViews: number +} + +export type VideosRedundancy = MostViewsRedundancyStrategy | TrendingRedundancyStrategy | RecentlyAddedStrategy From 2b62cccd75e9025fb66148bcb1feea2a458ee8e4 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 14 Sep 2018 11:09:34 +0200 Subject: [PATCH 05/44] Raw query to get video ids --- server/models/video/video.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/server/models/video/video.ts b/server/models/video/video.ts index ef8be7c86..23d1dedd6 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -221,6 +221,7 @@ type AvailableForListIDsOptions = { }, [ ScopeNames.AVAILABLE_FOR_LIST_IDS ]: (options: AvailableForListIDsOptions) => { const query: IFindOptions = { + raw: true, attributes: [ 'id' ], where: { id: { From 7348b1fd84dee869b3c36554aea6797f09d4ceed Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 14 Sep 2018 11:52:23 +0200 Subject: [PATCH 06/44] Speed up overviews route --- package.json | 2 + server/controllers/api/overviews.ts | 26 +++++++++---- server/helpers/utils.ts | 24 +++--------- server/initializers/constants.ts | 5 +++ .../schedulers/videos-redundancy-scheduler.ts | 2 +- server/models/video/video.ts | 27 ++++++------- yarn.lock | 38 +++++++++++++++++-- 7 files changed, 80 insertions(+), 44 deletions(-) diff --git a/package.json b/package.json index 5a8843b0c..1cb5be181 100644 --- a/package.json +++ b/package.json @@ -111,6 +111,7 @@ "jsonld-signatures": "https://github.com/Chocobozzz/jsonld-signatures#rsa2017", "lodash": "^4.17.10", "magnet-uri": "^5.1.4", + "memoizee": "^0.4.14", "morgan": "^1.5.3", "multer": "^1.1.0", "nodemailer": "^4.4.2", @@ -158,6 +159,7 @@ "@types/lodash": "^4.14.64", "@types/magnet-uri": "^5.1.1", "@types/maildev": "^0.0.1", + "@types/memoizee": "^0.4.2", "@types/mkdirp": "^0.5.1", "@types/mocha": "^5.0.0", "@types/morgan": "^1.7.32", diff --git a/server/controllers/api/overviews.ts b/server/controllers/api/overviews.ts index da941c0ac..cc3cc54a7 100644 --- a/server/controllers/api/overviews.ts +++ b/server/controllers/api/overviews.ts @@ -4,8 +4,9 @@ import { VideoModel } from '../../models/video/video' import { asyncMiddleware } from '../../middlewares' import { TagModel } from '../../models/video/tag' import { VideosOverview } from '../../../shared/models/overviews' -import { OVERVIEWS, ROUTE_CACHE_LIFETIME } from '../../initializers' +import { MEMOIZE_TTL, OVERVIEWS, ROUTE_CACHE_LIFETIME } from '../../initializers' import { cacheRoute } from '../../middlewares/cache' +import * as memoizee from 'memoizee' const overviewsRouter = express.Router() @@ -23,10 +24,17 @@ export { overviewsRouter } // This endpoint could be quite long, but we cache it async function getVideosOverview (req: express.Request, res: express.Response) { const attributes = await buildSamples() + + const [ categories, channels, tags ] = await Promise.all([ + Promise.all(attributes.categories.map(c => getVideosByCategory(c, res))), + Promise.all(attributes.channels.map(c => getVideosByChannel(c, res))), + Promise.all(attributes.tags.map(t => getVideosByTag(t, res))) + ]) + const result: VideosOverview = { - categories: await Promise.all(attributes.categories.map(c => getVideosByCategory(c, res))), - channels: await Promise.all(attributes.channels.map(c => getVideosByChannel(c, res))), - tags: await Promise.all(attributes.tags.map(t => getVideosByTag(t, res))) + categories, + channels, + tags } // Cleanup our object @@ -37,7 +45,7 @@ async function getVideosOverview (req: express.Request, res: express.Response) { return res.json(result) } -async function buildSamples () { +const buildSamples = memoizee(async function () { const [ categories, channels, tags ] = await Promise.all([ VideoModel.getRandomFieldSamples('category', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT), VideoModel.getRandomFieldSamples('channelId', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD ,OVERVIEWS.VIDEOS.SAMPLES_COUNT), @@ -45,7 +53,7 @@ async function buildSamples () { ]) return { categories, channels, tags } -} +}, { maxAge: MEMOIZE_TTL.OVERVIEWS_SAMPLE }) async function getVideosByTag (tag: string, res: express.Response) { const videos = await getVideos(res, { tagsOneOf: [ tag ] }) @@ -84,14 +92,16 @@ async function getVideos ( res: express.Response, where: { videoChannelId?: number, tagsOneOf?: string[], categoryOneOf?: number[] } ) { - const { data } = await VideoModel.listForApi(Object.assign({ + const query = Object.assign({ start: 0, count: 10, sort: '-createdAt', includeLocalVideos: true, nsfw: buildNSFWFilter(res), withFiles: false - }, where)) + }, where) + + const { data } = await VideoModel.listForApi(query, false) return data.map(d => d.toFormattedJSON()) } diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts index a1ed8e72d..a42474417 100644 --- a/server/helpers/utils.ts +++ b/server/helpers/utils.ts @@ -1,12 +1,12 @@ import { ResultList } from '../../shared' import { CONFIG } from '../initializers' -import { ActorModel } from '../models/activitypub/actor' import { ApplicationModel } from '../models/application/application' import { pseudoRandomBytesPromise, sha256 } from './core-utils' import { logger } from './logger' import { join } from 'path' import { Instance as ParseTorrent } from 'parse-torrent' import { remove } from 'fs-extra' +import * as memoizee from 'memoizee' function deleteFileAsync (path: string) { remove(path) @@ -36,24 +36,12 @@ function getFormattedObjects (objects: T[], obje } as ResultList } -async function getServerActor () { - if (getServerActor.serverActor === undefined) { - const application = await ApplicationModel.load() - if (!application) throw Error('Could not load Application from database.') +const getServerActor = memoizee(async function () { + const application = await ApplicationModel.load() + if (!application) throw Error('Could not load Application from database.') - getServerActor.serverActor = application.Account.Actor - } - - if (!getServerActor.serverActor) { - logger.error('Cannot load server actor.') - process.exit(0) - } - - return Promise.resolve(getServerActor.serverActor) -} -namespace getServerActor { - export let serverActor: ActorModel -} + return application.Account.Actor +}) function generateVideoTmpPath (target: string | ParseTorrent) { const id = typeof target === 'string' ? target : target.infoHash diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 5f7bcbd48..9cccb0919 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -592,6 +592,10 @@ const CACHE = { } } +const MEMOIZE_TTL = { + OVERVIEWS_SAMPLE: 1000 * 3600 * 4 // 4 hours +} + const REDUNDANCY = { VIDEOS: { EXPIRES_AFTER_MS: 48 * 3600 * 1000, // 2 days @@ -708,6 +712,7 @@ export { VIDEO_ABUSE_STATES, JOB_REQUEST_TIMEOUT, USER_PASSWORD_RESET_LIFETIME, + MEMOIZE_TTL, USER_EMAIL_VERIFY_LIFETIME, IMAGE_MIMETYPE_EXT, OVERVIEWS, diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index 8b91d750b..7079600a9 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts @@ -81,7 +81,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { } if (cache.strategy === 'recently-added') { - const minViews = (cache as RecentlyAddedStrategy).minViews + const minViews = cache.minViews return VideoRedundancyModel.findRecentlyAddedToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR, minViews) } } diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 23d1dedd6..b7d3f184f 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -929,7 +929,7 @@ export class VideoModel extends Model { videoChannelId?: number, actorId?: number trendingDays?: number - }) { + }, countVideos = true) { const query: IFindOptions = { offset: options.start, limit: options.count, @@ -962,7 +962,7 @@ export class VideoModel extends Model { trendingDays } - return VideoModel.getAvailableForApi(query, queryOptions) + return VideoModel.getAvailableForApi(query, queryOptions, countVideos) } static async searchAndPopulateAccountAndServer (options: { @@ -1164,7 +1164,14 @@ export class VideoModel extends Model { } // threshold corresponds to how many video the field should have to be returned - static getRandomFieldSamples (field: 'category' | 'channelId', threshold: number, count: number) { + static async getRandomFieldSamples (field: 'category' | 'channelId', threshold: number, count: number) { + const actorId = (await getServerActor()).id + + const scopeOptions = { + actorId, + includeLocalVideos: true + } + const query: IFindOptions = { attributes: [ field ], limit: count, @@ -1172,17 +1179,11 @@ export class VideoModel extends Model { having: Sequelize.where(Sequelize.fn('COUNT', Sequelize.col(field)), { [ Sequelize.Op.gte ]: threshold }) as any, // FIXME: typings - where: { - [ field ]: { - [ Sequelize.Op.not ]: null - }, - privacy: VideoPrivacy.PUBLIC, - state: VideoState.PUBLISHED - }, order: [ this.sequelize.random() ] } - return VideoModel.findAll(query) + return VideoModel.scope({ method: [ ScopeNames.AVAILABLE_FOR_LIST_IDS, scopeOptions ] }) + .findAll(query) .then(rows => rows.map(r => r[ field ])) } @@ -1210,7 +1211,7 @@ export class VideoModel extends Model { return {} } - private static async getAvailableForApi (query: IFindOptions, options: AvailableForListIDsOptions) { + private static async getAvailableForApi (query: IFindOptions, options: AvailableForListIDsOptions, countVideos = true) { const idsScope = { method: [ ScopeNames.AVAILABLE_FOR_LIST_IDS, options @@ -1227,7 +1228,7 @@ export class VideoModel extends Model { } const [ count, rowsId ] = await Promise.all([ - VideoModel.scope(countScope).count(countQuery), + countVideos ? VideoModel.scope(countScope).count(countQuery) : Promise.resolve(undefined), VideoModel.scope(idsScope).findAll(query) ]) const ids = rowsId.map(r => r.id) diff --git a/yarn.lock b/yarn.lock index c8fb21117..52ff895b1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -160,6 +160,10 @@ dependencies: "@types/node" "*" +"@types/memoizee@^0.4.2": + version "0.4.2" + resolved "https://registry.yarnpkg.com/@types/memoizee/-/memoizee-0.4.2.tgz#a500158999a8144a9b46cf9a9fb49b15f1853573" + "@types/mime@*": version "2.0.0" resolved "https://registry.yarnpkg.com/@types/mime/-/mime-2.0.0.tgz#5a7306e367c539b9f6543499de8dd519fac37a8b" @@ -2058,7 +2062,7 @@ error@^7.0.0: string-template "~0.2.1" xtend "~4.0.0" -es5-ext@^0.10.14, es5-ext@^0.10.35, es5-ext@^0.10.9, es5-ext@~0.10.14: +es5-ext@^0.10.14, es5-ext@^0.10.35, es5-ext@^0.10.45, es5-ext@^0.10.9, es5-ext@~0.10.14, es5-ext@~0.10.2: version "0.10.46" resolved "https://registry.yarnpkg.com/es5-ext/-/es5-ext-0.10.46.tgz#efd99f67c5a7ec789baa3daa7f79870388f7f572" dependencies: @@ -2110,7 +2114,7 @@ es6-symbol@3.1.1, es6-symbol@^3.1.1, es6-symbol@~3.1.1: d "1" es5-ext "~0.10.14" -es6-weak-map@^2.0.1: +es6-weak-map@^2.0.1, es6-weak-map@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/es6-weak-map/-/es6-weak-map-2.0.2.tgz#5e3ab32251ffd1538a1f8e5ffa1357772f92d96f" dependencies: @@ -2223,7 +2227,7 @@ etag@~1.8.1: version "1.8.1" resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887" -event-emitter@~0.3.5: +event-emitter@^0.3.5, event-emitter@~0.3.5: version "0.3.5" resolved "https://registry.yarnpkg.com/event-emitter/-/event-emitter-0.3.5.tgz#df8c69eef1647923c7157b9ce83840610b02cc39" dependencies: @@ -3757,7 +3761,7 @@ is-plain-object@^2.0.1, is-plain-object@^2.0.3, is-plain-object@^2.0.4: dependencies: isobject "^3.0.1" -is-promise@^2.1.0: +is-promise@^2.1, is-promise@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/is-promise/-/is-promise-2.1.0.tgz#79a2a9ece7f096e80f36d2b2f3bc16c1ff4bf3fa" @@ -4490,6 +4494,12 @@ lru-cache@4.1.x, lru-cache@^4.0.1: pseudomap "^1.0.2" yallist "^2.1.2" +lru-queue@0.1: + version "0.1.0" + resolved "https://registry.yarnpkg.com/lru-queue/-/lru-queue-0.1.0.tgz#2738bd9f0d3cf4f84490c5736c48699ac632cda3" + dependencies: + es5-ext "~0.10.2" + lru@^3.0.0, lru@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/lru/-/lru-3.1.0.tgz#ea7fb8546d83733396a13091d76cfeb4c06837d5" @@ -4594,6 +4604,19 @@ mem@^1.1.0: dependencies: mimic-fn "^1.0.0" +memoizee@^0.4.14: + version "0.4.14" + resolved "https://registry.yarnpkg.com/memoizee/-/memoizee-0.4.14.tgz#07a00f204699f9a95c2d9e77218271c7cd610d57" + dependencies: + d "1" + es5-ext "^0.10.45" + es6-weak-map "^2.0.2" + event-emitter "^0.3.5" + is-promise "^2.1" + lru-queue "0.1" + next-tick "1" + timers-ext "^0.1.5" + memory-chunk-store@^1.2.0: version "1.3.0" resolved "https://registry.yarnpkg.com/memory-chunk-store/-/memory-chunk-store-1.3.0.tgz#ae99e7e3b58b52db43d49d94722930d39459d0c4" @@ -7201,6 +7224,13 @@ timed-out@^4.0.0: version "4.0.1" resolved "https://registry.yarnpkg.com/timed-out/-/timed-out-4.0.1.tgz#f32eacac5a175bea25d7fab565ab3ed8741ef56f" +timers-ext@^0.1.5: + version "0.1.5" + resolved "https://registry.yarnpkg.com/timers-ext/-/timers-ext-0.1.5.tgz#77147dd4e76b660c2abb8785db96574cbbd12922" + dependencies: + es5-ext "~0.10.14" + next-tick "1" + tiny-lr@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/tiny-lr/-/tiny-lr-1.1.1.tgz#9fa547412f238fedb068ee295af8b682c98b2aab" From cfc16a6db88378f83fa3a501170fa0fc5e7d6636 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 14 Sep 2018 14:36:12 +0200 Subject: [PATCH 07/44] Fix tag search on overview page --- client/src/app/videos/video-list/video-overview.component.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/app/videos/video-list/video-overview.component.html b/client/src/app/videos/video-list/video-overview.component.html index 4150cd5e1..4dad6a6e4 100644 --- a/client/src/app/videos/video-list/video-overview.component.html +++ b/client/src/app/videos/video-list/video-overview.component.html @@ -12,7 +12,7 @@
- {{ object.tag }} + {{ object.tag }}
From 4b5384f6e7be62d072d21d8d964951ba572ab10e Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 14 Sep 2018 14:57:59 +0200 Subject: [PATCH 08/44] Add redundancy stats --- server/controllers/api/overviews.ts | 20 +++--- server/controllers/api/server/stats.ts | 14 +++- server/initializers/constants.ts | 3 +- server/models/redundancy/video-redundancy.ts | 31 ++++++++ server/tests/api/server/redundancy.ts | 75 +++++++++++++++----- server/tests/api/server/stats.ts | 5 +- server/tests/utils/server/stats.ts | 7 +- shared/models/server/server-stats.model.ts | 10 +++ 8 files changed, 132 insertions(+), 33 deletions(-) diff --git a/server/controllers/api/overviews.ts b/server/controllers/api/overviews.ts index cc3cc54a7..8b6773056 100644 --- a/server/controllers/api/overviews.ts +++ b/server/controllers/api/overviews.ts @@ -21,6 +21,16 @@ export { overviewsRouter } // --------------------------------------------------------------------------- +const buildSamples = memoizee(async function () { + const [ categories, channels, tags ] = await Promise.all([ + VideoModel.getRandomFieldSamples('category', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT), + VideoModel.getRandomFieldSamples('channelId', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD ,OVERVIEWS.VIDEOS.SAMPLES_COUNT), + TagModel.getRandomSamples(OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT) + ]) + + return { categories, channels, tags } +}, { maxAge: MEMOIZE_TTL.OVERVIEWS_SAMPLE }) + // This endpoint could be quite long, but we cache it async function getVideosOverview (req: express.Request, res: express.Response) { const attributes = await buildSamples() @@ -45,16 +55,6 @@ async function getVideosOverview (req: express.Request, res: express.Response) { return res.json(result) } -const buildSamples = memoizee(async function () { - const [ categories, channels, tags ] = await Promise.all([ - VideoModel.getRandomFieldSamples('category', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT), - VideoModel.getRandomFieldSamples('channelId', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD ,OVERVIEWS.VIDEOS.SAMPLES_COUNT), - TagModel.getRandomSamples(OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT) - ]) - - return { categories, channels, tags } -}, { maxAge: MEMOIZE_TTL.OVERVIEWS_SAMPLE }) - async function getVideosByTag (tag: string, res: express.Response) { const videos = await getVideos(res, { tagsOneOf: [ tag ] }) diff --git a/server/controllers/api/server/stats.ts b/server/controllers/api/server/stats.ts index 6f4fe938c..bb6311e81 100644 --- a/server/controllers/api/server/stats.ts +++ b/server/controllers/api/server/stats.ts @@ -5,10 +5,14 @@ import { UserModel } from '../../../models/account/user' import { ActorFollowModel } from '../../../models/activitypub/actor-follow' import { VideoModel } from '../../../models/video/video' import { VideoCommentModel } from '../../../models/video/video-comment' +import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' +import { CONFIG, ROUTE_CACHE_LIFETIME } from '../../../initializers/constants' +import { cacheRoute } from '../../../middlewares/cache' const statsRouter = express.Router() statsRouter.get('/stats', + asyncMiddleware(cacheRoute(ROUTE_CACHE_LIFETIME.STATS)), asyncMiddleware(getStats) ) @@ -18,6 +22,13 @@ async function getStats (req: express.Request, res: express.Response, next: expr const { totalUsers } = await UserModel.getStats() const { totalInstanceFollowers, totalInstanceFollowing } = await ActorFollowModel.getStats() + const videosRedundancyStats = await Promise.all( + CONFIG.REDUNDANCY.VIDEOS.map(r => { + return VideoRedundancyModel.getStats(r.strategy) + .then(stats => Object.assign(stats, { strategy: r.strategy, totalSize: r.size })) + }) + ) + const data: ServerStats = { totalLocalVideos, totalLocalVideoViews, @@ -26,7 +37,8 @@ async function getStats (req: express.Request, res: express.Response, next: expr totalVideoComments, totalUsers, totalInstanceFollowers, - totalInstanceFollowing + totalInstanceFollowing, + videosRedundancy: videosRedundancyStats } return res.json(data).end() diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 9cccb0919..e8dab21db 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -66,7 +66,8 @@ const ROUTE_CACHE_LIFETIME = { }, ACTIVITY_PUB: { VIDEOS: '1 second' // 1 second, cache concurrent requests after a broadcast for example - } + }, + STATS: '4 hours' } // --------------------------------------------------------------------------- diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts index b7454c617..6ae02efb9 100644 --- a/server/models/redundancy/video-redundancy.ts +++ b/server/models/redundancy/video-redundancy.ts @@ -245,6 +245,37 @@ export class VideoRedundancyModel extends Model { .findAll(query) } + static async getStats (strategy: VideoRedundancyStrategy) { + const actor = await getServerActor() + + const query = { + raw: true, + attributes: [ + [ Sequelize.fn('COALESCE', Sequelize.fn('SUM', Sequelize.col('VideoFile.size')), '0'), 'totalUsed' ], + [ Sequelize.fn('COUNT', Sequelize.fn('DISTINCT', 'videoId')), 'totalVideos' ], + [ Sequelize.fn('COUNT', 'videoFileId'), 'totalVideoFiles' ] + ], + where: { + strategy, + actorId: actor.id + }, + include: [ + { + attributes: [], + model: VideoFileModel, + required: true + } + ] + } + + return VideoRedundancyModel.find(query as any) // FIXME: typings + .then((r: any) => ({ + totalUsed: parseInt(r.totalUsed.toString(), 10), + totalVideos: r.totalVideos, + totalVideoFiles: r.totalVideoFiles + })) + } + toActivityPubObject (): CacheFileObject { return { id: this.url, diff --git a/server/tests/api/server/redundancy.ts b/server/tests/api/server/redundancy.ts index 6574e8ea9..c0ab251e6 100644 --- a/server/tests/api/server/redundancy.ts +++ b/server/tests/api/server/redundancy.ts @@ -23,6 +23,8 @@ import { ActorFollow } from '../../../../shared/models/actors' import { readdir } from 'fs-extra' import { join } from 'path' import { VideoRedundancyStrategy } from '../../../../shared/models/redundancy' +import { getStats } from '../../utils/server/stats' +import { ServerStats } from '../../../../shared/models/server/server-stats.model' const expect = chai.expect @@ -79,16 +81,32 @@ async function runServers (strategy: VideoRedundancyStrategy, additionalParams: await waitJobs(servers) } -async function check1WebSeed () { +async function check1WebSeed (strategy: VideoRedundancyStrategy) { const webseeds = [ 'http://localhost:9002/static/webseed/' + video1Server2UUID ] for (const server of servers) { - const res = await getVideo(server.url, video1Server2UUID) + { + const res = await getVideo(server.url, video1Server2UUID) - const video: VideoDetails = res.body - video.files.forEach(f => checkMagnetWebseeds(f, webseeds)) + const video: VideoDetails = res.body + video.files.forEach(f => checkMagnetWebseeds(f, webseeds)) + } + + { + const res = await getStats(server.url) + const data: ServerStats = res.body + + expect(data.videosRedundancy).to.have.lengthOf(1) + + const stat = data.videosRedundancy[0] + expect(stat.strategy).to.equal(strategy) + expect(stat.totalSize).to.equal(102400) + expect(stat.totalUsed).to.equal(0) + expect(stat.totalVideoFiles).to.equal(0) + expect(stat.totalVideos).to.equal(0) + } } } @@ -107,7 +125,7 @@ async function enableRedundancy () { expect(server2.following.hostRedundancyAllowed).to.be.true } -async function check2Webseeds () { +async function check2Webseeds (strategy: VideoRedundancyStrategy) { await waitJobs(servers) await wait(15000) await waitJobs(servers) @@ -118,12 +136,14 @@ async function check2Webseeds () { ] for (const server of servers) { - const res = await getVideo(server.url, video1Server2UUID) + { + const res = await getVideo(server.url, video1Server2UUID) - const video: VideoDetails = res.body + const video: VideoDetails = res.body - for (const file of video.files) { - checkMagnetWebseeds(file, webseeds) + for (const file of video.files) { + checkMagnetWebseeds(file, webseeds) + } } } @@ -133,6 +153,20 @@ async function check2Webseeds () { for (const resolution of [ 240, 360, 480, 720 ]) { expect(files.find(f => f === `${video1Server2UUID}-${resolution}.mp4`)).to.not.be.undefined } + + { + const res = await getStats(servers[0].url) + const data: ServerStats = res.body + + expect(data.videosRedundancy).to.have.lengthOf(1) + const stat = data.videosRedundancy[0] + + expect(stat.strategy).to.equal(strategy) + expect(stat.totalSize).to.equal(102400) + expect(stat.totalUsed).to.be.at.least(1).and.below(102401) + expect(stat.totalVideoFiles).to.equal(4) + expect(stat.totalVideos).to.equal(1) + } } async function cleanServers () { @@ -142,15 +176,16 @@ async function cleanServers () { describe('Test videos redundancy', function () { describe('With most-views strategy', function () { + const strategy = 'most-views' before(function () { this.timeout(120000) - return runServers('most-views') + return runServers(strategy) }) it('Should have 1 webseed on the first video', function () { - return check1WebSeed() + return check1WebSeed(strategy) }) it('Should enable redundancy on server 1', function () { @@ -160,7 +195,7 @@ describe('Test videos redundancy', function () { it('Should have 2 webseed on the first video', function () { this.timeout(40000) - return check2Webseeds() + return check2Webseeds(strategy) }) after(function () { @@ -169,15 +204,16 @@ describe('Test videos redundancy', function () { }) describe('With trending strategy', function () { + const strategy = 'trending' before(function () { this.timeout(120000) - return runServers('trending') + return runServers(strategy) }) it('Should have 1 webseed on the first video', function () { - return check1WebSeed() + return check1WebSeed(strategy) }) it('Should enable redundancy on server 1', function () { @@ -187,7 +223,7 @@ describe('Test videos redundancy', function () { it('Should have 2 webseed on the first video', function () { this.timeout(40000) - return check2Webseeds() + return check2Webseeds(strategy) }) after(function () { @@ -196,15 +232,16 @@ describe('Test videos redundancy', function () { }) describe('With recently added strategy', function () { + const strategy = 'recently-added' before(function () { this.timeout(120000) - return runServers('recently-added', { minViews: 3 }) + return runServers(strategy, { minViews: 3 }) }) it('Should have 1 webseed on the first video', function () { - return check1WebSeed() + return check1WebSeed(strategy) }) it('Should enable redundancy on server 1', function () { @@ -218,7 +255,7 @@ describe('Test videos redundancy', function () { await wait(15000) await waitJobs(servers) - return check1WebSeed() + return check1WebSeed(strategy) }) it('Should view 2 times the first video', async function () { @@ -234,7 +271,7 @@ describe('Test videos redundancy', function () { it('Should have 2 webseed on the first video', function () { this.timeout(40000) - return check2Webseeds() + return check2Webseeds(strategy) }) after(function () { diff --git a/server/tests/api/server/stats.ts b/server/tests/api/server/stats.ts index fc9b88805..d8a3268bb 100644 --- a/server/tests/api/server/stats.ts +++ b/server/tests/api/server/stats.ts @@ -21,7 +21,7 @@ import { waitJobs } from '../../utils/server/jobs' const expect = chai.expect -describe('Test stats', function () { +describe('Test stats (excluding redundancy)', function () { let servers: ServerInfo[] = [] before(async function () { @@ -65,6 +65,7 @@ describe('Test stats', function () { expect(data.totalVideos).to.equal(1) expect(data.totalInstanceFollowers).to.equal(2) expect(data.totalInstanceFollowing).to.equal(1) + expect(data.videosRedundancy).to.have.lengthOf(0) }) it('Should have the correct stats on instance 2', async function () { @@ -79,6 +80,7 @@ describe('Test stats', function () { expect(data.totalVideos).to.equal(1) expect(data.totalInstanceFollowers).to.equal(1) expect(data.totalInstanceFollowing).to.equal(1) + expect(data.videosRedundancy).to.have.lengthOf(0) }) it('Should have the correct stats on instance 3', async function () { @@ -93,6 +95,7 @@ describe('Test stats', function () { expect(data.totalVideos).to.equal(1) expect(data.totalInstanceFollowing).to.equal(1) expect(data.totalInstanceFollowers).to.equal(0) + expect(data.videosRedundancy).to.have.lengthOf(0) }) after(async function () { diff --git a/server/tests/utils/server/stats.ts b/server/tests/utils/server/stats.ts index 9cdec6cff..01989d952 100644 --- a/server/tests/utils/server/stats.ts +++ b/server/tests/utils/server/stats.ts @@ -1,11 +1,16 @@ import { makeGetRequest } from '../' -function getStats (url: string) { +function getStats (url: string, useCache = false) { const path = '/api/v1/server/stats' + const query = { + t: useCache ? undefined : new Date().getTime() + } + return makeGetRequest({ url, path, + query, statusCodeExpected: 200 }) } diff --git a/shared/models/server/server-stats.model.ts b/shared/models/server/server-stats.model.ts index 5c1bf3468..a6bd2d4d3 100644 --- a/shared/models/server/server-stats.model.ts +++ b/shared/models/server/server-stats.model.ts @@ -1,3 +1,5 @@ +import { VideoRedundancyStrategy } from '../redundancy' + export interface ServerStats { totalUsers: number totalLocalVideos: number @@ -9,4 +11,12 @@ export interface ServerStats { totalInstanceFollowers: number totalInstanceFollowing: number + + videosRedundancy: { + strategy: VideoRedundancyStrategy + totalSize: number + totalUsed: number + totalVideoFiles: number + totalVideos: number + }[] } From d61b817890d5d5bba61d447518321870498028d8 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 14 Sep 2018 16:47:15 +0200 Subject: [PATCH 09/44] Process inbox activities in a queue --- server/controllers/activitypub/inbox.ts | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/server/controllers/activitypub/inbox.ts b/server/controllers/activitypub/inbox.ts index 20bd20ed4..738d155eb 100644 --- a/server/controllers/activitypub/inbox.ts +++ b/server/controllers/activitypub/inbox.ts @@ -7,6 +7,8 @@ import { asyncMiddleware, checkSignature, localAccountValidator, localVideoChann import { activityPubValidator } from '../../middlewares/validators/activitypub/activity' import { VideoChannelModel } from '../../models/video/video-channel' import { AccountModel } from '../../models/account/account' +import { queue } from 'async' +import { ActorModel } from '../../models/activitypub/actor' const inboxRouter = express.Router() @@ -14,7 +16,7 @@ inboxRouter.post('/inbox', signatureValidator, asyncMiddleware(checkSignature), asyncMiddleware(activityPubValidator), - asyncMiddleware(inboxController) + inboxController ) inboxRouter.post('/accounts/:name/inbox', @@ -22,14 +24,14 @@ inboxRouter.post('/accounts/:name/inbox', asyncMiddleware(checkSignature), asyncMiddleware(localAccountValidator), asyncMiddleware(activityPubValidator), - asyncMiddleware(inboxController) + inboxController ) inboxRouter.post('/video-channels/:name/inbox', signatureValidator, asyncMiddleware(checkSignature), asyncMiddleware(localVideoChannelValidator), asyncMiddleware(activityPubValidator), - asyncMiddleware(inboxController) + inboxController ) // --------------------------------------------------------------------------- @@ -40,7 +42,12 @@ export { // --------------------------------------------------------------------------- -async function inboxController (req: express.Request, res: express.Response, next: express.NextFunction) { +const inboxQueue = queue<{ activities: Activity[], signatureActor?: ActorModel, inboxActor?: ActorModel }, Error>((task, cb) => { + processActivities(task.activities, task.signatureActor, task.inboxActor) + .then(() => cb()) +}) + +function inboxController (req: express.Request, res: express.Response, next: express.NextFunction) { const rootActivity: RootActivity = req.body let activities: Activity[] = [] @@ -66,7 +73,11 @@ async function inboxController (req: express.Request, res: express.Response, nex logger.info('Receiving inbox requests for %d activities by %s.', activities.length, res.locals.signature.actor.url) - await processActivities(activities, res.locals.signature.actor, accountOrChannel ? accountOrChannel.Actor : undefined) + inboxQueue.push({ + activities, + signatureActor: res.locals.signature.actor, + inboxActor: accountOrChannel ? accountOrChannel.Actor : undefined + }) - res.status(204).end() + return res.status(204).end() } From a2377d15ee09301cf4cc5434ad865a21918da15f Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 14 Sep 2018 16:51:35 +0200 Subject: [PATCH 10/44] Refractor activities sending --- server/controllers/api/videos/index.ts | 4 +- server/lib/activitypub/audience.ts | 10 +- .../lib/activitypub/process/process-delete.ts | 2 + server/lib/activitypub/send/send-announce.ts | 4 +- server/lib/activitypub/send/send-create.ts | 95 +++++++++---------- server/lib/activitypub/send/send-delete.ts | 13 +-- server/lib/activitypub/send/send-like.ts | 23 ++--- server/lib/activitypub/send/send-undo.ts | 95 ++++++++----------- server/lib/activitypub/send/send-update.ts | 16 ++-- server/lib/activitypub/send/utils.ts | 30 +++++- server/tests/api/server/stats.ts | 3 - 11 files changed, 140 insertions(+), 155 deletions(-) diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts index 0c9e6c2d1..8353a649a 100644 --- a/server/controllers/api/videos/index.ts +++ b/server/controllers/api/videos/index.ts @@ -393,9 +393,9 @@ async function viewVideo (req: express.Request, res: express.Response) { Redis.Instance.setIPVideoView(ip, videoInstance.uuid) ]) - const serverAccount = await getServerActor() + const serverActor = await getServerActor() - await sendCreateView(serverAccount, videoInstance, undefined) + await sendCreateView(serverActor, videoInstance, undefined) return res.status(204).end() } diff --git a/server/lib/activitypub/audience.ts b/server/lib/activitypub/audience.ts index 7b4067c11..a86428461 100644 --- a/server/lib/activitypub/audience.ts +++ b/server/lib/activitypub/audience.ts @@ -6,7 +6,7 @@ import { VideoModel } from '../../models/video/video' import { VideoCommentModel } from '../../models/video/video-comment' import { VideoShareModel } from '../../models/video/video-share' -function getVideoAudience (video: VideoModel, actorsInvolvedInVideo: ActorModel[]) { +function getRemoteVideoAudience (video: VideoModel, actorsInvolvedInVideo: ActorModel[]): ActivityAudience { return { to: [ video.VideoChannel.Account.Actor.url ], cc: actorsInvolvedInVideo.map(a => a.followersUrl) @@ -18,7 +18,7 @@ function getVideoCommentAudience ( threadParentComments: VideoCommentModel[], actorsInvolvedInVideo: ActorModel[], isOrigin = false -) { +): ActivityAudience { const to = [ ACTIVITY_PUB.PUBLIC ] const cc: string[] = [] @@ -41,7 +41,7 @@ function getVideoCommentAudience ( } } -function getObjectFollowersAudience (actorsInvolvedInObject: ActorModel[]) { +function getAudienceFromFollowersOf (actorsInvolvedInObject: ActorModel[]): ActivityAudience { return { to: [ ACTIVITY_PUB.PUBLIC ].concat(actorsInvolvedInObject.map(a => a.followersUrl)), cc: [] @@ -83,9 +83,9 @@ function audiencify (object: T, audience: ActivityAudience) { export { buildAudience, getAudience, - getVideoAudience, + getRemoteVideoAudience, getActorsInvolvedInVideo, - getObjectFollowersAudience, + getAudienceFromFollowersOf, audiencify, getVideoCommentAudience } diff --git a/server/lib/activitypub/process/process-delete.ts b/server/lib/activitypub/process/process-delete.ts index 3c830abea..4c034a81c 100644 --- a/server/lib/activitypub/process/process-delete.ts +++ b/server/lib/activitypub/process/process-delete.ts @@ -41,6 +41,8 @@ async function processDeleteActivity (activity: ActivityDelete) { { const videoInstance = await VideoModel.loadByUrlAndPopulateAccount(objectUrl) if (videoInstance) { + if (videoInstance.isOwned()) throw new Error(`Remote instance cannot delete owned video ${videoInstance.url}.`) + return retryTransactionWrapper(processDeleteVideo, actor, videoInstance) } } diff --git a/server/lib/activitypub/send/send-announce.ts b/server/lib/activitypub/send/send-announce.ts index f137217f8..cd0cab7ee 100644 --- a/server/lib/activitypub/send/send-announce.ts +++ b/server/lib/activitypub/send/send-announce.ts @@ -4,14 +4,14 @@ import { ActorModel } from '../../../models/activitypub/actor' import { VideoModel } from '../../../models/video/video' import { VideoShareModel } from '../../../models/video/video-share' import { broadcastToFollowers } from './utils' -import { audiencify, getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience } from '../audience' +import { audiencify, getActorsInvolvedInVideo, getAudience, getAudienceFromFollowersOf } from '../audience' import { logger } from '../../../helpers/logger' async function buildAnnounceWithVideoAudience (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) { const announcedObject = video.url const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) - const audience = getObjectFollowersAudience(actorsInvolvedInVideo) + const audience = getAudienceFromFollowersOf(actorsInvolvedInVideo) const activity = buildAnnounceActivity(videoShare.url, byActor, announcedObject, audience) diff --git a/server/lib/activitypub/send/send-create.ts b/server/lib/activitypub/send/send-create.ts index 6f89b1a22..285edba3b 100644 --- a/server/lib/activitypub/send/send-create.ts +++ b/server/lib/activitypub/send/send-create.ts @@ -1,21 +1,13 @@ import { Transaction } from 'sequelize' import { ActivityAudience, ActivityCreate } from '../../../../shared/models/activitypub' import { VideoPrivacy } from '../../../../shared/models/videos' -import { getServerActor } from '../../../helpers/utils' import { ActorModel } from '../../../models/activitypub/actor' import { VideoModel } from '../../../models/video/video' import { VideoAbuseModel } from '../../../models/video/video-abuse' import { VideoCommentModel } from '../../../models/video/video-comment' import { getVideoAbuseActivityPubUrl, getVideoDislikeActivityPubUrl, getVideoViewActivityPubUrl } from '../url' -import { broadcastToActors, broadcastToFollowers, unicastTo } from './utils' -import { - audiencify, - getActorsInvolvedInVideo, - getAudience, - getObjectFollowersAudience, - getVideoAudience, - getVideoCommentAudience -} from '../audience' +import { broadcastToActors, broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils' +import { audiencify, getActorsInvolvedInVideo, getAudience, getAudienceFromFollowersOf, getVideoCommentAudience } from '../audience' import { logger } from '../../../helpers/logger' import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' @@ -40,6 +32,7 @@ async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, logger.info('Creating job to send video abuse %s.', url) + // Custom audience, we only send the abuse to the origin instance const audience = { to: [ video.VideoChannel.Account.Actor.url ], cc: [] } const createActivity = buildCreateActivity(url, byActor, videoAbuse.toActivityPubObject(), audience) @@ -49,15 +42,15 @@ async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, async function sendCreateCacheFile (byActor: ActorModel, fileRedundancy: VideoRedundancyModel) { logger.info('Creating job to send file cache of %s.', fileRedundancy.url) + const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(fileRedundancy.VideoFile.Video.id) const redundancyObject = fileRedundancy.toActivityPubObject() - const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(fileRedundancy.VideoFile.Video.id) - const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, undefined) - - const audience = getVideoAudience(video, actorsInvolvedInVideo) - const createActivity = buildCreateActivity(fileRedundancy.url, byActor, redundancyObject, audience) - - return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) + return sendVideoRelatedCreateActivity({ + byActor, + video, + url: fileRedundancy.url, + object: redundancyObject + }) } async function sendCreateVideoComment (comment: VideoCommentModel, t: Transaction) { @@ -70,6 +63,7 @@ async function sendCreateVideoComment (comment: VideoCommentModel, t: Transactio const commentObject = comment.toActivityPubObject(threadParentComments) const actorsInvolvedInComment = await getActorsInvolvedInVideo(comment.Video, t) + // Add the actor that commented too actorsInvolvedInComment.push(byActor) const parentsCommentActors = threadParentComments.map(c => c.Account.Actor) @@ -78,7 +72,7 @@ async function sendCreateVideoComment (comment: VideoCommentModel, t: Transactio if (isOrigin) { audience = getVideoCommentAudience(comment, threadParentComments, actorsInvolvedInComment, isOrigin) } else { - audience = getObjectFollowersAudience(actorsInvolvedInComment.concat(parentsCommentActors)) + audience = getAudienceFromFollowersOf(actorsInvolvedInComment.concat(parentsCommentActors)) } const createActivity = buildCreateActivity(comment.url, byActor, commentObject, audience) @@ -103,24 +97,14 @@ async function sendCreateView (byActor: ActorModel, video: VideoModel, t: Transa const url = getVideoViewActivityPubUrl(byActor, video) const viewActivity = buildViewActivity(byActor, video) - const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) - - // Send to origin - if (video.isOwned() === false) { - const audience = getVideoAudience(video, actorsInvolvedInVideo) - const createActivity = buildCreateActivity(url, byActor, viewActivity, audience) - - return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) - } - - // Send to followers - const audience = getObjectFollowersAudience(actorsInvolvedInVideo) - const createActivity = buildCreateActivity(url, byActor, viewActivity, audience) - - // Use the server actor to send the view - const serverActor = await getServerActor() - const actorsException = [ byActor ] - return broadcastToFollowers(createActivity, serverActor, actorsInvolvedInVideo, t, actorsException) + return sendVideoRelatedCreateActivity({ + // Use the server actor to send the view + byActor, + video, + url, + object: viewActivity, + transaction: t + }) } async function sendCreateDislike (byActor: ActorModel, video: VideoModel, t: Transaction) { @@ -129,22 +113,13 @@ async function sendCreateDislike (byActor: ActorModel, video: VideoModel, t: Tra const url = getVideoDislikeActivityPubUrl(byActor, video) const dislikeActivity = buildDislikeActivity(byActor, video) - const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) - - // Send to origin - if (video.isOwned() === false) { - const audience = getVideoAudience(video, actorsInvolvedInVideo) - const createActivity = buildCreateActivity(url, byActor, dislikeActivity, audience) - - return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) - } - - // Send to followers - const audience = getObjectFollowersAudience(actorsInvolvedInVideo) - const createActivity = buildCreateActivity(url, byActor, dislikeActivity, audience) - - const actorsException = [ byActor ] - return broadcastToFollowers(createActivity, byActor, actorsInvolvedInVideo, t, actorsException) + return sendVideoRelatedCreateActivity({ + byActor, + video, + url, + object: dislikeActivity, + transaction: t + }) } function buildCreateActivity (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityCreate { @@ -189,3 +164,19 @@ export { sendCreateVideoComment, sendCreateCacheFile } + +// --------------------------------------------------------------------------- + +async function sendVideoRelatedCreateActivity (options: { + byActor: ActorModel, + video: VideoModel, + url: string, + object: any, + transaction?: Transaction +}) { + const activityBuilder = (audience: ActivityAudience) => { + return buildCreateActivity(options.url, options.byActor, options.object, audience) + } + + return sendVideoRelatedActivity(activityBuilder, options) +} diff --git a/server/lib/activitypub/send/send-delete.ts b/server/lib/activitypub/send/send-delete.ts index 479182543..18969433a 100644 --- a/server/lib/activitypub/send/send-delete.ts +++ b/server/lib/activitypub/send/send-delete.ts @@ -5,21 +5,22 @@ import { VideoModel } from '../../../models/video/video' import { VideoCommentModel } from '../../../models/video/video-comment' import { VideoShareModel } from '../../../models/video/video-share' import { getDeleteActivityPubUrl } from '../url' -import { broadcastToActors, broadcastToFollowers, unicastTo } from './utils' +import { broadcastToActors, broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils' import { audiencify, getActorsInvolvedInVideo, getVideoCommentAudience } from '../audience' import { logger } from '../../../helpers/logger' -async function sendDeleteVideo (video: VideoModel, t: Transaction) { +async function sendDeleteVideo (video: VideoModel, transaction: Transaction) { logger.info('Creating job to broadcast delete of video %s.', video.url) - const url = getDeleteActivityPubUrl(video.url) const byActor = video.VideoChannel.Account.Actor - const activity = buildDeleteActivity(url, video.url, byActor) + const activityBuilder = (audience: ActivityAudience) => { + const url = getDeleteActivityPubUrl(video.url) - const actorsInvolved = await getActorsInvolvedInVideo(video, t) + return buildDeleteActivity(url, video.url, byActor, audience) + } - return broadcastToFollowers(activity, byActor, actorsInvolved, t) + return sendVideoRelatedActivity(activityBuilder, { byActor, video, transaction }) } async function sendDeleteActor (byActor: ActorModel, t: Transaction) { diff --git a/server/lib/activitypub/send/send-like.ts b/server/lib/activitypub/send/send-like.ts index a5408ac6a..89307acc6 100644 --- a/server/lib/activitypub/send/send-like.ts +++ b/server/lib/activitypub/send/send-like.ts @@ -3,31 +3,20 @@ import { ActivityAudience, ActivityLike } from '../../../../shared/models/activi import { ActorModel } from '../../../models/activitypub/actor' import { VideoModel } from '../../../models/video/video' import { getVideoLikeActivityPubUrl } from '../url' -import { broadcastToFollowers, unicastTo } from './utils' -import { audiencify, getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience, getVideoAudience } from '../audience' +import { sendVideoRelatedActivity } from './utils' +import { audiencify, getAudience } from '../audience' import { logger } from '../../../helpers/logger' async function sendLike (byActor: ActorModel, video: VideoModel, t: Transaction) { logger.info('Creating job to like %s.', video.url) - const url = getVideoLikeActivityPubUrl(byActor, video) + const activityBuilder = (audience: ActivityAudience) => { + const url = getVideoLikeActivityPubUrl(byActor, video) - const accountsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) - - // Send to origin - if (video.isOwned() === false) { - const audience = getVideoAudience(video, accountsInvolvedInVideo) - const data = buildLikeActivity(url, byActor, video, audience) - - return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) + return buildLikeActivity(url, byActor, video, audience) } - // Send to followers - const audience = getObjectFollowersAudience(accountsInvolvedInVideo) - const activity = buildLikeActivity(url, byActor, video, audience) - - const followersException = [ byActor ] - return broadcastToFollowers(activity, byActor, accountsInvolvedInVideo, t, followersException) + return sendVideoRelatedActivity(activityBuilder, { byActor, video, transaction: t }) } function buildLikeActivity (url: string, byActor: ActorModel, video: VideoModel, audience?: ActivityAudience): ActivityLike { diff --git a/server/lib/activitypub/send/send-undo.ts b/server/lib/activitypub/send/send-undo.ts index a50673c79..5236d2cb3 100644 --- a/server/lib/activitypub/send/send-undo.ts +++ b/server/lib/activitypub/send/send-undo.ts @@ -11,8 +11,8 @@ import { ActorModel } from '../../../models/activitypub/actor' import { ActorFollowModel } from '../../../models/activitypub/actor-follow' import { VideoModel } from '../../../models/video/video' import { getActorFollowActivityPubUrl, getUndoActivityPubUrl, getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from '../url' -import { broadcastToFollowers, unicastTo } from './utils' -import { audiencify, getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience, getVideoAudience } from '../audience' +import { broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils' +import { audiencify, getAudience } from '../audience' import { buildCreateActivity, buildDislikeActivity } from './send-create' import { buildFollowActivity } from './send-follow' import { buildLikeActivity } from './send-like' @@ -39,53 +39,6 @@ async function sendUndoFollow (actorFollow: ActorFollowModel, t: Transaction) { return unicastTo(undoActivity, me, following.inboxUrl) } -async function sendUndoLike (byActor: ActorModel, video: VideoModel, t: Transaction) { - logger.info('Creating job to undo a like of video %s.', video.url) - - const likeUrl = getVideoLikeActivityPubUrl(byActor, video) - const undoUrl = getUndoActivityPubUrl(likeUrl) - - const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) - const likeActivity = buildLikeActivity(likeUrl, byActor, video) - - // Send to origin - if (video.isOwned() === false) { - const audience = getVideoAudience(video, actorsInvolvedInVideo) - const undoActivity = undoActivityData(undoUrl, byActor, likeActivity, audience) - - return unicastTo(undoActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) - } - - const audience = getObjectFollowersAudience(actorsInvolvedInVideo) - const undoActivity = undoActivityData(undoUrl, byActor, likeActivity, audience) - - const followersException = [ byActor ] - return broadcastToFollowers(undoActivity, byActor, actorsInvolvedInVideo, t, followersException) -} - -async function sendUndoDislike (byActor: ActorModel, video: VideoModel, t: Transaction) { - logger.info('Creating job to undo a dislike of video %s.', video.url) - - const dislikeUrl = getVideoDislikeActivityPubUrl(byActor, video) - const undoUrl = getUndoActivityPubUrl(dislikeUrl) - - const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) - const dislikeActivity = buildDislikeActivity(byActor, video) - const createDislikeActivity = buildCreateActivity(dislikeUrl, byActor, dislikeActivity) - - if (video.isOwned() === false) { - const audience = getVideoAudience(video, actorsInvolvedInVideo) - const undoActivity = undoActivityData(undoUrl, byActor, createDislikeActivity, audience) - - return unicastTo(undoActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) - } - - const undoActivity = undoActivityData(undoUrl, byActor, createDislikeActivity) - - const followersException = [ byActor ] - return broadcastToFollowers(undoActivity, byActor, actorsInvolvedInVideo, t, followersException) -} - async function sendUndoAnnounce (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) { logger.info('Creating job to undo announce %s.', videoShare.url) @@ -98,20 +51,32 @@ async function sendUndoAnnounce (byActor: ActorModel, videoShare: VideoShareMode return broadcastToFollowers(undoActivity, byActor, actorsInvolvedInVideo, t, followersException) } +async function sendUndoLike (byActor: ActorModel, video: VideoModel, t: Transaction) { + logger.info('Creating job to undo a like of video %s.', video.url) + + const likeUrl = getVideoLikeActivityPubUrl(byActor, video) + const likeActivity = buildLikeActivity(likeUrl, byActor, video) + + return sendUndoVideoRelatedActivity({ byActor, video, url: likeUrl, activity: likeActivity, transaction: t }) +} + +async function sendUndoDislike (byActor: ActorModel, video: VideoModel, t: Transaction) { + logger.info('Creating job to undo a dislike of video %s.', video.url) + + const dislikeUrl = getVideoDislikeActivityPubUrl(byActor, video) + const dislikeActivity = buildDislikeActivity(byActor, video) + const createDislikeActivity = buildCreateActivity(dislikeUrl, byActor, dislikeActivity) + + return sendUndoVideoRelatedActivity({ byActor, video, url: dislikeUrl, activity: createDislikeActivity, transaction: t }) +} + async function sendUndoCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel, t: Transaction) { logger.info('Creating job to undo cache file %s.', redundancyModel.url) - const undoUrl = getUndoActivityPubUrl(redundancyModel.url) - const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id) - const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) - - const audience = getVideoAudience(video, actorsInvolvedInVideo) const createActivity = buildCreateActivity(redundancyModel.url, byActor, redundancyModel.toActivityPubObject()) - const undoActivity = undoActivityData(undoUrl, byActor, createActivity, audience) - - return unicastTo(undoActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) + return sendUndoVideoRelatedActivity({ byActor, video, url: redundancyModel.url, activity: createActivity, transaction: t }) } // --------------------------------------------------------------------------- @@ -144,3 +109,19 @@ function undoActivityData ( audience ) } + +async function sendUndoVideoRelatedActivity (options: { + byActor: ActorModel, + video: VideoModel, + url: string, + activity: ActivityFollow | ActivityLike | ActivityCreate | ActivityAnnounce, + transaction: Transaction +}) { + const activityBuilder = (audience: ActivityAudience) => { + const undoUrl = getUndoActivityPubUrl(options.url) + + return undoActivityData(undoUrl, options.byActor, options.activity, audience) + } + + return sendVideoRelatedActivity(activityBuilder, options) +} diff --git a/server/lib/activitypub/send/send-update.ts b/server/lib/activitypub/send/send-update.ts index 605473338..ec46789b7 100644 --- a/server/lib/activitypub/send/send-update.ts +++ b/server/lib/activitypub/send/send-update.ts @@ -7,8 +7,8 @@ import { VideoModel } from '../../../models/video/video' import { VideoChannelModel } from '../../../models/video/video-channel' import { VideoShareModel } from '../../../models/video/video-share' import { getUpdateActivityPubUrl } from '../url' -import { broadcastToFollowers, unicastTo } from './utils' -import { audiencify, getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience } from '../audience' +import { broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils' +import { audiencify, getActorsInvolvedInVideo, getAudience, getAudienceFromFollowersOf } from '../audience' import { logger } from '../../../helpers/logger' import { VideoCaptionModel } from '../../../models/video/video-caption' import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' @@ -61,16 +61,16 @@ async function sendUpdateActor (accountOrChannel: AccountModel | VideoChannelMod async function sendUpdateCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel) { logger.info('Creating job to update cache file %s.', redundancyModel.url) - const url = getUpdateActivityPubUrl(redundancyModel.url, redundancyModel.updatedAt.toISOString()) const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id) - const redundancyObject = redundancyModel.toActivityPubObject() + const activityBuilder = (audience: ActivityAudience) => { + const redundancyObject = redundancyModel.toActivityPubObject() + const url = getUpdateActivityPubUrl(redundancyModel.url, redundancyModel.updatedAt.toISOString()) - const accountsInvolvedInVideo = await getActorsInvolvedInVideo(video, undefined) - const audience = getObjectFollowersAudience(accountsInvolvedInVideo) + return buildUpdateActivity(url, byActor, redundancyObject, audience) + } - const updateActivity = buildUpdateActivity(url, byActor, redundancyObject, audience) - return unicastTo(updateActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) + return sendVideoRelatedActivity(activityBuilder, { byActor, video }) } // --------------------------------------------------------------------------- diff --git a/server/lib/activitypub/send/utils.ts b/server/lib/activitypub/send/utils.ts index c20c15633..69706e620 100644 --- a/server/lib/activitypub/send/utils.ts +++ b/server/lib/activitypub/send/utils.ts @@ -1,13 +1,36 @@ import { Transaction } from 'sequelize' -import { Activity } from '../../../../shared/models/activitypub' +import { Activity, ActivityAudience } from '../../../../shared/models/activitypub' import { logger } from '../../../helpers/logger' import { ActorModel } from '../../../models/activitypub/actor' import { ActorFollowModel } from '../../../models/activitypub/actor-follow' import { JobQueue } from '../../job-queue' import { VideoModel } from '../../../models/video/video' -import { getActorsInvolvedInVideo } from '../audience' +import { getActorsInvolvedInVideo, getAudienceFromFollowersOf, getRemoteVideoAudience } from '../audience' import { getServerActor } from '../../../helpers/utils' +async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAudience) => Activity, options: { + byActor: ActorModel, + video: VideoModel, + transaction?: Transaction +}) { + const actorsInvolvedInVideo = await getActorsInvolvedInVideo(options.video, options.transaction) + + // Send to origin + if (options.video.isOwned() === false) { + const audience = getRemoteVideoAudience(options.video, actorsInvolvedInVideo) + const activity = activityBuilder(audience) + + return unicastTo(activity, options.byActor, options.video.VideoChannel.Account.Actor.sharedInboxUrl) + } + + // Send to followers + const audience = getAudienceFromFollowersOf(actorsInvolvedInVideo) + const activity = activityBuilder(audience) + + const actorsException = [ options.byActor ] + return broadcastToFollowers(activity, options.byActor, actorsInvolvedInVideo, options.transaction, actorsException) +} + async function forwardVideoRelatedActivity ( activity: Activity, t: Transaction, @@ -110,7 +133,8 @@ export { unicastTo, forwardActivity, broadcastToActors, - forwardVideoRelatedActivity + forwardVideoRelatedActivity, + sendVideoRelatedActivity } // --------------------------------------------------------------------------- diff --git a/server/tests/api/server/stats.ts b/server/tests/api/server/stats.ts index d8a3268bb..cb229e876 100644 --- a/server/tests/api/server/stats.ts +++ b/server/tests/api/server/stats.ts @@ -65,7 +65,6 @@ describe('Test stats (excluding redundancy)', function () { expect(data.totalVideos).to.equal(1) expect(data.totalInstanceFollowers).to.equal(2) expect(data.totalInstanceFollowing).to.equal(1) - expect(data.videosRedundancy).to.have.lengthOf(0) }) it('Should have the correct stats on instance 2', async function () { @@ -80,7 +79,6 @@ describe('Test stats (excluding redundancy)', function () { expect(data.totalVideos).to.equal(1) expect(data.totalInstanceFollowers).to.equal(1) expect(data.totalInstanceFollowing).to.equal(1) - expect(data.videosRedundancy).to.have.lengthOf(0) }) it('Should have the correct stats on instance 3', async function () { @@ -95,7 +93,6 @@ describe('Test stats (excluding redundancy)', function () { expect(data.totalVideos).to.equal(1) expect(data.totalInstanceFollowing).to.equal(1) expect(data.totalInstanceFollowers).to.equal(0) - expect(data.videosRedundancy).to.have.lengthOf(0) }) after(async function () { From ff587059b9d901c26eab34f4b7f0c06130aae10d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?victor=20h=C3=A9ry?= Date: Sun, 16 Sep 2018 17:53:57 +0200 Subject: [PATCH 11/44] Add possibility to manage trust_proxy list in docker image --- support/docker/production/.env | 1 + .../docker/production/config/custom-environment-variables.yaml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/support/docker/production/.env b/support/docker/production/.env index 51c4e0ace..8af161b2a 100644 --- a/support/docker/production/.env +++ b/support/docker/production/.env @@ -3,6 +3,7 @@ PEERTUBE_DB_PASSWORD=postgres_password PEERTUBE_WEBSERVER_HOSTNAME=domain.tld PEERTUBE_WEBSERVER_PORT=443 PEERTUBE_WEBSERVER_HTTPS=true +PEERTUBE_TRUST_PROXY=127.0.0.1 PEERTUBE_SMTP_USERNAME= PEERTUBE_SMTP_PASSWORD= PEERTUBE_SMTP_HOSTNAME= diff --git a/support/docker/production/config/custom-environment-variables.yaml b/support/docker/production/config/custom-environment-variables.yaml index 1c732e2e0..daf885813 100644 --- a/support/docker/production/config/custom-environment-variables.yaml +++ b/support/docker/production/config/custom-environment-variables.yaml @@ -7,6 +7,8 @@ webserver: __name: "PEERTUBE_WEBSERVER_HTTPS" __format: "json" +trust_proxy: "PEERTUBE_TRUST_PROXY" + database: hostname: "PEERTUBE_DB_HOSTNAME" port: From 743164fed1cbeeee0fd39e9848dbe5131b734e71 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 17 Sep 2018 09:39:08 +0200 Subject: [PATCH 12/44] Fix overviews tests --- server/initializers/constants.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index e8dab21db..2e0b32ce2 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -659,6 +659,7 @@ if (isTestInstance() === true) { JOB_ATTEMPTS['email'] = 1 CACHE.VIDEO_CAPTIONS.MAX_AGE = 3000 + MEMOIZE_TTL.OVERVIEWS_SAMPLE = 1 } updateWebserverConfig() From 2ff83ae2923aa93ccefb0fa989ae0bf138a46d77 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 17 Sep 2018 10:12:30 +0200 Subject: [PATCH 13/44] Handle actors search beginning with '@' Something like @toto@example.com --- server/controllers/api/search.ts | 3 +++ server/helpers/webfinger.ts | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/server/controllers/api/search.ts b/server/controllers/api/search.ts index 28a7a04ca..58851d0b5 100644 --- a/server/controllers/api/search.ts +++ b/server/controllers/api/search.ts @@ -56,6 +56,9 @@ function searchVideoChannels (req: express.Request, res: express.Response) { const isURISearch = search.startsWith('http://') || search.startsWith('https://') const parts = search.split('@') + + // Handle strings like @toto@example.com + if (parts.length === 3 && parts[0].length === 0) parts.shift() const isWebfingerSearch = parts.length === 2 && parts.every(p => p.indexOf(' ') === -1) if (isURISearch || isWebfingerSearch) return searchVideoChannelURI(search, isWebfingerSearch, res) diff --git a/server/helpers/webfinger.ts b/server/helpers/webfinger.ts index 10fcec462..156376943 100644 --- a/server/helpers/webfinger.ts +++ b/server/helpers/webfinger.ts @@ -12,7 +12,10 @@ const webfinger = new WebFinger({ request_timeout: 3000 }) -async function loadActorUrlOrGetFromWebfinger (uri: string) { +async function loadActorUrlOrGetFromWebfinger (uriArg: string) { + // Handle strings like @toto@example.com + const uri = uriArg.startsWith('@') ? uriArg.slice(1) : uriArg + const [ name, host ] = uri.split('@') let actor: ActorModel From 860cfb31e343f2317416da738f7155803ef4fe75 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 17 Sep 2018 10:28:38 +0200 Subject: [PATCH 14/44] Fix "no results" on overview page --- client/src/app/shared/overview/overview.service.ts | 2 ++ server/initializers/constants.ts | 1 + 2 files changed, 3 insertions(+) diff --git a/client/src/app/shared/overview/overview.service.ts b/client/src/app/shared/overview/overview.service.ts index 4a4714af6..097079e6d 100644 --- a/client/src/app/shared/overview/overview.service.ts +++ b/client/src/app/shared/overview/overview.service.ts @@ -56,6 +56,8 @@ export class OverviewService { } } + if (observables.length === 0) return of(videosOverviewResult) + return forkJoin(observables) .pipe( // Translate categories diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 2e0b32ce2..02363352e 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -660,6 +660,7 @@ if (isTestInstance() === true) { CACHE.VIDEO_CAPTIONS.MAX_AGE = 3000 MEMOIZE_TTL.OVERVIEWS_SAMPLE = 1 + ROUTE_CACHE_LIFETIME.OVERVIEWS.VIDEOS = '0' } updateWebserverConfig() From c07b6041111daa6dd5d611f31e31819db5992ba8 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 17 Sep 2018 11:28:08 +0200 Subject: [PATCH 15/44] Fix checker if we don't have redundancy strategies --- config/test.yaml | 6 +++--- server/initializers/checker.ts | 8 ++++---- server/initializers/constants.ts | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/config/test.yaml b/config/test.yaml index 517fc7449..16113211e 100644 --- a/config/test.yaml +++ b/config/test.yaml @@ -24,13 +24,13 @@ log: redundancy: videos: - - size: '100KB' + size: '10MB' strategy: 'most-views' - - size: '100KB' + size: '10MB' strategy: 'trending' - - size: '100KB' + size: '10MB' strategy: 'recently-added' minViews: 10 diff --git a/server/initializers/checker.ts b/server/initializers/checker.ts index 29f4f3036..b9dc1e725 100644 --- a/server/initializers/checker.ts +++ b/server/initializers/checker.ts @@ -52,11 +52,11 @@ function checkConfig () { if (filtered.length !== redundancyVideos.length) { return 'Redundancy video entries should have unique strategies' } - } - const recentlyAddedStrategy = redundancyVideos.find(r => r.strategy === 'recently-added') as RecentlyAddedStrategy - if (recentlyAddedStrategy && isNaN(recentlyAddedStrategy.minViews)) { - return 'Min views in recently added strategy is not a number' + const recentlyAddedStrategy = redundancyVideos.find(r => r.strategy === 'recently-added') as RecentlyAddedStrategy + if (recentlyAddedStrategy && isNaN(recentlyAddedStrategy.minViews)) { + return 'Min views in recently added strategy is not a number' + } } return null diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 02363352e..fa9093918 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -660,7 +660,7 @@ if (isTestInstance() === true) { CACHE.VIDEO_CAPTIONS.MAX_AGE = 3000 MEMOIZE_TTL.OVERVIEWS_SAMPLE = 1 - ROUTE_CACHE_LIFETIME.OVERVIEWS.VIDEOS = '0' + ROUTE_CACHE_LIFETIME.OVERVIEWS.VIDEOS = '0ms' } updateWebserverConfig() From b335ccec49b450052e3520f66f9acb6670e669f8 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 17 Sep 2018 15:00:46 +0200 Subject: [PATCH 16/44] Fix ios player playback/subtitles menu --- .../assets/player/peertube-videojs-plugin.ts | 243 +++++++++--------- .../src/assets/player/settings-menu-item.ts | 7 +- client/src/sass/application.scss | 2 +- .../sass/player/{player.scss => index.scss} | 0 client/src/sass/player/peertube-skin.scss | 5 +- client/src/standalone/videos/embed.scss | 2 +- package.json | 2 +- 7 files changed, 132 insertions(+), 129 deletions(-) rename client/src/sass/player/{player.scss => index.scss} (100%) diff --git a/client/src/assets/player/peertube-videojs-plugin.ts b/client/src/assets/player/peertube-videojs-plugin.ts index 4b0677fab..36b80bd72 100644 --- a/client/src/assets/player/peertube-videojs-plugin.ts +++ b/client/src/assets/player/peertube-videojs-plugin.ts @@ -4,7 +4,7 @@ import { VideoFile } from '../../../../shared/models/videos/video.model' import { renderVideo } from './video-renderer' import './settings-menu-button' import { PeertubePluginOptions, VideoJSCaption, VideoJSComponentInterface, videojsUntyped } from './peertube-videojs-typings' -import { isMobile, videoFileMaxByResolution, videoFileMinByResolution, timeToInt } from './utils' +import { isMobile, timeToInt, videoFileMaxByResolution, videoFileMinByResolution } from './utils' import * as CacheChunkStore from 'cache-chunk-store' import { PeertubeChunkStore } from './peertube-chunk-store' import { @@ -83,11 +83,6 @@ class PeerTubePlugin extends Plugin { this.videoCaptions = options.videoCaptions this.savePlayerSrcFunction = this.player.src - // Hack to "simulate" src link in video.js >= 6 - // Without this, we can't play the video after pausing it - // https://github.com/videojs/video.js/blob/master/src/js/player.js#L1633 - this.player.src = () => true - this.playerElement = options.playerElement if (this.autoplay === true) this.player.addClass('vjs-has-autoplay') @@ -104,9 +99,7 @@ class PeerTubePlugin extends Plugin { this.player.one('play', () => { // Don't run immediately scheduler, wait some seconds the TCP connections are made - this.runAutoQualitySchedulerTimer = setTimeout(() => { - this.runAutoQualityScheduler() - }, this.CONSTANTS.AUTO_QUALITY_SCHEDULER) + this.runAutoQualitySchedulerTimer = setTimeout(() => this.runAutoQualityScheduler(), this.CONSTANTS.AUTO_QUALITY_SCHEDULER) }) }) @@ -167,6 +160,9 @@ class PeerTubePlugin extends Plugin { // Do not display error to user because we will have multiple fallback this.disableErrorDisplay() + // Hack to "simulate" src link in video.js >= 6 + // Without this, we can't play the video after pausing it + // https://github.com/videojs/video.js/blob/master/src/js/player.js#L1633 this.player.src = () => true const oldPlaybackRate = this.player.playbackRate() @@ -181,102 +177,6 @@ class PeerTubePlugin extends Plugin { this.trigger('videoFileUpdate') } - addTorrent ( - magnetOrTorrentUrl: string, - previousVideoFile: VideoFile, - options: { - forcePlay?: boolean, - seek?: number, - delay?: number - }, - done: Function - ) { - console.log('Adding ' + magnetOrTorrentUrl + '.') - - const oldTorrent = this.torrent - const torrentOptions = { - store: (chunkLength, storeOpts) => new CacheChunkStore(new PeertubeChunkStore(chunkLength, storeOpts), { - max: 100 - }) - } - - this.torrent = this.webtorrent.add(magnetOrTorrentUrl, torrentOptions, torrent => { - console.log('Added ' + magnetOrTorrentUrl + '.') - - if (oldTorrent) { - // Pause the old torrent - oldTorrent.pause() - // Pause does not remove actual peers (in particular the webseed peer) - oldTorrent.removePeer(oldTorrent['ws']) - - // We use a fake renderer so we download correct pieces of the next file - if (options.delay) { - const fakeVideoElem = document.createElement('video') - renderVideo(torrent.files[0], fakeVideoElem, { autoplay: false, controls: false }, (err, renderer) => { - this.fakeRenderer = renderer - - if (err) console.error('Cannot render new torrent in fake video element.', err) - - // Load the future file at the correct time - fakeVideoElem.currentTime = this.player.currentTime() + (options.delay / 2000) - }) - } - } - - // Render the video in a few seconds? (on resolution change for example, we wait some seconds of the new video resolution) - this.addTorrentDelay = setTimeout(() => { - this.destroyFakeRenderer() - - const paused = this.player.paused() - - this.flushVideoFile(previousVideoFile) - - const renderVideoOptions = { autoplay: false, controls: true } - renderVideo(torrent.files[0], this.playerElement, renderVideoOptions,(err, renderer) => { - this.renderer = renderer - - if (err) return this.fallbackToHttp(done) - - return this.tryToPlay(err => { - if (err) return done(err) - - if (options.seek) this.seek(options.seek) - if (options.forcePlay === false && paused === true) this.player.pause() - - return done(err) - }) - }) - }, options.delay || 0) - }) - - this.torrent.on('error', err => console.error(err)) - - this.torrent.on('warning', (err: any) => { - // We don't support HTTP tracker but we don't care -> we use the web socket tracker - if (err.message.indexOf('Unsupported tracker protocol') !== -1) return - - // Users don't care about issues with WebRTC, but developers do so log it in the console - if (err.message.indexOf('Ice connection failed') !== -1) { - console.log(err) - return - } - - // Magnet hash is not up to date with the torrent file, add directly the torrent file - if (err.message.indexOf('incorrect info hash') !== -1) { - console.error('Incorrect info hash detected, falling back to torrent file.') - const newOptions = { forcePlay: true, seek: options.seek } - return this.addTorrent(this.torrent['xs'], previousVideoFile, newOptions, done) - } - - // Remote instance is down - if (err.message.indexOf('from xs param') !== -1) { - this.handleError(err) - } - - console.warn(err) - }) - } - updateResolution (resolutionId: number, delay = 0) { // Remember player state const currentTime = this.player.currentTime() @@ -336,6 +236,91 @@ class PeerTubePlugin extends Plugin { return this.torrent } + private addTorrent ( + magnetOrTorrentUrl: string, + previousVideoFile: VideoFile, + options: { + forcePlay?: boolean, + seek?: number, + delay?: number + }, + done: Function + ) { + console.log('Adding ' + magnetOrTorrentUrl + '.') + + const oldTorrent = this.torrent + const torrentOptions = { + store: (chunkLength, storeOpts) => new CacheChunkStore(new PeertubeChunkStore(chunkLength, storeOpts), { + max: 100 + }) + } + + this.torrent = this.webtorrent.add(magnetOrTorrentUrl, torrentOptions, torrent => { + console.log('Added ' + magnetOrTorrentUrl + '.') + + if (oldTorrent) { + // Pause the old torrent + this.stopTorrent(oldTorrent) + + // We use a fake renderer so we download correct pieces of the next file + if (options.delay) this.renderFileInFakeElement(torrent.files[ 0 ], options.delay) + } + + // Render the video in a few seconds? (on resolution change for example, we wait some seconds of the new video resolution) + this.addTorrentDelay = setTimeout(() => { + // We don't need the fake renderer anymore + this.destroyFakeRenderer() + + const paused = this.player.paused() + + this.flushVideoFile(previousVideoFile) + + const renderVideoOptions = { autoplay: false, controls: true } + renderVideo(torrent.files[ 0 ], this.playerElement, renderVideoOptions, (err, renderer) => { + this.renderer = renderer + + if (err) return this.fallbackToHttp(done) + + return this.tryToPlay(err => { + if (err) return done(err) + + if (options.seek) this.seek(options.seek) + if (options.forcePlay === false && paused === true) this.player.pause() + + return done(err) + }) + }) + }, options.delay || 0) + }) + + this.torrent.on('error', err => console.error(err)) + + this.torrent.on('warning', (err: any) => { + // We don't support HTTP tracker but we don't care -> we use the web socket tracker + if (err.message.indexOf('Unsupported tracker protocol') !== -1) return + + // Users don't care about issues with WebRTC, but developers do so log it in the console + if (err.message.indexOf('Ice connection failed') !== -1) { + console.log(err) + return + } + + // Magnet hash is not up to date with the torrent file, add directly the torrent file + if (err.message.indexOf('incorrect info hash') !== -1) { + console.error('Incorrect info hash detected, falling back to torrent file.') + const newOptions = { forcePlay: true, seek: options.seek } + return this.addTorrent(this.torrent[ 'xs' ], previousVideoFile, newOptions, done) + } + + // Remote instance is down + if (err.message.indexOf('from xs param') !== -1) { + this.handleError(err) + } + + console.warn(err) + }) + } + private tryToPlay (done?: Function) { if (!done) done = function () { /* empty */ } @@ -435,22 +420,22 @@ class PeerTubePlugin extends Plugin { if (this.autoplay === true) { this.player.posterImage.hide() + return this.updateVideoFile(undefined, { forcePlay: true, seek: this.startTime }) + } + + // Don't try on iOS that does not support MediaSource + if (this.isIOS()) { + this.currentVideoFile = this.pickAverageVideoFile() + return this.fallbackToHttp(undefined, false) + } + + // Proxy first play + const oldPlay = this.player.play.bind(this.player) + this.player.play = () => { + this.player.addClass('vjs-has-big-play-button-clicked') + this.player.play = oldPlay + this.updateVideoFile(undefined, { forcePlay: true, seek: this.startTime }) - } else { - // Don't try on iOS that does not support MediaSource - if (this.isIOS()) { - this.currentVideoFile = this.pickAverageVideoFile() - return this.fallbackToHttp(undefined, false) - } - - // Proxy first play - const oldPlay = this.player.play.bind(this.player) - this.player.play = () => { - this.player.addClass('vjs-has-big-play-button-clicked') - this.player.play = oldPlay - - this.updateVideoFile(undefined, { forcePlay: true, seek: this.startTime }) - } } } @@ -607,6 +592,24 @@ class PeerTubePlugin extends Plugin { return this.videoFiles[Math.floor(this.videoFiles.length / 2)] } + private stopTorrent (torrent: WebTorrent.Torrent) { + torrent.pause() + // Pause does not remove actual peers (in particular the webseed peer) + torrent.removePeer(torrent[ 'ws' ]) + } + + private renderFileInFakeElement (file: WebTorrent.TorrentFile, delay: number) { + const fakeVideoElem = document.createElement('video') + renderVideo(file, fakeVideoElem, { autoplay: false, controls: false }, (err, renderer) => { + this.fakeRenderer = renderer + + if (err) console.error('Cannot render new torrent in fake video element.', err) + + // Load the future file at the correct time (in delay MS - 2 seconds) + fakeVideoElem.currentTime = this.player.currentTime() + (delay - 2000) + }) + } + private destroyFakeRenderer () { if (this.fakeRenderer) { if (this.fakeRenderer.destroy) { diff --git a/client/src/assets/player/settings-menu-item.ts b/client/src/assets/player/settings-menu-item.ts index 6e2224e20..f6cf6d0f3 100644 --- a/client/src/assets/player/settings-menu-item.ts +++ b/client/src/assets/player/settings-menu-item.ts @@ -38,8 +38,11 @@ class SettingsMenuItem extends MenuItem { this.eventHandlers() player.ready(() => { - this.build() - this.reset() + // Voodoo magic for IOS + setTimeout(() => { + this.build() + this.reset() + }, 0) }) } diff --git a/client/src/sass/application.scss b/client/src/sass/application.scss index 8d2bfb077..c1135cd02 100644 --- a/client/src/sass/application.scss +++ b/client/src/sass/application.scss @@ -9,7 +9,7 @@ $icon-font-path: '../../node_modules/@neos21/bootstrap3-glyphicons/assets/fonts/ @import '~video.js/dist/video-js.css'; $assets-path: '../assets/'; -@import './player/player'; +@import './player/index'; @import './loading-bar'; @import './primeng-custom'; diff --git a/client/src/sass/player/player.scss b/client/src/sass/player/index.scss similarity index 100% rename from client/src/sass/player/player.scss rename to client/src/sass/player/index.scss diff --git a/client/src/sass/player/peertube-skin.scss b/client/src/sass/player/peertube-skin.scss index 185b00222..4e921e970 100644 --- a/client/src/sass/player/peertube-skin.scss +++ b/client/src/sass/player/peertube-skin.scss @@ -406,6 +406,7 @@ width: 37px; margin-right: 1px; + cursor: pointer; .vjs-icon-placeholder { transition: transform 0.2s ease; @@ -504,10 +505,6 @@ } } - .vjs-playback-rate { - display: none; - } - .vjs-peertube { padding: 0 !important; diff --git a/client/src/standalone/videos/embed.scss b/client/src/standalone/videos/embed.scss index 30650538f..c40ea1208 100644 --- a/client/src/standalone/videos/embed.scss +++ b/client/src/standalone/videos/embed.scss @@ -4,7 +4,7 @@ @import '~videojs-dock/dist/videojs-dock.css'; $assets-path: '../../assets/'; -@import '../../sass/player/player'; +@import '../../sass/player/index'; [hidden] { display: none !important; diff --git a/package.json b/package.json index 1cb5be181..53e07a72b 100644 --- a/package.json +++ b/package.json @@ -70,7 +70,7 @@ }, "lint-staged": { "*.scss": [ - "sass-lint -c .sass-lint.yml", + "sass-lint -c client/.sass-lint.yml", "git add" ] }, From e452d2e2b80b2fbea88a1fe9c40b49e8241eae92 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 17 Sep 2018 15:28:32 +0200 Subject: [PATCH 17/44] Fix description/comments max width --- client/src/app/videos/+video-watch/video-watch.component.scss | 1 + 1 file changed, 1 insertion(+) diff --git a/client/src/app/videos/+video-watch/video-watch.component.scss b/client/src/app/videos/+video-watch/video-watch.component.scss index fac4bdbe5..15adf0f61 100644 --- a/client/src/app/videos/+video-watch/video-watch.component.scss +++ b/client/src/app/videos/+video-watch/video-watch.component.scss @@ -81,6 +81,7 @@ flex-grow: 1; // Set min width for flex item min-width: 1px; + max-width: 100%; .video-info-first-row { display: flex; From 8c72543a4af7a613496e0581a939996fc284f861 Mon Sep 17 00:00:00 2001 From: Rigel Kent Date: Mon, 17 Sep 2018 16:44:41 +0200 Subject: [PATCH 18/44] adding missing i18n for schedule option --- .../video-add-components/video-upload.component.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/app/videos/+video-edit/video-add-components/video-upload.component.html b/client/src/app/videos/+video-edit/video-add-components/video-upload.component.html index 8c0723155..ff0e45413 100644 --- a/client/src/app/videos/+video-edit/video-add-components/video-upload.component.html +++ b/client/src/app/videos/+video-edit/video-add-components/video-upload.component.html @@ -22,7 +22,7 @@
From 415acc63cf3b51c91f70f75fe93ad0384f7d176a Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 17 Sep 2018 15:45:17 +0200 Subject: [PATCH 19/44] Add comments in nginx regarding blocks that can be safely removed --- README.md | 3 +-- support/nginx/peertube | 4 +++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index cd522301b..9fb89039b 100644 --- a/README.md +++ b/README.md @@ -97,11 +97,10 @@ BitTorrent) inside the web browser, as of today. ## Dependencies * nginx - * PostgreSQL + * **PostgreSQL >= 9.6** * **Redis >= 2.8.18** * **NodeJS >= 8.x** * yarn - * OpenSSL (cli) * **FFmpeg >= 3.x** ## Run in production diff --git a/support/nginx/peertube b/support/nginx/peertube index 0da427037..b00031133 100644 --- a/support/nginx/peertube +++ b/support/nginx/peertube @@ -58,12 +58,14 @@ server { root /var/www/certbot; } + # Bypass PeerTube for performance reasons. Could be removed location ~ ^/client/(.*\.(js|css|woff2|otf|ttf|woff|eot))$ { add_header Cache-Control "public, max-age=31536000, immutable"; alias /var/www/peertube/peertube-latest/client/dist/$1; } + # Bypass PeerTube for performance reasons. Could be removed location ~ ^/static/(thumbnails|avatars)/ { if ($request_method = 'OPTIONS') { add_header 'Access-Control-Allow-Origin' '*'; @@ -102,7 +104,7 @@ server { send_timeout 600; } - # Bypass PeerTube webseed route for better performances + # Bypass PeerTube for performance reasons. Could be removed location /static/webseed { # Clients usually have 4 simultaneous webseed connections, so the real limit is 3MB/s per client limit_rate 800k; From a8ecc6f6709bdb54c47c7dd7cd18ef371254c3af Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 17 Sep 2018 17:36:46 +0200 Subject: [PATCH 20/44] Try to improve infinite pagination --- .../app/shared/video/abstract-video-list.html | 2 +- .../app/shared/video/abstract-video-list.ts | 29 +++++++++++++++++-- .../video/infinite-scroller.directive.ts | 13 +++++++-- 3 files changed, 37 insertions(+), 7 deletions(-) diff --git a/client/src/app/shared/video/abstract-video-list.html b/client/src/app/shared/video/abstract-video-list.html index 0f48b9a64..4ad4e3568 100644 --- a/client/src/app/shared/video/abstract-video-list.html +++ b/client/src/app/shared/video/abstract-video-list.html @@ -7,7 +7,7 @@
No results.
diff --git a/client/src/app/shared/video/abstract-video-list.ts b/client/src/app/shared/video/abstract-video-list.ts index b8fd7f8eb..9df4cfc22 100644 --- a/client/src/app/shared/video/abstract-video-list.ts +++ b/client/src/app/shared/video/abstract-video-list.ts @@ -38,7 +38,7 @@ export abstract class AbstractVideoList implements OnInit, OnDestroy { ownerDisplayType: OwnerDisplayType = 'account' protected baseVideoWidth = 215 - protected baseVideoHeight = 230 + protected baseVideoHeight = 205 protected abstract notificationsService: NotificationsService protected abstract authService: AuthService @@ -55,6 +55,7 @@ export abstract class AbstractVideoList implements OnInit, OnDestroy { protected otherRouteParams = {} private resizeSubscription: Subscription + private firstLoadedPage: number abstract getVideosObservable (page: number): Observable<{ videos: Video[], totalVideos: number}> abstract generateSyndicationList () @@ -100,7 +101,11 @@ export abstract class AbstractVideoList implements OnInit, OnDestroy { this.loadMoreVideos(this.pagination.currentPage) } - loadMoreVideos (page: number) { + loadMoreVideos (page: number, loadOnTop = false) { + this.adjustVideoPageHeight() + + const currentY = window.scrollY + if (this.loadedPages[page] !== undefined) return if (this.loadingPage[page] === true) return @@ -111,6 +116,8 @@ export abstract class AbstractVideoList implements OnInit, OnDestroy { ({ videos, totalVideos }) => { this.loadingPage[page] = false + if (this.firstLoadedPage === undefined || this.firstLoadedPage > page) this.firstLoadedPage = page + // Paging is too high, return to the first one if (this.pagination.currentPage > 1 && totalVideos <= ((this.pagination.currentPage - 1) * this.pagination.itemsPerPage)) { this.pagination.currentPage = 1 @@ -125,8 +132,17 @@ export abstract class AbstractVideoList implements OnInit, OnDestroy { // Initialize infinite scroller now we loaded the first page if (Object.keys(this.loadedPages).length === 1) { // Wait elements creation - setTimeout(() => this.infiniteScroller.initialize(), 500) + setTimeout(() => { + this.infiniteScroller.initialize() + + // At our first load, we did not load the first page + // Load the previous page so the user can move on the top (and browser previous pages) + if (this.pagination.currentPage > 1) this.loadMoreVideos(this.pagination.currentPage - 1, true) + }, 500) } + + // Insert elements on the top but keep the scroll in the previous position + if (loadOnTop) setTimeout(() => { window.scrollTo(0, currentY + this.pageHeight) }, 0) }, error => { this.loadingPage[page] = false @@ -189,6 +205,13 @@ export abstract class AbstractVideoList implements OnInit, OnDestroy { this.videoPages = Object.values(this.loadedPages) } + protected adjustVideoPageHeight () { + const numberOfPagesLoaded = Object.keys(this.loadedPages).length + if (!numberOfPagesLoaded) return + + this.pageHeight = this.videosElement.nativeElement.offsetHeight / numberOfPagesLoaded + } + protected buildVideoHeight () { // Same ratios than base width/height return this.videosElement.nativeElement.offsetWidth * (this.baseVideoHeight / this.baseVideoWidth) diff --git a/client/src/app/shared/video/infinite-scroller.directive.ts b/client/src/app/shared/video/infinite-scroller.directive.ts index 4dc1f86e7..a02e9444a 100644 --- a/client/src/app/shared/video/infinite-scroller.directive.ts +++ b/client/src/app/shared/video/infinite-scroller.directive.ts @@ -6,10 +6,9 @@ import { fromEvent, Subscription } from 'rxjs' selector: '[myInfiniteScroller]' }) export class InfiniteScrollerDirective implements OnInit, OnDestroy { - private static PAGE_VIEW_TOP_MARGIN = 500 - @Input() containerHeight: number @Input() pageHeight: number + @Input() firstLoadedPage = 1 @Input() percentLimit = 70 @Input() autoInit = false @@ -23,6 +22,7 @@ export class InfiniteScrollerDirective implements OnInit, OnDestroy { private scrollDownSub: Subscription private scrollUpSub: Subscription private pageChangeSub: Subscription + private middleScreen: number constructor () { this.decimalLimit = this.percentLimit / 100 @@ -39,6 +39,8 @@ export class InfiniteScrollerDirective implements OnInit, OnDestroy { } initialize () { + this.middleScreen = window.innerHeight / 2 + // Emit the last value const throttleOptions = { leading: true, trailing: true } @@ -92,6 +94,11 @@ export class InfiniteScrollerDirective implements OnInit, OnDestroy { } private calculateCurrentPage (current: number) { - return Math.max(1, Math.round((current + InfiniteScrollerDirective.PAGE_VIEW_TOP_MARGIN) / this.pageHeight)) + const scrollY = current + this.middleScreen + + const page = Math.max(1, Math.ceil(scrollY / this.pageHeight)) + + // Offset page + return page + (this.firstLoadedPage - 1) } } From dae4a1c0f8d8af2528d7e04fef2b8b65b2d52122 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 17 Sep 2018 17:50:51 +0200 Subject: [PATCH 21/44] Improve webtorrent import error message when the torrent has multiple files --- server/helpers/webtorrent.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts index 2fdfd1876..f4b44bc4f 100644 --- a/server/helpers/webtorrent.ts +++ b/server/helpers/webtorrent.ts @@ -24,7 +24,7 @@ function downloadWebTorrentVideo (target: { magnetUri: string, torrentName?: str if (timer) clearTimeout(timer) return safeWebtorrentDestroy(webtorrent, torrentId, file.name, target.torrentName) - .then(() => rej(new Error('The number of files is not equal to 1 for ' + torrentId))) + .then(() => rej(new Error('Cannot import torrent ' + torrentId + ': there are multiple files in it'))) } file = torrent.files[ 0 ] From d5931e623320d0851a19e1001e90c7d8138d7a20 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 18 Sep 2018 08:21:04 +0200 Subject: [PATCH 22/44] Fix client build --- client/src/app/shared/video/abstract-video-list.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/app/shared/video/abstract-video-list.ts b/client/src/app/shared/video/abstract-video-list.ts index 9df4cfc22..53b044478 100644 --- a/client/src/app/shared/video/abstract-video-list.ts +++ b/client/src/app/shared/video/abstract-video-list.ts @@ -36,6 +36,7 @@ export abstract class AbstractVideoList implements OnInit, OnDestroy { videoHeight: number videoPages: Video[][] = [] ownerDisplayType: OwnerDisplayType = 'account' + firstLoadedPage: number protected baseVideoWidth = 215 protected baseVideoHeight = 205 @@ -55,7 +56,6 @@ export abstract class AbstractVideoList implements OnInit, OnDestroy { protected otherRouteParams = {} private resizeSubscription: Subscription - private firstLoadedPage: number abstract getVideosObservable (page: number): Observable<{ videos: Video[], totalVideos: number}> abstract generateSyndicationList () From df182b373fc49f20188d531494e1bff1a9ad247e Mon Sep 17 00:00:00 2001 From: Rigel Kent Date: Tue, 18 Sep 2018 11:18:51 +0200 Subject: [PATCH 23/44] normalize robot.txt and specify test servers as scope of security audits --- SECURITY.md | 2 +- config/default.yaml | 2 +- config/production.yaml.example | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index 37ed19246..5c668a2a3 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -30,7 +30,7 @@ To encourage vulnerability research and to avoid any confusion between good-fait - Avoid violating the privacy of others, disrupting our systems, destroying data, and/or harming user experience. - Use only the Official Channels to discuss vulnerability information with us. - Keep the details of any discovered vulnerabilities confidential until they are fixed, according to the Disclosure Terms in this policy. -- Perform testing only on in-scope systems, and respect systems and activities which are out-of-scope. +- Perform testing only on in-scope systems, and respect systems and activities which are out-of-scope. Systems currently considered in-scope are the official demonstration/test servers provided by the PeerTube development team. - If a vulnerability provides unintended access to data: Limit the amount of data you access to the minimum required for effectively demonstrating a Proof of Concept; and cease testing and submit a report immediately if you encounter any user data during testing, such as Personally Identifiable Information (PII), Personal Healthcare Information (PHI), credit card data, or proprietary information. - You should only interact with test accounts you own or with explicit permission from the account holder. - Do not engage in extortion. diff --git a/config/default.yaml b/config/default.yaml index adac9deeb..ab07bfedd 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -142,7 +142,7 @@ instance: # Robot.txt rules. To disallow robots to crawl your instance and disallow indexation of your site, add '/' to "Disallow:' robots: | User-agent: * - Disallow: '' + Disallow: # Security.txt rules. To discourage researchers from testing your instance and disable security.txt integration, set this to an empty string. securitytxt: "# If you would like to report a security issue\n# you may report it to:\nContact: https://github.com/Chocobozzz/PeerTube/blob/develop/SECURITY.md\nContact: mailto:" diff --git a/config/production.yaml.example b/config/production.yaml.example index ca7b936c2..f9557b8eb 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example @@ -156,7 +156,7 @@ instance: # Robot.txt rules. To disallow robots to crawl your instance and disallow indexation of your site, add '/' to "Disallow:' robots: | User-agent: * - Disallow: '' + Disallow: # Security.txt rules. To discourage researchers from testing your instance and disable security.txt integration, set this to an empty string. securitytxt: "# If you would like to report a security issue\n# you may report it to:\nContact: https://github.com/Chocobozzz/PeerTube/blob/develop/SECURITY.md\nContact: mailto:" From 098eb37797fdadd4adf660b76867da68061fd588 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 18 Sep 2018 11:02:51 +0200 Subject: [PATCH 24/44] Reduce video.ts file size by moving some methods in other files --- config/test.yaml | 2 +- server/lib/client-html.ts | 3 +- server/lib/job-queue/handlers/video-file.ts | 7 +- server/lib/video-transcoding.ts | 130 ++++++ server/models/redundancy/video-redundancy.ts | 2 +- server/models/video/video-format-utils.ts | 295 +++++++++++++ server/models/video/video.ts | 419 +------------------ 7 files changed, 455 insertions(+), 403 deletions(-) create mode 100644 server/lib/video-transcoding.ts create mode 100644 server/models/video/video-format-utils.ts diff --git a/config/test.yaml b/config/test.yaml index 16113211e..d3e0e49ac 100644 --- a/config/test.yaml +++ b/config/test.yaml @@ -32,7 +32,7 @@ redundancy: - size: '10MB' strategy: 'recently-added' - minViews: 10 + minViews: 1 cache: previews: diff --git a/server/lib/client-html.ts b/server/lib/client-html.ts index a69e09c32..b1088c096 100644 --- a/server/lib/client-html.ts +++ b/server/lib/client-html.ts @@ -8,6 +8,7 @@ import { VideoModel } from '../models/video/video' import * as validator from 'validator' import { VideoPrivacy } from '../../shared/models/videos' import { readFile } from 'fs-extra' +import { getActivityStreamDuration } from '../models/video/video-format-utils' export class ClientHtml { @@ -150,7 +151,7 @@ export class ClientHtml { description: videoDescriptionEscaped, thumbnailUrl: previewUrl, uploadDate: video.createdAt.toISOString(), - duration: video.getActivityStreamDuration(), + duration: getActivityStreamDuration(video.duration), contentUrl: videoUrl, embedUrl: embedUrl, interactionCount: video.views diff --git a/server/lib/job-queue/handlers/video-file.ts b/server/lib/job-queue/handlers/video-file.ts index c6308f7a6..2c9ca8e12 100644 --- a/server/lib/job-queue/handlers/video-file.ts +++ b/server/lib/job-queue/handlers/video-file.ts @@ -8,6 +8,7 @@ import { retryTransactionWrapper } from '../../../helpers/database-utils' import { sequelizeTypescript } from '../../../initializers' import * as Bluebird from 'bluebird' import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg-utils' +import { importVideoFile, transcodeOriginalVideofile, optimizeOriginalVideofile } from '../../video-transcoding' export type VideoFilePayload = { videoUUID: string @@ -32,7 +33,7 @@ async function processVideoFileImport (job: Bull.Job) { return undefined } - await video.importVideoFile(payload.filePath) + await importVideoFile(video, payload.filePath) await onVideoFileTranscoderOrImportSuccess(video) return video @@ -51,11 +52,11 @@ async function processVideoFile (job: Bull.Job) { // Transcoding in other resolution if (payload.resolution) { - await video.transcodeOriginalVideofile(payload.resolution, payload.isPortraitMode || false) + await transcodeOriginalVideofile(video, payload.resolution, payload.isPortraitMode || false) await retryTransactionWrapper(onVideoFileTranscoderOrImportSuccess, video) } else { - await video.optimizeOriginalVideofile() + await optimizeOriginalVideofile(video) await retryTransactionWrapper(onVideoFileOptimizerSuccess, video, payload.isNewVideo) } diff --git a/server/lib/video-transcoding.ts b/server/lib/video-transcoding.ts new file mode 100644 index 000000000..bf3ff78c2 --- /dev/null +++ b/server/lib/video-transcoding.ts @@ -0,0 +1,130 @@ +import { CONFIG } from '../initializers' +import { join, extname } from 'path' +import { getVideoFileFPS, getVideoFileResolution, transcode } from '../helpers/ffmpeg-utils' +import { copy, remove, rename, stat } from 'fs-extra' +import { logger } from '../helpers/logger' +import { VideoResolution } from '../../shared/models/videos' +import { VideoFileModel } from '../models/video/video-file' +import { VideoModel } from '../models/video/video' + +async function optimizeOriginalVideofile (video: VideoModel) { + const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR + const newExtname = '.mp4' + const inputVideoFile = video.getOriginalFile() + const videoInputPath = join(videosDirectory, video.getVideoFilename(inputVideoFile)) + const videoTranscodedPath = join(videosDirectory, video.id + '-transcoded' + newExtname) + + const transcodeOptions = { + inputPath: videoInputPath, + outputPath: videoTranscodedPath + } + + // Could be very long! + await transcode(transcodeOptions) + + try { + await remove(videoInputPath) + + // Important to do this before getVideoFilename() to take in account the new file extension + inputVideoFile.set('extname', newExtname) + + const videoOutputPath = video.getVideoFilePath(inputVideoFile) + await rename(videoTranscodedPath, videoOutputPath) + const stats = await stat(videoOutputPath) + const fps = await getVideoFileFPS(videoOutputPath) + + inputVideoFile.set('size', stats.size) + inputVideoFile.set('fps', fps) + + await video.createTorrentAndSetInfoHash(inputVideoFile) + await inputVideoFile.save() + } catch (err) { + // Auto destruction... + video.destroy().catch(err => logger.error('Cannot destruct video after transcoding failure.', { err })) + + throw err + } +} + +async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoResolution, isPortraitMode: boolean) { + const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR + const extname = '.mp4' + + // We are sure it's x264 in mp4 because optimizeOriginalVideofile was already executed + const videoInputPath = join(videosDirectory, video.getVideoFilename(video.getOriginalFile())) + + const newVideoFile = new VideoFileModel({ + resolution, + extname, + size: 0, + videoId: video.id + }) + const videoOutputPath = join(videosDirectory, video.getVideoFilename(newVideoFile)) + + const transcodeOptions = { + inputPath: videoInputPath, + outputPath: videoOutputPath, + resolution, + isPortraitMode + } + + await transcode(transcodeOptions) + + const stats = await stat(videoOutputPath) + const fps = await getVideoFileFPS(videoOutputPath) + + newVideoFile.set('size', stats.size) + newVideoFile.set('fps', fps) + + await video.createTorrentAndSetInfoHash(newVideoFile) + + await newVideoFile.save() + + video.VideoFiles.push(newVideoFile) +} + +async function importVideoFile (video: VideoModel, inputFilePath: string) { + const { videoFileResolution } = await getVideoFileResolution(inputFilePath) + const { size } = await stat(inputFilePath) + const fps = await getVideoFileFPS(inputFilePath) + + let updatedVideoFile = new VideoFileModel({ + resolution: videoFileResolution, + extname: extname(inputFilePath), + size, + fps, + videoId: video.id + }) + + const currentVideoFile = video.VideoFiles.find(videoFile => videoFile.resolution === updatedVideoFile.resolution) + + if (currentVideoFile) { + // Remove old file and old torrent + await video.removeFile(currentVideoFile) + await video.removeTorrent(currentVideoFile) + // Remove the old video file from the array + video.VideoFiles = video.VideoFiles.filter(f => f !== currentVideoFile) + + // Update the database + currentVideoFile.set('extname', updatedVideoFile.extname) + currentVideoFile.set('size', updatedVideoFile.size) + currentVideoFile.set('fps', updatedVideoFile.fps) + + updatedVideoFile = currentVideoFile + } + + const outputPath = video.getVideoFilePath(updatedVideoFile) + await copy(inputFilePath, outputPath) + + await video.createTorrentAndSetInfoHash(updatedVideoFile) + + await updatedVideoFile.save() + + video.VideoFiles.push(updatedVideoFile) +} + +export { + optimizeOriginalVideofile, + transcodeOriginalVideofile, + importVideoFile +} diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts index 6ae02efb9..fb07287a8 100644 --- a/server/models/redundancy/video-redundancy.ts +++ b/server/models/redundancy/video-redundancy.ts @@ -193,7 +193,7 @@ export class VideoRedundancyModel extends Model { // On VideoModel! const query = { attributes: [ 'id', 'publishedAt' ], - // logging: !isTestInstance(), + logging: !isTestInstance(), limit: randomizedFactor, order: getVideoSort('-publishedAt'), where: { diff --git a/server/models/video/video-format-utils.ts b/server/models/video/video-format-utils.ts new file mode 100644 index 000000000..fae38507b --- /dev/null +++ b/server/models/video/video-format-utils.ts @@ -0,0 +1,295 @@ +import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos' +import { VideoModel } from './video' +import { VideoFileModel } from './video-file' +import { ActivityUrlObject, VideoTorrentObject } from '../../../shared/models/activitypub/objects' +import { CONFIG, THUMBNAILS_SIZE, VIDEO_EXT_MIMETYPE } from '../../initializers' +import { VideoCaptionModel } from './video-caption' +import { + getVideoCommentsActivityPubUrl, + getVideoDislikesActivityPubUrl, + getVideoLikesActivityPubUrl, + getVideoSharesActivityPubUrl +} from '../../lib/activitypub' + +export type VideoFormattingJSONOptions = { + additionalAttributes: { + state?: boolean, + waitTranscoding?: boolean, + scheduledUpdate?: boolean, + blacklistInfo?: boolean + } +} +function videoModelToFormattedJSON (video: VideoModel, options?: VideoFormattingJSONOptions): Video { + const formattedAccount = video.VideoChannel.Account.toFormattedJSON() + const formattedVideoChannel = video.VideoChannel.toFormattedJSON() + + const videoObject: Video = { + id: video.id, + uuid: video.uuid, + name: video.name, + category: { + id: video.category, + label: VideoModel.getCategoryLabel(video.category) + }, + licence: { + id: video.licence, + label: VideoModel.getLicenceLabel(video.licence) + }, + language: { + id: video.language, + label: VideoModel.getLanguageLabel(video.language) + }, + privacy: { + id: video.privacy, + label: VideoModel.getPrivacyLabel(video.privacy) + }, + nsfw: video.nsfw, + description: video.getTruncatedDescription(), + isLocal: video.isOwned(), + duration: video.duration, + views: video.views, + likes: video.likes, + dislikes: video.dislikes, + thumbnailPath: video.getThumbnailStaticPath(), + previewPath: video.getPreviewStaticPath(), + embedPath: video.getEmbedStaticPath(), + createdAt: video.createdAt, + updatedAt: video.updatedAt, + publishedAt: video.publishedAt, + account: { + id: formattedAccount.id, + uuid: formattedAccount.uuid, + name: formattedAccount.name, + displayName: formattedAccount.displayName, + url: formattedAccount.url, + host: formattedAccount.host, + avatar: formattedAccount.avatar + }, + channel: { + id: formattedVideoChannel.id, + uuid: formattedVideoChannel.uuid, + name: formattedVideoChannel.name, + displayName: formattedVideoChannel.displayName, + url: formattedVideoChannel.url, + host: formattedVideoChannel.host, + avatar: formattedVideoChannel.avatar + } + } + + if (options) { + if (options.additionalAttributes.state === true) { + videoObject.state = { + id: video.state, + label: VideoModel.getStateLabel(video.state) + } + } + + if (options.additionalAttributes.waitTranscoding === true) { + videoObject.waitTranscoding = video.waitTranscoding + } + + if (options.additionalAttributes.scheduledUpdate === true && video.ScheduleVideoUpdate) { + videoObject.scheduledUpdate = { + updateAt: video.ScheduleVideoUpdate.updateAt, + privacy: video.ScheduleVideoUpdate.privacy || undefined + } + } + + if (options.additionalAttributes.blacklistInfo === true) { + videoObject.blacklisted = !!video.VideoBlacklist + videoObject.blacklistedReason = video.VideoBlacklist ? video.VideoBlacklist.reason : null + } + } + + return videoObject +} + +function videoModelToFormattedDetailsJSON (video: VideoModel): VideoDetails { + const formattedJson = video.toFormattedJSON({ + additionalAttributes: { + scheduledUpdate: true, + blacklistInfo: true + } + }) + + const detailsJson = { + support: video.support, + descriptionPath: video.getDescriptionPath(), + channel: video.VideoChannel.toFormattedJSON(), + account: video.VideoChannel.Account.toFormattedJSON(), + tags: video.Tags.map(t => t.name), + commentsEnabled: video.commentsEnabled, + waitTranscoding: video.waitTranscoding, + state: { + id: video.state, + label: VideoModel.getStateLabel(video.state) + }, + files: [] + } + + // Format and sort video files + detailsJson.files = videoFilesModelToFormattedJSON(video, video.VideoFiles) + + return Object.assign(formattedJson, detailsJson) +} + +function videoFilesModelToFormattedJSON (video: VideoModel, videoFiles: VideoFileModel[]): VideoFile[] { + const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() + + return videoFiles + .map(videoFile => { + let resolutionLabel = videoFile.resolution + 'p' + + return { + resolution: { + id: videoFile.resolution, + label: resolutionLabel + }, + magnetUri: video.generateMagnetUri(videoFile, baseUrlHttp, baseUrlWs), + size: videoFile.size, + fps: videoFile.fps, + torrentUrl: video.getTorrentUrl(videoFile, baseUrlHttp), + torrentDownloadUrl: video.getTorrentDownloadUrl(videoFile, baseUrlHttp), + fileUrl: video.getVideoFileUrl(videoFile, baseUrlHttp), + fileDownloadUrl: video.getVideoFileDownloadUrl(videoFile, baseUrlHttp) + } as VideoFile + }) + .sort((a, b) => { + if (a.resolution.id < b.resolution.id) return 1 + if (a.resolution.id === b.resolution.id) return 0 + return -1 + }) +} + +function videoModelToActivityPubObject (video: VideoModel): VideoTorrentObject { + const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() + if (!video.Tags) video.Tags = [] + + const tag = video.Tags.map(t => ({ + type: 'Hashtag' as 'Hashtag', + name: t.name + })) + + let language + if (video.language) { + language = { + identifier: video.language, + name: VideoModel.getLanguageLabel(video.language) + } + } + + let category + if (video.category) { + category = { + identifier: video.category + '', + name: VideoModel.getCategoryLabel(video.category) + } + } + + let licence + if (video.licence) { + licence = { + identifier: video.licence + '', + name: VideoModel.getLicenceLabel(video.licence) + } + } + + const url: ActivityUrlObject[] = [] + for (const file of video.VideoFiles) { + url.push({ + type: 'Link', + mimeType: VIDEO_EXT_MIMETYPE[ file.extname ] as any, + href: video.getVideoFileUrl(file, baseUrlHttp), + height: file.resolution, + size: file.size, + fps: file.fps + }) + + url.push({ + type: 'Link', + mimeType: 'application/x-bittorrent' as 'application/x-bittorrent', + href: video.getTorrentUrl(file, baseUrlHttp), + height: file.resolution + }) + + url.push({ + type: 'Link', + mimeType: 'application/x-bittorrent;x-scheme-handler/magnet' as 'application/x-bittorrent;x-scheme-handler/magnet', + href: video.generateMagnetUri(file, baseUrlHttp, baseUrlWs), + height: file.resolution + }) + } + + // Add video url too + url.push({ + type: 'Link', + mimeType: 'text/html', + href: CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid + }) + + const subtitleLanguage = [] + for (const caption of video.VideoCaptions) { + subtitleLanguage.push({ + identifier: caption.language, + name: VideoCaptionModel.getLanguageLabel(caption.language) + }) + } + + return { + type: 'Video' as 'Video', + id: video.url, + name: video.name, + duration: getActivityStreamDuration(video.duration), + uuid: video.uuid, + tag, + category, + licence, + language, + views: video.views, + sensitive: video.nsfw, + waitTranscoding: video.waitTranscoding, + state: video.state, + commentsEnabled: video.commentsEnabled, + published: video.publishedAt.toISOString(), + updated: video.updatedAt.toISOString(), + mediaType: 'text/markdown', + content: video.getTruncatedDescription(), + support: video.support, + subtitleLanguage, + icon: { + type: 'Image', + url: video.getThumbnailUrl(baseUrlHttp), + mediaType: 'image/jpeg', + width: THUMBNAILS_SIZE.width, + height: THUMBNAILS_SIZE.height + }, + url, + likes: getVideoLikesActivityPubUrl(video), + dislikes: getVideoDislikesActivityPubUrl(video), + shares: getVideoSharesActivityPubUrl(video), + comments: getVideoCommentsActivityPubUrl(video), + attributedTo: [ + { + type: 'Person', + id: video.VideoChannel.Account.Actor.url + }, + { + type: 'Group', + id: video.VideoChannel.Actor.url + } + ] + } +} + +function getActivityStreamDuration (duration: number) { + // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-duration + return 'PT' + duration + 'S' +} + +export { + videoModelToFormattedJSON, + videoModelToFormattedDetailsJSON, + videoFilesModelToFormattedJSON, + videoModelToActivityPubObject, + getActivityStreamDuration +} diff --git a/server/models/video/video.ts b/server/models/video/video.ts index b7d3f184f..ce856aed2 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -1,8 +1,8 @@ import * as Bluebird from 'bluebird' -import { map, maxBy } from 'lodash' +import { maxBy } from 'lodash' import * as magnetUtil from 'magnet-uri' import * as parseTorrent from 'parse-torrent' -import { extname, join } from 'path' +import { join } from 'path' import * as Sequelize from 'sequelize' import { AllowNull, @@ -27,7 +27,7 @@ import { Table, UpdatedAt } from 'sequelize-typescript' -import { ActivityUrlObject, VideoPrivacy, VideoResolution, VideoState } from '../../../shared' +import { VideoPrivacy, VideoState } from '../../../shared' import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos' import { VideoFilter } from '../../../shared/models/videos/video-query.type' @@ -45,7 +45,7 @@ import { isVideoStateValid, isVideoSupportValid } from '../../helpers/custom-validators/videos' -import { generateImageFromVideoFile, getVideoFileFPS, getVideoFileResolution, transcode } from '../../helpers/ffmpeg-utils' +import { generateImageFromVideoFile, getVideoFileResolution } from '../../helpers/ffmpeg-utils' import { logger } from '../../helpers/logger' import { getServerActor } from '../../helpers/utils' import { @@ -59,18 +59,11 @@ import { STATIC_PATHS, THUMBNAILS_SIZE, VIDEO_CATEGORIES, - VIDEO_EXT_MIMETYPE, VIDEO_LANGUAGES, VIDEO_LICENCES, VIDEO_PRIVACIES, VIDEO_STATES } from '../../initializers' -import { - getVideoCommentsActivityPubUrl, - getVideoDislikesActivityPubUrl, - getVideoLikesActivityPubUrl, - getVideoSharesActivityPubUrl -} from '../../lib/activitypub' import { sendDeleteVideo } from '../../lib/activitypub/send' import { AccountModel } from '../account/account' import { AccountVideoRateModel } from '../account/account-video-rate' @@ -88,9 +81,16 @@ import { VideoTagModel } from './video-tag' import { ScheduleVideoUpdateModel } from './schedule-video-update' import { VideoCaptionModel } from './video-caption' import { VideoBlacklistModel } from './video-blacklist' -import { copy, remove, rename, stat, writeFile } from 'fs-extra' +import { remove, writeFile } from 'fs-extra' import { VideoViewModel } from './video-views' import { VideoRedundancyModel } from '../redundancy/video-redundancy' +import { + videoFilesModelToFormattedJSON, + VideoFormattingJSONOptions, + videoModelToActivityPubObject, + videoModelToFormattedDetailsJSON, + videoModelToFormattedJSON +} from './video-format-utils' // FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation const indexes: Sequelize.DefineIndexesOptions[] = [ @@ -1257,23 +1257,23 @@ export class VideoModel extends Model { } } - private static getCategoryLabel (id: number) { + static getCategoryLabel (id: number) { return VIDEO_CATEGORIES[ id ] || 'Misc' } - private static getLicenceLabel (id: number) { + static getLicenceLabel (id: number) { return VIDEO_LICENCES[ id ] || 'Unknown' } - private static getLanguageLabel (id: string) { + static getLanguageLabel (id: string) { return VIDEO_LANGUAGES[ id ] || 'Unknown' } - private static getPrivacyLabel (id: number) { + static getPrivacyLabel (id: number) { return VIDEO_PRIVACIES[ id ] || 'Unknown' } - private static getStateLabel (id: number) { + static getStateLabel (id: number) { return VIDEO_STATES[ id ] || 'Unknown' } @@ -1369,273 +1369,20 @@ export class VideoModel extends Model { return join(STATIC_PATHS.PREVIEWS, this.getPreviewName()) } - toFormattedJSON (options?: { - additionalAttributes: { - state?: boolean, - waitTranscoding?: boolean, - scheduledUpdate?: boolean, - blacklistInfo?: boolean - } - }): Video { - const formattedAccount = this.VideoChannel.Account.toFormattedJSON() - const formattedVideoChannel = this.VideoChannel.toFormattedJSON() - - const videoObject: Video = { - id: this.id, - uuid: this.uuid, - name: this.name, - category: { - id: this.category, - label: VideoModel.getCategoryLabel(this.category) - }, - licence: { - id: this.licence, - label: VideoModel.getLicenceLabel(this.licence) - }, - language: { - id: this.language, - label: VideoModel.getLanguageLabel(this.language) - }, - privacy: { - id: this.privacy, - label: VideoModel.getPrivacyLabel(this.privacy) - }, - nsfw: this.nsfw, - description: this.getTruncatedDescription(), - isLocal: this.isOwned(), - duration: this.duration, - views: this.views, - likes: this.likes, - dislikes: this.dislikes, - thumbnailPath: this.getThumbnailStaticPath(), - previewPath: this.getPreviewStaticPath(), - embedPath: this.getEmbedStaticPath(), - createdAt: this.createdAt, - updatedAt: this.updatedAt, - publishedAt: this.publishedAt, - account: { - id: formattedAccount.id, - uuid: formattedAccount.uuid, - name: formattedAccount.name, - displayName: formattedAccount.displayName, - url: formattedAccount.url, - host: formattedAccount.host, - avatar: formattedAccount.avatar - }, - channel: { - id: formattedVideoChannel.id, - uuid: formattedVideoChannel.uuid, - name: formattedVideoChannel.name, - displayName: formattedVideoChannel.displayName, - url: formattedVideoChannel.url, - host: formattedVideoChannel.host, - avatar: formattedVideoChannel.avatar - } - } - - if (options) { - if (options.additionalAttributes.state === true) { - videoObject.state = { - id: this.state, - label: VideoModel.getStateLabel(this.state) - } - } - - if (options.additionalAttributes.waitTranscoding === true) { - videoObject.waitTranscoding = this.waitTranscoding - } - - if (options.additionalAttributes.scheduledUpdate === true && this.ScheduleVideoUpdate) { - videoObject.scheduledUpdate = { - updateAt: this.ScheduleVideoUpdate.updateAt, - privacy: this.ScheduleVideoUpdate.privacy || undefined - } - } - - if (options.additionalAttributes.blacklistInfo === true) { - videoObject.blacklisted = !!this.VideoBlacklist - videoObject.blacklistedReason = this.VideoBlacklist ? this.VideoBlacklist.reason : null - } - } - - return videoObject + toFormattedJSON (options?: VideoFormattingJSONOptions): Video { + return videoModelToFormattedJSON(this, options) } toFormattedDetailsJSON (): VideoDetails { - const formattedJson = this.toFormattedJSON({ - additionalAttributes: { - scheduledUpdate: true, - blacklistInfo: true - } - }) - - const detailsJson = { - support: this.support, - descriptionPath: this.getDescriptionPath(), - channel: this.VideoChannel.toFormattedJSON(), - account: this.VideoChannel.Account.toFormattedJSON(), - tags: map(this.Tags, 'name'), - commentsEnabled: this.commentsEnabled, - waitTranscoding: this.waitTranscoding, - state: { - id: this.state, - label: VideoModel.getStateLabel(this.state) - }, - files: [] - } - - // Format and sort video files - detailsJson.files = this.getFormattedVideoFilesJSON() - - return Object.assign(formattedJson, detailsJson) + return videoModelToFormattedDetailsJSON(this) } getFormattedVideoFilesJSON (): VideoFile[] { - const { baseUrlHttp, baseUrlWs } = this.getBaseUrls() - - return this.VideoFiles - .map(videoFile => { - let resolutionLabel = videoFile.resolution + 'p' - - return { - resolution: { - id: videoFile.resolution, - label: resolutionLabel - }, - magnetUri: this.generateMagnetUri(videoFile, baseUrlHttp, baseUrlWs), - size: videoFile.size, - fps: videoFile.fps, - torrentUrl: this.getTorrentUrl(videoFile, baseUrlHttp), - torrentDownloadUrl: this.getTorrentDownloadUrl(videoFile, baseUrlHttp), - fileUrl: this.getVideoFileUrl(videoFile, baseUrlHttp), - fileDownloadUrl: this.getVideoFileDownloadUrl(videoFile, baseUrlHttp) - } as VideoFile - }) - .sort((a, b) => { - if (a.resolution.id < b.resolution.id) return 1 - if (a.resolution.id === b.resolution.id) return 0 - return -1 - }) + return videoFilesModelToFormattedJSON(this, this.VideoFiles) } toActivityPubObject (): VideoTorrentObject { - const { baseUrlHttp, baseUrlWs } = this.getBaseUrls() - if (!this.Tags) this.Tags = [] - - const tag = this.Tags.map(t => ({ - type: 'Hashtag' as 'Hashtag', - name: t.name - })) - - let language - if (this.language) { - language = { - identifier: this.language, - name: VideoModel.getLanguageLabel(this.language) - } - } - - let category - if (this.category) { - category = { - identifier: this.category + '', - name: VideoModel.getCategoryLabel(this.category) - } - } - - let licence - if (this.licence) { - licence = { - identifier: this.licence + '', - name: VideoModel.getLicenceLabel(this.licence) - } - } - - const url: ActivityUrlObject[] = [] - for (const file of this.VideoFiles) { - url.push({ - type: 'Link', - mimeType: VIDEO_EXT_MIMETYPE[ file.extname ] as any, - href: this.getVideoFileUrl(file, baseUrlHttp), - height: file.resolution, - size: file.size, - fps: file.fps - }) - - url.push({ - type: 'Link', - mimeType: 'application/x-bittorrent' as 'application/x-bittorrent', - href: this.getTorrentUrl(file, baseUrlHttp), - height: file.resolution - }) - - url.push({ - type: 'Link', - mimeType: 'application/x-bittorrent;x-scheme-handler/magnet' as 'application/x-bittorrent;x-scheme-handler/magnet', - href: this.generateMagnetUri(file, baseUrlHttp, baseUrlWs), - height: file.resolution - }) - } - - // Add video url too - url.push({ - type: 'Link', - mimeType: 'text/html', - href: CONFIG.WEBSERVER.URL + '/videos/watch/' + this.uuid - }) - - const subtitleLanguage = [] - for (const caption of this.VideoCaptions) { - subtitleLanguage.push({ - identifier: caption.language, - name: VideoCaptionModel.getLanguageLabel(caption.language) - }) - } - - return { - type: 'Video' as 'Video', - id: this.url, - name: this.name, - duration: this.getActivityStreamDuration(), - uuid: this.uuid, - tag, - category, - licence, - language, - views: this.views, - sensitive: this.nsfw, - waitTranscoding: this.waitTranscoding, - state: this.state, - commentsEnabled: this.commentsEnabled, - published: this.publishedAt.toISOString(), - updated: this.updatedAt.toISOString(), - mediaType: 'text/markdown', - content: this.getTruncatedDescription(), - support: this.support, - subtitleLanguage, - icon: { - type: 'Image', - url: this.getThumbnailUrl(baseUrlHttp), - mediaType: 'image/jpeg', - width: THUMBNAILS_SIZE.width, - height: THUMBNAILS_SIZE.height - }, - url, - likes: getVideoLikesActivityPubUrl(this), - dislikes: getVideoDislikesActivityPubUrl(this), - shares: getVideoSharesActivityPubUrl(this), - comments: getVideoCommentsActivityPubUrl(this), - attributedTo: [ - { - type: 'Person', - id: this.VideoChannel.Account.Actor.url - }, - { - type: 'Group', - id: this.VideoChannel.Actor.url - } - ] - } + return videoModelToActivityPubObject(this) } getTruncatedDescription () { @@ -1645,123 +1392,6 @@ export class VideoModel extends Model { return peertubeTruncate(this.description, maxLength) } - async optimizeOriginalVideofile () { - const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR - const newExtname = '.mp4' - const inputVideoFile = this.getOriginalFile() - const videoInputPath = join(videosDirectory, this.getVideoFilename(inputVideoFile)) - const videoTranscodedPath = join(videosDirectory, this.id + '-transcoded' + newExtname) - - const transcodeOptions = { - inputPath: videoInputPath, - outputPath: videoTranscodedPath - } - - // Could be very long! - await transcode(transcodeOptions) - - try { - await remove(videoInputPath) - - // Important to do this before getVideoFilename() to take in account the new file extension - inputVideoFile.set('extname', newExtname) - - const videoOutputPath = this.getVideoFilePath(inputVideoFile) - await rename(videoTranscodedPath, videoOutputPath) - const stats = await stat(videoOutputPath) - const fps = await getVideoFileFPS(videoOutputPath) - - inputVideoFile.set('size', stats.size) - inputVideoFile.set('fps', fps) - - await this.createTorrentAndSetInfoHash(inputVideoFile) - await inputVideoFile.save() - - } catch (err) { - // Auto destruction... - this.destroy().catch(err => logger.error('Cannot destruct video after transcoding failure.', { err })) - - throw err - } - } - - async transcodeOriginalVideofile (resolution: VideoResolution, isPortraitMode: boolean) { - const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR - const extname = '.mp4' - - // We are sure it's x264 in mp4 because optimizeOriginalVideofile was already executed - const videoInputPath = join(videosDirectory, this.getVideoFilename(this.getOriginalFile())) - - const newVideoFile = new VideoFileModel({ - resolution, - extname, - size: 0, - videoId: this.id - }) - const videoOutputPath = join(videosDirectory, this.getVideoFilename(newVideoFile)) - - const transcodeOptions = { - inputPath: videoInputPath, - outputPath: videoOutputPath, - resolution, - isPortraitMode - } - - await transcode(transcodeOptions) - - const stats = await stat(videoOutputPath) - const fps = await getVideoFileFPS(videoOutputPath) - - newVideoFile.set('size', stats.size) - newVideoFile.set('fps', fps) - - await this.createTorrentAndSetInfoHash(newVideoFile) - - await newVideoFile.save() - - this.VideoFiles.push(newVideoFile) - } - - async importVideoFile (inputFilePath: string) { - const { videoFileResolution } = await getVideoFileResolution(inputFilePath) - const { size } = await stat(inputFilePath) - const fps = await getVideoFileFPS(inputFilePath) - - let updatedVideoFile = new VideoFileModel({ - resolution: videoFileResolution, - extname: extname(inputFilePath), - size, - fps, - videoId: this.id - }) - - const currentVideoFile = this.VideoFiles.find(videoFile => videoFile.resolution === updatedVideoFile.resolution) - - if (currentVideoFile) { - // Remove old file and old torrent - await this.removeFile(currentVideoFile) - await this.removeTorrent(currentVideoFile) - // Remove the old video file from the array - this.VideoFiles = this.VideoFiles.filter(f => f !== currentVideoFile) - - // Update the database - currentVideoFile.set('extname', updatedVideoFile.extname) - currentVideoFile.set('size', updatedVideoFile.size) - currentVideoFile.set('fps', updatedVideoFile.fps) - - updatedVideoFile = currentVideoFile - } - - const outputPath = this.getVideoFilePath(updatedVideoFile) - await copy(inputFilePath, outputPath) - - await this.createTorrentAndSetInfoHash(updatedVideoFile) - - await updatedVideoFile.save() - - this.VideoFiles.push(updatedVideoFile) - } - getOriginalFileResolution () { const originalFilePath = this.getVideoFilePath(this.getOriginalFile()) @@ -1796,11 +1426,6 @@ export class VideoModel extends Model { .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err })) } - getActivityStreamDuration () { - // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-duration - return 'PT' + this.duration + 'S' - } - isOutdated () { if (this.isOwned()) return false From e972e046dbe9b499944c4fab9220eee13e31ac1b Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 18 Sep 2018 11:59:05 +0200 Subject: [PATCH 25/44] Don't get recommended videos twice --- .../videos/+video-watch/video-watch.component.ts | 2 +- .../recent-videos-recommendation.service.ts | 4 ++-- .../recommendations/recommended-videos.store.ts | 14 +++++++++----- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/client/src/app/videos/+video-watch/video-watch.component.ts b/client/src/app/videos/+video-watch/video-watch.component.ts index 834428fa4..7a61e355a 100644 --- a/client/src/app/videos/+video-watch/video-watch.component.ts +++ b/client/src/app/videos/+video-watch/video-watch.component.ts @@ -1,4 +1,4 @@ -import { catchError, subscribeOn } from 'rxjs/operators' +import { catchError } from 'rxjs/operators' import { ChangeDetectorRef, Component, ElementRef, Inject, LOCALE_ID, NgZone, OnDestroy, OnInit, ViewChild } from '@angular/core' import { ActivatedRoute, Router } from '@angular/router' import { RedirectService } from '@app/core/routing/redirect.service' diff --git a/client/src/app/videos/recommendations/recent-videos-recommendation.service.ts b/client/src/app/videos/recommendations/recent-videos-recommendation.service.ts index 4723f7fd0..0ee34b9cb 100644 --- a/client/src/app/videos/recommendations/recent-videos-recommendation.service.ts +++ b/client/src/app/videos/recommendations/recent-videos-recommendation.service.ts @@ -25,8 +25,8 @@ export class RecentVideosRecommendationService implements RecommendationService getRecommendations (recommendation: RecommendationInfo): Observable { return this.fetchPage(1, recommendation) .pipe( - map(vids => { - const otherVideos = vids.filter(v => v.uuid !== recommendation.uuid) + map(videos => { + const otherVideos = videos.filter(v => v.uuid !== recommendation.uuid) return otherVideos.slice(0, this.pageSize) }) ) diff --git a/client/src/app/videos/recommendations/recommended-videos.store.ts b/client/src/app/videos/recommendations/recommended-videos.store.ts index eb5c9867f..858ec3a27 100644 --- a/client/src/app/videos/recommendations/recommended-videos.store.ts +++ b/client/src/app/videos/recommendations/recommended-videos.store.ts @@ -3,8 +3,8 @@ import { Observable, ReplaySubject } from 'rxjs' import { Video } from '@app/shared/video/video.model' import { RecommendationInfo } from '@app/shared/video/recommendation-info.model' import { RecentVideosRecommendationService } from '@app/videos/recommendations/recent-videos-recommendation.service' -import { RecommendationService, UUID } from '@app/videos/recommendations/recommendations.service' -import { map, switchMap, take } from 'rxjs/operators' +import { RecommendationService } from '@app/videos/recommendations/recommendations.service' +import { map, shareReplay, switchMap, take } from 'rxjs/operators' /** * This store is intended to provide data for the RecommendedVideosComponent. @@ -19,9 +19,13 @@ export class RecommendedVideosStore { @Inject(RecentVideosRecommendationService) private recommendations: RecommendationService ) { this.recommendations$ = this.requestsForLoad$$.pipe( - switchMap(requestedRecommendation => recommendations.getRecommendations(requestedRecommendation) - .pipe(take(1)) - )) + switchMap(requestedRecommendation => { + return recommendations.getRecommendations(requestedRecommendation) + .pipe(take(1)) + }), + shareReplay() + ) + this.hasRecommendations$ = this.recommendations$.pipe( map(otherVideos => otherVideos.length > 0) ) From 627621c1e8d37c33f7b3dd59f4c8907b12c630bc Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 18 Sep 2018 12:00:49 +0200 Subject: [PATCH 26/44] Optimize SQL requests of watch page API endpoints --- scripts/create-import-video-file-job.ts | 2 +- scripts/create-transcoding-job.ts | 2 +- scripts/prune-storage.ts | 2 +- server/controllers/api/users/me.ts | 2 +- server/controllers/api/videos/comment.ts | 2 +- server/helpers/custom-validators/videos.ts | 14 ++-- server/lib/cache/videos-caption-cache.ts | 2 +- server/lib/cache/videos-preview-cache.ts | 4 +- server/lib/client-html.ts | 6 +- server/lib/job-queue/handlers/video-file.ts | 8 +- server/lib/job-queue/handlers/video-import.ts | 2 +- server/middlewares/validators/users.ts | 2 +- .../middlewares/validators/video-captions.ts | 2 +- .../middlewares/validators/video-comments.ts | 4 +- server/models/video/video.ts | 74 ++++++++++--------- 15 files changed, 68 insertions(+), 60 deletions(-) diff --git a/scripts/create-import-video-file-job.ts b/scripts/create-import-video-file-job.ts index 2b636014a..c8c6c6429 100644 --- a/scripts/create-import-video-file-job.ts +++ b/scripts/create-import-video-file-job.ts @@ -25,7 +25,7 @@ run() async function run () { await initDatabaseModels(true) - const video = await VideoModel.loadByUUID(program['video']) + const video = await VideoModel.loadByUUIDWithFile(program['video']) if (!video) throw new Error('Video not found.') if (video.isOwned() === false) throw new Error('Cannot import files of a non owned video.') diff --git a/scripts/create-transcoding-job.ts b/scripts/create-transcoding-job.ts index 3ea30f98e..7e5b687bb 100755 --- a/scripts/create-transcoding-job.ts +++ b/scripts/create-transcoding-job.ts @@ -28,7 +28,7 @@ run() async function run () { await initDatabaseModels(true) - const video = await VideoModel.loadByUUID(program['video']) + const video = await VideoModel.loadByUUIDWithFile(program['video']) if (!video) throw new Error('Video not found.') const dataInput = { diff --git a/scripts/prune-storage.ts b/scripts/prune-storage.ts index 572283868..b00f20934 100755 --- a/scripts/prune-storage.ts +++ b/scripts/prune-storage.ts @@ -56,7 +56,7 @@ async function pruneDirectory (directory: string) { const uuid = getUUIDFromFilename(file) let video: VideoModel - if (uuid) video = await VideoModel.loadByUUID(uuid) + if (uuid) video = await VideoModel.loadByUUIDWithFile(uuid) if (!uuid || !video) toDelete.push(join(directory, file)) } diff --git a/server/controllers/api/users/me.ts b/server/controllers/api/users/me.ts index e886d4b2a..113563c39 100644 --- a/server/controllers/api/users/me.ts +++ b/server/controllers/api/users/me.ts @@ -293,7 +293,7 @@ async function getUserVideoQuotaUsed (req: express.Request, res: express.Respons } async function getUserVideoRating (req: express.Request, res: express.Response, next: express.NextFunction) { - const videoId = +req.params.videoId + const videoId = res.locals.video.id const accountId = +res.locals.oauth.token.User.Account.id const ratingObj = await AccountVideoRateModel.load(accountId, videoId, null) diff --git a/server/controllers/api/videos/comment.ts b/server/controllers/api/videos/comment.ts index e35247829..8d0692b2b 100644 --- a/server/controllers/api/videos/comment.ts +++ b/server/controllers/api/videos/comment.ts @@ -86,7 +86,7 @@ async function listVideoThreadComments (req: express.Request, res: express.Respo let resultList: ResultList if (video.commentsEnabled === true) { - resultList = await VideoCommentModel.listThreadCommentsForApi(res.locals.video.id, res.locals.videoCommentThread.id) + resultList = await VideoCommentModel.listThreadCommentsForApi(video.id, res.locals.videoCommentThread.id) } else { resultList = { total: 0, diff --git a/server/helpers/custom-validators/videos.ts b/server/helpers/custom-validators/videos.ts index edafba6e2..dd207c787 100644 --- a/server/helpers/custom-validators/videos.ts +++ b/server/helpers/custom-validators/videos.ts @@ -152,13 +152,15 @@ function checkUserCanManageVideo (user: UserModel, video: VideoModel, right: Use return true } -async function isVideoExist (id: string, res: Response) { +async function isVideoExist (id: string, res: Response, fetchType: 'all' | 'only-video' | 'id' | 'none' = 'all') { let video: VideoModel | null - if (validator.isInt(id)) { - video = await VideoModel.loadAndPopulateAccountAndServerAndTags(+id) - } else { // UUID - video = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(id) + if (fetchType === 'all') { + video = await VideoModel.loadAndPopulateAccountAndServerAndTags(id) + } else if (fetchType === 'only-video') { + video = await VideoModel.load(id) + } else if (fetchType === 'id' || fetchType === 'none') { + video = await VideoModel.loadOnlyId(id) } if (video === null) { @@ -169,7 +171,7 @@ async function isVideoExist (id: string, res: Response) { return false } - res.locals.video = video + if (fetchType !== 'none') res.locals.video = video return true } diff --git a/server/lib/cache/videos-caption-cache.ts b/server/lib/cache/videos-caption-cache.ts index 380d42b2c..f240affbc 100644 --- a/server/lib/cache/videos-caption-cache.ts +++ b/server/lib/cache/videos-caption-cache.ts @@ -38,7 +38,7 @@ class VideosCaptionCache extends AbstractVideoStaticFileCache { if (videoCaption.isOwned()) throw new Error('Cannot load remote caption of owned video.') // Used to fetch the path - const video = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(videoId) + const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId) if (!video) return undefined const remoteStaticPath = videoCaption.getCaptionStaticPath() diff --git a/server/lib/cache/videos-preview-cache.ts b/server/lib/cache/videos-preview-cache.ts index 22b6d9cb0..a5d6f5b62 100644 --- a/server/lib/cache/videos-preview-cache.ts +++ b/server/lib/cache/videos-preview-cache.ts @@ -16,7 +16,7 @@ class VideosPreviewCache extends AbstractVideoStaticFileCache { } async getFilePath (videoUUID: string) { - const video = await VideoModel.loadByUUID(videoUUID) + const video = await VideoModel.loadByUUIDWithFile(videoUUID) if (!video) return undefined if (video.isOwned()) return join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName()) @@ -25,7 +25,7 @@ class VideosPreviewCache extends AbstractVideoStaticFileCache { } protected async loadRemoteFile (key: string) { - const video = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(key) + const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(key) if (!video) return undefined if (video.isOwned()) throw new Error('Cannot load remote preview of owned video.') diff --git a/server/lib/client-html.ts b/server/lib/client-html.ts index b1088c096..fc013e0c3 100644 --- a/server/lib/client-html.ts +++ b/server/lib/client-html.ts @@ -39,10 +39,8 @@ export class ClientHtml { let videoPromise: Bluebird // Let Angular application handle errors - if (validator.isUUID(videoId, 4)) { - videoPromise = VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(videoId) - } else if (validator.isInt(videoId)) { - videoPromise = VideoModel.loadAndPopulateAccountAndServerAndTags(+videoId) + if (validator.isInt(videoId) || validator.isUUID(videoId, 4)) { + videoPromise = VideoModel.loadAndPopulateAccountAndServerAndTags(videoId) } else { return ClientHtml.getIndexHTML(req, res) } diff --git a/server/lib/job-queue/handlers/video-file.ts b/server/lib/job-queue/handlers/video-file.ts index 2c9ca8e12..1463c93fc 100644 --- a/server/lib/job-queue/handlers/video-file.ts +++ b/server/lib/job-queue/handlers/video-file.ts @@ -26,7 +26,7 @@ async function processVideoFileImport (job: Bull.Job) { const payload = job.data as VideoFileImportPayload logger.info('Processing video file import in job %d.', job.id) - const video = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(payload.videoUUID) + const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(payload.videoUUID) // No video, maybe deleted? if (!video) { logger.info('Do not process job %d, video does not exist.', job.id) @@ -43,7 +43,7 @@ async function processVideoFile (job: Bull.Job) { const payload = job.data as VideoFilePayload logger.info('Processing video file in job %d.', job.id) - const video = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(payload.videoUUID) + const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(payload.videoUUID) // No video, maybe deleted? if (!video) { logger.info('Do not process job %d, video does not exist.', job.id) @@ -69,7 +69,7 @@ async function onVideoFileTranscoderOrImportSuccess (video: VideoModel) { return sequelizeTypescript.transaction(async t => { // Maybe the video changed in database, refresh it - let videoDatabase = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(video.uuid, t) + let videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) // Video does not exist anymore if (!videoDatabase) return undefined @@ -99,7 +99,7 @@ async function onVideoFileOptimizerSuccess (video: VideoModel, isNewVideo: boole return sequelizeTypescript.transaction(async t => { // Maybe the video changed in database, refresh it - const videoDatabase = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(video.uuid, t) + const videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) // Video does not exist anymore if (!videoDatabase) return undefined diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index ebcb2090c..9e14e57e6 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts @@ -183,7 +183,7 @@ async function processFile (downloader: () => Promise, videoImport: Vide const videoUpdated = await video.save({ transaction: t }) // Now we can federate the video (reload from database, we need more attributes) - const videoForFederation = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(video.uuid, t) + const videoForFederation = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) await federateVideoIfNeeded(videoForFederation, true, t) // Update video import object diff --git a/server/middlewares/validators/users.ts b/server/middlewares/validators/users.ts index d13c50c84..d3ba1ae23 100644 --- a/server/middlewares/validators/users.ts +++ b/server/middlewares/validators/users.ts @@ -172,7 +172,7 @@ const usersVideoRatingValidator = [ logger.debug('Checking usersVideoRating parameters', { parameters: req.params }) if (areValidationErrors(req, res)) return - if (!await isVideoExist(req.params.videoId, res)) return + if (!await isVideoExist(req.params.videoId, res, 'id')) return return next() } diff --git a/server/middlewares/validators/video-captions.ts b/server/middlewares/validators/video-captions.ts index 4f393ea84..51ffd7f3c 100644 --- a/server/middlewares/validators/video-captions.ts +++ b/server/middlewares/validators/video-captions.ts @@ -58,7 +58,7 @@ const listVideoCaptionsValidator = [ logger.debug('Checking listVideoCaptions parameters', { parameters: req.params }) if (areValidationErrors(req, res)) return - if (!await isVideoExist(req.params.videoId, res)) return + if (!await isVideoExist(req.params.videoId, res, 'id')) return return next() } diff --git a/server/middlewares/validators/video-comments.ts b/server/middlewares/validators/video-comments.ts index 227bc1fca..4b15eed23 100644 --- a/server/middlewares/validators/video-comments.ts +++ b/server/middlewares/validators/video-comments.ts @@ -17,7 +17,7 @@ const listVideoCommentThreadsValidator = [ logger.debug('Checking listVideoCommentThreads parameters.', { parameters: req.params }) if (areValidationErrors(req, res)) return - if (!await isVideoExist(req.params.videoId, res)) return + if (!await isVideoExist(req.params.videoId, res, 'only-video')) return return next() } @@ -31,7 +31,7 @@ const listVideoThreadCommentsValidator = [ logger.debug('Checking listVideoThreadComments parameters.', { parameters: req.params }) if (areValidationErrors(req, res)) return - if (!await isVideoExist(req.params.videoId, res)) return + if (!await isVideoExist(req.params.videoId, res, 'only-video')) return if (!await isVideoCommentThreadExist(req.params.threadId, res.locals.video, res)) return return next() diff --git a/server/models/video/video.ts b/server/models/video/video.ts index ce856aed2..c7cd2890c 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -91,6 +91,7 @@ import { videoModelToFormattedDetailsJSON, videoModelToFormattedJSON } from './video-format-utils' +import * as validator from 'validator' // FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation const indexes: Sequelize.DefineIndexesOptions[] = [ @@ -466,6 +467,7 @@ type AvailableForListIDsOptions = { required: false, include: [ { + attributes: [ 'fileUrl' ], model: () => VideoRedundancyModel.unscoped(), required: false } @@ -1062,8 +1064,26 @@ export class VideoModel extends Model { return VideoModel.getAvailableForApi(query, queryOptions) } - static load (id: number, t?: Sequelize.Transaction) { - return VideoModel.findById(id, { transaction: t }) + static load (id: number | string, t?: Sequelize.Transaction) { + const where = VideoModel.buildWhereIdOrUUID(id) + const options = { + where, + transaction: t + } + + return VideoModel.findOne(options) + } + + static loadOnlyId (id: number | string, t?: Sequelize.Transaction) { + const where = VideoModel.buildWhereIdOrUUID(id) + + const options = { + attributes: [ 'id' ], + where, + transaction: t + } + + return VideoModel.findOne(options) } static loadWithFile (id: number, t?: Sequelize.Transaction, logging?: boolean) { @@ -1071,6 +1091,18 @@ export class VideoModel extends Model { .findById(id, { transaction: t, logging }) } + static loadByUUIDWithFile (uuid: string) { + const options = { + where: { + uuid + } + } + + return VideoModel + .scope([ ScopeNames.WITH_FILES ]) + .findOne(options) + } + static loadByUrlAndPopulateAccount (url: string, t?: Sequelize.Transaction) { const query: IFindOptions = { where: { @@ -1083,40 +1115,12 @@ export class VideoModel extends Model { return VideoModel.scope([ ScopeNames.WITH_ACCOUNT_DETAILS, ScopeNames.WITH_FILES ]).findOne(query) } - static loadAndPopulateAccountAndServerAndTags (id: number) { - const options = { - order: [ [ 'Tags', 'name', 'ASC' ] ] - } + static loadAndPopulateAccountAndServerAndTags (id: number | string, t?: Sequelize.Transaction) { + const where = VideoModel.buildWhereIdOrUUID(id) - return VideoModel - .scope([ - ScopeNames.WITH_TAGS, - ScopeNames.WITH_BLACKLISTED, - ScopeNames.WITH_FILES, - ScopeNames.WITH_ACCOUNT_DETAILS, - ScopeNames.WITH_SCHEDULED_UPDATE - ]) - .findById(id, options) - } - - static loadByUUID (uuid: string) { - const options = { - where: { - uuid - } - } - - return VideoModel - .scope([ ScopeNames.WITH_FILES ]) - .findOne(options) - } - - static loadByUUIDAndPopulateAccountAndServerAndTags (uuid: string, t?: Sequelize.Transaction) { const options = { order: [ [ 'Tags', 'name', 'ASC' ] ], - where: { - uuid - }, + where, transaction: t } @@ -1277,6 +1281,10 @@ export class VideoModel extends Model { return VIDEO_STATES[ id ] || 'Unknown' } + static buildWhereIdOrUUID (id: number | string) { + return validator.isInt('' + id) ? { id } : { uuid: id } + } + getOriginalFile () { if (Array.isArray(this.VideoFiles) === false) return undefined From ad76628b17ff8f25d3402d6d669b274116bbf76c Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 19 Sep 2018 09:53:49 +0200 Subject: [PATCH 27/44] Fix admin access to moderators --- client/src/app/+admin/moderation/moderation.routes.ts | 10 ++++++++++ client/src/app/menu/menu.component.ts | 10 +++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/client/src/app/+admin/moderation/moderation.routes.ts b/client/src/app/+admin/moderation/moderation.routes.ts index b133152d9..6d81b9b36 100644 --- a/client/src/app/+admin/moderation/moderation.routes.ts +++ b/client/src/app/+admin/moderation/moderation.routes.ts @@ -15,6 +15,16 @@ export const ModerationRoutes: Routes = [ redirectTo: 'video-abuses/list', pathMatch: 'full' }, + { + path: 'video-abuses', + redirectTo: 'video-abuses/list', + pathMatch: 'full' + }, + { + path: 'video-blacklist', + redirectTo: 'video-blacklist/list', + pathMatch: 'full' + }, { path: 'video-abuses/list', component: VideoAbuseListComponent, diff --git a/client/src/app/menu/menu.component.ts b/client/src/app/menu/menu.component.ts index 24cd5aa28..f13ecc2c7 100644 --- a/client/src/app/menu/menu.component.ts +++ b/client/src/app/menu/menu.component.ts @@ -19,8 +19,10 @@ export class MenuComponent implements OnInit { private routesPerRight = { [UserRight.MANAGE_USERS]: '/admin/users', [UserRight.MANAGE_SERVER_FOLLOW]: '/admin/friends', - [UserRight.MANAGE_VIDEO_ABUSES]: '/admin/video-abuses', - [UserRight.MANAGE_VIDEO_BLACKLIST]: '/admin/video-blacklist' + [UserRight.MANAGE_VIDEO_ABUSES]: '/admin/moderation/video-abuses', + [UserRight.MANAGE_VIDEO_BLACKLIST]: '/admin/moderation/video-blacklist', + [UserRight.MANAGE_JOBS]: '/admin/jobs', + [UserRight.MANAGE_CONFIGURATION]: '/admin/config' } constructor ( @@ -67,7 +69,9 @@ export class MenuComponent implements OnInit { UserRight.MANAGE_USERS, UserRight.MANAGE_SERVER_FOLLOW, UserRight.MANAGE_VIDEO_ABUSES, - UserRight.MANAGE_VIDEO_BLACKLIST + UserRight.MANAGE_VIDEO_BLACKLIST, + UserRight.MANAGE_JOBS, + UserRight.MANAGE_CONFIGURATION ] for (const adminRight of adminRights) { From 96f29c0f6d2e623fb088e88200934c5df8da9924 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 19 Sep 2018 10:16:44 +0200 Subject: [PATCH 28/44] Optimize SQL requests of videos AP endpoints --- server/controllers/activitypub/client.ts | 16 +++-- server/helpers/custom-validators/videos.ts | 3 +- server/lib/activitypub/videos.ts | 2 +- .../middlewares/validators/video-comments.ts | 2 +- server/middlewares/validators/videos.ts | 70 ++++++++++--------- server/models/video/video-format-utils.ts | 5 +- server/models/video/video.ts | 2 +- 7 files changed, 56 insertions(+), 44 deletions(-) diff --git a/server/controllers/activitypub/client.ts b/server/controllers/activitypub/client.ts index 2e168ea78..6229c44aa 100644 --- a/server/controllers/activitypub/client.ts +++ b/server/controllers/activitypub/client.ts @@ -6,7 +6,13 @@ import { CONFIG, ROUTE_CACHE_LIFETIME } from '../../initializers' import { buildAnnounceWithVideoAudience } from '../../lib/activitypub/send' import { audiencify, getAudience } from '../../lib/activitypub/audience' import { buildCreateActivity } from '../../lib/activitypub/send/send-create' -import { asyncMiddleware, executeIfActivityPub, localAccountValidator, localVideoChannelValidator } from '../../middlewares' +import { + asyncMiddleware, + executeIfActivityPub, + localAccountValidator, + localVideoChannelValidator, + videosCustomGetValidator +} from '../../middlewares' import { videosGetValidator, videosShareValidator } from '../../middlewares/validators' import { videoCommentGetValidator } from '../../middlewares/validators/video-comments' import { AccountModel } from '../../models/account/account' @@ -54,7 +60,7 @@ activityPubClientRouter.get('/videos/watch/:id/activity', executeIfActivityPub(asyncMiddleware(videoController)) ) activityPubClientRouter.get('/videos/watch/:id/announces', - executeIfActivityPub(asyncMiddleware(videosGetValidator)), + executeIfActivityPub(asyncMiddleware(videosCustomGetValidator('only-video'))), executeIfActivityPub(asyncMiddleware(videoAnnouncesController)) ) activityPubClientRouter.get('/videos/watch/:id/announces/:accountId', @@ -62,15 +68,15 @@ activityPubClientRouter.get('/videos/watch/:id/announces/:accountId', executeIfActivityPub(asyncMiddleware(videoAnnounceController)) ) activityPubClientRouter.get('/videos/watch/:id/likes', - executeIfActivityPub(asyncMiddleware(videosGetValidator)), + executeIfActivityPub(asyncMiddleware(videosCustomGetValidator('only-video'))), executeIfActivityPub(asyncMiddleware(videoLikesController)) ) activityPubClientRouter.get('/videos/watch/:id/dislikes', - executeIfActivityPub(asyncMiddleware(videosGetValidator)), + executeIfActivityPub(asyncMiddleware(videosCustomGetValidator('only-video'))), executeIfActivityPub(asyncMiddleware(videoDislikesController)) ) activityPubClientRouter.get('/videos/watch/:id/comments', - executeIfActivityPub(asyncMiddleware(videosGetValidator)), + executeIfActivityPub(asyncMiddleware(videosCustomGetValidator('only-video'))), executeIfActivityPub(asyncMiddleware(videoCommentsController)) ) activityPubClientRouter.get('/videos/watch/:videoId/comments/:commentId', diff --git a/server/helpers/custom-validators/videos.ts b/server/helpers/custom-validators/videos.ts index dd207c787..c9ef8445d 100644 --- a/server/helpers/custom-validators/videos.ts +++ b/server/helpers/custom-validators/videos.ts @@ -152,7 +152,8 @@ function checkUserCanManageVideo (user: UserModel, video: VideoModel, right: Use return true } -async function isVideoExist (id: string, res: Response, fetchType: 'all' | 'only-video' | 'id' | 'none' = 'all') { +export type VideoFetchType = 'all' | 'only-video' | 'id' | 'none' +async function isVideoExist (id: string, res: Response, fetchType: VideoFetchType = 'all') { let video: VideoModel | null if (fetchType === 'all') { diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index 783f78d3e..5150c9975 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -61,7 +61,7 @@ function fetchRemoteVideoStaticFile (video: VideoModel, path: string, reject: Fu async function fetchRemoteVideoDescription (video: VideoModel) { const host = video.VideoChannel.Account.Actor.Server.host - const path = video.getDescriptionPath() + const path = video.getDescriptionAPIPath() const options = { uri: REMOTE_SCHEME.HTTP + '://' + host + path, json: true diff --git a/server/middlewares/validators/video-comments.ts b/server/middlewares/validators/video-comments.ts index 4b15eed23..693852499 100644 --- a/server/middlewares/validators/video-comments.ts +++ b/server/middlewares/validators/video-comments.ts @@ -78,7 +78,7 @@ const videoCommentGetValidator = [ logger.debug('Checking videoCommentGetValidator parameters.', { parameters: req.params }) if (areValidationErrors(req, res)) return - if (!await isVideoExist(req.params.videoId, res)) return + if (!await isVideoExist(req.params.videoId, res, 'id')) return if (!await isVideoCommentExist(req.params.commentId, res.locals.video, res)) return return next() diff --git a/server/middlewares/validators/videos.ts b/server/middlewares/validators/videos.ts index 9befbc9ee..8aa7b3a39 100644 --- a/server/middlewares/validators/videos.ts +++ b/server/middlewares/validators/videos.ts @@ -26,7 +26,8 @@ import { isVideoPrivacyValid, isVideoRatingTypeValid, isVideoSupportValid, - isVideoTagsValid + isVideoTagsValid, + VideoFetchType } from '../../helpers/custom-validators/videos' import { getDurationFromVideoFile } from '../../helpers/ffmpeg-utils' import { logger } from '../../helpers/logger' @@ -128,47 +129,49 @@ const videosUpdateValidator = getCommonVideoAttributes().concat([ } ]) -const videosGetValidator = [ - param('id').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid id'), +const videosCustomGetValidator = (fetchType: VideoFetchType) => { + return [ + param('id').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid id'), - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - logger.debug('Checking videosGet parameters', { parameters: req.params }) + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking videosGet parameters', { parameters: req.params }) - if (areValidationErrors(req, res)) return - if (!await isVideoExist(req.params.id, res)) return + if (areValidationErrors(req, res)) return + if (!await isVideoExist(req.params.id, res, fetchType)) return - const video: VideoModel = res.locals.video + const video: VideoModel = res.locals.video - // Video private or blacklisted - if (video.privacy === VideoPrivacy.PRIVATE || video.VideoBlacklist) { - return authenticate(req, res, () => { - const user: UserModel = res.locals.oauth.token.User + // Video private or blacklisted + if (video.privacy === VideoPrivacy.PRIVATE || video.VideoBlacklist) { + return authenticate(req, res, () => { + const user: UserModel = res.locals.oauth.token.User - // Only the owner or a user that have blacklist rights can see the video - if (video.VideoChannel.Account.userId !== user.id && !user.hasRight(UserRight.MANAGE_VIDEO_BLACKLIST)) { - return res.status(403) - .json({ error: 'Cannot get this private or blacklisted video.' }) - .end() - } + // Only the owner or a user that have blacklist rights can see the video + if (video.VideoChannel.Account.userId !== user.id && !user.hasRight(UserRight.MANAGE_VIDEO_BLACKLIST)) { + return res.status(403) + .json({ error: 'Cannot get this private or blacklisted video.' }) + .end() + } - return next() - }) + return next() + }) + } - return + // Video is public, anyone can access it + if (video.privacy === VideoPrivacy.PUBLIC) return next() + + // Video is unlisted, check we used the uuid to fetch it + if (video.privacy === VideoPrivacy.UNLISTED) { + if (isUUIDValid(req.params.id)) return next() + + // Don't leak this unlisted video + return res.status(404).end() + } } + ] +} - // Video is public, anyone can access it - if (video.privacy === VideoPrivacy.PUBLIC) return next() - - // Video is unlisted, check we used the uuid to fetch it - if (video.privacy === VideoPrivacy.UNLISTED) { - if (isUUIDValid(req.params.id)) return next() - - // Don't leak this unlisted video - return res.status(404).end() - } - } -] +const videosGetValidator = videosCustomGetValidator('all') const videosRemoveValidator = [ param('id').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid id'), @@ -366,6 +369,7 @@ export { videosAddValidator, videosUpdateValidator, videosGetValidator, + videosCustomGetValidator, videosRemoveValidator, videosShareValidator, diff --git a/server/models/video/video-format-utils.ts b/server/models/video/video-format-utils.ts index fae38507b..a9a58624d 100644 --- a/server/models/video/video-format-utils.ts +++ b/server/models/video/video-format-utils.ts @@ -112,12 +112,13 @@ function videoModelToFormattedDetailsJSON (video: VideoModel): VideoDetails { } }) + const tags = video.Tags ? video.Tags.map(t => t.name) : [] const detailsJson = { support: video.support, - descriptionPath: video.getDescriptionPath(), + descriptionPath: video.getDescriptionAPIPath(), channel: video.VideoChannel.toFormattedJSON(), account: video.VideoChannel.Account.toFormattedJSON(), - tags: video.Tags.map(t => t.name), + tags, commentsEnabled: video.commentsEnabled, waitTranscoding: video.waitTranscoding, state: { diff --git a/server/models/video/video.ts b/server/models/video/video.ts index c7cd2890c..ce2153f87 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -1406,7 +1406,7 @@ export class VideoModel extends Model { return getVideoFileResolution(originalFilePath) } - getDescriptionPath () { + getDescriptionAPIPath () { return `/api/${API_VERSION}/videos/${this.uuid}/description` } From 4157cdb13748cb6e8ce7081d062a8778554cc5a7 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 19 Sep 2018 11:16:23 +0200 Subject: [PATCH 29/44] Refractor videos AP functions --- server/controllers/api/search.ts | 2 +- server/helpers/custom-validators/videos.ts | 12 +- server/helpers/video.ts | 25 + .../activitypub/process/process-announce.ts | 2 +- .../lib/activitypub/process/process-create.ts | 10 +- .../lib/activitypub/process/process-like.ts | 2 +- .../lib/activitypub/process/process-undo.ts | 6 +- .../lib/activitypub/process/process-update.ts | 4 +- server/lib/activitypub/video-comments.ts | 2 +- server/lib/activitypub/videos.ts | 436 +++++++++--------- .../handlers/activitypub-http-fetcher.ts | 4 +- server/middlewares/validators/videos.ts | 4 +- server/models/video/video.ts | 16 +- 13 files changed, 280 insertions(+), 245 deletions(-) create mode 100644 server/helpers/video.ts diff --git a/server/controllers/api/search.ts b/server/controllers/api/search.ts index 58851d0b5..ea3166f5f 100644 --- a/server/controllers/api/search.ts +++ b/server/controllers/api/search.ts @@ -139,7 +139,7 @@ async function searchVideoURI (url: string, res: express.Response) { refreshVideo: false } - const result = await getOrCreateVideoAndAccountAndChannel(url, syncParam) + const result = await getOrCreateVideoAndAccountAndChannel({ videoObject: url, syncParam }) video = result ? result.video : undefined } catch (err) { logger.info('Cannot search remote video %s.', url, { err }) diff --git a/server/helpers/custom-validators/videos.ts b/server/helpers/custom-validators/videos.ts index c9ef8445d..9875c68bd 100644 --- a/server/helpers/custom-validators/videos.ts +++ b/server/helpers/custom-validators/videos.ts @@ -18,6 +18,7 @@ import { exists, isArray, isFileValid } from './misc' import { VideoChannelModel } from '../../models/video/video-channel' import { UserModel } from '../../models/account/user' import * as magnetUtil from 'magnet-uri' +import { fetchVideo, VideoFetchType } from '../video' const VIDEOS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEOS @@ -152,17 +153,8 @@ function checkUserCanManageVideo (user: UserModel, video: VideoModel, right: Use return true } -export type VideoFetchType = 'all' | 'only-video' | 'id' | 'none' async function isVideoExist (id: string, res: Response, fetchType: VideoFetchType = 'all') { - let video: VideoModel | null - - if (fetchType === 'all') { - video = await VideoModel.loadAndPopulateAccountAndServerAndTags(id) - } else if (fetchType === 'only-video') { - video = await VideoModel.load(id) - } else if (fetchType === 'id' || fetchType === 'none') { - video = await VideoModel.loadOnlyId(id) - } + const video = await fetchVideo(id, fetchType) if (video === null) { res.status(404) diff --git a/server/helpers/video.ts b/server/helpers/video.ts new file mode 100644 index 000000000..b1577a6b0 --- /dev/null +++ b/server/helpers/video.ts @@ -0,0 +1,25 @@ +import { VideoModel } from '../models/video/video' + +type VideoFetchType = 'all' | 'only-video' | 'id' | 'none' + +function fetchVideo (id: number | string, fetchType: VideoFetchType) { + if (fetchType === 'all') return VideoModel.loadAndPopulateAccountAndServerAndTags(id) + + if (fetchType === 'only-video') return VideoModel.load(id) + + if (fetchType === 'id' || fetchType === 'none') return VideoModel.loadOnlyId(id) +} + +type VideoFetchByUrlType = 'all' | 'only-video' +function fetchVideoByUrl (url: string, fetchType: VideoFetchByUrlType) { + if (fetchType === 'all') return VideoModel.loadByUrlAndPopulateAccount(url) + + if (fetchType === 'only-video') return VideoModel.loadByUrl(url) +} + +export { + VideoFetchType, + VideoFetchByUrlType, + fetchVideo, + fetchVideoByUrl +} diff --git a/server/lib/activitypub/process/process-announce.ts b/server/lib/activitypub/process/process-announce.ts index 814556817..b968389b3 100644 --- a/server/lib/activitypub/process/process-announce.ts +++ b/server/lib/activitypub/process/process-announce.ts @@ -25,7 +25,7 @@ export { async function processVideoShare (actorAnnouncer: ActorModel, activity: ActivityAnnounce) { const objectUri = typeof activity.object === 'string' ? activity.object : activity.object.id - const { video } = await getOrCreateVideoAndAccountAndChannel(objectUri) + const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: objectUri }) return sequelizeTypescript.transaction(async t => { // Add share entry diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts index 32e555acf..99841da14 100644 --- a/server/lib/activitypub/process/process-create.ts +++ b/server/lib/activitypub/process/process-create.ts @@ -48,7 +48,7 @@ export { async function processCreateVideo (activity: ActivityCreate) { const videoToCreateData = activity.object as VideoTorrentObject - const { video } = await getOrCreateVideoAndAccountAndChannel(videoToCreateData) + const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoToCreateData }) return video } @@ -59,7 +59,7 @@ async function processCreateDislike (byActor: ActorModel, activity: ActivityCrea if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) - const { video } = await getOrCreateVideoAndAccountAndChannel(dislike.object) + const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislike.object }) return sequelizeTypescript.transaction(async t => { const rate = { @@ -86,7 +86,7 @@ async function processCreateDislike (byActor: ActorModel, activity: ActivityCrea async function processCreateView (byActor: ActorModel, activity: ActivityCreate) { const view = activity.object as ViewObject - const { video } = await getOrCreateVideoAndAccountAndChannel(view.object) + const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: view.object }) const actor = await ActorModel.loadByUrl(view.actor) if (!actor) throw new Error('Unknown actor ' + view.actor) @@ -103,7 +103,7 @@ async function processCreateView (byActor: ActorModel, activity: ActivityCreate) async function processCacheFile (byActor: ActorModel, activity: ActivityCreate) { const cacheFile = activity.object as CacheFileObject - const { video } = await getOrCreateVideoAndAccountAndChannel(cacheFile.object) + const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFile.object }) await createCacheFile(cacheFile, video, byActor) @@ -120,7 +120,7 @@ async function processCreateVideoAbuse (actor: ActorModel, videoAbuseToCreateDat const account = actor.Account if (!account) throw new Error('Cannot create dislike with the non account actor ' + actor.url) - const { video } = await getOrCreateVideoAndAccountAndChannel(videoAbuseToCreateData.object) + const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoAbuseToCreateData.object }) return sequelizeTypescript.transaction(async t => { const videoAbuseData = { diff --git a/server/lib/activitypub/process/process-like.ts b/server/lib/activitypub/process/process-like.ts index 9e1664fd8..631a9dde7 100644 --- a/server/lib/activitypub/process/process-like.ts +++ b/server/lib/activitypub/process/process-like.ts @@ -27,7 +27,7 @@ async function processLikeVideo (byActor: ActorModel, activity: ActivityLike) { const byAccount = byActor.Account if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url) - const { video } = await getOrCreateVideoAndAccountAndChannel(videoUrl) + const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoUrl }) return sequelizeTypescript.transaction(async t => { const rate = { diff --git a/server/lib/activitypub/process/process-undo.ts b/server/lib/activitypub/process/process-undo.ts index 0eb5fa392..b78de6697 100644 --- a/server/lib/activitypub/process/process-undo.ts +++ b/server/lib/activitypub/process/process-undo.ts @@ -54,7 +54,7 @@ export { async function processUndoLike (actorUrl: string, activity: ActivityUndo) { const likeActivity = activity.object as ActivityLike - const { video } = await getOrCreateVideoAndAccountAndChannel(likeActivity.object) + const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: likeActivity.object }) return sequelizeTypescript.transaction(async t => { const byAccount = await AccountModel.loadByUrl(actorUrl, t) @@ -78,7 +78,7 @@ async function processUndoLike (actorUrl: string, activity: ActivityUndo) { async function processUndoDislike (actorUrl: string, activity: ActivityUndo) { const dislike = activity.object.object as DislikeObject - const { video } = await getOrCreateVideoAndAccountAndChannel(dislike.object) + const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislike.object }) return sequelizeTypescript.transaction(async t => { const byAccount = await AccountModel.loadByUrl(actorUrl, t) @@ -102,7 +102,7 @@ async function processUndoDislike (actorUrl: string, activity: ActivityUndo) { async function processUndoCacheFile (actorUrl: string, activity: ActivityUndo) { const cacheFileObject = activity.object.object as CacheFileObject - const { video } = await getOrCreateVideoAndAccountAndChannel(cacheFileObject.object) + const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFileObject.object }) return sequelizeTypescript.transaction(async t => { const byActor = await ActorModel.loadByUrl(actorUrl) diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts index d3af1a181..935da5a54 100644 --- a/server/lib/activitypub/process/process-update.ts +++ b/server/lib/activitypub/process/process-update.ts @@ -48,7 +48,7 @@ async function processUpdateVideo (actor: ActorModel, activity: ActivityUpdate) return undefined } - const { video } = await getOrCreateVideoAndAccountAndChannel(videoObject.id) + const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoObject.id }) const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) return updateVideoFromAP(video, videoObject, actor.Account, channelActor.VideoChannel, activity.to) @@ -64,7 +64,7 @@ async function processUpdateCacheFile (byActor: ActorModel, activity: ActivityUp const redundancyModel = await VideoRedundancyModel.loadByUrl(cacheFileObject.id) if (!redundancyModel) { - const { video } = await getOrCreateVideoAndAccountAndChannel(cacheFileObject.id) + const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFileObject.id }) return createCacheFile(cacheFileObject, video, byActor) } diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts index ffbd3a64e..4ca8bf659 100644 --- a/server/lib/activitypub/video-comments.ts +++ b/server/lib/activitypub/video-comments.ts @@ -94,7 +94,7 @@ async function resolveThread (url: string, comments: VideoCommentModel[] = []) { try { // Maybe it's a reply to a video? // If yes, it's done: we resolved all the thread - const { video } = await getOrCreateVideoAndAccountAndChannel(url) + const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: url }) if (comments.length !== 0) { const firstReply = comments[ comments.length - 1 ] diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index 5150c9975..5aabd3e0d 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -3,7 +3,7 @@ import * as sequelize from 'sequelize' import * as magnetUtil from 'magnet-uri' import { join } from 'path' import * as request from 'request' -import { ActivityIconObject, ActivityVideoUrlObject, VideoState, ActivityUrlObject } from '../../../shared/index' +import { ActivityIconObject, ActivityUrlObject, ActivityVideoUrlObject, VideoState } from '../../../shared/index' import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' import { VideoPrivacy } from '../../../shared/models/videos' import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos' @@ -28,6 +28,7 @@ import { ActivitypubHttpFetcherPayload } from '../job-queue/handlers/activitypub import { createRates } from './video-rates' import { addVideoShares, shareVideoByServerAndChannel } from './share' import { AccountModel } from '../../models/account/account' +import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video' async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { // If the video is not private and published, we federate it @@ -50,13 +51,24 @@ async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, tr } } -function fetchRemoteVideoStaticFile (video: VideoModel, path: string, reject: Function) { - const host = video.VideoChannel.Account.Actor.Server.host +async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.RequestResponse, videoObject: VideoTorrentObject }> { + const options = { + uri: videoUrl, + method: 'GET', + json: true, + activityPub: true + } - // We need to provide a callback, if no we could have an uncaught exception - return request.get(REMOTE_SCHEME.HTTP + '://' + host + path, err => { - if (err) reject(err) - }) + logger.info('Fetching remote video %s.', videoUrl) + + const { response, body } = await doRequest(options) + + if (sanitizeAndCheckVideoTorrentObject(body) === false) { + logger.debug('Remote video JSON is not valid.', { body }) + return { response, videoObject: undefined } + } + + return { response, videoObject: body } } async function fetchRemoteVideoDescription (video: VideoModel) { @@ -71,6 +83,15 @@ async function fetchRemoteVideoDescription (video: VideoModel) { return body.description ? body.description : '' } +function fetchRemoteVideoStaticFile (video: VideoModel, path: string, reject: Function) { + const host = video.VideoChannel.Account.Actor.Server.host + + // We need to provide a callback, if no we could have an uncaught exception + return request.get(REMOTE_SCHEME.HTTP + '://' + host + path, err => { + if (err) reject(err) + }) +} + function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject) { const thumbnailName = video.getThumbnailName() const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName) @@ -82,94 +103,6 @@ function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject) return doRequestAndSaveToFile(options, thumbnailPath) } -async function videoActivityObjectToDBAttributes ( - videoChannel: VideoChannelModel, - videoObject: VideoTorrentObject, - to: string[] = [] -) { - const privacy = to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 ? VideoPrivacy.PUBLIC : VideoPrivacy.UNLISTED - const duration = videoObject.duration.replace(/[^\d]+/, '') - - let language: string | undefined - if (videoObject.language) { - language = videoObject.language.identifier - } - - let category: number | undefined - if (videoObject.category) { - category = parseInt(videoObject.category.identifier, 10) - } - - let licence: number | undefined - if (videoObject.licence) { - licence = parseInt(videoObject.licence.identifier, 10) - } - - const description = videoObject.content || null - const support = videoObject.support || null - - return { - name: videoObject.name, - uuid: videoObject.uuid, - url: videoObject.id, - category, - licence, - language, - description, - support, - nsfw: videoObject.sensitive, - commentsEnabled: videoObject.commentsEnabled, - waitTranscoding: videoObject.waitTranscoding, - state: videoObject.state, - channelId: videoChannel.id, - duration: parseInt(duration, 10), - createdAt: new Date(videoObject.published), - publishedAt: new Date(videoObject.published), - // FIXME: updatedAt does not seems to be considered by Sequelize - updatedAt: new Date(videoObject.updated), - views: videoObject.views, - likes: 0, - dislikes: 0, - remote: true, - privacy - } -} - -function videoFileActivityUrlToDBAttributes (videoCreated: VideoModel, videoObject: VideoTorrentObject) { - const fileUrls = videoObject.url.filter(u => isActivityVideoUrlObject(u)) as ActivityVideoUrlObject[] - - if (fileUrls.length === 0) { - throw new Error('Cannot find video files for ' + videoCreated.url) - } - - const attributes: VideoFileModel[] = [] - for (const fileUrl of fileUrls) { - // Fetch associated magnet uri - const magnet = videoObject.url.find(u => { - return u.mimeType === 'application/x-bittorrent;x-scheme-handler/magnet' && u.height === fileUrl.height - }) - - if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href) - - const parsed = magnetUtil.decode(magnet.href) - if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) { - throw new Error('Cannot parse magnet URI ' + magnet.href) - } - - const attribute = { - extname: VIDEO_MIMETYPE_EXT[ fileUrl.mimeType ], - infoHash: parsed.infoHash, - resolution: fileUrl.height, - size: fileUrl.size, - videoId: videoCreated.id, - fps: fileUrl.fps - } as VideoFileModel - attributes.push(attribute) - } - - return attributes -} - function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) { const channel = videoObject.attributedTo.find(a => a.type === 'Group') if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url) @@ -177,51 +110,6 @@ function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject return getOrCreateActorAndServerAndModel(channel.id) } -async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) { - logger.debug('Adding remote video %s.', videoObject.id) - - const videoCreated: VideoModel = await sequelizeTypescript.transaction(async t => { - const sequelizeOptions = { transaction: t } - - const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to) - const video = VideoModel.build(videoData) - - const videoCreated = await video.save(sequelizeOptions) - - // Process files - const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject) - if (videoFileAttributes.length === 0) { - throw new Error('Cannot find valid files for video %s ' + videoObject.url) - } - - const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t })) - await Promise.all(videoFilePromises) - - // Process tags - const tags = videoObject.tag.map(t => t.name) - const tagInstances = await TagModel.findOrCreateTags(tags, t) - await videoCreated.$set('Tags', tagInstances, sequelizeOptions) - - // Process captions - const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => { - return VideoCaptionModel.insertOrReplaceLanguage(videoCreated.id, c.identifier, t) - }) - await Promise.all(videoCaptionsPromises) - - logger.info('Remote video with uuid %s inserted.', videoObject.uuid) - - videoCreated.VideoChannel = channelActor.VideoChannel - return videoCreated - }) - - const p = generateThumbnailFromUrl(videoCreated, videoObject.icon) - .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err })) - - if (waitThumbnail === true) await p - - return videoCreated -} - type SyncParam = { likes: boolean dislikes: boolean @@ -230,28 +118,7 @@ type SyncParam = { thumbnail: boolean refreshVideo: boolean } -async function getOrCreateVideoAndAccountAndChannel ( - videoObject: VideoTorrentObject | string, - syncParam: SyncParam = { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false } -) { - const videoUrl = typeof videoObject === 'string' ? videoObject : videoObject.id - - let videoFromDatabase = await VideoModel.loadByUrlAndPopulateAccount(videoUrl) - if (videoFromDatabase) { - const p = retryTransactionWrapper(refreshVideoIfNeeded, videoFromDatabase) - if (syncParam.refreshVideo === true) videoFromDatabase = await p - - return { video: videoFromDatabase } - } - - const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl) - if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl) - - const channelActor = await getOrCreateVideoChannelFromVideoObject(fetchedVideo) - const video = await retryTransactionWrapper(createVideo, fetchedVideo, channelActor, syncParam.thumbnail) - - // Process outside the transaction because we could fetch remote data - +async function syncVideoExternalAttributes (video: VideoModel, fetchedVideo: VideoTorrentObject, syncParam: SyncParam) { logger.info('Adding likes/dislikes/shares/comments of video %s.', video.uuid) const jobPayloads: ActivitypubHttpFetcherPayload[] = [] @@ -285,56 +152,39 @@ async function getOrCreateVideoAndAccountAndChannel ( } await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })) +} + +async function getOrCreateVideoAndAccountAndChannel (options: { + videoObject: VideoTorrentObject | string, + syncParam?: SyncParam, + fetchType?: VideoFetchByUrlType +}) { + // Default params + const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false } + const fetchType = options.fetchType || 'all' + + // Get video url + const videoUrl = typeof options.videoObject === 'string' ? options.videoObject : options.videoObject.id + + let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType) + if (videoFromDatabase) { + const p = retryTransactionWrapper(refreshVideoIfNeeded, videoFromDatabase, fetchType, syncParam) + if (syncParam.refreshVideo === true) videoFromDatabase = await p + + return { video: videoFromDatabase } + } + + const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl) + if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl) + + const channelActor = await getOrCreateVideoChannelFromVideoObject(fetchedVideo) + const video = await retryTransactionWrapper(createVideo, fetchedVideo, channelActor, syncParam.thumbnail) + + await syncVideoExternalAttributes(video, fetchedVideo, syncParam) return { video } } -async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.RequestResponse, videoObject: VideoTorrentObject }> { - const options = { - uri: videoUrl, - method: 'GET', - json: true, - activityPub: true - } - - logger.info('Fetching remote video %s.', videoUrl) - - const { response, body } = await doRequest(options) - - if (sanitizeAndCheckVideoTorrentObject(body) === false) { - logger.debug('Remote video JSON is not valid.', { body }) - return { response, videoObject: undefined } - } - - return { response, videoObject: body } -} - -async function refreshVideoIfNeeded (video: VideoModel): Promise { - if (!video.isOutdated()) return video - - try { - const { response, videoObject } = await fetchRemoteVideo(video.url) - if (response.statusCode === 404) { - // Video does not exist anymore - await video.destroy() - return undefined - } - - if (videoObject === undefined) { - logger.warn('Cannot refresh remote video: invalid body.') - return video - } - - const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) - const account = await AccountModel.load(channelActor.VideoChannel.accountId) - - return updateVideoFromAP(video, videoObject, account, channelActor.VideoChannel) - } catch (err) { - logger.warn('Cannot refresh video.', { err }) - return video - } -} - async function updateVideoFromAP ( video: VideoModel, videoObject: VideoTorrentObject, @@ -433,12 +283,7 @@ export { fetchRemoteVideoStaticFile, fetchRemoteVideoDescription, generateThumbnailFromUrl, - videoActivityObjectToDBAttributes, - videoFileActivityUrlToDBAttributes, - createVideo, - getOrCreateVideoChannelFromVideoObject, - addVideoShares, - createRates + getOrCreateVideoChannelFromVideoObject } // --------------------------------------------------------------------------- @@ -448,3 +293,166 @@ function isActivityVideoUrlObject (url: ActivityUrlObject): url is ActivityVideo return mimeTypes.indexOf(url.mimeType) !== -1 && url.mimeType.startsWith('video/') } + +async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) { + logger.debug('Adding remote video %s.', videoObject.id) + + const videoCreated: VideoModel = await sequelizeTypescript.transaction(async t => { + const sequelizeOptions = { transaction: t } + + const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to) + const video = VideoModel.build(videoData) + + const videoCreated = await video.save(sequelizeOptions) + + // Process files + const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject) + if (videoFileAttributes.length === 0) { + throw new Error('Cannot find valid files for video %s ' + videoObject.url) + } + + const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t })) + await Promise.all(videoFilePromises) + + // Process tags + const tags = videoObject.tag.map(t => t.name) + const tagInstances = await TagModel.findOrCreateTags(tags, t) + await videoCreated.$set('Tags', tagInstances, sequelizeOptions) + + // Process captions + const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => { + return VideoCaptionModel.insertOrReplaceLanguage(videoCreated.id, c.identifier, t) + }) + await Promise.all(videoCaptionsPromises) + + logger.info('Remote video with uuid %s inserted.', videoObject.uuid) + + videoCreated.VideoChannel = channelActor.VideoChannel + return videoCreated + }) + + const p = generateThumbnailFromUrl(videoCreated, videoObject.icon) + .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err })) + + if (waitThumbnail === true) await p + + return videoCreated +} + +async function refreshVideoIfNeeded (videoArg: VideoModel, fetchedType: VideoFetchByUrlType, syncParam: SyncParam): Promise { + // We need more attributes if the argument video was fetched with not enough joints + const video = fetchedType === 'all' ? videoArg : await VideoModel.loadByUrlAndPopulateAccount(videoArg.url) + + if (!video.isOutdated()) return video + + try { + const { response, videoObject } = await fetchRemoteVideo(video.url) + if (response.statusCode === 404) { + // Video does not exist anymore + await video.destroy() + return undefined + } + + if (videoObject === undefined) { + logger.warn('Cannot refresh remote video: invalid body.') + return video + } + + const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) + const account = await AccountModel.load(channelActor.VideoChannel.accountId) + + await updateVideoFromAP(video, videoObject, account, channelActor.VideoChannel) + await syncVideoExternalAttributes(video, videoObject, syncParam) + } catch (err) { + logger.warn('Cannot refresh video.', { err }) + return video + } +} + +async function videoActivityObjectToDBAttributes ( + videoChannel: VideoChannelModel, + videoObject: VideoTorrentObject, + to: string[] = [] +) { + const privacy = to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 ? VideoPrivacy.PUBLIC : VideoPrivacy.UNLISTED + const duration = videoObject.duration.replace(/[^\d]+/, '') + + let language: string | undefined + if (videoObject.language) { + language = videoObject.language.identifier + } + + let category: number | undefined + if (videoObject.category) { + category = parseInt(videoObject.category.identifier, 10) + } + + let licence: number | undefined + if (videoObject.licence) { + licence = parseInt(videoObject.licence.identifier, 10) + } + + const description = videoObject.content || null + const support = videoObject.support || null + + return { + name: videoObject.name, + uuid: videoObject.uuid, + url: videoObject.id, + category, + licence, + language, + description, + support, + nsfw: videoObject.sensitive, + commentsEnabled: videoObject.commentsEnabled, + waitTranscoding: videoObject.waitTranscoding, + state: videoObject.state, + channelId: videoChannel.id, + duration: parseInt(duration, 10), + createdAt: new Date(videoObject.published), + publishedAt: new Date(videoObject.published), + // FIXME: updatedAt does not seems to be considered by Sequelize + updatedAt: new Date(videoObject.updated), + views: videoObject.views, + likes: 0, + dislikes: 0, + remote: true, + privacy + } +} + +function videoFileActivityUrlToDBAttributes (videoCreated: VideoModel, videoObject: VideoTorrentObject) { + const fileUrls = videoObject.url.filter(u => isActivityVideoUrlObject(u)) as ActivityVideoUrlObject[] + + if (fileUrls.length === 0) { + throw new Error('Cannot find video files for ' + videoCreated.url) + } + + const attributes: VideoFileModel[] = [] + for (const fileUrl of fileUrls) { + // Fetch associated magnet uri + const magnet = videoObject.url.find(u => { + return u.mimeType === 'application/x-bittorrent;x-scheme-handler/magnet' && u.height === fileUrl.height + }) + + if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href) + + const parsed = magnetUtil.decode(magnet.href) + if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) { + throw new Error('Cannot parse magnet URI ' + magnet.href) + } + + const attribute = { + extname: VIDEO_MIMETYPE_EXT[ fileUrl.mimeType ], + infoHash: parsed.infoHash, + resolution: fileUrl.height, + size: fileUrl.size, + videoId: videoCreated.id, + fps: fileUrl.fps + } as VideoFileModel + attributes.push(attribute) + } + + return attributes +} diff --git a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts index 72d670277..42217c27c 100644 --- a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts +++ b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts @@ -1,10 +1,10 @@ import * as Bull from 'bull' import { logger } from '../../../helpers/logger' import { processActivities } from '../../activitypub/process' -import { VideoModel } from '../../../models/video/video' -import { addVideoShares, createRates } from '../../activitypub/videos' import { addVideoComments } from '../../activitypub/video-comments' import { crawlCollectionPage } from '../../activitypub/crawl' +import { VideoModel } from '../../../models/video/video' +import { addVideoShares, createRates } from '../../activitypub' type FetchType = 'activity' | 'video-likes' | 'video-dislikes' | 'video-shares' | 'video-comments' diff --git a/server/middlewares/validators/videos.ts b/server/middlewares/validators/videos.ts index 8aa7b3a39..67eabe468 100644 --- a/server/middlewares/validators/videos.ts +++ b/server/middlewares/validators/videos.ts @@ -26,8 +26,7 @@ import { isVideoPrivacyValid, isVideoRatingTypeValid, isVideoSupportValid, - isVideoTagsValid, - VideoFetchType + isVideoTagsValid } from '../../helpers/custom-validators/videos' import { getDurationFromVideoFile } from '../../helpers/ffmpeg-utils' import { logger } from '../../helpers/logger' @@ -42,6 +41,7 @@ import { checkUserCanTerminateOwnershipChange, doesChangeVideoOwnershipExist } f import { VideoChangeOwnershipAccept } from '../../../shared/models/videos/video-change-ownership-accept.model' import { VideoChangeOwnershipModel } from '../../models/video/video-change-ownership' import { AccountModel } from '../../models/account/account' +import { VideoFetchType } from '../../helpers/video' const videosAddValidator = getCommonVideoAttributes().concat([ body('videofile') diff --git a/server/models/video/video.ts b/server/models/video/video.ts index ce2153f87..6c89c16bf 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -1103,14 +1103,24 @@ export class VideoModel extends Model { .findOne(options) } - static loadByUrlAndPopulateAccount (url: string, t?: Sequelize.Transaction) { + static loadByUrl (url: string, transaction?: Sequelize.Transaction) { const query: IFindOptions = { where: { url - } + }, + transaction } - if (t !== undefined) query.transaction = t + return VideoModel.findOne(query) + } + + static loadByUrlAndPopulateAccount (url: string, transaction?: Sequelize.Transaction) { + const query: IFindOptions = { + where: { + url + }, + transaction + } return VideoModel.scope([ ScopeNames.WITH_ACCOUNT_DETAILS, ScopeNames.WITH_FILES ]).findOne(query) } From d4defe07d26013a75577b30608841fe3f8334308 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 19 Sep 2018 11:41:21 +0200 Subject: [PATCH 30/44] Optimize video view AP processing --- .../lib/activitypub/process/process-create.ts | 10 +- .../lib/activitypub/process/process-update.ts | 10 +- server/lib/activitypub/videos.ts | 113 +++++++++++------- server/models/activitypub/actor.ts | 12 ++ 4 files changed, 95 insertions(+), 50 deletions(-) diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts index 99841da14..559a0c23c 100644 --- a/server/lib/activitypub/process/process-create.ts +++ b/server/lib/activitypub/process/process-create.ts @@ -86,10 +86,14 @@ async function processCreateDislike (byActor: ActorModel, activity: ActivityCrea async function processCreateView (byActor: ActorModel, activity: ActivityCreate) { const view = activity.object as ViewObject - const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: view.object }) + const options = { + videoObject: view.object, + fetchType: 'only-video' as 'only-video' + } + const { video } = await getOrCreateVideoAndAccountAndChannel(options) - const actor = await ActorModel.loadByUrl(view.actor) - if (!actor) throw new Error('Unknown actor ' + view.actor) + const actorExists = await ActorModel.isActorUrlExist(view.actor) + if (actorExists === false) throw new Error('Unknown actor ' + view.actor) await Redis.Instance.addVideoView(video.id) diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts index 935da5a54..0bceb370e 100644 --- a/server/lib/activitypub/process/process-update.ts +++ b/server/lib/activitypub/process/process-update.ts @@ -51,7 +51,15 @@ async function processUpdateVideo (actor: ActorModel, activity: ActivityUpdate) const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoObject.id }) const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) - return updateVideoFromAP(video, videoObject, actor.Account, channelActor.VideoChannel, activity.to) + const updateOptions = { + video, + videoObject, + account: actor.Account, + channel: channelActor.VideoChannel, + updateViews: true, + overrideTo: activity.to + } + return updateVideoFromAP(updateOptions) } async function processUpdateCacheFile (byActor: ActorModel, activity: ActivityUpdate) { diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index 5aabd3e0d..de22e3584 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -157,18 +157,26 @@ async function syncVideoExternalAttributes (video: VideoModel, fetchedVideo: Vid async function getOrCreateVideoAndAccountAndChannel (options: { videoObject: VideoTorrentObject | string, syncParam?: SyncParam, - fetchType?: VideoFetchByUrlType + fetchType?: VideoFetchByUrlType, + refreshViews?: boolean }) { // Default params const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false } const fetchType = options.fetchType || 'all' + const refreshViews = options.refreshViews || false // Get video url const videoUrl = typeof options.videoObject === 'string' ? options.videoObject : options.videoObject.id let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType) if (videoFromDatabase) { - const p = retryTransactionWrapper(refreshVideoIfNeeded, videoFromDatabase, fetchType, syncParam) + const refreshOptions = { + video: videoFromDatabase, + fetchedType: fetchType, + syncParam, + refreshViews + } + const p = retryTransactionWrapper(refreshVideoIfNeeded, refreshOptions) if (syncParam.refreshVideo === true) videoFromDatabase = await p return { video: videoFromDatabase } @@ -185,14 +193,15 @@ async function getOrCreateVideoAndAccountAndChannel (options: { return { video } } -async function updateVideoFromAP ( +async function updateVideoFromAP (options: { video: VideoModel, videoObject: VideoTorrentObject, account: AccountModel, channel: VideoChannelModel, + updateViews: boolean, overrideTo?: string[] -) { - logger.debug('Updating remote video "%s".', videoObject.uuid) +}) { + logger.debug('Updating remote video "%s".', options.videoObject.uuid) let videoFieldsSave: any try { @@ -201,72 +210,72 @@ async function updateVideoFromAP ( transaction: t } - videoFieldsSave = video.toJSON() + videoFieldsSave = options.video.toJSON() // Check actor has the right to update the video - const videoChannel = video.VideoChannel - if (videoChannel.Account.id !== account.id) { - throw new Error('Account ' + account.Actor.url + ' does not own video channel ' + videoChannel.Actor.url) + const videoChannel = options.video.VideoChannel + if (videoChannel.Account.id !== options.account.id) { + throw new Error('Account ' + options.account.Actor.url + ' does not own video channel ' + videoChannel.Actor.url) } - const to = overrideTo ? overrideTo : videoObject.to - const videoData = await videoActivityObjectToDBAttributes(channel, videoObject, to) - video.set('name', videoData.name) - video.set('uuid', videoData.uuid) - video.set('url', videoData.url) - video.set('category', videoData.category) - video.set('licence', videoData.licence) - video.set('language', videoData.language) - video.set('description', videoData.description) - video.set('support', videoData.support) - video.set('nsfw', videoData.nsfw) - video.set('commentsEnabled', videoData.commentsEnabled) - video.set('waitTranscoding', videoData.waitTranscoding) - video.set('state', videoData.state) - video.set('duration', videoData.duration) - video.set('createdAt', videoData.createdAt) - video.set('publishedAt', videoData.publishedAt) - video.set('views', videoData.views) - video.set('privacy', videoData.privacy) - video.set('channelId', videoData.channelId) + const to = options.overrideTo ? options.overrideTo : options.videoObject.to + const videoData = await videoActivityObjectToDBAttributes(options.channel, options.videoObject, to) + options.video.set('name', videoData.name) + options.video.set('uuid', videoData.uuid) + options.video.set('url', videoData.url) + options.video.set('category', videoData.category) + options.video.set('licence', videoData.licence) + options.video.set('language', videoData.language) + options.video.set('description', videoData.description) + options.video.set('support', videoData.support) + options.video.set('nsfw', videoData.nsfw) + options.video.set('commentsEnabled', videoData.commentsEnabled) + options.video.set('waitTranscoding', videoData.waitTranscoding) + options.video.set('state', videoData.state) + options.video.set('duration', videoData.duration) + options.video.set('createdAt', videoData.createdAt) + options.video.set('publishedAt', videoData.publishedAt) + options.video.set('privacy', videoData.privacy) + options.video.set('channelId', videoData.channelId) - await video.save(sequelizeOptions) + if (options.updateViews === true) options.video.set('views', videoData.views) + await options.video.save(sequelizeOptions) // Don't block on request - generateThumbnailFromUrl(video, videoObject.icon) - .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err })) + generateThumbnailFromUrl(options.video, options.videoObject.icon) + .catch(err => logger.warn('Cannot generate thumbnail of %s.', options.videoObject.id, { err })) // Remove old video files const videoFileDestroyTasks: Bluebird[] = [] - for (const videoFile of video.VideoFiles) { + for (const videoFile of options.video.VideoFiles) { videoFileDestroyTasks.push(videoFile.destroy(sequelizeOptions)) } await Promise.all(videoFileDestroyTasks) - const videoFileAttributes = videoFileActivityUrlToDBAttributes(video, videoObject) + const videoFileAttributes = videoFileActivityUrlToDBAttributes(options.video, options.videoObject) const tasks = videoFileAttributes.map(f => VideoFileModel.create(f, sequelizeOptions)) await Promise.all(tasks) // Update Tags - const tags = videoObject.tag.map(tag => tag.name) + const tags = options.videoObject.tag.map(tag => tag.name) const tagInstances = await TagModel.findOrCreateTags(tags, t) - await video.$set('Tags', tagInstances, sequelizeOptions) + await options.video.$set('Tags', tagInstances, sequelizeOptions) // Update captions - await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(video.id, t) + await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(options.video.id, t) - const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => { - return VideoCaptionModel.insertOrReplaceLanguage(video.id, c.identifier, t) + const videoCaptionsPromises = options.videoObject.subtitleLanguage.map(c => { + return VideoCaptionModel.insertOrReplaceLanguage(options.video.id, c.identifier, t) }) await Promise.all(videoCaptionsPromises) }) - logger.info('Remote video with uuid %s updated', videoObject.uuid) + logger.info('Remote video with uuid %s updated', options.videoObject.uuid) return updatedVideo } catch (err) { - if (video !== undefined && videoFieldsSave !== undefined) { - resetSequelizeInstance(video, videoFieldsSave) + if (options.video !== undefined && videoFieldsSave !== undefined) { + resetSequelizeInstance(options.video, videoFieldsSave) } // This is just a debug because we will retry the insert @@ -339,9 +348,14 @@ async function createVideo (videoObject: VideoTorrentObject, channelActor: Actor return videoCreated } -async function refreshVideoIfNeeded (videoArg: VideoModel, fetchedType: VideoFetchByUrlType, syncParam: SyncParam): Promise { +async function refreshVideoIfNeeded (options: { + video: VideoModel, + fetchedType: VideoFetchByUrlType, + syncParam: SyncParam, + refreshViews: boolean +}): Promise { // We need more attributes if the argument video was fetched with not enough joints - const video = fetchedType === 'all' ? videoArg : await VideoModel.loadByUrlAndPopulateAccount(videoArg.url) + const video = options.fetchedType === 'all' ? options.video : await VideoModel.loadByUrlAndPopulateAccount(options.video.url) if (!video.isOutdated()) return video @@ -361,8 +375,15 @@ async function refreshVideoIfNeeded (videoArg: VideoModel, fetchedType: VideoFet const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) const account = await AccountModel.load(channelActor.VideoChannel.accountId) - await updateVideoFromAP(video, videoObject, account, channelActor.VideoChannel) - await syncVideoExternalAttributes(video, videoObject, syncParam) + const updateOptions = { + video, + videoObject, + account, + channel: channelActor.VideoChannel, + updateViews: options.refreshViews + } + await updateVideoFromAP(updateOptions) + await syncVideoExternalAttributes(video, videoObject, options.syncParam) } catch (err) { logger.warn('Cannot refresh video.', { err }) return video diff --git a/server/models/activitypub/actor.ts b/server/models/activitypub/actor.ts index ef8dd9f7c..69c2eca57 100644 --- a/server/models/activitypub/actor.ts +++ b/server/models/activitypub/actor.ts @@ -266,6 +266,18 @@ export class ActorModel extends Model { return ActorModel.unscoped().findById(id) } + static isActorUrlExist (url: string) { + const query = { + raw: true, + where: { + url + } + } + + return ActorModel.unscoped().findOne(query) + .then(a => !!a) + } + static listByFollowersUrls (followersUrls: string[], transaction?: Sequelize.Transaction) { const query = { where: { From e587e0ecee5bec43a225995948faaa4bc97f080a Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 19 Sep 2018 14:44:20 +0200 Subject: [PATCH 31/44] Optimize activity actor load in AP processors --- server/controllers/api/search.ts | 2 +- server/helpers/actor.ts | 13 +++++++++ server/lib/activitypub/actor.ts | 21 ++++++++------ server/lib/activitypub/cache-file.ts | 7 ++--- .../lib/activitypub/process/process-accept.ts | 6 +--- .../activitypub/process/process-announce.ts | 6 +--- .../lib/activitypub/process/process-create.ts | 20 ++++++------- .../lib/activitypub/process/process-delete.ts | 28 +++++++++---------- .../lib/activitypub/process/process-follow.ts | 8 ++---- .../lib/activitypub/process/process-like.ts | 7 ++--- .../lib/activitypub/process/process-reject.ts | 6 +--- .../lib/activitypub/process/process-undo.ts | 23 ++++++--------- .../lib/activitypub/process/process-update.ts | 17 ++++++----- server/lib/activitypub/process/process.ts | 10 +++++-- server/lib/activitypub/videos.ts | 2 +- server/models/activitypub/actor.ts | 23 +++++++++++++++ 16 files changed, 110 insertions(+), 89 deletions(-) create mode 100644 server/helpers/actor.ts diff --git a/server/controllers/api/search.ts b/server/controllers/api/search.ts index ea3166f5f..fd4db7a54 100644 --- a/server/controllers/api/search.ts +++ b/server/controllers/api/search.ts @@ -89,7 +89,7 @@ async function searchVideoChannelURI (search: string, isWebfingerSearch: boolean if (isUserAbleToSearchRemoteURI(res)) { try { - const actor = await getOrCreateActorAndServerAndModel(uri, true, true) + const actor = await getOrCreateActorAndServerAndModel(uri, 'all', true, true) videoChannel = actor.VideoChannel } catch (err) { logger.info('Cannot search remote video channel %s.', uri, { err }) diff --git a/server/helpers/actor.ts b/server/helpers/actor.ts new file mode 100644 index 000000000..12a7ace9f --- /dev/null +++ b/server/helpers/actor.ts @@ -0,0 +1,13 @@ +import { ActorModel } from '../models/activitypub/actor' + +type ActorFetchByUrlType = 'all' | 'actor-and-association-ids' +function fetchActorByUrl (url: string, fetchType: ActorFetchByUrlType) { + if (fetchType === 'all') return ActorModel.loadByUrlAndPopulateAccountAndChannel(url) + + if (fetchType === 'actor-and-association-ids') return ActorModel.loadByUrl(url) +} + +export { + ActorFetchByUrlType, + fetchActorByUrl +} diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts index 3464add03..0bdb7d12e 100644 --- a/server/lib/activitypub/actor.ts +++ b/server/lib/activitypub/actor.ts @@ -21,6 +21,7 @@ import { ServerModel } from '../../models/server/server' import { VideoChannelModel } from '../../models/video/video-channel' import { JobQueue } from '../job-queue' import { getServerActor } from '../../helpers/utils' +import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor' // Set account keys, this could be long so process after the account creation and do not block the client function setAsyncActorKeys (actor: ActorModel) { @@ -38,13 +39,14 @@ function setAsyncActorKeys (actor: ActorModel) { async function getOrCreateActorAndServerAndModel ( activityActor: string | ActivityPubActor, + fetchType: ActorFetchByUrlType = 'actor-and-association-ids', recurseIfNeeded = true, updateCollections = false ) { const actorUrl = getActorUrl(activityActor) let created = false - let actor = await ActorModel.loadByUrl(actorUrl) + let actor = await fetchActorByUrl(actorUrl, fetchType) // Orphan actor (not associated to an account of channel) so recreate it if (actor && (!actor.Account && !actor.VideoChannel)) { await actor.destroy() @@ -65,7 +67,7 @@ async function getOrCreateActorAndServerAndModel ( try { // Assert we don't recurse another time - ownerActor = await getOrCreateActorAndServerAndModel(accountAttributedTo.id, false) + ownerActor = await getOrCreateActorAndServerAndModel(accountAttributedTo.id, 'all', false) } catch (err) { logger.error('Cannot get or create account attributed to video channel ' + actor.url) throw new Error(err) @@ -76,10 +78,7 @@ async function getOrCreateActorAndServerAndModel ( created = true } - if (actor.Account) actor.Account.Actor = actor - if (actor.VideoChannel) actor.VideoChannel.Actor = actor - - const { actor: actorRefreshed, refreshed } = await retryTransactionWrapper(refreshActorIfNeeded, actor) + const { actor: actorRefreshed, refreshed } = await retryTransactionWrapper(refreshActorIfNeeded, actor, fetchType) if (!actorRefreshed) throw new Error('Actor ' + actorRefreshed.url + ' does not exist anymore.') if ((created === true || refreshed === true) && updateCollections === true) { @@ -370,8 +369,14 @@ async function saveVideoChannel (actor: ActorModel, result: FetchRemoteActorResu return videoChannelCreated } -async function refreshActorIfNeeded (actor: ActorModel): Promise<{ actor: ActorModel, refreshed: boolean }> { - if (!actor.isOutdated()) return { actor, refreshed: false } +async function refreshActorIfNeeded ( + actorArg: ActorModel, + fetchedType: ActorFetchByUrlType +): Promise<{ actor: ActorModel, refreshed: boolean }> { + if (!actorArg.isOutdated()) return { actor: actorArg, refreshed: false } + + // We need more attributes + const actor = fetchedType === 'all' ? actorArg : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url) try { const actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost()) diff --git a/server/lib/activitypub/cache-file.ts b/server/lib/activitypub/cache-file.ts index 7325ddcb6..20558daf9 100644 --- a/server/lib/activitypub/cache-file.ts +++ b/server/lib/activitypub/cache-file.ts @@ -1,10 +1,9 @@ import { CacheFileObject } from '../../../shared/index' import { VideoModel } from '../../models/video/video' -import { ActorModel } from '../../models/activitypub/actor' import { sequelizeTypescript } from '../../initializers' import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' -function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: VideoModel, byActor: ActorModel) { +function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: VideoModel, byActor: { id?: number }) { const url = cacheFileObject.url const videoFile = video.VideoFiles.find(f => { @@ -23,7 +22,7 @@ function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject } } -function createCacheFile (cacheFileObject: CacheFileObject, video: VideoModel, byActor: ActorModel) { +function createCacheFile (cacheFileObject: CacheFileObject, video: VideoModel, byActor: { id?: number }) { return sequelizeTypescript.transaction(async t => { const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor) @@ -31,7 +30,7 @@ function createCacheFile (cacheFileObject: CacheFileObject, video: VideoModel, b }) } -function updateCacheFile (cacheFileObject: CacheFileObject, redundancyModel: VideoRedundancyModel, byActor: ActorModel) { +function updateCacheFile (cacheFileObject: CacheFileObject, redundancyModel: VideoRedundancyModel, byActor: { id?: number }) { const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, redundancyModel.VideoFile.Video, byActor) redundancyModel.set('expires', attributes.expiresOn) diff --git a/server/lib/activitypub/process/process-accept.ts b/server/lib/activitypub/process/process-accept.ts index 046370b79..89bda9c32 100644 --- a/server/lib/activitypub/process/process-accept.ts +++ b/server/lib/activitypub/process/process-accept.ts @@ -1,15 +1,11 @@ import { ActivityAccept } from '../../../../shared/models/activitypub' -import { getActorUrl } from '../../../helpers/activitypub' import { ActorModel } from '../../../models/activitypub/actor' import { ActorFollowModel } from '../../../models/activitypub/actor-follow' import { addFetchOutboxJob } from '../actor' -async function processAcceptActivity (activity: ActivityAccept, inboxActor?: ActorModel) { +async function processAcceptActivity (activity: ActivityAccept, targetActor: ActorModel, inboxActor?: ActorModel) { if (inboxActor === undefined) throw new Error('Need to accept on explicit inbox.') - const actorUrl = getActorUrl(activity.actor) - const targetActor = await ActorModel.loadByUrl(actorUrl) - return processAccept(inboxActor, targetActor) } diff --git a/server/lib/activitypub/process/process-announce.ts b/server/lib/activitypub/process/process-announce.ts index b968389b3..cc88b5423 100644 --- a/server/lib/activitypub/process/process-announce.ts +++ b/server/lib/activitypub/process/process-announce.ts @@ -2,15 +2,11 @@ import { ActivityAnnounce } from '../../../../shared/models/activitypub' import { retryTransactionWrapper } from '../../../helpers/database-utils' import { sequelizeTypescript } from '../../../initializers' import { ActorModel } from '../../../models/activitypub/actor' -import { VideoModel } from '../../../models/video/video' import { VideoShareModel } from '../../../models/video/video-share' -import { getOrCreateActorAndServerAndModel } from '../actor' import { forwardVideoRelatedActivity } from '../send/utils' import { getOrCreateVideoAndAccountAndChannel } from '../videos' -async function processAnnounceActivity (activity: ActivityAnnounce) { - const actorAnnouncer = await getOrCreateActorAndServerAndModel(activity.actor) - +async function processAnnounceActivity (activity: ActivityAnnounce, actorAnnouncer: ActorModel) { return retryTransactionWrapper(processVideoShare, actorAnnouncer, activity) } diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts index 559a0c23c..5197dac73 100644 --- a/server/lib/activitypub/process/process-create.ts +++ b/server/lib/activitypub/process/process-create.ts @@ -7,30 +7,28 @@ import { sequelizeTypescript } from '../../../initializers' import { AccountVideoRateModel } from '../../../models/account/account-video-rate' import { ActorModel } from '../../../models/activitypub/actor' import { VideoAbuseModel } from '../../../models/video/video-abuse' -import { getOrCreateActorAndServerAndModel } from '../actor' import { addVideoComment, resolveThread } from '../video-comments' import { getOrCreateVideoAndAccountAndChannel } from '../videos' import { forwardActivity, forwardVideoRelatedActivity } from '../send/utils' import { Redis } from '../../redis' import { createCacheFile } from '../cache-file' -async function processCreateActivity (activity: ActivityCreate) { +async function processCreateActivity (activity: ActivityCreate, byActor: ActorModel) { const activityObject = activity.object const activityType = activityObject.type - const actor = await getOrCreateActorAndServerAndModel(activity.actor) if (activityType === 'View') { - return processCreateView(actor, activity) + return processCreateView(byActor, activity) } else if (activityType === 'Dislike') { - return retryTransactionWrapper(processCreateDislike, actor, activity) + return retryTransactionWrapper(processCreateDislike, byActor, activity) } else if (activityType === 'Video') { return processCreateVideo(activity) } else if (activityType === 'Flag') { - return retryTransactionWrapper(processCreateVideoAbuse, actor, activityObject as VideoAbuseObject) + return retryTransactionWrapper(processCreateVideoAbuse, byActor, activityObject as VideoAbuseObject) } else if (activityType === 'Note') { - return retryTransactionWrapper(processCreateVideoComment, actor, activity) + return retryTransactionWrapper(processCreateVideoComment, byActor, activity) } else if (activityType === 'CacheFile') { - return retryTransactionWrapper(processCacheFile, actor, activity) + return retryTransactionWrapper(processCacheFile, byActor, activity) } logger.warn('Unknown activity object type %s when creating activity.', activityType, { activity: activity.id }) @@ -118,11 +116,11 @@ async function processCacheFile (byActor: ActorModel, activity: ActivityCreate) } } -async function processCreateVideoAbuse (actor: ActorModel, videoAbuseToCreateData: VideoAbuseObject) { +async function processCreateVideoAbuse (byActor: ActorModel, videoAbuseToCreateData: VideoAbuseObject) { logger.debug('Reporting remote abuse for video %s.', videoAbuseToCreateData.object) - const account = actor.Account - if (!account) throw new Error('Cannot create dislike with the non account actor ' + actor.url) + const account = byActor.Account + if (!account) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoAbuseToCreateData.object }) diff --git a/server/lib/activitypub/process/process-delete.ts b/server/lib/activitypub/process/process-delete.ts index 4c034a81c..bf2a4d114 100644 --- a/server/lib/activitypub/process/process-delete.ts +++ b/server/lib/activitypub/process/process-delete.ts @@ -7,34 +7,32 @@ import { ActorModel } from '../../../models/activitypub/actor' import { VideoModel } from '../../../models/video/video' import { VideoChannelModel } from '../../../models/video/video-channel' import { VideoCommentModel } from '../../../models/video/video-comment' -import { getOrCreateActorAndServerAndModel } from '../actor' import { forwardActivity } from '../send/utils' -async function processDeleteActivity (activity: ActivityDelete) { +async function processDeleteActivity (activity: ActivityDelete, byActor: ActorModel) { const objectUrl = typeof activity.object === 'string' ? activity.object : activity.object.id if (activity.actor === objectUrl) { - let actor = await ActorModel.loadByUrl(activity.actor) - if (!actor) return undefined + // We need more attributes (all the account and channel) + const byActorFull = await ActorModel.loadByUrlAndPopulateAccountAndChannel(byActor.url) - if (actor.type === 'Person') { - if (!actor.Account) throw new Error('Actor ' + actor.url + ' is a person but we cannot find it in database.') + if (byActorFull.type === 'Person') { + if (!byActorFull.Account) throw new Error('Actor ' + byActorFull.url + ' is a person but we cannot find it in database.') - actor.Account.Actor = await actor.Account.$get('Actor') as ActorModel - return retryTransactionWrapper(processDeleteAccount, actor.Account) - } else if (actor.type === 'Group') { - if (!actor.VideoChannel) throw new Error('Actor ' + actor.url + ' is a group but we cannot find it in database.') + byActorFull.Account.Actor = await byActorFull.Account.$get('Actor') as ActorModel + return retryTransactionWrapper(processDeleteAccount, byActorFull.Account) + } else if (byActorFull.type === 'Group') { + if (!byActorFull.VideoChannel) throw new Error('Actor ' + byActorFull.url + ' is a group but we cannot find it in database.') - actor.VideoChannel.Actor = await actor.VideoChannel.$get('Actor') as ActorModel - return retryTransactionWrapper(processDeleteVideoChannel, actor.VideoChannel) + byActorFull.VideoChannel.Actor = await byActorFull.VideoChannel.$get('Actor') as ActorModel + return retryTransactionWrapper(processDeleteVideoChannel, byActorFull.VideoChannel) } } - const actor = await getOrCreateActorAndServerAndModel(activity.actor) { const videoCommentInstance = await VideoCommentModel.loadByUrlAndPopulateAccount(objectUrl) if (videoCommentInstance) { - return retryTransactionWrapper(processDeleteVideoComment, actor, videoCommentInstance, activity) + return retryTransactionWrapper(processDeleteVideoComment, byActor, videoCommentInstance, activity) } } @@ -43,7 +41,7 @@ async function processDeleteActivity (activity: ActivityDelete) { if (videoInstance) { if (videoInstance.isOwned()) throw new Error(`Remote instance cannot delete owned video ${videoInstance.url}.`) - return retryTransactionWrapper(processDeleteVideo, actor, videoInstance) + return retryTransactionWrapper(processDeleteVideo, byActor, videoInstance) } } diff --git a/server/lib/activitypub/process/process-follow.ts b/server/lib/activitypub/process/process-follow.ts index f34fd66cc..24c9085f7 100644 --- a/server/lib/activitypub/process/process-follow.ts +++ b/server/lib/activitypub/process/process-follow.ts @@ -4,14 +4,12 @@ import { logger } from '../../../helpers/logger' import { sequelizeTypescript } from '../../../initializers' import { ActorModel } from '../../../models/activitypub/actor' import { ActorFollowModel } from '../../../models/activitypub/actor-follow' -import { getOrCreateActorAndServerAndModel } from '../actor' import { sendAccept } from '../send' -async function processFollowActivity (activity: ActivityFollow) { +async function processFollowActivity (activity: ActivityFollow, byActor: ActorModel) { const activityObject = activity.object - const actor = await getOrCreateActorAndServerAndModel(activity.actor) - return retryTransactionWrapper(processFollow, actor, activityObject) + return retryTransactionWrapper(processFollow, byActor, activityObject) } // --------------------------------------------------------------------------- @@ -24,7 +22,7 @@ export { async function processFollow (actor: ActorModel, targetActorURL: string) { await sequelizeTypescript.transaction(async t => { - const targetActor = await ActorModel.loadByUrl(targetActorURL, t) + const targetActor = await ActorModel.loadByUrlAndPopulateAccountAndChannel(targetActorURL, t) if (!targetActor) throw new Error('Unknown actor') if (targetActor.isOwned() === false) throw new Error('This is not a local actor.') diff --git a/server/lib/activitypub/process/process-like.ts b/server/lib/activitypub/process/process-like.ts index 631a9dde7..f7200db61 100644 --- a/server/lib/activitypub/process/process-like.ts +++ b/server/lib/activitypub/process/process-like.ts @@ -3,14 +3,11 @@ import { retryTransactionWrapper } from '../../../helpers/database-utils' import { sequelizeTypescript } from '../../../initializers' import { AccountVideoRateModel } from '../../../models/account/account-video-rate' import { ActorModel } from '../../../models/activitypub/actor' -import { getOrCreateActorAndServerAndModel } from '../actor' import { forwardVideoRelatedActivity } from '../send/utils' import { getOrCreateVideoAndAccountAndChannel } from '../videos' -async function processLikeActivity (activity: ActivityLike) { - const actor = await getOrCreateActorAndServerAndModel(activity.actor) - - return retryTransactionWrapper(processLikeVideo, actor, activity) +async function processLikeActivity (activity: ActivityLike, byActor: ActorModel) { + return retryTransactionWrapper(processLikeVideo, byActor, activity) } // --------------------------------------------------------------------------- diff --git a/server/lib/activitypub/process/process-reject.ts b/server/lib/activitypub/process/process-reject.ts index f06b03772..b0e678316 100644 --- a/server/lib/activitypub/process/process-reject.ts +++ b/server/lib/activitypub/process/process-reject.ts @@ -1,15 +1,11 @@ import { ActivityReject } from '../../../../shared/models/activitypub/activity' -import { getActorUrl } from '../../../helpers/activitypub' import { sequelizeTypescript } from '../../../initializers' import { ActorModel } from '../../../models/activitypub/actor' import { ActorFollowModel } from '../../../models/activitypub/actor-follow' -async function processRejectActivity (activity: ActivityReject, inboxActor?: ActorModel) { +async function processRejectActivity (activity: ActivityReject, targetActor: ActorModel, inboxActor?: ActorModel) { if (inboxActor === undefined) throw new Error('Need to reject on explicit inbox.') - const actorUrl = getActorUrl(activity.actor) - const targetActor = await ActorModel.loadByUrl(actorUrl) - return processReject(inboxActor, targetActor) } diff --git a/server/lib/activitypub/process/process-undo.ts b/server/lib/activitypub/process/process-undo.ts index b78de6697..c091d9678 100644 --- a/server/lib/activitypub/process/process-undo.ts +++ b/server/lib/activitypub/process/process-undo.ts @@ -13,7 +13,7 @@ import { getOrCreateVideoAndAccountAndChannel } from '../videos' import { VideoShareModel } from '../../../models/video/video-share' import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' -async function processUndoActivity (activity: ActivityUndo) { +async function processUndoActivity (activity: ActivityUndo, byActor: ActorModel) { const activityToUndo = activity.object const actorUrl = getActorUrl(activity.actor) @@ -26,16 +26,16 @@ async function processUndoActivity (activity: ActivityUndo) { if (activityToUndo.object.type === 'Dislike') { return retryTransactionWrapper(processUndoDislike, actorUrl, activity) } else if (activityToUndo.object.type === 'CacheFile') { - return retryTransactionWrapper(processUndoCacheFile, actorUrl, activity) + return retryTransactionWrapper(processUndoCacheFile, byActor, activity) } } if (activityToUndo.type === 'Follow') { - return retryTransactionWrapper(processUndoFollow, actorUrl, activityToUndo) + return retryTransactionWrapper(processUndoFollow, byActor, activityToUndo) } if (activityToUndo.type === 'Announce') { - return retryTransactionWrapper(processUndoAnnounce, actorUrl, activityToUndo) + return retryTransactionWrapper(processUndoAnnounce, byActor, activityToUndo) } logger.warn('Unknown activity object type %s -> %s when undo activity.', activityToUndo.type, { activity: activity.id }) @@ -99,15 +99,12 @@ async function processUndoDislike (actorUrl: string, activity: ActivityUndo) { }) } -async function processUndoCacheFile (actorUrl: string, activity: ActivityUndo) { +async function processUndoCacheFile (byActor: ActorModel, activity: ActivityUndo) { const cacheFileObject = activity.object.object as CacheFileObject const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFileObject.object }) return sequelizeTypescript.transaction(async t => { - const byActor = await ActorModel.loadByUrl(actorUrl) - if (!byActor) throw new Error('Unknown actor ' + actorUrl) - const cacheFile = await VideoRedundancyModel.loadByUrl(cacheFileObject.id) if (!cacheFile) throw new Error('Unknown video cache ' + cacheFile.url) @@ -122,10 +119,9 @@ async function processUndoCacheFile (actorUrl: string, activity: ActivityUndo) { }) } -function processUndoFollow (actorUrl: string, followActivity: ActivityFollow) { +function processUndoFollow (follower: ActorModel, followActivity: ActivityFollow) { return sequelizeTypescript.transaction(async t => { - const follower = await ActorModel.loadByUrl(actorUrl, t) - const following = await ActorModel.loadByUrl(followActivity.object, t) + const following = await ActorModel.loadByUrlAndPopulateAccountAndChannel(followActivity.object, t) const actorFollow = await ActorFollowModel.loadByActorAndTarget(follower.id, following.id, t) if (!actorFollow) throw new Error(`'Unknown actor follow ${follower.id} -> ${following.id}.`) @@ -136,11 +132,8 @@ function processUndoFollow (actorUrl: string, followActivity: ActivityFollow) { }) } -function processUndoAnnounce (actorUrl: string, announceActivity: ActivityAnnounce) { +function processUndoAnnounce (byActor: ActorModel, announceActivity: ActivityAnnounce) { return sequelizeTypescript.transaction(async t => { - const byActor = await ActorModel.loadByUrl(actorUrl, t) - if (!byActor) throw new Error('Unknown actor ' + actorUrl) - const share = await VideoShareModel.loadByUrl(announceActivity.id, t) if (!share) throw new Error(`Unknown video share ${announceActivity.id}.`) diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts index 0bceb370e..ed3489ebf 100644 --- a/server/lib/activitypub/process/process-update.ts +++ b/server/lib/activitypub/process/process-update.ts @@ -6,27 +6,30 @@ import { sequelizeTypescript } from '../../../initializers' import { AccountModel } from '../../../models/account/account' import { ActorModel } from '../../../models/activitypub/actor' import { VideoChannelModel } from '../../../models/video/video-channel' -import { fetchAvatarIfExists, getOrCreateActorAndServerAndModel, updateActorAvatarInstance, updateActorInstance } from '../actor' -import { getOrCreateVideoAndAccountAndChannel, updateVideoFromAP, getOrCreateVideoChannelFromVideoObject } from '../videos' +import { fetchAvatarIfExists, updateActorAvatarInstance, updateActorInstance } from '../actor' +import { getOrCreateVideoAndAccountAndChannel, getOrCreateVideoChannelFromVideoObject, updateVideoFromAP } from '../videos' import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-validators/activitypub/videos' import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file' import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' import { createCacheFile, updateCacheFile } from '../cache-file' -async function processUpdateActivity (activity: ActivityUpdate) { - const actor = await getOrCreateActorAndServerAndModel(activity.actor) +async function processUpdateActivity (activity: ActivityUpdate, byActor: ActorModel) { const objectType = activity.object.type if (objectType === 'Video') { - return retryTransactionWrapper(processUpdateVideo, actor, activity) + return retryTransactionWrapper(processUpdateVideo, byActor, activity) } if (objectType === 'Person' || objectType === 'Application' || objectType === 'Group') { - return retryTransactionWrapper(processUpdateActor, actor, activity) + // We need more attributes + const byActorFull = await ActorModel.loadByUrlAndPopulateAccountAndChannel(byActor.url) + return retryTransactionWrapper(processUpdateActor, byActorFull, activity) } if (objectType === 'CacheFile') { - return retryTransactionWrapper(processUpdateCacheFile, actor, activity) + // We need more attributes + const byActorFull = await ActorModel.loadByUrlAndPopulateAccountAndChannel(byActor.url) + return retryTransactionWrapper(processUpdateCacheFile, byActorFull, activity) } return undefined diff --git a/server/lib/activitypub/process/process.ts b/server/lib/activitypub/process/process.ts index da91675ce..35ad1696a 100644 --- a/server/lib/activitypub/process/process.ts +++ b/server/lib/activitypub/process/process.ts @@ -11,8 +11,9 @@ import { processLikeActivity } from './process-like' import { processRejectActivity } from './process-reject' import { processUndoActivity } from './process-undo' import { processUpdateActivity } from './process-update' +import { getOrCreateActorAndServerAndModel } from '../actor' -const processActivity: { [ P in ActivityType ]: (activity: Activity, inboxActor?: ActorModel) => Promise } = { +const processActivity: { [ P in ActivityType ]: (activity: Activity, byActor: ActorModel, inboxActor?: ActorModel) => Promise } = { Create: processCreateActivity, Update: processUpdateActivity, Delete: processDeleteActivity, @@ -25,6 +26,8 @@ const processActivity: { [ P in ActivityType ]: (activity: Activity, inboxActor? } async function processActivities (activities: Activity[], signatureActor?: ActorModel, inboxActor?: ActorModel) { + const actorsCache: { [ url: string ]: ActorModel } = {} + for (const activity of activities) { const actorUrl = getActorUrl(activity.actor) @@ -34,6 +37,9 @@ async function processActivities (activities: Activity[], signatureActor?: Actor continue } + const byActor = signatureActor || actorsCache[actorUrl] || await getOrCreateActorAndServerAndModel(actorUrl) + actorsCache[actorUrl] = byActor + const activityProcessor = processActivity[activity.type] if (activityProcessor === undefined) { logger.warn('Unknown activity type %s.', activity.type, { activityId: activity.id }) @@ -41,7 +47,7 @@ async function processActivities (activities: Activity[], signatureActor?: Actor } try { - await activityProcessor(activity, inboxActor) + await activityProcessor(activity, byActor, inboxActor) } catch (err) { logger.warn('Cannot process activity %s.', activity.type, { err }) } diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index de22e3584..91231a187 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -107,7 +107,7 @@ function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject const channel = videoObject.attributedTo.find(a => a.type === 'Group') if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url) - return getOrCreateActorAndServerAndModel(channel.id) + return getOrCreateActorAndServerAndModel(channel.id, 'all') } type SyncParam = { diff --git a/server/models/activitypub/actor.ts b/server/models/activitypub/actor.ts index 69c2eca57..f8bb59323 100644 --- a/server/models/activitypub/actor.ts +++ b/server/models/activitypub/actor.ts @@ -323,6 +323,29 @@ export class ActorModel extends Model { } static loadByUrl (url: string, transaction?: Sequelize.Transaction) { + const query = { + where: { + url + }, + transaction, + include: [ + { + attributes: [ 'id' ], + model: AccountModel.unscoped(), + required: false + }, + { + attributes: [ 'id' ], + model: VideoChannelModel.unscoped(), + required: false + } + ] + } + + return ActorModel.unscoped().findOne(query) + } + + static loadByUrlAndPopulateAccountAndChannel (url: string, transaction?: Sequelize.Transaction) { const query = { where: { url From 12ba460e9ebf4951f9c1caee8822a8ca1523563f Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 19 Sep 2018 15:47:55 +0200 Subject: [PATCH 32/44] Improve AP actor checks --- server/lib/activitypub/cache-file.ts | 4 +++ .../lib/activitypub/process/process-delete.ts | 4 +++ .../lib/activitypub/process/process-reject.ts | 6 ++-- .../lib/activitypub/process/process-undo.ts | 32 ++++++++----------- server/lib/activitypub/process/process.ts | 5 +++ 5 files changed, 30 insertions(+), 21 deletions(-) diff --git a/server/lib/activitypub/cache-file.ts b/server/lib/activitypub/cache-file.ts index 20558daf9..87f8a4162 100644 --- a/server/lib/activitypub/cache-file.ts +++ b/server/lib/activitypub/cache-file.ts @@ -31,6 +31,10 @@ function createCacheFile (cacheFileObject: CacheFileObject, video: VideoModel, b } function updateCacheFile (cacheFileObject: CacheFileObject, redundancyModel: VideoRedundancyModel, byActor: { id?: number }) { + if (redundancyModel.actorId !== byActor.id) { + throw new Error('Cannot update redundancy ' + redundancyModel.url + ' of another actor.') + } + const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, redundancyModel.VideoFile.Video, byActor) redundancyModel.set('expires', attributes.expiresOn) diff --git a/server/lib/activitypub/process/process-delete.ts b/server/lib/activitypub/process/process-delete.ts index bf2a4d114..038d8c4d3 100644 --- a/server/lib/activitypub/process/process-delete.ts +++ b/server/lib/activitypub/process/process-delete.ts @@ -94,6 +94,10 @@ function processDeleteVideoComment (byActor: ActorModel, videoComment: VideoComm logger.debug('Removing remote video comment "%s".', videoComment.url) return sequelizeTypescript.transaction(async t => { + if (videoComment.Account.id !== byActor.Account.id) { + throw new Error('Account ' + byActor.url + ' does not own video comment ' + videoComment.url) + } + await videoComment.destroy({ transaction: t }) if (videoComment.Video.isOwned()) { diff --git a/server/lib/activitypub/process/process-reject.ts b/server/lib/activitypub/process/process-reject.ts index b0e678316..709a65096 100644 --- a/server/lib/activitypub/process/process-reject.ts +++ b/server/lib/activitypub/process/process-reject.ts @@ -17,11 +17,11 @@ export { // --------------------------------------------------------------------------- -async function processReject (actor: ActorModel, targetActor: ActorModel) { +async function processReject (follower: ActorModel, targetActor: ActorModel) { return sequelizeTypescript.transaction(async t => { - const actorFollow = await ActorFollowModel.loadByActorAndTarget(actor.id, targetActor.id, t) + const actorFollow = await ActorFollowModel.loadByActorAndTarget(follower.id, targetActor.id, t) - if (!actorFollow) throw new Error(`'Unknown actor follow ${actor.id} -> ${targetActor.id}.`) + if (!actorFollow) throw new Error(`'Unknown actor follow ${follower.id} -> ${targetActor.id}.`) await actorFollow.destroy({ transaction: t }) diff --git a/server/lib/activitypub/process/process-undo.ts b/server/lib/activitypub/process/process-undo.ts index c091d9678..73ca0a17c 100644 --- a/server/lib/activitypub/process/process-undo.ts +++ b/server/lib/activitypub/process/process-undo.ts @@ -1,10 +1,8 @@ import { ActivityAnnounce, ActivityFollow, ActivityLike, ActivityUndo, CacheFileObject } from '../../../../shared/models/activitypub' import { DislikeObject } from '../../../../shared/models/activitypub/objects' -import { getActorUrl } from '../../../helpers/activitypub' import { retryTransactionWrapper } from '../../../helpers/database-utils' import { logger } from '../../../helpers/logger' import { sequelizeTypescript } from '../../../initializers' -import { AccountModel } from '../../../models/account/account' import { AccountVideoRateModel } from '../../../models/account/account-video-rate' import { ActorModel } from '../../../models/activitypub/actor' import { ActorFollowModel } from '../../../models/activitypub/actor-follow' @@ -16,15 +14,13 @@ import { VideoRedundancyModel } from '../../../models/redundancy/video-redundanc async function processUndoActivity (activity: ActivityUndo, byActor: ActorModel) { const activityToUndo = activity.object - const actorUrl = getActorUrl(activity.actor) - if (activityToUndo.type === 'Like') { - return retryTransactionWrapper(processUndoLike, actorUrl, activity) + return retryTransactionWrapper(processUndoLike, byActor, activity) } if (activityToUndo.type === 'Create') { if (activityToUndo.object.type === 'Dislike') { - return retryTransactionWrapper(processUndoDislike, actorUrl, activity) + return retryTransactionWrapper(processUndoDislike, byActor, activity) } else if (activityToUndo.object.type === 'CacheFile') { return retryTransactionWrapper(processUndoCacheFile, byActor, activity) } @@ -51,48 +47,46 @@ export { // --------------------------------------------------------------------------- -async function processUndoLike (actorUrl: string, activity: ActivityUndo) { +async function processUndoLike (byActor: ActorModel, activity: ActivityUndo) { const likeActivity = activity.object as ActivityLike const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: likeActivity.object }) return sequelizeTypescript.transaction(async t => { - const byAccount = await AccountModel.loadByUrl(actorUrl, t) - if (!byAccount) throw new Error('Unknown account ' + actorUrl) + if (!byActor.Account) throw new Error('Unknown account ' + byActor.url) - const rate = await AccountVideoRateModel.load(byAccount.id, video.id, t) - if (!rate) throw new Error(`Unknown rate by account ${byAccount.id} for video ${video.id}.`) + const rate = await AccountVideoRateModel.load(byActor.Account.id, video.id, t) + if (!rate) throw new Error(`Unknown rate by account ${byActor.Account.id} for video ${video.id}.`) await rate.destroy({ transaction: t }) await video.decrement('likes', { transaction: t }) if (video.isOwned()) { // Don't resend the activity to the sender - const exceptions = [ byAccount.Actor ] + const exceptions = [ byActor ] await forwardVideoRelatedActivity(activity, t, exceptions, video) } }) } -async function processUndoDislike (actorUrl: string, activity: ActivityUndo) { +async function processUndoDislike (byActor: ActorModel, activity: ActivityUndo) { const dislike = activity.object.object as DislikeObject const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislike.object }) return sequelizeTypescript.transaction(async t => { - const byAccount = await AccountModel.loadByUrl(actorUrl, t) - if (!byAccount) throw new Error('Unknown account ' + actorUrl) + if (!byActor.Account) throw new Error('Unknown account ' + byActor.url) - const rate = await AccountVideoRateModel.load(byAccount.id, video.id, t) - if (!rate) throw new Error(`Unknown rate by account ${byAccount.id} for video ${video.id}.`) + const rate = await AccountVideoRateModel.load(byActor.Account.id, video.id, t) + if (!rate) throw new Error(`Unknown rate by account ${byActor.Account.id} for video ${video.id}.`) await rate.destroy({ transaction: t }) await video.decrement('dislikes', { transaction: t }) if (video.isOwned()) { // Don't resend the activity to the sender - const exceptions = [ byAccount.Actor ] + const exceptions = [ byActor ] await forwardVideoRelatedActivity(activity, t, exceptions, video) } @@ -108,6 +102,8 @@ async function processUndoCacheFile (byActor: ActorModel, activity: ActivityUndo const cacheFile = await VideoRedundancyModel.loadByUrl(cacheFileObject.id) if (!cacheFile) throw new Error('Unknown video cache ' + cacheFile.url) + if (cacheFile.actorId !== byActor.id) throw new Error('Cannot delete redundancy ' + cacheFile.url + ' of another actor.') + await cacheFile.destroy() if (video.isOwned()) { diff --git a/server/lib/activitypub/process/process.ts b/server/lib/activitypub/process/process.ts index 35ad1696a..b263f1ea2 100644 --- a/server/lib/activitypub/process/process.ts +++ b/server/lib/activitypub/process/process.ts @@ -29,6 +29,11 @@ async function processActivities (activities: Activity[], signatureActor?: Actor const actorsCache: { [ url: string ]: ActorModel } = {} for (const activity of activities) { + if (!signatureActor && [ 'Create', 'Announce', 'Like' ].indexOf(activity.type) === -1) { + logger.error('Cannot process activity %s (type: %s) without the actor signature.', activity.id, activity.type) + continue + } + const actorUrl = getActorUrl(activity.actor) // When we fetch remote data, we don't have signature From d9bdd007d7a1368d2a13127ecb5c0a81a18a8c04 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 19 Sep 2018 16:12:07 +0200 Subject: [PATCH 33/44] Put config redundancy strategies in "strategies" subkey --- config/default.yaml | 21 ++++++++++--------- config/production.yaml.example | 21 ++++++++++--------- config/test.yaml | 21 ++++++++++--------- server/controllers/api/server/stats.ts | 2 +- server/initializers/checker.ts | 3 ++- server/initializers/constants.ts | 4 +++- server/lib/activitypub/actor.ts | 3 +++ .../schedulers/videos-redundancy-scheduler.ts | 6 ++---- server/models/video/tag.ts | 5 ++--- 9 files changed, 46 insertions(+), 40 deletions(-) diff --git a/config/default.yaml b/config/default.yaml index ab07bfedd..00eeaea8c 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -71,16 +71,17 @@ trending: # Once you have defined your strategies, choose which instances you want to cache in admin -> manage follows -> following redundancy: videos: -# - -# size: '10GB' -# strategy: 'most-views' # Cache videos that have the most views -# - -# size: '10GB' -# strategy: 'trending' # Cache trending videos -# - -# size: '10GB' -# strategy: 'recently-added' # Cache recently added videos -# minViews: 10 # Having at least x views + strategies: +# - +# size: '10GB' +# strategy: 'most-views' # Cache videos that have the most views +# - +# size: '10GB' +# strategy: 'trending' # Cache trending videos +# - +# size: '10GB' +# strategy: 'recently-added' # Cache recently added videos +# minViews: 10 # Having at least x views cache: previews: diff --git a/config/production.yaml.example b/config/production.yaml.example index f9557b8eb..28770e480 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example @@ -72,16 +72,17 @@ trending: # Once you have defined your strategies, choose which instances you want to cache in admin -> manage follows -> following redundancy: videos: -# - -# size: '10GB' -# strategy: 'most-views' # Cache videos that have the most views -# - -# size: '10GB' -# strategy: 'trending' # Cache trending videos -# - -# size: '10GB' -# strategy: 'recently-added' # Cache recently added videos -# minViews: 10 # Having at least x views + strategies: +# - +# size: '10GB' +# strategy: 'most-views' # Cache videos that have the most views +# - +# size: '10GB' +# strategy: 'trending' # Cache trending videos +# - +# size: '10GB' +# strategy: 'recently-added' # Cache recently added videos +# minViews: 10 # Having at least x views ############################################################################### # diff --git a/config/test.yaml b/config/test.yaml index d3e0e49ac..d36d90bbd 100644 --- a/config/test.yaml +++ b/config/test.yaml @@ -23,16 +23,17 @@ log: redundancy: videos: - - - size: '10MB' - strategy: 'most-views' - - - size: '10MB' - strategy: 'trending' - - - size: '10MB' - strategy: 'recently-added' - minViews: 1 + strategies: + - + size: '10MB' + strategy: 'most-views' + - + size: '10MB' + strategy: 'trending' + - + size: '10MB' + strategy: 'recently-added' + minViews: 1 cache: previews: diff --git a/server/controllers/api/server/stats.ts b/server/controllers/api/server/stats.ts index bb6311e81..85803f69e 100644 --- a/server/controllers/api/server/stats.ts +++ b/server/controllers/api/server/stats.ts @@ -23,7 +23,7 @@ async function getStats (req: express.Request, res: express.Response, next: expr const { totalInstanceFollowers, totalInstanceFollowing } = await ActorFollowModel.getStats() const videosRedundancyStats = await Promise.all( - CONFIG.REDUNDANCY.VIDEOS.map(r => { + CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.map(r => { return VideoRedundancyModel.getStats(r.strategy) .then(stats => Object.assign(stats, { strategy: r.strategy, totalSize: r.size })) }) diff --git a/server/initializers/checker.ts b/server/initializers/checker.ts index b9dc1e725..8b5280848 100644 --- a/server/initializers/checker.ts +++ b/server/initializers/checker.ts @@ -40,7 +40,7 @@ function checkConfig () { } // Redundancies - const redundancyVideos = config.get('redundancy.videos') + const redundancyVideos = config.get('redundancy.videos.strategies') if (isArray(redundancyVideos)) { for (const r of redundancyVideos) { if ([ 'most-views', 'trending', 'recently-added' ].indexOf(r.strategy) === -1) { @@ -75,6 +75,7 @@ function checkMissedConfig () { 'cache.previews.size', 'admin.email', 'signup.enabled', 'signup.limit', 'signup.requires_email_verification', 'signup.filters.cidr.whitelist', 'signup.filters.cidr.blacklist', + 'redundancy.videos.strategies', 'transcoding.enabled', 'transcoding.threads', 'import.videos.http.enabled', 'import.videos.torrent.enabled', 'trending.videos.interval_days', diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index fa9093918..881978753 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -212,7 +212,9 @@ const CONFIG = { } }, REDUNDANCY: { - VIDEOS: buildVideosRedundancy(config.get('redundancy.videos')) + VIDEOS: { + STRATEGIES: buildVideosRedundancy(config.get('redundancy.videos.strategies')) + } }, ADMIN: { get EMAIL () { return config.get('admin.email') } diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts index 0bdb7d12e..d37a695a7 100644 --- a/server/lib/activitypub/actor.ts +++ b/server/lib/activitypub/actor.ts @@ -78,6 +78,9 @@ async function getOrCreateActorAndServerAndModel ( created = true } + if (actor.Account) actor.Account.Actor = actor + if (actor.VideoChannel) actor.VideoChannel.Actor = actor + const { actor: actorRefreshed, refreshed } = await retryTransactionWrapper(refreshActorIfNeeded, actor, fetchType) if (!actorRefreshed) throw new Error('Actor ' + actorRefreshed.url + ' does not exist anymore.') diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index 7079600a9..5f9fd9911 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts @@ -1,10 +1,9 @@ import { AbstractScheduler } from './abstract-scheduler' import { CONFIG, JOB_TTL, REDUNDANCY, SCHEDULER_INTERVALS_MS } from '../../initializers' import { logger } from '../../helpers/logger' -import { RecentlyAddedStrategy, VideoRedundancyStrategy, VideosRedundancy } from '../../../shared/models/redundancy' +import { VideoRedundancyStrategy, VideosRedundancy } from '../../../shared/models/redundancy' import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' import { VideoFileModel } from '../../models/video/video-file' -import { sortBy } from 'lodash' import { downloadWebTorrentVideo } from '../../helpers/webtorrent' import { join } from 'path' import { rename } from 'fs-extra' @@ -12,7 +11,6 @@ import { getServerActor } from '../../helpers/utils' import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' import { VideoModel } from '../../models/video/video' import { getVideoCacheFileActivityPubUrl } from '../activitypub/url' -import { removeVideoRedundancy } from '../redundancy' import { isTestInstance } from '../../helpers/core-utils' export class VideosRedundancyScheduler extends AbstractScheduler { @@ -31,7 +29,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { this.executing = true - for (const obj of CONFIG.REDUNDANCY.VIDEOS) { + for (const obj of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) { try { const videoToDuplicate = await this.findVideoToDuplicate(obj) if (!videoToDuplicate) continue diff --git a/server/models/video/tag.ts b/server/models/video/tag.ts index e39a418cd..b39621eaf 100644 --- a/server/models/video/tag.ts +++ b/server/models/video/tag.ts @@ -48,11 +48,10 @@ export class TagModel extends Model { }, defaults: { name: tag - } + }, + transaction } - if (transaction) query['transaction'] = transaction - const promise = TagModel.findOrCreate(query) .then(([ tagInstance ]) => tagInstance) tasks.push(promise) From f9f899b9f803eb5159a67781f10649a0cf040677 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 19 Sep 2018 16:21:09 +0200 Subject: [PATCH 34/44] Add redundancy check interval in config --- config/default.yaml | 1 + config/production.yaml.example | 1 + config/test.yaml | 1 + server/initializers/checker.ts | 2 +- server/initializers/constants.ts | 7 +++---- .../schedulers/videos-redundancy-scheduler.ts | 18 +++++++++++------- 6 files changed, 18 insertions(+), 12 deletions(-) diff --git a/config/default.yaml b/config/default.yaml index 00eeaea8c..fa1fb628a 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -71,6 +71,7 @@ trending: # Once you have defined your strategies, choose which instances you want to cache in admin -> manage follows -> following redundancy: videos: + check_interval: '1 hour' # How often you want to check new videos to cache strategies: # - # size: '10GB' diff --git a/config/production.yaml.example b/config/production.yaml.example index 28770e480..4d8752206 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example @@ -72,6 +72,7 @@ trending: # Once you have defined your strategies, choose which instances you want to cache in admin -> manage follows -> following redundancy: videos: + check_interval: '1 hour' # How often you want to check new videos to cache strategies: # - # size: '10GB' diff --git a/config/test.yaml b/config/test.yaml index d36d90bbd..ad94b00cd 100644 --- a/config/test.yaml +++ b/config/test.yaml @@ -23,6 +23,7 @@ log: redundancy: videos: + check_interval: '5 seconds' strategies: - size: '10MB' diff --git a/server/initializers/checker.ts b/server/initializers/checker.ts index 8b5280848..a54f6155b 100644 --- a/server/initializers/checker.ts +++ b/server/initializers/checker.ts @@ -75,7 +75,7 @@ function checkMissedConfig () { 'cache.previews.size', 'admin.email', 'signup.enabled', 'signup.limit', 'signup.requires_email_verification', 'signup.filters.cidr.whitelist', 'signup.filters.cidr.blacklist', - 'redundancy.videos.strategies', + 'redundancy.videos.strategies', 'redundancy.videos.check_interval', 'transcoding.enabled', 'transcoding.threads', 'import.videos.http.enabled', 'import.videos.torrent.enabled', 'trending.videos.interval_days', diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 881978753..03424ffb8 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -5,7 +5,7 @@ import { ActivityPubActorType } from '../../shared/models/activitypub' import { FollowState } from '../../shared/models/actors' import { VideoAbuseState, VideoImportState, VideoPrivacy } from '../../shared/models/videos' // Do not use barrels, remain constants as independent as possible -import { buildPath, isTestInstance, root, sanitizeHost, sanitizeUrl } from '../helpers/core-utils' +import { buildPath, isTestInstance, parseDuration, root, sanitizeHost, sanitizeUrl } from '../helpers/core-utils' import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type' import { invert } from 'lodash' import { CronRepeatOptions, EveryRepeatOptions } from 'bull' @@ -139,8 +139,7 @@ let SCHEDULER_INTERVALS_MS = { badActorFollow: 60000 * 60, // 1 hour removeOldJobs: 60000 * 60, // 1 hour updateVideos: 60000, // 1 minute - youtubeDLUpdate: 60000 * 60 * 24, // 1 day - videosRedundancy: 60000 * 2 // 2 hours + youtubeDLUpdate: 60000 * 60 * 24 // 1 day } // --------------------------------------------------------------------------- @@ -213,6 +212,7 @@ const CONFIG = { }, REDUNDANCY: { VIDEOS: { + CHECK_INTERVAL: parseDuration(config.get('redundancy.videos.check_interval')), STRATEGIES: buildVideosRedundancy(config.get('redundancy.videos.strategies')) } }, @@ -651,7 +651,6 @@ if (isTestInstance() === true) { SCHEDULER_INTERVALS_MS.badActorFollow = 10000 SCHEDULER_INTERVALS_MS.removeOldJobs = 10000 SCHEDULER_INTERVALS_MS.updateVideos = 5000 - SCHEDULER_INTERVALS_MS.videosRedundancy = 5000 REPEAT_JOBS['videos-views'] = { every: 5000 } REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR = 1 diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index 5f9fd9911..960651712 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts @@ -18,7 +18,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { private static instance: AbstractScheduler private executing = false - protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.videosRedundancy + protected schedulerIntervalMs = CONFIG.REDUNDANCY.VIDEOS.CHECK_INTERVAL private constructor () { super() @@ -50,6 +50,16 @@ export class VideosRedundancyScheduler extends AbstractScheduler { } } + await this.removeExpired() + + this.executing = false + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } + + private async removeExpired () { const expired = await VideoRedundancyModel.listAllExpired() for (const m of expired) { @@ -61,12 +71,6 @@ export class VideosRedundancyScheduler extends AbstractScheduler { logger.error('Cannot remove %s video from our redundancy system.', this.buildEntryLogId(m)) } } - - this.executing = false - } - - static get Instance () { - return this.instance || (this.instance = new this()) } private findVideoToDuplicate (cache: VideosRedundancy) { From 606c946e74211c4123b16087288902226306198d Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 19 Sep 2018 16:24:24 +0200 Subject: [PATCH 35/44] Move youtubeDL upgrader in helpers/ --- server/helpers/youtube-dl.ts | 69 ++++++++++++++++++- .../schedulers/youtube-dl-update-scheduler.ts | 60 ++-------------- 2 files changed, 70 insertions(+), 59 deletions(-) diff --git a/server/helpers/youtube-dl.ts b/server/helpers/youtube-dl.ts index 6738090f3..db2bddf78 100644 --- a/server/helpers/youtube-dl.ts +++ b/server/helpers/youtube-dl.ts @@ -2,7 +2,11 @@ import { truncate } from 'lodash' import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES } from '../initializers' import { logger } from './logger' import { generateVideoTmpPath } from './utils' -import { YoutubeDlUpdateScheduler } from '../lib/schedulers/youtube-dl-update-scheduler' +import { join } from 'path' +import { root } from './core-utils' +import { ensureDir, writeFile } from 'fs-extra' +import * as request from 'request' +import { createWriteStream } from 'fs' export type YoutubeDLInfo = { name?: string @@ -40,7 +44,7 @@ function downloadYoutubeDLVideo (url: string) { return new Promise(async (res, rej) => { const youtubeDL = await safeGetYoutubeDL() - youtubeDL.exec(url, options, async (err, output) => { + youtubeDL.exec(url, options, err => { if (err) return rej(err) return res(path) @@ -48,9 +52,68 @@ function downloadYoutubeDLVideo (url: string) { }) } +// Thanks: https://github.com/przemyslawpluta/node-youtube-dl/blob/master/lib/downloader.js +// We rewrote it to avoid sync calls +async function updateYoutubeDLBinary () { + logger.info('Updating youtubeDL binary.') + + const binDirectory = join(root(), 'node_modules', 'youtube-dl', 'bin') + const bin = join(binDirectory, 'youtube-dl') + const detailsPath = join(binDirectory, 'details') + const url = 'https://yt-dl.org/downloads/latest/youtube-dl' + + await ensureDir(binDirectory) + + return new Promise(res => { + request.get(url, { followRedirect: false }, (err, result) => { + if (err) { + logger.error('Cannot update youtube-dl.', { err }) + return res() + } + + if (result.statusCode !== 302) { + logger.error('youtube-dl update error: did not get redirect for the latest version link. Status %d', result.statusCode) + return res() + } + + const url = result.headers.location + const downloadFile = request.get(url) + const newVersion = /yt-dl\.org\/downloads\/(\d{4}\.\d\d\.\d\d(\.\d)?)\/youtube-dl/.exec(url)[ 1 ] + + downloadFile.on('response', result => { + if (result.statusCode !== 200) { + logger.error('Cannot update youtube-dl: new version response is not 200, it\'s %d.', result.statusCode) + return res() + } + + downloadFile.pipe(createWriteStream(bin, { mode: 493 })) + }) + + downloadFile.on('error', err => { + logger.error('youtube-dl update error.', { err }) + return res() + }) + + downloadFile.on('end', () => { + const details = JSON.stringify({ version: newVersion, path: bin, exec: 'youtube-dl' }) + writeFile(detailsPath, details, { encoding: 'utf8' }, err => { + if (err) { + logger.error('youtube-dl update error: cannot write details.', { err }) + return res() + } + + logger.info('youtube-dl updated to version %s.', newVersion) + return res() + }) + }) + }) + }) +} + // --------------------------------------------------------------------------- export { + updateYoutubeDLBinary, downloadYoutubeDLVideo, getYoutubeDLInfo } @@ -64,7 +127,7 @@ async function safeGetYoutubeDL () { youtubeDL = require('youtube-dl') } catch (e) { // Download binary - await YoutubeDlUpdateScheduler.Instance.execute() + await updateYoutubeDLBinary() youtubeDL = require('youtube-dl') } diff --git a/server/lib/schedulers/youtube-dl-update-scheduler.ts b/server/lib/schedulers/youtube-dl-update-scheduler.ts index faadb4334..2fc8950fe 100644 --- a/server/lib/schedulers/youtube-dl-update-scheduler.ts +++ b/server/lib/schedulers/youtube-dl-update-scheduler.ts @@ -1,5 +1,4 @@ -// Thanks: https://github.com/przemyslawpluta/node-youtube-dl/blob/master/lib/downloader.js -// We rewrote it to avoid sync calls + import { AbstractScheduler } from './abstract-scheduler' import { SCHEDULER_INTERVALS_MS } from '../../initializers' @@ -8,6 +7,7 @@ import * as request from 'request' import { createWriteStream, ensureDir, writeFile } from 'fs-extra' import { join } from 'path' import { root } from '../../helpers/core-utils' +import { updateYoutubeDLBinary } from '../../helpers/youtube-dl' export class YoutubeDlUpdateScheduler extends AbstractScheduler { @@ -19,60 +19,8 @@ export class YoutubeDlUpdateScheduler extends AbstractScheduler { super() } - async execute () { - logger.info('Updating youtubeDL binary.') - - const binDirectory = join(root(), 'node_modules', 'youtube-dl', 'bin') - const bin = join(binDirectory, 'youtube-dl') - const detailsPath = join(binDirectory, 'details') - const url = 'https://yt-dl.org/downloads/latest/youtube-dl' - - await ensureDir(binDirectory) - - return new Promise(res => { - request.get(url, { followRedirect: false }, (err, result) => { - if (err) { - logger.error('Cannot update youtube-dl.', { err }) - return res() - } - - if (result.statusCode !== 302) { - logger.error('youtube-dl update error: did not get redirect for the latest version link. Status %d', result.statusCode) - return res() - } - - const url = result.headers.location - const downloadFile = request.get(url) - const newVersion = /yt-dl\.org\/downloads\/(\d{4}\.\d\d\.\d\d(\.\d)?)\/youtube-dl/.exec(url)[ 1 ] - - downloadFile.on('response', result => { - if (result.statusCode !== 200) { - logger.error('Cannot update youtube-dl: new version response is not 200, it\'s %d.', result.statusCode) - return res() - } - - downloadFile.pipe(createWriteStream(bin, { mode: 493 })) - }) - - downloadFile.on('error', err => { - logger.error('youtube-dl update error.', { err }) - return res() - }) - - downloadFile.on('end', () => { - const details = JSON.stringify({ version: newVersion, path: bin, exec: 'youtube-dl' }) - writeFile(detailsPath, details, { encoding: 'utf8' }, err => { - if (err) { - logger.error('youtube-dl update error: cannot write details.', { err }) - return res() - } - - logger.info('youtube-dl updated to version %s.', newVersion) - return res() - }) - }) - }) - }) + execute () { + return updateYoutubeDLBinary() } static get Instance () { From 993cef4b6e000ee425087e5195dfe40cd0840243 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 19 Sep 2018 17:02:16 +0200 Subject: [PATCH 36/44] Refractor audit user identifier --- server/controllers/api/config.ts | 9 +++------ server/controllers/api/users/index.ts | 17 ++++++++++------- server/controllers/api/users/me.ts | 15 ++++++++------- server/controllers/api/video-channel.ts | 13 +++++-------- server/controllers/api/videos/comment.ts | 8 ++++---- server/controllers/api/videos/import.ts | 6 +++--- server/controllers/api/videos/index.ts | 8 ++++---- server/helpers/audit-logger.ts | 8 ++++++++ server/tests/api/server/redundancy.ts | 22 +++++++++++++--------- 9 files changed, 58 insertions(+), 48 deletions(-) diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts index 6edbe4820..95549b724 100644 --- a/server/controllers/api/config.ts +++ b/server/controllers/api/config.ts @@ -8,7 +8,7 @@ import { CONFIG, CONSTRAINTS_FIELDS, reloadConfig } from '../../initializers' import { asyncMiddleware, authenticate, ensureUserHasRight } from '../../middlewares' import { customConfigUpdateValidator } from '../../middlewares/validators/config' import { ClientHtml } from '../../lib/client-html' -import { auditLoggerFactory, CustomConfigAuditView } from '../../helpers/audit-logger' +import { auditLoggerFactory, CustomConfigAuditView, getAuditIdFromRes } from '../../helpers/audit-logger' import { remove, writeJSON } from 'fs-extra' const packageJSON = require('../../../../package.json') @@ -134,10 +134,7 @@ async function getCustomConfig (req: express.Request, res: express.Response, nex async function deleteCustomConfig (req: express.Request, res: express.Response, next: express.NextFunction) { await remove(CONFIG.CUSTOM_FILE) - auditLogger.delete( - res.locals.oauth.token.User.Account.Actor.getIdentifier(), - new CustomConfigAuditView(customConfig()) - ) + auditLogger.delete(getAuditIdFromRes(res), new CustomConfigAuditView(customConfig())) reloadConfig() ClientHtml.invalidCache() @@ -183,7 +180,7 @@ async function updateCustomConfig (req: express.Request, res: express.Response, const data = customConfig() auditLogger.update( - res.locals.oauth.token.User.Account.Actor.getIdentifier(), + getAuditIdFromRes(res), new CustomConfigAuditView(data), oldCustomConfigAuditKeys ) diff --git a/server/controllers/api/users/index.ts b/server/controllers/api/users/index.ts index 07edf3727..a299167e8 100644 --- a/server/controllers/api/users/index.ts +++ b/server/controllers/api/users/index.ts @@ -27,12 +27,15 @@ import { usersUpdateValidator } from '../../../middlewares' import { - usersAskResetPasswordValidator, usersBlockingValidator, usersResetPasswordValidator, - usersAskSendVerifyEmailValidator, usersVerifyEmailValidator + usersAskResetPasswordValidator, + usersAskSendVerifyEmailValidator, + usersBlockingValidator, + usersResetPasswordValidator, + usersVerifyEmailValidator } from '../../../middlewares/validators' import { UserModel } from '../../../models/account/user' import { OAuthTokenModel } from '../../../models/oauth/oauth-token' -import { auditLoggerFactory, UserAuditView } from '../../../helpers/audit-logger' +import { auditLoggerFactory, getAuditIdFromRes, UserAuditView } from '../../../helpers/audit-logger' import { meRouter } from './me' const auditLogger = auditLoggerFactory('users') @@ -166,7 +169,7 @@ async function createUser (req: express.Request, res: express.Response) { const { user, account } = await createUserAccountAndChannel(userToCreate) - auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new UserAuditView(user.toFormattedJSON())) + auditLogger.create(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON())) logger.info('User %s with its channel and account created.', body.username) return res.json({ @@ -245,7 +248,7 @@ async function removeUser (req: express.Request, res: express.Response, next: ex await user.destroy() - auditLogger.delete(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new UserAuditView(user.toFormattedJSON())) + auditLogger.delete(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON())) return res.sendStatus(204) } @@ -269,7 +272,7 @@ async function updateUser (req: express.Request, res: express.Response, next: ex } auditLogger.update( - res.locals.oauth.token.User.Account.Actor.getIdentifier(), + getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON()), oldUserAuditView ) @@ -341,7 +344,7 @@ async function changeUserBlock (res: express.Response, user: UserModel, block: b await Emailer.Instance.addUserBlockJob(user, block, reason) auditLogger.update( - res.locals.oauth.token.User.Account.Actor.getIdentifier(), + getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON()), oldUserAuditView ) diff --git a/server/controllers/api/users/me.ts b/server/controllers/api/users/me.ts index 113563c39..d4b7e3715 100644 --- a/server/controllers/api/users/me.ts +++ b/server/controllers/api/users/me.ts @@ -5,7 +5,8 @@ import { getFormattedObjects } from '../../../helpers/utils' import { CONFIG, IMAGE_MIMETYPE_EXT, sequelizeTypescript } from '../../../initializers' import { sendUpdateActor } from '../../../lib/activitypub/send' import { - asyncMiddleware, asyncRetryTransactionMiddleware, + asyncMiddleware, + asyncRetryTransactionMiddleware, authenticate, commonVideosFiltersValidator, paginationValidator, @@ -17,11 +18,11 @@ import { usersVideoRatingValidator } from '../../../middlewares' import { + areSubscriptionsExistValidator, deleteMeValidator, userSubscriptionsSortValidator, videoImportsSortValidator, - videosSortValidator, - areSubscriptionsExistValidator + videosSortValidator } from '../../../middlewares/validators' import { AccountVideoRateModel } from '../../../models/account/account-video-rate' import { UserModel } from '../../../models/account/user' @@ -31,7 +32,7 @@ import { buildNSFWFilter, createReqFiles } from '../../../helpers/express-utils' import { UserVideoQuota } from '../../../../shared/models/users/user-video-quota.model' import { updateAvatarValidator } from '../../../middlewares/validators/avatar' import { updateActorAvatarFile } from '../../../lib/avatar' -import { auditLoggerFactory, UserAuditView } from '../../../helpers/audit-logger' +import { auditLoggerFactory, getAuditIdFromRes, UserAuditView } from '../../../helpers/audit-logger' import { VideoImportModel } from '../../../models/video/video-import' import { VideoFilter } from '../../../../shared/models/videos/video-query.type' import { ActorFollowModel } from '../../../models/activitypub/actor-follow' @@ -311,7 +312,7 @@ async function deleteMe (req: express.Request, res: express.Response) { await user.destroy() - auditLogger.delete(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new UserAuditView(user.toFormattedJSON())) + auditLogger.delete(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON())) return res.sendStatus(204) } @@ -337,7 +338,7 @@ async function updateMe (req: express.Request, res: express.Response, next: expr await sendUpdateActor(user.Account, t) auditLogger.update( - res.locals.oauth.token.User.Account.Actor.getIdentifier(), + getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON()), oldUserAuditView ) @@ -355,7 +356,7 @@ async function updateMyAvatar (req: express.Request, res: express.Response, next const avatar = await updateActorAvatarFile(avatarPhysicalFile, account.Actor, account) auditLogger.update( - res.locals.oauth.token.User.Account.Actor.getIdentifier(), + getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON()), oldUserAuditView ) diff --git a/server/controllers/api/video-channel.ts b/server/controllers/api/video-channel.ts index a7a36080b..50dc44f7c 100644 --- a/server/controllers/api/video-channel.ts +++ b/server/controllers/api/video-channel.ts @@ -27,7 +27,7 @@ import { logger } from '../../helpers/logger' import { VideoModel } from '../../models/video/video' import { updateAvatarValidator } from '../../middlewares/validators/avatar' import { updateActorAvatarFile } from '../../lib/avatar' -import { auditLoggerFactory, VideoChannelAuditView } from '../../helpers/audit-logger' +import { auditLoggerFactory, getAuditIdFromRes, VideoChannelAuditView } from '../../helpers/audit-logger' import { resetSequelizeInstance } from '../../helpers/database-utils' const auditLogger = auditLoggerFactory('channels') @@ -109,7 +109,7 @@ async function updateVideoChannelAvatar (req: express.Request, res: express.Resp const avatar = await updateActorAvatarFile(avatarPhysicalFile, videoChannel.Actor, videoChannel) auditLogger.update( - res.locals.oauth.token.User.Account.Actor.getIdentifier(), + getAuditIdFromRes(res), new VideoChannelAuditView(videoChannel.toFormattedJSON()), oldVideoChannelAuditKeys ) @@ -133,7 +133,7 @@ async function addVideoChannel (req: express.Request, res: express.Response) { .catch(err => logger.error('Cannot set async actor keys for account %s.', videoChannelCreated.Actor.uuid, { err })) auditLogger.create( - res.locals.oauth.token.User.Account.Actor.getIdentifier(), + getAuditIdFromRes(res), new VideoChannelAuditView(videoChannelCreated.toFormattedJSON()) ) logger.info('Video channel with uuid %s created.', videoChannelCreated.Actor.uuid) @@ -166,7 +166,7 @@ async function updateVideoChannel (req: express.Request, res: express.Response) await sendUpdateActor(videoChannelInstanceUpdated, t) auditLogger.update( - res.locals.oauth.token.User.Account.Actor.getIdentifier(), + getAuditIdFromRes(res), new VideoChannelAuditView(videoChannelInstanceUpdated.toFormattedJSON()), oldVideoChannelAuditKeys ) @@ -192,10 +192,7 @@ async function removeVideoChannel (req: express.Request, res: express.Response) await sequelizeTypescript.transaction(async t => { await videoChannelInstance.destroy({ transaction: t }) - auditLogger.delete( - res.locals.oauth.token.User.Account.Actor.getIdentifier(), - new VideoChannelAuditView(videoChannelInstance.toFormattedJSON()) - ) + auditLogger.delete(getAuditIdFromRes(res), new VideoChannelAuditView(videoChannelInstance.toFormattedJSON())) logger.info('Video channel with name %s and uuid %s deleted.', videoChannelInstance.name, videoChannelInstance.Actor.uuid) }) diff --git a/server/controllers/api/videos/comment.ts b/server/controllers/api/videos/comment.ts index 8d0692b2b..40ad54d09 100644 --- a/server/controllers/api/videos/comment.ts +++ b/server/controllers/api/videos/comment.ts @@ -23,7 +23,7 @@ import { } from '../../../middlewares/validators/video-comments' import { VideoModel } from '../../../models/video/video' import { VideoCommentModel } from '../../../models/video/video-comment' -import { auditLoggerFactory, CommentAuditView } from '../../../helpers/audit-logger' +import { auditLoggerFactory, CommentAuditView, getAuditIdFromRes } from '../../../helpers/audit-logger' const auditLogger = auditLoggerFactory('comments') const videoCommentRouter = express.Router() @@ -109,7 +109,7 @@ async function addVideoCommentThread (req: express.Request, res: express.Respons }, t) }) - auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new CommentAuditView(comment.toFormattedJSON())) + auditLogger.create(getAuditIdFromRes(res), new CommentAuditView(comment.toFormattedJSON())) return res.json({ comment: comment.toFormattedJSON() @@ -128,7 +128,7 @@ async function addVideoCommentReply (req: express.Request, res: express.Response }, t) }) - auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new CommentAuditView(comment.toFormattedJSON())) + auditLogger.create(getAuditIdFromRes(res), new CommentAuditView(comment.toFormattedJSON())) return res.json({ comment: comment.toFormattedJSON() @@ -143,7 +143,7 @@ async function removeVideoComment (req: express.Request, res: express.Response) }) auditLogger.delete( - res.locals.oauth.token.User.Account.Actor.getIdentifier(), + getAuditIdFromRes(res), new CommentAuditView(videoCommentInstance.toFormattedJSON()) ) logger.info('Video comment %d deleted.', videoCommentInstance.id) diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts index 44f15ef74..398fd5a7f 100644 --- a/server/controllers/api/videos/import.ts +++ b/server/controllers/api/videos/import.ts @@ -1,7 +1,7 @@ import * as express from 'express' import * as magnetUtil from 'magnet-uri' import 'multer' -import { auditLoggerFactory, VideoImportAuditView } from '../../../helpers/audit-logger' +import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger' import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoImportAddValidator } from '../../../middlewares' import { CONFIG, @@ -114,7 +114,7 @@ async function addTorrentImport (req: express.Request, res: express.Response, to } await JobQueue.Instance.createJob({ type: 'video-import', payload }) - auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new VideoImportAuditView(videoImport.toFormattedJSON())) + auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON())) return res.json(videoImport.toFormattedJSON()).end() } @@ -158,7 +158,7 @@ async function addYoutubeDLImport (req: express.Request, res: express.Response) } await JobQueue.Instance.createJob({ type: 'video-import', payload }) - auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new VideoImportAuditView(videoImport.toFormattedJSON())) + auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON())) return res.json(videoImport.toFormattedJSON()).end() } diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts index 8353a649a..581046782 100644 --- a/server/controllers/api/videos/index.ts +++ b/server/controllers/api/videos/index.ts @@ -4,7 +4,7 @@ import { VideoCreate, VideoPrivacy, VideoState, VideoUpdate } from '../../../../ import { getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils' import { processImage } from '../../../helpers/image-utils' import { logger } from '../../../helpers/logger' -import { auditLoggerFactory, VideoAuditView } from '../../../helpers/audit-logger' +import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger' import { getFormattedObjects, getServerActor } from '../../../helpers/utils' import { CONFIG, @@ -253,7 +253,7 @@ async function addVideo (req: express.Request, res: express.Response) { await federateVideoIfNeeded(video, true, t) - auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new VideoAuditView(videoCreated.toFormattedDetailsJSON())) + auditLogger.create(getAuditIdFromRes(res), new VideoAuditView(videoCreated.toFormattedDetailsJSON())) logger.info('Video with name %s and uuid %s created.', videoInfo.name, videoCreated.uuid) return videoCreated @@ -354,7 +354,7 @@ async function updateVideo (req: express.Request, res: express.Response) { await federateVideoIfNeeded(videoInstanceUpdated, isNewVideo, t) auditLogger.update( - res.locals.oauth.token.User.Account.Actor.getIdentifier(), + getAuditIdFromRes(res), new VideoAuditView(videoInstanceUpdated.toFormattedDetailsJSON()), oldVideoAuditView ) @@ -439,7 +439,7 @@ async function removeVideo (req: express.Request, res: express.Response) { await videoInstance.destroy({ transaction: t }) }) - auditLogger.delete(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new VideoAuditView(videoInstance.toFormattedDetailsJSON())) + auditLogger.delete(getAuditIdFromRes(res), new VideoAuditView(videoInstance.toFormattedDetailsJSON())) logger.info('Video with name %s and uuid %s deleted.', videoInstance.name, videoInstance.uuid) return res.type('json').status(204).end() diff --git a/server/helpers/audit-logger.ts b/server/helpers/audit-logger.ts index 7db72b69c..00311fce1 100644 --- a/server/helpers/audit-logger.ts +++ b/server/helpers/audit-logger.ts @@ -1,4 +1,5 @@ import * as path from 'path' +import * as express from 'express' import { diff } from 'deep-object-diff' import { chain } from 'lodash' import * as flatten from 'flat' @@ -8,6 +9,11 @@ import { jsonLoggerFormat, labelFormatter } from './logger' import { VideoDetails, User, VideoChannel, VideoAbuse, VideoImport } from '../../shared' import { VideoComment } from '../../shared/models/videos/video-comment.model' import { CustomConfig } from '../../shared/models/server/custom-config.model' +import { UserModel } from '../models/account/user' + +function getAuditIdFromRes (res: express.Response) { + return (res.locals.oauth.token.User as UserModel).username +} enum AUDIT_TYPE { CREATE = 'create', @@ -255,6 +261,8 @@ class CustomConfigAuditView extends EntityAuditView { } export { + getAuditIdFromRes, + auditLoggerFactory, VideoImportAuditView, VideoChannelAuditView, diff --git a/server/tests/api/server/redundancy.ts b/server/tests/api/server/redundancy.ts index c0ab251e6..6ce4b9dd1 100644 --- a/server/tests/api/server/redundancy.ts +++ b/server/tests/api/server/redundancy.ts @@ -6,15 +6,16 @@ import { VideoDetails } from '../../../../shared/models/videos' import { doubleFollow, flushAndRunMultipleServers, - flushTests, getFollowingListPaginationAndSort, getVideo, + immutableAssign, killallServers, + root, ServerInfo, setAccessTokensToServers, uploadVideo, - wait, - root, viewVideo, immutableAssign + viewVideo, + wait } from '../../utils' import { waitJobs } from '../../utils/server/jobs' import * as magnetUtil from 'magnet-uri' @@ -44,12 +45,15 @@ function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: numbe async function runServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) { const config = { redundancy: { - videos: [ - immutableAssign({ - strategy: strategy, - size: '100KB' - }, additionalParams) - ] + videos: { + check_interval: '5 seconds', + strategies: [ + immutableAssign({ + strategy: strategy, + size: '100KB' + }, additionalParams) + ] + } } } servers = await flushAndRunMultipleServers(3, config) From f41d6aacdf3b67e0c8d4e7a599b331d90aa607b7 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 19 Sep 2018 17:42:16 +0200 Subject: [PATCH 37/44] Fix jobs tests --- server/helpers/custom-validators/video-ownership.ts | 2 +- server/tests/api/server/jobs.ts | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/server/helpers/custom-validators/video-ownership.ts b/server/helpers/custom-validators/video-ownership.ts index aaa0c736b..a7771e07b 100644 --- a/server/helpers/custom-validators/video-ownership.ts +++ b/server/helpers/custom-validators/video-ownership.ts @@ -31,7 +31,7 @@ export function checkUserCanTerminateOwnershipChange ( videoChangeOwnership: VideoChangeOwnershipModel, res: Response ): boolean { - if (videoChangeOwnership.NextOwner.userId === user.Account.userId) { + if (videoChangeOwnership.NextOwner.userId === user.id) { return true } diff --git a/server/tests/api/server/jobs.ts b/server/tests/api/server/jobs.ts index b2922c5da..f5a19c5ea 100644 --- a/server/tests/api/server/jobs.ts +++ b/server/tests/api/server/jobs.ts @@ -45,7 +45,9 @@ describe('Test jobs', function () { expect(res.body.total).to.be.above(2) expect(res.body.data).to.have.lengthOf(1) - const job = res.body.data[0] + let job = res.body.data[0] + // Skip repeat jobs + if (job.type === 'videos-views') job = res.body.data[1] expect(job.state).to.equal('completed') expect(job.type).to.equal('activitypub-follow') From fcc7c060374c3a547257d96af847352c14d6144b Mon Sep 17 00:00:00 2001 From: BO41 Date: Wed, 19 Sep 2018 18:27:10 +0200 Subject: [PATCH 38/44] rename manifest --- client/angular.json | 4 ++-- client/src/index.html | 2 +- client/src/{manifest.json => manifest.webmanifest} | 2 +- server/controllers/client.ts | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) rename client/src/{manifest.json => manifest.webmanifest} (99%) diff --git a/client/angular.json b/client/angular.json index 789eeb3d0..2cf2ecd62 100644 --- a/client/angular.json +++ b/client/angular.json @@ -24,7 +24,7 @@ }, "assets": [ "src/assets/images", - "src/manifest.json" + "src/manifest.webmanifest" ], "styles": [ "src/sass/application.scss" @@ -105,7 +105,7 @@ ], "assets": [ "src/assets/images", - "src/manifest.json" + "src/manifest.webmanifest" ] } }, diff --git a/client/src/index.html b/client/src/index.html index f00af8bff..593de4ac6 100644 --- a/client/src/index.html +++ b/client/src/index.html @@ -7,7 +7,7 @@ - + diff --git a/client/src/manifest.json b/client/src/manifest.webmanifest similarity index 99% rename from client/src/manifest.json rename to client/src/manifest.webmanifest index 30914e35f..3d3c7d6d5 100644 --- a/client/src/manifest.json +++ b/client/src/manifest.webmanifest @@ -24,7 +24,7 @@ "src": "/client/assets/images/icons/icon-96x96.png", "sizes": "96x96", "type": "image/png" - }, + }, { "src": "/client/assets/images/icons/icon-144x144.png", "sizes": "144x144", diff --git a/server/controllers/client.ts b/server/controllers/client.ts index c33061289..73b40cf65 100644 --- a/server/controllers/client.ts +++ b/server/controllers/client.ts @@ -35,7 +35,7 @@ clientsRouter.use('' + // Static HTML/CSS/JS client files const staticClientFiles = [ - 'manifest.json', + 'manifest.webmanifest', 'ngsw-worker.js', 'ngsw.json' ] From 91411dba928678c15a5e99d9795ae061909e397d Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 20 Sep 2018 10:13:13 +0200 Subject: [PATCH 39/44] Limit associations fetch when loading token --- server/controllers/api/users/index.ts | 16 ++--------- server/controllers/api/users/me.ts | 28 ++++++++----------- server/controllers/api/video-channel.ts | 9 +++--- server/controllers/api/videos/abuse.ts | 23 +++++++-------- server/controllers/api/videos/comment.ts | 14 ++++++---- server/controllers/api/videos/ownership.ts | 13 +++++---- server/controllers/api/videos/rate.ts | 11 ++++---- server/lib/activitypub/videos.ts | 4 +-- .../schedulers/youtube-dl-update-scheduler.ts | 7 ----- server/models/account/account.ts | 4 +-- server/models/oauth/oauth-token.ts | 26 +++++++++++------ 11 files changed, 75 insertions(+), 80 deletions(-) diff --git a/server/controllers/api/users/index.ts b/server/controllers/api/users/index.ts index a299167e8..d1163900b 100644 --- a/server/controllers/api/users/index.ts +++ b/server/controllers/api/users/index.ts @@ -267,15 +267,9 @@ async function updateUser (req: express.Request, res: express.Response, next: ex const user = await userToUpdate.save() // Destroy user token to refresh rights - if (roleChanged) { - await OAuthTokenModel.deleteUserToken(userToUpdate.id) - } + if (roleChanged) await OAuthTokenModel.deleteUserToken(userToUpdate.id) - auditLogger.update( - getAuditIdFromRes(res), - new UserAuditView(user.toFormattedJSON()), - oldUserAuditView - ) + auditLogger.update(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON()), oldUserAuditView) // Don't need to send this update to followers, these attributes are not propagated @@ -343,9 +337,5 @@ async function changeUserBlock (res: express.Response, user: UserModel, block: b await Emailer.Instance.addUserBlockJob(user, block, reason) - auditLogger.update( - getAuditIdFromRes(res), - new UserAuditView(user.toFormattedJSON()), - oldUserAuditView - ) + auditLogger.update(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON()), oldUserAuditView) } diff --git a/server/controllers/api/users/me.ts b/server/controllers/api/users/me.ts index d4b7e3715..eba1e7edd 100644 --- a/server/controllers/api/users/me.ts +++ b/server/controllers/api/users/me.ts @@ -38,6 +38,7 @@ import { VideoFilter } from '../../../../shared/models/videos/video-query.type' import { ActorFollowModel } from '../../../models/activitypub/actor-follow' import { JobQueue } from '../../../lib/job-queue' import { logger } from '../../../helpers/logger' +import { AccountModel } from '../../../models/account/account' const auditLogger = auditLoggerFactory('users-me') @@ -329,19 +330,17 @@ async function updateMe (req: express.Request, res: express.Response, next: expr if (body.autoPlayVideo !== undefined) user.autoPlayVideo = body.autoPlayVideo await sequelizeTypescript.transaction(async t => { + const userAccount = await AccountModel.load(user.Account.id) + await user.save({ transaction: t }) - if (body.displayName !== undefined) user.Account.name = body.displayName - if (body.description !== undefined) user.Account.description = body.description - await user.Account.save({ transaction: t }) + if (body.displayName !== undefined) userAccount.name = body.displayName + if (body.description !== undefined) userAccount.description = body.description + await userAccount.save({ transaction: t }) - await sendUpdateActor(user.Account, t) + await sendUpdateActor(userAccount, t) - auditLogger.update( - getAuditIdFromRes(res), - new UserAuditView(user.toFormattedJSON()), - oldUserAuditView - ) + auditLogger.update(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON()), oldUserAuditView) }) return res.sendStatus(204) @@ -351,15 +350,12 @@ async function updateMyAvatar (req: express.Request, res: express.Response, next const avatarPhysicalFile = req.files[ 'avatarfile' ][ 0 ] const user: UserModel = res.locals.oauth.token.user const oldUserAuditView = new UserAuditView(user.toFormattedJSON()) - const account = user.Account - const avatar = await updateActorAvatarFile(avatarPhysicalFile, account.Actor, account) + const userAccount = await AccountModel.load(user.Account.id) - auditLogger.update( - getAuditIdFromRes(res), - new UserAuditView(user.toFormattedJSON()), - oldUserAuditView - ) + const avatar = await updateActorAvatarFile(avatarPhysicalFile, userAccount.Actor, userAccount) + + auditLogger.update(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON()), oldUserAuditView) return res.json({ avatar: avatar.toFormattedJSON() }) } diff --git a/server/controllers/api/video-channel.ts b/server/controllers/api/video-channel.ts index 50dc44f7c..8fc340224 100644 --- a/server/controllers/api/video-channel.ts +++ b/server/controllers/api/video-channel.ts @@ -29,6 +29,7 @@ import { updateAvatarValidator } from '../../middlewares/validators/avatar' import { updateActorAvatarFile } from '../../lib/avatar' import { auditLoggerFactory, getAuditIdFromRes, VideoChannelAuditView } from '../../helpers/audit-logger' import { resetSequelizeInstance } from '../../helpers/database-utils' +import { UserModel } from '../../models/account/user' const auditLogger = auditLoggerFactory('channels') const reqAvatarFile = createReqFiles([ 'avatarfile' ], IMAGE_MIMETYPE_EXT, { avatarfile: CONFIG.STORAGE.AVATARS_DIR }) @@ -123,19 +124,17 @@ async function updateVideoChannelAvatar (req: express.Request, res: express.Resp async function addVideoChannel (req: express.Request, res: express.Response) { const videoChannelInfo: VideoChannelCreate = req.body - const account: AccountModel = res.locals.oauth.token.User.Account const videoChannelCreated: VideoChannelModel = await sequelizeTypescript.transaction(async t => { + const account = await AccountModel.load((res.locals.oauth.token.User as UserModel).Account.id, t) + return createVideoChannel(videoChannelInfo, account, t) }) setAsyncActorKeys(videoChannelCreated.Actor) .catch(err => logger.error('Cannot set async actor keys for account %s.', videoChannelCreated.Actor.uuid, { err })) - auditLogger.create( - getAuditIdFromRes(res), - new VideoChannelAuditView(videoChannelCreated.toFormattedJSON()) - ) + auditLogger.create(getAuditIdFromRes(res), new VideoChannelAuditView(videoChannelCreated.toFormattedJSON())) logger.info('Video channel with uuid %s created.', videoChannelCreated.Actor.uuid) return res.json({ diff --git a/server/controllers/api/videos/abuse.ts b/server/controllers/api/videos/abuse.ts index 08e11b00b..d0c81804b 100644 --- a/server/controllers/api/videos/abuse.ts +++ b/server/controllers/api/videos/abuse.ts @@ -21,6 +21,7 @@ import { AccountModel } from '../../../models/account/account' import { VideoModel } from '../../../models/video/video' import { VideoAbuseModel } from '../../../models/video/video-abuse' import { auditLoggerFactory, VideoAbuseAuditView } from '../../../helpers/audit-logger' +import { UserModel } from '../../../models/account/user' const auditLogger = auditLoggerFactory('abuse') const abuseVideoRouter = express.Router() @@ -95,17 +96,18 @@ async function deleteVideoAbuse (req: express.Request, res: express.Response) { async function reportVideoAbuse (req: express.Request, res: express.Response) { const videoInstance = res.locals.video as VideoModel - const reporterAccount = res.locals.oauth.token.User.Account as AccountModel const body: VideoAbuseCreate = req.body - const abuseToCreate = { - reporterAccountId: reporterAccount.id, - reason: body.reason, - videoId: videoInstance.id, - state: VideoAbuseState.PENDING - } - const videoAbuse: VideoAbuseModel = await sequelizeTypescript.transaction(async t => { + const reporterAccount = await AccountModel.load((res.locals.oauth.token.User as UserModel).Account.id, t) + + const abuseToCreate = { + reporterAccountId: reporterAccount.id, + reason: body.reason, + videoId: videoInstance.id, + state: VideoAbuseState.PENDING + } + const videoAbuseInstance = await VideoAbuseModel.create(abuseToCreate, { transaction: t }) videoAbuseInstance.Video = videoInstance videoAbuseInstance.Account = reporterAccount @@ -121,7 +123,6 @@ async function reportVideoAbuse (req: express.Request, res: express.Response) { }) logger.info('Abuse report for video %s created.', videoInstance.name) - return res.json({ - videoAbuse: videoAbuse.toFormattedJSON() - }).end() + + return res.json({ videoAbuse: videoAbuse.toFormattedJSON() }).end() } diff --git a/server/controllers/api/videos/comment.ts b/server/controllers/api/videos/comment.ts index 40ad54d09..dc25e1e85 100644 --- a/server/controllers/api/videos/comment.ts +++ b/server/controllers/api/videos/comment.ts @@ -24,6 +24,8 @@ import { import { VideoModel } from '../../../models/video/video' import { VideoCommentModel } from '../../../models/video/video-comment' import { auditLoggerFactory, CommentAuditView, getAuditIdFromRes } from '../../../helpers/audit-logger' +import { AccountModel } from '../../../models/account/account' +import { UserModel } from '../../../models/account/user' const auditLogger = auditLoggerFactory('comments') const videoCommentRouter = express.Router() @@ -101,11 +103,13 @@ async function addVideoCommentThread (req: express.Request, res: express.Respons const videoCommentInfo: VideoCommentCreate = req.body const comment = await sequelizeTypescript.transaction(async t => { + const account = await AccountModel.load((res.locals.oauth.token.User as UserModel).Account.id, t) + return createVideoComment({ text: videoCommentInfo.text, inReplyToComment: null, video: res.locals.video, - account: res.locals.oauth.token.User.Account + account }, t) }) @@ -120,19 +124,19 @@ async function addVideoCommentReply (req: express.Request, res: express.Response const videoCommentInfo: VideoCommentCreate = req.body const comment = await sequelizeTypescript.transaction(async t => { + const account = await AccountModel.load((res.locals.oauth.token.User as UserModel).Account.id, t) + return createVideoComment({ text: videoCommentInfo.text, inReplyToComment: res.locals.videoComment, video: res.locals.video, - account: res.locals.oauth.token.User.Account + account }, t) }) auditLogger.create(getAuditIdFromRes(res), new CommentAuditView(comment.toFormattedJSON())) - return res.json({ - comment: comment.toFormattedJSON() - }).end() + return res.json({ comment: comment.toFormattedJSON() }).end() } async function removeVideoComment (req: express.Request, res: express.Response) { diff --git a/server/controllers/api/videos/ownership.ts b/server/controllers/api/videos/ownership.ts index d26ed6cfc..5ea7d7c6a 100644 --- a/server/controllers/api/videos/ownership.ts +++ b/server/controllers/api/videos/ownership.ts @@ -19,6 +19,7 @@ import { VideoChannelModel } from '../../../models/video/video-channel' import { getFormattedObjects } from '../../../helpers/utils' import { changeVideoChannelShare } from '../../../lib/activitypub' import { sendUpdateVideo } from '../../../lib/activitypub/send' +import { UserModel } from '../../../models/account/user' const ownershipVideoRouter = express.Router() @@ -58,26 +59,25 @@ export { async function giveVideoOwnership (req: express.Request, res: express.Response) { const videoInstance = res.locals.video as VideoModel - const initiatorAccount = res.locals.oauth.token.User.Account as AccountModel + const initiatorAccountId = (res.locals.oauth.token.User as UserModel).Account.id const nextOwner = res.locals.nextOwner as AccountModel await sequelizeTypescript.transaction(t => { return VideoChangeOwnershipModel.findOrCreate({ where: { - initiatorAccountId: initiatorAccount.id, + initiatorAccountId, nextOwnerAccountId: nextOwner.id, videoId: videoInstance.id, status: VideoChangeOwnershipStatus.WAITING }, defaults: { - initiatorAccountId: initiatorAccount.id, + initiatorAccountId, nextOwnerAccountId: nextOwner.id, videoId: videoInstance.id, status: VideoChangeOwnershipStatus.WAITING }, transaction: t }) - }) logger.info('Ownership change for video %s created.', videoInstance.name) @@ -85,9 +85,10 @@ async function giveVideoOwnership (req: express.Request, res: express.Response) } async function listVideoOwnership (req: express.Request, res: express.Response) { - const currentAccount = res.locals.oauth.token.User.Account as AccountModel + const currentAccountId = (res.locals.oauth.token.User as UserModel).Account.id + const resultList = await VideoChangeOwnershipModel.listForApi( - currentAccount.id, + currentAccountId, req.query.start || 0, req.query.count || 10, req.query.sort || 'createdAt' diff --git a/server/controllers/api/videos/rate.ts b/server/controllers/api/videos/rate.ts index b1732837d..dc322bb0c 100644 --- a/server/controllers/api/videos/rate.ts +++ b/server/controllers/api/videos/rate.ts @@ -28,10 +28,11 @@ async function rateVideo (req: express.Request, res: express.Response) { const body: UserVideoRateUpdate = req.body const rateType = body.rating const videoInstance: VideoModel = res.locals.video - const accountInstance: AccountModel = res.locals.oauth.token.User.Account await sequelizeTypescript.transaction(async t => { const sequelizeOptions = { transaction: t } + + const accountInstance = await AccountModel.load(res.locals.oauth.token.User.Account.id, t) const previousRate = await AccountVideoRateModel.load(accountInstance.id, videoInstance.id, t) let likesToIncrement = 0 @@ -47,10 +48,10 @@ async function rateVideo (req: express.Request, res: express.Response) { else if (previousRate.type === VIDEO_RATE_TYPES.DISLIKE) dislikesToIncrement-- if (rateType === 'none') { // Destroy previous rate - await previousRate.destroy({ transaction: t }) + await previousRate.destroy(sequelizeOptions) } else { // Update previous rate previousRate.type = rateType - await previousRate.save({ transaction: t }) + await previousRate.save(sequelizeOptions) } } else if (rateType !== 'none') { // There was not a previous rate, insert a new one if there is a rate const query = { @@ -70,9 +71,9 @@ async function rateVideo (req: express.Request, res: express.Response) { await videoInstance.increment(incrementQuery, sequelizeOptions) await sendVideoRateChange(accountInstance, videoInstance, likesToIncrement, dislikesToIncrement, t) - }) - logger.info('Account video rate for video %s of account %s updated.', videoInstance.name, accountInstance.name) + logger.info('Account video rate for video %s of account %s updated.', videoInstance.name, accountInstance.name) + }) return res.type('json').status(204).end() } diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index 91231a187..48c0e0a5c 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -354,11 +354,11 @@ async function refreshVideoIfNeeded (options: { syncParam: SyncParam, refreshViews: boolean }): Promise { + if (!options.video.isOutdated()) return options.video + // We need more attributes if the argument video was fetched with not enough joints const video = options.fetchedType === 'all' ? options.video : await VideoModel.loadByUrlAndPopulateAccount(options.video.url) - if (!video.isOutdated()) return video - try { const { response, videoObject } = await fetchRemoteVideo(video.url) if (response.statusCode === 404) { diff --git a/server/lib/schedulers/youtube-dl-update-scheduler.ts b/server/lib/schedulers/youtube-dl-update-scheduler.ts index 2fc8950fe..461cd045e 100644 --- a/server/lib/schedulers/youtube-dl-update-scheduler.ts +++ b/server/lib/schedulers/youtube-dl-update-scheduler.ts @@ -1,12 +1,5 @@ - - import { AbstractScheduler } from './abstract-scheduler' import { SCHEDULER_INTERVALS_MS } from '../../initializers' -import { logger } from '../../helpers/logger' -import * as request from 'request' -import { createWriteStream, ensureDir, writeFile } from 'fs-extra' -import { join } from 'path' -import { root } from '../../helpers/core-utils' import { updateYoutubeDLBinary } from '../../helpers/youtube-dl' export class YoutubeDlUpdateScheduler extends AbstractScheduler { diff --git a/server/models/account/account.ts b/server/models/account/account.ts index 6bbfc6f4e..580d920ce 100644 --- a/server/models/account/account.ts +++ b/server/models/account/account.ts @@ -134,8 +134,8 @@ export class AccountModel extends Model { return undefined } - static load (id: number) { - return AccountModel.findById(id) + static load (id: number, transaction?: Sequelize.Transaction) { + return AccountModel.findById(id, { transaction }) } static loadByUUID (uuid: string) { diff --git a/server/models/oauth/oauth-token.ts b/server/models/oauth/oauth-token.ts index 4c53848dc..1dd5e0289 100644 --- a/server/models/oauth/oauth-token.ts +++ b/server/models/oauth/oauth-token.ts @@ -1,9 +1,10 @@ import { AllowNull, BelongsTo, Column, CreatedAt, ForeignKey, Model, Scopes, Table, UpdatedAt } from 'sequelize-typescript' import { logger } from '../../helpers/logger' -import { AccountModel } from '../account/account' import { UserModel } from '../account/user' import { OAuthClientModel } from './oauth-client' import { Transaction } from 'sequelize' +import { AccountModel } from '../account/account' +import { ActorModel } from '../activitypub/actor' export type OAuthTokenInfo = { refreshToken: string @@ -17,18 +18,27 @@ export type OAuthTokenInfo = { } enum ScopeNames { - WITH_ACCOUNT = 'WITH_ACCOUNT' + WITH_USER = 'WITH_USER' } @Scopes({ - [ScopeNames.WITH_ACCOUNT]: { + [ScopeNames.WITH_USER]: { include: [ { - model: () => UserModel, + model: () => UserModel.unscoped(), + required: true, include: [ { - model: () => AccountModel, - required: true + attributes: [ 'id' ], + model: () => AccountModel.unscoped(), + required: true, + include: [ + { + attributes: [ 'id' ], + model: () => ActorModel.unscoped(), + required: true + } + ] } ] } @@ -138,7 +148,7 @@ export class OAuthTokenModel extends Model { } } - return OAuthTokenModel.scope(ScopeNames.WITH_ACCOUNT).findOne(query).then(token => { + return OAuthTokenModel.scope(ScopeNames.WITH_USER).findOne(query).then(token => { if (token) token['user'] = token.User return token @@ -152,7 +162,7 @@ export class OAuthTokenModel extends Model { } } - return OAuthTokenModel.scope(ScopeNames.WITH_ACCOUNT) + return OAuthTokenModel.scope(ScopeNames.WITH_USER) .findOne(query) .then(token => { if (token) { From f201a749929ec8094a7ba6bcab7b196870ca5a5e Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 20 Sep 2018 11:31:48 +0200 Subject: [PATCH 40/44] Cache user token --- server/controllers/api/users/index.ts | 5 ++-- server/controllers/api/users/me.ts | 2 +- server/controllers/api/video-channel.ts | 10 ++----- server/lib/avatar.ts | 11 ++----- server/lib/oauth-model.ts | 40 +++++++++++++++++++++++++ server/models/account/user.ts | 9 ++++++ server/models/oauth/oauth-token.ts | 21 ++++++++++++- 7 files changed, 79 insertions(+), 19 deletions(-) diff --git a/server/controllers/api/users/index.ts b/server/controllers/api/users/index.ts index d1163900b..8b8ebcd23 100644 --- a/server/controllers/api/users/index.ts +++ b/server/controllers/api/users/index.ts @@ -37,6 +37,7 @@ import { UserModel } from '../../../models/account/user' import { OAuthTokenModel } from '../../../models/oauth/oauth-token' import { auditLoggerFactory, getAuditIdFromRes, UserAuditView } from '../../../helpers/audit-logger' import { meRouter } from './me' +import { deleteUserToken } from '../../../lib/oauth-model' const auditLogger = auditLoggerFactory('users') @@ -267,7 +268,7 @@ async function updateUser (req: express.Request, res: express.Response, next: ex const user = await userToUpdate.save() // Destroy user token to refresh rights - if (roleChanged) await OAuthTokenModel.deleteUserToken(userToUpdate.id) + if (roleChanged) await deleteUserToken(userToUpdate.id) auditLogger.update(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON()), oldUserAuditView) @@ -330,7 +331,7 @@ async function changeUserBlock (res: express.Response, user: UserModel, block: b user.blockedReason = reason || null await sequelizeTypescript.transaction(async t => { - await OAuthTokenModel.deleteUserToken(user.id, t) + await deleteUserToken(user.id, t) await user.save({ transaction: t }) }) diff --git a/server/controllers/api/users/me.ts b/server/controllers/api/users/me.ts index eba1e7edd..ff3a87b7f 100644 --- a/server/controllers/api/users/me.ts +++ b/server/controllers/api/users/me.ts @@ -353,7 +353,7 @@ async function updateMyAvatar (req: express.Request, res: express.Response, next const userAccount = await AccountModel.load(user.Account.id) - const avatar = await updateActorAvatarFile(avatarPhysicalFile, userAccount.Actor, userAccount) + const avatar = await updateActorAvatarFile(avatarPhysicalFile, userAccount) auditLogger.update(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON()), oldUserAuditView) diff --git a/server/controllers/api/video-channel.ts b/server/controllers/api/video-channel.ts index 8fc340224..ff6bbe44c 100644 --- a/server/controllers/api/video-channel.ts +++ b/server/controllers/api/video-channel.ts @@ -56,7 +56,7 @@ videoChannelRouter.post('/:nameWithHost/avatar/pick', // Check the rights asyncMiddleware(videoChannelsUpdateValidator), updateAvatarValidator, - asyncMiddleware(updateVideoChannelAvatar) + asyncRetryTransactionMiddleware(updateVideoChannelAvatar) ) videoChannelRouter.put('/:nameWithHost', @@ -107,13 +107,9 @@ async function updateVideoChannelAvatar (req: express.Request, res: express.Resp const videoChannel = res.locals.videoChannel as VideoChannelModel const oldVideoChannelAuditKeys = new VideoChannelAuditView(videoChannel.toFormattedJSON()) - const avatar = await updateActorAvatarFile(avatarPhysicalFile, videoChannel.Actor, videoChannel) + const avatar = await updateActorAvatarFile(avatarPhysicalFile, videoChannel) - auditLogger.update( - getAuditIdFromRes(res), - new VideoChannelAuditView(videoChannel.toFormattedJSON()), - oldVideoChannelAuditKeys - ) + auditLogger.update(getAuditIdFromRes(res), new VideoChannelAuditView(videoChannel.toFormattedJSON()), oldVideoChannelAuditKeys) return res .json({ diff --git a/server/lib/avatar.ts b/server/lib/avatar.ts index 5cfb81fc7..14f0a05f5 100644 --- a/server/lib/avatar.ts +++ b/server/lib/avatar.ts @@ -3,23 +3,18 @@ import { sendUpdateActor } from './activitypub/send' import { AVATARS_SIZE, CONFIG, sequelizeTypescript } from '../initializers' import { updateActorAvatarInstance } from './activitypub' import { processImage } from '../helpers/image-utils' -import { ActorModel } from '../models/activitypub/actor' import { AccountModel } from '../models/account/account' import { VideoChannelModel } from '../models/video/video-channel' import { extname, join } from 'path' -async function updateActorAvatarFile ( - avatarPhysicalFile: Express.Multer.File, - actor: ActorModel, - accountOrChannel: AccountModel | VideoChannelModel -) { +async function updateActorAvatarFile (avatarPhysicalFile: Express.Multer.File, accountOrChannel: AccountModel | VideoChannelModel) { const extension = extname(avatarPhysicalFile.filename) - const avatarName = actor.uuid + extension + const avatarName = accountOrChannel.Actor.uuid + extension const destination = join(CONFIG.STORAGE.AVATARS_DIR, avatarName) await processImage(avatarPhysicalFile, destination, AVATARS_SIZE) return sequelizeTypescript.transaction(async t => { - const updatedActor = await updateActorAvatarInstance(actor, avatarName, t) + const updatedActor = await updateActorAvatarInstance(accountOrChannel.Actor, avatarName, t) await updatedActor.save({ transaction: t }) await sendUpdateActor(accountOrChannel, t) diff --git a/server/lib/oauth-model.ts b/server/lib/oauth-model.ts index 2f8667e19..5cbe60b82 100644 --- a/server/lib/oauth-model.ts +++ b/server/lib/oauth-model.ts @@ -4,15 +4,50 @@ import { UserModel } from '../models/account/user' import { OAuthClientModel } from '../models/oauth/oauth-client' import { OAuthTokenModel } from '../models/oauth/oauth-token' import { CONFIG } from '../initializers/constants' +import { Transaction } from 'sequelize' type TokenInfo = { accessToken: string, refreshToken: string, accessTokenExpiresAt: Date, refreshTokenExpiresAt: Date } +const accessTokenCache: { [ accessToken: string ]: OAuthTokenModel } = {} +const userHavingToken: { [ userId: number ]: string } = {} // --------------------------------------------------------------------------- +function deleteUserToken (userId: number, t?: Transaction) { + clearCacheByUserId(userId) + + return OAuthTokenModel.deleteUserToken(userId, t) +} + +function clearCacheByUserId (userId: number) { + const token = userHavingToken[userId] + if (token !== undefined) { + accessTokenCache[ token ] = undefined + userHavingToken[ userId ] = undefined + } +} + +function clearCacheByToken (token: string) { + const tokenModel = accessTokenCache[ token ] + if (tokenModel !== undefined) { + userHavingToken[tokenModel.userId] = undefined + accessTokenCache[ token ] = undefined + } +} + function getAccessToken (bearerToken: string) { logger.debug('Getting access token (bearerToken: ' + bearerToken + ').') + if (accessTokenCache[bearerToken] !== undefined) return accessTokenCache[bearerToken] + return OAuthTokenModel.getByTokenAndPopulateUser(bearerToken) + .then(tokenModel => { + if (tokenModel) { + accessTokenCache[ bearerToken ] = tokenModel + userHavingToken[ tokenModel.userId ] = tokenModel.accessToken + } + + return tokenModel + }) } function getClient (clientId: string, clientSecret: string) { @@ -48,6 +83,8 @@ async function getUser (usernameOrEmail: string, password: string) { async function revokeToken (tokenInfo: TokenInfo) { const token = await OAuthTokenModel.getByRefreshTokenAndPopulateUser(tokenInfo.refreshToken) if (token) { + clearCacheByToken(token.accessToken) + token.destroy() .catch(err => logger.error('Cannot destroy token when revoking token.', { err })) } @@ -85,6 +122,9 @@ async function saveToken (token: TokenInfo, client: OAuthClientModel, user: User // See https://github.com/oauthjs/node-oauth2-server/wiki/Model-specification for the model specifications export { + deleteUserToken, + clearCacheByUserId, + clearCacheByToken, getAccessToken, getClient, getRefreshToken, diff --git a/server/models/account/user.ts b/server/models/account/user.ts index 680b1d52d..e56b0bf40 100644 --- a/server/models/account/user.ts +++ b/server/models/account/user.ts @@ -1,5 +1,7 @@ import * as Sequelize from 'sequelize' import { + AfterDelete, + AfterUpdate, AllowNull, BeforeCreate, BeforeUpdate, @@ -39,6 +41,7 @@ import { AccountModel } from './account' import { NSFWPolicyType } from '../../../shared/models/videos/nsfw-policy.type' import { values } from 'lodash' import { NSFW_POLICY_TYPES } from '../../initializers' +import { clearCacheByUserId } from '../../lib/oauth-model' enum ScopeNames { WITH_VIDEO_CHANNEL = 'WITH_VIDEO_CHANNEL' @@ -168,6 +171,12 @@ export class UserModel extends Model { } } + @AfterUpdate + @AfterDelete + static removeTokenCache (instance: UserModel) { + return clearCacheByUserId(instance.id) + } + static countTotal () { return this.count() } diff --git a/server/models/oauth/oauth-token.ts b/server/models/oauth/oauth-token.ts index 1dd5e0289..ef9592c04 100644 --- a/server/models/oauth/oauth-token.ts +++ b/server/models/oauth/oauth-token.ts @@ -1,10 +1,23 @@ -import { AllowNull, BelongsTo, Column, CreatedAt, ForeignKey, Model, Scopes, Table, UpdatedAt } from 'sequelize-typescript' +import { + AfterDelete, + AfterUpdate, + AllowNull, + BelongsTo, + Column, + CreatedAt, + ForeignKey, + Model, + Scopes, + Table, + UpdatedAt +} from 'sequelize-typescript' import { logger } from '../../helpers/logger' import { UserModel } from '../account/user' import { OAuthClientModel } from './oauth-client' import { Transaction } from 'sequelize' import { AccountModel } from '../account/account' import { ActorModel } from '../activitypub/actor' +import { clearCacheByToken } from '../../lib/oauth-model' export type OAuthTokenInfo = { refreshToken: string @@ -112,6 +125,12 @@ export class OAuthTokenModel extends Model { }) OAuthClients: OAuthClientModel[] + @AfterUpdate + @AfterDelete + static removeTokenCache (token: OAuthTokenModel) { + return clearCacheByToken(token.accessToken) + } + static getByRefreshTokenAndPopulateClient (refreshToken: string) { const query = { where: { From d466dece0a07555b91f01fa35fcc5dcfe79d9e12 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 20 Sep 2018 11:55:00 +0200 Subject: [PATCH 41/44] Improve message when removing a user --- client/src/app/+admin/users/user-list/user-list.component.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/src/app/+admin/users/user-list/user-list.component.ts b/client/src/app/+admin/users/user-list/user-list.component.ts index 57e63d465..9697ce202 100644 --- a/client/src/app/+admin/users/user-list/user-list.component.ts +++ b/client/src/app/+admin/users/user-list/user-list.component.ts @@ -105,7 +105,8 @@ export class UserListComponent extends RestTable implements OnInit { return } - const res = await this.confirmService.confirm(this.i18n('Do you really want to delete this user?'), this.i18n('Delete')) + const message = this.i18n('If you remove this user, you will not be able to create another with the same username!') + const res = await this.confirmService.confirm(message, this.i18n('Delete')) if (res === false) return this.userService.removeUser(user).subscribe( From 89724816ae79e0c4f9fba6f47267711f505ec7af Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 20 Sep 2018 14:21:57 +0200 Subject: [PATCH 42/44] Improve videos list client performance --- client/src/app/shared/video/abstract-video-list.html | 4 ++-- client/src/app/shared/video/abstract-video-list.ts | 11 ++++++++++- .../app/shared/video/video-miniature.component.html | 4 ++-- .../src/app/shared/video/video-miniature.component.ts | 11 ++++++----- client/src/hmr.ts | 10 +++++++++- 5 files changed, 29 insertions(+), 11 deletions(-) diff --git a/client/src/app/shared/video/abstract-video-list.html b/client/src/app/shared/video/abstract-video-list.html index 4ad4e3568..d543ab7c1 100644 --- a/client/src/app/shared/video/abstract-video-list.html +++ b/client/src/app/shared/video/abstract-video-list.html @@ -11,8 +11,8 @@ (nearOfTop)="onNearOfTop()" (nearOfBottom)="onNearOfBottom()" (pageChanged)="onPageChanged($event)" class="videos" #videosElement > -
- +
+
diff --git a/client/src/app/shared/video/abstract-video-list.ts b/client/src/app/shared/video/abstract-video-list.ts index 53b044478..6a758ebe0 100644 --- a/client/src/app/shared/video/abstract-video-list.ts +++ b/client/src/app/shared/video/abstract-video-list.ts @@ -81,6 +81,15 @@ export abstract class AbstractVideoList implements OnInit, OnDestroy { if (this.resizeSubscription) this.resizeSubscription.unsubscribe() } + pageByVideoId (index: number, page: Video[]) { + // Video are unique in all pages + return page[0].id + } + + videoById (index: number, video: Video) { + return video.id + } + onNearOfTop () { this.previousPage() } @@ -166,7 +175,7 @@ export abstract class AbstractVideoList implements OnInit, OnDestroy { const min = this.minPageLoaded() if (min > 1) { - this.loadMoreVideos(min - 1) + this.loadMoreVideos(min - 1, true) } } diff --git a/client/src/app/shared/video/video-miniature.component.html b/client/src/app/shared/video/video-miniature.component.html index 9cf3fb321..cfc483018 100644 --- a/client/src/app/shared/video/video-miniature.component.html +++ b/client/src/app/shared/video/video-miniature.component.html @@ -1,11 +1,11 @@
- +
{{ video.name }} diff --git a/client/src/app/shared/video/video-miniature.component.ts b/client/src/app/shared/video/video-miniature.component.ts index 07193ebd5..27098f4b4 100644 --- a/client/src/app/shared/video/video-miniature.component.ts +++ b/client/src/app/shared/video/video-miniature.component.ts @@ -1,4 +1,4 @@ -import { Component, Input, OnInit } from '@angular/core' +import { ChangeDetectionStrategy, Component, Input, OnInit } from '@angular/core' import { User } from '../users' import { Video } from './video.model' import { ServerService } from '@app/core' @@ -8,13 +8,16 @@ export type OwnerDisplayType = 'account' | 'videoChannel' | 'auto' @Component({ selector: 'my-video-miniature', styleUrls: [ './video-miniature.component.scss' ], - templateUrl: './video-miniature.component.html' + templateUrl: './video-miniature.component.html', + changeDetection: ChangeDetectionStrategy.OnPush }) export class VideoMiniatureComponent implements OnInit { @Input() user: User @Input() video: Video @Input() ownerDisplayType: OwnerDisplayType = 'account' + isVideoBlur: boolean + private ownerDisplayTypeChosen: 'account' | 'videoChannel' constructor (private serverService: ServerService) { } @@ -35,10 +38,8 @@ export class VideoMiniatureComponent implements OnInit { } else { this.ownerDisplayTypeChosen = 'videoChannel' } - } - isVideoBlur () { - return this.video.isVideoNSFWForUser(this.user, this.serverService.getConfig()) + this.isVideoBlur = this.video.isVideoNSFWForUser(this.user, this.serverService.getConfig()) } displayOwnerAccount () { diff --git a/client/src/hmr.ts b/client/src/hmr.ts index 4d707a250..d5306a7a2 100644 --- a/client/src/hmr.ts +++ b/client/src/hmr.ts @@ -1,11 +1,19 @@ import { NgModuleRef, ApplicationRef } from '@angular/core' import { createNewHosts } from '@angularclass/hmr' +import { enableDebugTools } from '@angular/platform-browser' export const hmrBootstrap = (module: any, bootstrap: () => Promise>) => { let ngModule: NgModuleRef module.hot.accept() bootstrap() - .then(mod => ngModule = mod) + .then(mod => { + ngModule = mod + + const applicationRef = ngModule.injector.get(ApplicationRef); + const componentRef = applicationRef.components[ 0 ] + // allows to run `ng.profiler.timeChangeDetection();` + enableDebugTools(componentRef) + }) module.hot.dispose(() => { const appRef: ApplicationRef = ngModule.injector.get(ApplicationRef) const elements = appRef.components.map(c => c.location.nativeElement) From 93ea9c47d989e28405cf1039f89be71e592e36a5 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 20 Sep 2018 14:49:12 +0200 Subject: [PATCH 43/44] Improve responsive on small screens --- .../comment/video-comment-add.component.scss | 6 +++++ .../comment/video-comment.component.scss | 7 +++++ .../comment/video-comments.component.scss | 8 +++++- .../+video-watch/video-watch.component.scss | 27 +++++++++++++++++-- client/src/sass/include/_mixins.scss | 1 - 5 files changed, 45 insertions(+), 4 deletions(-) diff --git a/client/src/app/videos/+video-watch/comment/video-comment-add.component.scss b/client/src/app/videos/+video-watch/comment/video-comment-add.component.scss index a55e743fb..bb809296a 100644 --- a/client/src/app/videos/+video-watch/comment/video-comment-add.component.scss +++ b/client/src/app/videos/+video-watch/comment/video-comment-add.component.scss @@ -39,3 +39,9 @@ form { @include orange-button } } + +@media screen and (max-width: 450px) { + textarea, .submit-comment button { + font-size: 14px !important; + } +} \ No newline at end of file diff --git a/client/src/app/videos/+video-watch/comment/video-comment.component.scss b/client/src/app/videos/+video-watch/comment/video-comment.component.scss index f331fab80..84da5727e 100644 --- a/client/src/app/videos/+video-watch/comment/video-comment.component.scss +++ b/client/src/app/videos/+video-watch/comment/video-comment.component.scss @@ -35,6 +35,7 @@ .comment-account { @include disable-default-a-behaviour; + word-break: break-all; color: var(--mainForegroundColor); font-weight: $font-bold; } @@ -102,3 +103,9 @@ img { margin-right: 10px; } } } + +@media screen and (max-width: 450px) { + .root-comment { + font-size: 14px; + } +} \ No newline at end of file diff --git a/client/src/app/videos/+video-watch/comment/video-comments.component.scss b/client/src/app/videos/+video-watch/comment/video-comments.component.scss index d5af929d7..04518e079 100644 --- a/client/src/app/videos/+video-watch/comment/video-comments.component.scss +++ b/client/src/app/videos/+video-watch/comment/video-comments.component.scss @@ -31,4 +31,10 @@ my-help { .view-replies { margin-left: 46px; } -} \ No newline at end of file +} + +@media screen and (max-width: 450px) { + .view-replies { + font-size: 14px; + } +} diff --git a/client/src/app/videos/+video-watch/video-watch.component.scss b/client/src/app/videos/+video-watch/video-watch.component.scss index 15adf0f61..eb63cbde7 100644 --- a/client/src/app/videos/+video-watch/video-watch.component.scss +++ b/client/src/app/videos/+video-watch/video-watch.component.scss @@ -473,6 +473,7 @@ my-video-comments { margin: 20px 0 0 0; .video-info { + padding: 0; .video-info-first-row { @@ -485,6 +486,8 @@ my-video-comments { } /deep/ .other-videos { + padding-left: 0 !important; + /deep/ .video-miniature { flex-direction: column; } @@ -500,7 +503,27 @@ my-video-comments { } @media screen and (max-width: 450px) { - .video-bottom .action-button .icon-text { - display: none !important; + .video-bottom { + .action-button .icon-text { + display: none !important; + } + + .video-info .video-info-first-row { + .video-info-name { + font-size: 18px; + } + + .video-info-date-views { + font-size: 14px; + } + + .video-actions-rates { + margin-top: 10px; + } + } + + .video-info-description { + font-size: 14px !important; + } } } diff --git a/client/src/sass/include/_mixins.scss b/client/src/sass/include/_mixins.scss index d755e7df3..544f39957 100644 --- a/client/src/sass/include/_mixins.scss +++ b/client/src/sass/include/_mixins.scss @@ -53,7 +53,6 @@ -ms-hyphens: auto; -moz-hyphens: auto; hyphens: auto; - text-align: justify; } @mixin peertube-input-text($width) { From 6247b2057b792cea155a1abd9788c363ae7d2cc2 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 20 Sep 2018 15:45:11 +0200 Subject: [PATCH 44/44] Fix client e2e tests --- README.md | 4 ++-- client/e2e/src/po/video-watch.po.ts | 7 +++++-- client/e2e/src/videos.e2e-spec.ts | 4 ++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 9fb89039b..3985f38bd 100644 --- a/README.md +++ b/README.md @@ -24,8 +24,8 @@ directly in the web browser with devDependency Status - - + +

diff --git a/client/e2e/src/po/video-watch.po.ts b/client/e2e/src/po/video-watch.po.ts index 13f4ae945..e17aebc29 100644 --- a/client/e2e/src/po/video-watch.po.ts +++ b/client/e2e/src/po/video-watch.po.ts @@ -26,8 +26,11 @@ export class VideoWatchPage { .then((texts: any) => texts.map(t => t.trim())) } - waitWatchVideoName (videoName: string, isSafari: boolean) { - const elem = element(by.css('.video-info .video-info-name')) + waitWatchVideoName (videoName: string, isMobileDevice: boolean, isSafari: boolean) { + // On mobile we display the first node, on desktop the second + const index = isMobileDevice ? 0 : 1 + + const elem = element.all(by.css('.video-info .video-info-name')).get(index) if (isSafari) return browser.sleep(5000) diff --git a/client/e2e/src/videos.e2e-spec.ts b/client/e2e/src/videos.e2e-spec.ts index 3d4d46292..606b6ac5d 100644 --- a/client/e2e/src/videos.e2e-spec.ts +++ b/client/e2e/src/videos.e2e-spec.ts @@ -12,7 +12,7 @@ describe('Videos workflow', () => { let isSafari = false beforeEach(async () => { - browser.waitForAngularEnabled(false) + await browser.waitForAngularEnabled(false) videoWatchPage = new VideoWatchPage() pageUploadPage = new VideoUploadPage() @@ -62,7 +62,7 @@ describe('Videos workflow', () => { if (isMobileDevice || isSafari) videoNameToExcept = await videoWatchPage.clickOnFirstVideo() else await videoWatchPage.clickOnVideo(videoName) - return videoWatchPage.waitWatchVideoName(videoNameToExcept, isSafari) + return videoWatchPage.waitWatchVideoName(videoNameToExcept, isMobileDevice, isSafari) }) it('Should play the video', async () => {