From 161b061d4e51eb1e2b2b7e9482d5299489ef7c45 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 28 Sep 2018 10:07:05 +0200 Subject: [PATCH] Remove duplicated videos on unfollow/delete redundancy --- client/src/sass/application.scss | 2 +- client/src/sass/include/_fonts.scss | 2 +- server/controllers/api/server/follows.ts | 5 ++ server/controllers/api/server/redundancy.ts | 6 ++ server/lib/redundancy.ts | 9 +++ server/models/redundancy/video-redundancy.ts | 41 ++++++++++++ server/tests/api/server/redundancy.ts | 67 +++++++++++++++----- 7 files changed, 115 insertions(+), 17 deletions(-) diff --git a/client/src/sass/application.scss b/client/src/sass/application.scss index 0029c22ef..40a9ed231 100644 --- a/client/src/sass/application.scss +++ b/client/src/sass/application.scss @@ -1,4 +1,4 @@ -$icon-font-path: '../../node_modules/@neos21/bootstrap3-glyphicons/assets/fonts/'; +$icon-font-path: '~@neos21/bootstrap3-glyphicons/assets/fonts/'; @import '_bootstrap'; @import '_variables'; diff --git a/client/src/sass/include/_fonts.scss b/client/src/sass/include/_fonts.scss index 61717e6f5..d4694a747 100644 --- a/client/src/sass/include/_fonts.scss +++ b/client/src/sass/include/_fonts.scss @@ -1,4 +1,4 @@ -$FontPathSourceSansPro: '../../node_modules/npm-font-source-sans-pro/fonts'; +$FontPathSourceSansPro: '~npm-font-source-sans-pro/fonts'; @font-face { font-family: 'Source Sans Pro'; diff --git a/server/controllers/api/server/follows.ts b/server/controllers/api/server/follows.ts index a4eae6b45..d62400e42 100644 --- a/server/controllers/api/server/follows.ts +++ b/server/controllers/api/server/follows.ts @@ -17,6 +17,7 @@ import { import { followersSortValidator, followingSortValidator, followValidator } from '../../../middlewares/validators' import { ActorFollowModel } from '../../../models/activitypub/actor-follow' import { JobQueue } from '../../../lib/job-queue' +import { removeRedundancyOf } from '../../../lib/redundancy' const serverFollowsRouter = express.Router() serverFollowsRouter.get('/following', @@ -101,6 +102,10 @@ async function removeFollow (req: express.Request, res: express.Response, next: server.redundancyAllowed = false await server.save({ transaction: t }) + // Async, could be long + removeRedundancyOf(server.id) + .catch(err => logger.error('Cannot remove redundancy of %s.', server.host, err)) + await follow.destroy({ transaction: t }) }) diff --git a/server/controllers/api/server/redundancy.ts b/server/controllers/api/server/redundancy.ts index 4216b9e35..4140c4991 100644 --- a/server/controllers/api/server/redundancy.ts +++ b/server/controllers/api/server/redundancy.ts @@ -3,6 +3,8 @@ import { UserRight } from '../../../../shared/models/users' import { asyncMiddleware, authenticate, ensureUserHasRight } from '../../../middlewares' import { updateServerRedundancyValidator } from '../../../middlewares/validators/redundancy' import { ServerModel } from '../../../models/server/server' +import { removeRedundancyOf } from '../../../lib/redundancy' +import { logger } from '../../../helpers/logger' const serverRedundancyRouter = express.Router() @@ -28,5 +30,9 @@ async function updateRedundancy (req: express.Request, res: express.Response, ne await server.save() + // Async, could be long + removeRedundancyOf(server.id) + .catch(err => logger.error('Cannot remove redundancy of %s.', server.host, err)) + return res.sendStatus(204) } diff --git a/server/lib/redundancy.ts b/server/lib/redundancy.ts index 16b122658..04d3ded8f 100644 --- a/server/lib/redundancy.ts +++ b/server/lib/redundancy.ts @@ -12,8 +12,17 @@ async function removeVideoRedundancy (videoRedundancy: VideoRedundancyModel, t?: await videoRedundancy.destroy({ transaction: t }) } +async function removeRedundancyOf (serverId: number) { + const videosRedundancy = await VideoRedundancyModel.listLocalOfServer(serverId) + + for (const redundancy of videosRedundancy) { + await removeVideoRedundancy(redundancy) + } +} + // --------------------------------------------------------------------------- export { + removeRedundancyOf, removeVideoRedundancy } diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts index 3c87ec2c1..e67164802 100644 --- a/server/models/redundancy/video-redundancy.ts +++ b/server/models/redundancy/video-redundancy.ts @@ -286,6 +286,47 @@ export class VideoRedundancyModel extends Model { return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query) } + static async listLocalOfServer (serverId: number) { + const actor = await getServerActor() + + const query = { + where: { + actorId: actor.id + }, + include: [ + { + model: VideoFileModel, + required: true, + include: [ + { + model: VideoModel, + required: true, + include: [ + { + attributes: [], + model: VideoChannelModel.unscoped(), + required: true, + include: [ + { + attributes: [], + model: ActorModel.unscoped(), + required: true, + where: { + serverId + } + } + ] + } + ] + } + ] + } + ] + } + + return VideoRedundancyModel.findAll(query) + } + static async getStats (strategy: VideoRedundancyStrategy) { const actor = await getServerActor() diff --git a/server/tests/api/server/redundancy.ts b/server/tests/api/server/redundancy.ts index 9b3b1b4ad..e1709891d 100644 --- a/server/tests/api/server/redundancy.ts +++ b/server/tests/api/server/redundancy.ts @@ -12,7 +12,7 @@ import { killallServers, makeGetRequest, root, ServerInfo, - setAccessTokensToServers, + setAccessTokensToServers, unfollow, uploadVideo, viewVideo, wait @@ -39,6 +39,8 @@ function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: numbe const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`) expect(found, `Webseed ${ws} not found in ${file.magnetUri} on server ${server.url}`).to.not.be.undefined } + + expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length) } async function runServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) { @@ -136,23 +138,21 @@ async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: st ] for (const server of servers) { - { - const res = await getVideo(server.url, videoUUID) + const res = await getVideo(server.url, videoUUID) - const video: VideoDetails = res.body + const video: VideoDetails = res.body - for (const file of video.files) { - checkMagnetWebseeds(file, webseeds, server) + for (const file of video.files) { + checkMagnetWebseeds(file, webseeds, server) - // Only servers 1 and 2 have the video - if (server.serverNumber !== 3) { - await makeGetRequest({ - url: server.url, - statusCodeExpected: 200, - path: '/static/webseed/' + `${videoUUID}-${file.resolution.id}.mp4`, - contentType: null - }) - } + // Only servers 1 and 2 have the video + if (server.serverNumber !== 3) { + await makeGetRequest({ + url: server.url, + statusCodeExpected: 200, + path: '/static/webseed/' + `${videoUUID}-${file.resolution.id}.mp4`, + contentType: null + }) } } } @@ -182,6 +182,21 @@ async function enableRedundancyOnServer1 () { expect(server2.following.hostRedundancyAllowed).to.be.true } +async function disableRedundancyOnServer1 () { + await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, false) + + const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt') + const follows: ActorFollow[] = res.body.data + const server2 = follows.find(f => f.following.host === 'localhost:9002') + const server3 = follows.find(f => f.following.host === 'localhost:9003') + + expect(server3).to.not.be.undefined + expect(server3.following.hostRedundancyAllowed).to.be.false + + expect(server2).to.not.be.undefined + expect(server2.following.hostRedundancyAllowed).to.be.false +} + async function cleanServers () { killallServers(servers) } @@ -217,6 +232,17 @@ describe('Test videos redundancy', function () { await checkStatsWith2Webseed(strategy) }) + it('Should undo redundancy on server 1 and remove duplicated videos', async function () { + this.timeout(40000) + + await disableRedundancyOnServer1() + + await waitJobs(servers) + await wait(5000) + + await check1WebSeed(strategy) + }) + after(function () { return cleanServers() }) @@ -251,6 +277,17 @@ describe('Test videos redundancy', function () { await checkStatsWith2Webseed(strategy) }) + it('Should unfollow on server 1 and remove duplicated videos', async function () { + this.timeout(40000) + + await unfollow(servers[0].url, servers[0].accessToken, servers[1]) + + await waitJobs(servers) + await wait(5000) + + await check1WebSeed(strategy) + }) + after(function () { return cleanServers() })