From 9c5cc501335b9f7dba4aa492b8fda68d4881d28d Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 2 Jun 2023 15:18:39 +0200 Subject: [PATCH] Add script to generate storyboards --- package.json | 1 + scripts/create-generate-storyboard-job.ts | 85 +++++++++++++ scripts/create-move-video-storage-job.ts | 8 +- .../job-queue/handlers/generate-storyboard.ts | 5 + .../cli/create-generate-storyboard-job.ts | 120 ++++++++++++++++++ server/tests/cli/index.ts | 1 + support/doc/tools.md | 29 +++++ 7 files changed, 247 insertions(+), 2 deletions(-) create mode 100644 scripts/create-generate-storyboard-job.ts create mode 100644 server/tests/cli/create-generate-storyboard-job.ts diff --git a/package.json b/package.json index 223156098..e2feea5a2 100644 --- a/package.json +++ b/package.json @@ -49,6 +49,7 @@ "regenerate-thumbnails": "node ./dist/scripts/regenerate-thumbnails.js", "create-import-video-file-job": "node ./dist/scripts/create-import-video-file-job.js", "create-move-video-storage-job": "node ./dist/scripts/create-move-video-storage-job.js", + "create-generate-storyboard-job": "node ./dist/scripts/create-generate-storyboard-job.js", "test": "bash ./scripts/test.sh", "generate-cli-doc": "bash ./scripts/generate-cli-doc.sh", "generate-types-package": "ts-node ./packages/types/generate-package.ts", diff --git a/scripts/create-generate-storyboard-job.ts b/scripts/create-generate-storyboard-job.ts new file mode 100644 index 000000000..47c08edac --- /dev/null +++ b/scripts/create-generate-storyboard-job.ts @@ -0,0 +1,85 @@ +import { program } from 'commander' +import { toCompleteUUID } from '@server/helpers/custom-validators/misc' +import { initDatabaseModels } from '@server/initializers/database' +import { JobQueue } from '@server/lib/job-queue' +import { VideoModel } from '@server/models/video/video' +import { StoryboardModel } from '@server/models/video/storyboard' + +program + .description('Generate videos storyboard') + .option('-v, --video [videoUUID]', 'Generate the storyboard of a specific video') + .option('-a, --all-videos', 'Generate missing storyboards of local videos') + .parse(process.argv) + +const options = program.opts() + +if (!options['video'] && !options['allVideos']) { + console.error('You need to choose videos for storyboard generation.') + process.exit(-1) +} + +run() + .then(() => process.exit(0)) + .catch(err => { + console.error(err) + process.exit(-1) + }) + +async function run () { + await initDatabaseModels(true) + + JobQueue.Instance.init() + + let ids: number[] = [] + + if (options['video']) { + const video = await VideoModel.load(toCompleteUUID(options['video'])) + + if (!video) { + console.error('Unknown video ' + options['video']) + process.exit(-1) + } + + if (video.remote === true) { + console.error('Cannot process a remote video') + process.exit(-1) + } + + if (video.isLive) { + console.error('Cannot process live video') + process.exit(-1) + } + + ids.push(video.id) + } else { + ids = await listLocalMissingStoryboards() + } + + for (const id of ids) { + const videoFull = await VideoModel.load(id) + + if (videoFull.isLive) continue + + await JobQueue.Instance.createJob({ + type: 'generate-video-storyboard', + payload: { + videoUUID: videoFull.uuid, + federate: true + } + }) + + console.log(`Created generate-storyboard job for ${videoFull.name}.`) + } +} + +async function listLocalMissingStoryboards () { + const ids = await VideoModel.listLocalIds() + const results: number[] = [] + + for (const id of ids) { + const storyboard = await StoryboardModel.loadByVideo(id) + if (!storyboard) results.push(id) + } + + return results +} diff --git a/scripts/create-move-video-storage-job.ts b/scripts/create-move-video-storage-job.ts index c402115f0..8537114eb 100644 --- a/scripts/create-move-video-storage-job.ts +++ b/scripts/create-move-video-storage-job.ts @@ -1,4 +1,5 @@ import { program } from 'commander' +import { toCompleteUUID } from '@server/helpers/custom-validators/misc' import { CONFIG } from '@server/initializers/config' import { initDatabaseModels } from '@server/initializers/database' import { JobQueue } from '@server/lib/job-queue' @@ -32,7 +33,10 @@ if (options['toObjectStorage'] && !CONFIG.OBJECT_STORAGE.ENABLED) { run() .then(() => process.exit(0)) - .catch(err => console.error(err)) + .catch(err => { + console.error(err) + process.exit(-1) + }) async function run () { await initDatabaseModels(true) @@ -42,7 +46,7 @@ async function run () { let ids: number[] = [] if (options['video']) { - const video = await VideoModel.load(options['video']) + const video = await VideoModel.load(toCompleteUUID(options['video'])) if (!video) { console.error('Unknown video ' + options['video']) diff --git a/server/lib/job-queue/handlers/generate-storyboard.ts b/server/lib/job-queue/handlers/generate-storyboard.ts index 652cac272..09b746a3e 100644 --- a/server/lib/job-queue/handlers/generate-storyboard.ts +++ b/server/lib/job-queue/handlers/generate-storyboard.ts @@ -43,6 +43,11 @@ async function processGenerateStoryboard (job: Job): Promise { const destination = join(CONFIG.STORAGE.STORYBOARDS_DIR, filename) const totalSprites = buildTotalSprites(video) + if (totalSprites === 0) { + logger.info('Do not generate a storyboard of %s because the video is not long enough', payload.videoUUID, lTags) + return + } + const spriteDuration = Math.round(video.duration / totalSprites) const spritesCount = findGridSize({ diff --git a/server/tests/cli/create-generate-storyboard-job.ts b/server/tests/cli/create-generate-storyboard-job.ts new file mode 100644 index 000000000..02a4be8ae --- /dev/null +++ b/server/tests/cli/create-generate-storyboard-job.ts @@ -0,0 +1,120 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { readdir, remove } from 'fs-extra' +import { join } from 'path' +import { HttpStatusCode } from '@shared/models' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + makeGetRequest, + PeerTubeServer, + setAccessTokensToServers, + waitJobs +} from '@shared/server-commands' +import { SQLCommand } from '../shared' + +function listStoryboardFiles (server: PeerTubeServer) { + const storage = server.getDirectoryPath('storyboards') + + return readdir(storage) +} + +describe('Test create generate storyboard job', function () { + let servers: PeerTubeServer[] = [] + const uuids: string[] = [] + let sql: SQLCommand + let existingStoryboardName: string + + before(async function () { + this.timeout(120000) + + // Run server 2 to have transcoding enabled + servers = await createMultipleServers(2) + await setAccessTokensToServers(servers) + + await doubleFollow(servers[0], servers[1]) + + for (let i = 0; i < 3; i++) { + const { uuid } = await servers[0].videos.quickUpload({ name: 'video ' + i }) + uuids.push(uuid) + } + + await waitJobs(servers) + + const storage = servers[0].getDirectoryPath('storyboards') + for (const storyboard of await listStoryboardFiles(servers[0])) { + await remove(join(storage, storyboard)) + } + + sql = new SQLCommand(servers[0]) + await sql.deleteAll('storyboard') + + const { uuid } = await servers[0].videos.quickUpload({ name: 'video 4' }) + uuids.push(uuid) + + await waitJobs(servers) + + const storyboards = await listStoryboardFiles(servers[0]) + existingStoryboardName = storyboards[0] + }) + + it('Should create a storyboard of a video', async function () { + this.timeout(120000) + + for (const uuid of [ uuids[0], uuids[3] ]) { + const command = `npm run create-generate-storyboard-job -- -v ${uuid}` + await servers[0].cli.execWithEnv(command) + } + + await waitJobs(servers) + + { + const storyboards = await listStoryboardFiles(servers[0]) + expect(storyboards).to.have.lengthOf(2) + expect(storyboards).to.not.include(existingStoryboardName) + + existingStoryboardName = storyboards[0] + } + + for (const server of servers) { + for (const uuid of [ uuids[0], uuids[3] ]) { + const { storyboards } = await server.storyboard.list({ id: uuid }) + expect(storyboards).to.have.lengthOf(1) + + await makeGetRequest({ url: server.url, path: storyboards[0].storyboardPath, expectedStatus: HttpStatusCode.OK_200 }) + } + } + }) + + it('Should create missing storyboards', async function () { + this.timeout(120000) + + const command = `npm run create-generate-storyboard-job -- -a` + await servers[0].cli.execWithEnv(command) + + await waitJobs(servers) + + { + const storyboards = await listStoryboardFiles(servers[0]) + expect(storyboards).to.have.lengthOf(4) + expect(storyboards).to.include(existingStoryboardName) + } + + for (const server of servers) { + for (const uuid of uuids) { + const { storyboards } = await server.storyboard.list({ id: uuid }) + expect(storyboards).to.have.lengthOf(1) + + await makeGetRequest({ url: server.url, path: storyboards[0].storyboardPath, expectedStatus: HttpStatusCode.OK_200 }) + } + } + }) + + after(async function () { + await sql.cleanup() + + await cleanupTests(servers) + }) +}) diff --git a/server/tests/cli/index.ts b/server/tests/cli/index.ts index 8579be39c..94444ace3 100644 --- a/server/tests/cli/index.ts +++ b/server/tests/cli/index.ts @@ -1,5 +1,6 @@ // Order of the tests we want to execute import './create-import-video-file-job' +import './create-generate-storyboard-job' import './create-move-video-storage-job' import './peertube' import './plugins' diff --git a/support/doc/tools.md b/support/doc/tools.md index 39f5ab787..b2fb7cd2c 100644 --- a/support/doc/tools.md +++ b/support/doc/tools.md @@ -268,6 +268,35 @@ cd /var/www/peertube-docker docker-compose exec -u peertube peertube npm run create-move-video-storage-job -- --to-object-storage --all-videos ``` + ### prune-storage.js