diff --git a/client/src/app/+admin/admin.component.ts b/client/src/app/+admin/admin.component.ts index d4d912c40..49092ea2a 100644 --- a/client/src/app/+admin/admin.component.ts +++ b/client/src/app/+admin/admin.component.ts @@ -272,6 +272,8 @@ export class AdminComponent implements OnInit { private isRemoteRunnersEnabled () { const config = this.server.getHTMLConfig() - return config.transcoding.remoteRunners.enabled || config.live.transcoding.remoteRunners.enabled + return config.transcoding.remoteRunners.enabled || + config.live.transcoding.remoteRunners.enabled || + config.videoStudio.remoteRunners.enabled } } diff --git a/client/src/app/+admin/config/edit-custom-config/edit-configuration.service.ts b/client/src/app/+admin/config/edit-custom-config/edit-configuration.service.ts index 96f5b830e..6c431ce64 100644 --- a/client/src/app/+admin/config/edit-custom-config/edit-configuration.service.ts +++ b/client/src/app/+admin/config/edit-custom-config/edit-configuration.service.ts @@ -61,6 +61,10 @@ export class EditConfigurationService { return form.value['transcoding']['enabled'] === true } + isStudioEnabled (form: FormGroup) { + return form.value['videoStudio']['enabled'] === true + } + isLiveEnabled (form: FormGroup) { return form.value['live']['enabled'] === true } diff --git a/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts b/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts index 335aedb67..30e4aa5d5 100644 --- a/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts +++ b/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts @@ -218,7 +218,10 @@ export class EditCustomConfigComponent extends FormReactive implements OnInit { } }, videoStudio: { - enabled: null + enabled: null, + remoteRunners: { + enabled: null + } }, autoBlacklist: { videos: { diff --git a/client/src/app/+admin/config/edit-custom-config/edit-vod-transcoding.component.html b/client/src/app/+admin/config/edit-custom-config/edit-vod-transcoding.component.html index c11f560dd..b17c51532 100644 --- a/client/src/app/+admin/config/edit-custom-config/edit-vod-transcoding.component.html +++ b/client/src/app/+admin/config/edit-custom-config/edit-vod-transcoding.component.html @@ -230,6 +230,20 @@ + +
+ + + + Use remote runners to process studio transcoding tasks. + Remote runners has to register on your instance first. + + + +
diff --git a/client/src/app/+admin/config/edit-custom-config/edit-vod-transcoding.component.ts b/client/src/app/+admin/config/edit-custom-config/edit-vod-transcoding.component.ts index 184dfd921..e960533f9 100644 --- a/client/src/app/+admin/config/edit-custom-config/edit-vod-transcoding.component.ts +++ b/client/src/app/+admin/config/edit-custom-config/edit-vod-transcoding.component.ts @@ -62,10 +62,18 @@ export class EditVODTranscodingComponent implements OnInit, OnChanges { return this.editConfigurationService.isTranscodingEnabled(this.form) } + isStudioEnabled () { + return this.editConfigurationService.isStudioEnabled(this.form) + } + getTranscodingDisabledClass () { return { 'disabled-checkbox-extra': !this.isTranscodingEnabled() } } + getStudioDisabledClass () { + return { 'disabled-checkbox-extra': !this.isStudioEnabled() } + } + getTotalTranscodingThreads () { return this.editConfigurationService.getTotalTranscodingThreads(this.form) } diff --git a/config/default.yaml b/config/default.yaml index f3f29ecb9..14bb8d060 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -579,6 +579,12 @@ video_studio: # If enabled, users can create transcoding tasks as they wish enabled: false + # Enable remote runners to transcode studio tasks + # If enabled, your instance won't transcode the videos itself + # At least 1 remote runner must be configured to transcode your videos + remote_runners: + enabled: false + import: # Add ability for your users to import remote videos (from YouTube, torrent...) videos: diff --git a/config/production.yaml.example b/config/production.yaml.example index ea6d77306..db9c18cb8 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example @@ -589,6 +589,13 @@ video_studio: # If enabled, users can create transcoding tasks as they wish enabled: false + + # Enable remote runners to transcode studio tasks + # If enabled, your instance won't transcode the videos itself + # At least 1 remote runner must be configured to transcode your videos + remote_runners: + enabled: false + import: # Add ability for your users to import remote videos (from YouTube, torrent...) videos: diff --git a/packages/peertube-runner/server/process/process.ts b/packages/peertube-runner/server/process/process.ts index 39a929c59..ef231cb38 100644 --- a/packages/peertube-runner/server/process/process.ts +++ b/packages/peertube-runner/server/process/process.ts @@ -1,12 +1,14 @@ import { logger } from 'packages/peertube-runner/shared/logger' import { RunnerJobLiveRTMPHLSTranscodingPayload, + RunnerJobVideoEditionTranscodingPayload, RunnerJobVODAudioMergeTranscodingPayload, RunnerJobVODHLSTranscodingPayload, RunnerJobVODWebVideoTranscodingPayload } from '@shared/models' import { processAudioMergeTranscoding, processHLSTranscoding, ProcessOptions, processWebVideoTranscoding } from './shared' import { ProcessLiveRTMPHLSTranscoding } from './shared/process-live' +import { processStudioTranscoding } from './shared/process-studio' export async function processJob (options: ProcessOptions) { const { server, job } = options @@ -21,6 +23,8 @@ export async function processJob (options: ProcessOptions) { await processHLSTranscoding(options as ProcessOptions) } else if (job.type === 'live-rtmp-hls-transcoding') { await new ProcessLiveRTMPHLSTranscoding(options as ProcessOptions).process() + } else if (job.type === 'video-edition-transcoding') { + await processStudioTranscoding(options as ProcessOptions) } else { logger.error(`Unknown job ${job.type} to process`) return diff --git a/packages/peertube-runner/server/process/shared/common.ts b/packages/peertube-runner/server/process/shared/common.ts index 9b2c40728..3cac98388 100644 --- a/packages/peertube-runner/server/process/shared/common.ts +++ b/packages/peertube-runner/server/process/shared/common.ts @@ -2,11 +2,12 @@ import { throttle } from 'lodash' import { ConfigManager, downloadFile, logger } from 'packages/peertube-runner/shared' import { join } from 'path' import { buildUUID } from '@shared/extra-utils' -import { FFmpegLive, FFmpegVOD } from '@shared/ffmpeg' +import { FFmpegEdition, FFmpegLive, FFmpegVOD } from '@shared/ffmpeg' import { RunnerJob, RunnerJobPayload } from '@shared/models' import { PeerTubeServer } from '@shared/server-commands' import { getTranscodingLogger } from './transcoding-logger' import { getAvailableEncoders, getEncodersToTry } from './transcoding-profiles' +import { remove } from 'fs-extra' export type JobWithToken = RunnerJob & { jobToken: string } @@ -24,7 +25,14 @@ export async function downloadInputFile (options: { const { url, job, runnerToken } = options const destination = join(ConfigManager.Instance.getTranscodingDirectory(), buildUUID()) - await downloadFile({ url, jobToken: job.jobToken, runnerToken, destination }) + try { + await downloadFile({ url, jobToken: job.jobToken, runnerToken, destination }) + } catch (err) { + remove(destination) + .catch(err => logger.error({ err }, `Cannot remove ${destination}`)) + + throw err + } return destination } @@ -40,6 +48,8 @@ export async function updateTranscodingProgress (options: { return server.runnerJobs.update({ jobToken: job.jobToken, jobUUID: job.uuid, runnerToken, progress }) } +// --------------------------------------------------------------------------- + export function buildFFmpegVOD (options: { server: PeerTubeServer runnerToken: string @@ -58,26 +68,25 @@ export function buildFFmpegVOD (options: { .catch(err => logger.error({ err }, 'Cannot send job progress')) }, updateInterval, { trailing: false }) - const config = ConfigManager.Instance.getConfig() - return new FFmpegVOD({ - niceness: config.ffmpeg.nice, - threads: config.ffmpeg.threads, - tmpDirectory: ConfigManager.Instance.getTranscodingDirectory(), - profile: 'default', - availableEncoders: { - available: getAvailableEncoders(), - encodersToTry: getEncodersToTry() - }, - logger: getTranscodingLogger(), + ...getCommonFFmpegOptions(), + updateJobProgress }) } export function buildFFmpegLive () { + return new FFmpegLive(getCommonFFmpegOptions()) +} + +export function buildFFmpegEdition () { + return new FFmpegEdition(getCommonFFmpegOptions()) +} + +function getCommonFFmpegOptions () { const config = ConfigManager.Instance.getConfig() - return new FFmpegLive({ + return { niceness: config.ffmpeg.nice, threads: config.ffmpeg.threads, tmpDirectory: ConfigManager.Instance.getTranscodingDirectory(), @@ -87,5 +96,5 @@ export function buildFFmpegLive () { encodersToTry: getEncodersToTry() }, logger: getTranscodingLogger() - }) + } } diff --git a/packages/peertube-runner/server/process/shared/process-studio.ts b/packages/peertube-runner/server/process/shared/process-studio.ts new file mode 100644 index 000000000..f8262096e --- /dev/null +++ b/packages/peertube-runner/server/process/shared/process-studio.ts @@ -0,0 +1,138 @@ +import { remove } from 'fs-extra' +import { pick } from 'lodash' +import { logger } from 'packages/peertube-runner/shared' +import { extname, join } from 'path' +import { buildUUID } from '@shared/extra-utils' +import { + RunnerJobVideoEditionTranscodingPayload, + VideoEditionTranscodingSuccess, + VideoStudioTask, + VideoStudioTaskCutPayload, + VideoStudioTaskIntroPayload, + VideoStudioTaskOutroPayload, + VideoStudioTaskPayload, + VideoStudioTaskWatermarkPayload +} from '@shared/models' +import { ConfigManager } from '../../../shared/config-manager' +import { buildFFmpegEdition, downloadInputFile, JobWithToken, ProcessOptions } from './common' + +export async function processStudioTranscoding (options: ProcessOptions) { + const { server, job, runnerToken } = options + const payload = job.payload + + let outputPath: string + const inputPath = await downloadInputFile({ url: payload.input.videoFileUrl, runnerToken, job }) + let tmpInputFilePath = inputPath + + try { + for (const task of payload.tasks) { + const outputFilename = 'output-edition-' + buildUUID() + '.mp4' + outputPath = join(ConfigManager.Instance.getTranscodingDirectory(), outputFilename) + + await processTask({ + inputPath: tmpInputFilePath, + outputPath, + task, + job, + runnerToken + }) + + if (tmpInputFilePath) await remove(tmpInputFilePath) + + // For the next iteration + tmpInputFilePath = outputPath + } + + const successBody: VideoEditionTranscodingSuccess = { + videoFile: outputPath + } + + await server.runnerJobs.success({ + jobToken: job.jobToken, + jobUUID: job.uuid, + runnerToken, + payload: successBody + }) + } finally { + await remove(tmpInputFilePath) + await remove(outputPath) + } +} + +// --------------------------------------------------------------------------- +// Private +// --------------------------------------------------------------------------- + +type TaskProcessorOptions = { + inputPath: string + outputPath: string + task: T + runnerToken: string + job: JobWithToken +} + +const taskProcessors: { [id in VideoStudioTask['name']]: (options: TaskProcessorOptions) => Promise } = { + 'add-intro': processAddIntroOutro, + 'add-outro': processAddIntroOutro, + 'cut': processCut, + 'add-watermark': processAddWatermark +} + +async function processTask (options: TaskProcessorOptions) { + const { task } = options + + const processor = taskProcessors[options.task.name] + if (!process) throw new Error('Unknown task ' + task.name) + + return processor(options) +} + +async function processAddIntroOutro (options: TaskProcessorOptions) { + const { inputPath, task, runnerToken, job } = options + + logger.debug('Adding intro/outro to ' + inputPath) + + const introOutroPath = await downloadInputFile({ url: task.options.file, runnerToken, job }) + + return buildFFmpegEdition().addIntroOutro({ + ...pick(options, [ 'inputPath', 'outputPath' ]), + + introOutroPath, + type: task.name === 'add-intro' + ? 'intro' + : 'outro' + }) +} + +function processCut (options: TaskProcessorOptions) { + const { inputPath, task } = options + + logger.debug(`Cutting ${inputPath}`) + + return buildFFmpegEdition().cutVideo({ + ...pick(options, [ 'inputPath', 'outputPath' ]), + + start: task.options.start, + end: task.options.end + }) +} + +async function processAddWatermark (options: TaskProcessorOptions) { + const { inputPath, task, runnerToken, job } = options + + logger.debug('Adding watermark to ' + inputPath) + + const watermarkPath = await downloadInputFile({ url: task.options.file, runnerToken, job }) + + return buildFFmpegEdition().addWatermark({ + ...pick(options, [ 'inputPath', 'outputPath' ]), + + watermarkPath, + + videoFilters: { + watermarkSizeRatio: task.options.watermarkSizeRatio, + horitonzalMarginRatio: task.options.horitonzalMarginRatio, + verticalMarginRatio: task.options.verticalMarginRatio + } + }) +} diff --git a/packages/peertube-runner/server/process/shared/process-vod.ts b/packages/peertube-runner/server/process/shared/process-vod.ts index aae61e9c5..d84ece3cb 100644 --- a/packages/peertube-runner/server/process/shared/process-vod.ts +++ b/packages/peertube-runner/server/process/shared/process-vod.ts @@ -62,33 +62,36 @@ export async function processHLSTranscoding (options: ProcessOptions {}, + inputFileMutexReleaser: () => {}, - resolution: payload.output.resolution, - fps: payload.output.fps - }) + resolution: payload.output.resolution, + fps: payload.output.fps + }) - const successBody: VODHLSTranscodingSuccess = { - resolutionPlaylistFile: outputPath, - videoFile: videoPath + const successBody: VODHLSTranscodingSuccess = { + resolutionPlaylistFile: outputPath, + videoFile: videoPath + } + + await server.runnerJobs.success({ + jobToken: job.jobToken, + jobUUID: job.uuid, + runnerToken, + payload: successBody + }) + } finally { + await remove(inputPath) + await remove(outputPath) + await remove(videoPath) } - - await server.runnerJobs.success({ - jobToken: job.jobToken, - jobUUID: job.uuid, - runnerToken, - payload: successBody - }) - - await remove(outputPath) - await remove(videoPath) } export async function processAudioMergeTranscoding (options: ProcessOptions) { diff --git a/packages/peertube-runner/server/server.ts b/packages/peertube-runner/server/server.ts index e851dfc7c..8eff4bd2f 100644 --- a/packages/peertube-runner/server/server.ts +++ b/packages/peertube-runner/server/server.ts @@ -8,6 +8,7 @@ import { ConfigManager } from '../shared' import { IPCServer } from '../shared/ipc' import { logger } from '../shared/logger' import { JobWithToken, processJob } from './process' +import { isJobSupported } from './shared' type PeerTubeServer = PeerTubeServerCommand & { runnerToken: string @@ -199,12 +200,14 @@ export class RunnerServer { const { availableJobs } = await server.runnerJobs.request({ runnerToken: server.runnerToken }) - if (availableJobs.length === 0) { + const filtered = availableJobs.filter(j => isJobSupported(j)) + + if (filtered.length === 0) { logger.debug(`No job available on ${server.url} for runner ${server.runnerName}`) return undefined } - return availableJobs[0] + return filtered[0] } private async tryToExecuteJobAsync (server: PeerTubeServer, jobToAccept: { uuid: string }) { diff --git a/packages/peertube-runner/server/shared/index.ts b/packages/peertube-runner/server/shared/index.ts new file mode 100644 index 000000000..5c86bafc0 --- /dev/null +++ b/packages/peertube-runner/server/shared/index.ts @@ -0,0 +1 @@ +export * from './supported-job' diff --git a/packages/peertube-runner/server/shared/supported-job.ts b/packages/peertube-runner/server/shared/supported-job.ts new file mode 100644 index 000000000..87d5a39cc --- /dev/null +++ b/packages/peertube-runner/server/shared/supported-job.ts @@ -0,0 +1,43 @@ +import { + RunnerJobLiveRTMPHLSTranscodingPayload, + RunnerJobPayload, + RunnerJobType, + RunnerJobVideoEditionTranscodingPayload, + RunnerJobVODAudioMergeTranscodingPayload, + RunnerJobVODHLSTranscodingPayload, + RunnerJobVODWebVideoTranscodingPayload, + VideoStudioTaskPayload +} from '@shared/models' + +const supportedMatrix = { + 'vod-web-video-transcoding': (_payload: RunnerJobVODWebVideoTranscodingPayload) => { + return true + }, + 'vod-hls-transcoding': (_payload: RunnerJobVODHLSTranscodingPayload) => { + return true + }, + 'vod-audio-merge-transcoding': (_payload: RunnerJobVODAudioMergeTranscodingPayload) => { + return true + }, + 'live-rtmp-hls-transcoding': (_payload: RunnerJobLiveRTMPHLSTranscodingPayload) => { + return true + }, + 'video-edition-transcoding': (payload: RunnerJobVideoEditionTranscodingPayload) => { + const tasks = payload?.tasks + const supported = new Set([ 'add-intro', 'add-outro', 'add-watermark', 'cut' ]) + + if (!Array.isArray(tasks)) return false + + return tasks.every(t => t && supported.has(t.name)) + } +} + +export function isJobSupported (job: { + type: RunnerJobType + payload: RunnerJobPayload +}) { + const fn = supportedMatrix[job.type] + if (!fn) return false + + return fn(job.payload as any) +} diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts index 0b9aaffda..3b6230f4a 100644 --- a/server/controllers/api/config.ts +++ b/server/controllers/api/config.ts @@ -274,7 +274,10 @@ function customConfig (): CustomConfig { } }, videoStudio: { - enabled: CONFIG.VIDEO_STUDIO.ENABLED + enabled: CONFIG.VIDEO_STUDIO.ENABLED, + remoteRunners: { + enabled: CONFIG.VIDEO_STUDIO.REMOTE_RUNNERS.ENABLED + } }, import: { videos: { diff --git a/server/controllers/api/runners/jobs-files.ts b/server/controllers/api/runners/jobs-files.ts index e43ce35f5..4efa40b3a 100644 --- a/server/controllers/api/runners/jobs-files.ts +++ b/server/controllers/api/runners/jobs-files.ts @@ -2,9 +2,13 @@ import express from 'express' import { logger, loggerTagsFactory } from '@server/helpers/logger' import { proxifyHLS, proxifyWebTorrentFile } from '@server/lib/object-storage' import { VideoPathManager } from '@server/lib/video-path-manager' +import { getStudioTaskFilePath } from '@server/lib/video-studio' import { asyncMiddleware } from '@server/middlewares' import { jobOfRunnerGetValidator } from '@server/middlewares/validators/runners' -import { runnerJobGetVideoTranscodingFileValidator } from '@server/middlewares/validators/runners/job-files' +import { + runnerJobGetVideoStudioTaskFileValidator, + runnerJobGetVideoTranscodingFileValidator +} from '@server/middlewares/validators/runners/job-files' import { VideoStorage } from '@shared/models' const lTags = loggerTagsFactory('api', 'runner') @@ -23,6 +27,13 @@ runnerJobFilesRouter.post('/jobs/:jobUUID/files/videos/:videoId/previews/max-qua getMaxQualityVideoPreview ) +runnerJobFilesRouter.post('/jobs/:jobUUID/files/videos/:videoId/studio/task-files/:filename', + asyncMiddleware(jobOfRunnerGetValidator), + asyncMiddleware(runnerJobGetVideoTranscodingFileValidator), + runnerJobGetVideoStudioTaskFileValidator, + getVideoEditionTaskFile +) + // --------------------------------------------------------------------------- export { @@ -82,3 +93,17 @@ function getMaxQualityVideoPreview (req: express.Request, res: express.Response) return res.sendFile(file.getPath()) } + +function getVideoEditionTaskFile (req: express.Request, res: express.Response) { + const runnerJob = res.locals.runnerJob + const runner = runnerJob.Runner + const video = res.locals.videoAll + const filename = req.params.filename + + logger.info( + 'Get video edition task file %s of video %s of job %s for runner %s', filename, video.uuid, runnerJob.uuid, runner.name, + lTags(runner.name, runnerJob.id, runnerJob.type) + ) + + return res.sendFile(getStudioTaskFilePath(filename)) +} diff --git a/server/controllers/api/runners/jobs.ts b/server/controllers/api/runners/jobs.ts index 7d488ec11..8e34c07a3 100644 --- a/server/controllers/api/runners/jobs.ts +++ b/server/controllers/api/runners/jobs.ts @@ -17,6 +17,7 @@ import { import { abortRunnerJobValidator, acceptRunnerJobValidator, + cancelRunnerJobValidator, errorRunnerJobValidator, getRunnerFromTokenValidator, jobOfRunnerGetValidator, @@ -41,6 +42,7 @@ import { RunnerJobUpdateBody, RunnerJobUpdatePayload, UserRight, + VideoEditionTranscodingSuccess, VODAudioMergeTranscodingSuccess, VODHLSTranscodingSuccess, VODWebVideoTranscodingSuccess @@ -110,6 +112,7 @@ runnerJobsRouter.post('/jobs/:jobUUID/cancel', authenticate, ensureUserHasRight(UserRight.MANAGE_RUNNERS), asyncMiddleware(runnerJobGetValidator), + cancelRunnerJobValidator, asyncMiddleware(cancelRunnerJob) ) @@ -297,6 +300,14 @@ const jobSuccessPayloadBuilders: { } }, + 'video-edition-transcoding': (payload: VideoEditionTranscodingSuccess, files) => { + return { + ...payload, + + videoFile: files['payload[videoFile]'][0].path + } + }, + 'live-rtmp-hls-transcoding': () => ({}) } @@ -327,7 +338,7 @@ async function postRunnerJobSuccess (req: express.Request, res: express.Response async function cancelRunnerJob (req: express.Request, res: express.Response) { const runnerJob = res.locals.runnerJob - logger.info('Cancelling job %s (%s)', runnerJob.type, lTags(runnerJob.uuid, runnerJob.type)) + logger.info('Cancelling job %s (%s)', runnerJob.uuid, runnerJob.type, lTags(runnerJob.uuid, runnerJob.type)) const RunnerJobHandler = getRunnerJobHandlerClass(runnerJob) await new RunnerJobHandler().cancel({ runnerJob }) diff --git a/server/controllers/api/videos/studio.ts b/server/controllers/api/videos/studio.ts index 2ccb2fb89..7c31dfd2b 100644 --- a/server/controllers/api/videos/studio.ts +++ b/server/controllers/api/videos/studio.ts @@ -1,12 +1,10 @@ import Bluebird from 'bluebird' import express from 'express' import { move } from 'fs-extra' -import { basename, join } from 'path' +import { basename } from 'path' import { createAnyReqFiles } from '@server/helpers/express-utils' -import { CONFIG } from '@server/initializers/config' -import { MIMETYPES } from '@server/initializers/constants' -import { JobQueue } from '@server/lib/job-queue' -import { buildTaskFileFieldname, getTaskFileFromReq } from '@server/lib/video-studio' +import { MIMETYPES, VIDEO_FILTERS } from '@server/initializers/constants' +import { buildTaskFileFieldname, createVideoStudioJob, getStudioTaskFilePath, getTaskFileFromReq } from '@server/lib/video-studio' import { HttpStatusCode, VideoState, @@ -75,7 +73,11 @@ async function createEditionTasks (req: express.Request, res: express.Response) tasks: await Bluebird.mapSeries(body.tasks, (t, i) => buildTaskPayload(t, i, files)) } - JobQueue.Instance.createJobAsync({ type: 'video-studio-edition', payload }) + await createVideoStudioJob({ + user: res.locals.oauth.token.User, + payload, + video + }) return res.sendStatus(HttpStatusCode.NO_CONTENT_204) } @@ -124,13 +126,16 @@ async function buildWatermarkTask (task: VideoStudioTaskWatermark, indice: numbe return { name: task.name, options: { - file: destination + file: destination, + watermarkSizeRatio: VIDEO_FILTERS.WATERMARK.SIZE_RATIO, + horitonzalMarginRatio: VIDEO_FILTERS.WATERMARK.HORIZONTAL_MARGIN_RATIO, + verticalMarginRatio: VIDEO_FILTERS.WATERMARK.VERTICAL_MARGIN_RATIO } } } async function moveStudioFileToPersistentTMP (file: string) { - const destination = join(CONFIG.STORAGE.TMP_PERSISTENT_DIR, basename(file)) + const destination = getStudioTaskFilePath(basename(file)) await move(file, destination) diff --git a/server/helpers/custom-validators/misc.ts b/server/helpers/custom-validators/misc.ts index fa0f469f6..2c4cd1b9f 100644 --- a/server/helpers/custom-validators/misc.ts +++ b/server/helpers/custom-validators/misc.ts @@ -15,8 +15,12 @@ function isSafePath (p: string) { }) } -function isSafeFilename (filename: string, extension: string) { - return typeof filename === 'string' && !!filename.match(new RegExp(`^[a-z0-9-]+\\.${extension}$`)) +function isSafeFilename (filename: string, extension?: string) { + const regex = extension + ? new RegExp(`^[a-z0-9-]+\\.${extension}$`) + : new RegExp(`^[a-z0-9-]+\\.[a-z0-9]{1,8}$`) + + return typeof filename === 'string' && !!filename.match(regex) } function isSafePeerTubeFilenameWithoutExtension (filename: string) { diff --git a/server/helpers/custom-validators/runners/jobs.ts b/server/helpers/custom-validators/runners/jobs.ts index 5f755d5bb..934bd37c9 100644 --- a/server/helpers/custom-validators/runners/jobs.ts +++ b/server/helpers/custom-validators/runners/jobs.ts @@ -6,6 +6,7 @@ import { RunnerJobSuccessPayload, RunnerJobType, RunnerJobUpdatePayload, + VideoEditionTranscodingSuccess, VODAudioMergeTranscodingSuccess, VODHLSTranscodingSuccess, VODWebVideoTranscodingSuccess @@ -23,7 +24,8 @@ function isRunnerJobSuccessPayloadValid (value: RunnerJobSuccessPayload, type: R return isRunnerJobVODWebVideoResultPayloadValid(value as VODWebVideoTranscodingSuccess, type, files) || isRunnerJobVODHLSResultPayloadValid(value as VODHLSTranscodingSuccess, type, files) || isRunnerJobVODAudioMergeResultPayloadValid(value as VODHLSTranscodingSuccess, type, files) || - isRunnerJobLiveRTMPHLSResultPayloadValid(value as LiveRTMPHLSTranscodingSuccess, type) + isRunnerJobLiveRTMPHLSResultPayloadValid(value as LiveRTMPHLSTranscodingSuccess, type) || + isRunnerJobVideoEditionResultPayloadValid(value as VideoEditionTranscodingSuccess, type, files) } // --------------------------------------------------------------------------- @@ -35,6 +37,7 @@ function isRunnerJobProgressValid (value: string) { function isRunnerJobUpdatePayloadValid (value: RunnerJobUpdatePayload, type: RunnerJobType, files: UploadFilesForCheck) { return isRunnerJobVODWebVideoUpdatePayloadValid(value, type, files) || isRunnerJobVODHLSUpdatePayloadValid(value, type, files) || + isRunnerJobVideoEditionUpdatePayloadValid(value, type, files) || isRunnerJobVODAudioMergeUpdatePayloadValid(value, type, files) || isRunnerJobLiveRTMPHLSUpdatePayloadValid(value, type, files) } @@ -102,6 +105,15 @@ function isRunnerJobLiveRTMPHLSResultPayloadValid ( return type === 'live-rtmp-hls-transcoding' && (!value || (typeof value === 'object' && Object.keys(value).length === 0)) } +function isRunnerJobVideoEditionResultPayloadValid ( + _value: VideoEditionTranscodingSuccess, + type: RunnerJobType, + files: UploadFilesForCheck +) { + return type === 'video-edition-transcoding' && + isFileValid({ files, field: 'payload[videoFile]', mimeTypeRegex: null, maxSize: null }) +} + // --------------------------------------------------------------------------- function isRunnerJobVODWebVideoUpdatePayloadValid ( @@ -164,3 +176,12 @@ function isRunnerJobLiveRTMPHLSUpdatePayloadValid ( ) ) } + +function isRunnerJobVideoEditionUpdatePayloadValid ( + value: RunnerJobUpdatePayload, + type: RunnerJobType, + _files: UploadFilesForCheck +) { + return type === 'video-edition-transcoding' && + (!value || (typeof value === 'object' && Object.keys(value).length === 0)) +} diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts index 2361aa1eb..2f5a274e4 100644 --- a/server/initializers/checker-before-init.ts +++ b/server/initializers/checker-before-init.ts @@ -38,7 +38,7 @@ function checkMissedConfig () { 'transcoding.resolutions.0p', 'transcoding.resolutions.144p', 'transcoding.resolutions.240p', 'transcoding.resolutions.360p', 'transcoding.resolutions.480p', 'transcoding.resolutions.720p', 'transcoding.resolutions.1080p', 'transcoding.resolutions.1440p', 'transcoding.resolutions.2160p', 'transcoding.always_transcode_original_resolution', 'transcoding.remote_runners.enabled', - 'video_studio.enabled', + 'video_studio.enabled', 'video_studio.remote_runners.enabled', 'remote_runners.stalled_jobs.vod', 'remote_runners.stalled_jobs.live', 'import.videos.http.enabled', 'import.videos.torrent.enabled', 'import.videos.concurrency', 'import.videos.timeout', 'import.video_channel_synchronization.enabled', 'import.video_channel_synchronization.max_per_user', diff --git a/server/initializers/config.ts b/server/initializers/config.ts index f2d8f99b5..9c2705689 100644 --- a/server/initializers/config.ts +++ b/server/initializers/config.ts @@ -423,7 +423,10 @@ const CONFIG = { } }, VIDEO_STUDIO: { - get ENABLED () { return config.get('video_studio.enabled') } + get ENABLED () { return config.get('video_studio.enabled') }, + REMOTE_RUNNERS: { + get ENABLED () { return config.get('video_studio.remote_runners.enabled') } + } }, IMPORT: { VIDEOS: { diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 279e77421..6a757a0ff 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -229,7 +229,8 @@ const REPEAT_JOBS: { [ id in JobType ]?: RepeatOptions } = { } } const JOB_PRIORITY = { - TRANSCODING: 100 + TRANSCODING: 100, + VIDEO_STUDIO: 150 } const JOB_REMOVAL_OPTIONS = { diff --git a/server/lib/job-queue/handlers/video-studio-edition.ts b/server/lib/job-queue/handlers/video-studio-edition.ts index 5e8dd4f51..df73caf72 100644 --- a/server/lib/job-queue/handlers/video-studio-edition.ts +++ b/server/lib/job-queue/handlers/video-studio-edition.ts @@ -1,25 +1,18 @@ import { Job } from 'bullmq' -import { move, remove } from 'fs-extra' +import { remove } from 'fs-extra' import { join } from 'path' import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg' -import { createTorrentAndSetInfoHashFromPath } from '@server/helpers/webtorrent' import { CONFIG } from '@server/initializers/config' -import { VIDEO_FILTERS } from '@server/initializers/constants' -import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' -import { generateWebTorrentVideoFilename } from '@server/lib/paths' -import { createOptimizeOrMergeAudioJobs } from '@server/lib/transcoding/create-transcoding-job' import { VideoTranscodingProfilesManager } from '@server/lib/transcoding/default-transcoding-profiles' import { isAbleToUploadVideo } from '@server/lib/user' -import { buildFileMetadata, removeHLSPlaylist, removeWebTorrentFile } from '@server/lib/video-file' import { VideoPathManager } from '@server/lib/video-path-manager' -import { approximateIntroOutroAdditionalSize, safeCleanupStudioTMPFiles } from '@server/lib/video-studio' +import { approximateIntroOutroAdditionalSize, onVideoEditionEnded, safeCleanupStudioTMPFiles } from '@server/lib/video-studio' import { UserModel } from '@server/models/user/user' import { VideoModel } from '@server/models/video/video' -import { VideoFileModel } from '@server/models/video/video-file' -import { MVideo, MVideoFile, MVideoFullLight, MVideoId, MVideoWithAllFiles } from '@server/types/models' -import { getLowercaseExtension, pick } from '@shared/core-utils' -import { buildUUID, getFileSize } from '@shared/extra-utils' -import { FFmpegEdition, ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamDuration, getVideoStreamFPS } from '@shared/ffmpeg' +import { MVideo, MVideoFullLight } from '@server/types/models' +import { pick } from '@shared/core-utils' +import { buildUUID } from '@shared/extra-utils' +import { FFmpegEdition } from '@shared/ffmpeg' import { VideoStudioEditionPayload, VideoStudioTask, @@ -46,7 +39,7 @@ async function processVideoStudioEdition (job: Job) { if (!video) { logger.info('Can\'t process job %d, video does not exist.', job.id, lTags) - await safeCleanupStudioTMPFiles(payload) + await safeCleanupStudioTMPFiles(payload.tasks) return undefined } @@ -81,28 +74,9 @@ async function processVideoStudioEdition (job: Job) { logger.info('Video edition ended for video %s.', video.uuid, lTags) - const newFile = await buildNewFile(video, editionResultPath) - - const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newFile) - await move(editionResultPath, outputPath) - - await safeCleanupStudioTMPFiles(payload) - - await createTorrentAndSetInfoHashFromPath(video, newFile, outputPath) - await removeAllFiles(video, newFile) - - await newFile.save() - - video.duration = await getVideoStreamDuration(outputPath) - await video.save() - - await federateVideoIfNeeded(video, false, undefined) - - const user = await UserModel.loadByVideoId(video.id) - - await createOptimizeOrMergeAudioJobs({ video, videoFile: newFile, isNewVideo: false, user, videoFileAlreadyLocked: false }) + await onVideoEditionEnded({ video, editionResultPath, tasks: payload.tasks }) } catch (err) { - await safeCleanupStudioTMPFiles(payload) + await safeCleanupStudioTMPFiles(payload.tasks) throw err } @@ -181,44 +155,15 @@ function processAddWatermark (options: TaskProcessorOptions payload: RunnerJobLiveRTMPHLSTranscodingPayload privatePayload: RunnerJobLiveRTMPHLSTranscodingPrivatePayload + } | + { + type: Extract + payload: RunnerJobVideoEditionTranscodingPayload + privatePayload: RunnerJobVideoEditionTranscodingPrivatePayload } export abstract class AbstractJobHandler { @@ -62,6 +69,8 @@ export abstract class AbstractJobHandler { const { priority, dependsOnRunnerJob } = options + logger.debug('Creating runner job', { options, ...this.lTags(options.type) }) + const runnerJob = new RunnerJobModel({ ...pick(options, [ 'type', 'payload', 'privatePayload' ]), diff --git a/server/lib/runners/job-handlers/abstract-vod-transcoding-job-handler.ts b/server/lib/runners/job-handlers/abstract-vod-transcoding-job-handler.ts index 517645848..a910ae383 100644 --- a/server/lib/runners/job-handlers/abstract-vod-transcoding-job-handler.ts +++ b/server/lib/runners/job-handlers/abstract-vod-transcoding-job-handler.ts @@ -4,27 +4,19 @@ import { logger } from '@server/helpers/logger' import { moveToFailedTranscodingState, moveToNextState } from '@server/lib/video-state' import { VideoJobInfoModel } from '@server/models/video/video-job-info' import { MRunnerJob } from '@server/types/models/runners' -import { - LiveRTMPHLSTranscodingUpdatePayload, - RunnerJobSuccessPayload, - RunnerJobUpdatePayload, - RunnerJobVODPrivatePayload -} from '@shared/models' +import { RunnerJobSuccessPayload, RunnerJobUpdatePayload, RunnerJobVODPrivatePayload } from '@shared/models' import { AbstractJobHandler } from './abstract-job-handler' import { loadTranscodingRunnerVideo } from './shared' // eslint-disable-next-line max-len export abstract class AbstractVODTranscodingJobHandler extends AbstractJobHandler { - // --------------------------------------------------------------------------- - protected isAbortSupported () { return true } protected specificUpdate (_options: { runnerJob: MRunnerJob - updatePayload?: LiveRTMPHLSTranscodingUpdatePayload }) { // empty } diff --git a/server/lib/runners/job-handlers/index.ts b/server/lib/runners/job-handlers/index.ts index 0fca72b9a..a40cee865 100644 --- a/server/lib/runners/job-handlers/index.ts +++ b/server/lib/runners/job-handlers/index.ts @@ -1,6 +1,7 @@ export * from './abstract-job-handler' export * from './live-rtmp-hls-transcoding-job-handler' +export * from './runner-job-handlers' +export * from './video-edition-transcoding-job-handler' export * from './vod-audio-merge-transcoding-job-handler' export * from './vod-hls-transcoding-job-handler' export * from './vod-web-video-transcoding-job-handler' -export * from './runner-job-handlers' diff --git a/server/lib/runners/job-handlers/live-rtmp-hls-transcoding-job-handler.ts b/server/lib/runners/job-handlers/live-rtmp-hls-transcoding-job-handler.ts index c3d0e427d..48a70d891 100644 --- a/server/lib/runners/job-handlers/live-rtmp-hls-transcoding-job-handler.ts +++ b/server/lib/runners/job-handlers/live-rtmp-hls-transcoding-job-handler.ts @@ -70,7 +70,7 @@ export class LiveRTMPHLSTranscodingJobHandler extends AbstractJobHandler AbstractJobHandler { + + async create (options: CreateOptions) { + const { video, priority, tasks } = options + + const jobUUID = buildUUID() + const payload: RunnerJobVideoEditionTranscodingPayload = { + input: { + videoFileUrl: generateRunnerTranscodingVideoInputFileUrl(jobUUID, video.uuid) + }, + tasks: tasks.map(t => { + if (isVideoStudioTaskIntro(t) || isVideoStudioTaskOutro(t)) { + return { + ...t, + + options: { + ...t.options, + + file: generateRunnerEditionTranscodingVideoInputFileUrl(jobUUID, video.uuid, basename(t.options.file)) + } + } + } + + if (isVideoStudioTaskWatermark(t)) { + return { + ...t, + + options: { + ...t.options, + + file: generateRunnerEditionTranscodingVideoInputFileUrl(jobUUID, video.uuid, basename(t.options.file)) + } + } + } + + return t + }) + } + + const privatePayload: RunnerJobVideoEditionTranscodingPrivatePayload = { + videoUUID: video.uuid, + originalTasks: tasks + } + + const job = await this.createRunnerJob({ + type: 'video-edition-transcoding', + jobUUID, + payload, + privatePayload, + priority + }) + + return job + } + + // --------------------------------------------------------------------------- + + protected isAbortSupported () { + return true + } + + protected specificUpdate (_options: { + runnerJob: MRunnerJob + }) { + // empty + } + + protected specificAbort (_options: { + runnerJob: MRunnerJob + }) { + // empty + } + + protected async specificComplete (options: { + runnerJob: MRunnerJob + resultPayload: VideoEditionTranscodingSuccess + }) { + const { runnerJob, resultPayload } = options + const privatePayload = runnerJob.privatePayload as RunnerJobVideoEditionTranscodingPrivatePayload + + const video = await loadTranscodingRunnerVideo(runnerJob, this.lTags) + if (!video) { + await safeCleanupStudioTMPFiles(privatePayload.originalTasks) + + } + + const videoFilePath = resultPayload.videoFile as string + + await onVideoEditionEnded({ video, editionResultPath: videoFilePath, tasks: privatePayload.originalTasks }) + + logger.info( + 'Runner video edition transcoding job %s for %s ended.', + runnerJob.uuid, video.uuid, this.lTags(video.uuid, runnerJob.uuid) + ) + } + + protected specificError (options: { + runnerJob: MRunnerJob + nextState: RunnerJobState + }) { + if (options.nextState === RunnerJobState.ERRORED) { + return this.specificErrorOrCancel(options) + } + + return Promise.resolve() + } + + protected specificCancel (options: { + runnerJob: MRunnerJob + }) { + return this.specificErrorOrCancel(options) + } + + private async specificErrorOrCancel (options: { + runnerJob: MRunnerJob + }) { + const { runnerJob } = options + + const payload = runnerJob.privatePayload as RunnerJobVideoEditionTranscodingPrivatePayload + await safeCleanupStudioTMPFiles(payload.originalTasks) + + const video = await loadTranscodingRunnerVideo(options.runnerJob, this.lTags) + if (!video) return + + return video.setNewState(VideoState.PUBLISHED, false, undefined) + } +} diff --git a/server/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts b/server/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts index a7b33f87e..5f247d792 100644 --- a/server/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts +++ b/server/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts @@ -64,7 +64,7 @@ export class VODAudioMergeTranscodingJobHandler extends AbstractVODTranscodingJo // --------------------------------------------------------------------------- - async specificComplete (options: { + protected async specificComplete (options: { runnerJob: MRunnerJob resultPayload: VODAudioMergeTranscodingSuccess }) { diff --git a/server/lib/runners/job-handlers/vod-hls-transcoding-job-handler.ts b/server/lib/runners/job-handlers/vod-hls-transcoding-job-handler.ts index 02566b9d5..cc94bcbda 100644 --- a/server/lib/runners/job-handlers/vod-hls-transcoding-job-handler.ts +++ b/server/lib/runners/job-handlers/vod-hls-transcoding-job-handler.ts @@ -71,7 +71,7 @@ export class VODHLSTranscodingJobHandler extends AbstractVODTranscodingJobHandle // --------------------------------------------------------------------------- - async specificComplete (options: { + protected async specificComplete (options: { runnerJob: MRunnerJob resultPayload: VODHLSTranscodingSuccess }) { diff --git a/server/lib/runners/job-handlers/vod-web-video-transcoding-job-handler.ts b/server/lib/runners/job-handlers/vod-web-video-transcoding-job-handler.ts index 57761a7a1..663d3306e 100644 --- a/server/lib/runners/job-handlers/vod-web-video-transcoding-job-handler.ts +++ b/server/lib/runners/job-handlers/vod-web-video-transcoding-job-handler.ts @@ -62,7 +62,7 @@ export class VODWebVideoTranscodingJobHandler extends AbstractVODTranscodingJobH // --------------------------------------------------------------------------- - async specificComplete (options: { + protected async specificComplete (options: { runnerJob: MRunnerJob resultPayload: VODWebVideoTranscodingSuccess }) { diff --git a/server/lib/runners/runner-urls.ts b/server/lib/runners/runner-urls.ts index 329fb1170..a27060b33 100644 --- a/server/lib/runners/runner-urls.ts +++ b/server/lib/runners/runner-urls.ts @@ -7,3 +7,7 @@ export function generateRunnerTranscodingVideoInputFileUrl (jobUUID: string, vid export function generateRunnerTranscodingVideoPreviewFileUrl (jobUUID: string, videoUUID: string) { return WEBSERVER.URL + '/api/v1/runners/jobs/' + jobUUID + '/files/videos/' + videoUUID + '/previews/max-quality' } + +export function generateRunnerEditionTranscodingVideoInputFileUrl (jobUUID: string, videoUUID: string, filename: string) { + return WEBSERVER.URL + '/api/v1/runners/jobs/' + jobUUID + '/files/videos/' + videoUUID + '/studio/task-files/' + filename +} diff --git a/server/lib/server-config-manager.ts b/server/lib/server-config-manager.ts index ba7916363..924adb337 100644 --- a/server/lib/server-config-manager.ts +++ b/server/lib/server-config-manager.ts @@ -166,7 +166,10 @@ class ServerConfigManager { } }, videoStudio: { - enabled: CONFIG.VIDEO_STUDIO.ENABLED + enabled: CONFIG.VIDEO_STUDIO.ENABLED, + remoteRunners: { + enabled: CONFIG.VIDEO_STUDIO.REMOTE_RUNNERS.ENABLED + } }, import: { videos: { diff --git a/server/lib/transcoding/shared/job-builders/abstract-job-builder.ts b/server/lib/transcoding/shared/job-builders/abstract-job-builder.ts index 576e786d5..80dc05bfb 100644 --- a/server/lib/transcoding/shared/job-builders/abstract-job-builder.ts +++ b/server/lib/transcoding/shared/job-builders/abstract-job-builder.ts @@ -1,6 +1,4 @@ -import { JOB_PRIORITY } from '@server/initializers/constants' -import { VideoModel } from '@server/models/video/video' import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models' export abstract class AbstractJobBuilder { @@ -20,20 +18,4 @@ export abstract class AbstractJobBuilder { isNewVideo: boolean user: MUserId | null }): Promise - - protected async getTranscodingJobPriority (options: { - user: MUserId - fallback: number - }) { - const { user, fallback } = options - - if (!user) return fallback - - const now = new Date() - const lastWeek = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 7) - - const videoUploadedByUser = await VideoModel.countVideosUploadedByUserSince(user.id, lastWeek) - - return JOB_PRIORITY.TRANSCODING + videoUploadedByUser - } } diff --git a/server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts b/server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts index 5a9c93ee5..29ee2ca61 100644 --- a/server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts +++ b/server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts @@ -16,6 +16,7 @@ import { OptimizeTranscodingPayload, VideoTranscodingPayload } from '@shared/models' +import { getTranscodingJobPriority } from '../../transcoding-priority' import { canDoQuickTranscode } from '../../transcoding-quick-transcode' import { computeResolutionsToTranscode } from '../../transcoding-resolutions' import { AbstractJobBuilder } from './abstract-job-builder' @@ -178,7 +179,7 @@ export class TranscodingJobQueueBuilder extends AbstractJobBuilder { return { type: 'video-transcoding' as 'video-transcoding', - priority: await this.getTranscodingJobPriority({ user, fallback: undefined }), + priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: undefined }), payload } } diff --git a/server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts b/server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts index 274dce21b..90b035402 100644 --- a/server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts +++ b/server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts @@ -8,6 +8,7 @@ import { VideoPathManager } from '@server/lib/video-path-manager' import { MUserId, MVideoFile, MVideoFullLight, MVideoWithFileThumbnail } from '@server/types/models' import { MRunnerJob } from '@server/types/models/runners' import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS, hasAudioStream, isAudioFile } from '@shared/ffmpeg' +import { getTranscodingJobPriority } from '../../transcoding-priority' import { computeResolutionsToTranscode } from '../../transcoding-resolutions' import { AbstractJobBuilder } from './abstract-job-builder' @@ -49,7 +50,7 @@ export class TranscodingRunnerJobBuilder extends AbstractJobBuilder { : resolution const fps = computeOutputFPS({ inputFPS, resolution: maxResolution }) - const priority = await this.getTranscodingJobPriority({ user, fallback: 0 }) + const priority = await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) const mainRunnerJob = videoFile.isAudio() ? await new VODAudioMergeTranscodingJobHandler().create({ video, resolution: maxResolution, fps, isNewVideo, priority }) @@ -63,7 +64,7 @@ export class TranscodingRunnerJobBuilder extends AbstractJobBuilder { fps, isNewVideo, dependsOnRunnerJob: mainRunnerJob, - priority: await this.getTranscodingJobPriority({ user, fallback: 0 }) + priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) }) } @@ -96,7 +97,7 @@ export class TranscodingRunnerJobBuilder extends AbstractJobBuilder { const maxResolution = Math.max(...resolutions) const { fps: inputFPS } = await video.probeMaxQualityFile() const maxFPS = computeOutputFPS({ inputFPS, resolution: maxResolution }) - const priority = await this.getTranscodingJobPriority({ user, fallback: 0 }) + const priority = await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) const childrenResolutions = resolutions.filter(r => r !== maxResolution) @@ -121,7 +122,7 @@ export class TranscodingRunnerJobBuilder extends AbstractJobBuilder { isNewVideo, deleteWebVideoFiles: false, dependsOnRunnerJob, - priority: await this.getTranscodingJobPriority({ user, fallback: 0 }) + priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) }) continue } @@ -133,7 +134,7 @@ export class TranscodingRunnerJobBuilder extends AbstractJobBuilder { fps, isNewVideo, dependsOnRunnerJob, - priority: await this.getTranscodingJobPriority({ user, fallback: 0 }) + priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) }) continue } @@ -172,7 +173,7 @@ export class TranscodingRunnerJobBuilder extends AbstractJobBuilder { fps, isNewVideo, dependsOnRunnerJob: mainRunnerJob, - priority: await this.getTranscodingJobPriority({ user, fallback: 0 }) + priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) }) } @@ -184,7 +185,7 @@ export class TranscodingRunnerJobBuilder extends AbstractJobBuilder { isNewVideo, deleteWebVideoFiles: false, dependsOnRunnerJob: mainRunnerJob, - priority: await this.getTranscodingJobPriority({ user, fallback: 0 }) + priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) }) } } diff --git a/server/lib/transcoding/transcoding-priority.ts b/server/lib/transcoding/transcoding-priority.ts new file mode 100644 index 000000000..82ab6f2f1 --- /dev/null +++ b/server/lib/transcoding/transcoding-priority.ts @@ -0,0 +1,24 @@ +import { JOB_PRIORITY } from '@server/initializers/constants' +import { VideoModel } from '@server/models/video/video' +import { MUserId } from '@server/types/models' + +export async function getTranscodingJobPriority (options: { + user: MUserId + fallback: number + type: 'vod' | 'studio' +}) { + const { user, fallback, type } = options + + if (!user) return fallback + + const now = new Date() + const lastWeek = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 7) + + const videoUploadedByUser = await VideoModel.countVideosUploadedByUserSince(user.id, lastWeek) + + const base = type === 'vod' + ? JOB_PRIORITY.TRANSCODING + : JOB_PRIORITY.VIDEO_STUDIO + + return base + videoUploadedByUser +} diff --git a/server/lib/video-studio.ts b/server/lib/video-studio.ts index beda326a0..2c993faeb 100644 --- a/server/lib/video-studio.ts +++ b/server/lib/video-studio.ts @@ -1,19 +1,38 @@ -import { logger } from '@server/helpers/logger' -import { MVideoFullLight } from '@server/types/models' +import { move, remove } from 'fs-extra' +import { join } from 'path' +import { logger, loggerTagsFactory } from '@server/helpers/logger' +import { createTorrentAndSetInfoHashFromPath } from '@server/helpers/webtorrent' +import { CONFIG } from '@server/initializers/config' +import { UserModel } from '@server/models/user/user' +import { MUser, MVideo, MVideoFile, MVideoFullLight, MVideoWithAllFiles } from '@server/types/models' import { getVideoStreamDuration } from '@shared/ffmpeg' -import { VideoStudioEditionPayload, VideoStudioTask } from '@shared/models' -import { remove } from 'fs-extra' +import { VideoStudioEditionPayload, VideoStudioTask, VideoStudioTaskPayload } from '@shared/models' +import { federateVideoIfNeeded } from './activitypub/videos' +import { JobQueue } from './job-queue' +import { VideoEditionTranscodingJobHandler } from './runners' +import { createOptimizeOrMergeAudioJobs } from './transcoding/create-transcoding-job' +import { getTranscodingJobPriority } from './transcoding/transcoding-priority' +import { buildNewFile, removeHLSPlaylist, removeWebTorrentFile } from './video-file' +import { VideoPathManager } from './video-path-manager' -function buildTaskFileFieldname (indice: number, fieldName = 'file') { +const lTags = loggerTagsFactory('video-edition') + +export function buildTaskFileFieldname (indice: number, fieldName = 'file') { return `tasks[${indice}][options][${fieldName}]` } -function getTaskFileFromReq (files: Express.Multer.File[], indice: number, fieldName = 'file') { +export function getTaskFileFromReq (files: Express.Multer.File[], indice: number, fieldName = 'file') { return files.find(f => f.fieldname === buildTaskFileFieldname(indice, fieldName)) } -async function safeCleanupStudioTMPFiles (payload: VideoStudioEditionPayload) { - for (const task of payload.tasks) { +export function getStudioTaskFilePath (filename: string) { + return join(CONFIG.STORAGE.TMP_PERSISTENT_DIR, filename) +} + +export async function safeCleanupStudioTMPFiles (tasks: VideoStudioTaskPayload[]) { + logger.info('Removing studio task files', { tasks, ...lTags() }) + + for (const task of tasks) { try { if (task.name === 'add-intro' || task.name === 'add-outro') { await remove(task.options.file) @@ -26,7 +45,13 @@ async function safeCleanupStudioTMPFiles (payload: VideoStudioEditionPayload) { } } -async function approximateIntroOutroAdditionalSize (video: MVideoFullLight, tasks: VideoStudioTask[], fileFinder: (i: number) => string) { +// --------------------------------------------------------------------------- + +export async function approximateIntroOutroAdditionalSize ( + video: MVideoFullLight, + tasks: VideoStudioTask[], + fileFinder: (i: number) => string +) { let additionalDuration = 0 for (let i = 0; i < tasks.length; i++) { @@ -41,9 +66,65 @@ async function approximateIntroOutroAdditionalSize (video: MVideoFullLight, task return (video.getMaxQualityFile().size / video.duration) * additionalDuration } -export { - approximateIntroOutroAdditionalSize, - buildTaskFileFieldname, - getTaskFileFromReq, - safeCleanupStudioTMPFiles +// --------------------------------------------------------------------------- + +export async function createVideoStudioJob (options: { + video: MVideo + user: MUser + payload: VideoStudioEditionPayload +}) { + const { video, user, payload } = options + + const priority = await getTranscodingJobPriority({ user, type: 'studio', fallback: 0 }) + + if (CONFIG.VIDEO_STUDIO.REMOTE_RUNNERS.ENABLED) { + await new VideoEditionTranscodingJobHandler().create({ video, tasks: payload.tasks, priority }) + return + } + + await JobQueue.Instance.createJob({ type: 'video-studio-edition', payload, priority }) +} + +export async function onVideoEditionEnded (options: { + editionResultPath: string + tasks: VideoStudioTaskPayload[] + video: MVideoFullLight +}) { + const { video, tasks, editionResultPath } = options + + const newFile = await buildNewFile({ path: editionResultPath, mode: 'web-video' }) + newFile.videoId = video.id + + const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newFile) + await move(editionResultPath, outputPath) + + await safeCleanupStudioTMPFiles(tasks) + + await createTorrentAndSetInfoHashFromPath(video, newFile, outputPath) + await removeAllFiles(video, newFile) + + await newFile.save() + + video.duration = await getVideoStreamDuration(outputPath) + await video.save() + + await federateVideoIfNeeded(video, false, undefined) + + const user = await UserModel.loadByVideoId(video.id) + + await createOptimizeOrMergeAudioJobs({ video, videoFile: newFile, isNewVideo: false, user, videoFileAlreadyLocked: false }) +} + +// --------------------------------------------------------------------------- +// Private +// --------------------------------------------------------------------------- + +async function removeAllFiles (video: MVideoWithAllFiles, webTorrentFileException: MVideoFile) { + await removeHLSPlaylist(video) + + for (const file of video.VideoFiles) { + if (file.id === webTorrentFileException.id) continue + + await removeWebTorrentFile(video, file.id) + } } diff --git a/server/middlewares/validators/config.ts b/server/middlewares/validators/config.ts index b3e7e5011..a0074cb24 100644 --- a/server/middlewares/validators/config.ts +++ b/server/middlewares/validators/config.ts @@ -62,6 +62,7 @@ const customConfigUpdateValidator = [ body('transcoding.hls.enabled').isBoolean(), body('videoStudio.enabled').isBoolean(), + body('videoStudio.remoteRunners.enabled').isBoolean(), body('import.videos.concurrency').isInt({ min: 0 }), body('import.videos.http.enabled').isBoolean(), diff --git a/server/middlewares/validators/runners/job-files.ts b/server/middlewares/validators/runners/job-files.ts index 56afa39aa..e5afff0e5 100644 --- a/server/middlewares/validators/runners/job-files.ts +++ b/server/middlewares/validators/runners/job-files.ts @@ -1,5 +1,8 @@ import express from 'express' -import { HttpStatusCode } from '@shared/models' +import { param } from 'express-validator' +import { basename } from 'path' +import { isSafeFilename } from '@server/helpers/custom-validators/misc' +import { hasVideoStudioTaskFile, HttpStatusCode, RunnerJobVideoEditionTranscodingPayload } from '@shared/models' import { areValidationErrors, doesVideoExist, isValidVideoIdParam } from '../shared' const tags = [ 'runner' ] @@ -25,3 +28,33 @@ export const runnerJobGetVideoTranscodingFileValidator = [ return next() } ] + +export const runnerJobGetVideoStudioTaskFileValidator = [ + param('filename').custom(v => isSafeFilename(v)), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + const filename = req.params.filename + + const payload = res.locals.runnerJob.payload as RunnerJobVideoEditionTranscodingPayload + + const found = Array.isArray(payload?.tasks) && payload.tasks.some(t => { + if (hasVideoStudioTaskFile(t)) { + return basename(t.options.file) === filename + } + + return false + }) + + if (!found) { + return res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'File is not associated to this edition task', + tags: [ ...tags, res.locals.videoAll.uuid ] + }) + } + + return next() + } +] diff --git a/server/middlewares/validators/runners/jobs.ts b/server/middlewares/validators/runners/jobs.ts index 8cb87e946..de956a1ca 100644 --- a/server/middlewares/validators/runners/jobs.ts +++ b/server/middlewares/validators/runners/jobs.ts @@ -91,6 +91,28 @@ export const successRunnerJobValidator = [ } ] +export const cancelRunnerJobValidator = [ + (req: express.Request, res: express.Response, next: express.NextFunction) => { + const runnerJob = res.locals.runnerJob + + const allowedStates = new Set([ + RunnerJobState.PENDING, + RunnerJobState.PROCESSING, + RunnerJobState.WAITING_FOR_PARENT_JOB + ]) + + if (allowedStates.has(runnerJob.state) !== true) { + return res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'Cannot cancel this job that is not in "pending", "processing" or "waiting for parent job" state', + tags + }) + } + + return next() + } +] + export const runnerJobGetValidator = [ param('jobUUID').custom(isUUIDValid), diff --git a/server/tests/api/check-params/config.ts b/server/tests/api/check-params/config.ts index c5cda203e..472cad182 100644 --- a/server/tests/api/check-params/config.ts +++ b/server/tests/api/check-params/config.ts @@ -162,7 +162,10 @@ describe('Test config API validators', function () { } }, videoStudio: { - enabled: true + enabled: true, + remoteRunners: { + enabled: true + } }, import: { videos: { diff --git a/server/tests/api/check-params/runners.ts b/server/tests/api/check-params/runners.ts index 4da6fd91d..90a301392 100644 --- a/server/tests/api/check-params/runners.ts +++ b/server/tests/api/check-params/runners.ts @@ -1,6 +1,17 @@ +import { basename } from 'path' /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@server/tests/shared' -import { HttpStatusCode, RunnerJob, RunnerJobState, RunnerJobSuccessPayload, RunnerJobUpdatePayload, VideoPrivacy } from '@shared/models' +import { + HttpStatusCode, + isVideoStudioTaskIntro, + RunnerJob, + RunnerJobState, + RunnerJobSuccessPayload, + RunnerJobUpdatePayload, + RunnerJobVideoEditionTranscodingPayload, + VideoPrivacy, + VideoStudioTaskIntro +} from '@shared/models' import { cleanupTests, createSingleServer, @@ -10,6 +21,7 @@ import { setAccessTokensToServers, setDefaultVideoChannel, stopFfmpeg, + VideoStudioCommand, waitJobs } from '@shared/server-commands' @@ -53,7 +65,10 @@ describe('Test managing runners', function () { registrationTokenId = data[0].id await server.config.enableTranscoding(true, true) + await server.config.enableStudio() await server.config.enableRemoteTranscoding() + await server.config.enableRemoteStudio() + runnerToken = await server.runners.autoRegisterRunner() runnerToken2 = await server.runners.autoRegisterRunner() @@ -249,6 +264,10 @@ describe('Test managing runners', function () { await server.runnerJobs.cancelByAdmin({ jobUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) }) + it('Should fail with an already cancelled job', async function () { + await server.runnerJobs.cancelByAdmin({ jobUUID: cancelledJobUUID, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + it('Should succeed with the correct params', async function () { await server.runnerJobs.cancelByAdmin({ jobUUID }) }) @@ -296,9 +315,13 @@ describe('Test managing runners', function () { let pendingUUID: string - let liveAcceptedJob: RunnerJob & { jobToken: string } + let videoStudioUUID: string + let studioFile: string - async function fetchFiles (options: { + let liveAcceptedJob: RunnerJob & { jobToken: string } + let studioAcceptedJob: RunnerJob & { jobToken: string } + + async function fetchVideoInputFiles (options: { jobUUID: string videoUUID: string runnerToken: string @@ -315,6 +338,21 @@ describe('Test managing runners', function () { } } + async function fetchStudioFiles (options: { + jobUUID: string + videoUUID: string + runnerToken: string + jobToken: string + studioFile?: string + expectedStatus: HttpStatusCode + }) { + const { jobUUID, expectedStatus, videoUUID, runnerToken, jobToken, studioFile } = options + + const path = `/api/v1/runners/jobs/${jobUUID}/files/videos/${videoUUID}/studio/task-files/${studioFile}` + + await makePostBodyRequest({ url: server.url, path, fields: { runnerToken, jobToken }, expectedStatus }) + } + before(async function () { this.timeout(120000) @@ -352,6 +390,28 @@ describe('Test managing runners', function () { pendingUUID = availableJobs[0].uuid } + { + await server.config.disableTranscoding() + + const { uuid } = await server.videos.quickUpload({ name: 'video studio' }) + videoStudioUUID = uuid + + await server.config.enableTranscoding(true, true) + await server.config.enableStudio() + + await server.videoStudio.createEditionTasks({ + videoId: videoStudioUUID, + tasks: VideoStudioCommand.getComplexTask() + }) + + const { job } = await server.runnerJobs.autoAccept({ runnerToken, type: 'video-edition-transcoding' }) + studioAcceptedJob = job + + const tasks = (job.payload as RunnerJobVideoEditionTranscodingPayload).tasks + const fileUrl = (tasks.find(t => isVideoStudioTaskIntro(t)) as VideoStudioTaskIntro).options.file as string + studioFile = basename(fileUrl) + } + { await server.config.enableLive({ allowReplay: false, @@ -381,8 +441,6 @@ describe('Test managing runners', function () { jobToken: string expectedStatus: HttpStatusCode }) { - await fetchFiles({ ...options, videoUUID }) - await server.runnerJobs.abort({ ...options, reason: 'reason' }) await server.runnerJobs.update({ ...options }) await server.runnerJobs.error({ ...options, message: 'message' }) @@ -390,39 +448,95 @@ describe('Test managing runners', function () { } it('Should fail with an invalid job uuid', async function () { - await testEndpoints({ jobUUID: 'a', runnerToken, jobToken, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + const options = { jobUUID: 'a', runnerToken, expectedStatus: HttpStatusCode.BAD_REQUEST_400 } + + await testEndpoints({ ...options, jobToken }) + await fetchVideoInputFiles({ ...options, videoUUID, jobToken }) + await fetchStudioFiles({ ...options, videoUUID, jobToken: studioAcceptedJob.jobToken, studioFile }) }) it('Should fail with an unknown job uuid', async function () { - const jobUUID = badUUID - await testEndpoints({ jobUUID, runnerToken, jobToken, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + const options = { jobUUID: badUUID, runnerToken, expectedStatus: HttpStatusCode.NOT_FOUND_404 } + + await testEndpoints({ ...options, jobToken }) + await fetchVideoInputFiles({ ...options, videoUUID, jobToken }) + await fetchStudioFiles({ ...options, jobToken: studioAcceptedJob.jobToken, videoUUID, studioFile }) }) it('Should fail with an invalid runner token', async function () { - await testEndpoints({ jobUUID, runnerToken: '', jobToken, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + const options = { runnerToken: '', expectedStatus: HttpStatusCode.BAD_REQUEST_400 } + + await testEndpoints({ ...options, jobUUID, jobToken }) + await fetchVideoInputFiles({ ...options, jobUUID, videoUUID, jobToken }) + await fetchStudioFiles({ + ...options, + jobToken: studioAcceptedJob.jobToken, + jobUUID: studioAcceptedJob.uuid, + videoUUID: videoStudioUUID, + studioFile + }) }) it('Should fail with an unknown runner token', async function () { - const runnerToken = badUUID - await testEndpoints({ jobUUID, runnerToken, jobToken, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + const options = { runnerToken: badUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 } + + await testEndpoints({ ...options, jobUUID, jobToken }) + await fetchVideoInputFiles({ ...options, jobUUID, videoUUID, jobToken }) + await fetchStudioFiles({ + ...options, + jobToken: studioAcceptedJob.jobToken, + jobUUID: studioAcceptedJob.uuid, + videoUUID: videoStudioUUID, + studioFile + }) }) it('Should fail with an invalid job token job uuid', async function () { - await testEndpoints({ jobUUID, runnerToken, jobToken: '', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + const options = { runnerToken, jobToken: '', expectedStatus: HttpStatusCode.BAD_REQUEST_400 } + + await testEndpoints({ ...options, jobUUID }) + await fetchVideoInputFiles({ ...options, jobUUID, videoUUID }) + await fetchStudioFiles({ ...options, jobUUID: studioAcceptedJob.uuid, videoUUID: videoStudioUUID, studioFile }) }) it('Should fail with an unknown job token job uuid', async function () { - const jobToken = badUUID - await testEndpoints({ jobUUID, runnerToken, jobToken, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + const options = { runnerToken, jobToken: badUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 } + + await testEndpoints({ ...options, jobUUID }) + await fetchVideoInputFiles({ ...options, jobUUID, videoUUID }) + await fetchStudioFiles({ ...options, jobUUID: studioAcceptedJob.uuid, videoUUID: videoStudioUUID, studioFile }) }) it('Should fail with a runner token not associated to this job', async function () { - await testEndpoints({ jobUUID, runnerToken: runnerToken2, jobToken, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + const options = { runnerToken: runnerToken2, expectedStatus: HttpStatusCode.NOT_FOUND_404 } + + await testEndpoints({ ...options, jobUUID, jobToken }) + await fetchVideoInputFiles({ ...options, jobUUID, videoUUID, jobToken }) + await fetchStudioFiles({ + ...options, + jobToken: studioAcceptedJob.jobToken, + jobUUID: studioAcceptedJob.uuid, + videoUUID: videoStudioUUID, + studioFile + }) }) it('Should fail with a job uuid not associated to the job token', async function () { - await testEndpoints({ jobUUID: jobUUID2, runnerToken, jobToken, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - await testEndpoints({ jobUUID, runnerToken, jobToken: jobToken2, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + { + const options = { jobUUID: jobUUID2, runnerToken, expectedStatus: HttpStatusCode.NOT_FOUND_404 } + + await testEndpoints({ ...options, jobToken }) + await fetchVideoInputFiles({ ...options, jobToken, videoUUID }) + await fetchStudioFiles({ ...options, jobToken: studioAcceptedJob.jobToken, videoUUID: videoStudioUUID, studioFile }) + } + + { + const options = { runnerToken, jobToken: jobToken2, expectedStatus: HttpStatusCode.NOT_FOUND_404 } + + await testEndpoints({ ...options, jobUUID }) + await fetchVideoInputFiles({ ...options, jobUUID, videoUUID }) + await fetchStudioFiles({ ...options, jobUUID: studioAcceptedJob.uuid, videoUUID: videoStudioUUID, studioFile }) + } }) }) @@ -670,27 +784,82 @@ describe('Test managing runners', function () { }) }) }) + + describe('Video studio', function () { + + it('Should fail with an invalid video edition transcoding payload', async function () { + await server.runnerJobs.success({ + jobUUID: studioAcceptedJob.uuid, + jobToken: studioAcceptedJob.jobToken, + payload: { hello: 'video_short.mp4' } as any, + runnerToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + }) }) describe('Job files', function () { - describe('Video files', function () { + describe('Check video param for common job file routes', function () { + + async function fetchFiles (options: { + videoUUID?: string + expectedStatus: HttpStatusCode + }) { + await fetchVideoInputFiles({ videoUUID, ...options, jobToken, jobUUID, runnerToken }) + + await fetchStudioFiles({ + videoUUID: videoStudioUUID, + + ...options, + + jobToken: studioAcceptedJob.jobToken, + jobUUID: studioAcceptedJob.uuid, + runnerToken, + studioFile + }) + } it('Should fail with an invalid video id', async function () { - await fetchFiles({ videoUUID: 'a', jobUUID, runnerToken, jobToken, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + await fetchFiles({ + videoUUID: 'a', + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) }) it('Should fail with an unknown video id', async function () { const videoUUID = '910ec12a-d9e6-458b-a274-0abb655f9464' - await fetchFiles({ videoUUID, jobUUID, runnerToken, jobToken, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + + await fetchFiles({ + videoUUID, + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) }) it('Should fail with a video id not associated to this job', async function () { - await fetchFiles({ videoUUID: videoUUID2, jobUUID, runnerToken, jobToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + await fetchFiles({ + videoUUID: videoUUID2, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) }) it('Should succeed with the correct params', async function () { - await fetchFiles({ videoUUID, jobUUID, runnerToken, jobToken, expectedStatus: HttpStatusCode.OK_200 }) + await fetchFiles({ expectedStatus: HttpStatusCode.OK_200 }) + }) + }) + + describe('Video edition tasks file routes', function () { + + it('Should fail with an invalid studio filename', async function () { + await fetchStudioFiles({ + videoUUID: videoStudioUUID, + jobUUID: studioAcceptedJob.uuid, + runnerToken, + jobToken: studioAcceptedJob.jobToken, + studioFile: 'toto', + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) }) }) }) diff --git a/server/tests/api/runners/index.ts b/server/tests/api/runners/index.ts index 7f33ec8dd..642a3a96d 100644 --- a/server/tests/api/runners/index.ts +++ b/server/tests/api/runners/index.ts @@ -1,4 +1,5 @@ export * from './runner-common' export * from './runner-live-transcoding' export * from './runner-socket' +export * from './runner-studio-transcoding' export * from './runner-vod-transcoding' diff --git a/server/tests/api/runners/runner-common.ts b/server/tests/api/runners/runner-common.ts index a2204753b..554024190 100644 --- a/server/tests/api/runners/runner-common.ts +++ b/server/tests/api/runners/runner-common.ts @@ -2,7 +2,15 @@ import { expect } from 'chai' import { wait } from '@shared/core-utils' -import { HttpStatusCode, Runner, RunnerJob, RunnerJobAdmin, RunnerJobState, RunnerRegistrationToken } from '@shared/models' +import { + HttpStatusCode, + Runner, + RunnerJob, + RunnerJobAdmin, + RunnerJobState, + RunnerJobVODWebVideoTranscodingPayload, + RunnerRegistrationToken +} from '@shared/models' import { cleanupTests, createSingleServer, @@ -349,7 +357,7 @@ describe('Test runner common actions', function () { for (const job of availableJobs) { expect(job.uuid).to.exist expect(job.payload.input).to.exist - expect(job.payload.output).to.exist + expect((job.payload as RunnerJobVODWebVideoTranscodingPayload).output).to.exist expect((job as RunnerJobAdmin).privatePayload).to.not.exist } diff --git a/server/tests/api/runners/runner-studio-transcoding.ts b/server/tests/api/runners/runner-studio-transcoding.ts new file mode 100644 index 000000000..9ae629be6 --- /dev/null +++ b/server/tests/api/runners/runner-studio-transcoding.ts @@ -0,0 +1,168 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { readFile } from 'fs-extra' +import { checkPersistentTmpIsEmpty, checkVideoDuration } from '@server/tests/shared' +import { buildAbsoluteFixturePath } from '@shared/core-utils' +import { + RunnerJobVideoEditionTranscodingPayload, + VideoEditionTranscodingSuccess, + VideoState, + VideoStudioTask, + VideoStudioTaskIntro +} from '@shared/models' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + PeerTubeServer, + setAccessTokensToServers, + setDefaultVideoChannel, + VideoStudioCommand, + waitJobs +} from '@shared/server-commands' + +describe('Test runner video studio transcoding', function () { + let servers: PeerTubeServer[] = [] + let runnerToken: string + let videoUUID: string + let jobUUID: string + + async function renewStudio (tasks: VideoStudioTask[] = VideoStudioCommand.getComplexTask()) { + const { uuid } = await servers[0].videos.quickUpload({ name: 'video' }) + videoUUID = uuid + + await waitJobs(servers) + + await servers[0].videoStudio.createEditionTasks({ videoId: uuid, tasks }) + await waitJobs(servers) + + const { availableJobs } = await servers[0].runnerJobs.request({ runnerToken }) + expect(availableJobs).to.have.lengthOf(1) + + jobUUID = availableJobs[0].uuid + } + + before(async function () { + this.timeout(120_000) + + servers = await createMultipleServers(2) + + await setAccessTokensToServers(servers) + await setDefaultVideoChannel(servers) + + await doubleFollow(servers[0], servers[1]) + + await servers[0].config.enableTranscoding(true, true) + await servers[0].config.enableStudio() + await servers[0].config.enableRemoteStudio() + + runnerToken = await servers[0].runners.autoRegisterRunner() + }) + + it('Should error a studio transcoding job', async function () { + this.timeout(60000) + + await renewStudio() + + for (let i = 0; i < 5; i++) { + const { job } = await servers[0].runnerJobs.accept({ runnerToken, jobUUID }) + const jobToken = job.jobToken + + await servers[0].runnerJobs.error({ runnerToken, jobUUID, jobToken, message: 'Error' }) + } + + const video = await servers[0].videos.get({ id: videoUUID }) + expect(video.state.id).to.equal(VideoState.PUBLISHED) + + await checkPersistentTmpIsEmpty(servers[0]) + }) + + it('Should cancel a transcoding job', async function () { + this.timeout(60000) + + await renewStudio() + + await servers[0].runnerJobs.cancelByAdmin({ jobUUID }) + + const video = await servers[0].videos.get({ id: videoUUID }) + expect(video.state.id).to.equal(VideoState.PUBLISHED) + + await checkPersistentTmpIsEmpty(servers[0]) + }) + + it('Should execute a remote studio job', async function () { + this.timeout(240_000) + + const tasks = [ + { + name: 'add-outro' as 'add-outro', + options: { + file: 'video_short.webm' + } + }, + { + name: 'add-watermark' as 'add-watermark', + options: { + file: 'thumbnail.png' + } + }, + { + name: 'add-intro' as 'add-intro', + options: { + file: 'video_very_short_240p.mp4' + } + } + ] + + await renewStudio(tasks) + + for (const server of servers) { + await checkVideoDuration(server, videoUUID, 5) + } + + const { job } = await servers[0].runnerJobs.accept({ runnerToken, jobUUID }) + const jobToken = job.jobToken + + expect(job.type === 'video-edition-transcoding') + expect(job.payload.input.videoFileUrl).to.exist + + // Check video input file + { + await servers[0].runnerJobs.getJobFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) + } + + // Check task files + for (let i = 0; i < tasks.length; i++) { + const task = tasks[i] + const payloadTask = job.payload.tasks[i] + + expect(payloadTask.name).to.equal(task.name) + + const inputFile = await readFile(buildAbsoluteFixturePath(task.options.file)) + + const { body } = await servers[0].runnerJobs.getJobFile({ + url: (payloadTask as VideoStudioTaskIntro).options.file as string, + jobToken, + runnerToken + }) + + expect(body).to.deep.equal(inputFile) + } + + const payload: VideoEditionTranscodingSuccess = { videoFile: 'video_very_short_240p.mp4' } + await servers[0].runnerJobs.success({ runnerToken, jobUUID, jobToken, payload }) + + await waitJobs(servers) + + for (const server of servers) { + await checkVideoDuration(server, videoUUID, 2) + } + + await checkPersistentTmpIsEmpty(servers[0]) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/server/tests/api/runners/runner-vod-transcoding.ts b/server/tests/api/runners/runner-vod-transcoding.ts index 92a47ac3b..b08ee312c 100644 --- a/server/tests/api/runners/runner-vod-transcoding.ts +++ b/server/tests/api/runners/runner-vod-transcoding.ts @@ -155,7 +155,7 @@ describe('Test runner VOD transcoding', function () { expect(job.payload.output.resolution).to.equal(720) expect(job.payload.output.fps).to.equal(25) - const { body } = await servers[0].runnerJobs.getInputFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) + const { body } = await servers[0].runnerJobs.getJobFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) const inputFile = await readFile(buildAbsoluteFixturePath('video_short.webm')) expect(body).to.deep.equal(inputFile) @@ -200,7 +200,7 @@ describe('Test runner VOD transcoding', function () { const { job } = await servers[0].runnerJobs.accept({ runnerToken, jobUUID }) jobToken = job.jobToken - const { body } = await servers[0].runnerJobs.getInputFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) + const { body } = await servers[0].runnerJobs.getJobFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) const inputFile = await readFile(buildAbsoluteFixturePath('video_short.mp4')) expect(body).to.deep.equal(inputFile) @@ -221,7 +221,7 @@ describe('Test runner VOD transcoding', function () { const { job } = await servers[0].runnerJobs.accept({ runnerToken, jobUUID }) jobToken = job.jobToken - const { body } = await servers[0].runnerJobs.getInputFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) + const { body } = await servers[0].runnerJobs.getJobFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) const inputFile = await readFile(buildAbsoluteFixturePath('video_short.mp4')) expect(body).to.deep.equal(inputFile) @@ -293,7 +293,7 @@ describe('Test runner VOD transcoding', function () { const { job } = await servers[0].runnerJobs.accept({ runnerToken, jobUUID }) jobToken = job.jobToken - const { body } = await servers[0].runnerJobs.getInputFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) + const { body } = await servers[0].runnerJobs.getJobFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) const inputFile = await readFile(buildAbsoluteFixturePath('video_short.mp4')) expect(body).to.deep.equal(inputFile) @@ -337,7 +337,7 @@ describe('Test runner VOD transcoding', function () { const { job } = await servers[0].runnerJobs.accept({ runnerToken, jobUUID }) jobToken = job.jobToken - const { body } = await servers[0].runnerJobs.getInputFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) + const { body } = await servers[0].runnerJobs.getJobFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) const inputFile = await readFile(buildAbsoluteFixturePath(maxQualityFile)) expect(body).to.deep.equal(inputFile) @@ -446,13 +446,13 @@ describe('Test runner VOD transcoding', function () { expect(job.payload.output.resolution).to.equal(480) { - const { body } = await servers[0].runnerJobs.getInputFile({ url: job.payload.input.audioFileUrl, jobToken, runnerToken }) + const { body } = await servers[0].runnerJobs.getJobFile({ url: job.payload.input.audioFileUrl, jobToken, runnerToken }) const inputFile = await readFile(buildAbsoluteFixturePath('sample.ogg')) expect(body).to.deep.equal(inputFile) } { - const { body } = await servers[0].runnerJobs.getInputFile({ url: job.payload.input.previewFileUrl, jobToken, runnerToken }) + const { body } = await servers[0].runnerJobs.getJobFile({ url: job.payload.input.previewFileUrl, jobToken, runnerToken }) const video = await servers[0].videos.get({ id: videoUUID }) const { body: inputFile } = await makeGetRequest({ @@ -503,7 +503,7 @@ describe('Test runner VOD transcoding', function () { const { job } = await servers[0].runnerJobs.accept({ runnerToken, jobUUID }) jobToken = job.jobToken - const { body } = await servers[0].runnerJobs.getInputFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) + const { body } = await servers[0].runnerJobs.getJobFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) const inputFile = await readFile(buildAbsoluteFixturePath('video_short_480p.mp4')) expect(body).to.deep.equal(inputFile) diff --git a/server/tests/api/server/config.ts b/server/tests/api/server/config.ts index 54a40b994..011ba268c 100644 --- a/server/tests/api/server/config.ts +++ b/server/tests/api/server/config.ts @@ -102,6 +102,7 @@ function checkInitialConfig (server: PeerTubeServer, data: CustomConfig) { expect(data.live.transcoding.alwaysTranscodeOriginalResolution).to.be.true expect(data.videoStudio.enabled).to.be.false + expect(data.videoStudio.remoteRunners.enabled).to.be.false expect(data.import.videos.concurrency).to.equal(2) expect(data.import.videos.http.enabled).to.be.true @@ -211,6 +212,7 @@ function checkUpdatedConfig (data: CustomConfig) { expect(data.live.transcoding.alwaysTranscodeOriginalResolution).to.be.false expect(data.videoStudio.enabled).to.be.true + expect(data.videoStudio.remoteRunners.enabled).to.be.true expect(data.import.videos.concurrency).to.equal(4) expect(data.import.videos.http.enabled).to.be.false @@ -374,7 +376,10 @@ const newCustomConfig: CustomConfig = { } }, videoStudio: { - enabled: true + enabled: true, + remoteRunners: { + enabled: true + } }, import: { videos: { diff --git a/server/tests/api/transcoding/video-studio.ts b/server/tests/api/transcoding/video-studio.ts index 30f72e6e9..2f64ef6bd 100644 --- a/server/tests/api/transcoding/video-studio.ts +++ b/server/tests/api/transcoding/video-studio.ts @@ -1,5 +1,5 @@ import { expect } from 'chai' -import { checkPersistentTmpIsEmpty, expectStartWith } from '@server/tests/shared' +import { checkPersistentTmpIsEmpty, checkVideoDuration, expectStartWith } from '@server/tests/shared' import { areMockObjectStorageTestsDisabled, getAllFiles } from '@shared/core-utils' import { VideoStudioTask } from '@shared/models' import { @@ -18,20 +18,6 @@ describe('Test video studio', function () { let servers: PeerTubeServer[] = [] let videoUUID: string - async function checkDuration (server: PeerTubeServer, duration: number) { - const video = await server.videos.get({ id: videoUUID }) - - expect(video.duration).to.be.approximately(duration, 1) - - for (const file of video.files) { - const metadata = await server.videos.getFileMetadata({ url: file.metadataUrl }) - - for (const stream of metadata.streams) { - expect(Math.round(stream.duration)).to.be.approximately(duration, 1) - } - } - } - async function renewVideo (fixture = 'video_short.webm') { const video = await servers[0].videos.quickUpload({ name: 'video', fixture }) videoUUID = video.uuid @@ -79,7 +65,7 @@ describe('Test video studio', function () { ]) for (const server of servers) { - await checkDuration(server, 3) + await checkVideoDuration(server, videoUUID, 3) const video = await server.videos.get({ id: videoUUID }) expect(new Date(video.publishedAt)).to.be.below(beforeTasks) @@ -100,7 +86,7 @@ describe('Test video studio', function () { ]) for (const server of servers) { - await checkDuration(server, 2) + await checkVideoDuration(server, videoUUID, 2) } }) @@ -119,7 +105,7 @@ describe('Test video studio', function () { ]) for (const server of servers) { - await checkDuration(server, 4) + await checkVideoDuration(server, videoUUID, 4) } }) }) @@ -140,7 +126,7 @@ describe('Test video studio', function () { ]) for (const server of servers) { - await checkDuration(server, 10) + await checkVideoDuration(server, videoUUID, 10) } }) @@ -158,7 +144,7 @@ describe('Test video studio', function () { ]) for (const server of servers) { - await checkDuration(server, 7) + await checkVideoDuration(server, videoUUID, 7) } }) @@ -183,7 +169,7 @@ describe('Test video studio', function () { ]) for (const server of servers) { - await checkDuration(server, 12) + await checkVideoDuration(server, videoUUID, 12) } }) @@ -201,7 +187,7 @@ describe('Test video studio', function () { ]) for (const server of servers) { - await checkDuration(server, 7) + await checkVideoDuration(server, videoUUID, 7) } }) @@ -219,7 +205,7 @@ describe('Test video studio', function () { ]) for (const server of servers) { - await checkDuration(server, 10) + await checkVideoDuration(server, videoUUID, 10) } }) @@ -237,7 +223,7 @@ describe('Test video studio', function () { ]) for (const server of servers) { - await checkDuration(server, 10) + await checkVideoDuration(server, videoUUID, 10) } }) }) @@ -279,7 +265,7 @@ describe('Test video studio', function () { await createTasks(VideoStudioCommand.getComplexTask()) for (const server of servers) { - await checkDuration(server, 9) + await checkVideoDuration(server, videoUUID, 9) } }) }) @@ -309,7 +295,7 @@ describe('Test video studio', function () { const video = await server.videos.get({ id: videoUUID }) expect(video.files).to.have.lengthOf(0) - await checkDuration(server, 9) + await checkVideoDuration(server, videoUUID, 9) } }) }) @@ -351,7 +337,7 @@ describe('Test video studio', function () { expectStartWith(hlsFile.fileUrl, ObjectStorageCommand.getMockPlaylistBaseUrl()) } - await checkDuration(server, 9) + await checkVideoDuration(server, videoUUID, 9) } }) }) @@ -370,7 +356,7 @@ describe('Test video studio', function () { await waitJobs(servers) for (const server of servers) { - await checkDuration(server, 9) + await checkVideoDuration(server, videoUUID, 9) } }) diff --git a/server/tests/peertube-runner/index.ts b/server/tests/peertube-runner/index.ts index 6258d6eb2..470316417 100644 --- a/server/tests/peertube-runner/index.ts +++ b/server/tests/peertube-runner/index.ts @@ -1,3 +1,4 @@ export * from './client-cli' export * from './live-transcoding' +export * from './studio-transcoding' export * from './vod-transcoding' diff --git a/server/tests/peertube-runner/live-transcoding.ts b/server/tests/peertube-runner/live-transcoding.ts index f58e920ba..1e94eabcd 100644 --- a/server/tests/peertube-runner/live-transcoding.ts +++ b/server/tests/peertube-runner/live-transcoding.ts @@ -1,6 +1,12 @@ import { expect } from 'chai' /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ -import { expectStartWith, PeerTubeRunnerProcess, SQLCommand, testLiveVideoResolutions } from '@server/tests/shared' +import { + checkPeerTubeRunnerCacheIsEmpty, + expectStartWith, + PeerTubeRunnerProcess, + SQLCommand, + testLiveVideoResolutions +} from '@server/tests/shared' import { areMockObjectStorageTestsDisabled, wait } from '@shared/core-utils' import { HttpStatusCode, VideoPrivacy } from '@shared/models' import { @@ -169,6 +175,13 @@ describe('Test Live transcoding in peertube-runner program', function () { runSuite({ objectStorage: true }) }) + describe('Check cleanup', function () { + + it('Should have an empty cache directory', async function () { + await checkPeerTubeRunnerCacheIsEmpty() + }) + }) + after(async function () { await peertubeRunner.unregisterPeerTubeInstance({ server: servers[0] }) peertubeRunner.kill() diff --git a/server/tests/peertube-runner/studio-transcoding.ts b/server/tests/peertube-runner/studio-transcoding.ts new file mode 100644 index 000000000..cca905e2f --- /dev/null +++ b/server/tests/peertube-runner/studio-transcoding.ts @@ -0,0 +1,116 @@ + +import { expect } from 'chai' +import { checkPeerTubeRunnerCacheIsEmpty, checkVideoDuration, expectStartWith, PeerTubeRunnerProcess } from '@server/tests/shared' +import { areMockObjectStorageTestsDisabled, getAllFiles, wait } from '@shared/core-utils' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + ObjectStorageCommand, + PeerTubeServer, + setAccessTokensToServers, + setDefaultVideoChannel, + VideoStudioCommand, + waitJobs +} from '@shared/server-commands' + +describe('Test studio transcoding in peertube-runner program', function () { + let servers: PeerTubeServer[] = [] + let peertubeRunner: PeerTubeRunnerProcess + + function runSuite (options: { + objectStorage: boolean + }) { + const { objectStorage } = options + + it('Should run a complex studio transcoding', async function () { + this.timeout(120000) + + const { uuid } = await servers[0].videos.quickUpload({ name: 'mp4', fixture: 'video_short.mp4' }) + await waitJobs(servers) + + const video = await servers[0].videos.get({ id: uuid }) + const oldFileUrls = getAllFiles(video).map(f => f.fileUrl) + + await servers[0].videoStudio.createEditionTasks({ videoId: uuid, tasks: VideoStudioCommand.getComplexTask() }) + await waitJobs(servers, { runnerJobs: true }) + + for (const server of servers) { + const video = await server.videos.get({ id: uuid }) + const files = getAllFiles(video) + + for (const f of files) { + expect(oldFileUrls).to.not.include(f.fileUrl) + } + + if (objectStorage) { + for (const webtorrentFile of video.files) { + expectStartWith(webtorrentFile.fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl()) + } + + for (const hlsFile of video.streamingPlaylists[0].files) { + expectStartWith(hlsFile.fileUrl, ObjectStorageCommand.getMockPlaylistBaseUrl()) + } + } + + await checkVideoDuration(server, uuid, 9) + } + }) + } + + before(async function () { + this.timeout(120_000) + + servers = await createMultipleServers(2) + + await setAccessTokensToServers(servers) + await setDefaultVideoChannel(servers) + + await doubleFollow(servers[0], servers[1]) + + await servers[0].config.enableTranscoding(true, true) + await servers[0].config.enableStudio() + await servers[0].config.enableRemoteStudio() + + const registrationToken = await servers[0].runnerRegistrationTokens.getFirstRegistrationToken() + + peertubeRunner = new PeerTubeRunnerProcess() + await peertubeRunner.runServer({ hideLogs: false }) + await peertubeRunner.registerPeerTubeInstance({ server: servers[0], registrationToken, runnerName: 'runner' }) + }) + + describe('With videos on local filesystem storage', function () { + runSuite({ objectStorage: false }) + }) + + describe('With videos on object storage', function () { + if (areMockObjectStorageTestsDisabled()) return + + before(async function () { + await ObjectStorageCommand.prepareDefaultMockBuckets() + + await servers[0].kill() + + await servers[0].run(ObjectStorageCommand.getDefaultMockConfig()) + + // Wait for peertube runner socket reconnection + await wait(1500) + }) + + runSuite({ objectStorage: true }) + }) + + describe('Check cleanup', function () { + + it('Should have an empty cache directory', async function () { + await checkPeerTubeRunnerCacheIsEmpty() + }) + }) + + after(async function () { + await peertubeRunner.unregisterPeerTubeInstance({ server: servers[0] }) + peertubeRunner.kill() + + await cleanupTests(servers) + }) +}) diff --git a/server/tests/peertube-runner/vod-transcoding.ts b/server/tests/peertube-runner/vod-transcoding.ts index bdf798379..3a9abba93 100644 --- a/server/tests/peertube-runner/vod-transcoding.ts +++ b/server/tests/peertube-runner/vod-transcoding.ts @@ -1,6 +1,11 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ import { expect } from 'chai' -import { completeCheckHlsPlaylist, completeWebVideoFilesCheck, PeerTubeRunnerProcess } from '@server/tests/shared' +import { + checkPeerTubeRunnerCacheIsEmpty, + completeCheckHlsPlaylist, + completeWebVideoFilesCheck, + PeerTubeRunnerProcess +} from '@server/tests/shared' import { areMockObjectStorageTestsDisabled, getAllFiles, wait } from '@shared/core-utils' import { VideoPrivacy } from '@shared/models' import { @@ -321,6 +326,13 @@ describe('Test VOD transcoding in peertube-runner program', function () { }) }) + describe('Check cleanup', function () { + + it('Should have an empty cache directory', async function () { + await checkPeerTubeRunnerCacheIsEmpty() + }) + }) + after(async function () { await peertubeRunner.unregisterPeerTubeInstance({ server: servers[0] }) peertubeRunner.kill() diff --git a/server/tests/shared/checks.ts b/server/tests/shared/checks.ts index d7eb25bb5..feaef37c6 100644 --- a/server/tests/shared/checks.ts +++ b/server/tests/shared/checks.ts @@ -130,6 +130,22 @@ function checkBadSortPagination (url: string, path: string, token?: string, quer }) } +// --------------------------------------------------------------------------- + +async function checkVideoDuration (server: PeerTubeServer, videoUUID: string, duration: number) { + const video = await server.videos.get({ id: videoUUID }) + + expect(video.duration).to.be.approximately(duration, 1) + + for (const file of video.files) { + const metadata = await server.videos.getFileMetadata({ url: file.metadataUrl }) + + for (const stream of metadata.streams) { + expect(Math.round(stream.duration)).to.be.approximately(duration, 1) + } + } +} + export { dateIsValid, testImageSize, @@ -142,5 +158,6 @@ export { checkBadStartPagination, checkBadCountPagination, checkBadSortPagination, + checkVideoDuration, expectLogContain } diff --git a/server/tests/shared/directories.ts b/server/tests/shared/directories.ts index a614cef7c..4f4282554 100644 --- a/server/tests/shared/directories.ts +++ b/server/tests/shared/directories.ts @@ -2,9 +2,11 @@ import { expect } from 'chai' import { pathExists, readdir } from 'fs-extra' +import { homedir } from 'os' +import { join } from 'path' import { PeerTubeServer } from '@shared/server-commands' -async function checkTmpIsEmpty (server: PeerTubeServer) { +export async function checkTmpIsEmpty (server: PeerTubeServer) { await checkDirectoryIsEmpty(server, 'tmp', [ 'plugins-global.css', 'hls', 'resumable-uploads' ]) if (await pathExists(server.getDirectoryPath('tmp/hls'))) { @@ -12,11 +14,11 @@ async function checkTmpIsEmpty (server: PeerTubeServer) { } } -async function checkPersistentTmpIsEmpty (server: PeerTubeServer) { +export async function checkPersistentTmpIsEmpty (server: PeerTubeServer) { await checkDirectoryIsEmpty(server, 'tmp-persistent') } -async function checkDirectoryIsEmpty (server: PeerTubeServer, directory: string, exceptions: string[] = []) { +export async function checkDirectoryIsEmpty (server: PeerTubeServer, directory: string, exceptions: string[] = []) { const directoryPath = server.getDirectoryPath(directory) const directoryExists = await pathExists(directoryPath) @@ -28,8 +30,13 @@ async function checkDirectoryIsEmpty (server: PeerTubeServer, directory: string, expect(filtered).to.have.lengthOf(0) } -export { - checkTmpIsEmpty, - checkPersistentTmpIsEmpty, - checkDirectoryIsEmpty +export async function checkPeerTubeRunnerCacheIsEmpty () { + const directoryPath = join(homedir(), '.cache', 'peertube-runner-nodejs', 'test', 'transcoding') + + const directoryExists = await pathExists(directoryPath) + expect(directoryExists).to.be.true + + const files = await readdir(directoryPath) + + expect(files).to.have.lengthOf(0) } diff --git a/shared/models/runners/runner-job-payload.model.ts b/shared/models/runners/runner-job-payload.model.ts index 8f0c17135..9f0db0dc4 100644 --- a/shared/models/runners/runner-job-payload.model.ts +++ b/shared/models/runners/runner-job-payload.model.ts @@ -1,3 +1,5 @@ +import { VideoStudioTaskPayload } from '../server' + export type RunnerJobVODPayload = RunnerJobVODWebVideoTranscodingPayload | RunnerJobVODHLSTranscodingPayload | @@ -5,7 +7,8 @@ export type RunnerJobVODPayload = export type RunnerJobPayload = RunnerJobVODPayload | - RunnerJobLiveRTMPHLSTranscodingPayload + RunnerJobLiveRTMPHLSTranscodingPayload | + RunnerJobVideoEditionTranscodingPayload // --------------------------------------------------------------------------- @@ -43,6 +46,14 @@ export interface RunnerJobVODAudioMergeTranscodingPayload { } } +export interface RunnerJobVideoEditionTranscodingPayload { + input: { + videoFileUrl: string + } + + tasks: VideoStudioTaskPayload[] +} + // --------------------------------------------------------------------------- export function isAudioMergeTranscodingPayload (payload: RunnerJobPayload): payload is RunnerJobVODAudioMergeTranscodingPayload { diff --git a/shared/models/runners/runner-job-private-payload.model.ts b/shared/models/runners/runner-job-private-payload.model.ts index c1d8d1045..c8fe0a7d8 100644 --- a/shared/models/runners/runner-job-private-payload.model.ts +++ b/shared/models/runners/runner-job-private-payload.model.ts @@ -1,3 +1,5 @@ +import { VideoStudioTaskPayload } from '../server' + export type RunnerJobVODPrivatePayload = RunnerJobVODWebVideoTranscodingPrivatePayload | RunnerJobVODAudioMergeTranscodingPrivatePayload | @@ -5,7 +7,8 @@ export type RunnerJobVODPrivatePayload = export type RunnerJobPrivatePayload = RunnerJobVODPrivatePayload | - RunnerJobLiveRTMPHLSTranscodingPrivatePayload + RunnerJobLiveRTMPHLSTranscodingPrivatePayload | + RunnerJobVideoEditionTranscodingPrivatePayload // --------------------------------------------------------------------------- @@ -32,3 +35,10 @@ export interface RunnerJobLiveRTMPHLSTranscodingPrivatePayload { masterPlaylistName: string outputDirectory: string } + +// --------------------------------------------------------------------------- + +export interface RunnerJobVideoEditionTranscodingPrivatePayload { + videoUUID: string + originalTasks: VideoStudioTaskPayload[] +} diff --git a/shared/models/runners/runner-job-success-body.model.ts b/shared/models/runners/runner-job-success-body.model.ts index 223b7552d..17e921f69 100644 --- a/shared/models/runners/runner-job-success-body.model.ts +++ b/shared/models/runners/runner-job-success-body.model.ts @@ -11,7 +11,8 @@ export type RunnerJobSuccessPayload = VODWebVideoTranscodingSuccess | VODHLSTranscodingSuccess | VODAudioMergeTranscodingSuccess | - LiveRTMPHLSTranscodingSuccess + LiveRTMPHLSTranscodingSuccess | + VideoEditionTranscodingSuccess export interface VODWebVideoTranscodingSuccess { videoFile: Blob | string @@ -30,6 +31,10 @@ export interface LiveRTMPHLSTranscodingSuccess { } +export interface VideoEditionTranscodingSuccess { + videoFile: Blob | string +} + export function isWebVideoOrAudioMergeTranscodingPayloadSuccess ( payload: RunnerJobSuccessPayload ): payload is VODHLSTranscodingSuccess | VODAudioMergeTranscodingSuccess { diff --git a/shared/models/runners/runner-job-type.type.ts b/shared/models/runners/runner-job-type.type.ts index 36d3b9b25..3b997cb6e 100644 --- a/shared/models/runners/runner-job-type.type.ts +++ b/shared/models/runners/runner-job-type.type.ts @@ -2,4 +2,5 @@ export type RunnerJobType = 'vod-web-video-transcoding' | 'vod-hls-transcoding' | 'vod-audio-merge-transcoding' | - 'live-rtmp-hls-transcoding' + 'live-rtmp-hls-transcoding' | + 'video-edition-transcoding' diff --git a/shared/models/server/custom-config.model.ts b/shared/models/server/custom-config.model.ts index 5d2c10278..4202589f3 100644 --- a/shared/models/server/custom-config.model.ts +++ b/shared/models/server/custom-config.model.ts @@ -165,6 +165,10 @@ export interface CustomConfig { videoStudio: { enabled: boolean + + remoteRunners: { + enabled: boolean + } } import: { diff --git a/shared/models/server/job.model.ts b/shared/models/server/job.model.ts index 3fd5bf7f9..22ecee324 100644 --- a/shared/models/server/job.model.ts +++ b/shared/models/server/job.model.ts @@ -225,6 +225,10 @@ export type VideoStudioTaskWatermarkPayload = { options: { file: string + + watermarkSizeRatio: number + horitonzalMarginRatio: number + verticalMarginRatio: number } } diff --git a/shared/models/server/server-config.model.ts b/shared/models/server/server-config.model.ts index 38b9d0385..024ed35bf 100644 --- a/shared/models/server/server-config.model.ts +++ b/shared/models/server/server-config.model.ts @@ -1,6 +1,6 @@ -import { VideoPrivacy } from '../videos/video-privacy.enum' import { ClientScriptJSON } from '../plugins/plugin-package-json.model' import { NSFWPolicyType } from '../videos/nsfw-policy.type' +import { VideoPrivacy } from '../videos/video-privacy.enum' import { BroadcastMessageLevel } from './broadcast-message-level.type' export interface ServerConfigPlugin { @@ -186,6 +186,10 @@ export interface ServerConfig { videoStudio: { enabled: boolean + + remoteRunners: { + enabled: boolean + } } import: { diff --git a/shared/models/videos/studio/video-studio-create-edit.model.ts b/shared/models/videos/studio/video-studio-create-edit.model.ts index 001d65c90..5e8296dc9 100644 --- a/shared/models/videos/studio/video-studio-create-edit.model.ts +++ b/shared/models/videos/studio/video-studio-create-edit.model.ts @@ -40,3 +40,21 @@ export interface VideoStudioTaskWatermark { file: Blob | string } } + +// --------------------------------------------------------------------------- + +export function isVideoStudioTaskIntro (v: VideoStudioTask): v is VideoStudioTaskIntro { + return v.name === 'add-intro' +} + +export function isVideoStudioTaskOutro (v: VideoStudioTask): v is VideoStudioTaskOutro { + return v.name === 'add-outro' +} + +export function isVideoStudioTaskWatermark (v: VideoStudioTask): v is VideoStudioTaskWatermark { + return v.name === 'add-watermark' +} + +export function hasVideoStudioTaskFile (v: VideoStudioTask): v is VideoStudioTaskIntro | VideoStudioTaskOutro | VideoStudioTaskWatermark { + return isVideoStudioTaskIntro(v) || isVideoStudioTaskOutro(v) || isVideoStudioTaskWatermark(v) +} diff --git a/shared/server-commands/runners/runner-jobs-command.ts b/shared/server-commands/runners/runner-jobs-command.ts index 3b0f84b9d..26dbef77a 100644 --- a/shared/server-commands/runners/runner-jobs-command.ts +++ b/shared/server-commands/runners/runner-jobs-command.ts @@ -200,7 +200,7 @@ export class RunnerJobsCommand extends AbstractCommand { }) } - getInputFile (options: OverrideCommandOptions & { url: string, jobToken: string, runnerToken: string }) { + getJobFile (options: OverrideCommandOptions & { url: string, jobToken: string, runnerToken: string }) { const { host, protocol, pathname } = new URL(options.url) return this.postBodyRequest({ @@ -249,8 +249,15 @@ export class RunnerJobsCommand extends AbstractCommand { const { data } = await this.list({ count: 100 }) + const allowedStates = new Set([ + RunnerJobState.PENDING, + RunnerJobState.PROCESSING, + RunnerJobState.WAITING_FOR_PARENT_JOB + ]) + for (const job of data) { if (state && job.state.id !== state) continue + else if (allowedStates.has(job.state.id) !== true) continue await this.cancelByAdmin({ jobUUID: job.uuid }) } diff --git a/shared/server-commands/server/config-command.ts b/shared/server-commands/server/config-command.ts index 9a6e413f2..b94bd2625 100644 --- a/shared/server-commands/server/config-command.ts +++ b/shared/server-commands/server/config-command.ts @@ -195,6 +195,18 @@ export class ConfigCommand extends AbstractCommand { }) } + enableRemoteStudio () { + return this.updateExistingSubConfig({ + newConfig: { + videoStudio: { + remoteRunners: { + enabled: true + } + } + } + }) + } + // --------------------------------------------------------------------------- enableStudio () { @@ -442,7 +454,10 @@ export class ConfigCommand extends AbstractCommand { } }, videoStudio: { - enabled: false + enabled: false, + remoteRunners: { + enabled: false + } }, import: { videos: {