diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 04bff26aa..1fa7704df 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -71,6 +71,20 @@ jobs: ${{ runner.OS }}-fixtures- ${{ runner.OS }}- + - name: Cache PeerTube pip directory + uses: actions/cache@v4 + with: + path: | + ~/.cache/pip + key: ${{ runner.OS }}-${{ matrix.test_suite }}-pip-v1 + + - name: Cache Hugging Face models + uses: actions/cache@v4 + with: + path: | + ~/.cache/huggingface + key: ${{ runner.OS }}-${{ matrix.test_suite }}-hugging-face-v1 + - name: Set env test variable (schedule) if: github.event_name != 'schedule' run: | diff --git a/.gitignore b/.gitignore index 6865442eb..cf3b25a1c 100644 --- a/.gitignore +++ b/.gitignore @@ -12,8 +12,11 @@ yarn-error.log /test4/ /test5/ /test6/ + +# Big fixtures generated/downloaded on-demand /packages/tests/fixtures/video_high_bitrate_1080p.mp4 /packages/tests/fixtures/video_59fps.mp4 +/packages/tests/fixtures/transcription/models-v1/ # Production /storage diff --git a/apps/peertube-runner/src/server/process/process.ts b/apps/peertube-runner/src/server/process/process.ts index e8a1d7c28..b20c2297d 100644 --- a/apps/peertube-runner/src/server/process/process.ts +++ b/apps/peertube-runner/src/server/process/process.ts @@ -1,6 +1,7 @@ import { RunnerJobLiveRTMPHLSTranscodingPayload, RunnerJobStudioTranscodingPayload, + RunnerJobTranscriptionPayload, RunnerJobVODAudioMergeTranscodingPayload, RunnerJobVODHLSTranscodingPayload, RunnerJobVODWebVideoTranscodingPayload @@ -9,25 +10,41 @@ import { logger } from '../../shared/index.js' import { processAudioMergeTranscoding, processHLSTranscoding, ProcessOptions, processWebVideoTranscoding } from './shared/index.js' import { ProcessLiveRTMPHLSTranscoding } from './shared/process-live.js' import { processStudioTranscoding } from './shared/process-studio.js' +import { processVideoTranscription } from './shared/process-transcription.js' export async function processJob (options: ProcessOptions) { const { server, job } = options logger.info(`[${server.url}] Processing job of type ${job.type}: ${job.uuid}`, { payload: job.payload }) - if (job.type === 'vod-audio-merge-transcoding') { - await processAudioMergeTranscoding(options as ProcessOptions) - } else if (job.type === 'vod-web-video-transcoding') { - await processWebVideoTranscoding(options as ProcessOptions) - } else if (job.type === 'vod-hls-transcoding') { - await processHLSTranscoding(options as ProcessOptions) - } else if (job.type === 'live-rtmp-hls-transcoding') { - await new ProcessLiveRTMPHLSTranscoding(options as ProcessOptions).process() - } else if (job.type === 'video-studio-transcoding') { - await processStudioTranscoding(options as ProcessOptions) - } else { - logger.error(`Unknown job ${job.type} to process`) - return + switch (job.type) { + case 'vod-audio-merge-transcoding': + await processAudioMergeTranscoding(options as ProcessOptions) + break + + case 'vod-web-video-transcoding': + await processWebVideoTranscoding(options as ProcessOptions) + break + + case 'vod-hls-transcoding': + await processHLSTranscoding(options as ProcessOptions) + break + + case 'live-rtmp-hls-transcoding': + await new ProcessLiveRTMPHLSTranscoding(options as ProcessOptions).process() + break + + case 'video-studio-transcoding': + await processStudioTranscoding(options as ProcessOptions) + break + + case 'video-transcription': + await processVideoTranscription(options as ProcessOptions) + break + + default: + logger.error(`Unknown job ${job.type} to process`) + return } logger.info(`[${server.url}] Finished processing job of type ${job.type}: ${job.uuid}`) diff --git a/apps/peertube-runner/src/server/process/shared/common.ts b/apps/peertube-runner/src/server/process/shared/common.ts index 09241d93b..cf7682991 100644 --- a/apps/peertube-runner/src/server/process/shared/common.ts +++ b/apps/peertube-runner/src/server/process/shared/common.ts @@ -5,7 +5,7 @@ import { RunnerJob, RunnerJobPayload } from '@peertube/peertube-models' import { buildUUID } from '@peertube/peertube-node-utils' import { PeerTubeServer } from '@peertube/peertube-server-commands' import { ConfigManager, downloadFile, logger } from '../../../shared/index.js' -import { getTranscodingLogger } from './transcoding-logger.js' +import { getWinstonLogger } from './winston-logger.js' export type JobWithToken = RunnerJob & { jobToken: string } @@ -101,6 +101,6 @@ function getCommonFFmpegOptions () { available: getDefaultAvailableEncoders(), encodersToTry: getDefaultEncodersToTry() }, - logger: getTranscodingLogger() + logger: getWinstonLogger() } } diff --git a/apps/peertube-runner/src/server/process/shared/index.ts b/apps/peertube-runner/src/server/process/shared/index.ts index 638bf127f..67d556f91 100644 --- a/apps/peertube-runner/src/server/process/shared/index.ts +++ b/apps/peertube-runner/src/server/process/shared/index.ts @@ -1,3 +1,3 @@ export * from './common.js' export * from './process-vod.js' -export * from './transcoding-logger.js' +export * from './winston-logger.js' diff --git a/apps/peertube-runner/src/server/process/shared/process-transcription.ts b/apps/peertube-runner/src/server/process/shared/process-transcription.ts new file mode 100644 index 000000000..715433f09 --- /dev/null +++ b/apps/peertube-runner/src/server/process/shared/process-transcription.ts @@ -0,0 +1,79 @@ +import { hasAudioStream } from '@peertube/peertube-ffmpeg' +import { RunnerJobTranscriptionPayload, TranscriptionSuccess } from '@peertube/peertube-models' +import { buildSUUID } from '@peertube/peertube-node-utils' +import { TranscriptionModel, WhisperBuiltinModel, transcriberFactory } from '@peertube/peertube-transcription' +import { remove } from 'fs-extra/esm' +import { join } from 'path' +import { ConfigManager } from '../../../shared/config-manager.js' +import { logger } from '../../../shared/index.js' +import { ProcessOptions, downloadInputFile, scheduleTranscodingProgress } from './common.js' +import { getWinstonLogger } from './winston-logger.js' + +export async function processVideoTranscription (options: ProcessOptions) { + const { server, job, runnerToken } = options + + const config = ConfigManager.Instance.getConfig().transcription + + const payload = job.payload + + let inputPath: string + + const updateProgressInterval = scheduleTranscodingProgress({ + job, + server, + runnerToken, + progressGetter: () => undefined + }) + + const outputPath = join(ConfigManager.Instance.getTranscriptionDirectory(), buildSUUID()) + + const transcriber = transcriberFactory.createFromEngineName({ + engineName: config.engine, + enginePath: config.enginePath, + logger: getWinstonLogger() + }) + + try { + logger.info(`Downloading input file ${payload.input.videoFileUrl} for transcription job ${job.jobToken}`) + + inputPath = await downloadInputFile({ url: payload.input.videoFileUrl, runnerToken, job }) + + logger.info(`Downloaded input file ${payload.input.videoFileUrl} for job ${job.jobToken}. Running transcription.`) + + if (await hasAudioStream(inputPath) !== true) { + await server.runnerJobs.error({ + jobToken: job.jobToken, + jobUUID: job.uuid, + runnerToken, + message: 'This input file does not contain audio' + }) + + return + } + + const transcriptFile = await transcriber.transcribe({ + mediaFilePath: inputPath, + model: config.modelPath + ? await TranscriptionModel.fromPath(config.modelPath) + : new WhisperBuiltinModel(config.model), + format: 'vtt', + transcriptDirectory: outputPath + }) + + const successBody: TranscriptionSuccess = { + inputLanguage: transcriptFile.language, + vttFile: transcriptFile.path + } + + await server.runnerJobs.success({ + jobToken: job.jobToken, + jobUUID: job.uuid, + runnerToken, + payload: successBody + }) + } finally { + if (inputPath) await remove(inputPath) + if (outputPath) await remove(outputPath) + if (updateProgressInterval) clearInterval(updateProgressInterval) + } +} diff --git a/apps/peertube-runner/src/server/process/shared/transcoding-logger.ts b/apps/peertube-runner/src/server/process/shared/transcoding-logger.ts deleted file mode 100644 index d0775e13b..000000000 --- a/apps/peertube-runner/src/server/process/shared/transcoding-logger.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { LogFn } from 'pino' -import { logger } from '../../../shared/index.js' - -export function getTranscodingLogger () { - return { - info: buildWinstonLogger(logger.info.bind(logger)), - debug: buildWinstonLogger(logger.debug.bind(logger)), - warn: buildWinstonLogger(logger.warn.bind(logger)), - error: buildWinstonLogger(logger.error.bind(logger)) - } -} - -function buildWinstonLogger (log: LogFn) { - return (arg1: string, arg2?: object) => { - if (arg2) return log(arg2, arg1) - - return log(arg1) - } -} diff --git a/apps/peertube-runner/src/server/process/shared/winston-logger.ts b/apps/peertube-runner/src/server/process/shared/winston-logger.ts new file mode 100644 index 000000000..d2958ea6a --- /dev/null +++ b/apps/peertube-runner/src/server/process/shared/winston-logger.ts @@ -0,0 +1,19 @@ +import { LogFn } from 'pino' +import { logger } from '../../../shared/index.js' + +export function getWinstonLogger () { + return { + info: buildLogLevelFn(logger.info.bind(logger)), + debug: buildLogLevelFn(logger.debug.bind(logger)), + warn: buildLogLevelFn(logger.warn.bind(logger)), + error: buildLogLevelFn(logger.error.bind(logger)) + } +} + +function buildLogLevelFn (log: LogFn) { + return (arg1: string, arg2?: object) => { + if (arg2) return log(arg2, arg1) + + return log(arg1) + } +} diff --git a/apps/peertube-runner/src/server/shared/supported-job.ts b/apps/peertube-runner/src/server/shared/supported-job.ts index d905b5de2..38cd28140 100644 --- a/apps/peertube-runner/src/server/shared/supported-job.ts +++ b/apps/peertube-runner/src/server/shared/supported-job.ts @@ -1,15 +1,16 @@ import { RunnerJobLiveRTMPHLSTranscodingPayload, RunnerJobPayload, - RunnerJobType, RunnerJobStudioTranscodingPayload, + RunnerJobTranscriptionPayload, + RunnerJobType, RunnerJobVODAudioMergeTranscodingPayload, RunnerJobVODHLSTranscodingPayload, RunnerJobVODWebVideoTranscodingPayload, VideoStudioTaskPayload } from '@peertube/peertube-models' -const supportedMatrix = { +const supportedMatrix: { [ id in RunnerJobType ]: (payload: RunnerJobPayload) => boolean } = { 'vod-web-video-transcoding': (_payload: RunnerJobVODWebVideoTranscodingPayload) => { return true }, @@ -29,6 +30,9 @@ const supportedMatrix = { if (!Array.isArray(tasks)) return false return tasks.every(t => t && supported.has(t.name)) + }, + 'video-transcription': (_payload: RunnerJobTranscriptionPayload) => { + return true } } diff --git a/apps/peertube-runner/src/shared/config-manager.ts b/apps/peertube-runner/src/shared/config-manager.ts index 84a326a16..97a70204a 100644 --- a/apps/peertube-runner/src/shared/config-manager.ts +++ b/apps/peertube-runner/src/shared/config-manager.ts @@ -1,4 +1,5 @@ import { parse, stringify } from '@iarna/toml' +import { TranscriptionEngineName, WhisperBuiltinModelName } from '@peertube/peertube-transcription' import envPaths from 'env-paths' import { ensureDir, pathExists, remove } from 'fs-extra/esm' import { readFile, writeFile } from 'fs/promises' @@ -24,6 +25,13 @@ type Config = { runnerName: string runnerDescription?: string }[] + + transcription: { + engine: TranscriptionEngineName + enginePath: string | null + model: WhisperBuiltinModelName + modelPath: string | null + } } export class ConfigManager { @@ -37,6 +45,12 @@ export class ConfigManager { threads: 2, nice: 20 }, + transcription: { + engine: 'whisper-ctranslate2', + enginePath: null, + model: 'small', + modelPath: null + }, registeredInstances: [] } @@ -98,6 +112,10 @@ export class ConfigManager { return join(paths.cache, this.id, 'transcoding') } + getTranscriptionDirectory () { + return join(paths.cache, this.id, 'transcription') + } + getSocketDirectory () { return join(paths.data, this.id) } diff --git a/client/src/app/+admin/config/edit-custom-config/edit-basic-configuration.component.html b/client/src/app/+admin/config/edit-custom-config/edit-basic-configuration.component.html index 3de40666f..ea8094d5f 100644 --- a/client/src/app/+admin/config/edit-custom-config/edit-basic-configuration.component.html +++ b/client/src/app/+admin/config/edit-custom-config/edit-basic-configuration.component.html @@ -318,7 +318,7 @@ > - ⛔ You need to allow import with HTTP URL to be able to activate this feature. + ⛔ You need to allow import with HTTP URL to be able to activate this feature. @@ -359,7 +359,6 @@ -
+
+ +
+ + + Automatically create a subtitle file of uploaded/imported VOD videos + + + +
+ + + + Use remote runners to process transcription tasks. + Remote runners has to register on your instance first. + + + +
+
+
+
diff --git a/client/src/app/+admin/config/edit-custom-config/edit-basic-configuration.component.ts b/client/src/app/+admin/config/edit-custom-config/edit-basic-configuration.component.ts index 9f1cbc758..98510c7c3 100644 --- a/client/src/app/+admin/config/edit-custom-config/edit-basic-configuration.component.ts +++ b/client/src/app/+admin/config/edit-custom-config/edit-basic-configuration.component.ts @@ -137,6 +137,18 @@ export class EditBasicConfigurationComponent implements OnInit, OnChanges { return { 'disabled-checkbox-extra': !this.isSearchIndexEnabled() } } + // --------------------------------------------------------------------------- + + isTranscriptionEnabled () { + return this.form.value['videoTranscription']['enabled'] === true + } + + getTranscriptionRunnerDisabledClass () { + return { 'disabled-checkbox-extra': !this.isTranscriptionEnabled() } + } + + // --------------------------------------------------------------------------- + isAutoFollowIndexEnabled () { return this.form.value['followings']['instance']['autoFollowIndex']['enabled'] === true } diff --git a/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts b/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts index 6c14b5c55..a85b17165 100644 --- a/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts +++ b/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts @@ -267,6 +267,12 @@ export class EditCustomConfigComponent extends FormReactive implements OnInit { enabled: null } }, + videoTranscription: { + enabled: null, + remoteRunners: { + enabled: null + } + }, videoFile: { update: { enabled: null diff --git a/client/src/app/+admin/overview/videos/video-list.component.ts b/client/src/app/+admin/overview/videos/video-list.component.ts index 3eb932322..f7d88b752 100644 --- a/client/src/app/+admin/overview/videos/video-list.component.ts +++ b/client/src/app/+admin/overview/videos/video-list.component.ts @@ -1,7 +1,7 @@ import { DatePipe, NgClass, NgFor, NgIf } from '@angular/common' import { Component, OnInit, ViewChild } from '@angular/core' import { ActivatedRoute, Router, RouterLink } from '@angular/router' -import { AuthService, ConfirmService, Notifier, RestPagination, RestTable } from '@app/core' +import { AuthService, ConfirmService, Notifier, RestPagination, RestTable, ServerService } from '@app/core' import { formatICU, getAbsoluteAPIUrl } from '@app/helpers' import { VideoDetails } from '@app/shared/shared-main/video/video-details.model' import { VideoFileTokenService } from '@app/shared/shared-main/video/video-file-token.service' @@ -30,6 +30,7 @@ import { VideoActionsDropdownComponent } from '../../../shared/shared-video-miniature/video-actions-dropdown.component' import { VideoAdminService } from './video-admin.service' +import { VideoCaptionService } from '@app/shared/shared-main/video-caption/video-caption.service' @Component({ selector: 'my-video-list', @@ -84,7 +85,8 @@ export class VideoListComponent extends RestTable