Integrate transcription in PeerTube

.github/workflows/test.yml

@@ -71,6 +71,20 @@ jobs:
             ${{ runner.OS }}-fixtures-
             ${{ runner.OS }}-
 
+      - name: Cache PeerTube pip directory
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/pip
+          key: ${{ runner.OS }}-${{ matrix.test_suite }}-pip-v1
+
+      - name: Cache Hugging Face models
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/huggingface
+          key: ${{ runner.OS }}-${{ matrix.test_suite }}-hugging-face-v1
+
       - name: Set env test variable (schedule)
         if: github.event_name != 'schedule'
         run: |

.gitignore

@@ -12,8 +12,11 @@ yarn-error.log
 /test4/
 /test5/
 /test6/
+
+# Big fixtures generated/downloaded on-demand
 /packages/tests/fixtures/video_high_bitrate_1080p.mp4
 /packages/tests/fixtures/video_59fps.mp4
+/packages/tests/fixtures/transcription/models-v1/
 
 # Production
 /storage

apps/peertube-runner/src/server/process/process.ts

@@ -1,6 +1,7 @@
 import {
   RunnerJobLiveRTMPHLSTranscodingPayload,
   RunnerJobStudioTranscodingPayload,
+  RunnerJobTranscriptionPayload,
   RunnerJobVODAudioMergeTranscodingPayload,
   RunnerJobVODHLSTranscodingPayload,
   RunnerJobVODWebVideoTranscodingPayload
@@ -9,25 +10,41 @@ import { logger } from '../../shared/index.js'
 import { processAudioMergeTranscoding, processHLSTranscoding, ProcessOptions, processWebVideoTranscoding } from './shared/index.js'
 import { ProcessLiveRTMPHLSTranscoding } from './shared/process-live.js'
 import { processStudioTranscoding } from './shared/process-studio.js'
+import { processVideoTranscription } from './shared/process-transcription.js'
 
 export async function processJob (options: ProcessOptions) {
   const { server, job } = options
 
   logger.info(`[${server.url}] Processing job of type ${job.type}: ${job.uuid}`, { payload: job.payload })
 
-  if (job.type === 'vod-audio-merge-transcoding') {
-    await processAudioMergeTranscoding(options as ProcessOptions<RunnerJobVODAudioMergeTranscodingPayload>)
-  } else if (job.type === 'vod-web-video-transcoding') {
-    await processWebVideoTranscoding(options as ProcessOptions<RunnerJobVODWebVideoTranscodingPayload>)
-  } else if (job.type === 'vod-hls-transcoding') {
-    await processHLSTranscoding(options as ProcessOptions<RunnerJobVODHLSTranscodingPayload>)
-  } else if (job.type === 'live-rtmp-hls-transcoding') {
-    await new ProcessLiveRTMPHLSTranscoding(options as ProcessOptions<RunnerJobLiveRTMPHLSTranscodingPayload>).process()
-  } else if (job.type === 'video-studio-transcoding') {
-    await processStudioTranscoding(options as ProcessOptions<RunnerJobStudioTranscodingPayload>)
-  } else {
-    logger.error(`Unknown job ${job.type} to process`)
-    return
+  switch (job.type) {
+    case 'vod-audio-merge-transcoding':
+      await processAudioMergeTranscoding(options as ProcessOptions<RunnerJobVODAudioMergeTranscodingPayload>)
+      break
+
+    case 'vod-web-video-transcoding':
+      await processWebVideoTranscoding(options as ProcessOptions<RunnerJobVODWebVideoTranscodingPayload>)
+      break
+
+    case 'vod-hls-transcoding':
+      await processHLSTranscoding(options as ProcessOptions<RunnerJobVODHLSTranscodingPayload>)
+      break
+
+    case 'live-rtmp-hls-transcoding':
+      await new ProcessLiveRTMPHLSTranscoding(options as ProcessOptions<RunnerJobLiveRTMPHLSTranscodingPayload>).process()
+      break
+
+    case 'video-studio-transcoding':
+      await processStudioTranscoding(options as ProcessOptions<RunnerJobStudioTranscodingPayload>)
+      break
+
+    case 'video-transcription':
+      await processVideoTranscription(options as ProcessOptions<RunnerJobTranscriptionPayload>)
+      break
+
+    default:
+      logger.error(`Unknown job ${job.type} to process`)
+      return
   }
 
   logger.info(`[${server.url}] Finished processing job of type ${job.type}: ${job.uuid}`)

apps/peertube-runner/src/server/process/shared/common.ts

@@ -5,7 +5,7 @@ import { RunnerJob, RunnerJobPayload } from '@peertube/peertube-models'
 import { buildUUID } from '@peertube/peertube-node-utils'
 import { PeerTubeServer } from '@peertube/peertube-server-commands'
 import { ConfigManager, downloadFile, logger } from '../../../shared/index.js'
-import { getTranscodingLogger } from './transcoding-logger.js'
+import { getWinstonLogger } from './winston-logger.js'
 
 export type JobWithToken <T extends RunnerJobPayload = RunnerJobPayload> = RunnerJob<T> & { jobToken: string }
 
@@ -101,6 +101,6 @@ function getCommonFFmpegOptions () {
       available: getDefaultAvailableEncoders(),
       encodersToTry: getDefaultEncodersToTry()
     },
-    logger: getTranscodingLogger()
+    logger: getWinstonLogger()
   }
 }

apps/peertube-runner/src/server/process/shared/index.ts

@@ -1,3 +1,3 @@
 export * from './common.js'
 export * from './process-vod.js'
-export * from './transcoding-logger.js'
+export * from './winston-logger.js'

apps/peertube-runner/src/server/process/shared/process-transcription.ts

@@ -0,0 +1,79 @@
+import { hasAudioStream } from '@peertube/peertube-ffmpeg'
+import { RunnerJobTranscriptionPayload, TranscriptionSuccess } from '@peertube/peertube-models'
+import { buildSUUID } from '@peertube/peertube-node-utils'
+import { TranscriptionModel, WhisperBuiltinModel, transcriberFactory } from '@peertube/peertube-transcription'
+import { remove } from 'fs-extra/esm'
+import { join } from 'path'
+import { ConfigManager } from '../../../shared/config-manager.js'
+import { logger } from '../../../shared/index.js'
+import { ProcessOptions, downloadInputFile, scheduleTranscodingProgress } from './common.js'
+import { getWinstonLogger } from './winston-logger.js'
+
+export async function processVideoTranscription (options: ProcessOptions<RunnerJobTranscriptionPayload>) {
+  const { server, job, runnerToken } = options
+
+  const config = ConfigManager.Instance.getConfig().transcription
+
+  const payload = job.payload
+
+  let inputPath: string
+
+  const updateProgressInterval = scheduleTranscodingProgress({
+    job,
+    server,
+    runnerToken,
+    progressGetter: () => undefined
+  })
+
+  const outputPath = join(ConfigManager.Instance.getTranscriptionDirectory(), buildSUUID())
+
+  const transcriber = transcriberFactory.createFromEngineName({
+    engineName: config.engine,
+    enginePath: config.enginePath,
+    logger: getWinstonLogger()
+  })
+
+  try {
+    logger.info(`Downloading input file ${payload.input.videoFileUrl} for transcription job ${job.jobToken}`)
+
+    inputPath = await downloadInputFile({ url: payload.input.videoFileUrl, runnerToken, job })
+
+    logger.info(`Downloaded input file ${payload.input.videoFileUrl} for job ${job.jobToken}. Running transcription.`)
+
+    if (await hasAudioStream(inputPath) !== true) {
+      await server.runnerJobs.error({
+        jobToken: job.jobToken,
+        jobUUID: job.uuid,
+        runnerToken,
+        message: 'This input file does not contain audio'
+      })
+
+      return
+    }
+
+    const transcriptFile = await transcriber.transcribe({
+      mediaFilePath: inputPath,
+      model: config.modelPath
+        ? await TranscriptionModel.fromPath(config.modelPath)
+        : new WhisperBuiltinModel(config.model),
+      format: 'vtt',
+      transcriptDirectory: outputPath
+    })
+
+    const successBody: TranscriptionSuccess = {
+      inputLanguage: transcriptFile.language,
+      vttFile: transcriptFile.path
+    }
+
+    await server.runnerJobs.success({
+      jobToken: job.jobToken,
+      jobUUID: job.uuid,
+      runnerToken,
+      payload: successBody
+    })
+  } finally {
+    if (inputPath) await remove(inputPath)
+    if (outputPath) await remove(outputPath)
+    if (updateProgressInterval) clearInterval(updateProgressInterval)
+  }
+}

apps/peertube-runner/src/server/process/shared/transcoding-logger.ts

@@ -1,19 +0,0 @@
-import { LogFn } from 'pino'
-import { logger } from '../../../shared/index.js'
-
-export function getTranscodingLogger () {
-  return {
-    info: buildWinstonLogger(logger.info.bind(logger)),
-    debug: buildWinstonLogger(logger.debug.bind(logger)),
-    warn: buildWinstonLogger(logger.warn.bind(logger)),
-    error: buildWinstonLogger(logger.error.bind(logger))
-  }
-}
-
-function buildWinstonLogger (log: LogFn) {
-  return (arg1: string, arg2?: object) => {
-    if (arg2) return log(arg2, arg1)
-
-    return log(arg1)
-  }
-}

apps/peertube-runner/src/server/process/shared/winston-logger.ts

@@ -0,0 +1,19 @@
+import { LogFn } from 'pino'
+import { logger } from '../../../shared/index.js'
+
+export function getWinstonLogger () {
+  return {
+    info: buildLogLevelFn(logger.info.bind(logger)),
+    debug: buildLogLevelFn(logger.debug.bind(logger)),
+    warn: buildLogLevelFn(logger.warn.bind(logger)),
+    error: buildLogLevelFn(logger.error.bind(logger))
+  }
+}
+
+function buildLogLevelFn (log: LogFn) {
+  return (arg1: string, arg2?: object) => {
+    if (arg2) return log(arg2, arg1)
+
+    return log(arg1)
+  }
+}

apps/peertube-runner/src/server/shared/supported-job.ts

@@ -1,15 +1,16 @@
 import {
   RunnerJobLiveRTMPHLSTranscodingPayload,
   RunnerJobPayload,
-  RunnerJobType,
   RunnerJobStudioTranscodingPayload,
+  RunnerJobTranscriptionPayload,
+  RunnerJobType,
   RunnerJobVODAudioMergeTranscodingPayload,
   RunnerJobVODHLSTranscodingPayload,
   RunnerJobVODWebVideoTranscodingPayload,
   VideoStudioTaskPayload
 } from '@peertube/peertube-models'
 
-const supportedMatrix = {
+const supportedMatrix: { [ id in RunnerJobType ]: (payload: RunnerJobPayload) => boolean } = {
   'vod-web-video-transcoding': (_payload: RunnerJobVODWebVideoTranscodingPayload) => {
     return true
   },
@@ -29,6 +30,9 @@ const supportedMatrix = {
     if (!Array.isArray(tasks)) return false
 
     return tasks.every(t => t && supported.has(t.name))
+  },
+  'video-transcription': (_payload: RunnerJobTranscriptionPayload) => {
+    return true
   }
 }
 

apps/peertube-runner/src/shared/config-manager.ts

@@ -1,4 +1,5 @@
 import { parse, stringify } from '@iarna/toml'
+import { TranscriptionEngineName, WhisperBuiltinModelName } from '@peertube/peertube-transcription'
 import envPaths from 'env-paths'
 import { ensureDir, pathExists, remove } from 'fs-extra/esm'
 import { readFile, writeFile } from 'fs/promises'
@@ -24,6 +25,13 @@ type Config = {
     runnerName: string
     runnerDescription?: string
   }[]
+
+  transcription: {
+    engine: TranscriptionEngineName
+    enginePath: string | null
+    model: WhisperBuiltinModelName
+    modelPath: string | null
+  }
 }
 
 export class ConfigManager {
@@ -37,6 +45,12 @@ export class ConfigManager {
       threads: 2,
       nice: 20
     },
+    transcription: {
+      engine: 'whisper-ctranslate2',
+      enginePath: null,
+      model: 'small',
+      modelPath: null
+    },
     registeredInstances: []
   }
 
@@ -98,6 +112,10 @@ export class ConfigManager {
     return join(paths.cache, this.id, 'transcoding')
   }
 
+  getTranscriptionDirectory () {
+    return join(paths.cache, this.id, 'transcription')
+  }
+
   getSocketDirectory () {
     return join(paths.data, this.id)
   }

client/src/app/+admin/config/edit-custom-config/edit-basic-configuration.component.html

@@ -318,7 +318,7 @@
             >
             <ng-container ngProjectAs="description">
               <span i18n [hidden]="isImportVideosHttpEnabled()">
-                  ⛔ You need to allow  import with HTTP URL to be able to activate this feature.
+                ⛔ You need to allow  import with HTTP URL to be able to activate this feature.
               </span>
             </ng-container>
             </my-peertube-checkbox>
@@ -359,7 +359,6 @@
       </ng-container>
 
       <ng-container formGroupName="storyboards">
-
         <div class="form-group">
           <my-peertube-checkbox
             inputName="storyboardsEnabled" formControlName="enabled"
@@ -370,7 +369,35 @@
             </ng-container>
           </my-peertube-checkbox>
         </div>
+      </ng-container>
 
+      <ng-container formGroupName="videoTranscription">
+        <div class="form-group">
+          <my-peertube-checkbox
+            inputName="videoTranscriptionEnabled" formControlName="enabled"
+            i18n-labelText labelText="Enable video transcription"
+          >
+            <ng-container ngProjectAs="description">
+              <span i18n>Automatically create a subtitle file of uploaded/imported VOD videos</span>
+            </ng-container>
+
+            <ng-container ngProjectAs="extra">
+              <div class="form-group" formGroupName="remoteRunners" [ngClass]="getTranscriptionRunnerDisabledClass()">
+                <my-peertube-checkbox
+                  inputName="videoTranscriptionRemoteRunnersEnabled" formControlName="enabled"
+                  i18n-labelText labelText="Enable remote runners for transcription"
+                >
+                  <ng-container ngProjectAs="description">
+                    <span i18n>
+                      Use <a routerLink="/admin/system/runners/runners-list">remote runners</a> to process transcription tasks.
+                      Remote runners has to register on your instance first.
+                    </span>
+                  </ng-container>
+                </my-peertube-checkbox>
+              </div>
+            </ng-container>
+          </my-peertube-checkbox>
+        </div>
       </ng-container>
     </div>
   </div>

client/src/app/+admin/config/edit-custom-config/edit-basic-configuration.component.ts

@@ -137,6 +137,18 @@ export class EditBasicConfigurationComponent implements OnInit, OnChanges {
     return { 'disabled-checkbox-extra': !this.isSearchIndexEnabled() }
   }
 
+  // ---------------------------------------------------------------------------
+
+  isTranscriptionEnabled () {
+    return this.form.value['videoTranscription']['enabled'] === true
+  }
+
+  getTranscriptionRunnerDisabledClass () {
+    return { 'disabled-checkbox-extra': !this.isTranscriptionEnabled() }
+  }
+
+  // ---------------------------------------------------------------------------
+
   isAutoFollowIndexEnabled () {
     return this.form.value['followings']['instance']['autoFollowIndex']['enabled'] === true
   }

client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts

@@ -267,6 +267,12 @@ export class EditCustomConfigComponent extends FormReactive implements OnInit {
           enabled: null
         }
       },
+      videoTranscription: {
+        enabled: null,
+        remoteRunners: {
+          enabled: null
+        }
+      },
       videoFile: {
         update: {
           enabled: null

client/src/app/+admin/overview/videos/video-list.component.ts

@@ -1,7 +1,7 @@
 import { DatePipe, NgClass, NgFor, NgIf } from '@angular/common'
 import { Component, OnInit, ViewChild } from '@angular/core'
 import { ActivatedRoute, Router, RouterLink } from '@angular/router'
-import { AuthService, ConfirmService, Notifier, RestPagination, RestTable } from '@app/core'
+import { AuthService, ConfirmService, Notifier, RestPagination, RestTable, ServerService } from '@app/core'
 import { formatICU, getAbsoluteAPIUrl } from '@app/helpers'
 import { VideoDetails } from '@app/shared/shared-main/video/video-details.model'
 import { VideoFileTokenService } from '@app/shared/shared-main/video/video-file-token.service'
@@ -30,6 +30,7 @@ import {
   VideoActionsDropdownComponent
 } from '../../../shared/shared-video-miniature/video-actions-dropdown.component'
 import { VideoAdminService } from './video-admin.service'
+import { VideoCaptionService } from '@app/shared/shared-main/video-caption/video-caption.service'
 
 @Component({
   selector: 'my-video-list',
@@ -84,7 +85,8 @@ export class VideoListComponent extends RestTable <Video> implements OnInit {
     removeFiles: true,
     transcoding: true,
     studio: true,
-    stats: true
+    stats: true,
+    generateTranscription: true
   }
 
   loading = true
@@ -100,6 +102,8 @@ export class VideoListComponent extends RestTable <Video> implements OnInit {
     private videoService: VideoService,
     private videoAdminService: VideoAdminService,
     private videoBlockService: VideoBlockService,
+    private videoCaptionService: VideoCaptionService,
+    private server: ServerService,
     private videoFileTokenService: VideoFileTokenService
   ) {
     super()
@@ -109,6 +113,10 @@ export class VideoListComponent extends RestTable <Video> implements OnInit {
     return this.auth.getUser()
   }
 
+  get serverConfig () {
+    return this.server.getHTMLConfig()
+  }
+
   ngOnInit () {
     this.initialize()
 
@@ -160,6 +168,14 @@ export class VideoListComponent extends RestTable <Video> implements OnInit {
           isDisplayed: videos => videos.every(v => v.canRemoveFiles(this.authUser)),
           iconName: 'delete'
         }
+      ],
+      [
+        {
+          label: $localize`Generate caption`,
+          handler: videos => this.generateCaption(videos),
+          isDisplayed: videos => videos.every(v => v.canGenerateTranscription(this.authUser, this.serverConfig.videoTranscription.enabled)),
+          iconName: 'video-lang'
+        }
       ]
     ]
   }
@@ -399,4 +415,15 @@ export class VideoListComponent extends RestTable <Video> implements OnInit {
         error: err => this.notifier.error(err.message)
       })
   }
+
+  private generateCaption (videos: Video[]) {
+    this.videoCaptionService.generateCaption(videos.map(v => v.id))
+      .subscribe({
+        next: () => {
+          this.notifier.success($localize`Transcription jobs created.`)
+        },
+
+        error: err => this.notifier.error(err.message)
+      })
+  }
 }

client/src/app/+admin/system/jobs/jobs.component.ts

@@ -69,6 +69,7 @@ export class JobsComponent extends RestTable implements OnInit {
     'video-redundancy',
     'video-studio-edition',
     'video-transcoding',
+    'video-transcription',
     'videos-views-stats'
   ]
 

client/src/app/+my-account/my-account-settings/my-account-notification-preferences/my-account-notification-preferences.component.ts

@@ -49,7 +49,8 @@ export class MyAccountNotificationPreferencesComponent implements OnInit {
       abuseStateChange: $localize`One of your abuse reports has been accepted or rejected by moderators`,
       newPeerTubeVersion: $localize`A new PeerTube version is available`,
       newPluginVersion: $localize`One of your plugin/theme has a new available version`,
-      myVideoStudioEditionFinished: $localize`Video studio edition has finished`
+      myVideoStudioEditionFinished: $localize`Video studio edition has finished`,
+      myVideoTranscriptionGenerated: $localize`The transcription of your video has been generated`
     }
     this.notificationSettingGroups = [
       {
@@ -68,7 +69,8 @@ export class MyAccountNotificationPreferencesComponent implements OnInit {
           'blacklistOnMyVideo',
           'myVideoPublished',
           'myVideoImportFinished',
-          'myVideoStudioEditionFinished'
+          'myVideoStudioEditionFinished',
+          'myVideoTranscriptionGenerated'
         ]
       },
 

client/src/app/+videos/+video-edit/shared/video-edit.component.html

@@ -173,12 +173,8 @@
 
       <ng-template ngbNavContent>
         <div class="captions">
-
-          <div class="captions-header">
-            <button (click)="openAddCaptionModal()" class="peertube-create-button">
-              <my-global-icon iconName="add" aria-hidden="true"></my-global-icon>
-              <ng-container i18n>Add another caption</ng-container>
-            </button>
+          <div class="alert pt-alert-primary" *ngIf="displayTranscriptionInfo && isTranscriptionEnabled()" i18n>
+            A subtitle will be automatically generated from your video.
           </div>
 
           <div class="form-group" *ngFor="let videoCaption of videoCaptions">
@@ -226,6 +222,13 @@
             No captions for now.
           </div>
 
+          <div class="mt-3 mb-3">
+            <button (click)="openAddCaptionModal()" class="peertube-create-button">
+              <my-global-icon iconName="add" aria-hidden="true"></my-global-icon>
+              <ng-container i18n>Add a caption</ng-container>
+            </button>
+          </div>
+
         </div>
       </ng-template>
     </ng-container>

client/src/app/+videos/+video-edit/shared/video-edit.component.scss

@@ -24,11 +24,6 @@ my-peertube-checkbox {
   }
 }
 
-.captions-header {
-  text-align: end;
-  margin-bottom: 1rem;
-}
-
 .caption-entry {
   display: flex;
   height: 40px;

client/src/app/+videos/+video-edit/shared/video-edit.component.ts

@@ -1,5 +1,16 @@
 import { DatePipe, NgClass, NgFor, NgIf, NgTemplateOutlet } from '@angular/common'
-import { ChangeDetectorRef, Component, EventEmitter, Input, NgZone, OnDestroy, OnInit, Output, ViewChild } from '@angular/core'
+import {
+  ChangeDetectorRef,
+  Component,
+  EventEmitter,
+  Input,
+  NgZone,
+  OnDestroy,
+  OnInit,
+  Output,
+  ViewChild,
+  booleanAttribute
+} from '@angular/core'
 import { AbstractControl, FormArray, FormGroup, FormsModule, ReactiveFormsModule, Validators } from '@angular/forms'
 import { HooksService, PluginService, ServerService } from '@app/core'
 import { removeElementFromArray } from '@app/helpers'
@@ -63,10 +74,10 @@ import { HelpComponent } from '../../../shared/shared-main/misc/help.component'
 import { EmbedComponent } from '../../../shared/shared-main/video/embed.component'
 import { LiveDocumentationLinkComponent } from '../../../shared/shared-video-live/live-documentation-link.component'
 import { I18nPrimengCalendarService } from './i18n-primeng-calendar.service'
+import { ThumbnailManagerComponent } from './thumbnail-manager/thumbnail-manager.component'
 import { VideoCaptionAddModalComponent } from './video-caption-add-modal.component'
 import { VideoCaptionEditModalContentComponent } from './video-caption-edit-modal-content/video-caption-edit-modal-content.component'
 import { VideoEditType } from './video-edit.type'
-import { ThumbnailManagerComponent } from './thumbnail-manager/thumbnail-manager.component'
 
 type VideoLanguages = VideoConstant<string> & { group?: string }
 type PluginField = {
@@ -122,15 +133,17 @@ export class VideoEditComponent implements OnInit, OnDestroy {
   @Input() publishedVideo: VideoDetails
 
   @Input() userVideoChannels: SelectChannelItem[] = []
-  @Input() forbidScheduledPublication = true
+
+  @Input({ transform: booleanAttribute }) forbidScheduledPublication = true
+  @Input({ transform: booleanAttribute }) displayTranscriptionInfo = true
 
   @Input() videoCaptions: VideoCaptionWithPathEdit[] = []
   @Input() videoSource: VideoSource
 
   @Input() videoChapters: VideoChapter[] = []
 
-  @Input() hideWaitTranscoding = false
-  @Input() updateVideoFileEnabled = false
+  @Input({ transform: booleanAttribute }) hideWaitTranscoding = false
+  @Input({ transform: booleanAttribute }) updateVideoFileEnabled = false
 
   @Input() type: VideoEditType
   @Input() liveVideo: LiveVideo
@@ -405,6 +418,10 @@ export class VideoEditComponent implements OnInit, OnDestroy {
     return !!this.form.value['originallyPublishedAt']
   }
 
+  isTranscriptionEnabled () {
+    return this.serverConfig.videoTranscription.enabled
+  }
+
   // ---------------------------------------------------------------------------
 
   resetField (name: string) {

client/src/app/+videos/+video-edit/video-add-components/video-go-live.component.html

@@ -53,8 +53,8 @@
 <form [hidden]="!isInUpdateForm" novalidate [formGroup]="form">
   <my-video-edit
     [form]="form" [formErrors]="formErrors" [videoCaptions]="videoCaptions"
-    [forbidScheduledPublication]="true" [hideWaitTranscoding]="true"
     [validationMessages]="validationMessages" [userVideoChannels]="userVideoChannels" [liveVideo]="liveVideo"
+    forbidScheduledPublication="true" hideWaitTranscoding="true" displayTranscriptionInfo="false"
     type="go-live"
   ></my-video-edit>
 

client/src/app/+videos/+video-edit/video-add-components/video-import-torrent.component.html

@@ -59,8 +59,9 @@
 <!-- Hidden because we want to load the component -->
 <form [hidden]="!hasImportedVideo" novalidate [formGroup]="form">
   <my-video-edit
-    [form]="form" [formErrors]="formErrors" [videoCaptions]="videoCaptions" [forbidScheduledPublication]="true"
+    [form]="form" [formErrors]="formErrors" [videoCaptions]="videoCaptions"
     [validationMessages]="validationMessages" [userVideoChannels]="userVideoChannels"
+    forbidScheduledPublication="true"
     type="import-torrent"
   ></my-video-edit>
 

client/src/app/+videos/+video-edit/video-add-components/video-import-url.component.html

@@ -57,8 +57,9 @@
 <form [hidden]="!hasImportedVideo" novalidate [formGroup]="form">
   <my-video-edit
     #videoEdit
-    [form]="form" [formErrors]="formErrors" [videoCaptions]="videoCaptions" [forbidScheduledPublication]="true"
+    [form]="form" [formErrors]="formErrors" [videoCaptions]="videoCaptions"
     [validationMessages]="validationMessages" [userVideoChannels]="userVideoChannels"
+    forbidScheduledPublication="true"
     type="import-url"
   ></my-video-edit>
 

client/src/app/+videos/+video-edit/video-add-components/video-upload.component.html

@@ -67,7 +67,7 @@
   <my-video-edit
     [form]="form" [formErrors]="formErrors" [videoCaptions]="videoCaptions"
     [validationMessages]="validationMessages" [userVideoChannels]="userVideoChannels"
-    [forbidScheduledPublication]="false"
+    forbidScheduledPublication="false"
     type="upload"
   ></my-video-edit>
 

client/src/app/+videos/+video-edit/video-update.component.html

@@ -20,6 +20,7 @@
       type="update" (pluginFieldsAdded)="hydratePluginFieldsFromVideo()"
       [liveVideo]="liveVideo" [publishedVideo]="videoDetails"
       [videoSource]="videoSource" [updateVideoFileEnabled]="isUpdateVideoFileEnabled()"
+      displayTranscriptionInfo="false"
 
       (formBuilt)="onFormBuilt()"
     >

client/src/app/shared/shared-instance/instance-features-table.component.html

@@ -34,6 +34,13 @@
       </td>
     </tr>
 
+    <tr>
+      <th i18n class="sub-label" scope="row">Automatic transcription</th>
+      <td>
+        <my-feature-boolean [value]="serverConfig.videoTranscription.enabled"></my-feature-boolean>
+      </td>
+    </tr>
+
     <tr>
       <th i18n class="sub-label" scope="row">Video uploads</th>
       <td>

client/src/app/shared/shared-main/buttons/action-dropdown.component.scss

@@ -63,6 +63,10 @@
 
     &.with-icon {
       @include dropdown-with-icon-item;
+
+      .icon-video-lang {
+        top: 0;
+      }
     }
 
     a,

client/src/app/shared/shared-main/users/user-notification.model.ts

@@ -11,6 +11,7 @@ import {
   UserNotificationType,
   UserNotificationType_Type,
   UserRight,
+  VideoConstant,
   VideoInfo
 } from '@peertube/peertube-models'
 import { logger } from '@root-helpers/logger'
@@ -90,6 +91,12 @@ export class UserNotification implements UserNotificationServer {
     username: string
   }
 
+  videoCaption?: {
+    id: number
+    language: VideoConstant<string>
+    video: VideoInfo
+  }
+
   createdAt: string
   updatedAt: string
 
@@ -149,6 +156,8 @@ export class UserNotification implements UserNotificationServer {
       this.peertube = hash.peertube
       this.registration = hash.registration
 
+      this.videoCaption = hash.videoCaption
+
       this.createdAt = hash.createdAt
       this.updatedAt = hash.updatedAt
 
@@ -250,6 +259,10 @@ export class UserNotification implements UserNotificationServer {
           this.pluginQueryParams.pluginType = this.plugin.type + ''
           break
 
+        case UserNotificationType.MY_VIDEO_TRANSCRIPTION_GENERATED:
+          this.videoUrl = this.buildVideoUrl(this.videoCaption.video)
+          break
+
         case UserNotificationType.MY_VIDEO_STUDIO_EDITION_FINISHED:
           this.videoUrl = this.buildVideoUrl(this.video)
           break

client/src/app/shared/shared-main/video-caption/video-caption.service.ts

@@ -1,15 +1,15 @@
-import { Observable, of } from 'rxjs'
-import { catchError, map, switchMap } from 'rxjs/operators'
 import { HttpClient } from '@angular/common/http'
 import { Injectable } from '@angular/core'
 import { RestExtractor, ServerService } from '@app/core'
 import { objectToFormData } from '@app/helpers'
 import { peertubeTranslate, sortBy } from '@peertube/peertube-core-utils'
 import { ResultList, VideoCaption } from '@peertube/peertube-models'
+import { Observable, from, of } from 'rxjs'
+import { catchError, concatMap, map, switchMap, toArray } from 'rxjs/operators'
 import { environment } from '../../../../environments/environment'
-import { VideoCaptionEdit } from './video-caption-edit.model'
 import { VideoPasswordService } from '../video/video-password.service'
 import { VideoService } from '../video/video.service'
+import { VideoCaptionEdit } from './video-caption-edit.model'
 
 @Injectable()
 export class VideoCaptionService {
@@ -74,4 +74,13 @@ export class VideoCaptionService {
   getCaptionContent ({ captionPath }: Pick<VideoCaption, 'captionPath'>) {
     return this.authHttp.get(environment.originServerUrl + captionPath, { responseType: 'text' })
   }
+
+  generateCaption (videoIds: (number | string)[]) {
+    return from(videoIds)
+      .pipe(
+        concatMap(videoId => this.authHttp.post(`${VideoService.BASE_VIDEO_URL}/${videoId}/captions/generate`, {})),
+        toArray(),
+        catchError(err => this.restExtractor.handleError(err))
+      )
+  }
 }

client/src/app/shared/shared-main/video/video.model.ts

@@ -244,6 +244,10 @@ export class Video implements VideoServerModel {
       this.isUpdatableBy(user)
   }
 
+  canGenerateTranscription (user: AuthUser, transcriptionEnabled: boolean) {
+    return transcriptionEnabled && this.isLocal && user.hasRight(UserRight.UPDATE_ANY_VIDEO)
+  }
+
   // ---------------------------------------------------------------------------
 
   isOwner (user: AuthUser) {

client/src/app/shared/shared-video-miniature/video-actions-dropdown.component.ts

@@ -11,6 +11,7 @@ import {
   DropdownButtonSize,
   DropdownDirection
 } from '../shared-main/buttons/action-dropdown.component'
+import { VideoCaptionService } from '../shared-main/video-caption/video-caption.service'
 import { RedundancyService } from '../shared-main/video/redundancy.service'
 import { VideoDetails } from '../shared-main/video/video-details.model'
 import { Video } from '../shared-main/video/video.model'
@@ -37,6 +38,7 @@ export type VideoActionsDisplayType = {
   transcoding?: boolean
   studio?: boolean
   stats?: boolean
+  generateTranscription?: boolean
 }
 
 @Component({
@@ -115,6 +117,7 @@ export class VideoActionsDropdownComponent implements OnChanges {
     private videoBlocklistService: VideoBlockService,
     private screenService: ScreenService,
     private videoService: VideoService,
+    private videoCaptionService: VideoCaptionService,
     private redundancyService: RedundancyService,
     private serverService: ServerService
   ) { }
@@ -206,6 +209,10 @@ export class VideoActionsDropdownComponent implements OnChanges {
     return this.video.isLiveInfoAvailableBy(this.user)
   }
 
+  canGenerateTranscription () {
+    return this.video.canGenerateTranscription(this.user, this.serverService.getHTMLConfig().videoTranscription.enabled)
+  }
+
   isVideoDownloadableByAnonymous () {
     return (
       this.video &&
@@ -338,7 +345,7 @@ export class VideoActionsDropdownComponent implements OnChanges {
     this.videoService.runTranscoding({ videos: [ video ], type, askForForceTranscodingIfNeeded: true })
       .subscribe({
         next: () => {
-          this.notifier.success($localize`Transcoding jobs created for "${video.name}".`)
+          this.notifier.success($localize`Transcoding job created for "${video.name}".`)
           this.transcodingCreated.emit()
         },
 
@@ -346,6 +353,17 @@ export class VideoActionsDropdownComponent implements OnChanges {
       })
   }
 
+  generateCaption (video: Video) {
+    this.videoCaptionService.generateCaption([ video.id ])
+      .subscribe({
+        next: () => {
+          this.notifier.success($localize`Transcription job created for "${video.name}".`)
+        },
+
+        error: err => this.notifier.error(err.message)
+      })
+  }
+
   onVideoBlocked () {
     this.videoBlocked.emit()
   }
@@ -466,6 +484,14 @@ export class VideoActionsDropdownComponent implements OnChanges {
           iconName: 'delete'
         }
       ],
+      [
+        {
+          label: $localize`Generate caption`,
+          handler: ({ video }) => this.generateCaption(video),
+          isDisplayed: () => this.displayOptions.generateTranscription && this.canGenerateTranscription(),
+          iconName: 'video-lang'
+        }
+      ],
       [ // actions regarding the account/its server
         {
           label: $localize`Mute account`,

client/src/app/shared/standalone-notifications/user-notifications.component.html

@@ -239,6 +239,14 @@
         }
       </ng-container>
 
+      <ng-container *ngSwitchCase="22"> <!-- UserNotificationType.MY_VIDEO_TRANSCRIPTION_GENERATED -->
+        <my-global-icon iconName="video-lang" aria-hidden="true"></my-global-icon>
+
+        <div class="message" i18n>
+          <em>{{ notification.videoCaption.language.label }}</em> transcription of <a (click)="markAsRead(notification)" [routerLink]="notification.videoUrl">your video {{ notification.videoCaption.video.name }}</a> has been generated
+        </div>
+      </ng-container>
+
       <ng-container *ngSwitchDefault>
         <my-global-icon iconName="alert" aria-hidden="true"></my-global-icon>
 

config/default.yaml

@@ -716,6 +716,34 @@ video_studio:
   remote_runners:
     enabled: false
 
+video_transcription:
+  # Enable automatic transcription of videos
+  enabled: false
+
+  # Choose engine for local transcription
+  # Supported: 'openai-whisper' or 'whisper-ctranslate2'
+  engine: 'whisper-ctranslate2'
+
+  # You can set a custom engine path for local transcription
+  # If not provided, PeerTube will try to automatically install it in the PeerTube bin directory
+  engine_path: null
+
+  # Choose engine model for local transcription
+  # Available for 'openai-whisper' and 'whisper-ctranslate2': 'tiny', 'base', 'small', 'medium' or 'large-v3'
+  model: 'small'
+
+  # Or specify the model path:
+  #  * PyTorch model file path for 'openai-whisper'
+  #  * CTranslate2 Whisper model directory path for 'whisper-ctranslate2'
+  # If not provided, PeerTube will automatically download the model
+  model_path: null
+
+  # Enable remote runners to transcribe videos
+  # If enabled, your instance won't transcribe the videos itself
+  # At least 1 remote runner must be configured to transcribe your videos
+  remote_runners:
+    enabled: false
+
 video_file:
   update:
     # Add ability for users to replace the video file of an existing video

config/production.yaml.example

@@ -726,6 +726,34 @@ video_studio:
   remote_runners:
     enabled: false
 
+video_transcription:
+  # Enable automatic transcription of videos
+  enabled: false
+
+  # Choose engine for local transcription
+  # Supported: 'openai-whisper' or 'whisper-ctranslate2'
+  engine: 'whisper-ctranslate2'
+
+  # You can set a custom engine path for local transcription
+  # If not provided, PeerTube will try to automatically install it in the PeerTube bin directory
+  engine_path: null
+
+  # Choose engine model for local transcription
+  # Available for 'openai-whisper' and 'whisper-ctranslate2': 'tiny', 'base', 'small', 'medium' or 'large-v3'
+  model: 'small'
+
+  # Or specify the model path:
+  #  * PyTorch model file path for 'openai-whisper'
+  #  * CTranslate2 Whisper model directory path for 'whisper-ctranslate2'
+  # If not provided, PeerTube will automatically download the model
+  model_path: null
+
+  # Enable remote runners to transcribe videos
+  # If enabled, your instance won't transcribe the videos itself
+  # At least 1 remote runner must be configured to transcribe your videos
+  remote_runners:
+    enabled: false
+
 video_file:
   update:
     # Add ability for users to replace the video file of an existing video

config/test.yaml

@@ -166,3 +166,6 @@ open_telemetry:
 search:
   search_index:
     url: 'https://search.joinpeertube.org/'
+
+video_transcription:
+  model: 'tiny'

packages/ffmpeg/src/ffmpeg-command-wrapper.ts

@@ -1,14 +1,13 @@
 import { pick, promisify0 } from '@peertube/peertube-core-utils'
-import { AvailableEncoders, EncoderOptionsBuilder, EncoderOptionsBuilderParams, EncoderProfile } from '@peertube/peertube-models'
+import {
+  AvailableEncoders,
+  EncoderOptionsBuilder,
+  EncoderOptionsBuilderParams,
+  EncoderProfile,
+  SimpleLogger
+} from '@peertube/peertube-models'
 import ffmpeg, { FfmpegCommand } from 'fluent-ffmpeg'
 
-type FFmpegLogger = {
-  info: (msg: string, obj?: object) => void
-  debug: (msg: string, obj?: object) => void
-  warn: (msg: string, obj?: object) => void
-  error: (msg: string, obj?: object) => void
-}
-
 export interface FFmpegCommandWrapperOptions {
   availableEncoders?: AvailableEncoders
   profile?: string
@@ -17,7 +16,7 @@ export interface FFmpegCommandWrapperOptions {
   tmpDirectory: string
   threads: number
 
-  logger: FFmpegLogger
+  logger: SimpleLogger
   lTags?: { tags: string[] }
 
   updateJobProgress?: (progress?: number) => void
@@ -35,7 +34,7 @@ export class FFmpegCommandWrapper {
   private readonly tmpDirectory: string
   private readonly threads: number
 
-  private readonly logger: FFmpegLogger
+  private readonly logger: SimpleLogger
   private readonly lTags: { tags: string[] }
 
   private readonly updateJobProgress: (progress?: number) => void

packages/jiwer/README.md

@@ -1,37 +0,0 @@
-JiWER
-=====
-__JiWER__ CLI NodeJs wrapper.
-
-> *JiWER is a python tool for computing the word-error-rate of ASR systems.*
-> https://jitsi.github.io/jiwer/cli/
-
-__JiWER__ serves as a reference implementation to calculate errors rates between 2 text files:
-- WER (Word Error Rate)
-- CER (Character Error Rate)
-
-Build
------
-
-```sh
-npm run build
-```
-
-Usage
------
-```typescript
-const jiwerCLI = new JiwerClI('./reference.txt', './hypothesis.txt')
-
-// WER as a percentage, ex: 0.03 -> 3%
-console.log(await jiwerCLI.wer())
-
-// CER as a percentage: 0.01 -> 1%
-console.log(await jiwerCLI.cer())
-
-// Detailed comparison report
-console.log(await jiwerCLI.alignment())
-```
-
-Resources
----------
-- https://jitsi.github.io/jiwer/
-- https://github.com/rapidfuzz/RapidFuzz

packages/jiwer/requirements.txt

@@ -1 +0,0 @@
-jiwer==3.0.4

packages/jiwer/src/index.ts

@@ -1 +0,0 @@
-export * from './jiwer-cli.js'

packages/models/src/common/index.ts

@@ -1,2 +1,3 @@
 export * from './file-storage.enum.js'
 export * from './result-list.model.js'
+export * from './simple-logger.model.js'

packages/models/src/common/simple-logger.model.ts

@@ -0,0 +1,6 @@
+export type SimpleLogger = {
+  info: (msg: string, obj?: object) => void
+  debug: (msg: string, obj?: object) => void
+  warn: (msg: string, obj?: object) => void
+  error: (msg: string, obj?: object) => void
+}

packages/models/src/runners/runner-job-payload.model.ts

@@ -8,7 +8,8 @@ export type RunnerJobVODPayload =
 export type RunnerJobPayload =
   RunnerJobVODPayload |
   RunnerJobLiveRTMPHLSTranscodingPayload |
-  RunnerJobStudioTranscodingPayload
+  RunnerJobStudioTranscodingPayload |
+  RunnerJobTranscriptionPayload
 
 // ---------------------------------------------------------------------------
 
@@ -54,6 +55,12 @@ export interface RunnerJobStudioTranscodingPayload {
   tasks: VideoStudioTaskPayload[]
 }
 
+export interface RunnerJobTranscriptionPayload {
+  input: {
+    videoFileUrl: string
+  }
+}
+
 // ---------------------------------------------------------------------------
 
 export function isAudioMergeTranscodingPayload (payload: RunnerJobPayload): payload is RunnerJobVODAudioMergeTranscodingPayload {

packages/models/src/runners/runner-job-private-payload.model.ts

@@ -8,7 +8,8 @@ export type RunnerJobVODPrivatePayload =
 export type RunnerJobPrivatePayload =
   RunnerJobVODPrivatePayload |
   RunnerJobLiveRTMPHLSTranscodingPrivatePayload |
-  RunnerJobVideoStudioTranscodingPrivatePayload
+  RunnerJobVideoStudioTranscodingPrivatePayload |
+  RunnerJobTranscriptionPrivatePayload
 
 // ---------------------------------------------------------------------------
 
@@ -45,3 +46,9 @@ export interface RunnerJobVideoStudioTranscodingPrivatePayload {
   videoUUID: string
   originalTasks: VideoStudioTaskPayload[]
 }
+
+// ---------------------------------------------------------------------------
+
+export interface RunnerJobTranscriptionPrivatePayload {
+  videoUUID: string
+}

packages/models/src/runners/runner-job-success-body.model.ts

@@ -12,7 +12,8 @@ export type RunnerJobSuccessPayload =
   VODHLSTranscodingSuccess |
   VODAudioMergeTranscodingSuccess |
   LiveRTMPHLSTranscodingSuccess |
-  VideoStudioTranscodingSuccess
+  VideoStudioTranscodingSuccess |
+  TranscriptionSuccess
 
 export interface VODWebVideoTranscodingSuccess {
   videoFile: Blob | string
@@ -35,6 +36,12 @@ export interface VideoStudioTranscodingSuccess {
   videoFile: Blob | string
 }
 
+export interface TranscriptionSuccess {
+  inputLanguage: string
+
+  vttFile: Blob | string
+}
+
 export function isWebVideoOrAudioMergeTranscodingPayloadSuccess (
   payload: RunnerJobSuccessPayload
 ): payload is VODHLSTranscodingSuccess | VODAudioMergeTranscodingSuccess {
@@ -44,3 +51,7 @@ export function isWebVideoOrAudioMergeTranscodingPayloadSuccess (
 export function isHLSTranscodingPayloadSuccess (payload: RunnerJobSuccessPayload): payload is VODHLSTranscodingSuccess {
   return !!(payload as VODHLSTranscodingSuccess)?.resolutionPlaylistFile
 }
+
+export function isTranscriptionPayloadSuccess (payload: RunnerJobSuccessPayload): payload is TranscriptionSuccess {
+  return !!(payload as TranscriptionSuccess)?.vttFile
+}

packages/models/src/runners/runner-job-type.type.ts

@@ -3,4 +3,5 @@ export type RunnerJobType =
   'vod-hls-transcoding' |
   'vod-audio-merge-transcoding' |
   'live-rtmp-hls-transcoding' |
-  'video-studio-transcoding'
+  'video-studio-transcoding' |
+  'video-transcription'

packages/models/src/server/custom-config.model.ts

@@ -179,6 +179,14 @@ export interface CustomConfig {
     }
   }
 
+  videoTranscription: {
+    enabled: boolean
+
+    remoteRunners: {
+      enabled: boolean
+    }
+  }
+
   videoFile: {
     update: {
       enabled: boolean