2024-06-13 02:23:12 -05:00
|
|
|
import { hasAudioStream } from '@peertube/peertube-ffmpeg'
|
|
|
|
import { buildSUUID } from '@peertube/peertube-node-utils'
|
|
|
|
import { AbstractTranscriber, TranscriptionModel, WhisperBuiltinModel, transcriberFactory } from '@peertube/peertube-transcription'
|
2024-02-14 02:21:53 -06:00
|
|
|
import { moveAndProcessCaptionFile } from '@server/helpers/captions-utils.js'
|
2024-06-13 02:23:12 -05:00
|
|
|
import { isVideoCaptionLanguageValid } from '@server/helpers/custom-validators/video-captions.js'
|
|
|
|
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
|
|
|
|
import { CONFIG } from '@server/initializers/config.js'
|
|
|
|
import { DIRECTORIES } from '@server/initializers/constants.js'
|
2024-02-14 02:21:53 -06:00
|
|
|
import { sequelizeTypescript } from '@server/initializers/database.js'
|
|
|
|
import { VideoCaptionModel } from '@server/models/video/video-caption.js'
|
2024-06-13 02:23:12 -05:00
|
|
|
import { VideoJobInfoModel } from '@server/models/video/video-job-info.js'
|
|
|
|
import { VideoModel } from '@server/models/video/video.js'
|
|
|
|
import { MVideo, MVideoCaption, MVideoFullLight, MVideoUUID, MVideoUrl } from '@server/types/models/index.js'
|
|
|
|
import { ensureDir, remove } from 'fs-extra/esm'
|
|
|
|
import { join } from 'path'
|
|
|
|
import { federateVideoIfNeeded } from './activitypub/videos/federate.js'
|
|
|
|
import { JobQueue } from './job-queue/job-queue.js'
|
|
|
|
import { Notifier } from './notifier/notifier.js'
|
|
|
|
import { TranscriptionJobHandler } from './runners/index.js'
|
|
|
|
import { VideoPathManager } from './video-path-manager.js'
|
|
|
|
|
|
|
|
const lTags = loggerTagsFactory('video-caption')
|
2024-02-14 02:21:53 -06:00
|
|
|
|
|
|
|
export async function createLocalCaption (options: {
|
|
|
|
video: MVideo
|
|
|
|
path: string
|
|
|
|
language: string
|
2024-06-27 08:29:26 -05:00
|
|
|
automaticallyGenerated: boolean
|
2024-02-14 02:21:53 -06:00
|
|
|
}) {
|
2024-06-27 08:29:26 -05:00
|
|
|
const { language, path, video, automaticallyGenerated } = options
|
2024-02-14 02:21:53 -06:00
|
|
|
|
|
|
|
const videoCaption = new VideoCaptionModel({
|
|
|
|
videoId: video.id,
|
|
|
|
filename: VideoCaptionModel.generateCaptionName(language),
|
2024-06-27 08:29:26 -05:00
|
|
|
language,
|
|
|
|
automaticallyGenerated
|
2024-02-14 02:21:53 -06:00
|
|
|
}) as MVideoCaption
|
|
|
|
|
|
|
|
await moveAndProcessCaptionFile({ path }, videoCaption)
|
|
|
|
|
|
|
|
await sequelizeTypescript.transaction(async t => {
|
|
|
|
await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t)
|
|
|
|
})
|
|
|
|
|
2024-06-13 02:23:12 -05:00
|
|
|
return Object.assign(videoCaption, { Video: video })
|
|
|
|
}
|
|
|
|
|
|
|
|
export async function createTranscriptionTaskIfNeeded (video: MVideoUUID & MVideoUrl) {
|
|
|
|
if (CONFIG.VIDEO_TRANSCRIPTION.ENABLED !== true) return
|
|
|
|
|
|
|
|
logger.info(`Creating transcription job for ${video.url}`, lTags(video.uuid))
|
|
|
|
|
|
|
|
if (CONFIG.VIDEO_TRANSCRIPTION.REMOTE_RUNNERS.ENABLED === true) {
|
|
|
|
await new TranscriptionJobHandler().create({ video })
|
|
|
|
} else {
|
|
|
|
await JobQueue.Instance.createJob({ type: 'video-transcription', payload: { videoUUID: video.uuid } })
|
|
|
|
}
|
|
|
|
|
|
|
|
await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingTranscription')
|
|
|
|
}
|
|
|
|
|
|
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// Transcription task
|
|
|
|
// ---------------------------------------------------------------------------
|
|
|
|
|
|
|
|
let transcriber: AbstractTranscriber
|
|
|
|
|
|
|
|
export async function generateSubtitle (options: {
|
|
|
|
video: MVideoUUID
|
|
|
|
}) {
|
|
|
|
const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(options.video.uuid)
|
|
|
|
|
|
|
|
const outputPath = join(CONFIG.STORAGE.TMP_DIR, 'transcription', buildSUUID())
|
|
|
|
await ensureDir(outputPath)
|
|
|
|
|
|
|
|
const binDirectory = join(DIRECTORIES.LOCAL_PIP_DIRECTORY, 'bin')
|
|
|
|
|
|
|
|
try {
|
|
|
|
// Lazy load the transcriber
|
|
|
|
if (!transcriber) {
|
|
|
|
transcriber = transcriberFactory.createFromEngineName({
|
|
|
|
engineName: CONFIG.VIDEO_TRANSCRIPTION.ENGINE,
|
|
|
|
enginePath: CONFIG.VIDEO_TRANSCRIPTION.ENGINE_PATH,
|
|
|
|
logger,
|
|
|
|
binDirectory
|
|
|
|
})
|
|
|
|
|
|
|
|
if (!CONFIG.VIDEO_TRANSCRIPTION.ENGINE_PATH) {
|
|
|
|
logger.info(`Installing transcriber ${transcriber.engine.name} to generate subtitles`, lTags())
|
|
|
|
await transcriber.install(DIRECTORIES.LOCAL_PIP_DIRECTORY)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
const video = await VideoModel.loadFull(options.video.uuid)
|
|
|
|
const file = video.getMaxQualityFile().withVideoOrPlaylist(video)
|
|
|
|
|
|
|
|
await VideoPathManager.Instance.makeAvailableVideoFile(file, async videoInputPath => {
|
|
|
|
if (await hasAudioStream(videoInputPath) !== true) {
|
|
|
|
logger.info(
|
|
|
|
`Do not run transcription for ${video.uuid} in ${outputPath} because it does not contain an audio stream`,
|
|
|
|
lTags(video.uuid)
|
|
|
|
)
|
|
|
|
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
logger.info(`Running transcription for ${video.uuid} in ${outputPath}`, lTags(video.uuid))
|
|
|
|
|
|
|
|
const transcriptFile = await transcriber.transcribe({
|
|
|
|
mediaFilePath: videoInputPath,
|
|
|
|
|
|
|
|
model: CONFIG.VIDEO_TRANSCRIPTION.MODEL_PATH
|
|
|
|
? await TranscriptionModel.fromPath(CONFIG.VIDEO_TRANSCRIPTION.MODEL_PATH)
|
|
|
|
: new WhisperBuiltinModel(CONFIG.VIDEO_TRANSCRIPTION.MODEL),
|
|
|
|
|
|
|
|
transcriptDirectory: outputPath,
|
|
|
|
|
|
|
|
format: 'vtt'
|
|
|
|
})
|
|
|
|
|
|
|
|
await onTranscriptionEnded({ video, language: transcriptFile.language, vttPath: transcriptFile.path })
|
|
|
|
})
|
|
|
|
} finally {
|
|
|
|
if (outputPath) await remove(outputPath)
|
|
|
|
|
2024-07-01 07:38:19 -05:00
|
|
|
VideoJobInfoModel.decrease(options.video.uuid, 'pendingTranscription')
|
|
|
|
.catch(err => logger.error('Cannot decrease pendingTranscription job count', { err, ...lTags(options.video.uuid) }))
|
|
|
|
|
2024-06-13 02:23:12 -05:00
|
|
|
inputFileMutexReleaser()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
export async function onTranscriptionEnded (options: {
|
|
|
|
video: MVideoFullLight
|
|
|
|
language: string
|
|
|
|
vttPath: string
|
|
|
|
lTags?: (string | number)[]
|
|
|
|
}) {
|
|
|
|
const { video, language, vttPath, lTags: customLTags = [] } = options
|
|
|
|
|
|
|
|
if (!isVideoCaptionLanguageValid(language)) {
|
2024-07-03 01:32:14 -05:00
|
|
|
logger.warn(`Invalid transcription language for video ${video.uuid}`, lTags(video.uuid))
|
2024-06-13 02:23:12 -05:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!video.language) {
|
|
|
|
video.language = language
|
|
|
|
await video.save()
|
|
|
|
}
|
|
|
|
|
|
|
|
const caption = await createLocalCaption({
|
|
|
|
video,
|
|
|
|
language,
|
2024-06-27 08:29:26 -05:00
|
|
|
path: vttPath,
|
|
|
|
automaticallyGenerated: true
|
2024-06-13 02:23:12 -05:00
|
|
|
})
|
|
|
|
|
|
|
|
await sequelizeTypescript.transaction(async t => {
|
|
|
|
await federateVideoIfNeeded(video, false, t)
|
|
|
|
})
|
|
|
|
|
|
|
|
Notifier.Instance.notifyOfGeneratedVideoTranscription(caption)
|
|
|
|
|
|
|
|
logger.info(`Transcription ended for ${video.uuid}`, lTags(video.uuid, ...customLTags))
|
2024-02-14 02:21:53 -06:00
|
|
|
}
|