Add audio-only option to transcoders and player

This patch adds an audio-only option to PeerTube by means of a new transcoding configuration which creates mp4 files which only contain an audio stream. This new transcoder has a resolution of '0' and is presented in the preferences and in the player resolution menu as 'Audio-only' (localised). When playing such streams the player shows the file thumbnail as background and disables controls autohide.

Audio-only files can be shared and streamed just like any other file. They can be downloaded as well, the resulting file will be an mp4 container with a single audio stream.

This patch is a proof of concept to show the feasibility of 'true' audio-only support. There are better ways of doing this which also enable multiple audio streams for a given video stream (e.g. DASH) but as this would entail a fundamental change in the way PeerTube works it is a bridge too far for a simple proof of concept.

client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts

@@ -36,6 +36,10 @@ export class EditCustomConfigComponent extends FormReactive implements OnInit {
     super()
 
     this.resolutions = [
+       {
+        id: '0p',
+        label: this.i18n('Audio-only')
+      },
       {
         id: '240p',
         label: this.i18n('240p')

client/src/assets/player/videojs-components/resolution-menu-button.ts

@@ -79,7 +79,7 @@ class ResolutionMenuButton extends MenuButton {
         this.player_,
         {
           id: d.id,
-          label: d.label,
+          label: d.id == 0 ? this.player .localize('Audio-only') : d.label,
           selected: d.selected,
           callback: data.qualitySwitchCallback
         })

client/src/assets/player/webtorrent/webtorrent-plugin.ts

@@ -181,20 +181,29 @@ class WebTorrentPlugin extends Plugin {
     const currentTime = this.player.currentTime()
     const isPaused = this.player.paused()
 
-    // Remove poster to have black background
-    this.playerElement.poster = ''
-
     // Hide bigPlayButton
     if (!isPaused) {
       this.player.bigPlayButton.hide()
     }
 
+    // Audio-only (resolutionId == 0) gets special treatment
+    if (resolutionId > 0) {
+      // Hide poster to have black background 
+      this.player.removeClass('vjs-playing-audio-only-content')
+      this.player.posterImage.hide()
+    } else {
+      // Audio-only: show poster, do not auto-hide controls
+      this.player.addClass('vjs-playing-audio-only-content')
+      this.player.posterImage.show()
+    }
+
     const newVideoFile = this.videoFiles.find(f => f.resolution.id === resolutionId)
     const options = {
       forcePlay: false,
       delay,
       seek: currentTime + (delay / 1000)
     }
+
     this.updateVideoFile(newVideoFile, options)
   }
 
@@ -327,6 +336,7 @@ class WebTorrentPlugin extends Plugin {
                           this.player.posterImage.show()
                           this.player.removeClass('vjs-has-autoplay')
                           this.player.removeClass('vjs-has-big-play-button-clicked')
+                          this.player.removeClass('vjs-playing-audio-only-content')
 
                           return done()
                         })

config/default.yaml

@@ -203,6 +203,7 @@ transcoding:
   allow_audio_files: true
   threads: 1
   resolutions: # Only created if the original video has a higher resolution, uses more storage!
+    0p: false # audio-only (creates mp4 without video stream)
     240p: false
     360p: false
     480p: false

config/production.yaml.example

@@ -217,6 +217,7 @@ transcoding:
   allow_audio_files: true
   threads: 1
   resolutions: # Only created if the original video has a higher resolution, uses more storage!
+    0p: false # audio-only (creates mp4 without video stream)
     240p: false
     360p: false
     480p: false

config/test.yaml

@@ -72,6 +72,7 @@ transcoding:
   allow_audio_files: false
   threads: 2
   resolutions:
+    0p: true
     240p: true
     360p: true
     480p: true

server/controllers/api/config.ts

@@ -300,6 +300,7 @@ function customConfig (): CustomConfig {
       allowAudioFiles: CONFIG.TRANSCODING.ALLOW_AUDIO_FILES,
       threads: CONFIG.TRANSCODING.THREADS,
       resolutions: {
+        '0p': CONFIG.TRANSCODING.RESOLUTIONS[ '0p' ],
         '240p': CONFIG.TRANSCODING.RESOLUTIONS[ '240p' ],
         '360p': CONFIG.TRANSCODING.RESOLUTIONS[ '360p' ],
         '480p': CONFIG.TRANSCODING.RESOLUTIONS[ '480p' ],
@@ -356,6 +357,7 @@ function convertCustomConfigBody (body: CustomConfig) {
   function keyConverter (k: string) {
     // Transcoding resolutions exception
     if (/^\d{3,4}p$/.exec(k)) return k
+    if (/^0p$/.exec(k)) return k
 
     return snakeCase(k)
   }

server/helpers/ffmpeg-utils.ts

@@ -14,6 +14,7 @@ function computeResolutionsToTranscode (videoFileHeight: number) {
 
   // Put in the order we want to proceed jobs
   const resolutions = [
+    VideoResolution.H_NOVIDEO,
     VideoResolution.H_480P,
     VideoResolution.H_360P,
     VideoResolution.H_720P,
@@ -34,7 +35,12 @@ function computeResolutionsToTranscode (videoFileHeight: number) {
 async function getVideoFileSize (path: string) {
   const videoStream = await getVideoStreamFromFile(path)
 
-  return {
+  return videoStream == null 
+    ? {
+        width: 0,
+        height: 0
+      }
+    : {
         width: videoStream.width,
         height: videoStream.height
       }
@@ -52,6 +58,10 @@ async function getVideoFileResolution (path: string) {
 async function getVideoFileFPS (path: string) {
   const videoStream = await getVideoStreamFromFile(path)
 
+  if (videoStream == null) {
+      return 0
+  }
+
   for (const key of [ 'avg_frame_rate', 'r_frame_rate' ]) {
     const valuesText: string = videoStream[key]
     if (!valuesText) continue
@@ -118,7 +128,7 @@ async function generateImageFromVideoFile (fromPath: string, folder: string, ima
   }
 }
 
-type TranscodeOptionsType = 'hls' | 'quick-transcode' | 'video' | 'merge-audio'
+type TranscodeOptionsType = 'hls' | 'quick-transcode' | 'video' | 'merge-audio' | 'split-audio'
 
 interface BaseTranscodeOptions {
   type: TranscodeOptionsType
@@ -149,7 +159,11 @@ interface MergeAudioTranscodeOptions extends BaseTranscodeOptions {
   audioPath: string
 }
 
-type TranscodeOptions = HLSTranscodeOptions | VideoTranscodeOptions | MergeAudioTranscodeOptions | QuickTranscodeOptions
+interface SplitAudioTranscodeOptions extends BaseTranscodeOptions {
+  type: 'split-audio'
+}
+
+type TranscodeOptions = HLSTranscodeOptions | VideoTranscodeOptions | MergeAudioTranscodeOptions | SplitAudioTranscodeOptions | QuickTranscodeOptions
 
 function transcode (options: TranscodeOptions) {
   return new Promise<void>(async (res, rej) => {
@@ -163,6 +177,8 @@ function transcode (options: TranscodeOptions) {
         command = await buildHLSCommand(command, options)
       } else if (options.type === 'merge-audio') {
         command = await buildAudioMergeCommand(command, options)
+      } else if (options.type === 'split-audio') {
+        command = await buildAudioSplitCommand(command, options)
       } else {
         command = await buildx264Command(command, options)
       }
@@ -198,6 +214,7 @@ async function canDoQuickTranscode (path: string): Promise<boolean> {
   const resolution = await getVideoFileResolution(path)
 
   // check video params
+  if (videoStream == null) return false
   if (videoStream[ 'codec_name' ] !== 'h264') return false
   if (videoStream[ 'pix_fmt' ] !== 'yuv420p') return false
   if (fps < VIDEO_TRANSCODING_FPS.MIN || fps > VIDEO_TRANSCODING_FPS.MAX) return false
@@ -276,6 +293,12 @@ async function buildAudioMergeCommand (command: ffmpeg.FfmpegCommand, options: M
   return command
 }
 
+async function buildAudioSplitCommand (command: ffmpeg.FfmpegCommand, options: SplitAudioTranscodeOptions) {
+  command = await presetAudioSplit(command)
+
+  return command
+}
+
 async function buildQuickTranscodeCommand (command: ffmpeg.FfmpegCommand) {
   command = await presetCopy(command)
 
@@ -327,7 +350,7 @@ function getVideoStreamFromFile (path: string) {
       if (err) return rej(err)
 
       const videoStream = metadata.streams.find(s => s.codec_type === 'video')
-      if (!videoStream) return rej(new Error('Cannot find video stream of ' + path))
+      //if (!videoStream) return rej(new Error('Cannot find video stream of ' + path))
 
       return res(videoStream)
     })
@@ -482,3 +505,11 @@ async function presetCopy (command: ffmpeg.FfmpegCommand): Promise<ffmpeg.Ffmpeg
     .videoCodec('copy')
     .audioCodec('copy')
 }
+
+
+async function presetAudioSplit (command: ffmpeg.FfmpegCommand): Promise<ffmpeg.FfmpegCommand> {
+  return command
+    .format('mp4')
+    .audioCodec('copy')
+    .noVideo()
+}

server/initializers/config.ts

@@ -168,6 +168,7 @@ const CONFIG = {
     get ALLOW_AUDIO_FILES () { return config.get<boolean>('transcoding.allow_audio_files') },
     get THREADS () { return config.get<number>('transcoding.threads') },
     RESOLUTIONS: {
+      get '0p' () { return config.get<boolean>('transcoding.resolutions.0p') },
       get '240p' () { return config.get<boolean>('transcoding.resolutions.240p') },
       get '360p' () { return config.get<boolean>('transcoding.resolutions.360p') },
       get '480p' () { return config.get<boolean>('transcoding.resolutions.480p') },

server/lib/video-transcoding.ts

@@ -81,7 +81,14 @@ async function transcodeNewResolution (video: MVideoWithFile, resolution: VideoR
   const videoOutputPath = getVideoFilePath(video, newVideoFile)
   const videoTranscodedPath = join(transcodeDirectory, getVideoFilename(video, newVideoFile))
 
-  const transcodeOptions = {
+  const transcodeOptions = resolution === VideoResolution.H_NOVIDEO
+    ? {
+        type: 'split-audio' as 'split-audio',
+        inputPath: videoInputPath,
+        outputPath: videoTranscodedPath,
+        resolution,
+      }
+    : {
         type: 'video' as 'video',
         inputPath: videoInputPath,
         outputPath: videoTranscodedPath,
@@ -94,6 +101,39 @@ async function transcodeNewResolution (video: MVideoWithFile, resolution: VideoR
   return onVideoFileTranscoding(video, newVideoFile, videoTranscodedPath, videoOutputPath)
 }
 
+/**
+ * Extract audio into a separate audio-only mp4.
+ */
+async function splitAudioFile (video: MVideoWithFile) {
+  const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR
+  const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
+  const extname = '.mp4'
+  const resolution = VideoResolution.H_NOVIDEO
+
+  // We are sure it's x264 in mp4 because optimizeOriginalVideofile was already executed
+  const videoInputPath = join(videosDirectory, video.getVideoFilename(video.getOriginalFile()))
+
+  const newVideoFile = new VideoFileModel({
+    resolution,
+    extname,
+    size: 0,
+    videoId: video.id
+  })
+  const videoOutputPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(newVideoFile))
+  const videoTranscodedPath = join(transcodeDirectory, video.getVideoFilename(newVideoFile))
+
+  const transcodeOptions = {
+    type: 'split-audio' as 'split-audio',
+    inputPath: videoInputPath,
+    outputPath: videoTranscodedPath,
+    resolution
+  }
+
+  await transcode(transcodeOptions)
+
+  return onVideoFileTranscoding(video, newVideoFile, videoTranscodedPath, videoOutputPath)
+}
+
 async function mergeAudioVideofile (video: MVideoWithAllFiles, resolution: VideoResolution) {
   const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
   const newExtname = '.mp4'

server/middlewares/validators/config.ts

@@ -37,6 +37,7 @@ const customConfigUpdateValidator = [
   body('transcoding.enabled').isBoolean().withMessage('Should have a valid transcoding enabled boolean'),
   body('transcoding.allowAdditionalExtensions').isBoolean().withMessage('Should have a valid additional extensions boolean'),
   body('transcoding.threads').isInt().withMessage('Should have a valid transcoding threads number'),
+  body('transcoding.resolutions.0p').isBoolean().withMessage('Should have a valid transcoding 0p resolution enabled boolean'),
   body('transcoding.resolutions.240p').isBoolean().withMessage('Should have a valid transcoding 240p resolution enabled boolean'),
   body('transcoding.resolutions.360p').isBoolean().withMessage('Should have a valid transcoding 360p resolution enabled boolean'),
   body('transcoding.resolutions.480p').isBoolean().withMessage('Should have a valid transcoding 480p resolution enabled boolean'),

server/tests/api/check-params/config.ts

@@ -85,6 +85,7 @@ describe('Test config API validators', function () {
       allowAudioFiles: true,
       threads: 1,
       resolutions: {
+        '0p': false,
         '240p': false,
         '360p': true,
         '480p': true,

server/tests/api/server/config.ts

@@ -274,6 +274,7 @@ describe('Test config', function () {
         allowAudioFiles: true,
         threads: 1,
         resolutions: {
+          '0p': false,
           '240p': false,
           '360p': true,
           '480p': true,

shared/extra-utils/server/config.ts

@@ -111,6 +111,7 @@ function updateCustomSubConfig (url: string, token: string, newConfig: DeepParti
       allowAudioFiles: true,
       threads: 1,
       resolutions: {
+        '0p': false,
         '240p': false,
         '360p': true,
         '480p': true,

shared/models/server/custom-config.model.ts

@@ -75,6 +75,7 @@ export interface CustomConfig {
 
     threads: number
     resolutions: {
+      '0p': boolean
       '240p': boolean
       '360p': boolean
       '480p': boolean

shared/models/videos/video-resolution.enum.ts

@@ -1,6 +1,7 @@
 import { VideoTranscodingFPS } from './video-transcoding-fps.model'
 
 export enum VideoResolution {
+  H_NOVIDEO = 0,
   H_240P = 240,
   H_360P = 360,
   H_480P = 480,
@@ -18,6 +19,10 @@ export enum VideoResolution {
  */
 function getBaseBitrate (resolution: VideoResolution) {
   switch (resolution) {
+    case VideoResolution.H_NOVIDEO:
+      // audio-only
+      return 64 * 1000
+
     case VideoResolution.H_240P:
       // quality according to Google Live Encoder: 300 - 700 Kbps
       // Quality according to YouTube Video Info: 186 Kbps