Add storyboard support

client/src/app/+admin/config/edit-custom-config/edit-advanced-configuration.component.html

@@ -52,6 +52,20 @@
 
           <div *ngIf="formErrors.cache.torrents.size" class="form-error">{{ formErrors.cache.torrents.size }}</div>
         </div>
+
+        <div class="form-group" formGroupName="torrents">
+          <label i18n for="cacheTorrentsSize">Number of video storyboard images to keep in cache</label>
+
+          <div class="number-with-unit">
+            <input
+              type="number" min="0" id="cacheStoryboardsSize" class="form-control"
+              formControlName="size" [ngClass]="{ 'input-error': formErrors['cache.storyboards.size'] }"
+            >
+            <span i18n>{getCacheSize('storyboards'), plural, =1 {cached storyboard} other {cached storyboards}}</span>
+          </div>
+
+          <div *ngIf="formErrors.cache.storyboards.size" class="form-error">{{ formErrors.cache.storyboards.size }}</div>
+        </div>
       </ng-container>
 
     </div>

client/src/app/+admin/config/edit-custom-config/edit-advanced-configuration.component.ts

@@ -10,7 +10,7 @@ export class EditAdvancedConfigurationComponent {
   @Input() form: FormGroup
   @Input() formErrors: any
 
-  getCacheSize (type: 'captions' | 'previews' | 'torrents') {
+  getCacheSize (type: 'captions' | 'previews' | 'torrents' | 'storyboards') {
     return this.form.value['cache'][type]['size']
   }
 }

client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts

@@ -9,8 +9,7 @@ import { Notifier } from '@app/core'
 import { ServerService } from '@app/core/server/server.service'
 import {
   ADMIN_EMAIL_VALIDATOR,
-  CACHE_CAPTIONS_SIZE_VALIDATOR,
+  CACHE_SIZE_VALIDATOR,
-  CACHE_PREVIEWS_SIZE_VALIDATOR,
   CONCURRENCY_VALIDATOR,
   INDEX_URL_VALIDATOR,
   INSTANCE_NAME_VALIDATOR,
@@ -120,13 +119,16 @@ export class EditCustomConfigComponent extends FormReactive implements OnInit {
       },
       cache: {
         previews: {
-          size: CACHE_PREVIEWS_SIZE_VALIDATOR
+          size: CACHE_SIZE_VALIDATOR
         },
         captions: {
-          size: CACHE_CAPTIONS_SIZE_VALIDATOR
+          size: CACHE_SIZE_VALIDATOR
         },
         torrents: {
-          size: CACHE_CAPTIONS_SIZE_VALIDATOR
+          size: CACHE_SIZE_VALIDATOR
+        },
+        storyboards: {
+          size: CACHE_SIZE_VALIDATOR
         }
       },
       signup: {

client/src/app/+videos/+video-watch/video-watch.component.ts

@@ -33,6 +33,7 @@ import {
   LiveVideo,
   PeerTubeProblemDocument,
   ServerErrorCode,
+  Storyboard,
   VideoCaption,
   VideoPrivacy,
   VideoState
@@ -69,6 +70,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
   videoCaptions: VideoCaption[] = []
   liveVideo: LiveVideo
   videoPassword: string
+  storyboards: Storyboard[] = []
 
   playlistPosition: number
   playlist: VideoPlaylist = null
@@ -285,9 +287,10 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
     forkJoin([
       videoAndLiveObs,
       this.videoCaptionService.listCaptions(videoId, videoPassword),
+      this.videoService.getStoryboards(videoId),
       this.userService.getAnonymousOrLoggedUser()
     ]).subscribe({
-      next: ([ { video, live, videoFileToken }, captionsResult, loggedInOrAnonymousUser ]) => {
+      next: ([ { video, live, videoFileToken }, captionsResult, storyboards, loggedInOrAnonymousUser ]) => {
         const queryParams = this.route.snapshot.queryParams
 
         const urlOptions = {
@@ -309,6 +312,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
           video,
           live,
           videoCaptions: captionsResult.data,
+          storyboards,
           videoFileToken,
           videoPassword,
           loggedInOrAnonymousUser,
@@ -414,6 +418,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
     video: VideoDetails
     live: LiveVideo
     videoCaptions: VideoCaption[]
+    storyboards: Storyboard[]
     videoFileToken: string
     videoPassword: string
 
@@ -421,7 +426,17 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
     loggedInOrAnonymousUser: User
     forceAutoplay: boolean
   }) {
-    const { video, live, videoCaptions, urlOptions, videoFileToken, videoPassword, loggedInOrAnonymousUser, forceAutoplay } = options
+    const {
+      video,
+      live,
+      videoCaptions,
+      storyboards,
+      urlOptions,
+      videoFileToken,
+      videoPassword,
+      loggedInOrAnonymousUser,
+      forceAutoplay
+    } = options
 
     this.subscribeToLiveEventsIfNeeded(this.video, video)
 
@@ -430,6 +445,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
     this.liveVideo = live
     this.videoFileToken = videoFileToken
     this.videoPassword = videoPassword
+    this.storyboards = storyboards
 
     // Re init attributes
     this.playerPlaceholderImgSrc = undefined
@@ -485,6 +501,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
     const params = {
       video: this.video,
       videoCaptions: this.videoCaptions,
+      storyboards: this.storyboards,
       liveVideo: this.liveVideo,
       videoFileToken: this.videoFileToken,
       videoPassword: this.videoPassword,
@@ -636,6 +653,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
     video: VideoDetails
     liveVideo: LiveVideo
     videoCaptions: VideoCaption[]
+    storyboards: Storyboard[]
 
     videoFileToken: string
     videoPassword: string
@@ -646,7 +664,17 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
     forceAutoplay: boolean
     user?: AuthUser // Keep for plugins
   }) {
-    const { video, liveVideo, videoCaptions, videoFileToken, videoPassword, urlOptions, loggedInOrAnonymousUser, forceAutoplay } = params
+    const {
+      video,
+      liveVideo,
+      videoCaptions,
+      storyboards,
+      videoFileToken,
+      videoPassword,
+      urlOptions,
+      loggedInOrAnonymousUser,
+      forceAutoplay
+    } = params
 
     const getStartTime = () => {
       const byUrl = urlOptions.startTime !== undefined
@@ -673,6 +701,15 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
       src: environment.apiUrl + c.captionPath
     }))
 
+    const storyboard = storyboards.length !== 0
+      ? {
+        url: environment.apiUrl + storyboards[0].storyboardPath,
+        height: storyboards[0].spriteHeight,
+        width: storyboards[0].spriteWidth,
+        interval: storyboards[0].spriteDuration
+      }
+      : undefined
+
     const liveOptions = video.isLive
       ? { latencyMode: liveVideo.latencyMode }
       : undefined
@@ -734,6 +771,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
         videoPassword: () => videoPassword,
 
         videoCaptions: playerCaptions,
+        storyboard,
 
         videoShortUUID: video.shortUUID,
         videoUUID: video.uuid,
@@ -767,6 +805,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
       else mode = 'webtorrent'
     }
 
+    // FIXME: remove, we don't support these old web browsers anymore
     // p2p-media-loader needs TextEncoder, fallback on WebTorrent if not available
     if (typeof TextEncoder === 'undefined') {
       mode = 'webtorrent'

client/src/app/shared/form-validators/custom-config-validators.ts

@@ -22,21 +22,12 @@ export const SERVICES_TWITTER_USERNAME_VALIDATOR: BuildFormValidator = {
   }
 }
 
-export const CACHE_PREVIEWS_SIZE_VALIDATOR: BuildFormValidator = {
+export const CACHE_SIZE_VALIDATOR: BuildFormValidator = {
   VALIDATORS: [ Validators.required, Validators.min(1), Validators.pattern('[0-9]+') ],
   MESSAGES: {
-    required: $localize`Previews cache size is required.`,
+    required: $localize`Cache size is required.`,
-    min: $localize`Previews cache size must be greater than 1.`,
+    min: $localize`Cache size must be greater than 1.`,
-    pattern: $localize`Previews cache size must be a number.`
+    pattern: $localize`Cache size must be a number.`
-  }
-}
-
-export const CACHE_CAPTIONS_SIZE_VALIDATOR: BuildFormValidator = {
-  VALIDATORS: [ Validators.required, Validators.min(1), Validators.pattern('[0-9]+') ],
-  MESSAGES: {
-    required: $localize`Captions cache size is required.`,
-    min: $localize`Captions cache size must be greater than 1.`,
-    pattern: $localize`Captions cache size must be a number.`
   }
 }
 

client/src/app/shared/shared-main/video/video.service.ts

@@ -11,6 +11,7 @@ import {
   FeedFormat,
   NSFWPolicyType,
   ResultList,
+  Storyboard,
   UserVideoRate,
   UserVideoRateType,
   UserVideoRateUpdate,
@@ -344,6 +345,25 @@ export class VideoService {
                )
   }
 
+  // ---------------------------------------------------------------------------
+
+  getStoryboards (videoId: string | number) {
+    return this.authHttp
+      .get<{ storyboards: Storyboard[] }>(VideoService.BASE_VIDEO_URL + '/' + videoId + '/storyboards')
+      .pipe(
+        map(({ storyboards }) => storyboards),
+        catchError(err => {
+          if (err.status === 404) {
+            return of([])
+          }
+
+          this.restExtractor.handleError(err)
+        })
+      )
+  }
+
+  // ---------------------------------------------------------------------------
+
   getSource (videoId: number) {
     return this.authHttp
                .get<{ source: VideoSource }>(VideoService.BASE_VIDEO_URL + '/' + videoId + '/source')
@@ -358,6 +378,8 @@ export class VideoService {
                )
   }
 
+  // ---------------------------------------------------------------------------
+
   setVideoLike (id: string, videoPassword: string) {
     return this.setVideoRate(id, 'like', videoPassword)
   }
@@ -370,6 +392,8 @@ export class VideoService {
     return this.setVideoRate(id, 'none', videoPassword)
   }
 
+  // ---------------------------------------------------------------------------
+
   getUserVideoRating (id: string) {
     const url = UserService.BASE_USERS_URL + 'me/videos/' + id + '/rating'
 

client/src/assets/player/peertube-player-manager.ts

@@ -6,6 +6,7 @@ import './shared/stats/stats-plugin'
 import './shared/bezels/bezels-plugin'
 import './shared/peertube/peertube-plugin'
 import './shared/resolutions/peertube-resolutions-plugin'
+import './shared/control-bar/storyboard-plugin'
 import './shared/control-bar/next-previous-video-button'
 import './shared/control-bar/p2p-info-button'
 import './shared/control-bar/peertube-link-button'
@@ -42,6 +43,12 @@ CaptionsButton.prototype.controlText_ = 'Subtitles/CC'
 // We just want to display 'Off' instead of 'captions off', keep a space so the variable == true (hacky I know)
 CaptionsButton.prototype.label_ = ' '
 
+// TODO: remove when https://github.com/videojs/video.js/pull/7598 is merged
+const PlayProgressBar = videojs.getComponent('PlayProgressBar') as any
+if (PlayProgressBar.prototype.options_.children.includes('timeTooltip') !== true) {
+  PlayProgressBar.prototype.options_.children.push('timeTooltip')
+}
+
 export class PeertubePlayerManager {
   private static playerElementClassName: string
   private static playerElementAttributes: { name: string, value: string }[] = []
@@ -135,6 +142,10 @@ export class PeertubePlayerManager {
           p2pEnabled: options.common.p2pEnabled
         })
 
+        if (options.common.storyboard) {
+          player.storyboard(options.common.storyboard)
+        }
+
         player.on('p2pInfo', (_, data: PlayerNetworkInfo) => {
           if (data.source !== 'p2p-media-loader' || isNaN(data.bandwidthEstimate)) return
 

client/src/assets/player/shared/control-bar/index.ts

@@ -3,4 +3,5 @@ export * from './p2p-info-button'
 export * from './peertube-link-button'
 export * from './peertube-live-display'
 export * from './peertube-load-progress-bar'
+export * from './storyboard-plugin'
 export * from './theater-button'

client/src/assets/player/shared/control-bar/storyboard-plugin.ts

@@ -0,0 +1,184 @@
+import videojs from 'video.js'
+import { StoryboardOptions } from '../../types'
+
+// Big thanks to this beautiful plugin: https://github.com/phloxic/videojs-sprite-thumbnails
+// Adapted to respect peertube player style
+
+const Plugin = videojs.getPlugin('plugin')
+
+class StoryboardPlugin extends Plugin {
+  private url: string
+  private height: number
+  private width: number
+  private interval: number
+
+  private cached: boolean
+
+  private mouseTimeTooltip: videojs.MouseTimeDisplay
+  private seekBar: { el(): HTMLElement, mouseTimeDisplay: any, playProgressBar: any }
+  private progress: any
+
+  private spritePlaceholder: HTMLElement
+
+  private readonly sprites: { [id: string]: HTMLImageElement } = {}
+
+  private readonly boundedHijackMouseTooltip: typeof StoryboardPlugin.prototype.hijackMouseTooltip
+
+  constructor (player: videojs.Player, options: videojs.ComponentOptions & StoryboardOptions) {
+    super(player, options)
+
+    this.url = options.url
+    this.height = options.height
+    this.width = options.width
+    this.interval = options.interval
+
+    this.boundedHijackMouseTooltip = this.hijackMouseTooltip.bind(this)
+
+    this.player.ready(() => {
+      player.addClass('vjs-storyboard')
+
+      this.init()
+    })
+  }
+
+  init () {
+    const controls = this.player.controlBar as any
+
+    // default control bar component tree is expected
+    // https://docs.videojs.com/tutorial-components.html#default-component-tree
+    this.progress = controls?.progressControl
+    this.seekBar = this.progress?.seekBar
+
+    this.mouseTimeTooltip = this.seekBar?.mouseTimeDisplay?.timeTooltip
+
+    this.spritePlaceholder = videojs.dom.createEl('div', { className: 'vjs-storyboard-sprite-placeholder' }) as HTMLElement
+    this.seekBar?.el()?.appendChild(this.spritePlaceholder)
+
+    this.player.on([ 'ready', 'loadstart' ], evt => {
+      if (evt !== 'ready') {
+        const spriteSource = this.player.currentSources().find(source => {
+          return Object.prototype.hasOwnProperty.call(source, 'storyboard')
+        }) as any
+        const spriteOpts = spriteSource?.['storyboard'] as StoryboardOptions
+
+        if (spriteOpts) {
+          this.url = spriteOpts.url
+          this.height = spriteOpts.height
+          this.width = spriteOpts.width
+          this.interval = spriteOpts.interval
+        }
+      }
+
+      this.cached = !!this.sprites[this.url]
+
+      this.load()
+    })
+  }
+
+  private load () {
+    const spriteEvents = [ 'mousemove', 'touchmove' ]
+
+    if (this.isReady()) {
+      if (!this.cached) {
+        this.sprites[this.url] = videojs.dom.createEl('img', {
+          src: this.url
+        })
+      }
+      this.progress.on(spriteEvents, this.boundedHijackMouseTooltip)
+    } else {
+      this.progress.off(spriteEvents, this.boundedHijackMouseTooltip)
+
+      this.resetMouseTooltip()
+    }
+  }
+
+  private hijackMouseTooltip (evt: Event) {
+    const sprite = this.sprites[this.url]
+    const imgWidth = sprite.naturalWidth
+    const imgHeight = sprite.naturalHeight
+    const seekBarEl = this.seekBar.el()
+
+    if (!sprite.complete || !imgWidth || !imgHeight) {
+      this.resetMouseTooltip()
+      return
+    }
+
+    this.player.requestNamedAnimationFrame('StoryBoardPlugin#hijackMouseTooltip', () => {
+      const seekBarRect = videojs.dom.getBoundingClientRect(seekBarEl)
+      const playerRect = videojs.dom.getBoundingClientRect(this.player.el())
+
+      if (!seekBarRect || !playerRect) return
+
+      const seekBarX = videojs.dom.getPointerPosition(seekBarEl, evt).x
+      let position = seekBarX * this.player.duration()
+
+      const maxPosition = Math.round((imgHeight / this.height) * (imgWidth / this.width)) - 1
+      position = Math.min(position / this.interval, maxPosition)
+
+      const responsive = 600
+      const playerWidth = this.player.currentWidth()
+      const scaleFactor = responsive && playerWidth < responsive
+        ? playerWidth / responsive
+        : 1
+      const columns = imgWidth / this.width
+
+      const scaledWidth = this.width * scaleFactor
+      const scaledHeight = this.height * scaleFactor
+      const cleft = Math.floor(position % columns) * -scaledWidth
+      const ctop = Math.floor(position / columns) * -scaledHeight
+
+      const bgSize = `${imgWidth * scaleFactor}px ${imgHeight * scaleFactor}px`
+      const topOffset = -scaledHeight - 60
+
+      const previewHalfSize = Math.round(scaledWidth / 2)
+      let left = seekBarRect.width * seekBarX - previewHalfSize
+
+      // Seek bar doesn't take all the player width, so we can add/minus a few more pixels
+      const minLeft = playerRect.left - seekBarRect.left
+      const maxLeft = seekBarRect.width - scaledWidth + (playerRect.right - seekBarRect.right)
+
+      if (left < minLeft) left = minLeft
+      if (left > maxLeft) left = maxLeft
+
+      const tooltipStyle: { [id: string]: string } = {
+        'background-image': `url("${this.url}")`,
+        'background-repeat': 'no-repeat',
+        'background-position': `${cleft}px ${ctop}px`,
+        'background-size': bgSize,
+
+        'color': '#fff',
+        'text-shadow': '1px 1px #000',
+
+        'position': 'relative',
+
+        'top': `${topOffset}px`,
+
+        'border': '1px solid #000',
+
+        // border should not overlay thumbnail area
+        'width': `${scaledWidth + 2}px`,
+        'height': `${scaledHeight + 2}px`
+      }
+
+      tooltipStyle.left = `${left}px`
+
+      for (const [ key, value ] of Object.entries(tooltipStyle)) {
+        this.spritePlaceholder.style.setProperty(key, value)
+      }
+    })
+  }
+
+  private resetMouseTooltip () {
+    if (this.spritePlaceholder) {
+      this.spritePlaceholder.style.cssText = ''
+    }
+  }
+
+  private isReady () {
+    return this.mouseTimeTooltip && this.width && this.height && this.url
+  }
+}
+
+videojs.registerPlugin('storyboard', StoryboardPlugin)
+
+export { StoryboardPlugin }

client/src/assets/player/types/manager-options.ts

@@ -1,6 +1,6 @@
 import { PluginsManager } from '@root-helpers/plugins-manager'
 import { LiveVideoLatencyMode, VideoFile } from '@shared/models'
-import { PlaylistPluginOptions, VideoJSCaption } from './peertube-videojs-typings'
+import { PlaylistPluginOptions, VideoJSCaption, VideoJSStoryboard } from './peertube-videojs-typings'
 
 export type PlayerMode = 'webtorrent' | 'p2p-media-loader'
 
@@ -78,6 +78,7 @@ export interface CommonOptions extends CustomizationOptions {
   language?: string
 
   videoCaptions: VideoJSCaption[]
+  storyboard: VideoJSStoryboard
 
   videoUUID: string
   videoShortUUID: string

client/src/assets/player/types/peertube-videojs-typings.ts

@@ -49,6 +49,8 @@ declare module 'video.js' {
 
     stats (options?: StatsCardOptions): StatsForNerdsPlugin
 
+    storyboard (options: StoryboardOptions): void
+
     textTracks (): TextTrackList & {
       tracks_: (TextTrack & { id: string, label: string, src: string })[]
     }
@@ -89,6 +91,13 @@ type VideoJSCaption = {
   src: string
 }
 
+type VideoJSStoryboard = {
+  url: string
+  width: number
+  height: number
+  interval: number
+}
+
 type PeerTubePluginOptions = {
   mode: PlayerMode
 
@@ -118,6 +127,13 @@ type MetricsPluginOptions = {
   videoUUID: string
 }
 
+type StoryboardOptions = {
+  url: string
+  width: number
+  height: number
+  interval: number
+}
+
 type PlaylistPluginOptions = {
   elements: VideoPlaylistElement[]
 
@@ -238,6 +254,7 @@ type PlaylistItemOptions = {
 
 export {
   PlayerNetworkInfo,
+  VideoJSStoryboard,
   PlaylistItemOptions,
   NextPreviousVideoButtonOptions,
   ResolutionUpdateData,
@@ -251,6 +268,7 @@ export {
   PeerTubeResolution,
   VideoJSPluginOptions,
   LoadedQualityData,
+  StoryboardOptions,
   PeerTubeLinkButtonOptions,
   PeerTubeP2PInfoButtonOptions
 }

client/src/sass/player/control-bar.scss

@@ -3,6 +3,20 @@
 @use '_mixins' as *;
 @use './_player-variables' as *;
 
+// Like the time tooltip
+.video-js .vjs-progress-holder .vjs-storyboard-sprite-placeholder {
+  display: none;
+}
+
+.video-js .vjs-progress-control:hover .vjs-storyboard-sprite-placeholder,
+.video-js .vjs-progress-control:hover .vjs-progress-holder:focus .vjs-storyboard-sprite-placeholder {
+  display: block;
+
+  // Ensure that we maintain a font-size of ~10px.
+  font-size: 0.6em;
+  visibility: visible;
+}
+
 .video-js.vjs-peertube-skin .vjs-control-bar {
   z-index: 100;
 
@@ -79,6 +93,7 @@
           top: -0.3em;
         }
 
+        // Only used on mobile
         .vjs-time-tooltip {
           display: none;
         }

client/src/sass/player/mobile.scss

@@ -6,6 +6,31 @@
 /* Special mobile style */
 
 .video-js.vjs-peertube-skin.vjs-is-mobile {
+  // No hover means we can't display the storyboard/time tooltip on mouse hover
+  // Use the time tooltip in progress control instead
+  .vjs-mouse-display {
+    display: none !important;
+  }
+
+  .vjs-storyboard-sprite-placeholder {
+    display: none;
+  }
+
+  .vjs-progress-control .vjs-sliding {
+
+    .vjs-time-tooltip,
+    .vjs-storyboard-sprite-placeholder {
+      display: block !important;
+
+      visibility: visible !important;
+    }
+
+    .vjs-time-tooltip {
+      color: #fff;
+      background-color: rgba(0, 0, 0, 0.8);
+    }
+  }
+
   .vjs-control-bar {
     .vjs-progress-control .vjs-slider .vjs-play-progress {
       // Always display the circle on mobile

config/default.yaml

@@ -136,6 +136,7 @@ storage:
   logs: 'storage/logs/'
   previews: 'storage/previews/'
   thumbnails: 'storage/thumbnails/'
+  storyboards: 'storage/storyboards/'
   torrents: 'storage/torrents/'
   captions: 'storage/captions/'
   cache: 'storage/cache/'
@@ -396,6 +397,8 @@ cache:
     size: 500 # Max number of video captions/subtitles you want to cache
   torrents:
     size: 500 # Max number of video torrents you want to cache
+  storyboards:
+    size: 500 # Max number of video storyboards you want to cache
 
 admin:
   # Used to generate the root user at first startup

config/production.yaml.example

@@ -134,6 +134,7 @@ storage:
   logs: '/var/www/peertube/storage/logs/'
   previews: '/var/www/peertube/storage/previews/'
   thumbnails: '/var/www/peertube/storage/thumbnails/'
+  storyboards: '/var/www/peertube/storage/storyboards/'
   torrents: '/var/www/peertube/storage/torrents/'
   captions: '/var/www/peertube/storage/captions/'
   cache: '/var/www/peertube/storage/cache/'
@@ -406,6 +407,8 @@ cache:
     size: 500 # Max number of video captions/subtitles you want to cache
   torrents:
     size: 500 # Max number of video torrents you want to cache
+  storyboards:
+    size: 500 # Max number of video storyboards you want to cache
 
 admin:
   # Used to generate the root user at first startup

config/test-1.yaml

@@ -19,6 +19,7 @@ storage:
   logs: 'test1/logs/'
   previews: 'test1/previews/'
   thumbnails: 'test1/thumbnails/'
+  storyboards: 'test1/storyboards/'
   torrents: 'test1/torrents/'
   captions: 'test1/captions/'
   cache: 'test1/cache/'

config/test-2.yaml

@@ -19,6 +19,7 @@ storage:
   logs: 'test2/logs/'
   previews: 'test2/previews/'
   thumbnails: 'test2/thumbnails/'
+  storyboards: 'test2/storyboards/'
   torrents: 'test2/torrents/'
   captions: 'test2/captions/'
   cache: 'test2/cache/'

config/test-3.yaml

@@ -19,6 +19,7 @@ storage:
   logs: 'test3/logs/'
   previews: 'test3/previews/'
   thumbnails: 'test3/thumbnails/'
+  storyboards: 'test3/storyboards/'
   torrents: 'test3/torrents/'
   captions: 'test3/captions/'
   cache: 'test3/cache/'

config/test-4.yaml

@@ -19,6 +19,7 @@ storage:
   logs: 'test4/logs/'
   previews: 'test4/previews/'
   thumbnails: 'test4/thumbnails/'
+  storyboards: 'test4/storyboards/'
   torrents: 'test4/torrents/'
   captions: 'test4/captions/'
   cache: 'test4/cache/'

config/test-5.yaml

@@ -19,6 +19,7 @@ storage:
   logs: 'test5/logs/'
   previews: 'test5/previews/'
   thumbnails: 'test5/thumbnails/'
+  storyboards: 'test5/storyboards/'
   torrents: 'test5/torrents/'
   captions: 'test5/captions/'
   cache: 'test5/cache/'

config/test-6.yaml

@@ -19,6 +19,7 @@ storage:
   logs: 'test6/logs/'
   previews: 'test6/previews/'
   thumbnails: 'test6/thumbnails/'
+  storyboards: 'test6/storyboards/'
   torrents: 'test6/torrents/'
   captions: 'test6/captions/'
   cache: 'test6/cache/'

config/test.yaml

@@ -73,6 +73,8 @@ cache:
     size: 1
   torrents:
     size: 1
+  storyboards:
+    size: 1
 
 signup:
   enabled: true

server.ts

@@ -101,7 +101,7 @@ loadLanguages()
 import { installApplication } from './server/initializers/installer'
 import { Emailer } from './server/lib/emailer'
 import { JobQueue } from './server/lib/job-queue'
-import { VideosPreviewCache, VideosCaptionCache } from './server/lib/files-cache'
+import { VideosPreviewCache, VideosCaptionCache, VideosStoryboardCache } from './server/lib/files-cache'
 import {
   activityPubRouter,
   apiRouter,
@@ -316,6 +316,7 @@ async function startApplication () {
   VideosPreviewCache.Instance.init(CONFIG.CACHE.PREVIEWS.SIZE, FILES_CACHE.PREVIEWS.MAX_AGE)
   VideosCaptionCache.Instance.init(CONFIG.CACHE.VIDEO_CAPTIONS.SIZE, FILES_CACHE.VIDEO_CAPTIONS.MAX_AGE)
   VideosTorrentCache.Instance.init(CONFIG.CACHE.TORRENTS.SIZE, FILES_CACHE.TORRENTS.MAX_AGE)
+  VideosStoryboardCache.Instance.init(CONFIG.CACHE.STORYBOARDS.SIZE, FILES_CACHE.STORYBOARDS.MAX_AGE)
 
   // Enable Schedulers
   ActorFollowScheduler.Instance.enable()

server/controllers/activitypub/client.ts

@@ -33,7 +33,6 @@ import { videoPlaylistElementAPGetValidator, videoPlaylistsGetValidator } from '
 import { AccountModel } from '../../models/account/account'
 import { AccountVideoRateModel } from '../../models/account/account-video-rate'
 import { ActorFollowModel } from '../../models/actor/actor-follow'
-import { VideoCaptionModel } from '../../models/video/video-caption'
 import { VideoCommentModel } from '../../models/video/video-comment'
 import { VideoPlaylistModel } from '../../models/video/video-playlist'
 import { VideoShareModel } from '../../models/video/video-share'
@@ -242,14 +241,13 @@ async function videoController (req: express.Request, res: express.Response) {
   if (redirectIfNotOwned(video.url, res)) return
 
   // We need captions to render AP object
-  const captions = await VideoCaptionModel.listVideoCaptions(video.id)
+  const videoAP = await video.lightAPToFullAP(undefined)
-  const videoWithCaptions = Object.assign(video, { VideoCaptions: captions })
 
-  const audience = getAudience(videoWithCaptions.VideoChannel.Account.Actor, videoWithCaptions.privacy === VideoPrivacy.PUBLIC)
+  const audience = getAudience(videoAP.VideoChannel.Account.Actor, videoAP.privacy === VideoPrivacy.PUBLIC)
-  const videoObject = audiencify(await videoWithCaptions.toActivityPubObject(), audience)
+  const videoObject = audiencify(await videoAP.toActivityPubObject(), audience)
 
   if (req.path.endsWith('/activity')) {
-    const data = buildCreateActivity(videoWithCaptions.url, video.VideoChannel.Account.Actor, videoObject, audience)
+    const data = buildCreateActivity(videoAP.url, video.VideoChannel.Account.Actor, videoObject, audience)
     return activityPubResponse(activityPubContextify(data, 'Video'), res)
   }
 

server/controllers/api/config.ts

@@ -190,6 +190,9 @@ function customConfig (): CustomConfig {
       },
       torrents: {
         size: CONFIG.CACHE.TORRENTS.SIZE
+      },
+      storyboards: {
+        size: CONFIG.CACHE.STORYBOARDS.SIZE
       }
     },
     signup: {

server/controllers/api/videos/index.ts

@@ -41,6 +41,7 @@ import { liveRouter } from './live'
 import { ownershipVideoRouter } from './ownership'
 import { rateVideoRouter } from './rate'
 import { statsRouter } from './stats'
+import { storyboardRouter } from './storyboard'
 import { studioRouter } from './studio'
 import { tokenRouter } from './token'
 import { transcodingRouter } from './transcoding'
@@ -70,6 +71,7 @@ videosRouter.use('/', filesRouter)
 videosRouter.use('/', transcodingRouter)
 videosRouter.use('/', tokenRouter)
 videosRouter.use('/', videoPasswordRouter)
+videosRouter.use('/', storyboardRouter)
 
 videosRouter.get('/categories',
   openapiOperationDoc({ operationId: 'getCategories' }),

server/controllers/api/videos/storyboard.ts

@@ -0,0 +1,29 @@
+import express from 'express'
+import { getVideoWithAttributes } from '@server/helpers/video'
+import { StoryboardModel } from '@server/models/video/storyboard'
+import { asyncMiddleware, videosGetValidator } from '../../../middlewares'
+
+const storyboardRouter = express.Router()
+
+storyboardRouter.get('/:id/storyboards',
+  asyncMiddleware(videosGetValidator),
+  asyncMiddleware(listStoryboards)
+)
+
+// ---------------------------------------------------------------------------
+
+export {
+  storyboardRouter
+}
+
+// ---------------------------------------------------------------------------
+
+async function listStoryboards (req: express.Request, res: express.Response) {
+  const video = getVideoWithAttributes(res)
+
+  const storyboards = await StoryboardModel.listStoryboardsOf(video)
+
+  return res.json({
+    storyboards: storyboards.map(s => s.toFormattedJSON())
+  })
+}

server/controllers/api/videos/upload.ts

@@ -234,6 +234,15 @@ async function addVideoJobsAfterUpload (video: MVideoFullLight, videoFile: MVide
       }
     },
 
+    {
+      type: 'generate-video-storyboard' as 'generate-video-storyboard',
+      payload: {
+        videoUUID: video.uuid,
+        // No need to federate, we process these jobs sequentially
+        federate: false
+      }
+    },
+
     {
       type: 'notify',
       payload: {

server/controllers/lazy-static.ts

@@ -5,7 +5,7 @@ import { MActorImage } from '@server/types/models'
 import { HttpStatusCode } from '../../shared/models/http/http-error-codes'
 import { logger } from '../helpers/logger'
 import { ACTOR_IMAGES_SIZE, LAZY_STATIC_PATHS, STATIC_MAX_AGE } from '../initializers/constants'
-import { VideosCaptionCache, VideosPreviewCache } from '../lib/files-cache'
+import { VideosCaptionCache, VideosPreviewCache, VideosStoryboardCache } from '../lib/files-cache'
 import { actorImagePathUnsafeCache, downloadActorImageFromWorker } from '../lib/local-actor'
 import { asyncMiddleware, handleStaticError } from '../middlewares'
 import { ActorImageModel } from '../models/actor/actor-image'
@@ -32,6 +32,12 @@ lazyStaticRouter.use(
   handleStaticError
 )
 
+lazyStaticRouter.use(
+  LAZY_STATIC_PATHS.STORYBOARDS + ':filename',
+  asyncMiddleware(getStoryboard),
+  handleStaticError
+)
+
 lazyStaticRouter.use(
   LAZY_STATIC_PATHS.VIDEO_CAPTIONS + ':filename',
   asyncMiddleware(getVideoCaption),
@@ -126,6 +132,13 @@ async function getPreview (req: express.Request, res: express.Response) {
   return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE.LAZY_SERVER })
 }
 
+async function getStoryboard (req: express.Request, res: express.Response) {
+  const result = await VideosStoryboardCache.Instance.getFilePath(req.params.filename)
+  if (!result) return res.status(HttpStatusCode.NOT_FOUND_404).end()
+
+  return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE.LAZY_SERVER })
+}
+
 async function getVideoCaption (req: express.Request, res: express.Response) {
   const result = await VideosCaptionCache.Instance.getFilePath(req.params.filename)
   if (!result) return res.status(HttpStatusCode.NOT_FOUND_404).end()

server/helpers/custom-validators/activitypub/videos.ts

@@ -1,6 +1,6 @@
 import validator from 'validator'
 import { logger } from '@server/helpers/logger'
-import { ActivityTrackerUrlObject, ActivityVideoFileMetadataUrlObject } from '@shared/models'
+import { ActivityPubStoryboard, ActivityTrackerUrlObject, ActivityVideoFileMetadataUrlObject, VideoObject } from '@shared/models'
 import { LiveVideoLatencyMode, VideoState } from '../../../../shared/models/videos'
 import { ACTIVITY_PUB, CONSTRAINTS_FIELDS } from '../../../initializers/constants'
 import { peertubeTruncate } from '../../core-utils'
@@ -48,6 +48,10 @@ function sanitizeAndCheckVideoTorrentObject (video: any) {
     logger.debug('Video has invalid icons', { video })
     return false
   }
+  if (!setValidStoryboard(video)) {
+    logger.debug('Video has invalid preview (storyboard)', { video })
+    return false
+  }
 
   // Default attributes
   if (!isVideoStateValid(video.state)) video.state = VideoState.PUBLISHED
@@ -201,3 +205,36 @@ function setRemoteVideoContent (video: any) {
 
   return true
 }
+
+function setValidStoryboard (video: VideoObject) {
+  if (!video.preview) return true
+  if (!Array.isArray(video.preview)) return false
+
+  video.preview = video.preview.filter(p => isStorybordValid(p))
+
+  return true
+}
+
+function isStorybordValid (preview: ActivityPubStoryboard) {
+  if (!preview) return false
+
+  if (
+    preview.type !== 'Image' ||
+    !isArray(preview.rel) ||
+    !preview.rel.includes('storyboard')
+  ) {
+    return false
+  }
+
+  preview.url = preview.url.filter(u => {
+    return u.mediaType === 'image/jpeg' &&
+      isActivityPubUrlValid(u.href) &&
+      validator.isInt(u.width + '', { min: 0 }) &&
+      validator.isInt(u.height + '', { min: 0 }) &&
+      validator.isInt(u.tileWidth + '', { min: 0 }) &&
+      validator.isInt(u.tileHeight + '', { min: 0 }) &&
+      isActivityPubVideoDurationValid(u.tileDuration)
+  })
+
+  return preview.url.length !== 0
+}

server/initializers/checker-before-init.ts

@@ -29,7 +29,8 @@ function checkMissedConfig () {
     'video_channels.max_per_user',
     'csp.enabled', 'csp.report_only', 'csp.report_uri',
     'security.frameguard.enabled', 'security.powered_by_header.enabled',
-    'cache.previews.size', 'cache.captions.size', 'cache.torrents.size', 'admin.email', 'contact_form.enabled',
+    'cache.previews.size', 'cache.captions.size', 'cache.torrents.size', 'cache.storyboards.size',
+    'admin.email', 'contact_form.enabled',
     'signup.enabled', 'signup.limit', 'signup.requires_approval', 'signup.requires_email_verification', 'signup.minimum_age',
     'signup.filters.cidr.whitelist', 'signup.filters.cidr.blacklist',
     'redundancy.videos.strategies', 'redundancy.videos.check_interval',

server/initializers/config.ts

@@ -112,6 +112,7 @@ const CONFIG = {
     STREAMING_PLAYLISTS_DIR: buildPath(config.get<string>('storage.streaming_playlists')),
     REDUNDANCY_DIR: buildPath(config.get<string>('storage.redundancy')),
     THUMBNAILS_DIR: buildPath(config.get<string>('storage.thumbnails')),
+    STORYBOARDS_DIR: buildPath(config.get<string>('storage.storyboards')),
     PREVIEWS_DIR: buildPath(config.get<string>('storage.previews')),
     CAPTIONS_DIR: buildPath(config.get<string>('storage.captions')),
     TORRENTS_DIR: buildPath(config.get<string>('storage.torrents')),
@@ -482,6 +483,9 @@ const CONFIG = {
     },
     TORRENTS: {
       get SIZE () { return config.get<number>('cache.torrents.size') }
+    },
+    STORYBOARDS: {
+      get SIZE () { return config.get<number>('cache.storyboards.size') }
     }
   },
   INSTANCE: {

server/initializers/constants.ts

@@ -174,6 +174,7 @@ const JOB_ATTEMPTS: { [id in JobType]: number } = {
   'after-video-channel-import': 1,
   'move-to-object-storage': 3,
   'transcoding-job-builder': 1,
+  'generate-video-storyboard': 1,
   'notify': 1,
   'federate-video': 1
 }
@@ -198,6 +199,7 @@ const JOB_CONCURRENCY: { [id in Exclude<JobType, 'video-transcoding' | 'video-im
   'video-channel-import': 1,
   'after-video-channel-import': 1,
   'transcoding-job-builder': 1,
+  'generate-video-storyboard': 1,
   'notify': 5,
   'federate-video': 3
 }
@@ -218,6 +220,7 @@ const JOB_TTL: { [id in JobType]: number } = {
   'activitypub-refresher': 60000 * 10, // 10 minutes
   'video-redundancy': 1000 * 3600 * 3, // 3 hours
   'video-live-ending': 1000 * 60 * 10, // 10 minutes
+  'generate-video-storyboard': 1000 * 60 * 10, // 10 minutes
   'manage-video-torrent': 1000 * 3600 * 3, // 3 hours
   'move-to-object-storage': 1000 * 60 * 60 * 3, // 3 hours
   'video-channel-import': 1000 * 60 * 60 * 4, // 4 hours
@@ -766,7 +769,8 @@ const LAZY_STATIC_PATHS = {
   AVATARS: '/lazy-static/avatars/',
   PREVIEWS: '/lazy-static/previews/',
   VIDEO_CAPTIONS: '/lazy-static/video-captions/',
-  TORRENTS: '/lazy-static/torrents/'
+  TORRENTS: '/lazy-static/torrents/',
+  STORYBOARDS: '/lazy-static/storyboards/'
 }
 const OBJECT_STORAGE_PROXY_PATHS = {
   PRIVATE_WEBSEED: '/object-storage-proxy/webseed/private/',
@@ -813,6 +817,14 @@ const ACTOR_IMAGES_SIZE: { [key in ActorImageType]: { width: number, height: num
   ]
 }
 
+const STORYBOARD = {
+  SPRITE_SIZE: {
+    width: 192,
+    height: 108
+  },
+  SPRITES_MAX_EDGE_COUNT: 10
+}
+
 const EMBED_SIZE = {
   width: 560,
   height: 315
@@ -824,6 +836,10 @@ const FILES_CACHE = {
     DIRECTORY: join(CONFIG.STORAGE.CACHE_DIR, 'previews'),
     MAX_AGE: 1000 * 3600 * 3 // 3 hours
   },
+  STORYBOARDS: {
+    DIRECTORY: join(CONFIG.STORAGE.CACHE_DIR, 'storyboards'),
+    MAX_AGE: 1000 * 3600 * 24 // 24 hours
+  },
   VIDEO_CAPTIONS: {
     DIRECTORY: join(CONFIG.STORAGE.CACHE_DIR, 'video-captions'),
     MAX_AGE: 1000 * 3600 * 3 // 3 hours
@@ -1090,6 +1106,7 @@ export {
   RESUMABLE_UPLOAD_SESSION_LIFETIME,
   RUNNER_JOB_STATES,
   P2P_MEDIA_LOADER_PEER_VERSION,
+  STORYBOARD,
   ACTOR_IMAGES_SIZE,
   ACCEPT_HEADERS,
   BCRYPT_SALT_SIZE,

server/initializers/database.ts

@@ -10,6 +10,7 @@ import { UserModel } from '@server/models/user/user'
 import { UserNotificationModel } from '@server/models/user/user-notification'
 import { UserRegistrationModel } from '@server/models/user/user-registration'
 import { UserVideoHistoryModel } from '@server/models/user/user-video-history'
+import { StoryboardModel } from '@server/models/video/storyboard'
 import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
 import { VideoJobInfoModel } from '@server/models/video/video-job-info'
 import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting'
@@ -167,7 +168,8 @@ async function initDatabaseModels (silent: boolean) {
     VideoPasswordModel,
     RunnerRegistrationTokenModel,
     RunnerModel,
-    RunnerJobModel
+    RunnerJobModel,
+    StoryboardModel
   ])
 
   // Check extensions exist in the database

server/lib/activitypub/context.ts

@@ -46,6 +46,19 @@ const contextStore: { [ id in ContextType ]: (string | { [ id: string ]: string
 
     Infohash: 'pt:Infohash',
 
+    tileWidth: {
+      '@type': 'sc:Number',
+      '@id': 'pt:tileWidth'
+    },
+    tileHeight: {
+      '@type': 'sc:Number',
+      '@id': 'pt:tileHeight'
+    },
+    tileDuration: {
+      '@type': 'sc:Number',
+      '@id': 'pt:tileDuration'
+    },
+
     originallyPublishedAt: 'sc:datePublished',
     views: {
       '@type': 'sc:Number',

server/lib/activitypub/send/send-update.ts

@@ -10,8 +10,7 @@ import {
   MActor,
   MActorLight,
   MChannelDefault,
-  MVideoAP,
+  MVideoAPLight,
-  MVideoAPWithoutCaption,
   MVideoPlaylistFull,
   MVideoRedundancyVideo
 } from '../../../types/models'
@@ -20,10 +19,10 @@ import { getUpdateActivityPubUrl } from '../url'
 import { getActorsInvolvedInVideo } from './shared'
 import { broadcastToFollowers, sendVideoRelatedActivity } from './shared/send-utils'
 
-async function sendUpdateVideo (videoArg: MVideoAPWithoutCaption, transaction: Transaction, overriddenByActor?: MActor) {
+async function sendUpdateVideo (videoArg: MVideoAPLight, transaction: Transaction, overriddenByActor?: MActor) {
-  const video = videoArg as MVideoAP
+  if (!videoArg.hasPrivacyForFederation()) return undefined
 
-  if (!video.hasPrivacyForFederation()) return undefined
+  const video = await videoArg.lightAPToFullAP(transaction)
 
   logger.info('Creating job to update video %s.', video.url)
 
@@ -31,11 +30,6 @@ async function sendUpdateVideo (videoArg: MVideoAPWithoutCaption, transaction: T
 
   const url = getUpdateActivityPubUrl(video.url, video.updatedAt.toISOString())
 
-  // Needed to build the AP object
-  if (!video.VideoCaptions) {
-    video.VideoCaptions = await video.$get('VideoCaptions', { transaction })
-  }
-
   const videoObject = await video.toActivityPubObject()
   const audience = getAudience(byActor, video.privacy === VideoPrivacy.PUBLIC)
 

server/lib/activitypub/videos/federate.ts

@@ -1,10 +1,9 @@
 import { Transaction } from 'sequelize/types'
-import { isArray } from '@server/helpers/custom-validators/misc'
+import { MVideoAP, MVideoAPLight } from '@server/types/models'
-import { MVideoAP, MVideoAPWithoutCaption } from '@server/types/models'
 import { sendCreateVideo, sendUpdateVideo } from '../send'
 import { shareVideoByServerAndChannel } from '../share'
 
-async function federateVideoIfNeeded (videoArg: MVideoAPWithoutCaption, isNewVideo: boolean, transaction?: Transaction) {
+async function federateVideoIfNeeded (videoArg: MVideoAPLight, isNewVideo: boolean, transaction?: Transaction) {
   const video = videoArg as MVideoAP
 
   if (
@@ -13,13 +12,7 @@ async function federateVideoIfNeeded (videoArg: MVideoAPWithoutCaption, isNewVid
     // Check the video is public/unlisted and published
     video.hasPrivacyForFederation() && video.hasStateForFederation()
   ) {
-    // Fetch more attributes that we will need to serialize in AP object
+    const video = await videoArg.lightAPToFullAP(transaction)
-    if (isArray(video.VideoCaptions) === false) {
-      video.VideoCaptions = await video.$get('VideoCaptions', {
-        attributes: [ 'filename', 'language' ],
-        transaction
-      })
-    }
 
     if (isNewVideo) {
       // Now we'll add the video's meta data to our followers

server/lib/activitypub/videos/shared/abstract-builder.ts

@@ -3,6 +3,7 @@ import { deleteAllModels, filterNonExistingModels } from '@server/helpers/databa
 import { logger, LoggerTagsFn } from '@server/helpers/logger'
 import { updatePlaceholderThumbnail, updateVideoMiniatureFromUrl } from '@server/lib/thumbnail'
 import { setVideoTags } from '@server/lib/video'
+import { StoryboardModel } from '@server/models/video/storyboard'
 import { VideoCaptionModel } from '@server/models/video/video-caption'
 import { VideoFileModel } from '@server/models/video/video-file'
 import { VideoLiveModel } from '@server/models/video/video-live'
@@ -24,6 +25,7 @@ import {
   getFileAttributesFromUrl,
   getLiveAttributesFromObject,
   getPreviewFromIcons,
+  getStoryboardAttributeFromObject,
   getStreamingPlaylistAttributesFromObject,
   getTagsFromObject,
   getThumbnailFromIcons
@@ -107,6 +109,16 @@ export abstract class APVideoAbstractBuilder {
     }
   }
 
+  protected async insertOrReplaceStoryboard (video: MVideoFullLight, t: Transaction) {
+    const existingStoryboard = await StoryboardModel.loadByVideo(video.id, t)
+    if (existingStoryboard) await existingStoryboard.destroy({ transaction: t })
+
+    const storyboardAttributes = getStoryboardAttributeFromObject(video, this.videoObject)
+    if (!storyboardAttributes) return
+
+    return StoryboardModel.create(storyboardAttributes, { transaction: t })
+  }
+
   protected async insertOrReplaceLive (video: MVideoFullLight, transaction: Transaction) {
     const attributes = getLiveAttributesFromObject(video, this.videoObject)
     const [ videoLive ] = await VideoLiveModel.upsert(attributes, { transaction, returning: true })

server/lib/activitypub/videos/shared/creator.ts

@@ -48,6 +48,7 @@ export class APVideoCreator extends APVideoAbstractBuilder {
         await this.setTrackers(videoCreated, t)
         await this.insertOrReplaceCaptions(videoCreated, t)
         await this.insertOrReplaceLive(videoCreated, t)
+        await this.insertOrReplaceStoryboard(videoCreated, t)
 
         // We added a video in this channel, set it as updated
         await channel.setAsUpdated(t)

server/lib/activitypub/videos/shared/object-to-model-attributes.ts

@@ -1,6 +1,6 @@
 import { maxBy, minBy } from 'lodash'
 import { decode as magnetUriDecode } from 'magnet-uri'
-import { basename } from 'path'
+import { basename, extname } from 'path'
 import { isAPVideoFileUrlMetadataObject } from '@server/helpers/custom-validators/activitypub/videos'
 import { isVideoFileInfoHashValid } from '@server/helpers/custom-validators/videos'
 import { logger } from '@server/helpers/logger'
@@ -25,6 +25,9 @@ import {
   VideoStreamingPlaylistType
 } from '@shared/models'
 import { getDurationFromActivityStream } from '../../activity'
+import { isArray } from '@server/helpers/custom-validators/misc'
+import { generateImageFilename } from '@server/helpers/image-utils'
+import { arrayify } from '@shared/core-utils'
 
 function getThumbnailFromIcons (videoObject: VideoObject) {
   let validIcons = videoObject.icon.filter(i => i.width > THUMBNAILS_SIZE.minWidth)
@@ -166,6 +169,26 @@ function getCaptionAttributesFromObject (video: MVideoId, videoObject: VideoObje
   }))
 }
 
+function getStoryboardAttributeFromObject (video: MVideoId, videoObject: VideoObject) {
+  if (!isArray(videoObject.preview)) return undefined
+
+  const storyboard = videoObject.preview.find(p => p.rel.includes('storyboard'))
+  if (!storyboard) return undefined
+
+  const url = arrayify(storyboard.url).find(u => u.mediaType === 'image/jpeg')
+
+  return {
+    filename: generateImageFilename(extname(url.href)),
+    totalHeight: url.height,
+    totalWidth: url.width,
+    spriteHeight: url.tileHeight,
+    spriteWidth: url.tileWidth,
+    spriteDuration: getDurationFromActivityStream(url.tileDuration),
+    fileUrl: url.href,
+    videoId: video.id
+  }
+}
+
 function getVideoAttributesFromObject (videoChannel: MChannelId, videoObject: VideoObject, to: string[] = []) {
   const privacy = to.includes(ACTIVITY_PUB.PUBLIC)
     ? VideoPrivacy.PUBLIC
@@ -228,6 +251,7 @@ export {
 
   getLiveAttributesFromObject,
   getCaptionAttributesFromObject,
+  getStoryboardAttributeFromObject,
 
   getVideoAttributesFromObject
 }

server/lib/activitypub/videos/updater.ts

@@ -57,6 +57,7 @@ export class APVideoUpdater extends APVideoAbstractBuilder {
       await Promise.all([
         runInReadCommittedTransaction(t => this.setTags(videoUpdated, t)),
         runInReadCommittedTransaction(t => this.setTrackers(videoUpdated, t)),
+        runInReadCommittedTransaction(t => this.setStoryboard(videoUpdated, t)),
         this.setOrDeleteLive(videoUpdated),
         this.setPreview(videoUpdated)
       ])
@@ -138,6 +139,10 @@ export class APVideoUpdater extends APVideoAbstractBuilder {
     await this.insertOrReplaceCaptions(videoUpdated, t)
   }
 
+  private async setStoryboard (videoUpdated: MVideoFullLight, t: Transaction) {
+    await this.insertOrReplaceStoryboard(videoUpdated, t)
+  }
+
   private async setOrDeleteLive (videoUpdated: MVideoFullLight, transaction?: Transaction) {
     if (!this.video.isLive) return
 

server/lib/files-cache/index.ts

@@ -1,3 +1,4 @@
-export * from './videos-preview-cache'
 export * from './videos-caption-cache'
+export * from './videos-preview-cache'
+export * from './videos-storyboard-cache'
 export * from './videos-torrent-cache'

server/lib/files-cache/videos-storyboard-cache.ts

@@ -0,0 +1,53 @@
+import { join } from 'path'
+import { logger } from '@server/helpers/logger'
+import { doRequestAndSaveToFile } from '@server/helpers/requests'
+import { StoryboardModel } from '@server/models/video/storyboard'
+import { FILES_CACHE } from '../../initializers/constants'
+import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache'
+
+class VideosStoryboardCache extends AbstractVideoStaticFileCache <string> {
+
+  private static instance: VideosStoryboardCache
+
+  private constructor () {
+    super()
+  }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+
+  async getFilePathImpl (filename: string) {
+    const storyboard = await StoryboardModel.loadWithVideoByFilename(filename)
+    if (!storyboard) return undefined
+
+    if (storyboard.Video.isOwned()) return { isOwned: true, path: storyboard.getPath() }
+
+    return this.loadRemoteFile(storyboard.filename)
+  }
+
+  // Key is the storyboard filename
+  protected async loadRemoteFile (key: string) {
+    const storyboard = await StoryboardModel.loadWithVideoByFilename(key)
+    if (!storyboard) return undefined
+
+    const destPath = join(FILES_CACHE.STORYBOARDS.DIRECTORY, storyboard.filename)
+    const remoteUrl = storyboard.getOriginFileUrl(storyboard.Video)
+
+    try {
+      await doRequestAndSaveToFile(remoteUrl, destPath)
+
+      logger.debug('Fetched remote storyboard %s to %s.', remoteUrl, destPath)
+
+      return { isOwned: false, path: destPath }
+    } catch (err) {
+      logger.info('Cannot fetch remote storyboard file %s.', remoteUrl, { err })
+
+      return undefined
+    }
+  }
+}
+
+export {
+  VideosStoryboardCache
+}

server/lib/job-queue/handlers/generate-storyboard.ts

@@ -0,0 +1,138 @@
+import { Job } from 'bullmq'
+import { join } from 'path'
+import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg'
+import { generateImageFilename, getImageSize } from '@server/helpers/image-utils'
+import { logger, loggerTagsFactory } from '@server/helpers/logger'
+import { CONFIG } from '@server/initializers/config'
+import { STORYBOARD } from '@server/initializers/constants'
+import { federateVideoIfNeeded } from '@server/lib/activitypub/videos'
+import { VideoPathManager } from '@server/lib/video-path-manager'
+import { StoryboardModel } from '@server/models/video/storyboard'
+import { VideoModel } from '@server/models/video/video'
+import { MVideo } from '@server/types/models'
+import { FFmpegImage, isAudioFile } from '@shared/ffmpeg'
+import { GenerateStoryboardPayload } from '@shared/models'
+
+const lTagsBase = loggerTagsFactory('storyboard')
+
+async function processGenerateStoryboard (job: Job): Promise<void> {
+  const payload = job.data as GenerateStoryboardPayload
+  const lTags = lTagsBase(payload.videoUUID)
+
+  logger.info('Processing generate storyboard of %s in job %s.', payload.videoUUID, job.id, lTags)
+
+  const video = await VideoModel.loadFull(payload.videoUUID)
+  if (!video) {
+    logger.info('Video %s does not exist anymore, skipping storyboard generation.', payload.videoUUID, lTags)
+    return
+  }
+
+  const inputFile = video.getMaxQualityFile()
+
+  await VideoPathManager.Instance.makeAvailableVideoFile(inputFile, async videoPath => {
+    const isAudio = await isAudioFile(videoPath)
+
+    if (isAudio) {
+      logger.info('Do not generate a storyboard of %s since the video does not have a video stream', payload.videoUUID, lTags)
+      return
+    }
+
+    const ffmpeg = new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail'))
+
+    const filename = generateImageFilename()
+    const destination = join(CONFIG.STORAGE.STORYBOARDS_DIR, filename)
+
+    const totalSprites = buildTotalSprites(video)
+    const spriteDuration = Math.round(video.duration / totalSprites)
+
+    const spritesCount = findGridSize({
+      toFind: totalSprites,
+      maxEdgeCount: STORYBOARD.SPRITES_MAX_EDGE_COUNT
+    })
+
+    logger.debug(
+      'Generating storyboard from video of %s to %s', video.uuid, destination,
+      { ...lTags, spritesCount, spriteDuration, videoDuration: video.duration }
+    )
+
+    await ffmpeg.generateStoryboardFromVideo({
+      destination,
+      path: videoPath,
+      sprites: {
+        size: STORYBOARD.SPRITE_SIZE,
+        count: spritesCount,
+        duration: spriteDuration
+      }
+    })
+
+    const imageSize = await getImageSize(destination)
+
+    const existing = await StoryboardModel.loadByVideo(video.id)
+    if (existing) await existing.destroy()
+
+    await StoryboardModel.create({
+      filename,
+      totalHeight: imageSize.height,
+      totalWidth: imageSize.width,
+      spriteHeight: STORYBOARD.SPRITE_SIZE.height,
+      spriteWidth: STORYBOARD.SPRITE_SIZE.width,
+      spriteDuration,
+      videoId: video.id
+    })
+
+    logger.info('Storyboard generation %s ended for video %s.', destination, video.uuid, lTags)
+  })
+
+  if (payload.federate) {
+    await federateVideoIfNeeded(video, false)
+  }
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  processGenerateStoryboard
+}
+
+function buildTotalSprites (video: MVideo) {
+  const maxSprites = STORYBOARD.SPRITE_SIZE.height * STORYBOARD.SPRITE_SIZE.width
+  const totalSprites = Math.min(Math.ceil(video.duration), maxSprites)
+
+  // We can generate a single line
+  if (totalSprites <= STORYBOARD.SPRITES_MAX_EDGE_COUNT) return totalSprites
+
+  return findGridFit(totalSprites, STORYBOARD.SPRITES_MAX_EDGE_COUNT)
+}
+
+function findGridSize (options: {
+  toFind: number
+  maxEdgeCount: number
+}) {
+  const { toFind, maxEdgeCount } = options
+
+  for (let i = 1; i <= maxEdgeCount; i++) {
+    for (let j = i; j <= maxEdgeCount; j++) {
+      if (toFind === i * j) return { width: j, height: i }
+    }
+  }
+
+  throw new Error(`Could not find grid size (to find: ${toFind}, max edge count: ${maxEdgeCount}`)
+}
+
+function findGridFit (value: number, maxMultiplier: number) {
+  for (let i = value; i--; i > 0) {
+    if (!isPrimeWithin(i, maxMultiplier)) return i
+  }
+
+  throw new Error('Could not find prime number below ' + value)
+}
+
+function isPrimeWithin (value: number, maxMultiplier: number) {
+  if (value < 2) return false
+
+  for (let i = 2, end = Math.min(Math.sqrt(value), maxMultiplier); i <= end; i++) {
+    if (value % i === 0 && value / i <= maxMultiplier) return false
+  }
+
+  return true
+}

server/lib/job-queue/handlers/video-import.ts

@@ -306,6 +306,15 @@ async function afterImportSuccess (options: {
     Notifier.Instance.notifyOnNewVideoIfNeeded(video)
   }
 
+  // Generate the storyboard in the job queue, and don't forget to federate an update after
+  await JobQueue.Instance.createJob({
+    type: 'generate-video-storyboard' as 'generate-video-storyboard',
+    payload: {
+      videoUUID: video.uuid,
+      federate: true
+    }
+  })
+
   if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
     await JobQueue.Instance.createJob(
       await buildMoveToObjectStorageJob({ video, previousVideoState: VideoState.TO_IMPORT })

server/lib/job-queue/handlers/video-live-ending.ts

@@ -1,6 +1,8 @@
 import { Job } from 'bullmq'
 import { readdir, remove } from 'fs-extra'
 import { join } from 'path'
+import { peertubeTruncate } from '@server/helpers/core-utils'
+import { CONSTRAINTS_FIELDS } from '@server/initializers/constants'
 import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url'
 import { federateVideoIfNeeded } from '@server/lib/activitypub/videos'
 import { cleanupAndDestroyPermanentLive, cleanupTMPLiveFiles, cleanupUnsavedNormalLive } from '@server/lib/live'
@@ -20,8 +22,7 @@ import { MVideo, MVideoLive, MVideoLiveSession, MVideoWithAllFiles } from '@serv
 import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo, getVideoStreamFPS } from '@shared/ffmpeg'
 import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models'
 import { logger, loggerTagsFactory } from '../../../helpers/logger'
-import { peertubeTruncate } from '@server/helpers/core-utils'
+import { JobQueue } from '../job-queue'
-import { CONSTRAINTS_FIELDS } from '@server/initializers/constants'
 
 const lTags = loggerTagsFactory('live', 'job')
 
@@ -147,6 +148,8 @@ async function saveReplayToExternalVideo (options: {
   }
 
   await moveToNextState({ video: replayVideo, isNewVideo: true })
+
+  await createStoryboardJob(replayVideo)
 }
 
 async function replaceLiveByReplay (options: {
@@ -186,6 +189,7 @@ async function replaceLiveByReplay (options: {
 
   await assignReplayFilesToVideo({ video: videoWithFiles, replayDirectory })
 
+  // FIXME: should not happen in this function
   if (permanentLive) { // Remove session replay
     await remove(replayDirectory)
   } else { // We won't stream again in this live, we can delete the base replay directory
@@ -213,6 +217,8 @@ async function replaceLiveByReplay (options: {
 
   // We consider this is a new video
   await moveToNextState({ video: videoWithFiles, isNewVideo: true })
+
+  await createStoryboardJob(videoWithFiles)
 }
 
 async function assignReplayFilesToVideo (options: {
@@ -277,3 +283,13 @@ async function cleanupLiveAndFederate (options: {
     logger.warn('Cannot federate live after cleanup', { videoId: video.id, err })
   }
 }
+
+function createStoryboardJob (video: MVideo) {
+  return JobQueue.Instance.createJob({
+    type: 'generate-video-storyboard' as 'generate-video-storyboard',
+    payload: {
+      videoUUID: video.uuid,
+      federate: true
+    }
+  })
+}

server/lib/job-queue/job-queue.ts

@@ -25,6 +25,7 @@ import {
   DeleteResumableUploadMetaFilePayload,
   EmailPayload,
   FederateVideoPayload,
+  GenerateStoryboardPayload,
   JobState,
   JobType,
   ManageVideoTorrentPayload,
@@ -65,6 +66,7 @@ import { processVideoLiveEnding } from './handlers/video-live-ending'
 import { processVideoStudioEdition } from './handlers/video-studio-edition'
 import { processVideoTranscoding } from './handlers/video-transcoding'
 import { processVideosViewsStats } from './handlers/video-views-stats'
+import { processGenerateStoryboard } from './handlers/generate-storyboard'
 
 export type CreateJobArgument =
   { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } |
@@ -91,7 +93,8 @@ export type CreateJobArgument =
   { type: 'after-video-channel-import', payload: AfterVideoChannelImportPayload } |
   { type: 'notify', payload: NotifyPayload } |
   { type: 'move-to-object-storage', payload: MoveObjectStoragePayload } |
-  { type: 'federate-video', payload: FederateVideoPayload }
+  { type: 'federate-video', payload: FederateVideoPayload } |
+  { type: 'generate-video-storyboard', payload: GenerateStoryboardPayload }
 
 export type CreateJobOptions = {
   delay?: number
@@ -122,7 +125,8 @@ const handlers: { [id in JobType]: (job: Job) => Promise<any> } = {
   'video-redundancy': processVideoRedundancy,
   'video-studio-edition': processVideoStudioEdition,
   'video-transcoding': processVideoTranscoding,
-  'videos-views-stats': processVideosViewsStats
+  'videos-views-stats': processVideosViewsStats,
+  'generate-video-storyboard': processGenerateStoryboard
 }
 
 const errorHandlers: { [id in JobType]?: (job: Job, err: any) => Promise<any> } = {
@@ -141,10 +145,11 @@ const jobTypes: JobType[] = [
   'after-video-channel-import',
   'email',
   'federate-video',
-  'transcoding-job-builder',
+  'generate-video-storyboard',
   'manage-video-torrent',
   'move-to-object-storage',
   'notify',
+  'transcoding-job-builder',
   'video-channel-import',
   'video-file-import',
   'video-import',

server/lib/redis.ts

@@ -325,8 +325,8 @@ class Redis {
     const value = await this.getValue('resumable-upload-' + uploadId)
 
     return value
-      ? JSON.parse(value)
+      ? JSON.parse(value) as { video: { id: number, shortUUID: string, uuid: string } }
-      : ''
+      : undefined
   }
 
   deleteUploadSession (uploadId: string) {

server/lib/transcoding/web-transcoding.ts

@@ -9,6 +9,7 @@ import { ffprobePromise, getVideoStreamDuration, getVideoStreamFPS, TranscodeVOD
 import { VideoResolution, VideoStorage } from '@shared/models'
 import { CONFIG } from '../../initializers/config'
 import { VideoFileModel } from '../../models/video/video-file'
+import { JobQueue } from '../job-queue'
 import { generateWebTorrentVideoFilename } from '../paths'
 import { buildFileMetadata } from '../video-file'
 import { VideoPathManager } from '../video-path-manager'
@@ -198,7 +199,8 @@ export async function mergeAudioVideofile (options: {
       return onWebTorrentVideoFileTranscoding({
         video,
         videoFile: inputVideoFile,
-        videoOutputPath
+        videoOutputPath,
+        wasAudioFile: true
       })
     })
 
@@ -212,8 +214,9 @@ export async function onWebTorrentVideoFileTranscoding (options: {
   video: MVideoFullLight
   videoFile: MVideoFile
   videoOutputPath: string
+  wasAudioFile?: boolean // default false
 }) {
-  const { video, videoFile, videoOutputPath } = options
+  const { video, videoFile, videoOutputPath, wasAudioFile } = options
 
   const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
 
@@ -242,6 +245,17 @@ export async function onWebTorrentVideoFileTranscoding (options: {
     await VideoFileModel.customUpsert(videoFile, 'video', undefined)
     video.VideoFiles = await video.$get('VideoFiles')
 
+    if (wasAudioFile) {
+      await JobQueue.Instance.createJob({
+        type: 'generate-video-storyboard' as 'generate-video-storyboard',
+        payload: {
+          videoUUID: video.uuid,
+          // No need to federate, we process these jobs sequentially
+          federate: false
+        }
+      })
+    }
+
     return { video, videoFile }
   } finally {
     mutexReleaser()

server/middlewares/validators/config.ts

@@ -25,6 +25,7 @@ const customConfigUpdateValidator = [
   body('cache.previews.size').isInt(),
   body('cache.captions.size').isInt(),
   body('cache.torrents.size').isInt(),
+  body('cache.storyboards.size').isInt(),
 
   body('signup.enabled').isBoolean(),
   body('signup.limit').isInt(),

server/models/video/formatter/video-format-utils.ts

@@ -5,6 +5,7 @@ import { getLocalVideoFileMetadataUrl } from '@server/lib/video-urls'
 import { VideoViewsManager } from '@server/lib/views/video-views-manager'
 import { uuidToShort } from '@shared/extra-utils'
 import {
+  ActivityPubStoryboard,
   ActivityTagObject,
   ActivityUrlObject,
   Video,
@@ -347,29 +348,17 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject {
     name: t.name
   }))
 
-  let language
+  const language = video.language
-  if (video.language) {
+    ? { identifier: video.language, name: getLanguageLabel(video.language) }
-    language = {
+    : undefined
-      identifier: video.language,
-      name: getLanguageLabel(video.language)
-    }
-  }
 
-  let category
+  const category = video.category
-  if (video.category) {
+    ? { identifier: video.category + '', name: getCategoryLabel(video.category) }
-    category = {
+    : undefined
-      identifier: video.category + '',
-      name: getCategoryLabel(video.category)
-    }
-  }
 
-  let licence
+  const licence = video.licence
-  if (video.licence) {
+    ? { identifier: video.licence + '', name: getLicenceLabel(video.licence) }
-    licence = {
+    : undefined
-      identifier: video.licence + '',
-      name: getLicenceLabel(video.licence)
-    }
-  }
 
   const url: ActivityUrlObject[] = [
     // HTML url should be the first element in the array so Mastodon correctly displays the embed
@@ -465,6 +454,8 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject {
       height: i.height
     })),
 
+    preview: buildPreviewAPAttribute(video),
+
     url,
 
     likes: getLocalVideoLikesActivityPubUrl(video),
@@ -541,3 +532,30 @@ function buildLiveAPAttributes (video: MVideoAP) {
     latencyMode: video.VideoLive.latencyMode
   }
 }
+
+function buildPreviewAPAttribute (video: MVideoAP): ActivityPubStoryboard[] {
+  if (!video.Storyboard) return undefined
+
+  const storyboard = video.Storyboard
+
+  return [
+    {
+      type: 'Image',
+      rel: [ 'storyboard' ],
+      url: [
+        {
+          mediaType: 'image/jpeg',
+
+          href: storyboard.getOriginFileUrl(video),
+
+          width: storyboard.totalWidth,
+          height: storyboard.totalHeight,
+
+          tileWidth: storyboard.spriteWidth,
+          tileHeight: storyboard.spriteHeight,
+          tileDuration: getActivityStreamDuration(storyboard.spriteDuration)
+        }
+      ]
+    }
+  ]
+}

server/models/video/storyboard.ts

@@ -0,0 +1,169 @@
+import { remove } from 'fs-extra'
+import { join } from 'path'
+import { AfterDestroy, AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript'
+import { CONFIG } from '@server/initializers/config'
+import { MStoryboard, MStoryboardVideo, MVideo } from '@server/types/models'
+import { Storyboard } from '@shared/models'
+import { AttributesOnly } from '@shared/typescript-utils'
+import { logger } from '../../helpers/logger'
+import { CONSTRAINTS_FIELDS, LAZY_STATIC_PATHS, WEBSERVER } from '../../initializers/constants'
+import { VideoModel } from './video'
+import { Transaction } from 'sequelize'
+
+@Table({
+  tableName: 'storyboard',
+  indexes: [
+    {
+      fields: [ 'videoId' ],
+      unique: true
+    },
+    {
+      fields: [ 'filename' ],
+      unique: true
+    }
+  ]
+})
+export class StoryboardModel extends Model<Partial<AttributesOnly<StoryboardModel>>> {
+
+  @AllowNull(false)
+  @Column
+  filename: string
+
+  @AllowNull(false)
+  @Column
+  totalHeight: number
+
+  @AllowNull(false)
+  @Column
+  totalWidth: number
+
+  @AllowNull(false)
+  @Column
+  spriteHeight: number
+
+  @AllowNull(false)
+  @Column
+  spriteWidth: number
+
+  @AllowNull(false)
+  @Column
+  spriteDuration: number
+
+  @AllowNull(true)
+  @Column(DataType.STRING(CONSTRAINTS_FIELDS.COMMONS.URL.max))
+  fileUrl: string
+
+  @ForeignKey(() => VideoModel)
+  @Column
+  videoId: number
+
+  @BelongsTo(() => VideoModel, {
+    foreignKey: {
+      allowNull: true
+    },
+    onDelete: 'CASCADE'
+  })
+  Video: VideoModel
+
+  @CreatedAt
+  createdAt: Date
+
+  @UpdatedAt
+  updatedAt: Date
+
+  @AfterDestroy
+  static removeInstanceFile (instance: StoryboardModel) {
+    logger.info('Removing storyboard file %s.', instance.filename)
+
+    // Don't block the transaction
+    instance.removeFile()
+            .catch(err => logger.error('Cannot remove storyboard file %s.', instance.filename, { err }))
+  }
+
+  static loadByVideo (videoId: number, transaction?: Transaction): Promise<MStoryboard> {
+    const query = {
+      where: {
+        videoId
+      },
+      transaction
+    }
+
+    return StoryboardModel.findOne(query)
+  }
+
+  static loadByFilename (filename: string): Promise<MStoryboard> {
+    const query = {
+      where: {
+        filename
+      }
+    }
+
+    return StoryboardModel.findOne(query)
+  }
+
+  static loadWithVideoByFilename (filename: string): Promise<MStoryboardVideo> {
+    const query = {
+      where: {
+        filename
+      },
+      include: [
+        {
+          model: VideoModel.unscoped(),
+          required: true
+        }
+      ]
+    }
+
+    return StoryboardModel.findOne(query)
+  }
+
+  // ---------------------------------------------------------------------------
+
+  static async listStoryboardsOf (video: MVideo): Promise<MStoryboardVideo[]> {
+    const query = {
+      where: {
+        videoId: video.id
+      }
+    }
+
+    const storyboards = await StoryboardModel.findAll<MStoryboard>(query)
+
+    return storyboards.map(s => Object.assign(s, { Video: video }))
+  }
+
+  // ---------------------------------------------------------------------------
+
+  getOriginFileUrl (video: MVideo) {
+    if (video.isOwned()) {
+      return WEBSERVER.URL + this.getLocalStaticPath()
+    }
+
+    return this.fileUrl
+  }
+
+  getLocalStaticPath () {
+    return LAZY_STATIC_PATHS.STORYBOARDS + this.filename
+  }
+
+  getPath () {
+    return join(CONFIG.STORAGE.STORYBOARDS_DIR, this.filename)
+  }
+
+  removeFile () {
+    return remove(this.getPath())
+  }
+
+  toFormattedJSON (this: MStoryboardVideo): Storyboard {
+    return {
+      storyboardPath: this.getLocalStaticPath(),
+
+      totalHeight: this.totalHeight,
+      totalWidth: this.totalWidth,
+
+      spriteWidth: this.spriteWidth,
+      spriteHeight: this.spriteHeight,
+
+      spriteDuration: this.spriteDuration
+    }
+  }
+}

server/models/video/video-caption.ts

@@ -15,7 +15,7 @@ import {
   Table,
   UpdatedAt
 } from 'sequelize-typescript'
-import { MVideo, MVideoCaption, MVideoCaptionFormattable, MVideoCaptionVideo } from '@server/types/models'
+import { MVideo, MVideoCaption, MVideoCaptionFormattable, MVideoCaptionLanguageUrl, MVideoCaptionVideo } from '@server/types/models'
 import { buildUUID } from '@shared/extra-utils'
 import { AttributesOnly } from '@shared/typescript-utils'
 import { VideoCaption } from '../../../shared/models/videos/caption/video-caption.model'
@@ -225,7 +225,7 @@ export class VideoCaptionModel extends Model<Partial<AttributesOnly<VideoCaption
     }
   }
 
-  getCaptionStaticPath (this: MVideoCaption) {
+  getCaptionStaticPath (this: MVideoCaptionLanguageUrl) {
     return join(LAZY_STATIC_PATHS.VIDEO_CAPTIONS, this.filename)
   }
 
@@ -233,9 +233,7 @@ export class VideoCaptionModel extends Model<Partial<AttributesOnly<VideoCaption
     return remove(CONFIG.STORAGE.CAPTIONS_DIR + this.filename)
   }
 
-  getFileUrl (video: MVideo) {
+  getFileUrl (this: MVideoCaptionLanguageUrl, video: MVideo) {
-    if (!this.Video) this.Video = video as VideoModel
-
     if (video.isOwned()) return WEBSERVER.URL + this.getCaptionStaticPath()
 
     return this.fileUrl

server/models/video/video.ts

@@ -58,7 +58,7 @@ import {
 import { AttributesOnly } from '@shared/typescript-utils'
 import { peertubeTruncate } from '../../helpers/core-utils'
 import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
-import { exists, isBooleanValid, isUUIDValid } from '../../helpers/custom-validators/misc'
+import { exists, isArray, isBooleanValid, isUUIDValid } from '../../helpers/custom-validators/misc'
 import {
   isVideoDescriptionValid,
   isVideoDurationValid,
@@ -75,6 +75,7 @@ import {
   MChannel,
   MChannelAccountDefault,
   MChannelId,
+  MStoryboard,
   MStreamingPlaylist,
   MStreamingPlaylistFilesVideo,
   MUserAccountId,
@@ -83,6 +84,8 @@ import {
   MVideoAccountLight,
   MVideoAccountLightBlacklistAllFiles,
   MVideoAP,
+  MVideoAPLight,
+  MVideoCaptionLanguageUrl,
   MVideoDetails,
   MVideoFileVideo,
   MVideoFormattable,
@@ -126,6 +129,7 @@ import {
   VideosIdListQueryBuilder,
   VideosModelListQueryBuilder
 } from './sql/video'
+import { StoryboardModel } from './storyboard'
 import { TagModel } from './tag'
 import { ThumbnailModel } from './thumbnail'
 import { VideoBlacklistModel } from './video-blacklist'
@@ -753,6 +757,15 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
   })
   VideoJobInfo: VideoJobInfoModel
 
+  @HasOne(() => StoryboardModel, {
+    foreignKey: {
+      name: 'videoId',
+      allowNull: false
+    },
+    onDelete: 'cascade'
+  })
+  Storyboard: StoryboardModel
+
   @AfterCreate
   static notifyCreate (video: MVideo) {
     InternalEventEmitter.Instance.emit('video-created', { video })
@@ -903,6 +916,10 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
           model: VideoCaptionModel.unscoped(),
           required: false
         },
+        {
+          model: StoryboardModel.unscoped(),
+          required: false
+        },
         {
           attributes: [ 'id', 'url' ],
           model: VideoShareModel.unscoped(),
@@ -1768,6 +1785,32 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
     )
   }
 
+  async lightAPToFullAP (this: MVideoAPLight, transaction: Transaction): Promise<MVideoAP> {
+    const videoAP = this as MVideoAP
+
+    const getCaptions = () => {
+      if (isArray(videoAP.VideoCaptions)) return videoAP.VideoCaptions
+
+      return this.$get('VideoCaptions', {
+        attributes: [ 'filename', 'language', 'fileUrl' ],
+        transaction
+      }) as Promise<MVideoCaptionLanguageUrl[]>
+    }
+
+    const getStoryboard = () => {
+      if (videoAP.Storyboard) return videoAP.Storyboard
+
+      return this.$get('Storyboard', { transaction }) as Promise<MStoryboard>
+    }
+
+    const [ captions, storyboard ] = await Promise.all([ getCaptions(), getStoryboard() ])
+
+    return Object.assign(this, {
+      VideoCaptions: captions,
+      Storyboard: storyboard
+    })
+  }
+
   getTruncatedDescription () {
     if (!this.description) return null
 

server/tests/api/check-params/config.ts

@@ -74,6 +74,9 @@ describe('Test config API validators', function () {
       },
       torrents: {
         size: 4
+      },
+      storyboards: {
+        size: 5
       }
     },
     signup: {

server/tests/api/check-params/index.ts

@@ -34,6 +34,7 @@ import './video-comments'
 import './video-files'
 import './video-imports'
 import './video-playlists'
+import './video-storyboards'
 import './video-source'
 import './video-studio'
 import './video-token'

server/tests/api/check-params/video-storyboards.ts

@@ -0,0 +1,45 @@
+/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
+
+import { HttpStatusCode, VideoPrivacy } from '@shared/models'
+import { cleanupTests, createSingleServer, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands'
+
+describe('Test video storyboards API validator', function () {
+  let server: PeerTubeServer
+
+  let publicVideo: { uuid: string }
+  let privateVideo: { uuid: string }
+
+  // ---------------------------------------------------------------
+
+  before(async function () {
+    this.timeout(30000)
+
+    server = await createSingleServer(1)
+    await setAccessTokensToServers([ server ])
+
+    publicVideo = await server.videos.quickUpload({ name: 'public' })
+    privateVideo = await server.videos.quickUpload({ name: 'private', privacy: VideoPrivacy.PRIVATE })
+  })
+
+  it('Should fail without a valid uuid', async function () {
+    await server.storyboard.list({ id: '4da6fde3-88f7-4d16-b119-108df563d0b0', expectedStatus: HttpStatusCode.NOT_FOUND_404 })
+  })
+
+  it('Should receive 404 when passing a non existing video id', async function () {
+    await server.storyboard.list({ id: '4da6fde3-88f7-4d16-b119-108df5630b06', expectedStatus: HttpStatusCode.NOT_FOUND_404 })
+  })
+
+  it('Should not get the private storyboard without the appropriate token', async function () {
+    await server.storyboard.list({ id: privateVideo.uuid, expectedStatus: HttpStatusCode.UNAUTHORIZED_401, token: null })
+    await server.storyboard.list({ id: publicVideo.uuid, expectedStatus: HttpStatusCode.OK_200, token: null })
+  })
+
+  it('Should succeed with the correct parameters', async function () {
+    await server.storyboard.list({ id: privateVideo.uuid })
+    await server.storyboard.list({ id: publicVideo.uuid })
+  })
+
+  after(async function () {
+    await cleanupTests([ server ])
+  })
+})

server/tests/api/check-params/videos-overviews.ts

@@ -2,7 +2,7 @@
 
 import { cleanupTests, createSingleServer, PeerTubeServer } from '@shared/server-commands'
 
-describe('Test videos overview', function () {
+describe('Test videos overview API validator', function () {
   let server: PeerTubeServer
 
   // ---------------------------------------------------------------

server/tests/api/server/config.ts

@@ -46,6 +46,7 @@ function checkInitialConfig (server: PeerTubeServer, data: CustomConfig) {
   expect(data.cache.previews.size).to.equal(1)
   expect(data.cache.captions.size).to.equal(1)
   expect(data.cache.torrents.size).to.equal(1)
+  expect(data.cache.storyboards.size).to.equal(1)
 
   expect(data.signup.enabled).to.be.true
   expect(data.signup.limit).to.equal(4)
@@ -154,6 +155,7 @@ function checkUpdatedConfig (data: CustomConfig) {
   expect(data.cache.previews.size).to.equal(2)
   expect(data.cache.captions.size).to.equal(3)
   expect(data.cache.torrents.size).to.equal(4)
+  expect(data.cache.storyboards.size).to.equal(5)
 
   expect(data.signup.enabled).to.be.false
   expect(data.signup.limit).to.equal(5)
@@ -290,6 +292,9 @@ const newCustomConfig: CustomConfig = {
     },
     torrents: {
       size: 4
+    },
+    storyboards: {
+      size: 5
     }
   },
   signup: {

server/tests/api/videos/index.ts

@@ -20,3 +20,4 @@ import './videos-history'
 import './videos-overview'
 import './video-source'
 import './video-static-file-privacy'
+import './video-storyboard'

server/tests/api/videos/video-storyboard.ts

@@ -0,0 +1,184 @@
+/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
+
+import { expect } from 'chai'
+import { FIXTURE_URLS } from '@server/tests/shared'
+import { areHttpImportTestsDisabled } from '@shared/core-utils'
+import { HttpStatusCode, VideoPrivacy } from '@shared/models'
+import {
+  cleanupTests,
+  createMultipleServers,
+  doubleFollow,
+  makeGetRequest,
+  PeerTubeServer,
+  sendRTMPStream,
+  setAccessTokensToServers,
+  setDefaultVideoChannel,
+  stopFfmpeg,
+  waitJobs
+} from '@shared/server-commands'
+
+async function checkStoryboard (options: {
+  server: PeerTubeServer
+  uuid: string
+  tilesCount?: number
+  minSize?: number
+}) {
+  const { server, uuid, tilesCount, minSize = 1000 } = options
+
+  const { storyboards } = await server.storyboard.list({ id: uuid })
+
+  expect(storyboards).to.have.lengthOf(1)
+
+  const storyboard = storyboards[0]
+
+  expect(storyboard.spriteDuration).to.equal(1)
+  expect(storyboard.spriteHeight).to.equal(108)
+  expect(storyboard.spriteWidth).to.equal(192)
+  expect(storyboard.storyboardPath).to.exist
+
+  if (tilesCount) {
+    expect(storyboard.totalWidth).to.equal(192 * Math.min(tilesCount, 10))
+    expect(storyboard.totalHeight).to.equal(108 * Math.max((tilesCount / 10), 1))
+  }
+
+  const { body } = await makeGetRequest({ url: server.url, path: storyboard.storyboardPath, expectedStatus: HttpStatusCode.OK_200 })
+  expect(body.length).to.be.above(minSize)
+}
+
+describe('Test video storyboard', function () {
+  let servers: PeerTubeServer[]
+
+  before(async function () {
+    this.timeout(120000)
+
+    servers = await createMultipleServers(2)
+    await setAccessTokensToServers(servers)
+    await setDefaultVideoChannel(servers)
+
+    await doubleFollow(servers[0], servers[1])
+  })
+
+  it('Should generate a storyboard after upload without transcoding', async function () {
+    this.timeout(60000)
+
+    // 5s video
+    const { uuid } = await servers[0].videos.quickUpload({ name: 'upload', fixture: 'video_short.webm' })
+    await waitJobs(servers)
+
+    for (const server of servers) {
+      await checkStoryboard({ server, uuid, tilesCount: 5 })
+    }
+  })
+
+  it('Should generate a storyboard after upload without transcoding with a long video', async function () {
+    this.timeout(60000)
+
+    // 124s video
+    const { uuid } = await servers[0].videos.quickUpload({ name: 'upload', fixture: 'video_very_long_10p.mp4' })
+    await waitJobs(servers)
+
+    for (const server of servers) {
+      await checkStoryboard({ server, uuid, tilesCount: 100 })
+    }
+  })
+
+  it('Should generate a storyboard after upload with transcoding', async function () {
+    this.timeout(60000)
+
+    await servers[0].config.enableMinimumTranscoding()
+
+    // 5s video
+    const { uuid } = await servers[0].videos.quickUpload({ name: 'upload', fixture: 'video_short.webm' })
+    await waitJobs(servers)
+
+    for (const server of servers) {
+      await checkStoryboard({ server, uuid, tilesCount: 5 })
+    }
+  })
+
+  it('Should generate a storyboard after an audio upload', async function () {
+    this.timeout(60000)
+
+    // 6s audio
+    const attributes = { name: 'audio', fixture: 'sample.ogg' }
+    const { uuid } = await servers[0].videos.upload({ attributes, mode: 'legacy' })
+    await waitJobs(servers)
+
+    for (const server of servers) {
+      await checkStoryboard({ server, uuid, tilesCount: 6, minSize: 250 })
+    }
+  })
+
+  it('Should generate a storyboard after HTTP import', async function () {
+    this.timeout(60000)
+
+    if (areHttpImportTestsDisabled()) return
+
+    // 3s video
+    const { video } = await servers[0].imports.importVideo({
+      attributes: {
+        targetUrl: FIXTURE_URLS.goodVideo,
+        channelId: servers[0].store.channel.id,
+        privacy: VideoPrivacy.PUBLIC
+      }
+    })
+    await waitJobs(servers)
+
+    for (const server of servers) {
+      await checkStoryboard({ server, uuid: video.uuid, tilesCount: 3 })
+    }
+  })
+
+  it('Should generate a storyboard after torrent import', async function () {
+    this.timeout(60000)
+
+    if (areHttpImportTestsDisabled()) return
+
+    // 10s video
+    const { video } = await servers[0].imports.importVideo({
+      attributes: {
+        magnetUri: FIXTURE_URLS.magnet,
+        channelId: servers[0].store.channel.id,
+        privacy: VideoPrivacy.PUBLIC
+      }
+    })
+    await waitJobs(servers)
+
+    for (const server of servers) {
+      await checkStoryboard({ server, uuid: video.uuid, tilesCount: 10 })
+    }
+  })
+
+  it('Should generate a storyboard after a live', async function () {
+    this.timeout(240000)
+
+    await servers[0].config.enableLive({ allowReplay: true, transcoding: true, resolutions: 'min' })
+
+    const { live, video } = await servers[0].live.quickCreate({
+      saveReplay: true,
+      permanentLive: false,
+      privacy: VideoPrivacy.PUBLIC
+    })
+
+    const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey })
+    await servers[0].live.waitUntilPublished({ videoId: video.id })
+
+    await stopFfmpeg(ffmpegCommand)
+
+    await servers[0].live.waitUntilReplacedByReplay({ videoId: video.id })
+    await waitJobs(servers)
+
+    for (const server of servers) {
+      await checkStoryboard({ server, uuid: video.uuid })
+    }
+  })
+
+  it('Should generate a storyboard with different video durations', async function () {
+    this.timeout(60000)
+
+  })
+
+  after(async function () {
+    await cleanupTests(servers)
+  })
+})

server/types/models/video/index.ts

@@ -1,6 +1,7 @@
 export * from './local-video-viewer-watch-section'
 export * from './local-video-viewer-watch-section'
 export * from './local-video-viewer'
+export * from './storyboard'
 export * from './schedule-video-update'
 export * from './tag'
 export * from './thumbnail'

server/types/models/video/storyboard.ts

@@ -0,0 +1,15 @@
+import { StoryboardModel } from '@server/models/video/storyboard'
+import { PickWith } from '@shared/typescript-utils'
+import { MVideo } from './video'
+
+type Use<K extends keyof StoryboardModel, M> = PickWith<StoryboardModel, K, M>
+
+// ############################################################################
+
+export type MStoryboard = Omit<StoryboardModel, 'Video'>
+
+// ############################################################################
+
+export type MStoryboardVideo =
+  MStoryboard &
+  Use<'Video', MVideo>

server/types/models/video/video-caption.ts

@@ -11,7 +11,7 @@ export type MVideoCaption = Omit<VideoCaptionModel, 'Video'>
 // ############################################################################
 
 export type MVideoCaptionLanguage = Pick<MVideoCaption, 'language'>
-export type MVideoCaptionLanguageUrl = Pick<MVideoCaption, 'language' | 'fileUrl' | 'getFileUrl'>
+export type MVideoCaptionLanguageUrl = Pick<MVideoCaption, 'language' | 'fileUrl' | 'filename' | 'getFileUrl' | 'getCaptionStaticPath'>
 
 export type MVideoCaptionVideo =
   MVideoCaption &

server/types/models/video/video.ts

@@ -3,6 +3,7 @@ import { VideoModel } from '../../../models/video/video'
 import { MTrackerUrl } from '../server/tracker'
 import { MUserVideoHistoryTime } from '../user/user-video-history'
 import { MScheduleVideoUpdate } from './schedule-video-update'
+import { MStoryboard } from './storyboard'
 import { MTag } from './tag'
 import { MThumbnail } from './thumbnail'
 import { MVideoBlacklist, MVideoBlacklistLight, MVideoBlacklistUnfederated } from './video-blacklist'
@@ -32,7 +33,7 @@ type Use<K extends keyof VideoModel, M> = PickWith<VideoModel, K, M>
 export type MVideo =
   Omit<VideoModel, 'VideoChannel' | 'Tags' | 'Thumbnails' | 'VideoPlaylistElements' | 'VideoAbuses' |
   'VideoFiles' | 'VideoStreamingPlaylists' | 'VideoShares' | 'AccountVideoRates' | 'VideoComments' | 'VideoViews' | 'UserVideoHistories' |
-  'ScheduleVideoUpdate' | 'VideoBlacklist' | 'VideoImport' | 'VideoCaptions' | 'VideoLive' | 'Trackers' | 'VideoPasswords'>
+  'ScheduleVideoUpdate' | 'VideoBlacklist' | 'VideoImport' | 'VideoCaptions' | 'VideoLive' | 'Trackers' | 'VideoPasswords' | 'Storyboard'>
 
 // ############################################################################
 
@@ -173,9 +174,10 @@ export type MVideoAP =
   Use<'VideoBlacklist', MVideoBlacklistUnfederated> &
   Use<'VideoFiles', MVideoFileRedundanciesOpt[]> &
   Use<'Thumbnails', MThumbnail[]> &
-  Use<'VideoLive', MVideoLive>
+  Use<'VideoLive', MVideoLive> &
+  Use<'Storyboard', MStoryboard>
 
-export type MVideoAPWithoutCaption = Omit<MVideoAP, 'VideoCaptions'>
+export type MVideoAPLight = Omit<MVideoAP, 'VideoCaptions' | 'Storyboard'>
 
 export type MVideoDetails =
   MVideo &

shared/ffmpeg/ffmpeg-images.ts

@@ -56,4 +56,41 @@ export class FFmpegImage {
         .thumbnail(thumbnailOptions)
     })
   }
+
+  async generateStoryboardFromVideo (options: {
+    path: string
+    destination: string
+
+    sprites: {
+      size: {
+        width: number
+        height: number
+      }
+
+      count: {
+        width: number
+        height: number
+      }
+
+      duration: number
+    }
+  }) {
+    const { path, destination, sprites } = options
+
+    const command = this.commandWrapper.buildCommand(path)
+
+    const filter = [
+      `setpts=N/round(FRAME_RATE)/TB`,
+      `select='not(mod(t,${options.sprites.duration}))'`,
+      `scale=${sprites.size.width}:${sprites.size.height}`,
+      `tile=layout=${sprites.count.width}x${sprites.count.height}`
+    ].join(',')
+
+    command.outputOption('-filter_complex', filter)
+    command.outputOption('-frames:v', '1')
+    command.outputOption('-q:v', '2')
+    command.output(destination)
+
+    return this.commandWrapper.runCommand()
+  }
 }

shared/models/activitypub/objects/index.ts

@@ -6,5 +6,5 @@ export * from './object.model'
 export * from './playlist-element-object'
 export * from './playlist-object'
 export * from './video-comment-object'
-export * from './video-torrent-object'
+export * from './video-object'
 export * from './watch-action-object'

shared/models/activitypub/objects/video-object.ts

@@ -51,6 +51,22 @@ export interface VideoObject {
 
   attributedTo: ActivityPubAttributedTo[]
 
+  preview?: ActivityPubStoryboard[]
+
   to?: string[]
   cc?: string[]
 }
+
+export interface ActivityPubStoryboard {
+  type: 'Image'
+  rel: [ 'storyboard' ]
+  url: {
+    href: string
+    mediaType: string
+    width: number
+    height: number
+    tileWidth: number
+    tileHeight: number
+    tileDuration: string
+  }[]
+}

shared/models/server/custom-config.model.ts

@@ -78,6 +78,10 @@ export interface CustomConfig {
     torrents: {
       size: number
     }
+
+    storyboards: {
+      size: number
+    }
   }
 
   signup: {

shared/models/server/job.model.ts

@@ -30,6 +30,7 @@ export type JobType =
   | 'video-studio-edition'
   | 'video-transcoding'
   | 'videos-views-stats'
+  | 'generate-video-storyboard'
 
 export interface Job {
   id: number | string
@@ -294,3 +295,10 @@ export interface TranscodingJobBuilderPayload {
     priority?: number
   }[][]
 }
+
+// ---------------------------------------------------------------------------
+
+export interface GenerateStoryboardPayload {
+  videoUUID: string
+  federate: boolean
+}

shared/models/videos/index.ts

@@ -15,6 +15,7 @@ export * from './channel-sync'
 
 export * from './nsfw-policy.type'
 
+export * from './storyboard.model'
 export * from './thumbnail.type'
 
 export * from './video-constant.model'

shared/models/videos/storyboard.model.ts

@@ -0,0 +1,11 @@
+export interface Storyboard {
+  storyboardPath: string
+
+  totalHeight: number
+  totalWidth: number
+
+  spriteHeight: number
+  spriteWidth: number
+
+  spriteDuration: number
+}

shared/server-commands/server/config-command.ts

@@ -159,6 +159,10 @@ export class ConfigCommand extends AbstractCommand {
       newConfig: {
         transcoding: {
           enabled: true,
+
+          allowAudioFiles: true,
+          allowAdditionalExtensions: true,
+
           resolutions: {
             ...ConfigCommand.getCustomConfigResolutions(false),
 
@@ -368,6 +372,9 @@ export class ConfigCommand extends AbstractCommand {
         },
         torrents: {
           size: 4
+        },
+        storyboards: {
+          size: 5
         }
       },
       signup: {

shared/server-commands/server/jobs.ts

@@ -33,6 +33,8 @@ async function waitJobs (
 
     // Check if each server has pending request
     for (const server of servers) {
+      if (process.env.DEBUG) console.log('Checking ' + server.url)
+
       for (const state of states) {
 
         const jobPromise = server.jobs.list({
@@ -45,6 +47,10 @@ async function waitJobs (
           .then(jobs => {
             if (jobs.length !== 0) {
               pendingRequests = true
+
+              if (process.env.DEBUG) {
+                console.log(jobs)
+              }
             }
           })
 
@@ -55,6 +61,10 @@ async function waitJobs (
         .then(obj => {
           if (obj.activityPubMessagesWaiting !== 0) {
             pendingRequests = true
+
+            if (process.env.DEBUG) {
+              console.log('AP messages waiting: ' + obj.activityPubMessagesWaiting)
+            }
           }
         })
       tasks.push(debugPromise)
@@ -65,12 +75,15 @@ async function waitJobs (
             for (const job of data) {
               if (job.state.id !== RunnerJobState.COMPLETED) {
                 pendingRequests = true
+
+                if (process.env.DEBUG) {
+                  console.log(job)
+                }
               }
             }
           })
         tasks.push(runnerJobsPromise)
       }
-
     }
 
     return tasks

shared/server-commands/server/server.ts

@@ -35,6 +35,7 @@ import {
   VideoPasswordsCommand,
   PlaylistsCommand,
   ServicesCommand,
+  StoryboardCommand,
   StreamingPlaylistsCommand,
   VideosCommand,
   VideoStudioCommand,
@@ -149,6 +150,8 @@ export class PeerTubeServer {
   registrations?: RegistrationsCommand
   videoPasswords?: VideoPasswordsCommand
 
+  storyboard?: StoryboardCommand
+
   runners?: RunnersCommand
   runnerRegistrationTokens?: RunnerRegistrationTokensCommand
   runnerJobs?: RunnerJobsCommand
@@ -436,6 +439,8 @@ export class PeerTubeServer {
     this.videoToken = new VideoTokenCommand(this)
     this.registrations = new RegistrationsCommand(this)
 
+    this.storyboard = new StoryboardCommand(this)
+
     this.runners = new RunnersCommand(this)
     this.runnerRegistrationTokens = new RunnerRegistrationTokensCommand(this)
     this.runnerJobs = new RunnerJobsCommand(this)

shared/server-commands/videos/index.ts

@@ -11,6 +11,7 @@ export * from './live-command'
 export * from './live'
 export * from './playlists-command'
 export * from './services-command'
+export * from './storyboard-command'
 export * from './streaming-playlists-command'
 export * from './comments-command'
 export * from './video-studio-command'

shared/server-commands/videos/storyboard-command.ts

@@ -0,0 +1,19 @@
+import { HttpStatusCode, Storyboard } from '@shared/models'
+import { AbstractCommand, OverrideCommandOptions } from '../shared'
+
+export class StoryboardCommand extends AbstractCommand {
+
+  list (options: OverrideCommandOptions & {
+    id: number | string
+  }) {
+    const path = '/api/v1/videos/' + options.id + '/storyboards'
+
+    return this.getRequestBody<{ storyboards: Storyboard[] }>({
+      ...options,
+
+      path,
+      implicitToken: true,
+      defaultExpectedStatus: HttpStatusCode.OK_200
+    })
+  }
+}

support/doc/api/openapi.yaml

@@ -3668,6 +3668,27 @@ paths:
                     items:
                       $ref: '#/components/schemas/VideoBlacklist'
 
+  /api/v1/videos/{id}/storyboards:
+    get:
+      summary: List storyboards of a video
+      operationId: listVideoStoryboards
+      tags:
+        - Video
+      parameters:
+        - $ref: '#/components/parameters/idOrUUID'
+      responses:
+        '200':
+          description: successful operation
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  storyboards:
+                    type: array
+                    items:
+                      $ref: '#/components/schemas/Storyboard'
+
   /api/v1/videos/{id}/captions:
     get:
       summary: List captions of a video
@@ -7509,6 +7530,20 @@ components:
           type: array
           items:
             $ref: '#/components/schemas/VideoCommentThreadTree'
+    Storyboard:
+      properties:
+        storyboardPath:
+          type: string
+        totalHeight:
+          type: integer
+        totalWidth:
+          type: integer
+        spriteHeight:
+          type: integer
+        spriteWidth:
+          type: integer
+        spriteDuration:
+          type: integer
     VideoCaption:
       properties:
         language: