interesting
/
PeerTube
mirror of https://github.com/Chocobozzz/PeerTube.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Correctly handle large HLS files for redundancy

This commit is contained in:
Chocobozzz 2021-08-06 15:25:45 +02:00
parent a1c63fe1a2
commit 18998c45c0
No known key found for this signature in database
GPG Key ID: 583A612D890159BE
2 changed files with 16 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
server/lib/hls.ts
View File

@ -1,4 +1,4 @@
import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, writeFile } from 'fs-extra'
import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, stat, writeFile } from 'fs-extra'
import { flatten, uniq } from 'lodash'
import { basename, dirname, join } from 'path'
import { MStreamingPlaylistFilesVideo, MVideoWithFile } from '@server/types/models'
@ -108,8 +108,9 @@ async function buildSha256Segment (segmentPath: string) {
return sha256(buf)
}
function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number) {
function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number, bodyKBLimit: number) {
let timer
let remainingBodyKBLimit = bodyKBLimit
logger.info('Importing HLS playlist %s', playlistUrl)
@ -136,8 +137,12 @@ function downloadPlaylistSegments (playlistUrl: string, destinationDir: string,
for (const fileUrl of fileUrls) {
const destPath = join(tmpDirectory, basename(fileUrl))
const bodyKBLimit = 10 * 1000 * 1000 // 10GB
await doRequestAndSaveToFile(fileUrl, destPath, { bodyKBLimit })
await doRequestAndSaveToFile(fileUrl, destPath, { bodyKBLimit: remainingBodyKBLimit })
const { size } = await stat(destPath)
remainingBodyKBLimit -= (size / 1000)
logger.debug('Downloaded HLS playlist file %s with %d kB remained limit.', fileUrl, Math.floor(remainingBodyKBLimit))
}
clearTimeout(timer)

15
server/lib/schedulers/videos-redundancy-scheduler.ts
View File

@ -4,9 +4,7 @@ import { getServerActor } from '@server/models/application/application'
import { TrackerModel } from '@server/models/server/tracker'
import { VideoModel } from '@server/models/video/video'
import {
MStreamingPlaylist,
MStreamingPlaylistFiles,
MStreamingPlaylistVideo,
MVideoAccountLight,
MVideoFile,
MVideoFileVideo,
@ -249,7 +247,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
private async createStreamingPlaylistRedundancy (
redundancy: VideosRedundancyStrategy,
video: MVideoAccountLight,
playlistArg: MStreamingPlaylist
playlistArg: MStreamingPlaylistFiles
) {
let strategy = 'manual'
let expiresOn: Date = null
@ -259,16 +257,17 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
expiresOn = this.buildNewExpiration(redundancy.minLifetime)
}
const playlist = playlistArg as MStreamingPlaylistVideo
playlist.Video = video
const playlist = Object.assign(playlistArg, { Video: video })
const serverActor = await getServerActor()
logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, strategy)
const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid)
const masterPlaylistUrl = playlist.getMasterPlaylistUrl(video)
await downloadPlaylistSegments(masterPlaylistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT)
const maxSizeKB = this.getTotalFileSizes([], [ playlist ]) / 1000
const toleranceKB = maxSizeKB + ((5 * maxSizeKB) / 100) // 5% more tolerance
await downloadPlaylistSegments(masterPlaylistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT, toleranceKB)
const createdModel: MVideoRedundancyStreamingPlaylistVideo = await VideoRedundancyModel.create({
expiresOn,
@ -334,7 +333,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
return `${object.VideoStreamingPlaylist.getMasterPlaylistUrl(object.VideoStreamingPlaylist.Video)}`
}
private getTotalFileSizes (files: MVideoFile[], playlists: MStreamingPlaylistFiles[]) {
private getTotalFileSizes (files: MVideoFile[], playlists: MStreamingPlaylistFiles[]): number {
const fileReducer = (previous: number, current: MVideoFile) => previous + current.size
let allFiles = files