diff --git a/server/lib/hls.ts b/server/lib/hls.ts index 32b02bc26..0e77ab9fa 100644 --- a/server/lib/hls.ts +++ b/server/lib/hls.ts @@ -1,4 +1,4 @@ -import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, writeFile } from 'fs-extra' +import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, stat, writeFile } from 'fs-extra' import { flatten, uniq } from 'lodash' import { basename, dirname, join } from 'path' import { MStreamingPlaylistFilesVideo, MVideoWithFile } from '@server/types/models' @@ -108,8 +108,9 @@ async function buildSha256Segment (segmentPath: string) { return sha256(buf) } -function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number) { +function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number, bodyKBLimit: number) { let timer + let remainingBodyKBLimit = bodyKBLimit logger.info('Importing HLS playlist %s', playlistUrl) @@ -136,8 +137,12 @@ function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, for (const fileUrl of fileUrls) { const destPath = join(tmpDirectory, basename(fileUrl)) - const bodyKBLimit = 10 * 1000 * 1000 // 10GB - await doRequestAndSaveToFile(fileUrl, destPath, { bodyKBLimit }) + await doRequestAndSaveToFile(fileUrl, destPath, { bodyKBLimit: remainingBodyKBLimit }) + + const { size } = await stat(destPath) + remainingBodyKBLimit -= (size / 1000) + + logger.debug('Downloaded HLS playlist file %s with %d kB remained limit.', fileUrl, Math.floor(remainingBodyKBLimit)) } clearTimeout(timer) diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index 103ab1fab..137ae53a0 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts @@ -4,9 +4,7 @@ import { getServerActor } from '@server/models/application/application' import { TrackerModel } from '@server/models/server/tracker' import { VideoModel } from '@server/models/video/video' import { - MStreamingPlaylist, MStreamingPlaylistFiles, - MStreamingPlaylistVideo, MVideoAccountLight, MVideoFile, MVideoFileVideo, @@ -249,7 +247,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { private async createStreamingPlaylistRedundancy ( redundancy: VideosRedundancyStrategy, video: MVideoAccountLight, - playlistArg: MStreamingPlaylist + playlistArg: MStreamingPlaylistFiles ) { let strategy = 'manual' let expiresOn: Date = null @@ -259,16 +257,17 @@ export class VideosRedundancyScheduler extends AbstractScheduler { expiresOn = this.buildNewExpiration(redundancy.minLifetime) } - const playlist = playlistArg as MStreamingPlaylistVideo - playlist.Video = video - + const playlist = Object.assign(playlistArg, { Video: video }) const serverActor = await getServerActor() logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, strategy) const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid) const masterPlaylistUrl = playlist.getMasterPlaylistUrl(video) - await downloadPlaylistSegments(masterPlaylistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT) + + const maxSizeKB = this.getTotalFileSizes([], [ playlist ]) / 1000 + const toleranceKB = maxSizeKB + ((5 * maxSizeKB) / 100) // 5% more tolerance + await downloadPlaylistSegments(masterPlaylistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT, toleranceKB) const createdModel: MVideoRedundancyStreamingPlaylistVideo = await VideoRedundancyModel.create({ expiresOn, @@ -334,7 +333,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { return `${object.VideoStreamingPlaylist.getMasterPlaylistUrl(object.VideoStreamingPlaylist.Video)}` } - private getTotalFileSizes (files: MVideoFile[], playlists: MStreamingPlaylistFiles[]) { + private getTotalFileSizes (files: MVideoFile[], playlists: MStreamingPlaylistFiles[]): number { const fileReducer = (previous: number, current: MVideoFile) => previous + current.size let allFiles = files