From cf9166cf2fb7b51a1137a259eed9338798c73500 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 1 Oct 2018 12:00:05 +0200 Subject: [PATCH] Add timeout on youtube dl to cleaup files --- server/helpers/webtorrent.ts | 12 +++++------ server/helpers/youtube-dl.ts | 20 ++++++++++++++++--- server/initializers/constants.ts | 4 +++- server/lib/job-queue/handlers/video-import.ts | 6 +++--- .../schedulers/videos-redundancy-scheduler.ts | 5 ++--- 5 files changed, 30 insertions(+), 17 deletions(-) diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts index 924d630e7..ce35b87da 100644 --- a/server/helpers/webtorrent.ts +++ b/server/helpers/webtorrent.ts @@ -5,7 +5,7 @@ import { createWriteStream, ensureDir, remove } from 'fs-extra' import { CONFIG } from '../initializers' import { dirname, join } from 'path' -async function downloadWebTorrentVideo (target: { magnetUri: string, torrentName?: string }, timeout?: number) { +async function downloadWebTorrentVideo (target: { magnetUri: string, torrentName?: string }, timeout: number) { const id = target.magnetUri || target.torrentName let timer @@ -50,12 +50,10 @@ async function downloadWebTorrentVideo (target: { magnetUri: string, torrentName torrent.on('error', err => rej(err)) - if (timeout) { - timer = setTimeout(async () => { - return safeWebtorrentDestroy(webtorrent, torrentId, file ? { directoryPath, filepath: file.path } : undefined, target.torrentName) - .then(() => rej(new Error('Webtorrent download timeout.'))) - }, timeout) - } + timer = setTimeout(async () => { + return safeWebtorrentDestroy(webtorrent, torrentId, file ? { directoryPath, filepath: file.path } : undefined, target.torrentName) + .then(() => rej(new Error('Webtorrent download timeout.'))) + }, timeout) }) } diff --git a/server/helpers/youtube-dl.ts b/server/helpers/youtube-dl.ts index 748c67e6c..70b4e1b78 100644 --- a/server/helpers/youtube-dl.ts +++ b/server/helpers/youtube-dl.ts @@ -4,7 +4,7 @@ import { logger } from './logger' import { generateVideoTmpPath } from './utils' import { join } from 'path' import { root } from './core-utils' -import { ensureDir, writeFile } from 'fs-extra' +import { ensureDir, writeFile, remove } from 'fs-extra' import * as request from 'request' import { createWriteStream } from 'fs' @@ -39,8 +39,9 @@ function getYoutubeDLInfo (url: string, opts?: string[]): Promise }) } -function downloadYoutubeDLVideo (url: string) { +function downloadYoutubeDLVideo (url: string, timeout: number) { const path = generateVideoTmpPath(url) + let timer logger.info('Importing youtubeDL video %s', url) @@ -49,10 +50,23 @@ function downloadYoutubeDLVideo (url: string) { return new Promise(async (res, rej) => { const youtubeDL = await safeGetYoutubeDL() youtubeDL.exec(url, options, processOptions, err => { - if (err) return rej(err) + clearTimeout(timer) + + if (err) { + remove(path) + .catch(err => logger.error('Cannot delete path on YoutubeDL error.', { err })) + + return rej(err) + } return res(path) }) + + timer = setTimeout(async () => { + await remove(path) + + return rej(new Error('YoutubeDL download timeout.')) + }, timeout) }) } diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 09ecedfa5..1a3b52015 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -119,7 +119,7 @@ const JOB_TTL: { [ id in JobType ]: number } = { 'activitypub-follow': 60000 * 10, // 10 minutes 'video-file-import': 1000 * 3600, // 1 hour 'video-file': 1000 * 3600 * 48, // 2 days, transcoding could be long - 'video-import': 1000 * 3600, // 1 hour + 'video-import': 1000 * 3600 * 2, // hours 'email': 60000 * 10, // 10 minutes 'videos-views': undefined // Unlimited } @@ -133,6 +133,7 @@ const BROADCAST_CONCURRENCY = 10 // How many requests in parallel we do in activ const CRAWL_REQUEST_CONCURRENCY = 1 // How many requests in parallel to fetch remote data (likes, shares...) const JOB_REQUEST_TIMEOUT = 3000 // 3 seconds const JOB_COMPLETED_LIFETIME = 60000 * 60 * 24 * 2 // 2 days +const VIDEO_IMPORT_TIMEOUT = 1000 * 3600 // 1 hour // 1 hour let SCHEDULER_INTERVALS_MS = { @@ -700,6 +701,7 @@ export { TORRENT_MIMETYPE_EXT, STATIC_MAX_AGE, STATIC_PATHS, + VIDEO_IMPORT_TIMEOUT, ACTIVITY_PUB, ACTIVITY_PUB_ACTOR_TYPES, THUMBNAILS_SIZE, diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index 8f237dd91..e3f2a276c 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts @@ -6,7 +6,7 @@ import { VideoImportState } from '../../../../shared/models/videos' import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils' import { extname, join } from 'path' import { VideoFileModel } from '../../../models/video/video-file' -import { CONFIG, sequelizeTypescript } from '../../../initializers' +import { CONFIG, sequelizeTypescript, VIDEO_IMPORT_TIMEOUT } from '../../../initializers' import { doRequestAndSaveToFile } from '../../../helpers/requests' import { VideoState } from '../../../../shared' import { JobQueue } from '../index' @@ -65,7 +65,7 @@ async function processTorrentImport (job: Bull.Job, payload: VideoImportTorrentP torrentName: videoImport.torrentName ? getSecureTorrentName(videoImport.torrentName) : undefined, magnetUri: videoImport.magnetUri } - return processFile(() => downloadWebTorrentVideo(target), videoImport, options) + return processFile(() => downloadWebTorrentVideo(target, VIDEO_IMPORT_TIMEOUT), videoImport, options) } async function processYoutubeDLImport (job: Bull.Job, payload: VideoImportYoutubeDLPayload) { @@ -83,7 +83,7 @@ async function processYoutubeDLImport (job: Bull.Job, payload: VideoImportYoutub generatePreview: false } - return processFile(() => downloadYoutubeDLVideo(videoImport.targetUrl), videoImport, options) + return processFile(() => downloadYoutubeDLVideo(videoImport.targetUrl, VIDEO_IMPORT_TIMEOUT), videoImport, options) } async function getVideoImportOrDie (videoImportId: number) { diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index 11ee05a53..432bade1f 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts @@ -1,5 +1,5 @@ import { AbstractScheduler } from './abstract-scheduler' -import { CONFIG, JOB_TTL, REDUNDANCY } from '../../initializers' +import { CONFIG, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers' import { logger } from '../../helpers/logger' import { VideosRedundancy } from '../../../shared/models/redundancy' import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' @@ -9,7 +9,6 @@ import { join } from 'path' import { rename } from 'fs-extra' import { getServerActor } from '../../helpers/utils' import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' -import { VideoModel } from '../../models/video/video' import { getVideoCacheFileActivityPubUrl } from '../activitypub/url' import { removeVideoRedundancy } from '../redundancy' import { getOrCreateVideoAndAccountAndChannel } from '../activitypub' @@ -142,7 +141,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs) - const tmpPath = await downloadWebTorrentVideo({ magnetUri }, JOB_TTL['video-import']) + const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT) const destPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(file)) await rename(tmpPath, destPath)