From d61893f7236abbed30c25b1823e6ecad93a8e8dd Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 25 Feb 2021 13:56:07 +0100 Subject: [PATCH] Async torrent creation --- .../player/webtorrent/webtorrent-plugin.ts | 2 + server/controllers/api/videos/index.ts | 38 +++++++++++++++---- server/models/video/video-file.ts | 8 ++++ server/models/video/video-format-utils.ts | 2 +- shared/extra-utils/videos/videos.ts | 27 +++++++++++-- 5 files changed, 66 insertions(+), 11 deletions(-) diff --git a/client/src/assets/player/webtorrent/webtorrent-plugin.ts b/client/src/assets/player/webtorrent/webtorrent-plugin.ts index 4bb438e57..e557fe722 100644 --- a/client/src/assets/player/webtorrent/webtorrent-plugin.ts +++ b/client/src/assets/player/webtorrent/webtorrent-plugin.ts @@ -249,6 +249,8 @@ class WebTorrentPlugin extends Plugin { options: PlayOptions, done: Function ) { + if (!magnetOrTorrentUrl) return this.fallbackToHttp(options, done) + console.log('Adding ' + magnetOrTorrentUrl + '.') const oldTorrent = this.torrent diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts index e89315930..58ab72370 100644 --- a/server/controllers/api/videos/index.ts +++ b/server/controllers/api/videos/index.ts @@ -9,12 +9,12 @@ import { LiveManager } from '@server/lib/live-manager' import { addOptimizeOrMergeAudioJob, buildLocalVideoFromReq, buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video' import { generateVideoFilename, getVideoFilePath } from '@server/lib/video-paths' import { getServerActor } from '@server/models/application/application' -import { MVideoFullLight } from '@server/types/models' +import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' import { VideoCreate, VideoState, VideoUpdate } from '../../../../shared' import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes' import { VideoFilter } from '../../../../shared/models/videos/video-query.type' import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger' -import { resetSequelizeInstance } from '../../../helpers/database-utils' +import { resetSequelizeInstance, retryTransactionWrapper } from '../../../helpers/database-utils' import { buildNSFWFilter, createReqFiles, getCountVideos } from '../../../helpers/express-utils' import { getMetadataFromFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffprobe-utils' import { logger } from '../../../helpers/logger' @@ -221,9 +221,6 @@ async function addVideo (req: express.Request, res: express.Response) { fallback: type => generateVideoMiniature({ video, videoFile, type }) }) - // Create the torrent file - await createTorrentAndSetInfoHash(video, videoFile) - const { videoCreated } = await sequelizeTypescript.transaction(async t => { const sequelizeOptions = { transaction: t } @@ -258,7 +255,6 @@ async function addVideo (req: express.Request, res: express.Response) { isNew: true, transaction: t }) - await federateVideoIfNeeded(video, true, t) auditLogger.create(getAuditIdFromRes(res), new VideoAuditView(videoCreated.toFormattedDetailsJSON())) logger.info('Video with name %s and uuid %s created.', videoInfo.name, videoCreated.uuid) @@ -266,7 +262,21 @@ async function addVideo (req: express.Request, res: express.Response) { return { videoCreated } }) - Notifier.Instance.notifyOnNewVideoIfNeeded(videoCreated) + // Create the torrent file in async way because it could be long + createTorrentAndSetInfoHashAsync(video, videoFile) + .catch(err => logger.error('Cannot create torrent file for video %s', video.url, { err })) + .then(() => VideoModel.loadAndPopulateAccountAndServerAndTags(video.id)) + .then(refreshedVideo => { + if (!refreshedVideo) return + + // Only federate and notify after the torrent creation + Notifier.Instance.notifyOnNewVideoIfNeeded(refreshedVideo) + + return retryTransactionWrapper(() => { + return sequelizeTypescript.transaction(t => federateVideoIfNeeded(refreshedVideo, true, t)) + }) + }) + .catch(err => logger.error('Cannot federate or notify video creation %s', video.url, { err })) if (video.state === VideoState.TO_TRANSCODE) { await addOptimizeOrMergeAudioJob(videoCreated, videoFile, res.locals.oauth.token.User) @@ -526,3 +536,17 @@ async function removeVideo (req: express.Request, res: express.Response) { .status(HttpStatusCode.NO_CONTENT_204) .end() } + +async function createTorrentAndSetInfoHashAsync (video: MVideo, fileArg: MVideoFile) { + await createTorrentAndSetInfoHash(video, fileArg) + + // Refresh videoFile because the createTorrentAndSetInfoHash could be long + const refreshedFile = await VideoFileModel.loadWithVideo(fileArg.id) + // File does not exist anymore, remove the generated torrent + if (!refreshedFile) return fileArg.removeTorrent() + + refreshedFile.infoHash = fileArg.infoHash + refreshedFile.torrentFilename = fileArg.torrentFilename + + return refreshedFile.save() +} diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts index 4df2c20bc..1ad796104 100644 --- a/server/models/video/video-file.ts +++ b/server/models/video/video-file.ts @@ -457,18 +457,26 @@ export class VideoFileModel extends Model { // We proxify torrent requests so use a local URL getTorrentUrl () { + if (!this.torrentFilename) return null + return WEBSERVER.URL + this.getTorrentStaticPath() } getTorrentStaticPath () { + if (!this.torrentFilename) return null + return join(LAZY_STATIC_PATHS.TORRENTS, this.torrentFilename) } getTorrentDownloadUrl () { + if (!this.torrentFilename) return null + return WEBSERVER.URL + join(STATIC_DOWNLOAD_PATHS.TORRENTS, this.torrentFilename) } removeTorrent () { + if (!this.torrentFilename) return null + const torrentPath = getTorrentFilePath(this) return remove(torrentPath) .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err })) diff --git a/server/models/video/video-format-utils.ts b/server/models/video/video-format-utils.ts index a6a1a4f0d..bcba90093 100644 --- a/server/models/video/video-format-utils.ts +++ b/server/models/video/video-format-utils.ts @@ -205,7 +205,7 @@ function videoFilesModelToFormattedJSON ( label: videoFile.resolution + 'p' }, - magnetUri: includeMagnet + magnetUri: includeMagnet && videoFile.torrentFilename ? generateMagnetUri(video, videoFile, trackerUrls) : undefined, diff --git a/shared/extra-utils/videos/videos.ts b/shared/extra-utils/videos/videos.ts index 929eb42ca..0b6a54046 100644 --- a/shared/extra-utils/videos/videos.ts +++ b/shared/extra-utils/videos/videos.ts @@ -1,6 +1,5 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/no-floating-promises */ -import { HttpStatusCode } from '@shared/core-utils' import { expect } from 'chai' import { pathExists, readdir, readFile } from 'fs-extra' import * as parseTorrent from 'parse-torrent' @@ -8,9 +7,18 @@ import { extname, join } from 'path' import * as request from 'supertest' import { v4 as uuidv4 } from 'uuid' import validator from 'validator' +import { HttpStatusCode } from '@shared/core-utils' import { loadLanguages, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES, VIDEO_PRIVACIES } from '../../../server/initializers/constants' import { VideoDetails, VideoPrivacy } from '../../models/videos' -import { buildAbsoluteFixturePath, buildServerDirectory, dateIsValid, immutableAssign, testImage, webtorrentAdd } from '../miscs/miscs' +import { + buildAbsoluteFixturePath, + buildServerDirectory, + dateIsValid, + immutableAssign, + testImage, + wait, + webtorrentAdd +} from '../miscs/miscs' import { makeGetRequest, makePutBodyRequest, makeRawRequest, makeUploadRequest } from '../requests/requests' import { waitJobs } from '../server/jobs' import { ServerInfo } from '../server/servers' @@ -423,8 +431,21 @@ async function uploadVideo (url: string, accessToken: string, videoAttributesArg req.field('originallyPublishedAt', attributes.originallyPublishedAt) } - return req.attach('videofile', buildAbsoluteFixturePath(attributes.fixture)) + const res = await req.attach('videofile', buildAbsoluteFixturePath(attributes.fixture)) .expect(specialStatus) + + // Wait torrent generation + if (specialStatus === HttpStatusCode.OK_200) { + let video: VideoDetails + do { + const resVideo = await getVideoWithToken(url, accessToken, res.body.video.uuid) + video = resVideo.body + + await wait(50) + } while (!video.files[0].torrentUrl) + } + + return res } function updateVideo (