diff --git a/client/src/app/+admin/overview/videos/video-list.component.html b/client/src/app/+admin/overview/videos/video-list.component.html index fdd682ee2..06b9ab347 100644 --- a/client/src/app/+admin/overview/videos/video-list.component.html +++ b/client/src/app/+admin/overview/videos/video-list.component.html @@ -107,6 +107,11 @@ @@ -117,6 +122,11 @@ diff --git a/client/src/app/+admin/overview/videos/video-list.component.scss b/client/src/app/+admin/overview/videos/video-list.component.scss index dcd41a1b4..d538ca30a 100644 --- a/client/src/app/+admin/overview/videos/video-list.component.scss +++ b/client/src/app/+admin/overview/videos/video-list.component.scss @@ -13,6 +13,13 @@ my-embed { .video-info > div { display: flex; + + my-global-icon { + width: 16px; + margin-left: 3px; + position: relative; + top: -2px; + } } .loading { diff --git a/client/src/app/+admin/overview/videos/video-list.component.ts b/client/src/app/+admin/overview/videos/video-list.component.ts index 67e52d100..ed7ec54a1 100644 --- a/client/src/app/+admin/overview/videos/video-list.component.ts +++ b/client/src/app/+admin/overview/videos/video-list.component.ts @@ -8,7 +8,7 @@ import { AdvancedInputFilter } from '@app/shared/shared-forms' import { DropdownAction, Video, VideoService } from '@app/shared/shared-main' import { VideoBlockComponent, VideoBlockService } from '@app/shared/shared-moderation' import { VideoActionsDisplayType } from '@app/shared/shared-video-miniature' -import { UserRight, VideoPrivacy, VideoState, VideoStreamingPlaylistType } from '@shared/models' +import { UserRight, VideoFile, VideoPrivacy, VideoState, VideoStreamingPlaylistType } from '@shared/models' import { VideoAdminService } from './video-admin.service' @Component({ @@ -196,6 +196,22 @@ export class VideoListComponent extends RestTable implements OnInit { }) } + async removeVideoFile (video: Video, file: VideoFile, type: 'hls' | 'webtorrent') { + const message = $localize`Are you sure you want to delete this ${file.resolution.label} file?` + const res = await this.confirmService.confirm(message, $localize`Delete file`) + if (res === false) return + + this.videoService.removeFile(video.uuid, file.id, type) + .subscribe({ + next: () => { + this.notifier.success($localize`File removed.`) + this.reloadData() + }, + + error: err => this.notifier.error(err.message) + }) + } + private async removeVideos (videos: Video[]) { const message = prepareIcu($localize`Are you sure you want to delete {count, plural, =1 {this video} other {these {count} videos}}?`)( { count: videos.length }, diff --git a/client/src/app/shared/shared-main/video/video.service.ts b/client/src/app/shared/shared-main/video/video.service.ts index f2bf02695..8c8b1e08f 100644 --- a/client/src/app/shared/shared-main/video/video.service.ts +++ b/client/src/app/shared/shared-main/video/video.service.ts @@ -305,6 +305,11 @@ export class VideoService { ) } + removeFile (videoId: number | string, fileId: number, type: 'hls' | 'webtorrent') { + return this.authHttp.delete(VideoService.BASE_VIDEO_URL + '/' + videoId + '/' + type + '/' + fileId) + .pipe(catchError(err => this.restExtractor.handleError(err))) + } + runTranscoding (videoIds: (number | string)[], type: 'hls' | 'webtorrent') { const body: VideoTranscodingCreate = { transcodingType: type } diff --git a/package.json b/package.json index db433bfc2..a527a1880 100644 --- a/package.json +++ b/package.json @@ -150,6 +150,7 @@ "node-media-server": "^2.1.4", "nodemailer": "^6.0.0", "opentelemetry-instrumentation-sequelize": "^0.29.0", + "p-queue": "^6", "parse-torrent": "^9.1.0", "password-generator": "^2.0.2", "pg": "^8.2.1", diff --git a/server/controllers/api/videos/files.ts b/server/controllers/api/videos/files.ts index 0fbda280e..6d9c0b843 100644 --- a/server/controllers/api/videos/files.ts +++ b/server/controllers/api/videos/files.ts @@ -2,6 +2,7 @@ import express from 'express' import toInt from 'validator/lib/toInt' import { logger, loggerTagsFactory } from '@server/helpers/logger' import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' +import { removeAllWebTorrentFiles, removeHLSFile, removeHLSPlaylist, removeWebTorrentFile } from '@server/lib/video-file' import { VideoFileModel } from '@server/models/video/video-file' import { HttpStatusCode, UserRight } from '@shared/models' import { @@ -9,10 +10,13 @@ import { authenticate, ensureUserHasRight, videoFileMetadataGetValidator, + videoFilesDeleteHLSFileValidator, videoFilesDeleteHLSValidator, + videoFilesDeleteWebTorrentFileValidator, videoFilesDeleteWebTorrentValidator, videosGetValidator } from '../../../middlewares' +import { updatePlaylistAfterFileChange } from '@server/lib/hls' const lTags = loggerTagsFactory('api', 'video') const filesRouter = express.Router() @@ -27,14 +31,26 @@ filesRouter.delete('/:id/hls', authenticate, ensureUserHasRight(UserRight.MANAGE_VIDEO_FILES), asyncMiddleware(videoFilesDeleteHLSValidator), - asyncMiddleware(removeHLSPlaylist) + asyncMiddleware(removeHLSPlaylistController) +) +filesRouter.delete('/:id/hls/:videoFileId', + authenticate, + ensureUserHasRight(UserRight.MANAGE_VIDEO_FILES), + asyncMiddleware(videoFilesDeleteHLSFileValidator), + asyncMiddleware(removeHLSFileController) ) filesRouter.delete('/:id/webtorrent', authenticate, ensureUserHasRight(UserRight.MANAGE_VIDEO_FILES), asyncMiddleware(videoFilesDeleteWebTorrentValidator), - asyncMiddleware(removeWebTorrentFiles) + asyncMiddleware(removeAllWebTorrentFilesController) +) +filesRouter.delete('/:id/webtorrent/:videoFileId', + authenticate, + ensureUserHasRight(UserRight.MANAGE_VIDEO_FILES), + asyncMiddleware(videoFilesDeleteWebTorrentFileValidator), + asyncMiddleware(removeWebTorrentFileController) ) // --------------------------------------------------------------------------- @@ -51,33 +67,53 @@ async function getVideoFileMetadata (req: express.Request, res: express.Response return res.json(videoFile.metadata) } -async function removeHLSPlaylist (req: express.Request, res: express.Response) { +// --------------------------------------------------------------------------- + +async function removeHLSPlaylistController (req: express.Request, res: express.Response) { const video = res.locals.videoAll logger.info('Deleting HLS playlist of %s.', video.url, lTags(video.uuid)) - - const hls = video.getHLSPlaylist() - await video.removeStreamingPlaylistFiles(hls) - await hls.destroy() - - video.VideoStreamingPlaylists = video.VideoStreamingPlaylists.filter(p => p.id !== hls.id) + await removeHLSPlaylist(video) await federateVideoIfNeeded(video, false, undefined) return res.sendStatus(HttpStatusCode.NO_CONTENT_204) } -async function removeWebTorrentFiles (req: express.Request, res: express.Response) { +async function removeHLSFileController (req: express.Request, res: express.Response) { + const video = res.locals.videoAll + const videoFileId = +req.params.videoFileId + + logger.info('Deleting HLS file %d of %s.', videoFileId, video.url, lTags(video.uuid)) + + const playlist = await removeHLSFile(video, videoFileId) + if (playlist) await updatePlaylistAfterFileChange(video, playlist) + + await federateVideoIfNeeded(video, false, undefined) + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) +} + +// --------------------------------------------------------------------------- + +async function removeAllWebTorrentFilesController (req: express.Request, res: express.Response) { const video = res.locals.videoAll logger.info('Deleting WebTorrent files of %s.', video.url, lTags(video.uuid)) - for (const file of video.VideoFiles) { - await video.removeWebTorrentFileAndTorrent(file) - await file.destroy() - } - - video.VideoFiles = [] + await removeAllWebTorrentFiles(video) + await federateVideoIfNeeded(video, false, undefined) + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) +} + +async function removeWebTorrentFileController (req: express.Request, res: express.Response) { + const video = res.locals.videoAll + + const videoFileId = +req.params.videoFileId + logger.info('Deleting WebTorrent file %d of %s.', videoFileId, video.url, lTags(video.uuid)) + + await removeWebTorrentFile(video, videoFileId) await federateVideoIfNeeded(video, false, undefined) return res.sendStatus(HttpStatusCode.NO_CONTENT_204) diff --git a/server/helpers/ffmpeg/ffprobe-utils.ts b/server/helpers/ffmpeg/ffprobe-utils.ts index a9b4fb456..9529162eb 100644 --- a/server/helpers/ffmpeg/ffprobe-utils.ts +++ b/server/helpers/ffmpeg/ffprobe-utils.ts @@ -1,15 +1,15 @@ import { FfprobeData } from 'fluent-ffmpeg' import { getMaxBitrate } from '@shared/core-utils' import { + buildFileMetadata, ffprobePromise, getAudioStream, - getVideoStreamDuration, getMaxAudioBitrate, - buildFileMetadata, - getVideoStreamBitrate, - getVideoStreamFPS, getVideoStream, + getVideoStreamBitrate, getVideoStreamDimensionsInfo, + getVideoStreamDuration, + getVideoStreamFPS, hasAudioStream } from '@shared/extra-utils/ffprobe' import { VideoResolution, VideoTranscodingFPS } from '@shared/models' diff --git a/server/lib/activitypub/videos/shared/abstract-builder.ts b/server/lib/activitypub/videos/shared/abstract-builder.ts index f299ba4fd..c0b92c93d 100644 --- a/server/lib/activitypub/videos/shared/abstract-builder.ts +++ b/server/lib/activitypub/videos/shared/abstract-builder.ts @@ -1,4 +1,4 @@ -import { Transaction } from 'sequelize/types' +import { CreationAttributes, Transaction } from 'sequelize/types' import { deleteAllModels, filterNonExistingModels } from '@server/helpers/database-utils' import { logger, LoggerTagsFn } from '@server/helpers/logger' import { updatePlaceholderThumbnail, updateVideoMiniatureFromUrl } from '@server/lib/thumbnail' @@ -7,7 +7,15 @@ import { VideoCaptionModel } from '@server/models/video/video-caption' import { VideoFileModel } from '@server/models/video/video-file' import { VideoLiveModel } from '@server/models/video/video-live' import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' -import { MStreamingPlaylistFilesVideo, MThumbnail, MVideoCaption, MVideoFile, MVideoFullLight, MVideoThumbnail } from '@server/types/models' +import { + MStreamingPlaylistFiles, + MStreamingPlaylistFilesVideo, + MThumbnail, + MVideoCaption, + MVideoFile, + MVideoFullLight, + MVideoThumbnail +} from '@server/types/models' import { ActivityTagObject, ThumbnailType, VideoObject, VideoStreamingPlaylistType } from '@shared/models' import { getOrCreateAPActor } from '../../actors' import { checkUrlsSameHost } from '../../url' @@ -125,38 +133,39 @@ export abstract class APVideoAbstractBuilder { // Remove video playlists that do not exist anymore await deleteAllModels(filterNonExistingModels(video.VideoStreamingPlaylists || [], newStreamingPlaylists), t) + const oldPlaylists = video.VideoStreamingPlaylists video.VideoStreamingPlaylists = [] for (const playlistAttributes of streamingPlaylistAttributes) { const streamingPlaylistModel = await this.insertOrReplaceStreamingPlaylist(playlistAttributes, t) streamingPlaylistModel.Video = video - await this.setStreamingPlaylistFiles(video, streamingPlaylistModel, playlistAttributes.tagAPObject, t) + await this.setStreamingPlaylistFiles(oldPlaylists, streamingPlaylistModel, playlistAttributes.tagAPObject, t) video.VideoStreamingPlaylists.push(streamingPlaylistModel) } } - private async insertOrReplaceStreamingPlaylist (attributes: VideoStreamingPlaylistModel['_creationAttributes'], t: Transaction) { + private async insertOrReplaceStreamingPlaylist (attributes: CreationAttributes, t: Transaction) { const [ streamingPlaylist ] = await VideoStreamingPlaylistModel.upsert(attributes, { returning: true, transaction: t }) return streamingPlaylist as MStreamingPlaylistFilesVideo } - private getStreamingPlaylistFiles (video: MVideoFullLight, type: VideoStreamingPlaylistType) { - const playlist = video.VideoStreamingPlaylists.find(s => s.type === type) + private getStreamingPlaylistFiles (oldPlaylists: MStreamingPlaylistFiles[], type: VideoStreamingPlaylistType) { + const playlist = oldPlaylists.find(s => s.type === type) if (!playlist) return [] return playlist.VideoFiles } private async setStreamingPlaylistFiles ( - video: MVideoFullLight, + oldPlaylists: MStreamingPlaylistFiles[], playlistModel: MStreamingPlaylistFilesVideo, tagObjects: ActivityTagObject[], t: Transaction ) { - const oldStreamingPlaylistFiles = this.getStreamingPlaylistFiles(video, playlistModel.type) + const oldStreamingPlaylistFiles = this.getStreamingPlaylistFiles(oldPlaylists || [], playlistModel.type) const newVideoFiles: MVideoFile[] = getFileAttributesFromUrl(playlistModel, tagObjects).map(a => new VideoFileModel(a)) diff --git a/server/lib/hls.ts b/server/lib/hls.ts index 43043315b..20754219f 100644 --- a/server/lib/hls.ts +++ b/server/lib/hls.ts @@ -1,7 +1,8 @@ import { close, ensureDir, move, open, outputJSON, read, readFile, remove, stat, writeFile } from 'fs-extra' import { flatten, uniq } from 'lodash' +import PQueue from 'p-queue' import { basename, dirname, join } from 'path' -import { MStreamingPlaylistFilesVideo, MVideo, MVideoUUID } from '@server/types/models' +import { MStreamingPlaylist, MStreamingPlaylistFilesVideo, MVideo } from '@server/types/models' import { sha256 } from '@shared/extra-utils' import { VideoStorage } from '@shared/models' import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamDimensionsInfo } from '../helpers/ffmpeg' @@ -14,7 +15,7 @@ import { sequelizeTypescript } from '../initializers/database' import { VideoFileModel } from '../models/video/video-file' import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' import { storeHLSFile } from './object-storage' -import { getHlsResolutionPlaylistFilename } from './paths' +import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHlsResolutionPlaylistFilename } from './paths' import { VideoPathManager } from './video-path-manager' async function updateStreamingPlaylistsInfohashesIfNeeded () { @@ -33,80 +34,123 @@ async function updateStreamingPlaylistsInfohashesIfNeeded () { } } -async function updateMasterHLSPlaylist (video: MVideo, playlist: MStreamingPlaylistFilesVideo) { - const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ] +async function updatePlaylistAfterFileChange (video: MVideo, playlist: MStreamingPlaylist) { + let playlistWithFiles = await updateMasterHLSPlaylist(video, playlist) + playlistWithFiles = await updateSha256VODSegments(video, playlist) - for (const file of playlist.VideoFiles) { - const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) + // Refresh playlist, operations can take some time + playlistWithFiles = await VideoStreamingPlaylistModel.loadWithVideoAndFiles(playlist.id) + playlistWithFiles.assignP2PMediaLoaderInfoHashes(video, playlistWithFiles.VideoFiles) + await playlistWithFiles.save() - await VideoPathManager.Instance.makeAvailableVideoFile(file.withVideoOrPlaylist(playlist), async videoFilePath => { - const size = await getVideoStreamDimensionsInfo(videoFilePath) + video.setHLSPlaylist(playlistWithFiles) +} - const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file) - const resolution = `RESOLUTION=${size?.width || 0}x${size?.height || 0}` +// --------------------------------------------------------------------------- - let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}` - if (file.fps) line += ',FRAME-RATE=' + file.fps +// Avoid concurrency issues when updating streaming playlist files +const playlistFilesQueue = new PQueue({ concurrency: 1 }) - const codecs = await Promise.all([ - getVideoStreamCodec(videoFilePath), - getAudioStreamCodec(videoFilePath) - ]) +function updateMasterHLSPlaylist (video: MVideo, playlistArg: MStreamingPlaylist): Promise { + return playlistFilesQueue.add(async () => { + const playlist = await VideoStreamingPlaylistModel.loadWithVideoAndFiles(playlistArg.id) - line += `,CODECS="${codecs.filter(c => !!c).join(',')}"` + const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ] - masterPlaylists.push(line) - masterPlaylists.push(playlistFilename) - }) - } + for (const file of playlist.VideoFiles) { + const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) - await VideoPathManager.Instance.makeAvailablePlaylistFile(playlist, playlist.playlistFilename, async masterPlaylistPath => { + await VideoPathManager.Instance.makeAvailableVideoFile(file.withVideoOrPlaylist(playlist), async videoFilePath => { + const size = await getVideoStreamDimensionsInfo(videoFilePath) + + const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file) + const resolution = `RESOLUTION=${size?.width || 0}x${size?.height || 0}` + + let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}` + if (file.fps) line += ',FRAME-RATE=' + file.fps + + const codecs = await Promise.all([ + getVideoStreamCodec(videoFilePath), + getAudioStreamCodec(videoFilePath) + ]) + + line += `,CODECS="${codecs.filter(c => !!c).join(',')}"` + + masterPlaylists.push(line) + masterPlaylists.push(playlistFilename) + }) + } + + if (playlist.playlistFilename) { + await video.removeStreamingPlaylistFile(playlist, playlist.playlistFilename) + } + playlist.playlistFilename = generateHLSMasterPlaylistFilename(video.isLive) + + const masterPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.playlistFilename) await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') if (playlist.storage === VideoStorage.OBJECT_STORAGE) { - await storeHLSFile(playlist, playlist.playlistFilename, masterPlaylistPath) + playlist.playlistUrl = await storeHLSFile(playlist, playlist.playlistFilename) + await remove(masterPlaylistPath) } + + return playlist.save() }) } -async function updateSha256VODSegments (video: MVideoUUID, playlist: MStreamingPlaylistFilesVideo) { - const json: { [filename: string]: { [range: string]: string } } = {} +// --------------------------------------------------------------------------- - // For all the resolutions available for this video - for (const file of playlist.VideoFiles) { - const rangeHashes: { [range: string]: string } = {} - const fileWithPlaylist = file.withVideoOrPlaylist(playlist) +async function updateSha256VODSegments (video: MVideo, playlistArg: MStreamingPlaylist): Promise { + return playlistFilesQueue.add(async () => { + const json: { [filename: string]: { [range: string]: string } } = {} - await VideoPathManager.Instance.makeAvailableVideoFile(fileWithPlaylist, videoPath => { + const playlist = await VideoStreamingPlaylistModel.loadWithVideoAndFiles(playlistArg.id) - return VideoPathManager.Instance.makeAvailableResolutionPlaylistFile(fileWithPlaylist, async resolutionPlaylistPath => { - const playlistContent = await readFile(resolutionPlaylistPath) - const ranges = getRangesFromPlaylist(playlistContent.toString()) + // For all the resolutions available for this video + for (const file of playlist.VideoFiles) { + const rangeHashes: { [range: string]: string } = {} + const fileWithPlaylist = file.withVideoOrPlaylist(playlist) - const fd = await open(videoPath, 'r') - for (const range of ranges) { - const buf = Buffer.alloc(range.length) - await read(fd, buf, 0, range.length, range.offset) + await VideoPathManager.Instance.makeAvailableVideoFile(fileWithPlaylist, videoPath => { - rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf) - } - await close(fd) + return VideoPathManager.Instance.makeAvailableResolutionPlaylistFile(fileWithPlaylist, async resolutionPlaylistPath => { + const playlistContent = await readFile(resolutionPlaylistPath) + const ranges = getRangesFromPlaylist(playlistContent.toString()) - const videoFilename = file.filename - json[videoFilename] = rangeHashes + const fd = await open(videoPath, 'r') + for (const range of ranges) { + const buf = Buffer.alloc(range.length) + await read(fd, buf, 0, range.length, range.offset) + + rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf) + } + await close(fd) + + const videoFilename = file.filename + json[videoFilename] = rangeHashes + }) }) - }) - } + } - const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename) - await outputJSON(outputPath, json) + if (playlist.segmentsSha256Filename) { + await video.removeStreamingPlaylistFile(playlist, playlist.segmentsSha256Filename) + } + playlist.segmentsSha256Filename = generateHlsSha256SegmentsFilename(video.isLive) - if (playlist.storage === VideoStorage.OBJECT_STORAGE) { - await storeHLSFile(playlist, playlist.segmentsSha256Filename) - await remove(outputPath) - } + const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename) + await outputJSON(outputPath, json) + + if (playlist.storage === VideoStorage.OBJECT_STORAGE) { + playlist.segmentsSha256Url = await storeHLSFile(playlist, playlist.segmentsSha256Filename) + await remove(outputPath) + } + + return playlist.save() + }) } +// --------------------------------------------------------------------------- + async function buildSha256Segment (segmentPath: string) { const buf = await readFile(segmentPath) return sha256(buf) @@ -190,7 +234,8 @@ export { updateSha256VODSegments, buildSha256Segment, downloadPlaylistSegments, - updateStreamingPlaylistsInfohashesIfNeeded + updateStreamingPlaylistsInfohashesIfNeeded, + updatePlaylistAfterFileChange } // --------------------------------------------------------------------------- diff --git a/server/lib/job-queue/handlers/video-file-import.ts b/server/lib/job-queue/handlers/video-file-import.ts index 1c600e2a7..71c5444af 100644 --- a/server/lib/job-queue/handlers/video-file-import.ts +++ b/server/lib/job-queue/handlers/video-file-import.ts @@ -55,7 +55,7 @@ async function updateVideoFile (video: MVideoFullLight, inputFilePath: string) { if (currentVideoFile) { // Remove old file and old torrent - await video.removeWebTorrentFileAndTorrent(currentVideoFile) + await video.removeWebTorrentFile(currentVideoFile) // Remove the old video file from the array video.VideoFiles = video.VideoFiles.filter(f => f !== currentVideoFile) diff --git a/server/lib/job-queue/handlers/video-studio-edition.ts b/server/lib/job-queue/handlers/video-studio-edition.ts index 434d0ffe8..735150d57 100644 --- a/server/lib/job-queue/handlers/video-studio-edition.ts +++ b/server/lib/job-queue/handlers/video-studio-edition.ts @@ -9,6 +9,7 @@ import { generateWebTorrentVideoFilename } from '@server/lib/paths' import { VideoTranscodingProfilesManager } from '@server/lib/transcoding/default-transcoding-profiles' import { isAbleToUploadVideo } from '@server/lib/user' import { addOptimizeOrMergeAudioJob } from '@server/lib/video' +import { removeHLSPlaylist, removeWebTorrentFile } from '@server/lib/video-file' import { VideoPathManager } from '@server/lib/video-path-manager' import { approximateIntroOutroAdditionalSize } from '@server/lib/video-studio' import { UserModel } from '@server/models/user/user' @@ -27,12 +28,12 @@ import { } from '@shared/extra-utils' import { VideoStudioEditionPayload, - VideoStudioTaskPayload, + VideoStudioTask, VideoStudioTaskCutPayload, VideoStudioTaskIntroPayload, VideoStudioTaskOutroPayload, - VideoStudioTaskWatermarkPayload, - VideoStudioTask + VideoStudioTaskPayload, + VideoStudioTaskWatermarkPayload } from '@shared/models' import { logger, loggerTagsFactory } from '../../../helpers/logger' @@ -89,7 +90,6 @@ async function processVideoStudioEdition (job: Job) { await move(editionResultPath, outputPath) await createTorrentAndSetInfoHashFromPath(video, newFile, outputPath) - await removeAllFiles(video, newFile) await newFile.save() @@ -197,18 +197,12 @@ async function buildNewFile (video: MVideoId, path: string) { } async function removeAllFiles (video: MVideoWithAllFiles, webTorrentFileException: MVideoFile) { - const hls = video.getHLSPlaylist() - - if (hls) { - await video.removeStreamingPlaylistFiles(hls) - await hls.destroy() - } + await removeHLSPlaylist(video) for (const file of video.VideoFiles) { if (file.id === webTorrentFileException.id) continue - await video.removeWebTorrentFileAndTorrent(file) - await file.destroy() + await removeWebTorrentFile(video, file.id) } } diff --git a/server/lib/job-queue/handlers/video-transcoding.ts b/server/lib/job-queue/handlers/video-transcoding.ts index 5afca65ca..1b34ced14 100644 --- a/server/lib/job-queue/handlers/video-transcoding.ts +++ b/server/lib/job-queue/handlers/video-transcoding.ts @@ -149,7 +149,7 @@ async function onHlsPlaylistGeneration (video: MVideoFullLight, user: MUser, pay if (payload.isMaxQuality && payload.autoDeleteWebTorrentIfNeeded && CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false) { // Remove webtorrent files if not enabled for (const file of video.VideoFiles) { - await video.removeWebTorrentFileAndTorrent(file) + await video.removeWebTorrentFile(file) await file.destroy() } diff --git a/server/lib/transcoding/transcoding.ts b/server/lib/transcoding/transcoding.ts index 69a973fbd..924141d1c 100644 --- a/server/lib/transcoding/transcoding.ts +++ b/server/lib/transcoding/transcoding.ts @@ -5,9 +5,8 @@ import { toEven } from '@server/helpers/core-utils' import { retryTransactionWrapper } from '@server/helpers/database-utils' import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' import { sequelizeTypescript } from '@server/initializers/database' -import { MStreamingPlaylistFilesVideo, MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' +import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' import { VideoResolution, VideoStorage } from '../../../shared/models/videos' -import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' import { buildFileMetadata, canDoQuickTranscode, @@ -18,17 +17,10 @@ import { TranscodeVODOptionsType } from '../../helpers/ffmpeg' import { CONFIG } from '../../initializers/config' -import { P2P_MEDIA_LOADER_PEER_VERSION } from '../../initializers/constants' import { VideoFileModel } from '../../models/video/video-file' import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' -import { updateMasterHLSPlaylist, updateSha256VODSegments } from '../hls' -import { - generateHLSMasterPlaylistFilename, - generateHlsSha256SegmentsFilename, - generateHLSVideoFilename, - generateWebTorrentVideoFilename, - getHlsResolutionPlaylistFilename -} from '../paths' +import { updatePlaylistAfterFileChange } from '../hls' +import { generateHLSVideoFilename, generateWebTorrentVideoFilename, getHlsResolutionPlaylistFilename } from '../paths' import { VideoPathManager } from '../video-path-manager' import { VideoTranscodingProfilesManager } from './default-transcoding-profiles' @@ -260,7 +252,7 @@ async function onWebTorrentVideoFileTranscoding ( await createTorrentAndSetInfoHash(video, videoFile) const oldFile = await VideoFileModel.loadWebTorrentFile({ videoId: video.id, fps: videoFile.fps, resolution: videoFile.resolution }) - if (oldFile) await video.removeWebTorrentFileAndTorrent(oldFile) + if (oldFile) await video.removeWebTorrentFile(oldFile) await VideoFileModel.customUpsert(videoFile, 'video', undefined) video.VideoFiles = await video.$get('VideoFiles') @@ -314,35 +306,15 @@ async function generateHlsPlaylistCommon (options: { await transcodeVOD(transcodeOptions) // Create or update the playlist - const { playlist, oldPlaylistFilename, oldSegmentsSha256Filename } = await retryTransactionWrapper(() => { + const playlist = await retryTransactionWrapper(() => { return sequelizeTypescript.transaction(async transaction => { - const playlist = await VideoStreamingPlaylistModel.loadOrGenerate(video, transaction) - - const oldPlaylistFilename = playlist.playlistFilename - const oldSegmentsSha256Filename = playlist.segmentsSha256Filename - - playlist.playlistFilename = generateHLSMasterPlaylistFilename(video.isLive) - playlist.segmentsSha256Filename = generateHlsSha256SegmentsFilename(video.isLive) - - playlist.p2pMediaLoaderInfohashes = [] - playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION - - playlist.type = VideoStreamingPlaylistType.HLS - - await playlist.save({ transaction }) - - return { playlist, oldPlaylistFilename, oldSegmentsSha256Filename } + return VideoStreamingPlaylistModel.loadOrGenerate(video, transaction) }) }) - if (oldPlaylistFilename) await video.removeStreamingPlaylistFile(playlist, oldPlaylistFilename) - if (oldSegmentsSha256Filename) await video.removeStreamingPlaylistFile(playlist, oldSegmentsSha256Filename) - - // Build the new playlist file - const extname = extnameUtil(videoFilename) const newVideoFile = new VideoFileModel({ resolution, - extname, + extname: extnameUtil(videoFilename), size: 0, filename: videoFilename, fps: -1, @@ -350,8 +322,6 @@ async function generateHlsPlaylistCommon (options: { }) const videoFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(playlist, newVideoFile) - - // Move files from tmp transcoded directory to the appropriate place await ensureDir(VideoPathManager.Instance.getFSHLSOutputPath(video)) // Move playlist file @@ -369,21 +339,14 @@ async function generateHlsPlaylistCommon (options: { await createTorrentAndSetInfoHash(playlist, newVideoFile) const oldFile = await VideoFileModel.loadHLSFile({ playlistId: playlist.id, fps: newVideoFile.fps, resolution: newVideoFile.resolution }) - if (oldFile) await video.removeStreamingPlaylistVideoFile(playlist, oldFile) + if (oldFile) { + await video.removeStreamingPlaylistVideoFile(playlist, oldFile) + await oldFile.destroy() + } const savedVideoFile = await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined) - const playlistWithFiles = playlist as MStreamingPlaylistFilesVideo - playlistWithFiles.VideoFiles = await playlist.$get('VideoFiles') - playlist.assignP2PMediaLoaderInfoHashes(video, playlistWithFiles.VideoFiles) - playlist.storage = VideoStorage.FILE_SYSTEM - - await playlist.save() - - video.setHLSPlaylist(playlist) - - await updateMasterHLSPlaylist(video, playlistWithFiles) - await updateSha256VODSegments(video, playlistWithFiles) + await updatePlaylistAfterFileChange(video, playlist) return { resolutionPlaylistPath, videoFile: savedVideoFile } } diff --git a/server/lib/video-file.ts b/server/lib/video-file.ts new file mode 100644 index 000000000..2ab7190f1 --- /dev/null +++ b/server/lib/video-file.ts @@ -0,0 +1,69 @@ +import { logger } from '@server/helpers/logger' +import { MVideoWithAllFiles } from '@server/types/models' +import { lTags } from './object-storage/shared' + +async function removeHLSPlaylist (video: MVideoWithAllFiles) { + const hls = video.getHLSPlaylist() + if (!hls) return + + await video.removeStreamingPlaylistFiles(hls) + await hls.destroy() + + video.VideoStreamingPlaylists = video.VideoStreamingPlaylists.filter(p => p.id !== hls.id) +} + +async function removeHLSFile (video: MVideoWithAllFiles, fileToDeleteId: number) { + logger.info('Deleting HLS file %d of %s.', fileToDeleteId, video.url, lTags(video.uuid)) + + const hls = video.getHLSPlaylist() + const files = hls.VideoFiles + + if (files.length === 1) { + await removeHLSPlaylist(video) + return undefined + } + + const toDelete = files.find(f => f.id === fileToDeleteId) + await video.removeStreamingPlaylistVideoFile(video.getHLSPlaylist(), toDelete) + await toDelete.destroy() + + hls.VideoFiles = hls.VideoFiles.filter(f => f.id !== toDelete.id) + + return hls +} + +// --------------------------------------------------------------------------- + +async function removeAllWebTorrentFiles (video: MVideoWithAllFiles) { + for (const file of video.VideoFiles) { + await video.removeWebTorrentFile(file) + await file.destroy() + } + + video.VideoFiles = [] + + return video +} + +async function removeWebTorrentFile (video: MVideoWithAllFiles, fileToDeleteId: number) { + const files = video.VideoFiles + + if (files.length === 1) { + return removeAllWebTorrentFiles(video) + } + + const toDelete = files.find(f => f.id === fileToDeleteId) + await video.removeWebTorrentFile(toDelete) + await toDelete.destroy() + + video.VideoFiles = files.filter(f => f.id !== toDelete.id) + + return video +} + +export { + removeHLSPlaylist, + removeHLSFile, + removeAllWebTorrentFiles, + removeWebTorrentFile +} diff --git a/server/middlewares/validators/videos/video-files.ts b/server/middlewares/validators/videos/video-files.ts index 35b0ac757..b3db3f4f7 100644 --- a/server/middlewares/validators/videos/video-files.ts +++ b/server/middlewares/validators/videos/video-files.ts @@ -3,6 +3,8 @@ import { MVideo } from '@server/types/models' import { HttpStatusCode } from '@shared/models' import { logger } from '../../../helpers/logger' import { areValidationErrors, doesVideoExist, isValidVideoIdParam } from '../shared' +import { isIdValid } from '@server/helpers/custom-validators/misc' +import { param } from 'express-validator' const videoFilesDeleteWebTorrentValidator = [ isValidVideoIdParam('id'), @@ -35,6 +37,43 @@ const videoFilesDeleteWebTorrentValidator = [ } ] +const videoFilesDeleteWebTorrentFileValidator = [ + isValidVideoIdParam('id'), + + param('videoFileId') + .custom(isIdValid).withMessage('Should have a valid file id'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking videoFilesDeleteWebTorrentFile parameters', { parameters: req.params }) + + if (areValidationErrors(req, res)) return + if (!await doesVideoExist(req.params.id, res)) return + + const video = res.locals.videoAll + + if (!checkLocalVideo(video, res)) return + + const files = video.VideoFiles + if (!files.find(f => f.id === +req.params.videoFileId)) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'This video does not have this WebTorrent file id' + }) + } + + if (files.length === 1 && !video.getHLSPlaylist()) { + return res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'Cannot delete WebTorrent files since this video does not have HLS playlist' + }) + } + + return next() + } +] + +// --------------------------------------------------------------------------- + const videoFilesDeleteHLSValidator = [ isValidVideoIdParam('id'), @@ -66,9 +105,55 @@ const videoFilesDeleteHLSValidator = [ } ] +const videoFilesDeleteHLSFileValidator = [ + isValidVideoIdParam('id'), + + param('videoFileId') + .custom(isIdValid).withMessage('Should have a valid file id'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking videoFilesDeleteHLSFile parameters', { parameters: req.params }) + + if (areValidationErrors(req, res)) return + if (!await doesVideoExist(req.params.id, res)) return + + const video = res.locals.videoAll + + if (!checkLocalVideo(video, res)) return + + if (!video.getHLSPlaylist()) { + return res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'This video does not have HLS files' + }) + } + + const hlsFiles = video.getHLSPlaylist().VideoFiles + if (!hlsFiles.find(f => f.id === +req.params.videoFileId)) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'This HLS playlist does not have this file id' + }) + } + + // Last file to delete + if (hlsFiles.length === 1 && !video.hasWebTorrentFiles()) { + return res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'Cannot delete last HLS playlist file since this video does not have WebTorrent files' + }) + } + + return next() + } +] + export { videoFilesDeleteWebTorrentValidator, - videoFilesDeleteHLSValidator + videoFilesDeleteWebTorrentFileValidator, + + videoFilesDeleteHLSValidator, + videoFilesDeleteHLSFileValidator } // --------------------------------------------------------------------------- diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts index b363afb28..15909d5f3 100644 --- a/server/models/redundancy/video-redundancy.ts +++ b/server/models/redundancy/video-redundancy.ts @@ -162,7 +162,7 @@ export class VideoRedundancyModel extends Model logger.error('Cannot delete %s files.', logIdentifier, { err })) } diff --git a/server/models/video/video-streaming-playlist.ts b/server/models/video/video-streaming-playlist.ts index 2c4dbd8ec..f587989dc 100644 --- a/server/models/video/video-streaming-playlist.ts +++ b/server/models/video/video-streaming-playlist.ts @@ -16,8 +16,9 @@ import { UpdatedAt } from 'sequelize-typescript' import { getHLSPublicFileUrl } from '@server/lib/object-storage' +import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename } from '@server/lib/paths' import { VideoFileModel } from '@server/models/video/video-file' -import { MStreamingPlaylist, MVideo } from '@server/types/models' +import { MStreamingPlaylist, MStreamingPlaylistFilesVideo, MVideo } from '@server/types/models' import { sha1 } from '@shared/extra-utils' import { VideoStorage } from '@shared/models' import { AttributesOnly } from '@shared/typescript-utils' @@ -167,6 +168,22 @@ export class VideoStreamingPlaylistModel extends Model(id, options) + } + static loadWithVideo (id: number) { const options = { include: [ @@ -194,9 +211,22 @@ export class VideoStreamingPlaylistModel extends Model>> { // Remove physical files and torrents instance.VideoFiles.forEach(file => { - tasks.push(instance.removeWebTorrentFileAndTorrent(file)) + tasks.push(instance.removeWebTorrentFile(file)) }) // Remove playlists file @@ -1783,7 +1783,7 @@ export class VideoModel extends Model>> { .concat(toAdd) } - removeWebTorrentFileAndTorrent (videoFile: MVideoFile, isRedundancy = false) { + removeWebTorrentFile (videoFile: MVideoFile, isRedundancy = false) { const filePath = isRedundancy ? VideoPathManager.Instance.getFSRedundancyVideoFilePath(this, videoFile) : VideoPathManager.Instance.getFSVideoFileOutputPath(this, videoFile) @@ -1829,8 +1829,12 @@ export class VideoModel extends Model>> { await videoFile.removeTorrent() await remove(filePath) + const resolutionFilename = getHlsResolutionPlaylistFilename(videoFile.filename) + await remove(VideoPathManager.Instance.getFSHLSOutputPath(this, resolutionFilename)) + if (videoFile.storage === VideoStorage.OBJECT_STORAGE) { await removeHLSFileObjectStorage(streamingPlaylist.withVideo(this), videoFile.filename) + await removeHLSFileObjectStorage(streamingPlaylist.withVideo(this), resolutionFilename) } } diff --git a/server/tests/api/check-params/video-files.ts b/server/tests/api/check-params/video-files.ts index 8c0795092..c698bea82 100644 --- a/server/tests/api/check-params/video-files.ts +++ b/server/tests/api/check-params/video-files.ts @@ -24,6 +24,12 @@ describe('Test videos files', function () { let validId1: string let validId2: string + let hlsFileId: number + let webtorrentFileId: number + + let remoteHLSFileId: number + let remoteWebtorrentFileId: number + // --------------------------------------------------------------- before(async function () { @@ -39,7 +45,12 @@ describe('Test videos files', function () { { const { uuid } = await servers[1].videos.quickUpload({ name: 'remote video' }) - remoteId = uuid + await waitJobs(servers) + + const video = await servers[1].videos.get({ id: uuid }) + remoteId = video.uuid + remoteHLSFileId = video.streamingPlaylists[0].files[0].id + remoteWebtorrentFileId = video.files[0].id } { @@ -47,7 +58,12 @@ describe('Test videos files', function () { { const { uuid } = await servers[0].videos.quickUpload({ name: 'both 1' }) - validId1 = uuid + await waitJobs(servers) + + const video = await servers[0].videos.get({ id: uuid }) + validId1 = video.uuid + hlsFileId = video.streamingPlaylists[0].files[0].id + webtorrentFileId = video.files[0].id } { @@ -76,43 +92,67 @@ describe('Test videos files', function () { }) it('Should not delete files of a unknown video', async function () { - await servers[0].videos.removeHLSFiles({ videoId: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - await servers[0].videos.removeWebTorrentFiles({ videoId: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + const expectedStatus = HttpStatusCode.NOT_FOUND_404 + + await servers[0].videos.removeHLSPlaylist({ videoId: 404, expectedStatus }) + await servers[0].videos.removeAllWebTorrentFiles({ videoId: 404, expectedStatus }) + + await servers[0].videos.removeHLSFile({ videoId: 404, fileId: hlsFileId, expectedStatus }) + await servers[0].videos.removeWebTorrentFile({ videoId: 404, fileId: webtorrentFileId, expectedStatus }) + }) + + it('Should not delete unknown files', async function () { + const expectedStatus = HttpStatusCode.NOT_FOUND_404 + + await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: webtorrentFileId, expectedStatus }) + await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: hlsFileId, expectedStatus }) }) it('Should not delete files of a remote video', async function () { - await servers[0].videos.removeHLSFiles({ videoId: remoteId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - await servers[0].videos.removeWebTorrentFiles({ videoId: remoteId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + const expectedStatus = HttpStatusCode.BAD_REQUEST_400 + + await servers[0].videos.removeHLSPlaylist({ videoId: remoteId, expectedStatus }) + await servers[0].videos.removeAllWebTorrentFiles({ videoId: remoteId, expectedStatus }) + + await servers[0].videos.removeHLSFile({ videoId: remoteId, fileId: remoteHLSFileId, expectedStatus }) + await servers[0].videos.removeWebTorrentFile({ videoId: remoteId, fileId: remoteWebtorrentFileId, expectedStatus }) }) it('Should not delete files by a non admin user', async function () { const expectedStatus = HttpStatusCode.FORBIDDEN_403 - await servers[0].videos.removeHLSFiles({ videoId: validId1, token: userToken, expectedStatus }) - await servers[0].videos.removeHLSFiles({ videoId: validId1, token: moderatorToken, expectedStatus }) + await servers[0].videos.removeHLSPlaylist({ videoId: validId1, token: userToken, expectedStatus }) + await servers[0].videos.removeHLSPlaylist({ videoId: validId1, token: moderatorToken, expectedStatus }) - await servers[0].videos.removeWebTorrentFiles({ videoId: validId1, token: userToken, expectedStatus }) - await servers[0].videos.removeWebTorrentFiles({ videoId: validId1, token: moderatorToken, expectedStatus }) + await servers[0].videos.removeAllWebTorrentFiles({ videoId: validId1, token: userToken, expectedStatus }) + await servers[0].videos.removeAllWebTorrentFiles({ videoId: validId1, token: moderatorToken, expectedStatus }) + + await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: hlsFileId, token: userToken, expectedStatus }) + await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: hlsFileId, token: moderatorToken, expectedStatus }) + + await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: webtorrentFileId, token: userToken, expectedStatus }) + await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: webtorrentFileId, token: moderatorToken, expectedStatus }) }) it('Should not delete files if the files are not available', async function () { - await servers[0].videos.removeHLSFiles({ videoId: hlsId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - await servers[0].videos.removeWebTorrentFiles({ videoId: webtorrentId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + await servers[0].videos.removeHLSPlaylist({ videoId: hlsId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + await servers[0].videos.removeAllWebTorrentFiles({ videoId: webtorrentId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + + await servers[0].videos.removeHLSFile({ videoId: hlsId, fileId: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + await servers[0].videos.removeWebTorrentFile({ videoId: webtorrentId, fileId: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) }) it('Should not delete files if no both versions are available', async function () { - await servers[0].videos.removeHLSFiles({ videoId: hlsId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - await servers[0].videos.removeWebTorrentFiles({ videoId: webtorrentId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should not delete files if no both versions are available', async function () { - await servers[0].videos.removeHLSFiles({ videoId: hlsId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - await servers[0].videos.removeWebTorrentFiles({ videoId: webtorrentId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + await servers[0].videos.removeHLSPlaylist({ videoId: hlsId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + await servers[0].videos.removeAllWebTorrentFiles({ videoId: webtorrentId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) }) it('Should delete files if both versions are available', async function () { - await servers[0].videos.removeHLSFiles({ videoId: validId1 }) - await servers[0].videos.removeWebTorrentFiles({ videoId: validId2 }) + await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: hlsFileId }) + await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: webtorrentFileId }) + + await servers[0].videos.removeHLSPlaylist({ videoId: validId1 }) + await servers[0].videos.removeAllWebTorrentFiles({ videoId: validId2 }) }) after(async function () { diff --git a/server/tests/api/transcoding/create-transcoding.ts b/server/tests/api/transcoding/create-transcoding.ts index e3867fdad..b59bef772 100644 --- a/server/tests/api/transcoding/create-transcoding.ts +++ b/server/tests/api/transcoding/create-transcoding.ts @@ -122,7 +122,7 @@ function runTests (objectStorage: boolean) { it('Should generate WebTorrent from HLS only video', async function () { this.timeout(60000) - await servers[0].videos.removeWebTorrentFiles({ videoId: videoUUID }) + await servers[0].videos.removeAllWebTorrentFiles({ videoId: videoUUID }) await waitJobs(servers) await servers[0].videos.runTranscoding({ videoId: videoUUID, transcodingType: 'webtorrent' }) @@ -142,7 +142,7 @@ function runTests (objectStorage: boolean) { it('Should only generate WebTorrent', async function () { this.timeout(60000) - await servers[0].videos.removeHLSFiles({ videoId: videoUUID }) + await servers[0].videos.removeHLSPlaylist({ videoId: videoUUID }) await waitJobs(servers) await servers[0].videos.runTranscoding({ videoId: videoUUID, transcodingType: 'webtorrent' }) diff --git a/server/tests/api/videos/video-files.ts b/server/tests/api/videos/video-files.ts index b0ef4a2e9..313f020e9 100644 --- a/server/tests/api/videos/video-files.ts +++ b/server/tests/api/videos/video-files.ts @@ -2,10 +2,12 @@ import 'mocha' import { expect } from 'chai' +import { HttpStatusCode } from '@shared/models' import { cleanupTests, createMultipleServers, doubleFollow, + makeRawRequest, PeerTubeServer, setAccessTokensToServers, waitJobs @@ -13,8 +15,6 @@ import { describe('Test videos files', function () { let servers: PeerTubeServer[] - let validId1: string - let validId2: string // --------------------------------------------------------------- @@ -27,48 +27,160 @@ describe('Test videos files', function () { await doubleFollow(servers[0], servers[1]) await servers[0].config.enableTranscoding(true, true) - - { - const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1' }) - validId1 = uuid - } - - { - const { uuid } = await servers[0].videos.quickUpload({ name: 'video 2' }) - validId2 = uuid - } - - await waitJobs(servers) }) - it('Should delete webtorrent files', async function () { - this.timeout(30_000) + describe('When deleting all files', function () { + let validId1: string + let validId2: string - await servers[0].videos.removeWebTorrentFiles({ videoId: validId1 }) + before(async function () { + { + const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1' }) + validId1 = uuid + } - await waitJobs(servers) + { + const { uuid } = await servers[0].videos.quickUpload({ name: 'video 2' }) + validId2 = uuid + } - for (const server of servers) { - const video = await server.videos.get({ id: validId1 }) + await waitJobs(servers) + }) - expect(video.files).to.have.lengthOf(0) - expect(video.streamingPlaylists).to.have.lengthOf(1) - } + it('Should delete webtorrent files', async function () { + this.timeout(30_000) + + await servers[0].videos.removeAllWebTorrentFiles({ videoId: validId1 }) + + await waitJobs(servers) + + for (const server of servers) { + const video = await server.videos.get({ id: validId1 }) + + expect(video.files).to.have.lengthOf(0) + expect(video.streamingPlaylists).to.have.lengthOf(1) + } + }) + + it('Should delete HLS files', async function () { + this.timeout(30_000) + + await servers[0].videos.removeHLSPlaylist({ videoId: validId2 }) + + await waitJobs(servers) + + for (const server of servers) { + const video = await server.videos.get({ id: validId2 }) + + expect(video.files).to.have.length.above(0) + expect(video.streamingPlaylists).to.have.lengthOf(0) + } + }) }) - it('Should delete HLS files', async function () { - this.timeout(30_000) + describe('When deleting a specific file', function () { + let webtorrentId: string + let hlsId: string - await servers[0].videos.removeHLSFiles({ videoId: validId2 }) + before(async function () { + { + const { uuid } = await servers[0].videos.quickUpload({ name: 'webtorrent' }) + webtorrentId = uuid + } - await waitJobs(servers) + { + const { uuid } = await servers[0].videos.quickUpload({ name: 'hls' }) + hlsId = uuid + } - for (const server of servers) { - const video = await server.videos.get({ id: validId2 }) + await waitJobs(servers) + }) - expect(video.files).to.have.length.above(0) - expect(video.streamingPlaylists).to.have.lengthOf(0) - } + it('Shoulde delete a webtorrent file', async function () { + const video = await servers[0].videos.get({ id: webtorrentId }) + const files = video.files + + await servers[0].videos.removeWebTorrentFile({ videoId: webtorrentId, fileId: files[0].id }) + + await waitJobs(servers) + + for (const server of servers) { + const video = await server.videos.get({ id: webtorrentId }) + + expect(video.files).to.have.lengthOf(files.length - 1) + expect(video.files.find(f => f.id === files[0].id)).to.not.exist + } + }) + + it('Should delete all webtorrent files', async function () { + const video = await servers[0].videos.get({ id: webtorrentId }) + const files = video.files + + for (const file of files) { + await servers[0].videos.removeWebTorrentFile({ videoId: webtorrentId, fileId: file.id }) + } + + await waitJobs(servers) + + for (const server of servers) { + const video = await server.videos.get({ id: webtorrentId }) + + expect(video.files).to.have.lengthOf(0) + } + }) + + it('Should delete a hls file', async function () { + const video = await servers[0].videos.get({ id: hlsId }) + const files = video.streamingPlaylists[0].files + const toDelete = files[0] + + await servers[0].videos.removeHLSFile({ videoId: hlsId, fileId: toDelete.id }) + + await waitJobs(servers) + + for (const server of servers) { + const video = await server.videos.get({ id: hlsId }) + + expect(video.streamingPlaylists[0].files).to.have.lengthOf(files.length - 1) + expect(video.streamingPlaylists[0].files.find(f => f.id === toDelete.id)).to.not.exist + + const { text } = await makeRawRequest(video.streamingPlaylists[0].playlistUrl) + + expect(text.includes(`-${toDelete.resolution.id}.m3u8`)).to.be.false + expect(text.includes(`-${video.streamingPlaylists[0].files[0].resolution.id}.m3u8`)).to.be.true + } + }) + + it('Should delete all hls files', async function () { + const video = await servers[0].videos.get({ id: hlsId }) + const files = video.streamingPlaylists[0].files + + for (const file of files) { + await servers[0].videos.removeHLSFile({ videoId: hlsId, fileId: file.id }) + } + + await waitJobs(servers) + + for (const server of servers) { + const video = await server.videos.get({ id: hlsId }) + + expect(video.streamingPlaylists).to.have.lengthOf(0) + } + }) + + it('Should not delete last file of a video', async function () { + const webtorrentOnly = await servers[0].videos.get({ id: hlsId }) + const hlsOnly = await servers[0].videos.get({ id: webtorrentId }) + + for (let i = 0; i < 4; i++) { + await servers[0].videos.removeWebTorrentFile({ videoId: webtorrentOnly.id, fileId: webtorrentOnly.files[i].id }) + await servers[0].videos.removeHLSFile({ videoId: hlsOnly.id, fileId: hlsOnly.streamingPlaylists[0].files[i].id }) + } + + const expectedStatus = HttpStatusCode.BAD_REQUEST_400 + await servers[0].videos.removeWebTorrentFile({ videoId: webtorrentOnly.id, fileId: webtorrentOnly.files[4].id, expectedStatus }) + await servers[0].videos.removeHLSFile({ videoId: hlsOnly.id, fileId: hlsOnly.streamingPlaylists[0].files[4].id, expectedStatus }) + }) }) after(async function () { diff --git a/shared/server-commands/videos/videos-command.ts b/shared/server-commands/videos/videos-command.ts index e952c9777..c0b36d95b 100644 --- a/shared/server-commands/videos/videos-command.ts +++ b/shared/server-commands/videos/videos-command.ts @@ -20,10 +20,10 @@ import { VideosCommonQuery, VideoTranscodingCreate } from '@shared/models' +import { VideoSource } from '@shared/models/videos/video-source' import { unwrapBody } from '../requests' import { waitJobs } from '../server' import { AbstractCommand, OverrideCommandOptions } from '../shared' -import { VideoSource } from '@shared/models/videos/video-source' export type VideoEdit = Partial> & { fixture?: string @@ -605,7 +605,7 @@ export class VideosCommand extends AbstractCommand { // --------------------------------------------------------------------------- - removeHLSFiles (options: OverrideCommandOptions & { + removeHLSPlaylist (options: OverrideCommandOptions & { videoId: number | string }) { const path = '/api/v1/videos/' + options.videoId + '/hls' @@ -619,7 +619,22 @@ export class VideosCommand extends AbstractCommand { }) } - removeWebTorrentFiles (options: OverrideCommandOptions & { + removeHLSFile (options: OverrideCommandOptions & { + videoId: number | string + fileId: number + }) { + const path = '/api/v1/videos/' + options.videoId + '/hls/' + options.fileId + + return this.deleteRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + removeAllWebTorrentFiles (options: OverrideCommandOptions & { videoId: number | string }) { const path = '/api/v1/videos/' + options.videoId + '/webtorrent' @@ -633,6 +648,21 @@ export class VideosCommand extends AbstractCommand { }) } + removeWebTorrentFile (options: OverrideCommandOptions & { + videoId: number | string + fileId: number + }) { + const path = '/api/v1/videos/' + options.videoId + '/webtorrent/' + options.fileId + + return this.deleteRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + runTranscoding (options: OverrideCommandOptions & { videoId: number | string transcodingType: 'hls' | 'webtorrent' diff --git a/yarn.lock b/yarn.lock index 05fd3370a..090abda20 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4635,6 +4635,11 @@ eventemitter-asyncresource@^1.0.0: resolved "https://registry.yarnpkg.com/eventemitter-asyncresource/-/eventemitter-asyncresource-1.0.0.tgz#734ff2e44bf448e627f7748f905d6bdd57bdb65b" integrity sha512-39F7TBIV0G7gTelxwbEqnwhp90eqCPON1k0NwNfwhgKn4Co4ybUbj2pECcXT0B3ztRKZ7Pw1JujUUgmQJHcVAQ== +eventemitter3@^4.0.4: + version "4.0.7" + resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-4.0.7.tgz#2de9b68f6528d5644ef5c59526a1b4a07306169f" + integrity sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw== + events@3.3.0, events@^3.3.0: version "3.3.0" resolved "https://registry.yarnpkg.com/events/-/events-3.3.0.tgz#31a95ad0a924e2d2c419a813aeb2c4e878ea7400" @@ -7122,6 +7127,14 @@ p-map@^2.1.0: resolved "https://registry.yarnpkg.com/p-map/-/p-map-2.1.0.tgz#310928feef9c9ecc65b68b17693018a665cea175" integrity sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw== +p-queue@^6: + version "6.6.2" + resolved "https://registry.yarnpkg.com/p-queue/-/p-queue-6.6.2.tgz#2068a9dcf8e67dd0ec3e7a2bcb76810faa85e426" + integrity sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ== + dependencies: + eventemitter3 "^4.0.4" + p-timeout "^3.2.0" + p-timeout@^3.0.0, p-timeout@^3.1.0, p-timeout@^3.2.0: version "3.2.0" resolved "https://registry.yarnpkg.com/p-timeout/-/p-timeout-3.2.0.tgz#c7e17abc971d2a7962ef83626b35d635acf23dfe"