From e57c3024f42c7f84d77ddee1dc7bc71b845f9a68 Mon Sep 17 00:00:00 2001 From: chagai95 <31655082+chagai95@users.noreply.github.com> Date: Fri, 15 Mar 2024 15:47:18 +0100 Subject: [PATCH] Add config option to keep original video file (basic first version) (#6157) * testing not removing old file and adding columb to db * implement feature * remove unnecessary config changes * use only keptOriginalFileName, change keptOriginalFileName to keptOriginalFilename for consistency with with videoFile table, slight refactor with basename() * save original video files to dedicated directory original-video-files * begin implementing object storage (bucket) support --------- Co-authored-by: chagai.friedlander Co-authored-by: Ian Co-authored-by: Chocobozzz --- .../edit-custom-config.component.ts | 3 + .../edit-vod-transcoding.component.html | 15 +- .../shared/shared-main/video/video.service.ts | 2 +- .../video-download.component.html | 105 +++++--- .../video-download.component.scss | 27 +- .../video-download.component.ts | 133 ++++++---- client/src/assets/images/misc/shield.svg | 2 +- config/default.yaml | 14 +- config/dev.yaml | 3 + config/production.yaml.example | 12 + config/test-1.yaml | 1 + config/test-2.yaml | 1 + config/test-3.yaml | 1 + config/test-4.yaml | 1 + config/test-5.yaml | 1 + config/test-6.yaml | 1 + .../video-export.model.ts | 13 +- .../models/src/server/custom-config.model.ts | 4 + .../models/src/videos/video-source.model.ts | 21 +- .../src/server/config-command.ts | 28 +- .../src/server/object-storage-command.ts | 26 +- packages/server-commands/src/server/server.ts | 1 + .../src/server/servers-command.ts | 20 +- .../src/users/user-exports-command.ts | 23 +- packages/tests/src/api/check-params/config.ts | 240 +----------------- .../src/api/check-params/video-source.ts | 65 ++++- packages/tests/src/api/server/config.ts | 5 + packages/tests/src/api/users/user-export.ts | 39 ++- packages/tests/src/api/users/user-import.ts | 91 +++++-- packages/tests/src/api/videos/video-source.ts | 186 +++++++++++--- packages/tests/src/peertube-runner/index.ts | 1 + .../tests/src/peertube-runner/replace-file.ts | 86 +++++++ packages/tests/src/shared/videos.ts | 61 ++++- server/core/controllers/api/config.ts | 3 + server/core/controllers/api/videos/live.ts | 4 +- server/core/controllers/api/videos/source.ts | 25 +- server/core/controllers/api/videos/upload.ts | 21 +- server/core/controllers/download.ts | 53 +++- server/core/controllers/static.ts | 4 +- .../core/initializers/checker-after-init.ts | 58 ++++- .../core/initializers/checker-before-init.ts | 7 +- server/core/initializers/config.ts | 9 + server/core/initializers/constants.ts | 9 +- server/core/initializers/installer.ts | 4 +- .../migrations/0830-keep-original-file.ts | 91 +++++++ .../job-queue/handlers/video-live-ending.ts | 5 +- server/core/lib/local-video-creator.ts | 53 ++-- server/core/lib/object-storage/keys.ts | 15 +- .../lib/object-storage/pre-signed-urls.ts | 31 ++- server/core/lib/object-storage/urls.ts | 33 +-- server/core/lib/object-storage/videos.ts | 95 ++++--- server/core/lib/thumbnail.ts | 5 +- .../transcoding-runner-job-builder.ts | 8 +- .../core/lib/transcoding/web-transcoding.ts | 27 +- .../exporters/videos-exporter.ts | 96 ++++--- .../importers/videos-importer.ts | 52 ++-- .../lib/user-import-export/user-exporter.ts | 14 +- server/core/lib/video-file.ts | 118 +++++++-- server/core/lib/video-path-manager.ts | 16 +- server/core/lib/video-privacy.ts | 4 +- server/core/middlewares/validators/config.ts | 1 + .../middlewares/validators/shared/videos.ts | 82 +++--- .../validators/videos/shared/upload.ts | 11 +- .../validators/videos/video-source.ts | 35 ++- .../middlewares/validators/videos/videos.ts | 33 +-- server/core/models/user/user-export.ts | 6 +- .../video/formatter/video-api-format.ts | 20 +- server/core/models/video/video-file.ts | 13 +- server/core/models/video/video-source.ts | 89 ++++++- .../models/video/video-streaming-playlist.ts | 6 +- server/core/models/video/video.ts | 21 +- server/core/types/express.d.ts | 9 +- server/scripts/migrations/peertube-5.0.ts | 2 +- server/scripts/prune-storage.ts | 6 +- support/doc/api/openapi.yaml | 28 ++ 75 files changed, 1653 insertions(+), 801 deletions(-) create mode 100644 packages/tests/src/peertube-runner/replace-file.ts create mode 100644 server/core/initializers/migrations/0830-keep-original-file.ts diff --git a/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts b/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts index 08fb9e29e..6c14b5c55 100644 --- a/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts +++ b/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts @@ -226,6 +226,9 @@ export class EditCustomConfigComponent extends FormReactive implements OnInit { concurrency: CONCURRENCY_VALIDATOR, resolutions: {}, alwaysTranscodeOriginalResolution: null, + originalFile: { + keep: null + }, hls: { enabled: null }, diff --git a/client/src/app/+admin/config/edit-custom-config/edit-vod-transcoding.component.html b/client/src/app/+admin/config/edit-custom-config/edit-vod-transcoding.component.html index 588829a82..b82850858 100644 --- a/client/src/app/+admin/config/edit-custom-config/edit-vod-transcoding.component.html +++ b/client/src/app/+admin/config/edit-custom-config/edit-vod-transcoding.component.html @@ -39,7 +39,7 @@
-

Input formats

+

Input

+ +
+ + +
If enabled, the input file is not deleted after transcoding but moved in a dedicated folder or object storage
+ + +
-

Output formats

+

Output

diff --git a/client/src/app/shared/shared-main/video/video.service.ts b/client/src/app/shared/shared-main/video/video.service.ts index f2ee6f3dc..24bc9fee4 100644 --- a/client/src/app/shared/shared-main/video/video.service.ts +++ b/client/src/app/shared/shared-main/video/video.service.ts @@ -405,7 +405,7 @@ export class VideoService { getSource (videoId: number) { return this.authHttp - .get<{ source: VideoSource }>(VideoService.BASE_VIDEO_URL + '/' + videoId + '/source') + .get(VideoService.BASE_VIDEO_URL + '/' + videoId + '/source') .pipe( catchError(err => { if (err.status === 404) { diff --git a/client/src/app/shared/shared-video-miniature/video-download.component.html b/client/src/app/shared/shared-video-miniature/video-download.component.html index 7426aafe1..dd76754bb 100644 --- a/client/src/app/shared/shared-video-miniature/video-download.component.html +++ b/client/src/app/shared/shared-video-miniature/video-download.component.html @@ -16,7 +16,7 @@
-
+
The following link contains a private token and should not be shared with anyone.
@@ -45,13 +45,30 @@
- - {{ file.resolution.label }} + + +
+ +
+ + + + + Original file + + + -
- -
+ + + + + + {{ file.resolution.label }} + + +
@@ -60,47 +77,59 @@
+ + +
+ {{ item.value.label }} + + @if (item.value.value) { + {{ item.value.value }} + } @else { + Unknown + } +
+ + Format +
-
- {{ item.value.label }} - {{ item.value.value }} -
+ @for (item of videoFileMetadataFormat | keyvalue; track item) { + + }
+ - - Video stream - -
-
- {{ item.value.label }} - {{ item.value.value }} -
-
- - + + Video stream - - Audio stream - -
-
- {{ item.value.label }} - {{ item.value.value }} -
-
- - + +
+ @for (item of videoFileMetadataVideoStream | keyvalue; track item) { + + } +
+ + + + Audio stream + + +
+ @for (item of videoFileMetadataAudioStream | keyvalue; track item) { + + } +
+
-
+
-
+
@@ -121,17 +150,13 @@ - - Advanced - + More information/options - - Simple - + Less information/options diff --git a/client/src/app/shared/shared-video-miniature/video-download.component.scss b/client/src/app/shared/shared-video-miniature/video-download.component.scss index 407bdadf2..56d6dc5f6 100644 --- a/client/src/app/shared/shared-video-miniature/video-download.component.scss +++ b/client/src/app/shared/shared-video-miniature/video-download.component.scss @@ -5,6 +5,13 @@ margin-top: 30px; } +my-global-icon[iconName=shield] { + @include margin-left(10px); + + width: 16px; + margin-top: -3px; +} + .advanced-filters-button { display: flex; justify-content: center; @@ -53,7 +60,7 @@ display: block; margin-bottom: 12px; - .metadata-attribute-label { + > span:first-child { @include padding-right(5px); min-width: 142px; @@ -61,22 +68,4 @@ color: pvar(--greyForegroundColor); font-weight: $font-bold; } - - a.metadata-attribute-value { - @include disable-default-a-behaviour; - - color: pvar(--mainForegroundColor); - - &:hover { - opacity: 0.9; - } - } - - &.metadata-attribute-tags { - .metadata-attribute-value:not(:nth-child(2)) { - &::before { - content: ', '; - } - } - } } diff --git a/client/src/app/shared/shared-video-miniature/video-download.component.ts b/client/src/app/shared/shared-video-miniature/video-download.component.ts index 377dbdd8b..13d686471 100644 --- a/client/src/app/shared/shared-video-miniature/video-download.component.ts +++ b/client/src/app/shared/shared-video-miniature/video-download.component.ts @@ -1,30 +1,31 @@ -import { mapValues } from 'lodash-es' -import { firstValueFrom } from 'rxjs' -import { tap } from 'rxjs/operators' +import { KeyValuePipe, NgClass, NgFor, NgIf, NgTemplateOutlet } from '@angular/common' import { Component, ElementRef, Inject, Input, LOCALE_ID, ViewChild } from '@angular/core' -import { HooksService } from '@app/core' +import { FormsModule } from '@angular/forms' +import { AuthService, HooksService } from '@app/core' import { + NgbCollapse, NgbModal, NgbModalRef, NgbNav, + NgbNavContent, NgbNavItem, NgbNavLink, NgbNavLinkBase, - NgbNavContent, NgbNavOutlet, - NgbCollapse + NgbTooltip } from '@ng-bootstrap/ng-bootstrap' +import { objectKeysTyped, pick } from '@peertube/peertube-core-utils' +import { VideoCaption, VideoFile, VideoFileMetadata, VideoSource } from '@peertube/peertube-models' import { logger } from '@root-helpers/logger' import { videoRequiresFileToken } from '@root-helpers/video' -import { objectKeysTyped, pick } from '@peertube/peertube-core-utils' -import { VideoCaption, VideoFile } from '@peertube/peertube-models' +import { mapValues } from 'lodash-es' +import { firstValueFrom, of } from 'rxjs' +import { tap } from 'rxjs/operators' import { InputTextComponent } from '../shared-forms/input-text.component' import { GlobalIconComponent } from '../shared-icons/global-icon.component' -import { FormsModule } from '@angular/forms' -import { NgIf, NgFor, KeyValuePipe } from '@angular/common' -import { VideoDetails } from '../shared-main/video/video-details.model' import { BytesPipe } from '../shared-main/angular/bytes.pipe' import { NumberFormatterPipe } from '../shared-main/angular/number-formatter.pipe' +import { VideoDetails } from '../shared-main/video/video-details.model' import { VideoFileTokenService } from '../shared-main/video/video-file-token.service' import { VideoService } from '../shared-main/video/video.service' @@ -49,7 +50,10 @@ type FileMetadata = { [key: string]: { label: string, value: string | number } } InputTextComponent, NgbNavOutlet, NgbCollapse, - KeyValuePipe + KeyValuePipe, + NgbTooltip, + NgTemplateOutlet, + NgClass ] }) export class VideoDownloadComponent { @@ -59,7 +63,7 @@ export class VideoDownloadComponent { downloadType: 'direct' | 'torrent' = 'direct' - resolutionId: number | string = -1 + resolutionId: number | 'original' = -1 subtitleLanguageId: string videoFileMetadataFormat: FileMetadata @@ -72,6 +76,10 @@ export class VideoDownloadComponent { videoFileToken: string + originalVideoFile: VideoSource + + loaded = false + private activeModal: NgbModalRef private bytesPipe: BytesPipe @@ -83,6 +91,7 @@ export class VideoDownloadComponent { constructor ( @Inject(LOCALE_ID) private localeId: string, private modalService: NgbModal, + private authService: AuthService, private videoService: VideoService, private videoFileTokenService: VideoFileTokenService, private hooks: HooksService @@ -110,7 +119,10 @@ export class VideoDownloadComponent { } show (video: VideoDetails, videoCaptions?: VideoCaption[]) { + this.loaded = false + this.videoFileToken = undefined + this.originalVideoFile = undefined this.video = video this.videoCaptions = videoCaptions @@ -125,16 +137,40 @@ export class VideoDownloadComponent { this.subtitleLanguageId = this.videoCaptions[0].language.id } - if (this.isConfidentialVideo()) { - this.videoFileTokenService.getVideoFileToken({ videoUUID: this.video.uuid, videoPassword: this.videoPassword }) - .subscribe(({ token }) => this.videoFileToken = token) - } + this.getOriginalVideoFileObs() + .subscribe(source => { + if (source?.fileDownloadUrl) { + this.originalVideoFile = source + } + + if (this.originalVideoFile || this.isConfidentialVideo()) { + this.videoFileTokenService.getVideoFileToken({ videoUUID: this.video.uuid, videoPassword: this.videoPassword }) + .subscribe(({ token }) => { + this.videoFileToken = token + + this.loaded = true + }) + } else { + this.loaded = true + } + }) this.activeModal.shown.subscribe(() => { this.hooks.runAction('action:modal.video-download.shown', 'common') }) } + private getOriginalVideoFileObs () { + if (!this.authService.isLoggedIn()) return of(undefined) + const user = this.authService.getUser() + + if (!this.video.isOwnerOrHasSeeAllVideosRight(user)) return of(undefined) + + return this.videoService.getSource(this.video.id) + } + + // --------------------------------------------------------------------------- + onClose () { this.video = undefined this.videoCaptions = undefined @@ -152,28 +188,29 @@ export class VideoDownloadComponent { : this.getVideoFileLink() } - async onResolutionIdChange (resolutionId: number) { + async onResolutionIdChange (resolutionId: number | 'original') { this.resolutionId = resolutionId - const videoFile = this.getVideoFile() + let metadata: VideoFileMetadata - if (!videoFile.metadata) { - if (!videoFile.metadataUrl) return + if (this.resolutionId === 'original') { + metadata = this.originalVideoFile.metadata + } else { + const videoFile = this.getVideoFile() + if (!videoFile) return - await this.hydrateMetadataFromMetadataUrl(videoFile) + if (!videoFile.metadata && videoFile.metadataUrl) { + await this.hydrateMetadataFromMetadataUrl(videoFile) + } + + metadata = videoFile.metadata } - if (!videoFile.metadata) return + if (!metadata) return - this.videoFileMetadataFormat = videoFile - ? this.getMetadataFormat(videoFile.metadata.format) - : undefined - this.videoFileMetadataVideoStream = videoFile - ? this.getMetadataStream(videoFile.metadata.streams, 'video') - : undefined - this.videoFileMetadataAudioStream = videoFile - ? this.getMetadataStream(videoFile.metadata.streams, 'audio') - : undefined + this.videoFileMetadataFormat = this.getMetadataFormat(metadata.format) + this.videoFileMetadataVideoStream = this.getMetadataStream(metadata.streams, 'video') + this.videoFileMetadataAudioStream = this.getMetadataStream(metadata.streams, 'audio') } onSubtitleIdChange (subtitleId: string) { @@ -185,8 +222,10 @@ export class VideoDownloadComponent { } getVideoFile () { + if (this.resolutionId === 'original') return undefined + const file = this.getVideoFiles() - .find(f => f.resolution.id === this.resolutionId) + .find(f => f.resolution.id === this.resolutionId) if (!file) { logger.error(`Could not find file with resolution ${this.resolutionId}`) @@ -197,13 +236,17 @@ export class VideoDownloadComponent { } getVideoFileLink () { - const file = this.getVideoFile() - if (!file) return '' - - const suffix = this.isConfidentialVideo() + const suffix = this.resolutionId === 'original' || this.isConfidentialVideo() ? '?videoFileToken=' + this.videoFileToken : '' + if (this.resolutionId === 'original') { + return this.originalVideoFile.fileDownloadUrl + suffix + } + + const file = this.getVideoFile() + if (!file) return '' + switch (this.downloadType) { case 'direct': return file.fileDownloadUrl + suffix @@ -219,7 +262,7 @@ export class VideoDownloadComponent { getCaption () { const caption = this.getCaptions() - .find(c => c.language.id === this.subtitleLanguageId) + .find(c => c.language.id === this.subtitleLanguageId) if (!caption) { logger.error(`Cannot find caption ${this.subtitleLanguageId}`) @@ -237,19 +280,15 @@ export class VideoDownloadComponent { } isConfidentialVideo () { - return videoRequiresFileToken(this.video) - + return this.resolutionId === 'original' || videoRequiresFileToken(this.video) } switchToType (type: DownloadType) { this.type = type } - getFileMetadata () { - const file = this.getVideoFile() - if (!file) return undefined - - return file.metadata + hasMetadata () { + return !!this.videoFileMetadataFormat } private getMetadataFormat (format: any) { @@ -282,7 +321,9 @@ export class VideoDownloadComponent { profile: (value: string) => ({ label: $localize`Profile`, value }), bit_rate: (value: number | string) => ({ label: $localize`Bitrate`, - value: `${this.numbersPipe.transform(+value)}bps` + value: isNaN(+value) + ? undefined + : `${this.numbersPipe.transform(+value)}bps` }) } diff --git a/client/src/assets/images/misc/shield.svg b/client/src/assets/images/misc/shield.svg index b6ef58229..74b137538 100644 --- a/client/src/assets/images/misc/shield.svg +++ b/client/src/assets/images/misc/shield.svg @@ -1,5 +1,5 @@ diff --git a/config/default.yaml b/config/default.yaml index 4bd0c9a7a..8e7e5bd59 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -152,6 +152,7 @@ storage: avatars: 'storage/avatars/' web_videos: 'storage/web-videos/' streaming_playlists: 'storage/streaming-playlists/' + original_video_files: 'storage/original-video-files/' redundancy: 'storage/redundancy/' logs: 'storage/logs/' previews: 'storage/previews/' @@ -238,6 +239,12 @@ object_storage: prefix: '' base_url: '' + # Same settings but for original video files + original_video_files: + bucket_name: 'original-video-files' + prefix: '' + base_url: '' + log: level: 'info' # 'debug' | 'info' | 'warn' | 'error' @@ -526,6 +533,11 @@ video_channels: transcoding: enabled: true + original_file: + # If false the uploaded file is deleted after transcoding + # If yes it is not deleted but moved in a dedicated folder or object storage + keep: false + # Allow your users to upload .mkv, .mov, .avi, .wmv, .flv, .f4v, .3g2, .3gp, .mts, m2ts, .mxf, .nut videos allow_additional_extensions: true @@ -844,7 +856,7 @@ services: # Cards configuration to format video in Twitter/X # All other social media (Facebook, Mastodon, etc.) are supported out of the box twitter: - # Indicates the Twitter account for the website or platform where the content was published + # Indicates the Twitter/X account for the website or platform where the content was published # This is just an information injected in HTML that is required by Twitter/X username: '@Chocobozzz' diff --git a/config/dev.yaml b/config/dev.yaml index 2e0cf910c..690310333 100644 --- a/config/dev.yaml +++ b/config/dev.yaml @@ -128,3 +128,6 @@ geo_ip: video_studio: enabled: true + +transcoding: + keep_original_file: false diff --git a/config/production.yaml.example b/config/production.yaml.example index b645c5fda..f05cd2a74 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example @@ -150,6 +150,7 @@ storage: avatars: '/var/www/peertube/storage/avatars/' web_videos: '/var/www/peertube/storage/web-videos/' streaming_playlists: '/var/www/peertube/storage/streaming-playlists/' + original_video_files: '/var/www/peertube/storage/original-video-files/' redundancy: '/var/www/peertube/storage/redundancy/' logs: '/var/www/peertube/storage/logs/' previews: '/var/www/peertube/storage/previews/' @@ -236,6 +237,12 @@ object_storage: prefix: '' base_url: '' + # Same settings but for original video files + original_video_files: + bucket_name: 'original-video-files' + prefix: '' + base_url: '' + log: level: 'info' # 'debug' | 'info' | 'warn' | 'error' @@ -536,6 +543,11 @@ video_channels: transcoding: enabled: true + original_file: + # If false the uploaded file is deleted after transcoding + # If yes it is not deleted but moved in a dedicated folder or object storage + keep: false + # Allow your users to upload .mkv, .mov, .avi, .wmv, .flv, .f4v, .3g2, .3gp, .mts, m2ts, .mxf, .nut videos allow_additional_extensions: true diff --git a/config/test-1.yaml b/config/test-1.yaml index b86b48438..4207d5b33 100644 --- a/config/test-1.yaml +++ b/config/test-1.yaml @@ -15,6 +15,7 @@ storage: avatars: 'test1/avatars/' web_videos: 'test1/web-videos/' streaming_playlists: 'test1/streaming-playlists/' + original_video_files: 'test1/original-video-files/' redundancy: 'test1/redundancy/' logs: 'test1/logs/' previews: 'test1/previews/' diff --git a/config/test-2.yaml b/config/test-2.yaml index 266e44c17..1c0da1b05 100644 --- a/config/test-2.yaml +++ b/config/test-2.yaml @@ -15,6 +15,7 @@ storage: avatars: 'test2/avatars/' web_videos: 'test2/web-videos/' streaming_playlists: 'test2/streaming-playlists/' + original_video_files: 'test2/original-video-files/' redundancy: 'test2/redundancy/' logs: 'test2/logs/' previews: 'test2/previews/' diff --git a/config/test-3.yaml b/config/test-3.yaml index b31d37765..841398c4d 100644 --- a/config/test-3.yaml +++ b/config/test-3.yaml @@ -15,6 +15,7 @@ storage: avatars: 'test3/avatars/' web_videos: 'test3/web-videos/' streaming_playlists: 'test3/streaming-playlists/' + original_video_files: 'test3/original-video-files/' redundancy: 'test3/redundancy/' logs: 'test3/logs/' previews: 'test3/previews/' diff --git a/config/test-4.yaml b/config/test-4.yaml index d73b09b56..442b4ee28 100644 --- a/config/test-4.yaml +++ b/config/test-4.yaml @@ -15,6 +15,7 @@ storage: avatars: 'test4/avatars/' web_videos: 'test4/web-videos/' streaming_playlists: 'test4/streaming-playlists/' + original_video_files: 'test4/original-video-files/' redundancy: 'test4/redundancy/' logs: 'test4/logs/' previews: 'test4/previews/' diff --git a/config/test-5.yaml b/config/test-5.yaml index 56cdc5242..8a714520e 100644 --- a/config/test-5.yaml +++ b/config/test-5.yaml @@ -15,6 +15,7 @@ storage: avatars: 'test5/avatars/' web_videos: 'test5/web-videos/' streaming_playlists: 'test5/streaming-playlists/' + original_video_files: 'test5/original-video-files/' redundancy: 'test5/redundancy/' logs: 'test5/logs/' previews: 'test5/previews/' diff --git a/config/test-6.yaml b/config/test-6.yaml index 0e212c699..891c4c53d 100644 --- a/config/test-6.yaml +++ b/config/test-6.yaml @@ -15,6 +15,7 @@ storage: avatars: 'test6/avatars/' web_videos: 'test6/web-videos/' streaming_playlists: 'test6/streaming-playlists/' + original_video_files: 'test6/original-video-files/' redundancy: 'test6/redundancy/' logs: 'test6/logs/' previews: 'test6/previews/' diff --git a/packages/models/src/import-export/peertube-export-format/video-export.model.ts b/packages/models/src/import-export/peertube-export-format/video-export.model.ts index 016b469c4..a23a015ac 100644 --- a/packages/models/src/import-export/peertube-export-format/video-export.model.ts +++ b/packages/models/src/import-export/peertube-export-format/video-export.model.ts @@ -1,5 +1,6 @@ import { LiveVideoLatencyModeType, + VideoFileMetadata, VideoPrivacyType, VideoStateType, VideoStreamingPlaylistType_Type @@ -85,7 +86,17 @@ export interface VideoExportJSON { }[] source?: { - filename: string + inputFilename: string + + resolution: number + size: number + + width: number + height: number + + fps: number + + metadata: VideoFileMetadata } archiveFiles: { diff --git a/packages/models/src/server/custom-config.model.ts b/packages/models/src/server/custom-config.model.ts index 9b77147b9..a443525df 100644 --- a/packages/models/src/server/custom-config.model.ts +++ b/packages/models/src/server/custom-config.model.ts @@ -117,6 +117,10 @@ export interface CustomConfig { transcoding: { enabled: boolean + originalFile: { + keep: boolean + } + allowAdditionalExtensions: boolean allowAudioFiles: boolean diff --git a/packages/models/src/videos/video-source.model.ts b/packages/models/src/videos/video-source.model.ts index bf4ad2453..4dcbb35d9 100644 --- a/packages/models/src/videos/video-source.model.ts +++ b/packages/models/src/videos/video-source.model.ts @@ -1,4 +1,23 @@ +import { VideoFileMetadata } from './file/index.js' +import { VideoConstant } from './video-constant.model.js' + export interface VideoSource { - filename: string + inputFilename: string + + resolution?: VideoConstant + size?: number // Bytes + + width?: number + height?: number + + fileDownloadUrl: string + + fps?: number + + metadata?: VideoFileMetadata + createdAt: string | Date + + // TODO: remove, deprecated in 6.1 + filename: string } diff --git a/packages/server-commands/src/server/config-command.ts b/packages/server-commands/src/server/config-command.ts index 64be64bf5..20d1d350c 100644 --- a/packages/server-commands/src/server/config-command.ts +++ b/packages/server-commands/src/server/config-command.ts @@ -106,6 +106,19 @@ export class ConfigCommand extends AbstractCommand { // --------------------------------------------------------------------------- + keepSourceFile () { + return this.updateExistingSubConfig({ + newConfig: { + transcoding: { + originalFile: { + keep: true + } + } + } + }) + } + // --------------------------------------------------------------------------- + enableChannelSync () { return this.setChannelSyncEnabled(true) } @@ -234,13 +247,17 @@ export class ConfigCommand extends AbstractCommand { webVideo?: boolean // default true hls?: boolean // default true with0p?: boolean // default false + keepOriginal?: boolean // default false } = {}) { - const { webVideo = true, hls = true, with0p = false } = options + const { webVideo = true, hls = true, with0p = false, keepOriginal = false } = options return this.updateExistingSubConfig({ newConfig: { transcoding: { enabled: true, + originalFile: { + keep: keepOriginal + }, allowAudioFiles: true, allowAdditionalExtensions: true, @@ -261,13 +278,17 @@ export class ConfigCommand extends AbstractCommand { enableMinimumTranscoding (options: { webVideo?: boolean // default true hls?: boolean // default true + keepOriginal?: boolean // default false } = {}) { - const { webVideo = true, hls = true } = options + const { webVideo = true, hls = true, keepOriginal = false } = options return this.updateExistingSubConfig({ newConfig: { transcoding: { enabled: true, + originalFile: { + keep: keepOriginal + }, allowAudioFiles: true, allowAdditionalExtensions: true, @@ -560,6 +581,9 @@ export class ConfigCommand extends AbstractCommand { }, transcoding: { enabled: true, + originalFile: { + keep: false + }, remoteRunners: { enabled: false }, diff --git a/packages/server-commands/src/server/object-storage-command.ts b/packages/server-commands/src/server/object-storage-command.ts index ed9b3a394..7959dceaa 100644 --- a/packages/server-commands/src/server/object-storage-command.ts +++ b/packages/server-commands/src/server/object-storage-command.ts @@ -1,5 +1,5 @@ -import { randomInt } from 'crypto' import { HttpStatusCode } from '@peertube/peertube-models' +import { randomInt } from 'crypto' import { makePostBodyRequest } from '../requests/index.js' export class ObjectStorageCommand { @@ -50,6 +50,14 @@ export class ObjectStorageCommand { web_videos: { bucket_name: this.getMockWebVideosBucketName() + }, + + user_exports: { + bucket_name: this.getMockUserExportBucketName() + }, + + original_video_files: { + bucket_name: this.getMockOriginalFileBucketName() } } } @@ -63,6 +71,14 @@ export class ObjectStorageCommand { return `http://${this.getMockStreamingPlaylistsBucketName()}.${ObjectStorageCommand.getMockEndpointHost()}/` } + getMockUserExportBaseUrl () { + return `http://${this.getMockUserExportBucketName()}.${ObjectStorageCommand.getMockEndpointHost()}/` + } + + getMockOriginalFileBaseUrl () { + return `http://${this.getMockOriginalFileBucketName()}.${ObjectStorageCommand.getMockEndpointHost()}/` + } + async prepareDefaultMockBuckets () { await this.createMockBucket(this.getMockStreamingPlaylistsBucketName()) await this.createMockBucket(this.getMockWebVideosBucketName()) @@ -100,6 +116,14 @@ export class ObjectStorageCommand { return this.getMockBucketName(name) } + getMockUserExportBucketName (name = 'user-exports') { + return this.getMockBucketName(name) + } + + getMockOriginalFileBucketName (name = 'original-video-files') { + return this.getMockBucketName(name) + } + getMockBucketName (name: string) { return `${this.seed}-${name}` } diff --git a/packages/server-commands/src/server/server.ts b/packages/server-commands/src/server/server.ts index ef0491c73..8e5e1eaf0 100644 --- a/packages/server-commands/src/server/server.ts +++ b/packages/server-commands/src/server/server.ts @@ -379,6 +379,7 @@ export class PeerTubeServer { avatars: this.getDirectoryPath('avatars') + '/', web_videos: this.getDirectoryPath('web-videos') + '/', streaming_playlists: this.getDirectoryPath('streaming-playlists') + '/', + original_video_files: this.getDirectoryPath('original-video-files') + '/', redundancy: this.getDirectoryPath('redundancy') + '/', logs: this.getDirectoryPath('logs') + '/', previews: this.getDirectoryPath('previews') + '/', diff --git a/packages/server-commands/src/server/servers-command.ts b/packages/server-commands/src/server/servers-command.ts index 0b722b62f..b6190e6ce 100644 --- a/packages/server-commands/src/server/servers-command.ts +++ b/packages/server-commands/src/server/servers-command.ts @@ -1,10 +1,10 @@ -import { exec } from 'child_process' -import { copy, ensureDir, remove } from 'fs-extra/esm' -import { readdir, readFile } from 'fs/promises' -import { basename, join } from 'path' import { wait } from '@peertube/peertube-core-utils' import { HttpStatusCode } from '@peertube/peertube-models' -import { getFileSize, isGithubCI, root } from '@peertube/peertube-node-utils' +import { isGithubCI, root } from '@peertube/peertube-node-utils' +import { exec } from 'child_process' +import { copy, ensureDir, remove } from 'fs-extra/esm' +import { readFile, readdir } from 'fs/promises' +import { basename, join } from 'path' import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' export class ServersCommand extends AbstractCommand { @@ -84,6 +84,8 @@ export class ServersCommand extends AbstractCommand { return files.length } + // --------------------------------------------------------------------------- + buildWebVideoFilePath (fileUrl: string) { return this.buildDirectory(join('web-videos', basename(fileUrl))) } @@ -92,13 +94,9 @@ export class ServersCommand extends AbstractCommand { return this.buildDirectory(join('streaming-playlists', 'hls', videoUUID, basename(fileUrl))) } + // --------------------------------------------------------------------------- + getLogContent () { return readFile(this.buildDirectory('logs/peertube.log')) } - - async getServerFileSize (subPath: string) { - const path = this.server.servers.buildDirectory(subPath) - - return getFileSize(path) - } } diff --git a/packages/server-commands/src/users/user-exports-command.ts b/packages/server-commands/src/users/user-exports-command.ts index 2cac8e6e6..79ff1e41a 100644 --- a/packages/server-commands/src/users/user-exports-command.ts +++ b/packages/server-commands/src/users/user-exports-command.ts @@ -1,7 +1,8 @@ -import { HttpStatusCode, ResultList, UserExport, UserExportRequestResult, UserExportState } from '@peertube/peertube-models' -import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' import { wait } from '@peertube/peertube-core-utils' -import { unwrapBody } from '../requests/requests.js' +import { HttpStatusCode, ResultList, UserExport, UserExportRequestResult, UserExportState } from '@peertube/peertube-models' +import { writeFile } from 'fs/promises' +import { makeRawRequest, unwrapBody } from '../requests/requests.js' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' export class UserExportsCommand extends AbstractCommand { @@ -49,6 +50,22 @@ export class UserExportsCommand extends AbstractCommand { }) } + async downloadLatestArchive (options: OverrideCommandOptions & { + userId: number + destination: string + }) { + const { data } = await this.list(options) + + const res = await makeRawRequest({ + url: data[0].privateDownloadUrl, + responseType: 'arraybuffer', + redirects: 1, + expectedStatus: HttpStatusCode.OK_200 + }) + + await writeFile(options.destination, res.body) + } + async deleteAllArchives (options: OverrideCommandOptions & { userId: number }) { diff --git a/packages/tests/src/api/check-params/config.ts b/packages/tests/src/api/check-params/config.ts index 3eef13e6f..f6f3aea7e 100644 --- a/packages/tests/src/api/check-params/config.ts +++ b/packages/tests/src/api/check-params/config.ts @@ -19,244 +19,7 @@ describe('Test config API validators', function () { let server: PeerTubeServer let userAccessToken: string - const updateParams: CustomConfig = { - instance: { - name: 'PeerTube updated', - shortDescription: 'my short description', - description: 'my super description', - terms: 'my super terms', - codeOfConduct: 'my super coc', - - creationReason: 'my super reason', - moderationInformation: 'my super moderation information', - administrator: 'Kuja', - maintenanceLifetime: 'forever', - businessModel: 'my super business model', - hardwareInformation: '2vCore 3GB RAM', - - languages: [ 'en', 'es' ], - categories: [ 1, 2 ], - - isNSFW: true, - defaultNSFWPolicy: 'blur', - - defaultClientRoute: '/videos/recently-added', - - customizations: { - javascript: 'alert("coucou")', - css: 'body { background-color: red; }' - } - }, - theme: { - default: 'default' - }, - services: { - twitter: { - username: '@MySuperUsername' - } - }, - client: { - videos: { - miniature: { - preferAuthorDisplayName: false - } - }, - menu: { - login: { - redirectOnSingleExternalAuth: false - } - } - }, - cache: { - previews: { - size: 2 - }, - captions: { - size: 3 - }, - torrents: { - size: 4 - }, - storyboards: { - size: 5 - } - }, - signup: { - enabled: false, - limit: 5, - requiresApproval: false, - requiresEmailVerification: false, - minimumAge: 16 - }, - admin: { - email: 'superadmin1@example.com' - }, - contactForm: { - enabled: false - }, - user: { - history: { - videos: { - enabled: true - } - }, - videoQuota: 5242881, - videoQuotaDaily: 318742, - defaultChannelName: 'Main $1 channel' - }, - videoChannels: { - maxPerUser: 20 - }, - transcoding: { - enabled: true, - remoteRunners: { - enabled: true - }, - allowAdditionalExtensions: true, - allowAudioFiles: true, - concurrency: 1, - threads: 1, - profile: 'vod_profile', - resolutions: { - '0p': false, - '144p': false, - '240p': false, - '360p': true, - '480p': true, - '720p': false, - '1080p': false, - '1440p': false, - '2160p': false - }, - alwaysTranscodeOriginalResolution: false, - webVideos: { - enabled: true - }, - hls: { - enabled: false - } - }, - live: { - enabled: true, - - allowReplay: false, - latencySetting: { - enabled: false - }, - maxDuration: 30, - maxInstanceLives: -1, - maxUserLives: 50, - - transcoding: { - enabled: true, - remoteRunners: { - enabled: true - }, - threads: 4, - profile: 'live_profile', - resolutions: { - '144p': true, - '240p': true, - '360p': true, - '480p': true, - '720p': true, - '1080p': true, - '1440p': true, - '2160p': true - }, - alwaysTranscodeOriginalResolution: false - } - }, - videoStudio: { - enabled: true, - remoteRunners: { - enabled: true - } - }, - videoFile: { - update: { - enabled: true - } - }, - import: { - videos: { - concurrency: 1, - http: { - enabled: false - }, - torrent: { - enabled: false - } - }, - videoChannelSynchronization: { - enabled: false, - maxPerUser: 10 - }, - users: { - enabled: false - } - }, - export: { - users: { - enabled: false, - maxUserVideoQuota: 40, - exportExpiration: 10 - } - }, - trending: { - videos: { - algorithms: { - enabled: [ 'hot', 'most-viewed', 'most-liked' ], - default: 'most-viewed' - } - } - }, - autoBlacklist: { - videos: { - ofUsers: { - enabled: false - } - } - }, - followers: { - instance: { - enabled: false, - manualApproval: true - } - }, - followings: { - instance: { - autoFollowBack: { - enabled: true - }, - autoFollowIndex: { - enabled: true, - indexUrl: 'https://index.example.com' - } - } - }, - broadcastMessage: { - enabled: true, - dismissable: true, - message: 'super message', - level: 'warning' - }, - search: { - remoteUri: { - users: true, - anonymous: true - }, - searchIndex: { - enabled: true, - url: 'https://search.joinpeertube.org', - disableLocalSearch: true, - isDefaultSearch: true - } - }, - storyboards: { - enabled: false - } - } + let updateParams: CustomConfig // --------------------------------------------------------------- @@ -266,6 +29,7 @@ describe('Test config API validators', function () { server = await createSingleServer(1) await setAccessTokensToServers([ server ]) + updateParams = await server.config.getCustomConfig() const user = { username: 'user1', diff --git a/packages/tests/src/api/check-params/video-source.ts b/packages/tests/src/api/check-params/video-source.ts index 918182b8d..8076fa8bd 100644 --- a/packages/tests/src/api/check-params/video-source.ts +++ b/packages/tests/src/api/check-params/video-source.ts @@ -1,8 +1,9 @@ -import { HttpStatusCode } from '@peertube/peertube-models' +import { HttpStatusCode, VideoSource } from '@peertube/peertube-models' import { + PeerTubeServer, cleanupTests, createSingleServer, - PeerTubeServer, + makeRawRequest, setAccessTokensToServers, setDefaultVideoChannel, waitJobs @@ -148,6 +149,66 @@ describe('Test video sources API validator', function () { }) }) + describe('When downloading the source file', function () { + let videoFileToken: string + let videoId: string + let source: VideoSource + let user3: string + let user4: string + + before(async function () { + this.timeout(60000) + + user3 = await server.users.generateUserAndToken('user3') + user4 = await server.users.generateUserAndToken('user4') + + await server.config.enableMinimumTranscoding({ hls: true, keepOriginal: true, webVideo: true }) + + const { uuid } = await server.videos.quickUpload({ name: 'video', token: user3 }) + + videoId = uuid + videoFileToken = await server.videoToken.getVideoFileToken({ videoId: uuid, token: user3 }) + + await waitJobs([ server ]) + + source = await server.videos.getSource({ id: videoId, token: user3 }) + }) + + it('Should fail with an invalid filename', async function () { + await makeRawRequest({ url: server.url + '/download/original-video-files/hello.mp4', expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should fail without header token or video file token', async function () { + await makeRawRequest({ url: source.fileDownloadUrl, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + + it('Should fail with an invalid header token', async function () { + await makeRawRequest({ url: source.fileDownloadUrl, token: 'toto', expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should fail with an invalid video file token', async function () { + await makeRawRequest({ url: source.fileDownloadUrl, query: { videoFileToken: 'toto' }, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + + it('Should fail with header token of another user', async function () { + await makeRawRequest({ url: source.fileDownloadUrl, token: user4, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + + it('Should fail with video file token of another user', async function () { + const videoFileToken = await server.videoToken.getVideoFileToken({ videoId: uuid, token: user4 }) + + await makeRawRequest({ url: source.fileDownloadUrl, query: { videoFileToken }, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + + it('Should succeed with a valid header token', async function () { + await makeRawRequest({ url: source.fileDownloadUrl, token: user3, expectedStatus: HttpStatusCode.OK_200 }) + }) + + it('Should succeed with a valid header token', async function () { + await makeRawRequest({ url: source.fileDownloadUrl, query: { videoFileToken }, expectedStatus: HttpStatusCode.OK_200 }) + }) + }) + after(async function () { await cleanupTests([ server ]) }) diff --git a/packages/tests/src/api/server/config.ts b/packages/tests/src/api/server/config.ts index e503541a8..217396de9 100644 --- a/packages/tests/src/api/server/config.ts +++ b/packages/tests/src/api/server/config.ts @@ -84,6 +84,7 @@ function checkInitialConfig (server: PeerTubeServer, data: CustomConfig) { expect(data.transcoding.alwaysTranscodeOriginalResolution).to.be.true expect(data.transcoding.webVideos.enabled).to.be.true expect(data.transcoding.hls.enabled).to.be.true + expect(data.transcoding.originalFile.keep).to.be.false expect(data.live.enabled).to.be.false expect(data.live.allowReplay).to.be.false @@ -205,6 +206,7 @@ function checkUpdatedConfig (data: CustomConfig) { expect(data.transcoding.alwaysTranscodeOriginalResolution).to.be.false expect(data.transcoding.hls.enabled).to.be.false expect(data.transcoding.webVideos.enabled).to.be.true + expect(data.transcoding.originalFile.keep).to.be.true expect(data.live.enabled).to.be.true expect(data.live.allowReplay).to.be.true @@ -349,6 +351,9 @@ const newCustomConfig: CustomConfig = { remoteRunners: { enabled: true }, + originalFile: { + keep: true + }, allowAdditionalExtensions: true, allowAudioFiles: true, threads: 1, diff --git a/packages/tests/src/api/users/user-export.ts b/packages/tests/src/api/users/user-export.ts index cb39f57ea..ca51c403d 100644 --- a/packages/tests/src/api/users/user-export.ts +++ b/packages/tests/src/api/users/user-export.ts @@ -1,14 +1,6 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ -import { MockSmtpServer } from '@tests/shared/mock-servers/index.js' -import { - cleanupTests, getRedirectionUrl, makeActivityPubRawRequest, - makeRawRequest, - ObjectStorageCommand, - PeerTubeServer, - waitJobs -} from '@peertube/peertube-server-commands' -import { expect } from 'chai' +import { wait } from '@peertube/peertube-core-utils' import { AccountExportJSON, ActivityPubActor, ActivityPubOrderedCollection, @@ -34,6 +26,15 @@ import { VideoPlaylistType, VideoPrivacy } from '@peertube/peertube-models' +import { areMockObjectStorageTestsDisabled } from '@peertube/peertube-node-utils' +import { + cleanupTests, getRedirectionUrl, makeActivityPubRawRequest, + makeRawRequest, + ObjectStorageCommand, + PeerTubeServer, + waitJobs +} from '@peertube/peertube-server-commands' +import { expectStartWith } from '@tests/shared/checks.js' import { checkExportFileExists, checkFileExistsInZIP, @@ -44,8 +45,8 @@ import { prepareImportExportTests, regenerateExport } from '@tests/shared/import-export.js' -import { areMockObjectStorageTestsDisabled } from '@peertube/peertube-node-utils' -import { wait } from '@peertube/peertube-core-utils' +import { MockSmtpServer } from '@tests/shared/mock-servers/index.js' +import { expect } from 'chai' function runTest (withObjectStorage: boolean) { let server: PeerTubeServer @@ -69,10 +70,12 @@ function runTest (withObjectStorage: boolean) { let noahExportId: number + let objectStorage: ObjectStorageCommand + before(async function () { this.timeout(240000) - const objectStorage = withObjectStorage + objectStorage = withObjectStorage ? new ObjectStorageCommand() : undefined; @@ -126,6 +129,10 @@ function runTest (withObjectStorage: boolean) { expect(data[0].size).to.be.greaterThan(0) expect(data[0].state.id).to.equal(UserExportState.COMPLETED) expect(data[0].state.label).to.equal('Completed') + + if (objectStorage) { + expectStartWith(await getRedirectionUrl(data[0].privateDownloadUrl), objectStorage.getMockUserExportBaseUrl()) + } } await waitJobs([ server ]) @@ -526,6 +533,14 @@ function runTest (withObjectStorage: boolean) { for (const url of urls) { await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 }) } + + expect(publicVideo.source.inputFilename).to.equal('video_short.webm') + expect(publicVideo.source.fps).to.equal(25) + expect(publicVideo.source.height).to.equal(720) + expect(publicVideo.source.width).to.equal(1280) + expect(publicVideo.source.metadata?.streams).to.exist + expect(publicVideo.source.resolution).to.equal(720) + expect(publicVideo.source.size).to.equal(218910) } { diff --git a/packages/tests/src/api/users/user-import.ts b/packages/tests/src/api/users/user-import.ts index 02df833f0..4416cd1b2 100644 --- a/packages/tests/src/api/users/user-import.ts +++ b/packages/tests/src/api/users/user-import.ts @@ -1,11 +1,5 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ -import { MockSmtpServer } from '@tests/shared/mock-servers/index.js' -import { - cleanupTests, makeRawRequest, - ObjectStorageCommand, - PeerTubeServer, waitJobs -} from '@peertube/peertube-server-commands' import { HttpStatusCode, LiveVideoLatencyMode, @@ -17,14 +11,20 @@ import { VideoPrivacy, VideoState } from '@peertube/peertube-models' -import { prepareImportExportTests } from '@tests/shared/import-export.js' import { areMockObjectStorageTestsDisabled } from '@peertube/peertube-node-utils' -import { writeFile } from 'fs/promises' -import { join } from 'path' -import { expect } from 'chai' -import { testImage, testAvatarSize } from '@tests/shared/checks.js' -import { completeVideoCheck } from '@tests/shared/videos.js' +import { + ObjectStorageCommand, + PeerTubeServer, + cleanupTests, + waitJobs +} from '@peertube/peertube-server-commands' +import { testAvatarSize, testImage } from '@tests/shared/checks.js' +import { prepareImportExportTests } from '@tests/shared/import-export.js' +import { MockSmtpServer } from '@tests/shared/mock-servers/index.js' import { completeCheckHlsPlaylist } from '@tests/shared/streaming-playlists.js' +import { completeVideoCheck } from '@tests/shared/videos.js' +import { expect } from 'chai' +import { join } from 'path' function runTest (withObjectStorage: boolean) { let server: PeerTubeServer @@ -115,17 +115,8 @@ function runTest (withObjectStorage: boolean) { await server.userExports.request({ userId: noahId, withVideoFiles: true }) await server.userExports.waitForCreation({ userId: noahId }) - const { data } = await server.userExports.list({ userId: noahId }) - - const res = await makeRawRequest({ - url: data[0].privateDownloadUrl, - responseType: 'arraybuffer', - redirects: 1, - expectedStatus: HttpStatusCode.OK_200 - }) - archivePath = join(server.getDirectoryPath('tmp'), 'archive.zip') - await writeFile(archivePath, res.body) + await server.userExports.downloadLatestArchive({ userId: noahId, destination: archivePath }) }) it('Should import an archive with video files', async function () { @@ -444,6 +435,11 @@ function runTest (withObjectStorage: boolean) { const source = await remoteServer.videos.getSource({ id: otherVideo.uuid }) expect(source.filename).to.equal('video_short.webm') + expect(source.inputFilename).to.equal('video_short.webm') + expect(source.fileDownloadUrl).to.not.exist + + expect(source.metadata?.format).to.exist + expect(source.metadata?.streams).to.be.an('array') } { @@ -572,6 +568,57 @@ function runTest (withObjectStorage: boolean) { } }) + it('Should import original file if included in the export', async function () { + this.timeout(120000) + + await server.config.enableMinimumTranscoding({ keepOriginal: true }) + await remoteServer.config.keepSourceFile() + + const archivePath = join(server.getDirectoryPath('tmp'), 'archive2.zip') + const fixture = 'video_short1.webm' + + { + const { token, userId } = await server.users.generate('claire') + + await server.videos.quickUpload({ name: 'claire video', token, fixture }) + + await waitJobs([ server ]) + + await server.userExports.request({ userId, token, withVideoFiles: true }) + await server.userExports.waitForCreation({ userId, token }) + + await server.userExports.downloadLatestArchive({ userId, token, destination: archivePath }) + } + + { + const { token, userId } = await remoteServer.users.generate('external_claire') + + await remoteServer.userImports.importArchive({ fixture: archivePath, userId, token }) + await waitJobs([ remoteServer ]) + + { + const { data } = await remoteServer.videos.listMyVideos({ token }) + expect(data).to.have.lengthOf(1) + + const source = await remoteServer.videos.getSource({ id: data[0].id }) + expect(source.filename).to.equal(fixture) + expect(source.inputFilename).to.equal(fixture) + expect(source.fileDownloadUrl).to.exist + + expect(source.metadata?.format).to.exist + expect(source.metadata?.streams).to.be.an('array') + expect(source.metadata.format['format_name']).to.include('webm') + + expect(source.createdAt).to.exist + expect(source.fps).to.equal(25) + expect(source.height).to.equal(720) + expect(source.width).to.equal(1280) + expect(source.resolution.id).to.equal(720) + expect(source.size).to.equal(572456) + } + } + }) + after(async function () { MockSmtpServer.Instance.kill() diff --git a/packages/tests/src/api/videos/video-source.ts b/packages/tests/src/api/videos/video-source.ts index b33e84e66..73d620aa1 100644 --- a/packages/tests/src/api/videos/video-source.ts +++ b/packages/tests/src/api/videos/video-source.ts @@ -1,24 +1,26 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ -import { expect } from 'chai' import { getAllFiles } from '@peertube/peertube-core-utils' -import { HttpStatusCode } from '@peertube/peertube-models' -import { expectStartWith } from '@tests/shared/checks.js' +import { HttpStatusCode, VideoPrivacy } from '@peertube/peertube-models' import { areMockObjectStorageTestsDisabled } from '@peertube/peertube-node-utils' import { - cleanupTests, - createMultipleServers, - doubleFollow, - makeGetRequest, - makeRawRequest, ObjectStorageCommand, PeerTubeServer, + cleanupTests, + createMultipleServers, + doubleFollow, makeGetRequest, + makeRawRequest, setAccessTokensToServers, setDefaultAccountAvatar, setDefaultVideoChannel, waitJobs } from '@peertube/peertube-server-commands' +import { expectStartWith } from '@tests/shared/checks.js' +import { checkDirectoryIsEmpty } from '@tests/shared/directories.js' +import { FIXTURE_URLS } from '@tests/shared/tests.js' +import { checkSourceFile } from '@tests/shared/videos.js' +import { expect } from 'chai' -describe('Test a video file replacement', function () { +describe('Test video source management', function () { let servers: PeerTubeServer[] = [] let replaceDate: Date @@ -36,6 +38,7 @@ describe('Test a video file replacement', function () { await setDefaultAccountAvatar(servers) await servers[0].config.enableFileUpdate() + await servers[0].config.enableMinimumTranscoding() userToken = await servers[0].users.generateUserAndToken('user1') @@ -44,30 +47,95 @@ describe('Test a video file replacement', function () { }) describe('Getting latest video source', () => { - const fixture = 'video_short.webm' + const fixture1 = 'video_short.webm' + const fixture2 = 'video_short1.webm' + const uuids: string[] = [] - it('Should get the source filename with legacy upload', async function () { + it('Should get the source filename with legacy upload with disabled keep original file', async function () { this.timeout(30000) - const { uuid } = await servers[0].videos.upload({ attributes: { name: 'my video', fixture }, mode: 'legacy' }) + const { uuid } = await servers[0].videos.upload({ attributes: { name: 'my video', fixture: fixture1 }, mode: 'legacy' }) uuids.push(uuid) + await waitJobs(servers) + const source = await servers[0].videos.getSource({ id: uuid }) - expect(source.filename).to.equal(fixture) + expect(source.filename).to.equal(fixture1) + expect(source.inputFilename).to.equal(fixture1) + expect(source.fileDownloadUrl).to.be.null + + expect(source.createdAt).to.exist + expect(source.fps).to.equal(25) + expect(source.height).to.equal(720) + expect(source.width).to.equal(1280) + expect(source.resolution.id).to.equal(720) + expect(source.size).to.equal(218910) + + expect(source.metadata?.format).to.exist + expect(source.metadata?.streams).to.be.an('array') + + await checkDirectoryIsEmpty(servers[0], 'original-video-files') }) - it('Should get the source filename with resumable upload', async function () { + it('Should get the source filename with resumable upload and enabled keep original file', async function () { this.timeout(30000) - const { uuid } = await servers[0].videos.upload({ attributes: { name: 'my video', fixture }, mode: 'resumable' }) + await servers[0].config.keepSourceFile() + + const { uuid } = await servers[0].videos.upload({ attributes: { name: 'my video', fixture: fixture2 }, mode: 'resumable' }) uuids.push(uuid) + await waitJobs(servers) + const source = await servers[0].videos.getSource({ id: uuid }) - expect(source.filename).to.equal(fixture) + expect(source.filename).to.equal(fixture2) + expect(source.inputFilename).to.equal(fixture2) + expect(source.fileDownloadUrl).to.exist + + expect(source.createdAt).to.exist + expect(source.fps).to.equal(25) + expect(source.height).to.equal(720) + expect(source.width).to.equal(1280) + expect(source.resolution.id).to.equal(720) + expect(source.size).to.equal(572456) + + expect(source.metadata?.format).to.exist + expect(source.metadata?.streams).to.be.an('array') }) - after(async function () { + it('Should have kept original video file', async function () { + await checkSourceFile({ server: servers[0], fsCount: 1, fixture: fixture2, uuid: uuids[uuids.length - 1] }) + }) + + it('Should transcode a file but do not replace original file', async function () { + await servers[0].videos.runTranscoding({ transcodingType: 'web-video', videoId: uuids[0] }) + await servers[0].videos.runTranscoding({ transcodingType: 'web-video', videoId: uuids[1] }) + + await checkSourceFile({ server: servers[0], fsCount: 1, fixture: fixture2, uuid: uuids[uuids.length - 1] }) + }) + + it('Should also keep audio files', async function () { + const fixture = 'sample.ogg' + const { uuid } = await servers[0].videos.quickUpload({ name: 'audio', fixture }) + uuids.push(uuid) + + await waitJobs(servers) + const source = await checkSourceFile({ server: servers[0], fsCount: 2, fixture, uuid }) + + expect(source.createdAt).to.exist + expect(source.fps).to.equal(0) + expect(source.height).to.equal(0) + expect(source.width).to.equal(0) + expect(source.resolution.id).to.equal(0) + expect(source.resolution.label).to.equal('Audio') + expect(source.size).to.equal(105243) + + expect(source.metadata?.format).to.exist + expect(source.metadata?.streams).to.be.an('array') + }) + + it('Should delete all videos and do not have original files anymore', async function () { this.timeout(60000) for (const uuid of uuids) { @@ -75,6 +143,23 @@ describe('Test a video file replacement', function () { } await waitJobs(servers) + + await checkDirectoryIsEmpty(servers[0], 'original-video-files') + }) + + it('Should not have source on import', async function () { + const { video: { uuid } } = await servers[0].videoImports.importVideo({ + attributes: { + channelId: servers[0].store.channel.id, + targetUrl: FIXTURE_URLS.goodVideo, + privacy: VideoPrivacy.PUBLIC + } + }) + + await waitJobs(servers) + + await servers[0].videos.getSource({ id: uuid, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + await checkDirectoryIsEmpty(servers[0], 'original-video-files') }) }) @@ -110,18 +195,25 @@ describe('Test a video file replacement', function () { } }) + it('Should not have kept original video file', async function () { + await checkDirectoryIsEmpty(servers[0], 'original-video-files') + }) + it('Should replace a video file with transcoding enabled', async function () { this.timeout(240000) const previousPaths: string[] = [] - await servers[0].config.enableTranscoding({ hls: true, webVideo: true, with0p: true }) + await servers[0].config.enableTranscoding({ hls: true, webVideo: true, with0p: true, keepOriginal: true }) - const { uuid: videoUUID } = await servers[0].videos.quickUpload({ name: 'fs with transcoding', fixture: 'video_short_720p.mp4' }) + const uploadFixture = 'video_short_720p.mp4' + const { uuid: videoUUID } = await servers[0].videos.quickUpload({ name: 'fs with transcoding', fixture: uploadFixture }) uuid = videoUUID await waitJobs(servers) + await checkSourceFile({ server: servers[0], fsCount: 1, uuid, fixture: uploadFixture }) + for (const server of servers) { const video = await server.videos.get({ id: uuid }) expect(video.inputFileUpdatedAt).to.be.null @@ -151,9 +243,23 @@ describe('Test a video file replacement', function () { replaceDate = new Date() - await servers[0].videos.replaceSourceFile({ videoId: uuid, fixture: 'video_short_360p.mp4' }) + const replaceFixture = 'video_short_360p.mp4' + await servers[0].videos.replaceSourceFile({ videoId: uuid, fixture: replaceFixture }) await waitJobs(servers) + const source = await checkSourceFile({ server: servers[0], fsCount: 1, uuid, fixture: replaceFixture }) + + expect(source.createdAt).to.exist + expect(source.fps).to.equal(25) + expect(source.height).to.equal(360) + expect(source.width).to.equal(640) + expect(source.resolution.id).to.equal(360) + expect(source.resolution.label).to.equal('360p') + expect(source.size).to.equal(30620) + + expect(source.metadata?.format).to.exist + expect(source.metadata?.streams).to.be.an('array') + for (const server of servers) { const video = await server.videos.get({ id: uuid }) @@ -189,35 +295,36 @@ describe('Test a video file replacement', function () { } } - await servers[0].config.enableMinimumTranscoding() + await servers[0].config.enableMinimumTranscoding({ keepOriginal: true }) }) it('Should have cleaned up old files', async function () { { const count = await servers[0].servers.countFiles('storyboards') - expect(count).to.equal(2) + expect(count).to.equal(3) } { const count = await servers[0].servers.countFiles('web-videos') - expect(count).to.equal(5 + 1) // +1 for private directory + expect(count).to.equal(6 + 1) // +1 for private directory } { const count = await servers[0].servers.countFiles('streaming-playlists/hls') - expect(count).to.equal(1 + 1) // +1 for private directory + expect(count).to.equal(2 + 1) // +1 for private directory } { const count = await servers[0].servers.countFiles('torrents') - expect(count).to.equal(9) + expect(count).to.equal(11) } }) - it('Should have the correct source input', async function () { + it('Should have the correct source input filename', async function () { const source = await servers[0].videos.getSource({ id: uuid }) expect(source.filename).to.equal('video_short_360p.mp4') + expect(source.inputFilename).to.equal('video_short_360p.mp4') expect(new Date(source.createdAt)).to.be.above(replaceDate) }) @@ -367,6 +474,9 @@ describe('Test a video file replacement', function () { expect(files[0].resolution.id).to.equal(360) expectStartWith(files[0].fileUrl, objectStorage.getMockWebVideosBaseUrl()) } + + const source = await servers[0].videos.getSource({ id: uuid }) + expect(source.fileDownloadUrl).to.not.exist }) it('Should replace a video file with transcoding enabled', async function () { @@ -374,16 +484,25 @@ describe('Test a video file replacement', function () { const previousPaths: string[] = [] - await servers[0].config.enableTranscoding({ hls: true, webVideo: true, with0p: true }) + await servers[0].config.enableTranscoding({ hls: true, webVideo: true, with0p: true, keepOriginal: true }) + const fixture1 = 'video_short_360p.mp4' const { uuid: videoUUID } = await servers[0].videos.quickUpload({ name: 'object storage with transcoding', - fixture: 'video_short_360p.mp4' + fixture: fixture1 }) uuid = videoUUID await waitJobs(servers) + await checkSourceFile({ + server: servers[0], + fixture: fixture1, + fsCount: 0, + uuid, + objectStorageBaseUrl: objectStorage?.getMockOriginalFileBaseUrl() + }) + for (const server of servers) { const video = await server.videos.get({ id: uuid }) @@ -403,9 +522,18 @@ describe('Test a video file replacement', function () { } } - await servers[0].videos.replaceSourceFile({ videoId: uuid, fixture: 'video_short_240p.mp4' }) + const fixture2 = 'video_short_240p.mp4' + await servers[0].videos.replaceSourceFile({ videoId: uuid, fixture: fixture2 }) await waitJobs(servers) + await checkSourceFile({ + server: servers[0], + fixture: fixture2, + fsCount: 0, + uuid, + objectStorageBaseUrl: objectStorage?.getMockOriginalFileBaseUrl() + }) + for (const server of servers) { const video = await server.videos.get({ id: uuid }) diff --git a/packages/tests/src/peertube-runner/index.ts b/packages/tests/src/peertube-runner/index.ts index 29f21694f..f02611046 100644 --- a/packages/tests/src/peertube-runner/index.ts +++ b/packages/tests/src/peertube-runner/index.ts @@ -1,4 +1,5 @@ export * from './client-cli.js' export * from './live-transcoding.js' +export * from './replace-file.js' export * from './studio-transcoding.js' export * from './vod-transcoding.js' diff --git a/packages/tests/src/peertube-runner/replace-file.ts b/packages/tests/src/peertube-runner/replace-file.ts new file mode 100644 index 000000000..c5b0f4565 --- /dev/null +++ b/packages/tests/src/peertube-runner/replace-file.ts @@ -0,0 +1,86 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ +import { getAllFiles } from '@peertube/peertube-core-utils' +import { + cleanupTests, + createSingleServer, + PeerTubeServer, + setAccessTokensToServers, + setDefaultVideoChannel, + waitJobs +} from '@peertube/peertube-server-commands' +import { PeerTubeRunnerProcess } from '@tests/shared/peertube-runner-process.js' +import { checkSourceFile } from '@tests/shared/videos.js' +import { expect } from 'chai' + +describe('Test replace file using peertube-runner program', function () { + let server: PeerTubeServer + let peertubeRunner: PeerTubeRunnerProcess + let uuid: string + + before(async function () { + this.timeout(120_000) + + server = await createSingleServer(1) + + await setAccessTokensToServers([ server ]) + await setDefaultVideoChannel([ server ]) + + await server.config.enableRemoteTranscoding() + await server.config.enableFileUpdate() + await server.config.enableMinimumTranscoding({ hls: true, keepOriginal: true, webVideo: true }) + + const registrationToken = await server.runnerRegistrationTokens.getFirstRegistrationToken() + + peertubeRunner = new PeerTubeRunnerProcess(server) + await peertubeRunner.runServer() + await peertubeRunner.registerPeerTubeInstance({ registrationToken, runnerName: 'runner' }) + }) + + it('Should upload a webm video, transcode it and keep original file', async function () { + this.timeout(240000) + + const fixture = 'video_short.webm'; + ({ uuid } = await server.videos.quickUpload({ name: 'video', fixture })) + + await waitJobs(server, { runnerJobs: true }) + + const video = await server.videos.get({ id: uuid }) + + const files = getAllFiles(video) + expect(files).to.have.lengthOf(4) + expect(files[0].resolution.id).to.equal(720) + + await checkSourceFile({ server, fsCount: 1, fixture, uuid }) + }) + + it('Should upload an audio file, transcode it and keep original file', async function () { + const fixture = 'sample.ogg' + const { uuid } = await server.videos.quickUpload({ name: 'audio', fixture }) + + await waitJobs([ server ], { runnerJobs: true }) + await checkSourceFile({ server, fsCount: 2, fixture, uuid }) + }) + + it('Should replace the video', async function () { + const fixture = 'video_short_360p.mp4' + await server.videos.replaceSourceFile({ videoId: uuid, fixture }) + await waitJobs(server, { runnerJobs: true }) + + const video = await server.videos.get({ id: uuid }) + + const files = getAllFiles(video) + expect(files).to.have.lengthOf(4) + expect(files[0].resolution.id).to.equal(360) + + await checkSourceFile({ server, fsCount: 2, fixture, uuid }) + }) + + after(async function () { + if (peertubeRunner) { + await peertubeRunner.unregisterPeerTubeInstance({ runnerName: 'runner' }) + peertubeRunner.kill() + } + + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/shared/videos.ts b/packages/tests/src/shared/videos.ts index 3ad5852e6..acf647ac1 100644 --- a/packages/tests/src/shared/videos.ts +++ b/packages/tests/src/shared/videos.ts @@ -1,23 +1,23 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/no-floating-promises */ +import { uuidRegex } from '@peertube/peertube-core-utils' +import { HttpStatusCode, HttpStatusCodeType, VideoCaption, VideoDetails, VideoPrivacy, VideoResolution } from '@peertube/peertube-models' +import { buildAbsoluteFixturePath, getFileSize, getFilenameFromUrl, getLowercaseExtension } from '@peertube/peertube-node-utils' +import { PeerTubeServer, VideoEdit, getRedirectionUrl, makeRawRequest, waitJobs } from '@peertube/peertube-server-commands' +import { + VIDEO_CATEGORIES, + VIDEO_LANGUAGES, + VIDEO_LICENCES, + VIDEO_PRIVACIES, + loadLanguages +} from '@peertube/peertube-server/core/initializers/constants.js' import { expect } from 'chai' import { pathExists } from 'fs-extra/esm' import { readdir } from 'fs/promises' import { basename, join } from 'path' -import { uuidRegex } from '@peertube/peertube-core-utils' -import { HttpStatusCode, HttpStatusCodeType, VideoCaption, VideoDetails, VideoPrivacy, VideoResolution } from '@peertube/peertube-models' -import { - loadLanguages, - VIDEO_CATEGORIES, - VIDEO_LANGUAGES, - VIDEO_LICENCES, - VIDEO_PRIVACIES -} from '@peertube/peertube-server/core/initializers/constants.js' -import { getLowercaseExtension } from '@peertube/peertube-node-utils' -import { makeRawRequest, PeerTubeServer, VideoEdit, waitJobs } from '@peertube/peertube-server-commands' import { dateIsValid, expectStartWith, testImageGeneratedByFFmpeg } from './checks.js' -import { checkWebTorrentWorks } from './webtorrent.js' import { completeCheckHlsPlaylist } from './streaming-playlists.js' +import { checkWebTorrentWorks } from './webtorrent.js' export async function completeWebVideoFilesCheck (options: { server: PeerTubeServer @@ -369,3 +369,40 @@ export async function uploadRandomVideoOnServers ( return res } + +export async function checkSourceFile (options: { + server: PeerTubeServer + fsCount: number + uuid: string + fixture: string + objectStorageBaseUrl?: string // default false +}) { + const { server, fsCount, fixture, uuid, objectStorageBaseUrl } = options + + const source = await server.videos.getSource({ id: uuid }) + const fixtureFileSize = await getFileSize(buildAbsoluteFixturePath(fixture)) + + if (fsCount > 0) { + expect(await server.servers.countFiles('original-video-files')).to.equal(fsCount) + + const keptFilePath = join(server.servers.buildDirectory('original-video-files'), getFilenameFromUrl(source.fileDownloadUrl)) + expect(await getFileSize(keptFilePath)).to.equal(fixtureFileSize) + } + + expect(source.fileDownloadUrl).to.exist + if (objectStorageBaseUrl) { + const token = await server.videoToken.getVideoFileToken({ videoId: uuid }) + expectStartWith(await getRedirectionUrl(source.fileDownloadUrl + '?videoFileToken=' + token), objectStorageBaseUrl) + } + + const { body } = await makeRawRequest({ + url: source.fileDownloadUrl, + token: server.accessToken, + redirects: 1, + expectedStatus: HttpStatusCode.OK_200 + }) + + expect(body).to.have.lengthOf(fixtureFileSize) + + return source +} diff --git a/server/core/controllers/api/config.ts b/server/core/controllers/api/config.ts index d61c92026..9f44e4a89 100644 --- a/server/core/controllers/api/config.ts +++ b/server/core/controllers/api/config.ts @@ -320,6 +320,9 @@ function customConfig (): CustomConfig { }, transcoding: { enabled: CONFIG.TRANSCODING.ENABLED, + originalFile: { + keep: CONFIG.TRANSCODING.ORIGINAL_FILE.KEEP + }, remoteRunners: { enabled: CONFIG.TRANSCODING.REMOTE_RUNNERS.ENABLED }, diff --git a/server/core/controllers/api/videos/live.ts b/server/core/controllers/api/videos/live.ts index f21833a0d..4c3f3c324 100644 --- a/server/core/controllers/api/videos/live.ts +++ b/server/core/controllers/api/videos/live.ts @@ -184,9 +184,9 @@ async function addLiveVideo (req: express.Request, res: express.Response) { duration: 0, state: VideoState.WAITING_FOR_LIVE, isLive: true, - filename: null + inputFilename: null }, - videoFilePath: undefined, + videoFile: undefined, user: res.locals.oauth.token.User, thumbnails }) diff --git a/server/core/controllers/api/videos/source.ts b/server/core/controllers/api/videos/source.ts index fd2631110..3b77a7d75 100644 --- a/server/core/controllers/api/videos/source.ts +++ b/server/core/controllers/api/videos/source.ts @@ -1,20 +1,20 @@ -import express from 'express' -import { move } from 'fs-extra/esm' +import { buildAspectRatio } from '@peertube/peertube-core-utils' +import { VideoState } from '@peertube/peertube-models' import { sequelizeTypescript } from '@server/initializers/database.js' import { CreateJobArgument, CreateJobOptions, JobQueue } from '@server/lib/job-queue/index.js' import { Hooks } from '@server/lib/plugins/hooks.js' import { regenerateMiniaturesIfNeeded } from '@server/lib/thumbnail.js' import { setupUploadResumableRoutes } from '@server/lib/uploadx.js' -import { buildMoveJob, buildStoryboardJobIfNeeded } from '@server/lib/video-jobs.js' import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist.js' -import { buildNewFile } from '@server/lib/video-file.js' +import { buildNewFile, createVideoSource } from '@server/lib/video-file.js' +import { buildMoveJob, buildStoryboardJobIfNeeded } from '@server/lib/video-jobs.js' import { VideoPathManager } from '@server/lib/video-path-manager.js' import { buildNextVideoState } from '@server/lib/video-state.js' import { openapiOperationDoc } from '@server/middlewares/doc.js' import { VideoModel } from '@server/models/video/video.js' -import { VideoSourceModel } from '@server/models/video/video-source.js' import { MStreamingPlaylistFiles, MVideo, MVideoFile, MVideoFullLight } from '@server/types/models/index.js' -import { VideoState } from '@peertube/peertube-models' +import express from 'express' +import { move } from 'fs-extra/esm' import { logger, loggerTagsFactory } from '../../../helpers/logger.js' import { asyncMiddleware, @@ -23,7 +23,6 @@ import { replaceVideoSourceResumableValidator, videoSourceGetLatestValidator } from '../../../middlewares/index.js' -import { buildAspectRatio } from '@peertube/peertube-core-utils' const lTags = loggerTagsFactory('api', 'video') @@ -61,7 +60,7 @@ async function replaceVideoSourceResumable (req: express.Request, res: express.R const videoPhysicalFile = res.locals.updateVideoFileResumable const user = res.locals.oauth.token.User - const videoFile = await buildNewFile({ path: videoPhysicalFile.path, mode: 'web-video' }) + const videoFile = await buildNewFile({ path: videoPhysicalFile.path, mode: 'web-video', ffprobe: res.locals.ffprobe }) const originalFilename = videoPhysicalFile.originalname const videoFileMutexReleaser = await VideoPathManager.Instance.lockFiles(res.locals.videoAll.uuid) @@ -114,13 +113,15 @@ async function replaceVideoSourceResumable (req: express.Request, res: express.R await removeOldFiles({ video, files: oldWebVideoFiles, playlists: oldStreamingPlaylists }) - const source = await VideoSourceModel.create({ - filename: originalFilename, - videoId: video.id, + const source = await createVideoSource({ + inputFilename: originalFilename, + inputProbe: res.locals.ffprobe, + inputPath: destination, + video, createdAt: inputFileUpdatedAt }) - await regenerateMiniaturesIfNeeded(video) + await regenerateMiniaturesIfNeeded(video, res.locals.ffprobe) await video.VideoChannel.setAsUpdated() await addVideoJobsAfterUpload(video, video.getMaxQualityFile()) diff --git a/server/core/controllers/api/videos/upload.ts b/server/core/controllers/api/videos/upload.ts index 477b342ac..4e5efcce7 100644 --- a/server/core/controllers/api/videos/upload.ts +++ b/server/core/controllers/api/videos/upload.ts @@ -1,12 +1,14 @@ -import express from 'express' +import { ffprobePromise, getChaptersFromContainer } from '@peertube/peertube-ffmpeg' +import { HttpStatusCode, ThumbnailType, VideoCreate } from '@peertube/peertube-models' +import { uuidToShort } from '@peertube/peertube-node-utils' import { getResumableUploadPath } from '@server/helpers/upload.js' +import { LocalVideoCreator } from '@server/lib/local-video-creator.js' import { Redis } from '@server/lib/redis.js' import { setupUploadResumableRoutes, uploadx } from '@server/lib/uploadx.js' import { buildNextVideoState } from '@server/lib/video-state.js' import { openapiOperationDoc } from '@server/middlewares/doc.js' -import { uuidToShort } from '@peertube/peertube-node-utils' -import { HttpStatusCode, ThumbnailType, VideoCreate } from '@peertube/peertube-models' -import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger.js' +import express from 'express' +import { VideoAuditView, auditLoggerFactory, getAuditIdFromRes } from '../../../helpers/audit-logger.js' import { createReqFiles } from '../../../helpers/express-utils.js' import { logger, loggerTagsFactory } from '../../../helpers/logger.js' import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../../initializers/constants.js' @@ -19,8 +21,6 @@ import { videosAddResumableInitValidator, videosAddResumableValidator } from '../../../middlewares/index.js' -import { ffprobePromise, getChaptersFromContainer } from '@peertube/peertube-ffmpeg' -import { LocalVideoCreator } from '@server/lib/local-video-creator.js' const lTags = loggerTagsFactory('api', 'video') const auditLogger = auditLoggerFactory('videos') @@ -134,7 +134,12 @@ async function addVideo (options: { const localVideoCreator = new LocalVideoCreator({ lTags, - videoFilePath: videoPhysicalFile.path, + + videoFile: { + path: videoPhysicalFile.path, + probe: res.locals.ffprobe + }, + user: res.locals.oauth.token.User, channel: res.locals.videoChannel, @@ -148,7 +153,7 @@ async function addVideo (options: { ...videoInfo, duration: videoPhysicalFile.duration, - filename: videoPhysicalFile.originalname, + inputFilename: videoPhysicalFile.originalname, state: buildNextVideoState(), isLive: false }, diff --git a/server/core/controllers/download.ts b/server/core/controllers/download.ts index 26c9ce0d4..92f5c6e08 100644 --- a/server/core/controllers/download.ts +++ b/server/core/controllers/download.ts @@ -1,12 +1,14 @@ -import cors from 'cors' -import express from 'express' +import { forceNumber } from '@peertube/peertube-core-utils' +import { FileStorage, HttpStatusCode, VideoStreamingPlaylistType } from '@peertube/peertube-models' import { logger } from '@server/helpers/logger.js' import { VideoTorrentsSimpleFileCache } from '@server/lib/files-cache/index.js' import { generateHLSFilePresignedUrl, + generateOriginalFilePresignedUrl, generateUserExportPresignedUrl, generateWebVideoPresignedUrl } from '@server/lib/object-storage/index.js' +import { getFSUserExportFilePath } from '@server/lib/paths.js' import { Hooks } from '@server/lib/plugins/hooks.js' import { VideoPathManager } from '@server/lib/video-path-manager.js' import { @@ -17,15 +19,16 @@ import { MVideoFile, MVideoFullLight } from '@server/types/models/index.js' -import { forceNumber } from '@peertube/peertube-core-utils' -import { HttpStatusCode, FileStorage, VideoStreamingPlaylistType } from '@peertube/peertube-models' +import { MVideoSource } from '@server/types/models/video/video-source.js' +import cors from 'cors' +import express from 'express' import { STATIC_DOWNLOAD_PATHS } from '../initializers/constants.js' import { asyncMiddleware, optionalAuthenticate, + originalVideoFileDownloadValidator, userExportDownloadValidator, videosDownloadValidator } from '../middlewares/index.js' -import { getFSUserExportFilePath } from '@server/lib/paths.js' const downloadRouter = express.Router() @@ -40,7 +43,7 @@ downloadRouter.use( STATIC_DOWNLOAD_PATHS.VIDEOS + ':id-:resolution([0-9]+).:extension', optionalAuthenticate, asyncMiddleware(videosDownloadValidator), - asyncMiddleware(downloadVideoFile) + asyncMiddleware(downloadWebVideoFile) ) downloadRouter.use( @@ -51,11 +54,18 @@ downloadRouter.use( ) downloadRouter.use( - STATIC_DOWNLOAD_PATHS.USER_EXPORT + ':filename', + STATIC_DOWNLOAD_PATHS.USER_EXPORTS + ':filename', asyncMiddleware(userExportDownloadValidator), // Include JWT token authentication asyncMiddleware(downloadUserExport) ) +downloadRouter.use( + STATIC_DOWNLOAD_PATHS.ORIGINAL_VIDEO_FILE + ':filename', + optionalAuthenticate, + asyncMiddleware(originalVideoFileDownloadValidator), + asyncMiddleware(downloadOriginalFile) +) + // --------------------------------------------------------------------------- export { @@ -91,7 +101,7 @@ async function downloadTorrent (req: express.Request, res: express.Response) { return res.download(result.path, result.downloadName) } -async function downloadVideoFile (req: express.Request, res: express.Response) { +async function downloadWebVideoFile (req: express.Request, res: express.Response) { const video = res.locals.videoAll const videoFile = getVideoFile(req, video.VideoFiles) @@ -184,6 +194,19 @@ function downloadUserExport (req: express.Request, res: express.Response) { return Promise.resolve() } +function downloadOriginalFile (req: express.Request, res: express.Response) { + const videoSource = res.locals.videoSource + + const downloadFilename = videoSource.inputFilename + + if (videoSource.storage === FileStorage.OBJECT_STORAGE) { + return redirectOriginalFileToObjectStorage({ res, videoSource, downloadFilename }) + } + + res.download(VideoPathManager.Instance.getFSOriginalVideoFilePath(videoSource.keptOriginalFilename), downloadFilename) + return Promise.resolve() +} + // --------------------------------------------------------------------------- function getVideoFile (req: express.Request, files: MVideoFile[]) { @@ -262,3 +285,17 @@ async function redirectUserExportToObjectStorage (options: { return res.redirect(url) } + +async function redirectOriginalFileToObjectStorage (options: { + res: express.Response + downloadFilename: string + videoSource: MVideoSource +}) { + const { res, downloadFilename, videoSource } = options + + const url = await generateOriginalFilePresignedUrl({ videoSource, downloadFilename }) + + logger.debug('Generating pre-signed URL %s for original video file %s', url, videoSource.keptOriginalFilename) + + return res.redirect(url) +} diff --git a/server/core/controllers/static.ts b/server/core/controllers/static.ts index 06807a9d5..85618f64b 100644 --- a/server/core/controllers/static.ts +++ b/server/core/controllers/static.ts @@ -31,12 +31,12 @@ const privateWebVideoStaticMiddlewares = CONFIG.STATIC_FILES.PRIVATE_FILES_REQUI staticRouter.use( [ STATIC_PATHS.PRIVATE_WEB_VIDEOS, STATIC_PATHS.LEGACY_PRIVATE_WEB_VIDEOS ], ...privateWebVideoStaticMiddlewares, - express.static(DIRECTORIES.VIDEOS.PRIVATE, { fallthrough: false }), + express.static(DIRECTORIES.WEB_VIDEOS.PRIVATE, { fallthrough: false }), handleStaticError ) staticRouter.use( [ STATIC_PATHS.WEB_VIDEOS, STATIC_PATHS.LEGACY_WEB_VIDEOS ], - express.static(DIRECTORIES.VIDEOS.PUBLIC, { fallthrough: false }), + express.static(DIRECTORIES.WEB_VIDEOS.PUBLIC, { fallthrough: false }), handleStaticError ) diff --git a/server/core/initializers/checker-after-init.ts b/server/core/initializers/checker-after-init.ts index 692c80471..75da82c46 100644 --- a/server/core/initializers/checker-after-init.ts +++ b/server/core/initializers/checker-after-init.ts @@ -303,34 +303,68 @@ function checkLiveConfig () { } function checkObjectStorageConfig () { - if (CONFIG.OBJECT_STORAGE.ENABLED === true) { + if (CONFIG.OBJECT_STORAGE.ENABLED !== true) return - if (!CONFIG.OBJECT_STORAGE.WEB_VIDEOS.BUCKET_NAME) { - throw new Error('videos_bucket should be set when object storage support is enabled.') + if (!CONFIG.OBJECT_STORAGE.WEB_VIDEOS.BUCKET_NAME) { + throw new Error('videos_bucket should be set when object storage support is enabled.') + } + + if (!CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET_NAME) { + throw new Error('streaming_playlists_bucket should be set when object storage support is enabled.') + } + + // Check web videos and hls videos are not in the same bucket or directory + if ( + CONFIG.OBJECT_STORAGE.WEB_VIDEOS.BUCKET_NAME === CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET_NAME && + CONFIG.OBJECT_STORAGE.WEB_VIDEOS.PREFIX === CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.PREFIX + ) { + if (CONFIG.OBJECT_STORAGE.WEB_VIDEOS.PREFIX === '') { + throw new Error('Bucket prefixes should be set when the same bucket is used for both types of video.') } - if (!CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET_NAME) { - throw new Error('streaming_playlists_bucket should be set when object storage support is enabled.') + throw new Error( + 'Bucket prefixes should be set to different values when the same bucket is used for both types of video.' + ) + } + + if (CONFIG.TRANSCODING.ORIGINAL_FILE.KEEP) { + + if (!CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES.BUCKET_NAME) { + throw new Error('original_video_files_bucket should be set when object storage support is enabled.') } + // Check web videos/hls videos are not in the same bucket or directory as original video files if ( - CONFIG.OBJECT_STORAGE.WEB_VIDEOS.BUCKET_NAME === CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET_NAME && - CONFIG.OBJECT_STORAGE.WEB_VIDEOS.PREFIX === CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.PREFIX + CONFIG.OBJECT_STORAGE.WEB_VIDEOS.BUCKET_NAME === CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES.BUCKET_NAME && + CONFIG.OBJECT_STORAGE.WEB_VIDEOS.PREFIX === CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES.PREFIX ) { if (CONFIG.OBJECT_STORAGE.WEB_VIDEOS.PREFIX === '') { - throw new Error('Object storage bucket prefixes should be set when the same bucket is used for both types of video.') + throw new Error('Bucket prefixes should be set when the same bucket is used for both original and web video files.') } throw new Error( - 'Object storage bucket prefixes should be set to different values when the same bucket is used for both types of video.' + 'Bucket prefixes should be set to different values when the same bucket is used for both original and web video files.' ) } - if (CONFIG.OBJECT_STORAGE.MAX_UPLOAD_PART > parseBytes('250MB')) { - // eslint-disable-next-line max-len - logger.warn(`Object storage max upload part seems to have a big value (${CONFIG.OBJECT_STORAGE.MAX_UPLOAD_PART} bytes). Consider using a lower one (like 100MB).`) + if ( + CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET_NAME === CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES.BUCKET_NAME && + CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.PREFIX === CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES.PREFIX + ) { + if (CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.PREFIX === '') { + throw new Error('Bucket prefixes should be set when the same bucket is used for both original and hls files.') + } + + throw new Error( + 'Bucket prefixes should be set to different values when the same bucket is used for both original and hls files.' + ) } } + + if (CONFIG.OBJECT_STORAGE.MAX_UPLOAD_PART > parseBytes('250MB')) { + // eslint-disable-next-line max-len + logger.warn(`Object storage max upload part seems to have a big value (${CONFIG.OBJECT_STORAGE.MAX_UPLOAD_PART} bytes). Consider using a lower one (like 100MB).`) + } } function checkVideoStudioConfig () { diff --git a/server/core/initializers/checker-before-init.ts b/server/core/initializers/checker-before-init.ts index a75a4ecab..9e97ca5c1 100644 --- a/server/core/initializers/checker-before-init.ts +++ b/server/core/initializers/checker-before-init.ts @@ -32,8 +32,8 @@ function checkMissedConfig () { 'signup.enabled', 'signup.limit', 'signup.requires_approval', 'signup.requires_email_verification', 'signup.minimum_age', 'signup.filters.cidr.whitelist', 'signup.filters.cidr.blacklist', 'redundancy.videos.strategies', 'redundancy.videos.check_interval', - 'transcoding.enabled', 'transcoding.threads', 'transcoding.allow_additional_extensions', 'transcoding.web_videos.enabled', - 'transcoding.hls.enabled', 'transcoding.profile', 'transcoding.concurrency', + 'transcoding.enabled', 'transcoding.original_file.keep', 'transcoding.threads', 'transcoding.allow_additional_extensions', + 'transcoding.web_videos.enabled', 'transcoding.hls.enabled', 'transcoding.profile', 'transcoding.concurrency', 'transcoding.resolutions.0p', 'transcoding.resolutions.144p', 'transcoding.resolutions.240p', 'transcoding.resolutions.360p', 'transcoding.resolutions.480p', 'transcoding.resolutions.720p', 'transcoding.resolutions.1080p', 'transcoding.resolutions.1440p', 'transcoding.resolutions.2160p', 'transcoding.always_transcode_original_resolution', 'transcoding.remote_runners.enabled', @@ -66,7 +66,8 @@ function checkMissedConfig () { 'object_storage.upload_acl.private', 'object_storage.proxy.proxify_private_files', 'object_storage.credentials.access_key_id', 'object_storage.credentials.secret_access_key', 'object_storage.max_upload_part', 'object_storage.streaming_playlists.bucket_name', 'object_storage.streaming_playlists.prefix', 'object_storage.streaming_playlists.base_url', 'object_storage.web_videos.bucket_name', - 'object_storage.web_videos.prefix', 'object_storage.web_videos.base_url', + 'object_storage.web_videos.prefix', 'object_storage.web_videos.base_url', 'object_storage.original_video_files.bucket_name', + 'object_storage.original_video_files.prefix', 'object_storage.original_video_files.base_url', 'theme.default', 'feeds.videos.count', 'feeds.comments.count', 'geo_ip.enabled', 'geo_ip.country.database_url', 'geo_ip.city.database_url', diff --git a/server/core/initializers/config.ts b/server/core/initializers/config.ts index 51845b4a0..2e03b5cc2 100644 --- a/server/core/initializers/config.ts +++ b/server/core/initializers/config.ts @@ -114,6 +114,7 @@ const CONFIG = { LOG_DIR: buildPath(config.get('storage.logs')), WEB_VIDEOS_DIR: buildPath(config.get('storage.web_videos')), STREAMING_PLAYLISTS_DIR: buildPath(config.get('storage.streaming_playlists')), + ORIGINAL_VIDEO_FILES_DIR: buildPath(config.get('storage.original_video_files')), REDUNDANCY_DIR: buildPath(config.get('storage.redundancy')), THUMBNAILS_DIR: buildPath(config.get('storage.thumbnails')), STORYBOARDS_DIR: buildPath(config.get('storage.storyboards')), @@ -159,6 +160,11 @@ const CONFIG = { BUCKET_NAME: config.get('object_storage.user_exports.bucket_name'), PREFIX: config.get('object_storage.user_exports.prefix'), BASE_URL: config.get('object_storage.user_exports.base_url') + }, + ORIGINAL_VIDEO_FILES: { + BUCKET_NAME: config.get('object_storage.original_video_files.bucket_name'), + PREFIX: config.get('object_storage.original_video_files.prefix'), + BASE_URL: config.get('object_storage.original_video_files.base_url') } }, WEBSERVER: { @@ -412,6 +418,9 @@ const CONFIG = { }, TRANSCODING: { get ENABLED () { return config.get('transcoding.enabled') }, + ORIGINAL_FILE: { + get KEEP () { return config.get('transcoding.original_file.keep') } + }, get ALLOW_ADDITIONAL_EXTENSIONS () { return config.get('transcoding.allow_additional_extensions') }, get ALLOW_AUDIO_FILES () { return config.get('transcoding.allow_audio_files') }, get THREADS () { return config.get('transcoding.threads') }, diff --git a/server/core/initializers/constants.ts b/server/core/initializers/constants.ts index 03259f9d6..763c89a2e 100644 --- a/server/core/initializers/constants.ts +++ b/server/core/initializers/constants.ts @@ -45,7 +45,7 @@ import { cpus } from 'os' // --------------------------------------------------------------------------- -const LAST_MIGRATION_VERSION = 825 +const LAST_MIGRATION_VERSION = 830 // --------------------------------------------------------------------------- @@ -857,7 +857,8 @@ const STATIC_DOWNLOAD_PATHS = { TORRENTS: '/download/torrents/', VIDEOS: '/download/videos/', HLS_VIDEOS: '/download/streaming-playlists/hls/videos/', - USER_EXPORT: '/download/user-export/' + USER_EXPORTS: '/download/user-exports/', + ORIGINAL_VIDEO_FILE: '/download/original-video-files/' } const LAZY_STATIC_PATHS = { THUMBNAILS: '/lazy-static/thumbnails/', @@ -981,11 +982,13 @@ const DIRECTORIES = { PRIVATE: join(CONFIG.STORAGE.STREAMING_PLAYLISTS_DIR, 'hls', 'private') }, - VIDEOS: { + WEB_VIDEOS: { PUBLIC: CONFIG.STORAGE.WEB_VIDEOS_DIR, PRIVATE: join(CONFIG.STORAGE.WEB_VIDEOS_DIR, 'private') }, + ORIGINAL_VIDEOS: CONFIG.STORAGE.ORIGINAL_VIDEO_FILES_DIR, + HLS_REDUNDANCY: join(CONFIG.STORAGE.REDUNDANCY_DIR, 'hls') } diff --git a/server/core/initializers/installer.ts b/server/core/initializers/installer.ts index ff25f1909..f28d74f16 100644 --- a/server/core/initializers/installer.ts +++ b/server/core/initializers/installer.ts @@ -96,8 +96,8 @@ function createDirectoriesIfNotExist () { tasks.push(ensureDir(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE)) tasks.push(ensureDir(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC)) - tasks.push(ensureDir(DIRECTORIES.VIDEOS.PUBLIC)) - tasks.push(ensureDir(DIRECTORIES.VIDEOS.PRIVATE)) + tasks.push(ensureDir(DIRECTORIES.WEB_VIDEOS.PUBLIC)) + tasks.push(ensureDir(DIRECTORIES.WEB_VIDEOS.PRIVATE)) // Resumable upload directory tasks.push(ensureDir(DIRECTORIES.RESUMABLE_UPLOAD)) diff --git a/server/core/initializers/migrations/0830-keep-original-file.ts b/server/core/initializers/migrations/0830-keep-original-file.ts new file mode 100644 index 000000000..2eccd4083 --- /dev/null +++ b/server/core/initializers/migrations/0830-keep-original-file.ts @@ -0,0 +1,91 @@ +import * as Sequelize from 'sequelize' + +async function up (utils: { + transaction: Sequelize.Transaction + queryInterface: Sequelize.QueryInterface + sequelize: Sequelize.Sequelize +}): Promise { + const { transaction } = utils + + { + await utils.queryInterface.addColumn('videoSource', 'keptOriginalFilename', { + type: Sequelize.STRING, + allowNull: true + }, { transaction }) + } + + { + await utils.queryInterface.addColumn('videoSource', 'storage', { + type: Sequelize.INTEGER, + allowNull: true + }, { transaction }) + } + + { + await utils.queryInterface.addColumn('videoSource', 'resolution', { + type: Sequelize.INTEGER, + allowNull: true + }, { transaction }) + } + + { + await utils.queryInterface.addColumn('videoSource', 'width', { + type: Sequelize.INTEGER, + allowNull: true + }, { transaction }) + } + + { + await utils.queryInterface.addColumn('videoSource', 'height', { + type: Sequelize.INTEGER, + allowNull: true + }, { transaction }) + } + + { + await utils.queryInterface.addColumn('videoSource', 'fps', { + type: Sequelize.INTEGER, + allowNull: true + }, { transaction }) + } + + { + await utils.queryInterface.addColumn('videoSource', 'size', { + type: Sequelize.INTEGER, + allowNull: true + }, { transaction }) + } + + { + await utils.queryInterface.addColumn('videoSource', 'metadata', { + type: Sequelize.JSONB, + allowNull: true + }, { transaction }) + } + + { + await utils.queryInterface.addColumn('videoSource', 'fileUrl', { + type: Sequelize.STRING, + allowNull: true + }, { transaction }) + } + + { + await utils.queryInterface.renameColumn('videoSource', 'filename', 'inputFilename', { transaction }) + } + + { + await utils.queryInterface.addColumn('userExport', 'fileUrl', { + type: Sequelize.STRING, + allowNull: true + }, { transaction }) + } +} + +function down (options) { + throw new Error('Not implemented.') +} + +export { + down, up +} diff --git a/server/core/lib/job-queue/handlers/video-live-ending.ts b/server/core/lib/job-queue/handlers/video-live-ending.ts index 1a326645d..e151ef876 100644 --- a/server/core/lib/job-queue/handlers/video-live-ending.ts +++ b/server/core/lib/job-queue/handlers/video-live-ending.ts @@ -166,7 +166,8 @@ async function saveReplayToExternalVideo (options: { const thumbnails = await generateLocalVideoMiniature({ video: replayVideo, videoFile: replayVideo.getMaxQualityFile(), - types: [ ThumbnailType.MINIATURE, ThumbnailType.PREVIEW ] + types: [ ThumbnailType.MINIATURE, ThumbnailType.PREVIEW ], + ffprobe: undefined }) for (const thumbnail of thumbnails) { @@ -238,7 +239,7 @@ async function replaceLiveByReplay (options: { } // Regenerate the thumbnail & preview? - await regenerateMiniaturesIfNeeded(videoWithFiles) + await regenerateMiniaturesIfNeeded(videoWithFiles, undefined) // We consider this is a new video await moveToNextState({ video: videoWithFiles, isNewVideo: true }) diff --git a/server/core/lib/local-video-creator.ts b/server/core/lib/local-video-creator.ts index 5affaf2fa..1565f8bb0 100644 --- a/server/core/lib/local-video-creator.ts +++ b/server/core/lib/local-video-creator.ts @@ -1,5 +1,4 @@ import { buildAspectRatio } from '@peertube/peertube-core-utils' -import { ffprobePromise } from '@peertube/peertube-ffmpeg' import { LiveVideoCreate, LiveVideoLatencyMode, @@ -18,11 +17,10 @@ import { ScheduleVideoUpdateModel } from '@server/models/video/schedule-video-up import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting.js' import { VideoLiveModel } from '@server/models/video/video-live.js' import { VideoPasswordModel } from '@server/models/video/video-password.js' -import { VideoSourceModel } from '@server/models/video/video-source.js' import { VideoModel } from '@server/models/video/video.js' import { MChannel, MChannelAccountLight, MThumbnail, MUser, MVideoFile, MVideoFullLight } from '@server/types/models/index.js' import { FilteredModelAttributes } from '@server/types/sequelize.js' -import Ffmpeg from 'fluent-ffmpeg' +import { FfprobeData } from 'fluent-ffmpeg' import { move } from 'fs-extra/esm' import { getLocalVideoActivityPubUrl } from './activitypub/url.js' import { federateVideoIfNeeded } from './activitypub/videos/federate.js' @@ -30,7 +28,7 @@ import { Hooks } from './plugins/hooks.js' import { generateLocalVideoMiniature, updateLocalVideoMiniatureFromExisting } from './thumbnail.js' import { autoBlacklistVideoIfNeeded } from './video-blacklist.js' import { replaceChapters, replaceChaptersFromDescriptionIfNeeded } from './video-chapters.js' -import { buildNewFile } from './video-file.js' +import { buildNewFile, createVideoSource } from './video-file.js' import { addVideoJobsAfterCreation } from './video-jobs.js' import { VideoPathManager } from './video-path-manager.js' import { setVideoTags } from './video.js' @@ -39,7 +37,7 @@ type VideoAttributes = Omit & { duration: number isLive: boolean state: VideoStateType - filename: string + inputFilename: string } type LiveAttributes = Pick & { @@ -64,6 +62,8 @@ export class LocalVideoCreator { private readonly lTags: LoggerTagsFn private readonly videoFilePath: string | undefined + private readonly videoFileProbe: FfprobeData + private readonly videoAttributes: VideoAttributes private readonly liveAttributes: LiveAttributes | undefined @@ -72,12 +72,15 @@ export class LocalVideoCreator { private video: MVideoFullLight private videoFile: MVideoFile - private ffprobe: Ffmpeg.FfprobeData + private videoPath: string constructor (private readonly options: { lTags: LoggerTagsFn - videoFilePath: string + videoFile: { + path: string + probe: FfprobeData + } videoAttributes: VideoAttributes liveAttributes: LiveAttributes @@ -93,7 +96,8 @@ export class LocalVideoCreator { finalFallback: ChaptersOption | undefined } }) { - this.videoFilePath = options.videoFilePath + this.videoFilePath = options.videoFile?.path + this.videoFileProbe = options.videoFile?.probe this.videoAttributes = options.videoAttributes this.liveAttributes = options.liveAttributes @@ -112,11 +116,10 @@ export class LocalVideoCreator { this.video.url = getLocalVideoActivityPubUrl(this.video) if (this.videoFilePath) { - this.ffprobe = await ffprobePromise(this.videoFilePath) - this.videoFile = await buildNewFile({ path: this.videoFilePath, mode: 'web-video', ffprobe: this.ffprobe }) + this.videoFile = await buildNewFile({ path: this.videoFilePath, mode: 'web-video', ffprobe: this.videoFileProbe }) - const destination = VideoPathManager.Instance.getFSVideoFileOutputPath(this.video, this.videoFile) - await move(this.videoFilePath, destination) + this.videoPath = VideoPathManager.Instance.getFSVideoFileOutputPath(this.video, this.videoFile) + await move(this.videoFilePath, this.videoPath) this.video.aspectRatio = buildAspectRatio({ width: this.videoFile.width, height: this.videoFile.height }) } @@ -166,13 +169,6 @@ export class LocalVideoCreator { transaction }) - if (this.videoAttributes.filename) { - await VideoSourceModel.create({ - filename: this.videoAttributes.filename, - videoId: this.video.id - }, { transaction }) - } - if (this.videoAttributes.privacy === VideoPrivacy.PASSWORD_PROTECTED) { await VideoPasswordModel.addPasswords(this.videoAttributes.videoPasswords, this.video.id, transaction) } @@ -197,10 +193,11 @@ export class LocalVideoCreator { videoLive.videoId = this.video.id this.video.VideoLive = await videoLive.save({ transaction }) } + if (this.videoFile) { transaction.afterCommit(() => { addVideoJobsAfterCreation({ video: this.video, videoFile: this.videoFile }) - .catch(err => logger.error('Cannot build new video jobs of %s.', this.video.uuid, { err, ...this.lTags(this.video.uuid) })) + .catch(err => logger.error('Cannot build new video jobs of %s.', this.video.uuid, { err, ...this.lTags(this.video.uuid) })) }) } else { await federateVideoIfNeeded(this.video, true, transaction) @@ -218,6 +215,15 @@ export class LocalVideoCreator { }) }) + if (this.videoAttributes.inputFilename) { + await createVideoSource({ + inputFilename: this.videoAttributes.inputFilename, + inputPath: this.videoPath, + inputProbe: this.videoFileProbe, + video: this.video + }) + } + // Channel has a new content, set as updated await this.channel.setAsUpdated() @@ -248,7 +254,12 @@ export class LocalVideoCreator { return [ ...await Promise.all(promises), - ...await generateLocalVideoMiniature({ video: this.video, videoFile: this.videoFile, types: toGenerate, ffprobe: this.ffprobe }) + ...await generateLocalVideoMiniature({ + video: this.video, + videoFile: this.videoFile, + types: toGenerate, + ffprobe: this.videoFileProbe + }) ] } diff --git a/server/core/lib/object-storage/keys.ts b/server/core/lib/object-storage/keys.ts index d5381454c..75e5d60d7 100644 --- a/server/core/lib/object-storage/keys.ts +++ b/server/core/lib/object-storage/keys.ts @@ -1,25 +1,22 @@ import { join } from 'path' import { MStreamingPlaylistVideo } from '@server/types/models/index.js' -function generateHLSObjectStorageKey (playlist: MStreamingPlaylistVideo, filename: string) { +export function generateHLSObjectStorageKey (playlist: MStreamingPlaylistVideo, filename: string) { return join(generateHLSObjectBaseStorageKey(playlist), filename) } -function generateHLSObjectBaseStorageKey (playlist: MStreamingPlaylistVideo) { +export function generateHLSObjectBaseStorageKey (playlist: MStreamingPlaylistVideo) { return join(playlist.getStringType(), playlist.Video.uuid) } -function generateWebVideoObjectStorageKey (filename: string) { +export function generateWebVideoObjectStorageKey (filename: string) { return filename } -function generateUserExportObjectStorageKey (filename: string) { +export function generateOriginalVideoObjectStorageKey (filename: string) { return filename } -export { - generateHLSObjectStorageKey, - generateHLSObjectBaseStorageKey, - generateWebVideoObjectStorageKey, - generateUserExportObjectStorageKey +export function generateUserExportObjectStorageKey (filename: string) { + return filename } diff --git a/server/core/lib/object-storage/pre-signed-urls.ts b/server/core/lib/object-storage/pre-signed-urls.ts index 81d11307b..3abf55612 100644 --- a/server/core/lib/object-storage/pre-signed-urls.ts +++ b/server/core/lib/object-storage/pre-signed-urls.ts @@ -1,8 +1,14 @@ import { CONFIG } from '@server/initializers/config.js' import { MStreamingPlaylistVideo, MUserExport, MVideoFile } from '@server/types/models/index.js' -import { generateHLSObjectStorageKey, generateUserExportObjectStorageKey, generateWebVideoObjectStorageKey } from './keys.js' +import { MVideoSource } from '@server/types/models/video/video-source.js' +import { + generateHLSObjectStorageKey, + generateOriginalVideoObjectStorageKey, + generateUserExportObjectStorageKey, + generateWebVideoObjectStorageKey +} from './keys.js' import { buildKey, getClient } from './shared/index.js' -import { getHLSPublicFileUrl, getWebVideoPublicFileUrl } from './urls.js' +import { getObjectStoragePublicFileUrl } from './urls.js' export async function generateWebVideoPresignedUrl (options: { file: MVideoFile @@ -16,7 +22,7 @@ export async function generateWebVideoPresignedUrl (options: { downloadFilename }) - return getWebVideoPublicFileUrl(url) + return getObjectStoragePublicFileUrl(url, CONFIG.OBJECT_STORAGE.WEB_VIDEOS) } export async function generateHLSFilePresignedUrl (options: { @@ -32,7 +38,7 @@ export async function generateHLSFilePresignedUrl (options: { downloadFilename }) - return getHLSPublicFileUrl(url) + return getObjectStoragePublicFileUrl(url, CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) } export async function generateUserExportPresignedUrl (options: { @@ -47,7 +53,22 @@ export async function generateUserExportPresignedUrl (options: { downloadFilename }) - return getHLSPublicFileUrl(url) + return getObjectStoragePublicFileUrl(url, CONFIG.OBJECT_STORAGE.USER_EXPORTS) +} + +export async function generateOriginalFilePresignedUrl (options: { + videoSource: MVideoSource + downloadFilename: string +}) { + const { videoSource, downloadFilename } = options + + const url = await generatePresignedUrl({ + bucket: CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES.BUCKET_NAME, + key: buildKey(generateOriginalVideoObjectStorageKey(videoSource.keptOriginalFilename), CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES), + downloadFilename + }) + + return getObjectStoragePublicFileUrl(url, CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES) } // --------------------------------------------------------------------------- diff --git a/server/core/lib/object-storage/urls.ts b/server/core/lib/object-storage/urls.ts index 0bd469ea8..ca185710c 100644 --- a/server/core/lib/object-storage/urls.ts +++ b/server/core/lib/object-storage/urls.ts @@ -1,23 +1,15 @@ -import { CONFIG } from '@server/initializers/config.js' import { OBJECT_STORAGE_PROXY_PATHS, WEBSERVER } from '@server/initializers/constants.js' import { MVideoUUID } from '@server/types/models/index.js' import { BucketInfo, buildKey, getEndpointParsed } from './shared/index.js' -function getInternalUrl (config: BucketInfo, keyWithoutPrefix: string) { +export function getInternalUrl (config: BucketInfo, keyWithoutPrefix: string) { return getBaseUrl(config) + buildKey(keyWithoutPrefix, config) } // --------------------------------------------------------------------------- -function getWebVideoPublicFileUrl (fileUrl: string) { - const baseUrl = CONFIG.OBJECT_STORAGE.WEB_VIDEOS.BASE_URL - if (!baseUrl) return fileUrl - - return replaceByBaseUrl(fileUrl, baseUrl) -} - -function getHLSPublicFileUrl (fileUrl: string) { - const baseUrl = CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BASE_URL +export function getObjectStoragePublicFileUrl (fileUrl: string, objectStorageConfig: { BASE_URL: string }) { + const baseUrl = objectStorageConfig.BASE_URL if (!baseUrl) return fileUrl return replaceByBaseUrl(fileUrl, baseUrl) @@ -25,28 +17,13 @@ function getHLSPublicFileUrl (fileUrl: string) { // --------------------------------------------------------------------------- -function getHLSPrivateFileUrl (video: MVideoUUID, filename: string) { +export function getHLSPrivateFileUrl (video: MVideoUUID, filename: string) { return WEBSERVER.URL + OBJECT_STORAGE_PROXY_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS + video.uuid + `/${filename}` } -function getWebVideoPrivateFileUrl (filename: string) { +export function getWebVideoPrivateFileUrl (filename: string) { return WEBSERVER.URL + OBJECT_STORAGE_PROXY_PATHS.PRIVATE_WEB_VIDEOS + filename } - -// --------------------------------------------------------------------------- - -export { - getInternalUrl, - - getWebVideoPublicFileUrl, - getHLSPublicFileUrl, - - getHLSPrivateFileUrl, - getWebVideoPrivateFileUrl, - - replaceByBaseUrl -} - // --------------------------------------------------------------------------- function getBaseUrl (bucketInfo: BucketInfo, baseUrl?: string) { diff --git a/server/core/lib/object-storage/videos.ts b/server/core/lib/object-storage/videos.ts index 46d32b566..737490491 100644 --- a/server/core/lib/object-storage/videos.ts +++ b/server/core/lib/object-storage/videos.ts @@ -1,14 +1,20 @@ -import { basename, join } from 'path' import { logger } from '@server/helpers/logger.js' import { CONFIG } from '@server/initializers/config.js' import { MStreamingPlaylistVideo, MVideo, MVideoFile } from '@server/types/models/index.js' +import { MVideoSource } from '@server/types/models/video/video-source.js' +import { basename, join } from 'path' import { getHLSDirectory } from '../paths.js' import { VideoPathManager } from '../video-path-manager.js' -import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebVideoObjectStorageKey } from './keys.js' +import { + generateHLSObjectBaseStorageKey, + generateHLSObjectStorageKey, + generateOriginalVideoObjectStorageKey, + generateWebVideoObjectStorageKey +} from './keys.js' import { createObjectReadStream, - listKeysOfPrefix, lTags, + listKeysOfPrefix, makeAvailable, removeObject, removeObjectByFullKey, @@ -19,13 +25,13 @@ import { updatePrefixACL } from './shared/index.js' -function listHLSFileKeysOf (playlist: MStreamingPlaylistVideo) { +export function listHLSFileKeysOf (playlist: MStreamingPlaylistVideo) { return listKeysOfPrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) } // --------------------------------------------------------------------------- -function storeHLSFileFromFilename (playlist: MStreamingPlaylistVideo, filename: string) { +export function storeHLSFileFromFilename (playlist: MStreamingPlaylistVideo, filename: string) { return storeObject({ inputPath: join(getHLSDirectory(playlist.Video), filename), objectStorageKey: generateHLSObjectStorageKey(playlist, filename), @@ -34,7 +40,7 @@ function storeHLSFileFromFilename (playlist: MStreamingPlaylistVideo, filename: }) } -function storeHLSFileFromPath (playlist: MStreamingPlaylistVideo, path: string) { +export function storeHLSFileFromPath (playlist: MStreamingPlaylistVideo, path: string) { return storeObject({ inputPath: path, objectStorageKey: generateHLSObjectStorageKey(playlist, basename(path)), @@ -43,7 +49,7 @@ function storeHLSFileFromPath (playlist: MStreamingPlaylistVideo, path: string) }) } -function storeHLSFileFromContent (playlist: MStreamingPlaylistVideo, path: string, content: string) { +export function storeHLSFileFromContent (playlist: MStreamingPlaylistVideo, path: string, content: string) { return storeContent({ content, inputPath: path, @@ -55,7 +61,7 @@ function storeHLSFileFromContent (playlist: MStreamingPlaylistVideo, path: strin // --------------------------------------------------------------------------- -function storeWebVideoFile (video: MVideo, file: MVideoFile) { +export function storeWebVideoFile (video: MVideo, file: MVideoFile) { return storeObject({ inputPath: VideoPathManager.Instance.getFSVideoFileOutputPath(video, file), objectStorageKey: generateWebVideoObjectStorageKey(file.filename), @@ -66,7 +72,18 @@ function storeWebVideoFile (video: MVideo, file: MVideoFile) { // --------------------------------------------------------------------------- -async function updateWebVideoFileACL (video: MVideo, file: MVideoFile) { +export function storeOriginalVideoFile (inputPath: string, filename: string) { + return storeObject({ + inputPath, + objectStorageKey: generateOriginalVideoObjectStorageKey(filename), + bucketInfo: CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES, + isPrivate: true + }) +} + +// --------------------------------------------------------------------------- + +export async function updateWebVideoFileACL (video: MVideo, file: MVideoFile) { await updateObjectACL({ objectStorageKey: generateWebVideoObjectStorageKey(file.filename), bucketInfo: CONFIG.OBJECT_STORAGE.WEB_VIDEOS, @@ -74,7 +91,7 @@ async function updateWebVideoFileACL (video: MVideo, file: MVideoFile) { }) } -async function updateHLSFilesACL (playlist: MStreamingPlaylistVideo) { +export async function updateHLSFilesACL (playlist: MStreamingPlaylistVideo) { await updatePrefixACL({ prefix: generateHLSObjectBaseStorageKey(playlist), bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, @@ -84,31 +101,37 @@ async function updateHLSFilesACL (playlist: MStreamingPlaylistVideo) { // --------------------------------------------------------------------------- -function removeHLSObjectStorage (playlist: MStreamingPlaylistVideo) { +export function removeHLSObjectStorage (playlist: MStreamingPlaylistVideo) { return removePrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) } -function removeHLSFileObjectStorageByFilename (playlist: MStreamingPlaylistVideo, filename: string) { +export function removeHLSFileObjectStorageByFilename (playlist: MStreamingPlaylistVideo, filename: string) { return removeObject(generateHLSObjectStorageKey(playlist, filename), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) } -function removeHLSFileObjectStorageByPath (playlist: MStreamingPlaylistVideo, path: string) { +export function removeHLSFileObjectStorageByPath (playlist: MStreamingPlaylistVideo, path: string) { return removeObject(generateHLSObjectStorageKey(playlist, basename(path)), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) } -function removeHLSFileObjectStorageByFullKey (key: string) { +export function removeHLSFileObjectStorageByFullKey (key: string) { return removeObjectByFullKey(key, CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) } // --------------------------------------------------------------------------- -function removeWebVideoObjectStorage (videoFile: MVideoFile) { +export function removeWebVideoObjectStorage (videoFile: MVideoFile) { return removeObject(generateWebVideoObjectStorageKey(videoFile.filename), CONFIG.OBJECT_STORAGE.WEB_VIDEOS) } // --------------------------------------------------------------------------- -async function makeHLSFileAvailable (playlist: MStreamingPlaylistVideo, filename: string, destination: string) { +export function removeOriginalFileObjectStorage (videoSource: MVideoSource) { + return removeObject(generateOriginalVideoObjectStorageKey(videoSource.keptOriginalFilename), CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES) +} + +// --------------------------------------------------------------------------- + +export async function makeHLSFileAvailable (playlist: MStreamingPlaylistVideo, filename: string, destination: string) { const key = generateHLSObjectStorageKey(playlist, filename) logger.info('Fetching HLS file %s from object storage to %s.', key, destination, lTags()) @@ -122,7 +145,7 @@ async function makeHLSFileAvailable (playlist: MStreamingPlaylistVideo, filename return destination } -async function makeWebVideoFileAvailable (filename: string, destination: string) { +export async function makeWebVideoFileAvailable (filename: string, destination: string) { const key = generateWebVideoObjectStorageKey(filename) logger.info('Fetching Web Video file %s from object storage to %s.', key, destination, lTags()) @@ -138,7 +161,7 @@ async function makeWebVideoFileAvailable (filename: string, destination: string) // --------------------------------------------------------------------------- -function getWebVideoFileReadStream (options: { +export function getWebVideoFileReadStream (options: { filename: string rangeHeader: string }) { @@ -153,7 +176,7 @@ function getWebVideoFileReadStream (options: { }) } -function getHLSFileReadStream (options: { +export function getHLSFileReadStream (options: { playlist: MStreamingPlaylistVideo filename: string rangeHeader: string @@ -169,29 +192,17 @@ function getHLSFileReadStream (options: { }) } -// --------------------------------------------------------------------------- +export function getOriginalFileReadStream (options: { + keptOriginalFilename: string + rangeHeader: string +}) { + const { keptOriginalFilename, rangeHeader } = options -export { - listHLSFileKeysOf, + const key = generateOriginalVideoObjectStorageKey(keptOriginalFilename) - storeWebVideoFile, - storeHLSFileFromFilename, - storeHLSFileFromPath, - storeHLSFileFromContent, - - updateWebVideoFileACL, - updateHLSFilesACL, - - removeHLSObjectStorage, - removeHLSFileObjectStorageByFilename, - removeHLSFileObjectStorageByPath, - removeHLSFileObjectStorageByFullKey, - - removeWebVideoObjectStorage, - - makeWebVideoFileAvailable, - makeHLSFileAvailable, - - getWebVideoFileReadStream, - getHLSFileReadStream + return createObjectReadStream({ + key, + bucketInfo: CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES, + rangeHeader + }) } diff --git a/server/core/lib/thumbnail.ts b/server/core/lib/thumbnail.ts index 41479638c..3fee1f313 100644 --- a/server/core/lib/thumbnail.ts +++ b/server/core/lib/thumbnail.ts @@ -100,7 +100,7 @@ function generateLocalVideoMiniature (options: { video: MVideoThumbnail videoFile: MVideoFile types: ThumbnailType_Type[] - ffprobe?: FfprobeData + ffprobe: FfprobeData }): Promise { const { video, videoFile, types, ffprobe } = options @@ -223,7 +223,7 @@ function updateRemoteVideoThumbnail (options: { // --------------------------------------------------------------------------- -async function regenerateMiniaturesIfNeeded (video: MVideoWithAllFiles) { +async function regenerateMiniaturesIfNeeded (video: MVideoWithAllFiles, ffprobe: FfprobeData) { const thumbnailsToGenerate: ThumbnailType_Type[] = [] if (video.getMiniature().automaticallyGenerated === true) { @@ -237,6 +237,7 @@ async function regenerateMiniaturesIfNeeded (video: MVideoWithAllFiles) { const models = await generateLocalVideoMiniature({ video, videoFile: video.getMaxQualityFile(), + ffprobe, types: thumbnailsToGenerate }) diff --git a/server/core/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts b/server/core/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts index 914a649fd..e00fa6dcf 100644 --- a/server/core/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts +++ b/server/core/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts @@ -1,3 +1,4 @@ +import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS, hasAudioStream, isAudioFile } from '@peertube/peertube-ffmpeg' import { computeOutputFPS } from '@server/helpers/ffmpeg/index.js' import { logger, loggerTagsFactory } from '@server/helpers/logger.js' import { CONFIG } from '@server/initializers/config.js' @@ -11,7 +12,6 @@ import { import { VideoPathManager } from '@server/lib/video-path-manager.js' import { MUserId, MVideoFile, MVideoFullLight, MVideoWithFileThumbnail } from '@server/types/models/index.js' import { MRunnerJob } from '@server/types/models/runners/index.js' -import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS, hasAudioStream, isAudioFile } from '@peertube/peertube-ffmpeg' import { getTranscodingJobPriority } from '../../transcoding-priority.js' import { buildOriginalFileResolution, computeResolutionsToTranscode } from '../../transcoding-resolutions.js' import { AbstractJobBuilder } from './abstract-job-builder.js' @@ -60,11 +60,7 @@ export class TranscodingRunnerJobBuilder extends AbstractJobBuilder { const fps = computeOutputFPS({ inputFPS, resolution: maxResolution }) const priority = await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) - const deleteInputFileId = isAudioInput || maxResolution !== resolution - ? videoFile.id - : null - - const jobPayload = { video, resolution: maxResolution, fps, isNewVideo, priority, deleteInputFileId } + const jobPayload = { video, resolution: maxResolution, fps, isNewVideo, priority, deleteInputFileId: videoFile.id } const mainRunnerJob = videoFile.isAudio() ? await new VODAudioMergeTranscodingJobHandler().create(jobPayload) diff --git a/server/core/lib/transcoding/web-transcoding.ts b/server/core/lib/transcoding/web-transcoding.ts index 22c6ef030..ad82eb269 100644 --- a/server/core/lib/transcoding/web-transcoding.ts +++ b/server/core/lib/transcoding/web-transcoding.ts @@ -1,22 +1,22 @@ -import { Job } from 'bullmq' -import { move, remove } from 'fs-extra/esm' -import { copyFile } from 'fs/promises' -import { basename, join } from 'path' +import { buildAspectRatio } from '@peertube/peertube-core-utils' +import { TranscodeVODOptionsType, getVideoStreamDuration } from '@peertube/peertube-ffmpeg' import { computeOutputFPS } from '@server/helpers/ffmpeg/index.js' import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent.js' import { VideoModel } from '@server/models/video/video.js' import { MVideoFile, MVideoFullLight } from '@server/types/models/index.js' -import { getVideoStreamDuration, TranscodeVODOptionsType } from '@peertube/peertube-ffmpeg' +import { Job } from 'bullmq' +import { move, remove } from 'fs-extra/esm' +import { copyFile } from 'fs/promises' +import { basename, join } from 'path' import { CONFIG } from '../../initializers/config.js' import { VideoFileModel } from '../../models/video/video-file.js' import { JobQueue } from '../job-queue/index.js' import { generateWebVideoFilename } from '../paths.js' -import { buildNewFile } from '../video-file.js' +import { buildNewFile, saveNewOriginalFileIfNeeded } from '../video-file.js' +import { buildStoryboardJobIfNeeded } from '../video-jobs.js' import { VideoPathManager } from '../video-path-manager.js' import { buildFFmpegVOD } from './shared/index.js' import { buildOriginalFileResolution } from './transcoding-resolutions.js' -import { buildStoryboardJobIfNeeded } from '../video-jobs.js' -import { buildAspectRatio } from '@peertube/peertube-core-utils' // Optimize the original video file and replace it. The resolution is not changed. export async function optimizeOriginalVideofile (options: { @@ -73,7 +73,7 @@ export async function optimizeOriginalVideofile (options: { } } -// Transcode the original video file to a lower resolution compatible with web browsers +// Transcode the original/old/source video file to a lower resolution compatible with web browsers export async function transcodeNewWebVideoResolution (options: { video: MVideoFullLight resolution: number @@ -162,7 +162,6 @@ export async function mergeAudioVideofile (options: { try { await buildFFmpegVOD(job).transcode(transcodeOptions) - await remove(audioInputPath) await remove(tmpPreviewPath) } catch (err) { await remove(tmpPreviewPath) @@ -213,14 +212,16 @@ export async function onWebVideoFileTranscoding (options: { await createTorrentAndSetInfoHash(video, videoFile) - const oldFile = await VideoFileModel.loadWebVideoFile({ videoId: video.id, fps: videoFile.fps, resolution: videoFile.resolution }) - if (oldFile) await video.removeWebVideoFile(oldFile) - if (deleteWebInputVideoFile) { + await saveNewOriginalFileIfNeeded(video, deleteWebInputVideoFile) + await video.removeWebVideoFile(deleteWebInputVideoFile) await deleteWebInputVideoFile.destroy() } + const existingFile = await VideoFileModel.loadWebVideoFile({ videoId: video.id, fps: videoFile.fps, resolution: videoFile.resolution }) + if (existingFile) await video.removeWebVideoFile(existingFile) + await VideoFileModel.customUpsert(videoFile, 'video', undefined) video.VideoFiles = await video.$get('VideoFiles') diff --git a/server/core/lib/user-import-export/exporters/videos-exporter.ts b/server/core/lib/user-import-export/exporters/videos-exporter.ts index 8ba4b08cb..887a5b0ef 100644 --- a/server/core/lib/user-import-export/exporters/videos-exporter.ts +++ b/server/core/lib/user-import-export/exporters/videos-exporter.ts @@ -1,8 +1,19 @@ -import { VideoModel } from '@server/models/video/video.js' +import { pick } from '@peertube/peertube-core-utils' +import { ActivityCreate, FileStorage, VideoExportJSON, VideoObject, VideoPrivacy } from '@peertube/peertube-models' +import { logger } from '@server/helpers/logger.js' +import { USER_EXPORT_MAX_ITEMS } from '@server/initializers/constants.js' +import { audiencify, getAudience } from '@server/lib/activitypub/audience.js' +import { buildCreateActivity } from '@server/lib/activitypub/send/send-create.js' +import { buildChaptersAPHasPart } from '@server/lib/activitypub/video-chapters.js' +import { getHLSFileReadStream, getOriginalFileReadStream, getWebVideoFileReadStream } from '@server/lib/object-storage/videos.js' +import { VideoPathManager } from '@server/lib/video-path-manager.js' import { VideoCaptionModel } from '@server/models/video/video-caption.js' import { VideoChannelModel } from '@server/models/video/video-channel.js' +import { VideoChapterModel } from '@server/models/video/video-chapter.js' import { VideoLiveModel } from '@server/models/video/video-live.js' -import { ExportResult, AbstractUserExporter } from './abstract-user-exporter.js' +import { VideoPasswordModel } from '@server/models/video/video-password.js' +import { VideoSourceModel } from '@server/models/video/video-source.js' +import { VideoModel } from '@server/models/video/video.js' import { MStreamingPlaylistFiles, MThumbnail, MVideo, MVideoAP, MVideoCaption, @@ -12,23 +23,12 @@ import { MVideoFullLight, MVideoLiveWithSetting, MVideoPassword } from '@server/types/models/index.js' -import { logger } from '@server/helpers/logger.js' -import { ActivityCreate, VideoExportJSON, VideoObject, VideoPrivacy, FileStorage } from '@peertube/peertube-models' +import { MVideoSource } from '@server/types/models/video/video-source.js' import Bluebird from 'bluebird' -import { getHLSFileReadStream, getWebVideoFileReadStream } from '@server/lib/object-storage/videos.js' import { createReadStream } from 'fs' -import { VideoPathManager } from '@server/lib/video-path-manager.js' import { extname, join } from 'path' import { Readable } from 'stream' -import { getAudience, audiencify } from '@server/lib/activitypub/audience.js' -import { buildCreateActivity } from '@server/lib/activitypub/send/send-create.js' -import { pick } from '@peertube/peertube-core-utils' -import { VideoPasswordModel } from '@server/models/video/video-password.js' -import { MVideoSource } from '@server/types/models/video/video-source.js' -import { VideoSourceModel } from '@server/models/video/video-source.js' -import { VideoChapterModel } from '@server/models/video/video-chapter.js' -import { buildChaptersAPHasPart } from '@server/lib/activitypub/video-chapters.js' -import { USER_EXPORT_MAX_ITEMS } from '@server/initializers/constants.js' +import { AbstractUserExporter, ExportResult } from './abstract-user-exporter.js' export class VideosExporter extends AbstractUserExporter { @@ -89,7 +89,7 @@ export class VideosExporter extends AbstractUserExporter { // Then fetch more attributes for AP serialization const videoAP = await video.lightAPToFullAP(undefined) - const { relativePathsFromJSON, staticFiles } = this.exportVideoFiles({ video, captions }) + const { relativePathsFromJSON, staticFiles } = await this.exportVideoFiles({ video, captions }) return { json: this.exportVideoJSON({ video, captions, live, passwords, source, chapters, archiveFiles: relativePathsFromJSON }), @@ -168,9 +168,7 @@ export class VideosExporter extends AbstractUserExporter { streamingPlaylists: this.exportStreamingPlaylistsJSON(video, video.VideoStreamingPlaylists), - source: source - ? { filename: source.filename } - : null, + source: this.exportVideoSourceJSON(source), archiveFiles } @@ -228,6 +226,24 @@ export class VideosExporter extends AbstractUserExporter { })) } + private exportVideoSourceJSON (source: MVideoSource) { + if (!source) return null + + return { + inputFilename: source.inputFilename, + + resolution: source.resolution, + size: source.size, + + width: source.width, + height: source.height, + + fps: source.fps, + + metadata: source.metadata + } + } + // --------------------------------------------------------------------------- private async exportVideoAP (video: MVideoAP, chapters: MVideoChapter[]): Promise> { @@ -271,7 +287,7 @@ export class VideosExporter extends AbstractUserExporter { // --------------------------------------------------------------------------- - private exportVideoFiles (options: { + private async exportVideoFiles (options: { video: MVideoFullLight captions: MVideoCaption[] }) { @@ -284,15 +300,27 @@ export class VideosExporter extends AbstractUserExporter { captions: {} as { [ lang: string ]: string } } - const videoFile = video.getMaxQualityFile() + if (this.options.withVideoFiles) { + const source = await VideoSourceModel.loadLatest(video.id) + const maxQualityFile = video.getMaxQualityFile() - if (this.options.withVideoFiles && videoFile) { - staticFiles.push({ - archivePath: this.getArchiveVideoFilePath(video, videoFile), - createrReadStream: () => this.generateVideoFileReadStream(video, videoFile) - }) + // Prefer using original file if possible + const file = source?.keptOriginalFilename + ? source + : maxQualityFile - relativePathsFromJSON.videoFile = join(this.relativeStaticDirPath, this.getArchiveVideoFilePath(video, videoFile)) + if (file) { + const videoPath = this.getArchiveVideoFilePath(video, file) + + staticFiles.push({ + archivePath: videoPath, + createrReadStream: () => file === source + ? this.generateVideoSourceReadStream(source) + : this.generateVideoFileReadStream(video, maxQualityFile) + }) + + relativePathsFromJSON.videoFile = join(this.relativeStaticDirPath, videoPath) + } } for (const caption of captions) { @@ -317,6 +345,16 @@ export class VideosExporter extends AbstractUserExporter { return { staticFiles, relativePathsFromJSON } } + private async generateVideoSourceReadStream (source: MVideoSource): Promise { + if (source.storage === FileStorage.FILE_SYSTEM) { + return createReadStream(VideoPathManager.Instance.getFSOriginalVideoFilePath(source.keptOriginalFilename)) + } + + const { stream } = await getOriginalFileReadStream({ keptOriginalFilename: source.keptOriginalFilename, rangeHeader: undefined }) + + return stream + } + private async generateVideoFileReadStream (video: MVideoFullLight, videoFile: MVideoFile): Promise { if (videoFile.storage === FileStorage.FILE_SYSTEM) { return createReadStream(VideoPathManager.Instance.getFSVideoFileOutputPath(video, videoFile)) @@ -329,8 +367,8 @@ export class VideosExporter extends AbstractUserExporter { return stream } - private getArchiveVideoFilePath (video: MVideo, videoFile: MVideoFile) { - return join('video-files', video.uuid + extname(videoFile.filename)) + private getArchiveVideoFilePath (video: MVideo, file: { filename?: string, keptOriginalFilename?: string }) { + return join('video-files', video.uuid + extname(file.filename || file.keptOriginalFilename)) } private getArchiveCaptionFilePath (video: MVideo, caption: MVideoCaptionLanguageUrl) { diff --git a/server/core/lib/user-import-export/importers/videos-importer.ts b/server/core/lib/user-import-export/importers/videos-importer.ts index 2183b58ab..b648ecc45 100644 --- a/server/core/lib/user-import-export/importers/videos-importer.ts +++ b/server/core/lib/user-import-export/importers/videos-importer.ts @@ -1,15 +1,12 @@ -import { LiveVideoLatencyMode, ThumbnailType, VideoExportJSON, VideoPrivacy, VideoState } from '@peertube/peertube-models' -import { logger, loggerTagsFactory } from '@server/helpers/logger.js' -import { Hooks } from '@server/lib/plugins/hooks.js' -import { buildNextVideoState } from '@server/lib/video-state.js' -import { VideoModel } from '@server/models/video/video.js' import { pick } from '@peertube/peertube-core-utils' -import { buildUUID, getFileSize } from '@peertube/peertube-node-utils' -import { MChannelId, MVideoFullLight } from '@server/types/models/index.js' import { ffprobePromise, getVideoStreamDuration } from '@peertube/peertube-ffmpeg' -import { VideoChannelModel } from '@server/models/video/video-channel.js' -import { AbstractUserImporter } from './abstract-user-importer.js' -import { isUserQuotaValid } from '@server/lib/user.js' +import { LiveVideoLatencyMode, ThumbnailType, VideoExportJSON, VideoPrivacy, VideoState } from '@peertube/peertube-models' +import { buildUUID, getFileSize } from '@peertube/peertube-node-utils' +import { isArray, isBooleanValid, isUUIDValid } from '@server/helpers/custom-validators/misc.js' +import { isVideoCaptionLanguageValid } from '@server/helpers/custom-validators/video-captions.js' +import { isVideoChannelUsernameValid } from '@server/helpers/custom-validators/video-channels.js' +import { isVideoChapterTimecodeValid, isVideoChapterTitleValid } from '@server/helpers/custom-validators/video-chapters.js' +import { isLiveLatencyModeValid } from '@server/helpers/custom-validators/video-lives.js' import { isPasswordValid, isVideoCategoryValid, @@ -25,17 +22,21 @@ import { isVideoSupportValid, isVideoTagValid } from '@server/helpers/custom-validators/videos.js' -import { isVideoChannelUsernameValid } from '@server/helpers/custom-validators/video-channels.js' -import { CONSTRAINTS_FIELDS } from '@server/initializers/constants.js' -import { isArray, isBooleanValid, isUUIDValid } from '@server/helpers/custom-validators/misc.js' +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' import { CONFIG } from '@server/initializers/config.js' -import { isVideoCaptionLanguageValid } from '@server/helpers/custom-validators/video-captions.js' -import { isLiveLatencyModeValid } from '@server/helpers/custom-validators/video-lives.js' -import { parse } from 'path' -import { isLocalVideoFileAccepted } from '@server/lib/moderation.js' +import { CONSTRAINTS_FIELDS } from '@server/initializers/constants.js' import { LocalVideoCreator, ThumbnailOptions } from '@server/lib/local-video-creator.js' -import { isVideoChapterTimecodeValid, isVideoChapterTitleValid } from '@server/helpers/custom-validators/video-chapters.js' +import { isLocalVideoFileAccepted } from '@server/lib/moderation.js' +import { Hooks } from '@server/lib/plugins/hooks.js' +import { isUserQuotaValid } from '@server/lib/user.js' import { createLocalCaption } from '@server/lib/video-captions.js' +import { buildNextVideoState } from '@server/lib/video-state.js' +import { VideoChannelModel } from '@server/models/video/video-channel.js' +import { VideoModel } from '@server/models/video/video.js' +import { MChannelId, MVideoFullLight } from '@server/types/models/index.js' +import { FfprobeData } from 'fluent-ffmpeg' +import { parse } from 'path' +import { AbstractUserImporter } from './abstract-user-importer.js' const lTags = loggerTagsFactory('user-import') @@ -69,7 +70,7 @@ export class VideosImporter extends AbstractUserImporter (res => output.on('close', () => res())) + endPromise = new Promise(res => output.on('close', () => res(''))) } else { output = new PassThrough() endPromise = storeUserExportFile(output as PassThrough, exportModel) @@ -56,12 +56,16 @@ export class UserExporter { await this.createZip({ exportModel, user, output }) - await endPromise + const fileUrl = await endPromise + + if (exportModel.storage === FileStorage.OBJECT_STORAGE) { + exportModel.fileUrl = fileUrl + exportModel.size = await getUserExportFileObjectStorageSize(exportModel) + } else if (exportModel.storage === FileStorage.FILE_SYSTEM) { + exportModel.size = await getFileSize(getFSUserExportFilePath(exportModel)) + } exportModel.state = UserExportState.COMPLETED - exportModel.size = exportModel.storage === FileStorage.FILE_SYSTEM - ? await getFileSize(getFSUserExportFilePath(exportModel)) - : await getUserExportFileObjectStorageSize(exportModel) await saveInTransactionWithRetries(exportModel) } catch (err) { diff --git a/server/core/lib/video-file.ts b/server/core/lib/video-file.ts index 70194aa93..5134f76c9 100644 --- a/server/core/lib/video-file.ts +++ b/server/core/lib/video-file.ts @@ -1,16 +1,20 @@ -import { FfprobeData } from 'fluent-ffmpeg' -import { VideoFileMetadata, VideoResolution } from '@peertube/peertube-models' -import { logger } from '@server/helpers/logger.js' -import { VideoFileModel } from '@server/models/video/video-file.js' -import { MVideoWithAllFiles } from '@server/types/models/index.js' -import { getFileSize, getLowercaseExtension } from '@peertube/peertube-node-utils' import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS, isAudioFile } from '@peertube/peertube-ffmpeg' +import { FileStorage, VideoFileMetadata, VideoResolution } from '@peertube/peertube-models' +import { getFileSize, getLowercaseExtension } from '@peertube/peertube-node-utils' +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' +import { CONFIG } from '@server/initializers/config.js' +import { MIMETYPES } from '@server/initializers/constants.js' +import { VideoFileModel } from '@server/models/video/video-file.js' +import { VideoSourceModel } from '@server/models/video/video-source.js' +import { MVideo, MVideoFile, MVideoId, MVideoWithAllFiles } from '@server/types/models/index.js' +import { FfprobeData } from 'fluent-ffmpeg' +import { move, remove } from 'fs-extra' import { lTags } from './object-storage/shared/index.js' +import { storeOriginalVideoFile } from './object-storage/videos.js' import { generateHLSVideoFilename, generateWebVideoFilename } from './paths.js' import { VideoPathManager } from './video-path-manager.js' -import { MIMETYPES } from '@server/initializers/constants.js' -async function buildNewFile (options: { +export async function buildNewFile (options: { path: string mode: 'web-video' | 'hls' ffprobe?: FfprobeData @@ -48,7 +52,7 @@ async function buildNewFile (options: { // --------------------------------------------------------------------------- -async function removeHLSPlaylist (video: MVideoWithAllFiles) { +export async function removeHLSPlaylist (video: MVideoWithAllFiles) { const hls = video.getHLSPlaylist() if (!hls) return @@ -64,7 +68,7 @@ async function removeHLSPlaylist (video: MVideoWithAllFiles) { } } -async function removeHLSFile (video: MVideoWithAllFiles, fileToDeleteId: number) { +export async function removeHLSFile (video: MVideoWithAllFiles, fileToDeleteId: number) { logger.info('Deleting HLS file %d of %s.', fileToDeleteId, video.url, lTags(video.uuid)) const hls = video.getHLSPlaylist() @@ -92,7 +96,7 @@ async function removeHLSFile (video: MVideoWithAllFiles, fileToDeleteId: number) // --------------------------------------------------------------------------- -async function removeAllWebVideoFiles (video: MVideoWithAllFiles) { +export async function removeAllWebVideoFiles (video: MVideoWithAllFiles) { const videoFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) try { @@ -109,7 +113,7 @@ async function removeAllWebVideoFiles (video: MVideoWithAllFiles) { return video } -async function removeWebVideoFile (video: MVideoWithAllFiles, fileToDeleteId: number) { +export async function removeWebVideoFile (video: MVideoWithAllFiles, fileToDeleteId: number) { const files = video.VideoFiles if (files.length === 1) { @@ -132,13 +136,13 @@ async function removeWebVideoFile (video: MVideoWithAllFiles, fileToDeleteId: nu // --------------------------------------------------------------------------- -async function buildFileMetadata (path: string, existingProbe?: FfprobeData) { +export async function buildFileMetadata (path: string, existingProbe?: FfprobeData) { const metadata = existingProbe || await ffprobePromise(path) return new VideoFileMetadata(metadata) } -function getVideoFileMimeType (extname: string, isAudio: boolean) { +export function getVideoFileMimeType (extname: string, isAudio: boolean) { return isAudio && extname === '.mp4' // We use .mp4 even for audio file only ? MIMETYPES.AUDIO.EXT_MIMETYPE['.m4a'] : MIMETYPES.VIDEO.EXT_MIMETYPE[extname] @@ -146,14 +150,84 @@ function getVideoFileMimeType (extname: string, isAudio: boolean) { // --------------------------------------------------------------------------- -export { - buildNewFile, +export async function createVideoSource (options: { + inputFilename: string + inputProbe: FfprobeData + inputPath: string + video: MVideoId + createdAt?: Date +}) { + const { inputFilename, inputPath, inputProbe, video, createdAt } = options - removeHLSPlaylist, - removeHLSFile, - removeAllWebVideoFiles, - removeWebVideoFile, + const videoSource = new VideoSourceModel({ + inputFilename, + videoId: video.id, + createdAt + }) - buildFileMetadata, - getVideoFileMimeType + if (inputPath) { + const probe = inputProbe ?? await ffprobePromise(inputPath) + + if (await isAudioFile(inputPath, probe)) { + videoSource.fps = 0 + videoSource.resolution = VideoResolution.H_NOVIDEO + videoSource.width = 0 + videoSource.height = 0 + } else { + const dimensions = await getVideoStreamDimensionsInfo(inputPath, probe) + videoSource.fps = await getVideoStreamFPS(inputPath, probe) + videoSource.resolution = dimensions.resolution + videoSource.width = dimensions.width + videoSource.height = dimensions.height + } + + videoSource.metadata = await buildFileMetadata(inputPath, probe) + videoSource.size = await getFileSize(inputPath) + } + + return videoSource.save() +} + +export async function saveNewOriginalFileIfNeeded (video: MVideo, videoFile: MVideoFile) { + if (!CONFIG.TRANSCODING.ORIGINAL_FILE.KEEP) return + + const videoSource = await VideoSourceModel.loadLatest(video.id) + + // Already have saved an original file + if (!videoSource || videoSource.keptOriginalFilename) return + videoSource.keptOriginalFilename = videoFile.filename + + const lTags = loggerTagsFactory(video.uuid) + + logger.info(`Storing original video file ${videoSource.keptOriginalFilename} of video ${video.name}`, lTags()) + + const sourcePath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, videoFile) + + if (CONFIG.OBJECT_STORAGE.ENABLED) { + const fileUrl = await storeOriginalVideoFile(sourcePath, videoSource.keptOriginalFilename) + await remove(sourcePath) + + videoSource.storage = FileStorage.OBJECT_STORAGE + videoSource.fileUrl = fileUrl + } else { + const destinationPath = VideoPathManager.Instance.getFSOriginalVideoFilePath(videoSource.keptOriginalFilename) + await move(sourcePath, destinationPath) + + videoSource.storage = FileStorage.FILE_SYSTEM + } + + await videoSource.save() + + // Delete previously kept video files + const allSources = await VideoSourceModel.listAll(video.id) + for (const oldSource of allSources) { + if (!oldSource.keptOriginalFilename) continue + if (oldSource.id === videoSource.id) continue + + try { + await video.removeOriginalFile(oldSource) + } catch (err) { + logger.error('Cannot delete old original file ' + oldSource.keptOriginalFilename, { err, ...lTags() }) + } + } } diff --git a/server/core/lib/video-path-manager.ts b/server/core/lib/video-path-manager.ts index ecf8ccbc6..eb76585ad 100644 --- a/server/core/lib/video-path-manager.ts +++ b/server/core/lib/video-path-manager.ts @@ -1,7 +1,5 @@ -import { Mutex } from 'async-mutex' -import { remove } from 'fs-extra/esm' -import { extname, join } from 'path' import { FileStorage } from '@peertube/peertube-models' +import { buildUUID } from '@peertube/peertube-node-utils' import { logger, loggerTagsFactory } from '@server/helpers/logger.js' import { extractVideo } from '@server/helpers/video.js' import { CONFIG } from '@server/initializers/config.js' @@ -13,7 +11,9 @@ import { MVideoFileStreamingPlaylistVideo, MVideoFileVideo } from '@server/types/models/index.js' -import { buildUUID } from '@peertube/peertube-node-utils' +import { Mutex } from 'async-mutex' +import { remove } from 'fs-extra/esm' +import { extname, join } from 'path' import { makeHLSFileAvailable, makeWebVideoFileAvailable } from './object-storage/index.js' import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from './paths.js' import { isVideoInPrivateDirectory } from './video-privacy.js' @@ -56,10 +56,14 @@ class VideoPathManager { } if (isVideoInPrivateDirectory(video.privacy)) { - return join(DIRECTORIES.VIDEOS.PRIVATE, videoFile.filename) + return join(DIRECTORIES.WEB_VIDEOS.PRIVATE, videoFile.filename) } - return join(DIRECTORIES.VIDEOS.PUBLIC, videoFile.filename) + return join(DIRECTORIES.WEB_VIDEOS.PUBLIC, videoFile.filename) + } + + getFSOriginalVideoFilePath (filename: string) { + return join(DIRECTORIES.ORIGINAL_VIDEOS, filename) } async makeAvailableVideoFile (videoFile: MVideoFileVideo | MVideoFileStreamingPlaylistVideo, cb: MakeAvailableCB) { diff --git a/server/core/lib/video-privacy.ts b/server/core/lib/video-privacy.ts index d32643da3..5727a747b 100644 --- a/server/core/lib/video-privacy.ts +++ b/server/core/lib/video-privacy.ts @@ -101,10 +101,10 @@ async function moveWebVideoFileOnFS (type: MoveType, video: MVideo, file: MVideo function getWebVideoDirectories (moveType: MoveType) { if (moveType === 'private-to-public') { - return { old: DIRECTORIES.VIDEOS.PRIVATE, new: DIRECTORIES.VIDEOS.PUBLIC } + return { old: DIRECTORIES.WEB_VIDEOS.PRIVATE, new: DIRECTORIES.WEB_VIDEOS.PUBLIC } } - return { old: DIRECTORIES.VIDEOS.PUBLIC, new: DIRECTORIES.VIDEOS.PRIVATE } + return { old: DIRECTORIES.WEB_VIDEOS.PUBLIC, new: DIRECTORIES.WEB_VIDEOS.PRIVATE } } // --------------------------------------------------------------------------- diff --git a/server/core/middlewares/validators/config.ts b/server/core/middlewares/validators/config.ts index f3bab5851..f286cb0a3 100644 --- a/server/core/middlewares/validators/config.ts +++ b/server/core/middlewares/validators/config.ts @@ -41,6 +41,7 @@ const customConfigUpdateValidator = [ body('videoChannels.maxPerUser').isInt(), body('transcoding.enabled').isBoolean(), + body('transcoding.originalFile.keep').isBoolean(), body('transcoding.allowAdditionalExtensions').isBoolean(), body('transcoding.threads').isInt(), body('transcoding.concurrency').isInt({ min: 1 }), diff --git a/server/core/middlewares/validators/shared/videos.ts b/server/core/middlewares/validators/shared/videos.ts index 28c90a081..1e6a2e7ba 100644 --- a/server/core/middlewares/validators/shared/videos.ts +++ b/server/core/middlewares/validators/shared/videos.ts @@ -1,7 +1,6 @@ -import { Request, Response } from 'express' import { HttpStatusCode, ServerErrorCode, UserRight, UserRightType, VideoPrivacy } from '@peertube/peertube-models' import { exists } from '@server/helpers/custom-validators/misc.js' -import { loadVideo, VideoLoadType } from '@server/lib/model-loaders/index.js' +import { VideoLoadType, loadVideo } from '@server/lib/model-loaders/index.js' import { isUserQuotaValid } from '@server/lib/user.js' import { VideoTokensManager } from '@server/lib/video-tokens-manager.js' import { authenticatePromise } from '@server/middlewares/auth.js' @@ -20,10 +19,12 @@ import { MVideoId, MVideoImmutable, MVideoThumbnail, + MVideoUUID, MVideoWithRights } from '@server/types/models/index.js' +import { Request, Response } from 'express' -async function doesVideoExist (id: number | string, res: Response, fetchType: VideoLoadType = 'all') { +export async function doesVideoExist (id: number | string, res: Response, fetchType: VideoLoadType = 'all') { const userId = res.locals.oauth ? res.locals.oauth.token.User.id : undefined const video = await loadVideo(id, fetchType, userId) @@ -64,7 +65,7 @@ async function doesVideoExist (id: number | string, res: Response, fetchType: Vi // --------------------------------------------------------------------------- -async function doesVideoFileOfVideoExist (id: number, videoIdOrUUID: number | string, res: Response) { +export async function doesVideoFileOfVideoExist (id: number, videoIdOrUUID: number | string, res: Response) { if (!await VideoFileModel.doesVideoExistForVideoFile(id, videoIdOrUUID)) { res.fail({ status: HttpStatusCode.NOT_FOUND_404, @@ -78,7 +79,7 @@ async function doesVideoFileOfVideoExist (id: number, videoIdOrUUID: number | st // --------------------------------------------------------------------------- -async function doesVideoChannelOfAccountExist (channelId: number, user: MUserAccountId, res: Response) { +export async function doesVideoChannelOfAccountExist (channelId: number, user: MUserAccountId, res: Response) { const videoChannel = await VideoChannelModel.loadAndPopulateAccount(channelId) if (videoChannel === null) { @@ -105,7 +106,7 @@ async function doesVideoChannelOfAccountExist (channelId: number, user: MUserAcc // --------------------------------------------------------------------------- -async function checkCanSeeVideo (options: { +export async function checkCanSeeVideo (options: { req: Request res: Response paramId: string @@ -128,7 +129,7 @@ async function checkCanSeeVideo (options: { throw new Error('Unknown video privacy when checking video right ' + video.url) } -async function checkCanSeeUserAuthVideo (options: { +export async function checkCanSeeUserAuthVideo (options: { req: Request res: Response video: MVideoId | MVideoWithRights @@ -174,7 +175,7 @@ async function checkCanSeeUserAuthVideo (options: { return fail() } -async function checkCanSeePasswordProtectedVideo (options: { +export async function checkCanSeePasswordProtectedVideo (options: { req: Request res: Response video: MVideo @@ -215,13 +216,13 @@ async function checkCanSeePasswordProtectedVideo (options: { return false } -function canUserAccessVideo (user: MUser, video: MVideoWithRights | MVideoAccountLight, right: UserRightType) { +export function canUserAccessVideo (user: MUser, video: MVideoWithRights | MVideoAccountLight, right: UserRightType) { const isOwnedByUser = video.VideoChannel.Account.userId === user.id return isOwnedByUser || user.hasRight(right) } -async function getVideoWithRights (video: MVideoWithRights): Promise { +export async function getVideoWithRights (video: MVideoWithRights): Promise { return video.VideoChannel?.Account?.userId ? video : VideoModel.loadFull(video.id) @@ -229,7 +230,7 @@ async function getVideoWithRights (video: MVideoWithRights): Promise { + if (areValidationErrors(req, res)) return + + const videoSource = await VideoSourceModel.loadByKeptOriginalFilename(req.params.filename) + if (!videoSource) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Original video file not found' + }) + } + + if (!await checkCanAccessVideoSourceFile({ req, res, videoId: videoSource.videoId })) return + + res.locals.videoSource = videoSource + + return next() + } +] + // --------------------------------------------------------------------------- // Private // --------------------------------------------------------------------------- diff --git a/server/core/middlewares/validators/videos/videos.ts b/server/core/middlewares/validators/videos/videos.ts index b36be0f31..37e911b9e 100644 --- a/server/core/middlewares/validators/videos/videos.ts +++ b/server/core/middlewares/validators/videos/videos.ts @@ -1,5 +1,3 @@ -import express from 'express' -import { body, param, query, ValidationChain } from 'express-validator' import { arrayify } from '@peertube/peertube-core-utils' import { HttpStatusCode, ServerErrorCode, UserRight, VideoState } from '@peertube/peertube-models' import { Redis } from '@server/lib/redis.js' @@ -7,6 +5,8 @@ import { buildUploadXFile, safeUploadXCleanup } from '@server/lib/uploadx.js' import { getServerActor } from '@server/models/application/application.js' import { ExpressPromiseHandler } from '@server/types/express-handler.js' import { MUserAccountId, MVideoFullLight } from '@server/types/models/index.js' +import express from 'express' +import { ValidationChain, body, param, query } from 'express-validator' import { exists, isBooleanValid, @@ -41,8 +41,7 @@ import { CONFIG } from '../../../initializers/config.js' import { CONSTRAINTS_FIELDS, OVERVIEWS } from '../../../initializers/constants.js' import { VideoModel } from '../../../models/video/video.js' import { - areValidationErrors, - checkCanAccessVideoStaticFiles, + areValidationErrors, checkCanAccessVideoStaticFiles, checkCanSeeVideo, checkUserCanManageVideo, doesVideoChannelOfAccountExist, @@ -501,23 +500,19 @@ const commonVideosFiltersValidator = [ // --------------------------------------------------------------------------- export { - videosAddLegacyValidator, - videosAddResumableValidator, - videosAddResumableInitValidator, - - videosUpdateValidator, - videosGetValidator, - videoFileMetadataGetValidator, - videosDownloadValidator, checkVideoFollowConstraints, - videosCustomGetValidator, - videosRemoveValidator, - - getCommonVideoEditAttributes, - commonVideosFiltersValidator, - - videosOverviewValidator + getCommonVideoEditAttributes, + videoFileMetadataGetValidator, + videosAddLegacyValidator, + videosAddResumableInitValidator, + videosAddResumableValidator, + videosCustomGetValidator, + videosDownloadValidator, + videosGetValidator, + videosOverviewValidator, + videosRemoveValidator, + videosUpdateValidator } // --------------------------------------------------------------------------- diff --git a/server/core/models/user/user-export.ts b/server/core/models/user/user-export.ts index 8a67c81a5..9c78fdc5a 100644 --- a/server/core/models/user/user-export.ts +++ b/server/core/models/user/user-export.ts @@ -62,6 +62,10 @@ export class UserExportModel extends SequelizeModel { @Column storage: FileStorageType + @AllowNull(true) + @Column + fileUrl: string + @ForeignKey(() => UserModel) @Column userId: number @@ -188,7 +192,7 @@ export class UserExportModel extends SequelizeModel { getFileDownloadUrl () { if (this.state !== UserExportState.COMPLETED) return null - return WEBSERVER.URL + join(STATIC_DOWNLOAD_PATHS.USER_EXPORT, this.filename) + '?jwt=' + this.generateJWT() + return WEBSERVER.URL + join(STATIC_DOWNLOAD_PATHS.USER_EXPORTS, this.filename) + '?jwt=' + this.generateJWT() } // --------------------------------------------------------------------------- diff --git a/server/core/models/video/formatter/video-api-format.ts b/server/core/models/video/formatter/video-api-format.ts index 7e6bcb431..8fce56a5f 100644 --- a/server/core/models/video/formatter/video-api-format.ts +++ b/server/core/models/video/formatter/video-api-format.ts @@ -1,8 +1,3 @@ -import { generateMagnetUri } from '@server/helpers/webtorrent.js' -import { tracer } from '@server/lib/opentelemetry/tracing.js' -import { getLocalVideoFileMetadataUrl } from '@server/lib/video-urls.js' -import { VideoViewsManager } from '@server/lib/views/video-views-manager.js' -import { uuidToShort } from '@peertube/peertube-node-utils' import { Video, VideoAdditionalAttributes, @@ -12,6 +7,11 @@ import { VideosCommonQueryAfterSanitize, VideoStreamingPlaylist } from '@peertube/peertube-models' +import { uuidToShort } from '@peertube/peertube-node-utils' +import { generateMagnetUri } from '@server/helpers/webtorrent.js' +import { tracer } from '@server/lib/opentelemetry/tracing.js' +import { getLocalVideoFileMetadataUrl } from '@server/lib/video-urls.js' +import { VideoViewsManager } from '@server/lib/views/video-views-manager.js' import { isArray } from '../../../helpers/custom-validators/misc.js' import { VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES, VIDEO_PRIVACIES, VIDEO_STATES } from '../../../initializers/constants.js' import { MServer, MStreamingPlaylistRedundanciesOpt, MVideoFormattable, MVideoFormattableDetails } from '../../../types/models/index.js' @@ -211,9 +211,7 @@ export function videoFilesModelToFormattedJSON ( resolution: { id: videoFile.resolution, - label: videoFile.resolution === 0 - ? 'Audio' - : `${videoFile.resolution}p` + label: getResolutionLabel(videoFile.resolution) }, width: videoFile.width, @@ -259,6 +257,12 @@ export function getStateLabel (id: number) { return VIDEO_STATES[id] || 'Unknown' } +export function getResolutionLabel (resolution: number) { + if (resolution === 0) return 'Audio' + + return `${resolution}p` +} + // --------------------------------------------------------------------------- // Private // --------------------------------------------------------------------------- diff --git a/server/core/models/video/video-file.ts b/server/core/models/video/video-file.ts index 31b2323cb..374e3806f 100644 --- a/server/core/models/video/video-file.ts +++ b/server/core/models/video/video-file.ts @@ -1,15 +1,15 @@ -import { ActivityVideoUrlObject, VideoResolution, FileStorage, type FileStorageType } from '@peertube/peertube-models' +import { ActivityVideoUrlObject, FileStorage, VideoResolution, type FileStorageType } from '@peertube/peertube-models' import { logger } from '@server/helpers/logger.js' import { extractVideo } from '@server/helpers/video.js' import { CONFIG } from '@server/initializers/config.js' import { buildRemoteUrl } from '@server/lib/activitypub/url.js' import { getHLSPrivateFileUrl, - getHLSPublicFileUrl, - getWebVideoPrivateFileUrl, - getWebVideoPublicFileUrl + getObjectStoragePublicFileUrl, + getWebVideoPrivateFileUrl } from '@server/lib/object-storage/index.js' import { getFSTorrentFilePath } from '@server/lib/paths.js' +import { getVideoFileMimeType } from '@server/lib/video-file.js' import { isVideoInPrivateDirectory } from '@server/lib/video-privacy.js' import { MStreamingPlaylistVideo, MVideo, MVideoWithHost, isStreamingPlaylist } from '@server/types/models/index.js' import { remove } from 'fs-extra/esm' @@ -51,7 +51,6 @@ import { VideoRedundancyModel } from '../redundancy/video-redundancy.js' import { SequelizeModel, doesExist, parseAggregateResult, throwIfNotValid } from '../shared/index.js' import { VideoStreamingPlaylistModel } from './video-streaming-playlist.js' import { VideoModel } from './video.js' -import { getVideoFileMimeType } from '@server/lib/video-file.js' export enum ScopeNames { WITH_VIDEO = 'WITH_VIDEO', @@ -534,10 +533,10 @@ export class VideoFileModel extends SequelizeModel { private getPublicObjectStorageUrl () { if (this.isHLS()) { - return getHLSPublicFileUrl(this.fileUrl) + return getObjectStoragePublicFileUrl(this.fileUrl, CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) } - return getWebVideoPublicFileUrl(this.fileUrl) + return getObjectStoragePublicFileUrl(this.fileUrl, CONFIG.OBJECT_STORAGE.WEB_VIDEOS) } // --------------------------------------------------------------------------- diff --git a/server/core/models/video/video-source.ts b/server/core/models/video/video-source.ts index a336b83d1..e808d47cc 100644 --- a/server/core/models/video/video-source.ts +++ b/server/core/models/video/video-source.ts @@ -1,8 +1,12 @@ +import type { FileStorageType, VideoSource } from '@peertube/peertube-models' +import { STATIC_DOWNLOAD_PATHS, WEBSERVER } from '@server/initializers/constants.js' +import { join } from 'path' import { Transaction } from 'sequelize' -import { AllowNull, BelongsTo, Column, CreatedAt, ForeignKey, Table, UpdatedAt } from 'sequelize-typescript' -import { VideoSource } from '@peertube/peertube-models' +import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Table, UpdatedAt } from 'sequelize-typescript' import { SequelizeModel, getSort } from '../shared/index.js' +import { getResolutionLabel } from './formatter/video-api-format.js' import { VideoModel } from './video.js' +import { MVideoSource } from '@server/types/models/video/video-source.js' @Table({ tableName: 'videoSource', @@ -12,6 +16,10 @@ import { VideoModel } from './video.js' }, { fields: [ { name: 'createdAt', order: 'DESC' } ] + }, + { + fields: [ 'keptOriginalFilename' ], + unique: true } ] }) @@ -24,7 +32,43 @@ export class VideoSourceModel extends SequelizeModel { @AllowNull(false) @Column - filename: string + inputFilename: string + + @AllowNull(true) + @Column + keptOriginalFilename: string + + @AllowNull(true) + @Column + resolution: number + + @AllowNull(true) + @Column + width: number + + @AllowNull(true) + @Column + height: number + + @AllowNull(true) + @Column + fps: number + + @AllowNull(true) + @Column(DataType.BIGINT) + size: number + + @AllowNull(true) + @Column(DataType.JSONB) + metadata: any + + @AllowNull(true) + @Column + storage: FileStorageType + + @AllowNull(true) + @Column + fileUrl: string @ForeignKey(() => VideoModel) @Column @@ -39,16 +83,51 @@ export class VideoSourceModel extends SequelizeModel { Video: Awaited static loadLatest (videoId: number, transaction?: Transaction) { - return VideoSourceModel.findOne({ + return VideoSourceModel.findOne({ where: { videoId }, order: getSort('-createdAt'), transaction }) } + static loadByKeptOriginalFilename (keptOriginalFilename: string) { + return VideoSourceModel.findOne({ + where: { keptOriginalFilename } + }) + } + + static listAll (videoId: number, transaction?: Transaction) { + return VideoSourceModel.findAll({ + where: { videoId }, + transaction + }) + } + + getFileDownloadUrl () { + if (!this.keptOriginalFilename) return null + + return WEBSERVER.URL + join(STATIC_DOWNLOAD_PATHS.ORIGINAL_VIDEO_FILE, this.keptOriginalFilename) + } + toFormattedJSON (): VideoSource { return { - filename: this.filename, + filename: this.inputFilename, + inputFilename: this.inputFilename, + fileDownloadUrl: this.getFileDownloadUrl(), + + resolution: { + id: this.resolution, + label: getResolutionLabel(this.resolution) + }, + size: this.size, + + width: this.width, + height: this.height, + + fps: this.fps, + + metadata: this.metadata, + createdAt: this.createdAt.toISOString() } } diff --git a/server/core/models/video/video-streaming-playlist.ts b/server/core/models/video/video-streaming-playlist.ts index 2f6fb2f1c..4f4cfbee8 100644 --- a/server/core/models/video/video-streaming-playlist.ts +++ b/server/core/models/video/video-streaming-playlist.ts @@ -6,7 +6,7 @@ import { } from '@peertube/peertube-models' import { sha1 } from '@peertube/peertube-node-utils' import { CONFIG } from '@server/initializers/config.js' -import { getHLSPrivateFileUrl, getHLSPublicFileUrl } from '@server/lib/object-storage/index.js' +import { getHLSPrivateFileUrl, getObjectStoragePublicFileUrl } from '@server/lib/object-storage/index.js' import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename } from '@server/lib/paths.js' import { isVideoInPrivateDirectory } from '@server/lib/video-privacy.js' import { VideoFileModel } from '@server/models/video/video-file.js' @@ -266,7 +266,7 @@ export class VideoStreamingPlaylistModel extends SequelizeModel { for (const p of instance.VideoStreamingPlaylists) { tasks.push(instance.removeStreamingPlaylistFiles(p)) } + + // Remove source files + const promiseRemoveSources = VideoSourceModel.listAll(instance.id, options.transaction) + .then(sources => Promise.all(sources.map(s => instance.removeOriginalFile(s)))) + + tasks.push(promiseRemoveSources) } // Do not wait video deletion because we could be in a transaction @@ -2022,6 +2030,17 @@ export class VideoModel extends SequelizeModel { } } + async removeOriginalFile (videoSource: MVideoSource) { + if (!videoSource.keptOriginalFilename) return + + const filePath = VideoPathManager.Instance.getFSOriginalVideoFilePath(videoSource.keptOriginalFilename) + await remove(filePath) + + if (videoSource.storage === FileStorage.OBJECT_STORAGE) { + await removeOriginalFileObjectStorage(videoSource) + } + } + isOutdated () { if (this.isOwned()) return false diff --git a/server/core/types/express.d.ts b/server/core/types/express.d.ts index c80e21c6a..9f106cc11 100644 --- a/server/core/types/express.d.ts +++ b/server/core/types/express.d.ts @@ -1,5 +1,3 @@ -import { OutgoingHttpHeaders } from 'http' -import { Writable } from 'stream' import { HttpMethodType, PeerTubeProblemDocumentData, VideoCreate } from '@peertube/peertube-models' import { RegisterServerAuthExternalOptions } from '@server/types/index.js' import { @@ -29,7 +27,10 @@ import { MPlugin, MServer, MServerBlocklist } from '@server/types/models/server. import { MVideoImportDefault } from '@server/types/models/video/video-import.js' import { MVideoPlaylistElement, MVideoPlaylistElementVideoUrlPlaylistPrivacy } from '@server/types/models/video/video-playlist-element.js' import { MAccountVideoRateAccountVideo } from '@server/types/models/video/video-rate.js' -import { File as UploadXFile, Metadata } from '@uploadx/core' +import { Metadata, File as UploadXFile } from '@uploadx/core' +import { FfprobeData } from 'fluent-ffmpeg' +import { OutgoingHttpHeaders } from 'http' +import { Writable } from 'stream' import { RegisteredPlugin } from '../../lib/plugins/plugin-manager.js' import { MAccountDefault, @@ -127,6 +128,8 @@ declare module 'express' { docUrl?: string + ffprobe?: FfprobeData + videoAPI?: MVideoFormattableDetails videoAll?: MVideoFullLight onlyImmutableVideo?: MVideoImmutable diff --git a/server/scripts/migrations/peertube-5.0.ts b/server/scripts/migrations/peertube-5.0.ts index 6139abd08..72f241bc2 100644 --- a/server/scripts/migrations/peertube-5.0.ts +++ b/server/scripts/migrations/peertube-5.0.ts @@ -19,7 +19,7 @@ async function run () { console.log('Moving private video files in dedicated folders.') await ensureDir(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE) - await ensureDir(DIRECTORIES.VIDEOS.PRIVATE) + await ensureDir(DIRECTORIES.WEB_VIDEOS.PRIVATE) await initDatabaseModels(true) diff --git a/server/scripts/prune-storage.ts b/server/scripts/prune-storage.ts index 9b895cf59..7efc633d2 100755 --- a/server/scripts/prune-storage.ts +++ b/server/scripts/prune-storage.ts @@ -38,8 +38,8 @@ async function run () { console.log('Detecting files to remove, it could take a while...') toDelete = toDelete.concat( - await pruneDirectory(DIRECTORIES.VIDEOS.PUBLIC, doesWebVideoFileExist()), - await pruneDirectory(DIRECTORIES.VIDEOS.PRIVATE, doesWebVideoFileExist()), + await pruneDirectory(DIRECTORIES.WEB_VIDEOS.PUBLIC, doesWebVideoFileExist()), + await pruneDirectory(DIRECTORIES.WEB_VIDEOS.PRIVATE, doesWebVideoFileExist()), await pruneDirectory(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, doesHLSPlaylistExist()), await pruneDirectory(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, doesHLSPlaylistExist()), @@ -97,7 +97,7 @@ async function pruneDirectory (directory: string, existFun: ExistFun) { function doesWebVideoFileExist () { return (filePath: string) => { // Don't delete private directory - if (filePath === DIRECTORIES.VIDEOS.PRIVATE) return true + if (filePath === DIRECTORIES.WEB_VIDEOS.PRIVATE) return true return VideoFileModel.doesOwnedWebVideoFileExist(basename(filePath)) } diff --git a/support/doc/api/openapi.yaml b/support/doc/api/openapi.yaml index e6884fee9..3e68f79a3 100644 --- a/support/doc/api/openapi.yaml +++ b/support/doc/api/openapi.yaml @@ -8136,6 +8136,29 @@ components: properties: filename: type: string + deprecated: true + description: 'Deprecated in 6.1, use inputFilename instead' + inputFilename: + type: string + description: 'Uploaded/imported filename' + fileDownloadUrl: + type: string + description: "**PeerTube >= 6.1** If enabled by the admin, the video source file is kept on the server and can be downloaded by the owner" + resolution: + $ref: '#/components/schemas/VideoResolutionConstant' + description: "**PeerTube >= 6.1**" + size: + type: integer + description: "**PeerTube >= 6.1** Video file size in bytes" + fps: + type: number + description: "**PeerTube >= 6.1** Frames per second of the video file" + width: + type: number + description: "**PeerTube >= 6.1** Video stream width" + height: + type: number + description: "**PeerTube >= 6.1** Video stream height" createdAt: type: string format: date-time @@ -8792,6 +8815,11 @@ components: properties: enabled: type: boolean + originalFile: + type: object + properties: + keep: + type: boolean allowAdditionalExtensions: type: boolean description: Allow your users to upload .mkv, .mov, .avi, .wmv, .flv, .f4v, .3g2, .3gp, .mts, m2ts, .mxf, .nut videos