interesting
/
PeerTube
mirror of https://github.com/Chocobozzz/PeerTube.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add config option to keep original video file (basic first version) (#6157) 

* testing not removing old file and adding columb to db

* implement feature

* remove unnecessary config changes

* use only keptOriginalFileName, change keptOriginalFileName to keptOriginalFilename for consistency with with videoFile table, slight refactor with basename()

* save original video files to dedicated directory original-video-files

* begin implementing object storage (bucket) support

---------

Co-authored-by: chagai.friedlander <chagai.friedlander@fairkom.eu>
Co-authored-by: Ian <ian.kraft@hotmail.com>
Co-authored-by: Chocobozzz <me@florianbigard.com>
This commit is contained in:
chagai95 2024-03-15 15:47:18 +01:00 committed by GitHub
parent ae31e90c30
commit e57c3024f4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
75 changed files with 1653 additions and 801 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts
View File

@ -226,6 +226,9 @@ export class EditCustomConfigComponent extends FormReactive implements OnInit {
concurrency: CONCURRENCY_VALIDATOR,
resolutions: {},
alwaysTranscodeOriginalResolution: null,
originalFile: {
keep: null
},
hls: {
enabled: null
},

15
client/src/app/+admin/config/edit-custom-config/edit-vod-transcoding.component.html
View File

@ -39,7 +39,7 @@
<ng-container ngProjectAs="extra">
<div class="callout callout-light pt-2 pb-0">
<h3 class="callout-title" i18n>Input formats</h3>
<h3 class="callout-title" i18n>Input</h3>
<div class="form-group" [ngClass]="getTranscodingDisabledClass()">
<my-peertube-checkbox
@ -63,10 +63,21 @@
</ng-container>
</my-peertube-checkbox>
</div>
<div class="form-group" formGroupName="originalFile" [ngClass]="getTranscodingDisabledClass()">
<my-peertube-checkbox
inputName="transcodingOriginalFileKeep" formControlName="keep"
i18n-labelText labelText="Keep a version of the input file"
>
<ng-container ngProjectAs="description">
<div i18n>If enabled, the input file is not deleted after transcoding but moved in a dedicated folder or object storage</div>
</ng-container>
</my-peertube-checkbox>
</div>
</div>
<div class="callout callout-light pt-2 mt-2 pb-0">
<h3 class="callout-title" i18n>Output formats</h3>
<h3 class="callout-title" i18n>Output</h3>
<ng-container formGroupName="webVideos">
<div class="form-group" [ngClass]="getTranscodingDisabledClass()">

2
client/src/app/shared/shared-main/video/video.service.ts
View File

@ -405,7 +405,7 @@ export class VideoService {
getSource (videoId: number) {
return this.authHttp
.get<{ source: VideoSource }>(VideoService.BASE_VIDEO_URL + '/' + videoId + '/source')
.get<VideoSource>(VideoService.BASE_VIDEO_URL + '/' + videoId + '/source')
.pipe(
catchError(err => {
if (err.status === 404) {

87
client/src/app/shared/shared-video-miniature/video-download.component.html
View File

@ -16,7 +16,7 @@
</button>
</div>
<div class="modal-body">
<div class="modal-body" [ngClass]="{ 'opacity-0': !loaded }">
<div class="alert alert-warning" *ngIf="isConfidentialVideo()" i18n>
The following link contains a private token and should not be shared with anyone.
</div>
@ -45,14 +45,31 @@
<!-- Video tab -->
<ng-container *ngIf="type === 'video'">
<div ngbNav #resolutionNav="ngbNav" class="nav-tabs" [activeId]="resolutionId" (activeIdChange)="onResolutionIdChange($event)">
<ng-container *ngFor="let file of getVideoFiles()" [ngbNavItem]="file.resolution.id">
<a ngbNavLink i18n>{{ file.resolution.label }}</a>
<ng-template ngbNavContent>
<ng-template #rootNavContent>
<div class="nav-content">
<my-input-text [show]="true" [readonly]="true" [withCopy]="true" [withToggle]="false" [value]="getLink()"></my-input-text>
</div>
</ng-template>
<ng-container *ngIf="originalVideoFile" ngbNavItem="original">
<a ngbNavLink i18n>
<ng-container>Original file</ng-container>
<my-global-icon ngbTooltip="Other users cannot download the original file" iconName="shield"></my-global-icon>
</a>
<ng-template ngbNavContent>
<ng-template [ngTemplateOutlet]="rootNavContent"></ng-template>
</ng-template>
</ng-container>
<ng-container *ngFor="let file of getVideoFiles()" [ngbNavItem]="file.resolution.id">
<a ngbNavLink>{{ file.resolution.label }}</a>
<ng-template ngbNavContent>
<ng-template [ngTemplateOutlet]="rootNavContent"></ng-template>
</ng-template>
</ng-container>
</div>
@ -60,47 +77,59 @@
<div class="advanced-filters" [ngbCollapse]="isAdvancedCustomizationCollapsed" [animation]="true">
<div ngbNav #navMetadata="ngbNav" class="nav-tabs nav-metadata">
<ng-container ngbNavItem>
<a ngbNavLink i18n>Format</a>
<ng-template ngbNavContent>
<div class="file-metadata">
<div class="metadata-attribute metadata-attribute-tags" *ngFor="let item of videoFileMetadataFormat | keyvalue">
<span i18n class="metadata-attribute-label">{{ item.value.label }}</span>
<span class="metadata-attribute-value">{{ item.value.value }}</span>
</div>
<ng-template #metadataInfo let-item>
<div class="metadata-attribute">
<span>{{ item.value.label }}</span>
@if (item.value.value) {
<span>{{ item.value.value }}</span>
} @else {
<span i18n>Unknown</span>
}
</div>
</ng-template>
<ng-container ngbNavItem [disabled]="videoFileMetadataVideoStream === undefined">
<a ngbNavLink i18n>Video stream</a>
<ng-container ngbNavItem>
<a ngbNavLink i18n>Format</a>
<ng-template ngbNavContent>
<div class="file-metadata">
<div class="metadata-attribute metadata-attribute-tags" *ngFor="let item of videoFileMetadataVideoStream | keyvalue">
<span i18n class="metadata-attribute-label">{{ item.value.label }}</span>
<span class="metadata-attribute-value">{{ item.value.value }}</span>
@for (item of videoFileMetadataFormat | keyvalue; track item) {
<ng-template [ngTemplateOutlet]="metadataInfo" [ngTemplateOutletContext]="{ $implicit: item }"></ng-template>
}
</div>
</ng-template>
</ng-container>
<ng-container ngbNavItem [disabled]="videoFileMetadataVideoStream === undefined">
<a ngbNavLink i18n>Video stream</a>
<ng-template ngbNavContent>
<div class="file-metadata">
@for (item of videoFileMetadataVideoStream | keyvalue; track item) {
<ng-template [ngTemplateOutlet]="metadataInfo" [ngTemplateOutletContext]="{ $implicit: item }"></ng-template>
}
</div>
</ng-template>
</ng-container>
<ng-container ngbNavItem [disabled]="videoFileMetadataAudioStream === undefined">
<a ngbNavLink i18n>Audio stream</a>
<ng-template ngbNavContent>
<div class="file-metadata">
<div class="metadata-attribute metadata-attribute-tags" *ngFor="let item of videoFileMetadataAudioStream | keyvalue">
<span i18n class="metadata-attribute-label">{{ item.value.label }}</span>
<span class="metadata-attribute-value">{{ item.value.value }}</span>
</div>
@for (item of videoFileMetadataAudioStream | keyvalue; track item) {
<ng-template [ngTemplateOutlet]="metadataInfo" [ngTemplateOutletContext]="{ $implicit: item }"></ng-template>
}
</div>
</ng-template>
</ng-container>
</ng-container>
</div>
<div *ngIf="getFileMetadata()" [ngbNavOutlet]="navMetadata"></div>
<div *ngIf="hasMetadata()" [ngbNavOutlet]="navMetadata"></div>
<div class="download-type">
<div [hidden]="originalVideoFile" class="download-type">
<div class="peertube-radio-container">
<input type="radio" name="download" id="download-direct" [(ngModel)]="downloadType" value="direct">
<label i18n for="download-direct">Direct download</label>
@ -121,17 +150,13 @@
<ng-container *ngIf="isAdvancedCustomizationCollapsed">
<span class="chevron-down"></span>
<ng-container i18n>
Advanced
</ng-container>
<ng-container i18n>More information/options</ng-container>
</ng-container>
<ng-container *ngIf="!isAdvancedCustomizationCollapsed">
<span class="chevron-up"></span>
<ng-container i18n>
Simple
</ng-container>
<ng-container i18n>Less information/options</ng-container>
</ng-container>
</button>
</ng-container>

27
client/src/app/shared/shared-video-miniature/video-download.component.scss
View File

@ -5,6 +5,13 @@
margin-top: 30px;
}
my-global-icon[iconName=shield] {
@include margin-left(10px);
width: 16px;
margin-top: -3px;
}
.advanced-filters-button {
display: flex;
justify-content: center;
@ -53,7 +60,7 @@
display: block;
margin-bottom: 12px;
.metadata-attribute-label {
> span:first-child {
@include padding-right(5px);
min-width: 142px;
@ -61,22 +68,4 @@
color: pvar(--greyForegroundColor);
font-weight: $font-bold;
}
a.metadata-attribute-value {
@include disable-default-a-behaviour;
color: pvar(--mainForegroundColor);
&:hover {
opacity: 0.9;
}
}
&.metadata-attribute-tags {
.metadata-attribute-value:not(:nth-child(2)) {
&::before {
content: ', ';
}
}
}
}

125
client/src/app/shared/shared-video-miniature/video-download.component.ts
View File

@ -1,30 +1,31 @@
import { mapValues } from 'lodash-es'
import { firstValueFrom } from 'rxjs'
import { tap } from 'rxjs/operators'
import { KeyValuePipe, NgClass, NgFor, NgIf, NgTemplateOutlet } from '@angular/common'
import { Component, ElementRef, Inject, Input, LOCALE_ID, ViewChild } from '@angular/core'
import { HooksService } from '@app/core'
import { FormsModule } from '@angular/forms'
import { AuthService, HooksService } from '@app/core'
import {
NgbCollapse,
NgbModal,
NgbModalRef,
NgbNav,
NgbNavContent,
NgbNavItem,
NgbNavLink,
NgbNavLinkBase,
NgbNavContent,
NgbNavOutlet,
NgbCollapse
NgbTooltip
} from '@ng-bootstrap/ng-bootstrap'
import { objectKeysTyped, pick } from '@peertube/peertube-core-utils'
import { VideoCaption, VideoFile, VideoFileMetadata, VideoSource } from '@peertube/peertube-models'
import { logger } from '@root-helpers/logger'
import { videoRequiresFileToken } from '@root-helpers/video'
import { objectKeysTyped, pick } from '@peertube/peertube-core-utils'
import { VideoCaption, VideoFile } from '@peertube/peertube-models'
import { mapValues } from 'lodash-es'
import { firstValueFrom, of } from 'rxjs'
import { tap } from 'rxjs/operators'
import { InputTextComponent } from '../shared-forms/input-text.component'
import { GlobalIconComponent } from '../shared-icons/global-icon.component'
import { FormsModule } from '@angular/forms'
import { NgIf, NgFor, KeyValuePipe } from '@angular/common'
import { VideoDetails } from '../shared-main/video/video-details.model'
import { BytesPipe } from '../shared-main/angular/bytes.pipe'
import { NumberFormatterPipe } from '../shared-main/angular/number-formatter.pipe'
import { VideoDetails } from '../shared-main/video/video-details.model'
import { VideoFileTokenService } from '../shared-main/video/video-file-token.service'
import { VideoService } from '../shared-main/video/video.service'
@ -49,7 +50,10 @@ type FileMetadata = { [key: string]: { label: string, value: string | number } }
InputTextComponent,
NgbNavOutlet,
NgbCollapse,
KeyValuePipe
KeyValuePipe,
NgbTooltip,
NgTemplateOutlet,
NgClass
]
})
export class VideoDownloadComponent {
@ -59,7 +63,7 @@ export class VideoDownloadComponent {
downloadType: 'direct' | 'torrent' = 'direct'
resolutionId: number | string = -1
resolutionId: number | 'original' = -1
subtitleLanguageId: string
videoFileMetadataFormat: FileMetadata
@ -72,6 +76,10 @@ export class VideoDownloadComponent {
videoFileToken: string
originalVideoFile: VideoSource
loaded = false
private activeModal: NgbModalRef
private bytesPipe: BytesPipe
@ -83,6 +91,7 @@ export class VideoDownloadComponent {
constructor (
@Inject(LOCALE_ID) private localeId: string,
private modalService: NgbModal,
private authService: AuthService,
private videoService: VideoService,
private videoFileTokenService: VideoFileTokenService,
private hooks: HooksService
@ -110,7 +119,10 @@ export class VideoDownloadComponent {
}
show (video: VideoDetails, videoCaptions?: VideoCaption[]) {
this.loaded = false
this.videoFileToken = undefined
this.originalVideoFile = undefined
this.video = video
this.videoCaptions = videoCaptions
@ -125,16 +137,40 @@ export class VideoDownloadComponent {
this.subtitleLanguageId = this.videoCaptions[0].language.id
}
if (this.isConfidentialVideo()) {
this.videoFileTokenService.getVideoFileToken({ videoUUID: this.video.uuid, videoPassword: this.videoPassword })
.subscribe(({ token }) => this.videoFileToken = token)
this.getOriginalVideoFileObs()
.subscribe(source => {
if (source?.fileDownloadUrl) {
this.originalVideoFile = source
}
if (this.originalVideoFile || this.isConfidentialVideo()) {
this.videoFileTokenService.getVideoFileToken({ videoUUID: this.video.uuid, videoPassword: this.videoPassword })
.subscribe(({ token }) => {
this.videoFileToken = token
this.loaded = true
})
} else {
this.loaded = true
}
})
this.activeModal.shown.subscribe(() => {
this.hooks.runAction('action:modal.video-download.shown', 'common')
})
}
private getOriginalVideoFileObs () {
if (!this.authService.isLoggedIn()) return of(undefined)
const user = this.authService.getUser()
if (!this.video.isOwnerOrHasSeeAllVideosRight(user)) return of(undefined)
return this.videoService.getSource(this.video.id)
}
// ---------------------------------------------------------------------------
onClose () {
this.video = undefined
this.videoCaptions = undefined
@ -152,28 +188,29 @@ export class VideoDownloadComponent {
: this.getVideoFileLink()
}
async onResolutionIdChange (resolutionId: number) {
async onResolutionIdChange (resolutionId: number | 'original') {
this.resolutionId = resolutionId
let metadata: VideoFileMetadata
if (this.resolutionId === 'original') {
metadata = this.originalVideoFile.metadata
} else {
const videoFile = this.getVideoFile()
if (!videoFile) return
if (!videoFile.metadata) {
if (!videoFile.metadataUrl) return
if (!videoFile.metadata && videoFile.metadataUrl) {
await this.hydrateMetadataFromMetadataUrl(videoFile)
}
if (!videoFile.metadata) return
metadata = videoFile.metadata
}
this.videoFileMetadataFormat = videoFile
? this.getMetadataFormat(videoFile.metadata.format)
: undefined
this.videoFileMetadataVideoStream = videoFile
? this.getMetadataStream(videoFile.metadata.streams, 'video')
: undefined
this.videoFileMetadataAudioStream = videoFile
? this.getMetadataStream(videoFile.metadata.streams, 'audio')
: undefined
if (!metadata) return
this.videoFileMetadataFormat = this.getMetadataFormat(metadata.format)
this.videoFileMetadataVideoStream = this.getMetadataStream(metadata.streams, 'video')
this.videoFileMetadataAudioStream = this.getMetadataStream(metadata.streams, 'audio')
}
onSubtitleIdChange (subtitleId: string) {
@ -185,6 +222,8 @@ export class VideoDownloadComponent {
}
getVideoFile () {
if (this.resolutionId === 'original') return undefined
const file = this.getVideoFiles()
.find(f => f.resolution.id === this.resolutionId)
@ -197,13 +236,17 @@ export class VideoDownloadComponent {
}
getVideoFileLink () {
const file = this.getVideoFile()
if (!file) return ''
const suffix = this.isConfidentialVideo()
const suffix = this.resolutionId === 'original' || this.isConfidentialVideo()
? '?videoFileToken=' + this.videoFileToken
: ''
if (this.resolutionId === 'original') {
return this.originalVideoFile.fileDownloadUrl + suffix
}
const file = this.getVideoFile()
if (!file) return ''
switch (this.downloadType) {
case 'direct':
return file.fileDownloadUrl + suffix
@ -237,19 +280,15 @@ export class VideoDownloadComponent {
}
isConfidentialVideo () {
return videoRequiresFileToken(this.video)
return this.resolutionId === 'original' || videoRequiresFileToken(this.video)
}
switchToType (type: DownloadType) {
this.type = type
}
getFileMetadata () {
const file = this.getVideoFile()
if (!file) return undefined
return file.metadata
hasMetadata () {
return !!this.videoFileMetadataFormat
}
private getMetadataFormat (format: any) {
@ -282,7 +321,9 @@ export class VideoDownloadComponent {
profile: (value: string) => ({ label: $localize`Profile`, value }),
bit_rate: (value: number | string) => ({
label: $localize`Bitrate`,
value: `${this.numbersPipe.transform(+value)}bps`
value: isNaN(+value)
? undefined
: `${this.numbersPipe.transform(+value)}bps`
})
}

2
client/src/assets/images/misc/shield.svg
View File

@ -1,5 +1,5 @@
<svg
xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512" fill="currentColor"
xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 512 512" fill="currentColor"
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"
>
<path d="M466.5 83.7l-192-80a48.15 48.15 0 0 0-36.9 0l-192 80C27.7 91.1 16 108.6 16 128c0 198.5 114.5 335.7 221.5 380.3 11.8 4.9 25.1 4.9 36.9 0C360.1 472.6 496 349.3 496 128c0-19.4-11.7-36.9-29.5-44.3zM256.1 446.3l-.1-381 175.9 73.3c-3.3 151.4-82.1 261.1-175.8 307.7z"></path>

Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 486 B

After

Width:  |  Height:  |  Size: 484 B

14
config/default.yaml
View File

@ -152,6 +152,7 @@ storage:
avatars: 'storage/avatars/'
web_videos: 'storage/web-videos/'
streaming_playlists: 'storage/streaming-playlists/'
original_video_files: 'storage/original-video-files/'
redundancy: 'storage/redundancy/'
logs: 'storage/logs/'
previews: 'storage/previews/'
@ -238,6 +239,12 @@ object_storage:
prefix: ''
base_url: ''
# Same settings but for original video files
original_video_files:
bucket_name: 'original-video-files'
prefix: ''
base_url: ''
log:
level: 'info' # 'debug' | 'info' | 'warn' | 'error'
@ -526,6 +533,11 @@ video_channels:
transcoding:
enabled: true
original_file:
# If false the uploaded file is deleted after transcoding
# If yes it is not deleted but moved in a dedicated folder or object storage
keep: false
# Allow your users to upload .mkv, .mov, .avi, .wmv, .flv, .f4v, .3g2, .3gp, .mts, m2ts, .mxf, .nut videos
allow_additional_extensions: true
@ -844,7 +856,7 @@ services:
# Cards configuration to format video in Twitter/X
# All other social media (Facebook, Mastodon, etc.) are supported out of the box
twitter:
# Indicates the Twitter account for the website or platform where the content was published
# Indicates the Twitter/X account for the website or platform where the content was published
# This is just an information injected in HTML that is required by Twitter/X
username: '@Chocobozzz'

3
config/dev.yaml
View File

@ -128,3 +128,6 @@ geo_ip:
video_studio:
enabled: true
transcoding:
keep_original_file: false

12
config/production.yaml.example
View File

@ -150,6 +150,7 @@ storage:
avatars: '/var/www/peertube/storage/avatars/'
web_videos: '/var/www/peertube/storage/web-videos/'
streaming_playlists: '/var/www/peertube/storage/streaming-playlists/'
original_video_files: '/var/www/peertube/storage/original-video-files/'
redundancy: '/var/www/peertube/storage/redundancy/'
logs: '/var/www/peertube/storage/logs/'
previews: '/var/www/peertube/storage/previews/'
@ -236,6 +237,12 @@ object_storage:
prefix: ''
base_url: ''
# Same settings but for original video files
original_video_files:
bucket_name: 'original-video-files'
prefix: ''
base_url: ''
log:
level: 'info' # 'debug' | 'info' | 'warn' | 'error'
@ -536,6 +543,11 @@ video_channels:
transcoding:
enabled: true
original_file:
# If false the uploaded file is deleted after transcoding
# If yes it is not deleted but moved in a dedicated folder or object storage
keep: false
# Allow your users to upload .mkv, .mov, .avi, .wmv, .flv, .f4v, .3g2, .3gp, .mts, m2ts, .mxf, .nut videos
allow_additional_extensions: true

1
config/test-1.yaml
View File

@ -15,6 +15,7 @@ storage:
avatars: 'test1/avatars/'
web_videos: 'test1/web-videos/'
streaming_playlists: 'test1/streaming-playlists/'
original_video_files: 'test1/original-video-files/'
redundancy: 'test1/redundancy/'
logs: 'test1/logs/'
previews: 'test1/previews/'

1
config/test-2.yaml
View File

@ -15,6 +15,7 @@ storage:
avatars: 'test2/avatars/'
web_videos: 'test2/web-videos/'
streaming_playlists: 'test2/streaming-playlists/'
original_video_files: 'test2/original-video-files/'
redundancy: 'test2/redundancy/'
logs: 'test2/logs/'
previews: 'test2/previews/'

1
config/test-3.yaml
View File

@ -15,6 +15,7 @@ storage:
avatars: 'test3/avatars/'
web_videos: 'test3/web-videos/'
streaming_playlists: 'test3/streaming-playlists/'
original_video_files: 'test3/original-video-files/'
redundancy: 'test3/redundancy/'
logs: 'test3/logs/'
previews: 'test3/previews/'

1
config/test-4.yaml
View File

@ -15,6 +15,7 @@ storage:
avatars: 'test4/avatars/'
web_videos: 'test4/web-videos/'
streaming_playlists: 'test4/streaming-playlists/'
original_video_files: 'test4/original-video-files/'
redundancy: 'test4/redundancy/'
logs: 'test4/logs/'
previews: 'test4/previews/'

1
config/test-5.yaml
View File

@ -15,6 +15,7 @@ storage:
avatars: 'test5/avatars/'
web_videos: 'test5/web-videos/'
streaming_playlists: 'test5/streaming-playlists/'
original_video_files: 'test5/original-video-files/'
redundancy: 'test5/redundancy/'
logs: 'test5/logs/'
previews: 'test5/previews/'

1
config/test-6.yaml
View File

@ -15,6 +15,7 @@ storage:
avatars: 'test6/avatars/'
web_videos: 'test6/web-videos/'
streaming_playlists: 'test6/streaming-playlists/'
original_video_files: 'test6/original-video-files/'
redundancy: 'test6/redundancy/'
logs: 'test6/logs/'
previews: 'test6/previews/'

13
packages/models/src/import-export/peertube-export-format/video-export.model.ts
View File

@ -1,5 +1,6 @@
import {
LiveVideoLatencyModeType,
VideoFileMetadata,
VideoPrivacyType,
VideoStateType,
VideoStreamingPlaylistType_Type
@ -85,7 +86,17 @@ export interface VideoExportJSON {
}[]
source?: {
filename: string
inputFilename: string
resolution: number
size: number
width: number
height: number
fps: number
metadata: VideoFileMetadata
}
archiveFiles: {

4
packages/models/src/server/custom-config.model.ts
View File

@ -117,6 +117,10 @@ export interface CustomConfig {
transcoding: {
enabled: boolean
originalFile: {
keep: boolean
}
allowAdditionalExtensions: boolean
allowAudioFiles: boolean

21
packages/models/src/videos/video-source.model.ts
View File

@ -1,4 +1,23 @@
import { VideoFileMetadata } from './file/index.js'
import { VideoConstant } from './video-constant.model.js'
export interface VideoSource {
filename: string
inputFilename: string
resolution?: VideoConstant<number>
size?: number // Bytes
width?: number
height?: number
fileDownloadUrl: string
fps?: number
metadata?: VideoFileMetadata
createdAt: string | Date
// TODO: remove, deprecated in 6.1
filename: string
}

28
packages/server-commands/src/server/config-command.ts
View File

@ -106,6 +106,19 @@ export class ConfigCommand extends AbstractCommand {
// ---------------------------------------------------------------------------
keepSourceFile () {
return this.updateExistingSubConfig({
newConfig: {
transcoding: {
originalFile: {
keep: true
}
}
}
})
}
// ---------------------------------------------------------------------------
enableChannelSync () {
return this.setChannelSyncEnabled(true)
}
@ -234,13 +247,17 @@ export class ConfigCommand extends AbstractCommand {
webVideo?: boolean // default true
hls?: boolean // default true
with0p?: boolean // default false
keepOriginal?: boolean // default false
} = {}) {
const { webVideo = true, hls = true, with0p = false } = options
const { webVideo = true, hls = true, with0p = false, keepOriginal = false } = options
return this.updateExistingSubConfig({
newConfig: {
transcoding: {
enabled: true,
originalFile: {
keep: keepOriginal
},
allowAudioFiles: true,
allowAdditionalExtensions: true,
@ -261,13 +278,17 @@ export class ConfigCommand extends AbstractCommand {
enableMinimumTranscoding (options: {
webVideo?: boolean // default true
hls?: boolean // default true
keepOriginal?: boolean // default false
} = {}) {
const { webVideo = true, hls = true } = options
const { webVideo = true, hls = true, keepOriginal = false } = options
return this.updateExistingSubConfig({
newConfig: {
transcoding: {
enabled: true,
originalFile: {
keep: keepOriginal
},
allowAudioFiles: true,
allowAdditionalExtensions: true,
@ -560,6 +581,9 @@ export class ConfigCommand extends AbstractCommand {
},
transcoding: {
enabled: true,
originalFile: {
keep: false
},
remoteRunners: {
enabled: false
},

26
packages/server-commands/src/server/object-storage-command.ts
View File

@ -1,5 +1,5 @@
import { randomInt } from 'crypto'
import { HttpStatusCode } from '@peertube/peertube-models'
import { randomInt } from 'crypto'
import { makePostBodyRequest } from '../requests/index.js'
export class ObjectStorageCommand {
@ -50,6 +50,14 @@ export class ObjectStorageCommand {
web_videos: {
bucket_name: this.getMockWebVideosBucketName()
},
user_exports: {
bucket_name: this.getMockUserExportBucketName()
},
original_video_files: {
bucket_name: this.getMockOriginalFileBucketName()
}
}
}
@ -63,6 +71,14 @@ export class ObjectStorageCommand {
return `http://${this.getMockStreamingPlaylistsBucketName()}.${ObjectStorageCommand.getMockEndpointHost()}/`
}
getMockUserExportBaseUrl () {
return `http://${this.getMockUserExportBucketName()}.${ObjectStorageCommand.getMockEndpointHost()}/`
}
getMockOriginalFileBaseUrl () {
return `http://${this.getMockOriginalFileBucketName()}.${ObjectStorageCommand.getMockEndpointHost()}/`
}
async prepareDefaultMockBuckets () {
await this.createMockBucket(this.getMockStreamingPlaylistsBucketName())
await this.createMockBucket(this.getMockWebVideosBucketName())
@ -100,6 +116,14 @@ export class ObjectStorageCommand {
return this.getMockBucketName(name)
}
getMockUserExportBucketName (name = 'user-exports') {
return this.getMockBucketName(name)
}
getMockOriginalFileBucketName (name = 'original-video-files') {
return this.getMockBucketName(name)
}
getMockBucketName (name: string) {
return `${this.seed}-${name}`
}

1
packages/server-commands/src/server/server.ts
View File

@ -379,6 +379,7 @@ export class PeerTubeServer {
avatars: this.getDirectoryPath('avatars') + '/',
web_videos: this.getDirectoryPath('web-videos') + '/',
streaming_playlists: this.getDirectoryPath('streaming-playlists') + '/',
original_video_files: this.getDirectoryPath('original-video-files') + '/',
redundancy: this.getDirectoryPath('redundancy') + '/',
logs: this.getDirectoryPath('logs') + '/',
previews: this.getDirectoryPath('previews') + '/',

20
packages/server-commands/src/server/servers-command.ts
View File

@ -1,10 +1,10 @@
import { exec } from 'child_process'
import { copy, ensureDir, remove } from 'fs-extra/esm'
import { readdir, readFile } from 'fs/promises'
import { basename, join } from 'path'
import { wait } from '@peertube/peertube-core-utils'
import { HttpStatusCode } from '@peertube/peertube-models'
import { getFileSize, isGithubCI, root } from '@peertube/peertube-node-utils'
import { isGithubCI, root } from '@peertube/peertube-node-utils'
import { exec } from 'child_process'
import { copy, ensureDir, remove } from 'fs-extra/esm'
import { readFile, readdir } from 'fs/promises'
import { basename, join } from 'path'
import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js'
export class ServersCommand extends AbstractCommand {
@ -84,6 +84,8 @@ export class ServersCommand extends AbstractCommand {
return files.length
}
// ---------------------------------------------------------------------------
buildWebVideoFilePath (fileUrl: string) {
return this.buildDirectory(join('web-videos', basename(fileUrl)))
}
@ -92,13 +94,9 @@ export class ServersCommand extends AbstractCommand {
return this.buildDirectory(join('streaming-playlists', 'hls', videoUUID, basename(fileUrl)))
}
// ---------------------------------------------------------------------------
getLogContent () {
return readFile(this.buildDirectory('logs/peertube.log'))
}
async getServerFileSize (subPath: string) {
const path = this.server.servers.buildDirectory(subPath)
return getFileSize(path)
}
}

23
packages/server-commands/src/users/user-exports-command.ts
View File

@ -1,7 +1,8 @@
import { HttpStatusCode, ResultList, UserExport, UserExportRequestResult, UserExportState } from '@peertube/peertube-models'
import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js'
import { wait } from '@peertube/peertube-core-utils'
import { unwrapBody } from '../requests/requests.js'
import { HttpStatusCode, ResultList, UserExport, UserExportRequestResult, UserExportState } from '@peertube/peertube-models'
import { writeFile } from 'fs/promises'
import { makeRawRequest, unwrapBody } from '../requests/requests.js'
import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js'
export class UserExportsCommand extends AbstractCommand {
@ -49,6 +50,22 @@ export class UserExportsCommand extends AbstractCommand {
})
}
async downloadLatestArchive (options: OverrideCommandOptions & {
userId: number
destination: string
}) {
const { data } = await this.list(options)
const res = await makeRawRequest({
url: data[0].privateDownloadUrl,
responseType: 'arraybuffer',
redirects: 1,
expectedStatus: HttpStatusCode.OK_200
})
await writeFile(options.destination, res.body)
}
async deleteAllArchives (options: OverrideCommandOptions & {
userId: number
}) {

240
packages/tests/src/api/check-params/config.ts
View File

@ -19,244 +19,7 @@ describe('Test config API validators', function () {
let server: PeerTubeServer
let userAccessToken: string
const updateParams: CustomConfig = {
instance: {
name: 'PeerTube updated',
shortDescription: 'my short description',
description: 'my super description',
terms: 'my super terms',
codeOfConduct: 'my super coc',
creationReason: 'my super reason',
moderationInformation: 'my super moderation information',
administrator: 'Kuja',
maintenanceLifetime: 'forever',
businessModel: 'my super business model',
hardwareInformation: '2vCore 3GB RAM',
languages: [ 'en', 'es' ],
categories: [ 1, 2 ],
isNSFW: true,
defaultNSFWPolicy: 'blur',
defaultClientRoute: '/videos/recently-added',
customizations: {
javascript: 'alert("coucou")',
css: 'body { background-color: red; }'
}
},
theme: {
default: 'default'
},
services: {
twitter: {
username: '@MySuperUsername'
}
},
client: {
videos: {
miniature: {
preferAuthorDisplayName: false
}
},
menu: {
login: {
redirectOnSingleExternalAuth: false
}
}
},
cache: {
previews: {
size: 2
},
captions: {
size: 3
},
torrents: {
size: 4
},
storyboards: {
size: 5
}
},
signup: {
enabled: false,
limit: 5,
requiresApproval: false,
requiresEmailVerification: false,
minimumAge: 16
},
admin: {
email: 'superadmin1@example.com'
},
contactForm: {
enabled: false
},
user: {
history: {
videos: {
enabled: true
}
},
videoQuota: 5242881,
videoQuotaDaily: 318742,
defaultChannelName: 'Main $1 channel'
},
videoChannels: {
maxPerUser: 20
},
transcoding: {
enabled: true,
remoteRunners: {
enabled: true
},
allowAdditionalExtensions: true,
allowAudioFiles: true,
concurrency: 1,
threads: 1,
profile: 'vod_profile',
resolutions: {
'0p': false,
'144p': false,
'240p': false,
'360p': true,
'480p': true,
'720p': false,
'1080p': false,
'1440p': false,
'2160p': false
},
alwaysTranscodeOriginalResolution: false,
webVideos: {
enabled: true
},
hls: {
enabled: false
}
},
live: {
enabled: true,
allowReplay: false,
latencySetting: {
enabled: false
},
maxDuration: 30,
maxInstanceLives: -1,
maxUserLives: 50,
transcoding: {
enabled: true,
remoteRunners: {
enabled: true
},
threads: 4,
profile: 'live_profile',
resolutions: {
'144p': true,
'240p': true,
'360p': true,
'480p': true,
'720p': true,
'1080p': true,
'1440p': true,
'2160p': true
},
alwaysTranscodeOriginalResolution: false
}
},
videoStudio: {
enabled: true,
remoteRunners: {
enabled: true
}
},
videoFile: {
update: {
enabled: true
}
},
import: {
videos: {
concurrency: 1,
http: {
enabled: false
},
torrent: {
enabled: false
}
},
videoChannelSynchronization: {
enabled: false,
maxPerUser: 10
},
users: {
enabled: false
}
},
export: {
users: {
enabled: false,
maxUserVideoQuota: 40,
exportExpiration: 10
}
},
trending: {
videos: {
algorithms: {
enabled: [ 'hot', 'most-viewed', 'most-liked' ],
default: 'most-viewed'
}
}
},
autoBlacklist: {
videos: {
ofUsers: {
enabled: false
}
}
},
followers: {
instance: {
enabled: false,
manualApproval: true
}
},
followings: {
instance: {
autoFollowBack: {
enabled: true
},
autoFollowIndex: {
enabled: true,
indexUrl: 'https://index.example.com'
}
}
},
broadcastMessage: {
enabled: true,
dismissable: true,
message: 'super message',
level: 'warning'
},
search: {
remoteUri: {
users: true,
anonymous: true
},
searchIndex: {
enabled: true,
url: 'https://search.joinpeertube.org',
disableLocalSearch: true,
isDefaultSearch: true
}
},
storyboards: {
enabled: false
}
}
let updateParams: CustomConfig
// ---------------------------------------------------------------
@ -266,6 +29,7 @@ describe('Test config API validators', function () {
server = await createSingleServer(1)
await setAccessTokensToServers([ server ])
updateParams = await server.config.getCustomConfig()
const user = {
username: 'user1',

65
packages/tests/src/api/check-params/video-source.ts
View File

@ -1,8 +1,9 @@
import { HttpStatusCode } from '@peertube/peertube-models'
import { HttpStatusCode, VideoSource } from '@peertube/peertube-models'
import {
PeerTubeServer,
cleanupTests,
createSingleServer,
PeerTubeServer,
makeRawRequest,
setAccessTokensToServers,
setDefaultVideoChannel,
waitJobs
@ -148,6 +149,66 @@ describe('Test video sources API validator', function () {
})
})
describe('When downloading the source file', function () {
let videoFileToken: string
let videoId: string
let source: VideoSource
let user3: string
let user4: string
before(async function () {
this.timeout(60000)
user3 = await server.users.generateUserAndToken('user3')
user4 = await server.users.generateUserAndToken('user4')
await server.config.enableMinimumTranscoding({ hls: true, keepOriginal: true, webVideo: true })
const { uuid } = await server.videos.quickUpload({ name: 'video', token: user3 })
videoId = uuid
videoFileToken = await server.videoToken.getVideoFileToken({ videoId: uuid, token: user3 })
await waitJobs([ server ])
source = await server.videos.getSource({ id: videoId, token: user3 })
})
it('Should fail with an invalid filename', async function () {
await makeRawRequest({ url: server.url + '/download/original-video-files/hello.mp4', expectedStatus: HttpStatusCode.NOT_FOUND_404 })
})
it('Should fail without header token or video file token', async function () {
await makeRawRequest({ url: source.fileDownloadUrl, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
})
it('Should fail with an invalid header token', async function () {
await makeRawRequest({ url: source.fileDownloadUrl, token: 'toto', expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
})
it('Should fail with an invalid video file token', async function () {
await makeRawRequest({ url: source.fileDownloadUrl, query: { videoFileToken: 'toto' }, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
})
it('Should fail with header token of another user', async function () {
await makeRawRequest({ url: source.fileDownloadUrl, token: user4, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
})
it('Should fail with video file token of another user', async function () {
const videoFileToken = await server.videoToken.getVideoFileToken({ videoId: uuid, token: user4 })
await makeRawRequest({ url: source.fileDownloadUrl, query: { videoFileToken }, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
})
it('Should succeed with a valid header token', async function () {
await makeRawRequest({ url: source.fileDownloadUrl, token: user3, expectedStatus: HttpStatusCode.OK_200 })
})
it('Should succeed with a valid header token', async function () {
await makeRawRequest({ url: source.fileDownloadUrl, query: { videoFileToken }, expectedStatus: HttpStatusCode.OK_200 })
})
})
after(async function () {
await cleanupTests([ server ])
})

5
packages/tests/src/api/server/config.ts
View File

@ -84,6 +84,7 @@ function checkInitialConfig (server: PeerTubeServer, data: CustomConfig) {
expect(data.transcoding.alwaysTranscodeOriginalResolution).to.be.true
expect(data.transcoding.webVideos.enabled).to.be.true
expect(data.transcoding.hls.enabled).to.be.true
expect(data.transcoding.originalFile.keep).to.be.false
expect(data.live.enabled).to.be.false
expect(data.live.allowReplay).to.be.false
@ -205,6 +206,7 @@ function checkUpdatedConfig (data: CustomConfig) {
expect(data.transcoding.alwaysTranscodeOriginalResolution).to.be.false
expect(data.transcoding.hls.enabled).to.be.false
expect(data.transcoding.webVideos.enabled).to.be.true
expect(data.transcoding.originalFile.keep).to.be.true
expect(data.live.enabled).to.be.true
expect(data.live.allowReplay).to.be.true
@ -349,6 +351,9 @@ const newCustomConfig: CustomConfig = {
remoteRunners: {
enabled: true
},
originalFile: {
keep: true
},
allowAdditionalExtensions: true,
allowAudioFiles: true,
threads: 1,

39
packages/tests/src/api/users/user-export.ts
View File

@ -1,14 +1,6 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import { MockSmtpServer } from '@tests/shared/mock-servers/index.js'
import {
cleanupTests, getRedirectionUrl, makeActivityPubRawRequest,
makeRawRequest,
ObjectStorageCommand,
PeerTubeServer,
waitJobs
} from '@peertube/peertube-server-commands'
import { expect } from 'chai'
import { wait } from '@peertube/peertube-core-utils'
import {
AccountExportJSON, ActivityPubActor,
ActivityPubOrderedCollection,
@ -34,6 +26,15 @@ import {
VideoPlaylistType,
VideoPrivacy
} from '@peertube/peertube-models'
import { areMockObjectStorageTestsDisabled } from '@peertube/peertube-node-utils'
import {
cleanupTests, getRedirectionUrl, makeActivityPubRawRequest,
makeRawRequest,
ObjectStorageCommand,
PeerTubeServer,
waitJobs
} from '@peertube/peertube-server-commands'
import { expectStartWith } from '@tests/shared/checks.js'
import {
checkExportFileExists,
checkFileExistsInZIP,
@ -44,8 +45,8 @@ import {
prepareImportExportTests,
regenerateExport
} from '@tests/shared/import-export.js'
import { areMockObjectStorageTestsDisabled } from '@peertube/peertube-node-utils'
import { wait } from '@peertube/peertube-core-utils'
import { MockSmtpServer } from '@tests/shared/mock-servers/index.js'
import { expect } from 'chai'
function runTest (withObjectStorage: boolean) {
let server: PeerTubeServer
@ -69,10 +70,12 @@ function runTest (withObjectStorage: boolean) {
let noahExportId: number
let objectStorage: ObjectStorageCommand
before(async function () {
this.timeout(240000)
const objectStorage = withObjectStorage
objectStorage = withObjectStorage
? new ObjectStorageCommand()
: undefined;
@ -126,6 +129,10 @@ function runTest (withObjectStorage: boolean) {
expect(data[0].size).to.be.greaterThan(0)
expect(data[0].state.id).to.equal(UserExportState.COMPLETED)
expect(data[0].state.label).to.equal('Completed')
if (objectStorage) {
expectStartWith(await getRedirectionUrl(data[0].privateDownloadUrl), objectStorage.getMockUserExportBaseUrl())
}
}
await waitJobs([ server ])
@ -526,6 +533,14 @@ function runTest (withObjectStorage: boolean) {
for (const url of urls) {
await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 })
}
expect(publicVideo.source.inputFilename).to.equal('video_short.webm')
expect(publicVideo.source.fps).to.equal(25)
expect(publicVideo.source.height).to.equal(720)
expect(publicVideo.source.width).to.equal(1280)
expect(publicVideo.source.metadata?.streams).to.exist
expect(publicVideo.source.resolution).to.equal(720)
expect(publicVideo.source.size).to.equal(218910)
}
{

91
packages/tests/src/api/users/user-import.ts
View File

@ -1,11 +1,5 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import { MockSmtpServer } from '@tests/shared/mock-servers/index.js'
import {
cleanupTests, makeRawRequest,
ObjectStorageCommand,
PeerTubeServer, waitJobs
} from '@peertube/peertube-server-commands'
import {
HttpStatusCode,
LiveVideoLatencyMode,
@ -17,14 +11,20 @@ import {
VideoPrivacy,
VideoState
} from '@peertube/peertube-models'
import { prepareImportExportTests } from '@tests/shared/import-export.js'
import { areMockObjectStorageTestsDisabled } from '@peertube/peertube-node-utils'
import { writeFile } from 'fs/promises'
import { join } from 'path'
import { expect } from 'chai'
import { testImage, testAvatarSize } from '@tests/shared/checks.js'
import { completeVideoCheck } from '@tests/shared/videos.js'
import {
ObjectStorageCommand,
PeerTubeServer,
cleanupTests,
waitJobs
} from '@peertube/peertube-server-commands'
import { testAvatarSize, testImage } from '@tests/shared/checks.js'
import { prepareImportExportTests } from '@tests/shared/import-export.js'
import { MockSmtpServer } from '@tests/shared/mock-servers/index.js'
import { completeCheckHlsPlaylist } from '@tests/shared/streaming-playlists.js'
import { completeVideoCheck } from '@tests/shared/videos.js'
import { expect } from 'chai'
import { join } from 'path'
function runTest (withObjectStorage: boolean) {
let server: PeerTubeServer
@ -115,17 +115,8 @@ function runTest (withObjectStorage: boolean) {
await server.userExports.request({ userId: noahId, withVideoFiles: true })
await server.userExports.waitForCreation({ userId: noahId })
const { data } = await server.userExports.list({ userId: noahId })
const res = await makeRawRequest({
url: data[0].privateDownloadUrl,
responseType: 'arraybuffer',
redirects: 1,
expectedStatus: HttpStatusCode.OK_200
})
archivePath = join(server.getDirectoryPath('tmp'), 'archive.zip')
await writeFile(archivePath, res.body)
await server.userExports.downloadLatestArchive({ userId: noahId, destination: archivePath })
})
it('Should import an archive with video files', async function () {
@ -444,6 +435,11 @@ function runTest (withObjectStorage: boolean) {
const source = await remoteServer.videos.getSource({ id: otherVideo.uuid })
expect(source.filename).to.equal('video_short.webm')
expect(source.inputFilename).to.equal('video_short.webm')
expect(source.fileDownloadUrl).to.not.exist
expect(source.metadata?.format).to.exist
expect(source.metadata?.streams).to.be.an('array')
}
{
@ -572,6 +568,57 @@ function runTest (withObjectStorage: boolean) {
}
})
it('Should import original file if included in the export', async function () {
this.timeout(120000)
await server.config.enableMinimumTranscoding({ keepOriginal: true })
await remoteServer.config.keepSourceFile()
const archivePath = join(server.getDirectoryPath('tmp'), 'archive2.zip')
const fixture = 'video_short1.webm'
{
const { token, userId } = await server.users.generate('claire')
await server.videos.quickUpload({ name: 'claire video', token, fixture })
await waitJobs([ server ])
await server.userExports.request({ userId, token, withVideoFiles: true })
await server.userExports.waitForCreation({ userId, token })
await server.userExports.downloadLatestArchive({ userId, token, destination: archivePath })
}
{
const { token, userId } = await remoteServer.users.generate('external_claire')
await remoteServer.userImports.importArchive({ fixture: archivePath, userId, token })
await waitJobs([ remoteServer ])
{
const { data } = await remoteServer.videos.listMyVideos({ token })
expect(data).to.have.lengthOf(1)
const source = await remoteServer.videos.getSource({ id: data[0].id })
expect(source.filename).to.equal(fixture)
expect(source.inputFilename).to.equal(fixture)
expect(source.fileDownloadUrl).to.exist
expect(source.metadata?.format).to.exist
expect(source.metadata?.streams).to.be.an('array')
expect(source.metadata.format['format_name']).to.include('webm')
expect(source.createdAt).to.exist
expect(source.fps).to.equal(25)
expect(source.height).to.equal(720)
expect(source.width).to.equal(1280)
expect(source.resolution.id).to.equal(720)
expect(source.size).to.equal(572456)
}
}
})
after(async function () {
MockSmtpServer.Instance.kill()

186
packages/tests/src/api/videos/video-source.ts
View File

@ -1,24 +1,26 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import { expect } from 'chai'
import { getAllFiles } from '@peertube/peertube-core-utils'
import { HttpStatusCode } from '@peertube/peertube-models'
import { expectStartWith } from '@tests/shared/checks.js'
import { HttpStatusCode, VideoPrivacy } from '@peertube/peertube-models'
import { areMockObjectStorageTestsDisabled } from '@peertube/peertube-node-utils'
import {
cleanupTests,
createMultipleServers,
doubleFollow,
makeGetRequest,
makeRawRequest,
ObjectStorageCommand,
PeerTubeServer,
cleanupTests,
createMultipleServers,
doubleFollow, makeGetRequest,
makeRawRequest,
setAccessTokensToServers,
setDefaultAccountAvatar,
setDefaultVideoChannel,
waitJobs
} from '@peertube/peertube-server-commands'
import { expectStartWith } from '@tests/shared/checks.js'
import { checkDirectoryIsEmpty } from '@tests/shared/directories.js'
import { FIXTURE_URLS } from '@tests/shared/tests.js'
import { checkSourceFile } from '@tests/shared/videos.js'
import { expect } from 'chai'
describe('Test a video file replacement', function () {
describe('Test video source management', function () {
let servers: PeerTubeServer[] = []
let replaceDate: Date
@ -36,6 +38,7 @@ describe('Test a video file replacement', function () {
await setDefaultAccountAvatar(servers)
await servers[0].config.enableFileUpdate()
await servers[0].config.enableMinimumTranscoding()
userToken = await servers[0].users.generateUserAndToken('user1')
@ -44,30 +47,95 @@ describe('Test a video file replacement', function () {
})
describe('Getting latest video source', () => {
const fixture = 'video_short.webm'
const fixture1 = 'video_short.webm'
const fixture2 = 'video_short1.webm'
const uuids: string[] = []
it('Should get the source filename with legacy upload', async function () {
it('Should get the source filename with legacy upload with disabled keep original file', async function () {
this.timeout(30000)
const { uuid } = await servers[0].videos.upload({ attributes: { name: 'my video', fixture }, mode: 'legacy' })
const { uuid } = await servers[0].videos.upload({ attributes: { name: 'my video', fixture: fixture1 }, mode: 'legacy' })
uuids.push(uuid)
await waitJobs(servers)
const source = await servers[0].videos.getSource({ id: uuid })
expect(source.filename).to.equal(fixture)
expect(source.filename).to.equal(fixture1)
expect(source.inputFilename).to.equal(fixture1)
expect(source.fileDownloadUrl).to.be.null
expect(source.createdAt).to.exist
expect(source.fps).to.equal(25)
expect(source.height).to.equal(720)
expect(source.width).to.equal(1280)
expect(source.resolution.id).to.equal(720)
expect(source.size).to.equal(218910)
expect(source.metadata?.format).to.exist
expect(source.metadata?.streams).to.be.an('array')
await checkDirectoryIsEmpty(servers[0], 'original-video-files')
})
it('Should get the source filename with resumable upload', async function () {
it('Should get the source filename with resumable upload and enabled keep original file', async function () {
this.timeout(30000)
const { uuid } = await servers[0].videos.upload({ attributes: { name: 'my video', fixture }, mode: 'resumable' })
await servers[0].config.keepSourceFile()
const { uuid } = await servers[0].videos.upload({ attributes: { name: 'my video', fixture: fixture2 }, mode: 'resumable' })
uuids.push(uuid)
await waitJobs(servers)
const source = await servers[0].videos.getSource({ id: uuid })
expect(source.filename).to.equal(fixture)
expect(source.filename).to.equal(fixture2)
expect(source.inputFilename).to.equal(fixture2)
expect(source.fileDownloadUrl).to.exist
expect(source.createdAt).to.exist
expect(source.fps).to.equal(25)
expect(source.height).to.equal(720)
expect(source.width).to.equal(1280)
expect(source.resolution.id).to.equal(720)
expect(source.size).to.equal(572456)
expect(source.metadata?.format).to.exist
expect(source.metadata?.streams).to.be.an('array')
})
after(async function () {
it('Should have kept original video file', async function () {
await checkSourceFile({ server: servers[0], fsCount: 1, fixture: fixture2, uuid: uuids[uuids.length - 1] })
})
it('Should transcode a file but do not replace original file', async function () {
await servers[0].videos.runTranscoding({ transcodingType: 'web-video', videoId: uuids[0] })
await servers[0].videos.runTranscoding({ transcodingType: 'web-video', videoId: uuids[1] })
await checkSourceFile({ server: servers[0], fsCount: 1, fixture: fixture2, uuid: uuids[uuids.length - 1] })
})
it('Should also keep audio files', async function () {
const fixture = 'sample.ogg'
const { uuid } = await servers[0].videos.quickUpload({ name: 'audio', fixture })
uuids.push(uuid)
await waitJobs(servers)
const source = await checkSourceFile({ server: servers[0], fsCount: 2, fixture, uuid })
expect(source.createdAt).to.exist
expect(source.fps).to.equal(0)
expect(source.height).to.equal(0)
expect(source.width).to.equal(0)
expect(source.resolution.id).to.equal(0)
expect(source.resolution.label).to.equal('Audio')
expect(source.size).to.equal(105243)
expect(source.metadata?.format).to.exist
expect(source.metadata?.streams).to.be.an('array')
})
it('Should delete all videos and do not have original files anymore', async function () {
this.timeout(60000)
for (const uuid of uuids) {
@ -75,6 +143,23 @@ describe('Test a video file replacement', function () {
}
await waitJobs(servers)
await checkDirectoryIsEmpty(servers[0], 'original-video-files')
})
it('Should not have source on import', async function () {
const { video: { uuid } } = await servers[0].videoImports.importVideo({
attributes: {
channelId: servers[0].store.channel.id,
targetUrl: FIXTURE_URLS.goodVideo,
privacy: VideoPrivacy.PUBLIC
}
})
await waitJobs(servers)
await servers[0].videos.getSource({ id: uuid, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
await checkDirectoryIsEmpty(servers[0], 'original-video-files')
})
})
@ -110,18 +195,25 @@ describe('Test a video file replacement', function () {
}
})
it('Should not have kept original video file', async function () {
await checkDirectoryIsEmpty(servers[0], 'original-video-files')
})
it('Should replace a video file with transcoding enabled', async function () {
this.timeout(240000)
const previousPaths: string[] = []
await servers[0].config.enableTranscoding({ hls: true, webVideo: true, with0p: true })
await servers[0].config.enableTranscoding({ hls: true, webVideo: true, with0p: true, keepOriginal: true })
const { uuid: videoUUID } = await servers[0].videos.quickUpload({ name: 'fs with transcoding', fixture: 'video_short_720p.mp4' })
const uploadFixture = 'video_short_720p.mp4'
const { uuid: videoUUID } = await servers[0].videos.quickUpload({ name: 'fs with transcoding', fixture: uploadFixture })
uuid = videoUUID
await waitJobs(servers)
await checkSourceFile({ server: servers[0], fsCount: 1, uuid, fixture: uploadFixture })
for (const server of servers) {
const video = await server.videos.get({ id: uuid })
expect(video.inputFileUpdatedAt).to.be.null
@ -151,9 +243,23 @@ describe('Test a video file replacement', function () {
replaceDate = new Date()
await servers[0].videos.replaceSourceFile({ videoId: uuid, fixture: 'video_short_360p.mp4' })
const replaceFixture = 'video_short_360p.mp4'
await servers[0].videos.replaceSourceFile({ videoId: uuid, fixture: replaceFixture })
await waitJobs(servers)
const source = await checkSourceFile({ server: servers[0], fsCount: 1, uuid, fixture: replaceFixture })
expect(source.createdAt).to.exist
expect(source.fps).to.equal(25)
expect(source.height).to.equal(360)
expect(source.width).to.equal(640)
expect(source.resolution.id).to.equal(360)
expect(source.resolution.label).to.equal('360p')
expect(source.size).to.equal(30620)
expect(source.metadata?.format).to.exist
expect(source.metadata?.streams).to.be.an('array')
for (const server of servers) {
const video = await server.videos.get({ id: uuid })
@ -189,35 +295,36 @@ describe('Test a video file replacement', function () {
}
}
await servers[0].config.enableMinimumTranscoding()
await servers[0].config.enableMinimumTranscoding({ keepOriginal: true })
})
it('Should have cleaned up old files', async function () {
{
const count = await servers[0].servers.countFiles('storyboards')
expect(count).to.equal(2)
expect(count).to.equal(3)
}
{
const count = await servers[0].servers.countFiles('web-videos')
expect(count).to.equal(5 + 1) // +1 for private directory
expect(count).to.equal(6 + 1) // +1 for private directory
}
{
const count = await servers[0].servers.countFiles('streaming-playlists/hls')
expect(count).to.equal(1 + 1) // +1 for private directory
expect(count).to.equal(2 + 1) // +1 for private directory
}
{
const count = await servers[0].servers.countFiles('torrents')
expect(count).to.equal(9)
expect(count).to.equal(11)
}
})
it('Should have the correct source input', async function () {
it('Should have the correct source input filename', async function () {
const source = await servers[0].videos.getSource({ id: uuid })
expect(source.filename).to.equal('video_short_360p.mp4')
expect(source.inputFilename).to.equal('video_short_360p.mp4')
expect(new Date(source.createdAt)).to.be.above(replaceDate)
})
@ -367,6 +474,9 @@ describe('Test a video file replacement', function () {
expect(files[0].resolution.id).to.equal(360)
expectStartWith(files[0].fileUrl, objectStorage.getMockWebVideosBaseUrl())
}
const source = await servers[0].videos.getSource({ id: uuid })
expect(source.fileDownloadUrl).to.not.exist
})
it('Should replace a video file with transcoding enabled', async function () {
@ -374,16 +484,25 @@ describe('Test a video file replacement', function () {
const previousPaths: string[] = []
await servers[0].config.enableTranscoding({ hls: true, webVideo: true, with0p: true })
await servers[0].config.enableTranscoding({ hls: true, webVideo: true, with0p: true, keepOriginal: true })
const fixture1 = 'video_short_360p.mp4'
const { uuid: videoUUID } = await servers[0].videos.quickUpload({
name: 'object storage with transcoding',
fixture: 'video_short_360p.mp4'
fixture: fixture1
})
uuid = videoUUID
await waitJobs(servers)
await checkSourceFile({
server: servers[0],
fixture: fixture1,
fsCount: 0,
uuid,
objectStorageBaseUrl: objectStorage?.getMockOriginalFileBaseUrl()
})
for (const server of servers) {
const video = await server.videos.get({ id: uuid })
@ -403,9 +522,18 @@ describe('Test a video file replacement', function () {
}
}
await servers[0].videos.replaceSourceFile({ videoId: uuid, fixture: 'video_short_240p.mp4' })
const fixture2 = 'video_short_240p.mp4'
await servers[0].videos.replaceSourceFile({ videoId: uuid, fixture: fixture2 })
await waitJobs(servers)
await checkSourceFile({
server: servers[0],
fixture: fixture2,
fsCount: 0,
uuid,
objectStorageBaseUrl: objectStorage?.getMockOriginalFileBaseUrl()
})
for (const server of servers) {
const video = await server.videos.get({ id: uuid })

1
packages/tests/src/peertube-runner/index.ts
View File

@ -1,4 +1,5 @@
export * from './client-cli.js'
export * from './live-transcoding.js'
export * from './replace-file.js'
export * from './studio-transcoding.js'
export * from './vod-transcoding.js'

86
packages/tests/src/peertube-runner/replace-file.ts Normal file
View File

@ -0,0 +1,86 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import { getAllFiles } from '@peertube/peertube-core-utils'
import {
cleanupTests,
createSingleServer,
PeerTubeServer,
setAccessTokensToServers,
setDefaultVideoChannel,
waitJobs
} from '@peertube/peertube-server-commands'
import { PeerTubeRunnerProcess } from '@tests/shared/peertube-runner-process.js'
import { checkSourceFile } from '@tests/shared/videos.js'
import { expect } from 'chai'
describe('Test replace file using peertube-runner program', function () {
let server: PeerTubeServer
let peertubeRunner: PeerTubeRunnerProcess
let uuid: string
before(async function () {
this.timeout(120_000)
server = await createSingleServer(1)
await setAccessTokensToServers([ server ])
await setDefaultVideoChannel([ server ])
await server.config.enableRemoteTranscoding()
await server.config.enableFileUpdate()
await server.config.enableMinimumTranscoding({ hls: true, keepOriginal: true, webVideo: true })
const registrationToken = await server.runnerRegistrationTokens.getFirstRegistrationToken()
peertubeRunner = new PeerTubeRunnerProcess(server)
await peertubeRunner.runServer()
await peertubeRunner.registerPeerTubeInstance({ registrationToken, runnerName: 'runner' })
})
it('Should upload a webm video, transcode it and keep original file', async function () {
this.timeout(240000)
const fixture = 'video_short.webm';
({ uuid } = await server.videos.quickUpload({ name: 'video', fixture }))
await waitJobs(server, { runnerJobs: true })
const video = await server.videos.get({ id: uuid })
const files = getAllFiles(video)
expect(files).to.have.lengthOf(4)
expect(files[0].resolution.id).to.equal(720)
await checkSourceFile({ server, fsCount: 1, fixture, uuid })
})
it('Should upload an audio file, transcode it and keep original file', async function () {
const fixture = 'sample.ogg'
const { uuid } = await server.videos.quickUpload({ name: 'audio', fixture })
await waitJobs([ server ], { runnerJobs: true })
await checkSourceFile({ server, fsCount: 2, fixture, uuid })
})
it('Should replace the video', async function () {
const fixture = 'video_short_360p.mp4'
await server.videos.replaceSourceFile({ videoId: uuid, fixture })
await waitJobs(server, { runnerJobs: true })
const video = await server.videos.get({ id: uuid })
const files = getAllFiles(video)
expect(files).to.have.lengthOf(4)
expect(files[0].resolution.id).to.equal(360)
await checkSourceFile({ server, fsCount: 2, fixture, uuid })
})
after(async function () {
if (peertubeRunner) {
await peertubeRunner.unregisterPeerTubeInstance({ runnerName: 'runner' })
peertubeRunner.kill()
}
await cleanupTests([ server ])
})
})

61
packages/tests/src/shared/videos.ts
View File

@ -1,23 +1,23 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/no-floating-promises */
import { uuidRegex } from '@peertube/peertube-core-utils'
import { HttpStatusCode, HttpStatusCodeType, VideoCaption, VideoDetails, VideoPrivacy, VideoResolution } from '@peertube/peertube-models'
import { buildAbsoluteFixturePath, getFileSize, getFilenameFromUrl, getLowercaseExtension } from '@peertube/peertube-node-utils'
import { PeerTubeServer, VideoEdit, getRedirectionUrl, makeRawRequest, waitJobs } from '@peertube/peertube-server-commands'
import {
VIDEO_CATEGORIES,
VIDEO_LANGUAGES,
VIDEO_LICENCES,
VIDEO_PRIVACIES,
loadLanguages
} from '@peertube/peertube-server/core/initializers/constants.js'
import { expect } from 'chai'
import { pathExists } from 'fs-extra/esm'
import { readdir } from 'fs/promises'
import { basename, join } from 'path'
import { uuidRegex } from '@peertube/peertube-core-utils'
import { HttpStatusCode, HttpStatusCodeType, VideoCaption, VideoDetails, VideoPrivacy, VideoResolution } from '@peertube/peertube-models'
import {
loadLanguages,
VIDEO_CATEGORIES,
VIDEO_LANGUAGES,
VIDEO_LICENCES,
VIDEO_PRIVACIES
} from '@peertube/peertube-server/core/initializers/constants.js'
import { getLowercaseExtension } from '@peertube/peertube-node-utils'
import { makeRawRequest, PeerTubeServer, VideoEdit, waitJobs } from '@peertube/peertube-server-commands'
import { dateIsValid, expectStartWith, testImageGeneratedByFFmpeg } from './checks.js'
import { checkWebTorrentWorks } from './webtorrent.js'
import { completeCheckHlsPlaylist } from './streaming-playlists.js'
import { checkWebTorrentWorks } from './webtorrent.js'
export async function completeWebVideoFilesCheck (options: {
server: PeerTubeServer
@ -369,3 +369,40 @@ export async function uploadRandomVideoOnServers (
return res
}
export async function checkSourceFile (options: {
server: PeerTubeServer
fsCount: number
uuid: string
fixture: string
objectStorageBaseUrl?: string // default false
}) {
const { server, fsCount, fixture, uuid, objectStorageBaseUrl } = options
const source = await server.videos.getSource({ id: uuid })
const fixtureFileSize = await getFileSize(buildAbsoluteFixturePath(fixture))
if (fsCount > 0) {
expect(await server.servers.countFiles('original-video-files')).to.equal(fsCount)
const keptFilePath = join(server.servers.buildDirectory('original-video-files'), getFilenameFromUrl(source.fileDownloadUrl))
expect(await getFileSize(keptFilePath)).to.equal(fixtureFileSize)
}
expect(source.fileDownloadUrl).to.exist
if (objectStorageBaseUrl) {
const token = await server.videoToken.getVideoFileToken({ videoId: uuid })
expectStartWith(await getRedirectionUrl(source.fileDownloadUrl + '?videoFileToken=' + token), objectStorageBaseUrl)
}
const { body } = await makeRawRequest({
url: source.fileDownloadUrl,
token: server.accessToken,
redirects: 1,
expectedStatus: HttpStatusCode.OK_200
})
expect(body).to.have.lengthOf(fixtureFileSize)
return source
}

3
server/core/controllers/api/config.ts
View File

@ -320,6 +320,9 @@ function customConfig (): CustomConfig {
},
transcoding: {
enabled: CONFIG.TRANSCODING.ENABLED,
originalFile: {
keep: CONFIG.TRANSCODING.ORIGINAL_FILE.KEEP
},
remoteRunners: {
enabled: CONFIG.TRANSCODING.REMOTE_RUNNERS.ENABLED
},

4
server/core/controllers/api/videos/live.ts
View File

@ -184,9 +184,9 @@ async function addLiveVideo (req: express.Request, res: express.Response) {
duration: 0,
state: VideoState.WAITING_FOR_LIVE,
isLive: true,
filename: null
inputFilename: null
},
videoFilePath: undefined,
videoFile: undefined,
user: res.locals.oauth.token.User,
thumbnails
})

25
server/core/controllers/api/videos/source.ts
View File

@ -1,20 +1,20 @@
import express from 'express'
import { move } from 'fs-extra/esm'
import { buildAspectRatio } from '@peertube/peertube-core-utils'
import { VideoState } from '@peertube/peertube-models'
import { sequelizeTypescript } from '@server/initializers/database.js'
import { CreateJobArgument, CreateJobOptions, JobQueue } from '@server/lib/job-queue/index.js'
import { Hooks } from '@server/lib/plugins/hooks.js'
import { regenerateMiniaturesIfNeeded } from '@server/lib/thumbnail.js'
import { setupUploadResumableRoutes } from '@server/lib/uploadx.js'
import { buildMoveJob, buildStoryboardJobIfNeeded } from '@server/lib/video-jobs.js'
import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist.js'
import { buildNewFile } from '@server/lib/video-file.js'
import { buildNewFile, createVideoSource } from '@server/lib/video-file.js'
import { buildMoveJob, buildStoryboardJobIfNeeded } from '@server/lib/video-jobs.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { buildNextVideoState } from '@server/lib/video-state.js'
import { openapiOperationDoc } from '@server/middlewares/doc.js'
import { VideoModel } from '@server/models/video/video.js'
import { VideoSourceModel } from '@server/models/video/video-source.js'
import { MStreamingPlaylistFiles, MVideo, MVideoFile, MVideoFullLight } from '@server/types/models/index.js'
import { VideoState } from '@peertube/peertube-models'
import express from 'express'
import { move } from 'fs-extra/esm'
import { logger, loggerTagsFactory } from '../../../helpers/logger.js'
import {
asyncMiddleware,
@ -23,7 +23,6 @@ import {
replaceVideoSourceResumableValidator,
videoSourceGetLatestValidator
} from '../../../middlewares/index.js'
import { buildAspectRatio } from '@peertube/peertube-core-utils'
const lTags = loggerTagsFactory('api', 'video')
@ -61,7 +60,7 @@ async function replaceVideoSourceResumable (req: express.Request, res: express.R
const videoPhysicalFile = res.locals.updateVideoFileResumable
const user = res.locals.oauth.token.User
const videoFile = await buildNewFile({ path: videoPhysicalFile.path, mode: 'web-video' })
const videoFile = await buildNewFile({ path: videoPhysicalFile.path, mode: 'web-video', ffprobe: res.locals.ffprobe })
const originalFilename = videoPhysicalFile.originalname
const videoFileMutexReleaser = await VideoPathManager.Instance.lockFiles(res.locals.videoAll.uuid)
@ -114,13 +113,15 @@ async function replaceVideoSourceResumable (req: express.Request, res: express.R
await removeOldFiles({ video, files: oldWebVideoFiles, playlists: oldStreamingPlaylists })
const source = await VideoSourceModel.create({
filename: originalFilename,
videoId: video.id,
const source = await createVideoSource({
inputFilename: originalFilename,
inputProbe: res.locals.ffprobe,
inputPath: destination,
video,
createdAt: inputFileUpdatedAt
})
await regenerateMiniaturesIfNeeded(video)
await regenerateMiniaturesIfNeeded(video, res.locals.ffprobe)
await video.VideoChannel.setAsUpdated()
await addVideoJobsAfterUpload(video, video.getMaxQualityFile())

21
server/core/controllers/api/videos/upload.ts
View File

@ -1,12 +1,14 @@
import express from 'express'
import { ffprobePromise, getChaptersFromContainer } from '@peertube/peertube-ffmpeg'
import { HttpStatusCode, ThumbnailType, VideoCreate } from '@peertube/peertube-models'
import { uuidToShort } from '@peertube/peertube-node-utils'
import { getResumableUploadPath } from '@server/helpers/upload.js'
import { LocalVideoCreator } from '@server/lib/local-video-creator.js'
import { Redis } from '@server/lib/redis.js'
import { setupUploadResumableRoutes, uploadx } from '@server/lib/uploadx.js'
import { buildNextVideoState } from '@server/lib/video-state.js'
import { openapiOperationDoc } from '@server/middlewares/doc.js'
import { uuidToShort } from '@peertube/peertube-node-utils'
import { HttpStatusCode, ThumbnailType, VideoCreate } from '@peertube/peertube-models'
import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger.js'
import express from 'express'
import { VideoAuditView, auditLoggerFactory, getAuditIdFromRes } from '../../../helpers/audit-logger.js'
import { createReqFiles } from '../../../helpers/express-utils.js'
import { logger, loggerTagsFactory } from '../../../helpers/logger.js'
import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../../initializers/constants.js'
@ -19,8 +21,6 @@ import {
videosAddResumableInitValidator,
videosAddResumableValidator
} from '../../../middlewares/index.js'
import { ffprobePromise, getChaptersFromContainer } from '@peertube/peertube-ffmpeg'
import { LocalVideoCreator } from '@server/lib/local-video-creator.js'
const lTags = loggerTagsFactory('api', 'video')
const auditLogger = auditLoggerFactory('videos')
@ -134,7 +134,12 @@ async function addVideo (options: {
const localVideoCreator = new LocalVideoCreator({
lTags,
videoFilePath: videoPhysicalFile.path,
videoFile: {
path: videoPhysicalFile.path,
probe: res.locals.ffprobe
},
user: res.locals.oauth.token.User,
channel: res.locals.videoChannel,
@ -148,7 +153,7 @@ async function addVideo (options: {
...videoInfo,
duration: videoPhysicalFile.duration,
filename: videoPhysicalFile.originalname,
inputFilename: videoPhysicalFile.originalname,
state: buildNextVideoState(),
isLive: false
},

53
server/core/controllers/download.ts
View File

@ -1,12 +1,14 @@
import cors from 'cors'
import express from 'express'
import { forceNumber } from '@peertube/peertube-core-utils'
import { FileStorage, HttpStatusCode, VideoStreamingPlaylistType } from '@peertube/peertube-models'
import { logger } from '@server/helpers/logger.js'
import { VideoTorrentsSimpleFileCache } from '@server/lib/files-cache/index.js'
import {
generateHLSFilePresignedUrl,
generateOriginalFilePresignedUrl,
generateUserExportPresignedUrl,
generateWebVideoPresignedUrl
} from '@server/lib/object-storage/index.js'
import { getFSUserExportFilePath } from '@server/lib/paths.js'
import { Hooks } from '@server/lib/plugins/hooks.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
import {
@ -17,15 +19,16 @@ import {
MVideoFile,
MVideoFullLight
} from '@server/types/models/index.js'
import { forceNumber } from '@peertube/peertube-core-utils'
import { HttpStatusCode, FileStorage, VideoStreamingPlaylistType } from '@peertube/peertube-models'
import { MVideoSource } from '@server/types/models/video/video-source.js'
import cors from 'cors'
import express from 'express'
import { STATIC_DOWNLOAD_PATHS } from '../initializers/constants.js'
import {
asyncMiddleware, optionalAuthenticate,
originalVideoFileDownloadValidator,
userExportDownloadValidator,
videosDownloadValidator
} from '../middlewares/index.js'
import { getFSUserExportFilePath } from '@server/lib/paths.js'
const downloadRouter = express.Router()
@ -40,7 +43,7 @@ downloadRouter.use(
STATIC_DOWNLOAD_PATHS.VIDEOS + ':id-:resolution([0-9]+).:extension',
optionalAuthenticate,
asyncMiddleware(videosDownloadValidator),
asyncMiddleware(downloadVideoFile)
asyncMiddleware(downloadWebVideoFile)
)
downloadRouter.use(
@ -51,11 +54,18 @@ downloadRouter.use(
)
downloadRouter.use(
STATIC_DOWNLOAD_PATHS.USER_EXPORT + ':filename',
STATIC_DOWNLOAD_PATHS.USER_EXPORTS + ':filename',
asyncMiddleware(userExportDownloadValidator), // Include JWT token authentication
asyncMiddleware(downloadUserExport)
)
downloadRouter.use(
STATIC_DOWNLOAD_PATHS.ORIGINAL_VIDEO_FILE + ':filename',
optionalAuthenticate,
asyncMiddleware(originalVideoFileDownloadValidator),
asyncMiddleware(downloadOriginalFile)
)
// ---------------------------------------------------------------------------
export {
@ -91,7 +101,7 @@ async function downloadTorrent (req: express.Request, res: express.Response) {
return res.download(result.path, result.downloadName)
}
async function downloadVideoFile (req: express.Request, res: express.Response) {
async function downloadWebVideoFile (req: express.Request, res: express.Response) {
const video = res.locals.videoAll
const videoFile = getVideoFile(req, video.VideoFiles)
@ -184,6 +194,19 @@ function downloadUserExport (req: express.Request, res: express.Response) {
return Promise.resolve()
}
function downloadOriginalFile (req: express.Request, res: express.Response) {
const videoSource = res.locals.videoSource
const downloadFilename = videoSource.inputFilename
if (videoSource.storage === FileStorage.OBJECT_STORAGE) {
return redirectOriginalFileToObjectStorage({ res, videoSource, downloadFilename })
}
res.download(VideoPathManager.Instance.getFSOriginalVideoFilePath(videoSource.keptOriginalFilename), downloadFilename)
return Promise.resolve()
}
// ---------------------------------------------------------------------------
function getVideoFile (req: express.Request, files: MVideoFile[]) {
@ -262,3 +285,17 @@ async function redirectUserExportToObjectStorage (options: {
return res.redirect(url)
}
async function redirectOriginalFileToObjectStorage (options: {
res: express.Response
downloadFilename: string
videoSource: MVideoSource
}) {
const { res, downloadFilename, videoSource } = options
const url = await generateOriginalFilePresignedUrl({ videoSource, downloadFilename })
logger.debug('Generating pre-signed URL %s for original video file %s', url, videoSource.keptOriginalFilename)
return res.redirect(url)
}

4
server/core/controllers/static.ts
View File

@ -31,12 +31,12 @@ const privateWebVideoStaticMiddlewares = CONFIG.STATIC_FILES.PRIVATE_FILES_REQUI
staticRouter.use(
[ STATIC_PATHS.PRIVATE_WEB_VIDEOS, STATIC_PATHS.LEGACY_PRIVATE_WEB_VIDEOS ],
...privateWebVideoStaticMiddlewares,
express.static(DIRECTORIES.VIDEOS.PRIVATE, { fallthrough: false }),
express.static(DIRECTORIES.WEB_VIDEOS.PRIVATE, { fallthrough: false }),
handleStaticError
)
staticRouter.use(
[ STATIC_PATHS.WEB_VIDEOS, STATIC_PATHS.LEGACY_WEB_VIDEOS ],
express.static(DIRECTORIES.VIDEOS.PUBLIC, { fallthrough: false }),
express.static(DIRECTORIES.WEB_VIDEOS.PUBLIC, { fallthrough: false }),
handleStaticError
)

42
server/core/initializers/checker-after-init.ts
View File

@ -303,7 +303,7 @@ function checkLiveConfig () {
}
function checkObjectStorageConfig () {
if (CONFIG.OBJECT_STORAGE.ENABLED === true) {
if (CONFIG.OBJECT_STORAGE.ENABLED !== true) return
if (!CONFIG.OBJECT_STORAGE.WEB_VIDEOS.BUCKET_NAME) {
throw new Error('videos_bucket should be set when object storage support is enabled.')
@ -313,25 +313,59 @@ function checkObjectStorageConfig () {
throw new Error('streaming_playlists_bucket should be set when object storage support is enabled.')
}
// Check web videos and hls videos are not in the same bucket or directory
if (
CONFIG.OBJECT_STORAGE.WEB_VIDEOS.BUCKET_NAME === CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET_NAME &&
CONFIG.OBJECT_STORAGE.WEB_VIDEOS.PREFIX === CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.PREFIX
) {
if (CONFIG.OBJECT_STORAGE.WEB_VIDEOS.PREFIX === '') {
throw new Error('Object storage bucket prefixes should be set when the same bucket is used for both types of video.')
throw new Error('Bucket prefixes should be set when the same bucket is used for both types of video.')
}
throw new Error(
'Object storage bucket prefixes should be set to different values when the same bucket is used for both types of video.'
'Bucket prefixes should be set to different values when the same bucket is used for both types of video.'
)
}
if (CONFIG.TRANSCODING.ORIGINAL_FILE.KEEP) {
if (!CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES.BUCKET_NAME) {
throw new Error('original_video_files_bucket should be set when object storage support is enabled.')
}
// Check web videos/hls videos are not in the same bucket or directory as original video files
if (
CONFIG.OBJECT_STORAGE.WEB_VIDEOS.BUCKET_NAME === CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES.BUCKET_NAME &&
CONFIG.OBJECT_STORAGE.WEB_VIDEOS.PREFIX === CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES.PREFIX
) {
if (CONFIG.OBJECT_STORAGE.WEB_VIDEOS.PREFIX === '') {
throw new Error('Bucket prefixes should be set when the same bucket is used for both original and web video files.')
}
throw new Error(
'Bucket prefixes should be set to different values when the same bucket is used for both original and web video files.'
)
}
if (
CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET_NAME === CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES.BUCKET_NAME &&
CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.PREFIX === CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES.PREFIX
) {
if (CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.PREFIX === '') {
throw new Error('Bucket prefixes should be set when the same bucket is used for both original and hls files.')
}
throw new Error(
'Bucket prefixes should be set to different values when the same bucket is used for both original and hls files.'
)
}
}
if (CONFIG.OBJECT_STORAGE.MAX_UPLOAD_PART > parseBytes('250MB')) {
// eslint-disable-next-line max-len
logger.warn(`Object storage max upload part seems to have a big value (${CONFIG.OBJECT_STORAGE.MAX_UPLOAD_PART} bytes). Consider using a lower one (like 100MB).`)
}
}
}
function checkVideoStudioConfig () {
if (CONFIG.VIDEO_STUDIO.ENABLED === true && CONFIG.TRANSCODING.ENABLED === false) {

7
server/core/initializers/checker-before-init.ts
View File

@ -32,8 +32,8 @@ function checkMissedConfig () {
'signup.enabled', 'signup.limit', 'signup.requires_approval', 'signup.requires_email_verification', 'signup.minimum_age',
'signup.filters.cidr.whitelist', 'signup.filters.cidr.blacklist',
'redundancy.videos.strategies', 'redundancy.videos.check_interval',
'transcoding.enabled', 'transcoding.threads', 'transcoding.allow_additional_extensions', 'transcoding.web_videos.enabled',
'transcoding.hls.enabled', 'transcoding.profile', 'transcoding.concurrency',
'transcoding.enabled', 'transcoding.original_file.keep', 'transcoding.threads', 'transcoding.allow_additional_extensions',
'transcoding.web_videos.enabled', 'transcoding.hls.enabled', 'transcoding.profile', 'transcoding.concurrency',
'transcoding.resolutions.0p', 'transcoding.resolutions.144p', 'transcoding.resolutions.240p', 'transcoding.resolutions.360p',
'transcoding.resolutions.480p', 'transcoding.resolutions.720p', 'transcoding.resolutions.1080p', 'transcoding.resolutions.1440p',
'transcoding.resolutions.2160p', 'transcoding.always_transcode_original_resolution', 'transcoding.remote_runners.enabled',
@ -66,7 +66,8 @@ function checkMissedConfig () {
'object_storage.upload_acl.private', 'object_storage.proxy.proxify_private_files', 'object_storage.credentials.access_key_id',
'object_storage.credentials.secret_access_key', 'object_storage.max_upload_part', 'object_storage.streaming_playlists.bucket_name',
'object_storage.streaming_playlists.prefix', 'object_storage.streaming_playlists.base_url', 'object_storage.web_videos.bucket_name',
'object_storage.web_videos.prefix', 'object_storage.web_videos.base_url',
'object_storage.web_videos.prefix', 'object_storage.web_videos.base_url', 'object_storage.original_video_files.bucket_name',
'object_storage.original_video_files.prefix', 'object_storage.original_video_files.base_url',
'theme.default',
'feeds.videos.count', 'feeds.comments.count',
'geo_ip.enabled', 'geo_ip.country.database_url', 'geo_ip.city.database_url',

9
server/core/initializers/config.ts
View File

@ -114,6 +114,7 @@ const CONFIG = {
LOG_DIR: buildPath(config.get<string>('storage.logs')),
WEB_VIDEOS_DIR: buildPath(config.get<string>('storage.web_videos')),
STREAMING_PLAYLISTS_DIR: buildPath(config.get<string>('storage.streaming_playlists')),
ORIGINAL_VIDEO_FILES_DIR: buildPath(config.get<string>('storage.original_video_files')),
REDUNDANCY_DIR: buildPath(config.get<string>('storage.redundancy')),
THUMBNAILS_DIR: buildPath(config.get<string>('storage.thumbnails')),
STORYBOARDS_DIR: buildPath(config.get<string>('storage.storyboards')),
@ -159,6 +160,11 @@ const CONFIG = {
BUCKET_NAME: config.get<string>('object_storage.user_exports.bucket_name'),
PREFIX: config.get<string>('object_storage.user_exports.prefix'),
BASE_URL: config.get<string>('object_storage.user_exports.base_url')
},
ORIGINAL_VIDEO_FILES: {
BUCKET_NAME: config.get<string>('object_storage.original_video_files.bucket_name'),
PREFIX: config.get<string>('object_storage.original_video_files.prefix'),
BASE_URL: config.get<string>('object_storage.original_video_files.base_url')
}
},
WEBSERVER: {
@ -412,6 +418,9 @@ const CONFIG = {
},
TRANSCODING: {
get ENABLED () { return config.get<boolean>('transcoding.enabled') },
ORIGINAL_FILE: {
get KEEP () { return config.get<boolean>('transcoding.original_file.keep') }
},
get ALLOW_ADDITIONAL_EXTENSIONS () { return config.get<boolean>('transcoding.allow_additional_extensions') },
get ALLOW_AUDIO_FILES () { return config.get<boolean>('transcoding.allow_audio_files') },
get THREADS () { return config.get<number>('transcoding.threads') },

9
server/core/initializers/constants.ts
View File

@ -45,7 +45,7 @@ import { cpus } from 'os'
// ---------------------------------------------------------------------------
const LAST_MIGRATION_VERSION = 825
const LAST_MIGRATION_VERSION = 830
// ---------------------------------------------------------------------------
@ -857,7 +857,8 @@ const STATIC_DOWNLOAD_PATHS = {
TORRENTS: '/download/torrents/',
VIDEOS: '/download/videos/',
HLS_VIDEOS: '/download/streaming-playlists/hls/videos/',
USER_EXPORT: '/download/user-export/'
USER_EXPORTS: '/download/user-exports/',
ORIGINAL_VIDEO_FILE: '/download/original-video-files/'
}
const LAZY_STATIC_PATHS = {
THUMBNAILS: '/lazy-static/thumbnails/',
@ -981,11 +982,13 @@ const DIRECTORIES = {
PRIVATE: join(CONFIG.STORAGE.STREAMING_PLAYLISTS_DIR, 'hls', 'private')
},
VIDEOS: {
WEB_VIDEOS: {
PUBLIC: CONFIG.STORAGE.WEB_VIDEOS_DIR,
PRIVATE: join(CONFIG.STORAGE.WEB_VIDEOS_DIR, 'private')
},
ORIGINAL_VIDEOS: CONFIG.STORAGE.ORIGINAL_VIDEO_FILES_DIR,
HLS_REDUNDANCY: join(CONFIG.STORAGE.REDUNDANCY_DIR, 'hls')
}

4
server/core/initializers/installer.ts
View File

@ -96,8 +96,8 @@ function createDirectoriesIfNotExist () {
tasks.push(ensureDir(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE))
tasks.push(ensureDir(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC))
tasks.push(ensureDir(DIRECTORIES.VIDEOS.PUBLIC))
tasks.push(ensureDir(DIRECTORIES.VIDEOS.PRIVATE))
tasks.push(ensureDir(DIRECTORIES.WEB_VIDEOS.PUBLIC))
tasks.push(ensureDir(DIRECTORIES.WEB_VIDEOS.PRIVATE))
// Resumable upload directory
tasks.push(ensureDir(DIRECTORIES.RESUMABLE_UPLOAD))

91
server/core/initializers/migrations/0830-keep-original-file.ts Normal file
View File

@ -0,0 +1,91 @@
import * as Sequelize from 'sequelize'
async function up (utils: {
transaction: Sequelize.Transaction
queryInterface: Sequelize.QueryInterface
sequelize: Sequelize.Sequelize
}): Promise<void> {
const { transaction } = utils
{
await utils.queryInterface.addColumn('videoSource', 'keptOriginalFilename', {
type: Sequelize.STRING,
allowNull: true
}, { transaction })
}
{
await utils.queryInterface.addColumn('videoSource', 'storage', {
type: Sequelize.INTEGER,
allowNull: true
}, { transaction })
}
{
await utils.queryInterface.addColumn('videoSource', 'resolution', {
type: Sequelize.INTEGER,
allowNull: true
}, { transaction })
}
{
await utils.queryInterface.addColumn('videoSource', 'width', {
type: Sequelize.INTEGER,
allowNull: true
}, { transaction })
}
{
await utils.queryInterface.addColumn('videoSource', 'height', {
type: Sequelize.INTEGER,
allowNull: true
}, { transaction })
}
{
await utils.queryInterface.addColumn('videoSource', 'fps', {
type: Sequelize.INTEGER,
allowNull: true
}, { transaction })
}
{
await utils.queryInterface.addColumn('videoSource', 'size', {
type: Sequelize.INTEGER,
allowNull: true
}, { transaction })
}
{
await utils.queryInterface.addColumn('videoSource', 'metadata', {
type: Sequelize.JSONB,
allowNull: true
}, { transaction })
}
{
await utils.queryInterface.addColumn('videoSource', 'fileUrl', {
type: Sequelize.STRING,
allowNull: true
}, { transaction })
}
{
await utils.queryInterface.renameColumn('videoSource', 'filename', 'inputFilename', { transaction })
}
{
await utils.queryInterface.addColumn('userExport', 'fileUrl', {
type: Sequelize.STRING,
allowNull: true
}, { transaction })
}
}
function down (options) {
throw new Error('Not implemented.')
}
export {
down, up
}

5
server/core/lib/job-queue/handlers/video-live-ending.ts
View File

@ -166,7 +166,8 @@ async function saveReplayToExternalVideo (options: {
const thumbnails = await generateLocalVideoMiniature({
video: replayVideo,
videoFile: replayVideo.getMaxQualityFile(),
types: [ ThumbnailType.MINIATURE, ThumbnailType.PREVIEW ]
types: [ ThumbnailType.MINIATURE, ThumbnailType.PREVIEW ],
ffprobe: undefined
})
for (const thumbnail of thumbnails) {
@ -238,7 +239,7 @@ async function replaceLiveByReplay (options: {
}
// Regenerate the thumbnail & preview?
await regenerateMiniaturesIfNeeded(videoWithFiles)
await regenerateMiniaturesIfNeeded(videoWithFiles, undefined)
// We consider this is a new video
await moveToNextState({ video: videoWithFiles, isNewVideo: true })

51
server/core/lib/local-video-creator.ts
View File

@ -1,5 +1,4 @@
import { buildAspectRatio } from '@peertube/peertube-core-utils'
import { ffprobePromise } from '@peertube/peertube-ffmpeg'
import {
LiveVideoCreate,
LiveVideoLatencyMode,
@ -18,11 +17,10 @@ import { ScheduleVideoUpdateModel } from '@server/models/video/schedule-video-up
import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting.js'
import { VideoLiveModel } from '@server/models/video/video-live.js'
import { VideoPasswordModel } from '@server/models/video/video-password.js'
import { VideoSourceModel } from '@server/models/video/video-source.js'
import { VideoModel } from '@server/models/video/video.js'
import { MChannel, MChannelAccountLight, MThumbnail, MUser, MVideoFile, MVideoFullLight } from '@server/types/models/index.js'
import { FilteredModelAttributes } from '@server/types/sequelize.js'
import Ffmpeg from 'fluent-ffmpeg'
import { FfprobeData } from 'fluent-ffmpeg'
import { move } from 'fs-extra/esm'
import { getLocalVideoActivityPubUrl } from './activitypub/url.js'
import { federateVideoIfNeeded } from './activitypub/videos/federate.js'
@ -30,7 +28,7 @@ import { Hooks } from './plugins/hooks.js'
import { generateLocalVideoMiniature, updateLocalVideoMiniatureFromExisting } from './thumbnail.js'
import { autoBlacklistVideoIfNeeded } from './video-blacklist.js'
import { replaceChapters, replaceChaptersFromDescriptionIfNeeded } from './video-chapters.js'
import { buildNewFile } from './video-file.js'
import { buildNewFile, createVideoSource } from './video-file.js'
import { addVideoJobsAfterCreation } from './video-jobs.js'
import { VideoPathManager } from './video-path-manager.js'
import { setVideoTags } from './video.js'
@ -39,7 +37,7 @@ type VideoAttributes = Omit<VideoCreate, 'channelId'> & {
duration: number
isLive: boolean
state: VideoStateType
filename: string
inputFilename: string
}
type LiveAttributes = Pick<LiveVideoCreate, 'permanentLive' | 'latencyMode' | 'saveReplay' | 'replaySettings'> & {
@ -64,6 +62,8 @@ export class LocalVideoCreator {
private readonly lTags: LoggerTagsFn
private readonly videoFilePath: string | undefined
private readonly videoFileProbe: FfprobeData
private readonly videoAttributes: VideoAttributes
private readonly liveAttributes: LiveAttributes | undefined
@ -72,12 +72,15 @@ export class LocalVideoCreator {
private video: MVideoFullLight
private videoFile: MVideoFile
private ffprobe: Ffmpeg.FfprobeData
private videoPath: string
constructor (private readonly options: {
lTags: LoggerTagsFn
videoFilePath: string
videoFile: {
path: string
probe: FfprobeData
}
videoAttributes: VideoAttributes
liveAttributes: LiveAttributes
@ -93,7 +96,8 @@ export class LocalVideoCreator {
finalFallback: ChaptersOption | undefined
}
}) {
this.videoFilePath = options.videoFilePath
this.videoFilePath = options.videoFile?.path
this.videoFileProbe = options.videoFile?.probe
this.videoAttributes = options.videoAttributes
this.liveAttributes = options.liveAttributes
@ -112,11 +116,10 @@ export class LocalVideoCreator {
this.video.url = getLocalVideoActivityPubUrl(this.video)
if (this.videoFilePath) {
this.ffprobe = await ffprobePromise(this.videoFilePath)
this.videoFile = await buildNewFile({ path: this.videoFilePath, mode: 'web-video', ffprobe: this.ffprobe })
this.videoFile = await buildNewFile({ path: this.videoFilePath, mode: 'web-video', ffprobe: this.videoFileProbe })
const destination = VideoPathManager.Instance.getFSVideoFileOutputPath(this.video, this.videoFile)
await move(this.videoFilePath, destination)
this.videoPath = VideoPathManager.Instance.getFSVideoFileOutputPath(this.video, this.videoFile)
await move(this.videoFilePath, this.videoPath)
this.video.aspectRatio = buildAspectRatio({ width: this.videoFile.width, height: this.videoFile.height })
}
@ -166,13 +169,6 @@ export class LocalVideoCreator {
transaction
})
if (this.videoAttributes.filename) {
await VideoSourceModel.create({
filename: this.videoAttributes.filename,
videoId: this.video.id
}, { transaction })
}
if (this.videoAttributes.privacy === VideoPrivacy.PASSWORD_PROTECTED) {
await VideoPasswordModel.addPasswords(this.videoAttributes.videoPasswords, this.video.id, transaction)
}
@ -197,6 +193,7 @@ export class LocalVideoCreator {
videoLive.videoId = this.video.id
this.video.VideoLive = await videoLive.save({ transaction })
}
if (this.videoFile) {
transaction.afterCommit(() => {
addVideoJobsAfterCreation({ video: this.video, videoFile: this.videoFile })
@ -218,6 +215,15 @@ export class LocalVideoCreator {
})
})
if (this.videoAttributes.inputFilename) {
await createVideoSource({
inputFilename: this.videoAttributes.inputFilename,
inputPath: this.videoPath,
inputProbe: this.videoFileProbe,
video: this.video
})
}
// Channel has a new content, set as updated
await this.channel.setAsUpdated()
@ -248,7 +254,12 @@ export class LocalVideoCreator {
return [
...await Promise.all(promises),
...await generateLocalVideoMiniature({ video: this.video, videoFile: this.videoFile, types: toGenerate, ffprobe: this.ffprobe })
...await generateLocalVideoMiniature({
video: this.video,
videoFile: this.videoFile,
types: toGenerate,
ffprobe: this.videoFileProbe
})
]
}

15
server/core/lib/object-storage/keys.ts
View File

@ -1,25 +1,22 @@
import { join } from 'path'
import { MStreamingPlaylistVideo } from '@server/types/models/index.js'
function generateHLSObjectStorageKey (playlist: MStreamingPlaylistVideo, filename: string) {
export function generateHLSObjectStorageKey (playlist: MStreamingPlaylistVideo, filename: string) {
return join(generateHLSObjectBaseStorageKey(playlist), filename)
}
function generateHLSObjectBaseStorageKey (playlist: MStreamingPlaylistVideo) {
export function generateHLSObjectBaseStorageKey (playlist: MStreamingPlaylistVideo) {
return join(playlist.getStringType(), playlist.Video.uuid)
}
function generateWebVideoObjectStorageKey (filename: string) {
export function generateWebVideoObjectStorageKey (filename: string) {
return filename
}
function generateUserExportObjectStorageKey (filename: string) {
export function generateOriginalVideoObjectStorageKey (filename: string) {
return filename
}
export {
generateHLSObjectStorageKey,
generateHLSObjectBaseStorageKey,
generateWebVideoObjectStorageKey,
generateUserExportObjectStorageKey
export function generateUserExportObjectStorageKey (filename: string) {
return filename
}

31
server/core/lib/object-storage/pre-signed-urls.ts
View File

@ -1,8 +1,14 @@
import { CONFIG } from '@server/initializers/config.js'
import { MStreamingPlaylistVideo, MUserExport, MVideoFile } from '@server/types/models/index.js'
import { generateHLSObjectStorageKey, generateUserExportObjectStorageKey, generateWebVideoObjectStorageKey } from './keys.js'
import { MVideoSource } from '@server/types/models/video/video-source.js'
import {
generateHLSObjectStorageKey,
generateOriginalVideoObjectStorageKey,
generateUserExportObjectStorageKey,
generateWebVideoObjectStorageKey
} from './keys.js'
import { buildKey, getClient } from './shared/index.js'
import { getHLSPublicFileUrl, getWebVideoPublicFileUrl } from './urls.js'
import { getObjectStoragePublicFileUrl } from './urls.js'
export async function generateWebVideoPresignedUrl (options: {
file: MVideoFile
@ -16,7 +22,7 @@ export async function generateWebVideoPresignedUrl (options: {
downloadFilename
})
return getWebVideoPublicFileUrl(url)
return getObjectStoragePublicFileUrl(url, CONFIG.OBJECT_STORAGE.WEB_VIDEOS)
}
export async function generateHLSFilePresignedUrl (options: {
@ -32,7 +38,7 @@ export async function generateHLSFilePresignedUrl (options: {
downloadFilename
})
return getHLSPublicFileUrl(url)
return getObjectStoragePublicFileUrl(url, CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
}
export async function generateUserExportPresignedUrl (options: {
@ -47,7 +53,22 @@ export async function generateUserExportPresignedUrl (options: {
downloadFilename
})
return getHLSPublicFileUrl(url)
return getObjectStoragePublicFileUrl(url, CONFIG.OBJECT_STORAGE.USER_EXPORTS)
}
export async function generateOriginalFilePresignedUrl (options: {
videoSource: MVideoSource
downloadFilename: string
}) {
const { videoSource, downloadFilename } = options
const url = await generatePresignedUrl({
bucket: CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES.BUCKET_NAME,
key: buildKey(generateOriginalVideoObjectStorageKey(videoSource.keptOriginalFilename), CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES),
downloadFilename
})
return getObjectStoragePublicFileUrl(url, CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES)
}
// ---------------------------------------------------------------------------

33
server/core/lib/object-storage/urls.ts
View File

@ -1,23 +1,15 @@
import { CONFIG } from '@server/initializers/config.js'
import { OBJECT_STORAGE_PROXY_PATHS, WEBSERVER } from '@server/initializers/constants.js'
import { MVideoUUID } from '@server/types/models/index.js'
import { BucketInfo, buildKey, getEndpointParsed } from './shared/index.js'
function getInternalUrl (config: BucketInfo, keyWithoutPrefix: string) {
export function getInternalUrl (config: BucketInfo, keyWithoutPrefix: string) {
return getBaseUrl(config) + buildKey(keyWithoutPrefix, config)
}
// ---------------------------------------------------------------------------
function getWebVideoPublicFileUrl (fileUrl: string) {
const baseUrl = CONFIG.OBJECT_STORAGE.WEB_VIDEOS.BASE_URL
if (!baseUrl) return fileUrl
return replaceByBaseUrl(fileUrl, baseUrl)
}
function getHLSPublicFileUrl (fileUrl: string) {
const baseUrl = CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BASE_URL
export function getObjectStoragePublicFileUrl (fileUrl: string, objectStorageConfig: { BASE_URL: string }) {
const baseUrl = objectStorageConfig.BASE_URL
if (!baseUrl) return fileUrl
return replaceByBaseUrl(fileUrl, baseUrl)
@ -25,28 +17,13 @@ function getHLSPublicFileUrl (fileUrl: string) {
// ---------------------------------------------------------------------------
function getHLSPrivateFileUrl (video: MVideoUUID, filename: string) {
export function getHLSPrivateFileUrl (video: MVideoUUID, filename: string) {
return WEBSERVER.URL + OBJECT_STORAGE_PROXY_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS + video.uuid + `/${filename}`
}
function getWebVideoPrivateFileUrl (filename: string) {
export function getWebVideoPrivateFileUrl (filename: string) {
return WEBSERVER.URL + OBJECT_STORAGE_PROXY_PATHS.PRIVATE_WEB_VIDEOS + filename
}
// ---------------------------------------------------------------------------
export {
getInternalUrl,
getWebVideoPublicFileUrl,
getHLSPublicFileUrl,
getHLSPrivateFileUrl,
getWebVideoPrivateFileUrl,
replaceByBaseUrl
}
// ---------------------------------------------------------------------------
function getBaseUrl (bucketInfo: BucketInfo, baseUrl?: string) {

95
server/core/lib/object-storage/videos.ts
View File

@ -1,14 +1,20 @@
import { basename, join } from 'path'
import { logger } from '@server/helpers/logger.js'
import { CONFIG } from '@server/initializers/config.js'
import { MStreamingPlaylistVideo, MVideo, MVideoFile } from '@server/types/models/index.js'
import { MVideoSource } from '@server/types/models/video/video-source.js'
import { basename, join } from 'path'
import { getHLSDirectory } from '../paths.js'
import { VideoPathManager } from '../video-path-manager.js'
import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebVideoObjectStorageKey } from './keys.js'
import {
generateHLSObjectBaseStorageKey,
generateHLSObjectStorageKey,
generateOriginalVideoObjectStorageKey,
generateWebVideoObjectStorageKey
} from './keys.js'
import {
createObjectReadStream,
listKeysOfPrefix,
lTags,
listKeysOfPrefix,
makeAvailable,
removeObject,
removeObjectByFullKey,
@ -19,13 +25,13 @@ import {
updatePrefixACL
} from './shared/index.js'
function listHLSFileKeysOf (playlist: MStreamingPlaylistVideo) {
export function listHLSFileKeysOf (playlist: MStreamingPlaylistVideo) {
return listKeysOfPrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
}
// ---------------------------------------------------------------------------
function storeHLSFileFromFilename (playlist: MStreamingPlaylistVideo, filename: string) {
export function storeHLSFileFromFilename (playlist: MStreamingPlaylistVideo, filename: string) {
return storeObject({
inputPath: join(getHLSDirectory(playlist.Video), filename),
objectStorageKey: generateHLSObjectStorageKey(playlist, filename),
@ -34,7 +40,7 @@ function storeHLSFileFromFilename (playlist: MStreamingPlaylistVideo, filename:
})
}
function storeHLSFileFromPath (playlist: MStreamingPlaylistVideo, path: string) {
export function storeHLSFileFromPath (playlist: MStreamingPlaylistVideo, path: string) {
return storeObject({
inputPath: path,
objectStorageKey: generateHLSObjectStorageKey(playlist, basename(path)),
@ -43,7 +49,7 @@ function storeHLSFileFromPath (playlist: MStreamingPlaylistVideo, path: string)
})
}
function storeHLSFileFromContent (playlist: MStreamingPlaylistVideo, path: string, content: string) {
export function storeHLSFileFromContent (playlist: MStreamingPlaylistVideo, path: string, content: string) {
return storeContent({
content,
inputPath: path,
@ -55,7 +61,7 @@ function storeHLSFileFromContent (playlist: MStreamingPlaylistVideo, path: strin
// ---------------------------------------------------------------------------
function storeWebVideoFile (video: MVideo, file: MVideoFile) {
export function storeWebVideoFile (video: MVideo, file: MVideoFile) {
return storeObject({
inputPath: VideoPathManager.Instance.getFSVideoFileOutputPath(video, file),
objectStorageKey: generateWebVideoObjectStorageKey(file.filename),
@ -66,7 +72,18 @@ function storeWebVideoFile (video: MVideo, file: MVideoFile) {
// ---------------------------------------------------------------------------
async function updateWebVideoFileACL (video: MVideo, file: MVideoFile) {
export function storeOriginalVideoFile (inputPath: string, filename: string) {
return storeObject({
inputPath,
objectStorageKey: generateOriginalVideoObjectStorageKey(filename),
bucketInfo: CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES,
isPrivate: true
})
}
// ---------------------------------------------------------------------------
export async function updateWebVideoFileACL (video: MVideo, file: MVideoFile) {
await updateObjectACL({
objectStorageKey: generateWebVideoObjectStorageKey(file.filename),
bucketInfo: CONFIG.OBJECT_STORAGE.WEB_VIDEOS,
@ -74,7 +91,7 @@ async function updateWebVideoFileACL (video: MVideo, file: MVideoFile) {
})
}
async function updateHLSFilesACL (playlist: MStreamingPlaylistVideo) {
export async function updateHLSFilesACL (playlist: MStreamingPlaylistVideo) {
await updatePrefixACL({
prefix: generateHLSObjectBaseStorageKey(playlist),
bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS,
@ -84,31 +101,37 @@ async function updateHLSFilesACL (playlist: MStreamingPlaylistVideo) {
// ---------------------------------------------------------------------------
function removeHLSObjectStorage (playlist: MStreamingPlaylistVideo) {
export function removeHLSObjectStorage (playlist: MStreamingPlaylistVideo) {
return removePrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
}
function removeHLSFileObjectStorageByFilename (playlist: MStreamingPlaylistVideo, filename: string) {
export function removeHLSFileObjectStorageByFilename (playlist: MStreamingPlaylistVideo, filename: string) {
return removeObject(generateHLSObjectStorageKey(playlist, filename), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
}
function removeHLSFileObjectStorageByPath (playlist: MStreamingPlaylistVideo, path: string) {
export function removeHLSFileObjectStorageByPath (playlist: MStreamingPlaylistVideo, path: string) {
return removeObject(generateHLSObjectStorageKey(playlist, basename(path)), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
}
function removeHLSFileObjectStorageByFullKey (key: string) {
export function removeHLSFileObjectStorageByFullKey (key: string) {
return removeObjectByFullKey(key, CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
}
// ---------------------------------------------------------------------------
function removeWebVideoObjectStorage (videoFile: MVideoFile) {
export function removeWebVideoObjectStorage (videoFile: MVideoFile) {
return removeObject(generateWebVideoObjectStorageKey(videoFile.filename), CONFIG.OBJECT_STORAGE.WEB_VIDEOS)
}
// ---------------------------------------------------------------------------
async function makeHLSFileAvailable (playlist: MStreamingPlaylistVideo, filename: string, destination: string) {
export function removeOriginalFileObjectStorage (videoSource: MVideoSource) {
return removeObject(generateOriginalVideoObjectStorageKey(videoSource.keptOriginalFilename), CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES)
}
// ---------------------------------------------------------------------------
export async function makeHLSFileAvailable (playlist: MStreamingPlaylistVideo, filename: string, destination: string) {
const key = generateHLSObjectStorageKey(playlist, filename)
logger.info('Fetching HLS file %s from object storage to %s.', key, destination, lTags())
@ -122,7 +145,7 @@ async function makeHLSFileAvailable (playlist: MStreamingPlaylistVideo, filename
return destination
}
async function makeWebVideoFileAvailable (filename: string, destination: string) {
export async function makeWebVideoFileAvailable (filename: string, destination: string) {
const key = generateWebVideoObjectStorageKey(filename)
logger.info('Fetching Web Video file %s from object storage to %s.', key, destination, lTags())
@ -138,7 +161,7 @@ async function makeWebVideoFileAvailable (filename: string, destination: string)
// ---------------------------------------------------------------------------
function getWebVideoFileReadStream (options: {
export function getWebVideoFileReadStream (options: {
filename: string
rangeHeader: string
}) {
@ -153,7 +176,7 @@ function getWebVideoFileReadStream (options: {
})
}
function getHLSFileReadStream (options: {
export function getHLSFileReadStream (options: {
playlist: MStreamingPlaylistVideo
filename: string
rangeHeader: string
@ -169,29 +192,17 @@ function getHLSFileReadStream (options: {
})
}
// ---------------------------------------------------------------------------
export function getOriginalFileReadStream (options: {
keptOriginalFilename: string
rangeHeader: string
}) {
const { keptOriginalFilename, rangeHeader } = options
export {
listHLSFileKeysOf,
const key = generateOriginalVideoObjectStorageKey(keptOriginalFilename)
storeWebVideoFile,
storeHLSFileFromFilename,
storeHLSFileFromPath,
storeHLSFileFromContent,
updateWebVideoFileACL,
updateHLSFilesACL,
removeHLSObjectStorage,
removeHLSFileObjectStorageByFilename,
removeHLSFileObjectStorageByPath,
removeHLSFileObjectStorageByFullKey,
removeWebVideoObjectStorage,
makeWebVideoFileAvailable,
makeHLSFileAvailable,
getWebVideoFileReadStream,
getHLSFileReadStream
return createObjectReadStream({
key,
bucketInfo: CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES,
rangeHeader
})
}

5
server/core/lib/thumbnail.ts
View File

@ -100,7 +100,7 @@ function generateLocalVideoMiniature (options: {
video: MVideoThumbnail
videoFile: MVideoFile
types: ThumbnailType_Type[]
ffprobe?: FfprobeData
ffprobe: FfprobeData
}): Promise<MThumbnail[]> {
const { video, videoFile, types, ffprobe } = options
@ -223,7 +223,7 @@ function updateRemoteVideoThumbnail (options: {
// ---------------------------------------------------------------------------
async function regenerateMiniaturesIfNeeded (video: MVideoWithAllFiles) {
async function regenerateMiniaturesIfNeeded (video: MVideoWithAllFiles, ffprobe: FfprobeData) {
const thumbnailsToGenerate: ThumbnailType_Type[] = []
if (video.getMiniature().automaticallyGenerated === true) {
@ -237,6 +237,7 @@ async function regenerateMiniaturesIfNeeded (video: MVideoWithAllFiles) {
const models = await generateLocalVideoMiniature({
video,
videoFile: video.getMaxQualityFile(),
ffprobe,
types: thumbnailsToGenerate
})

8
server/core/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts
View File

@ -1,3 +1,4 @@
import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS, hasAudioStream, isAudioFile } from '@peertube/peertube-ffmpeg'
import { computeOutputFPS } from '@server/helpers/ffmpeg/index.js'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { CONFIG } from '@server/initializers/config.js'
@ -11,7 +12,6 @@ import {
import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { MUserId, MVideoFile, MVideoFullLight, MVideoWithFileThumbnail } from '@server/types/models/index.js'
import { MRunnerJob } from '@server/types/models/runners/index.js'
import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS, hasAudioStream, isAudioFile } from '@peertube/peertube-ffmpeg'
import { getTranscodingJobPriority } from '../../transcoding-priority.js'
import { buildOriginalFileResolution, computeResolutionsToTranscode } from '../../transcoding-resolutions.js'
import { AbstractJobBuilder } from './abstract-job-builder.js'
@ -60,11 +60,7 @@ export class TranscodingRunnerJobBuilder extends AbstractJobBuilder {
const fps = computeOutputFPS({ inputFPS, resolution: maxResolution })
const priority = await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 })
const deleteInputFileId = isAudioInput || maxResolution !== resolution
? videoFile.id
: null
const jobPayload = { video, resolution: maxResolution, fps, isNewVideo, priority, deleteInputFileId }
const jobPayload = { video, resolution: maxResolution, fps, isNewVideo, priority, deleteInputFileId: videoFile.id }
const mainRunnerJob = videoFile.isAudio()
? await new VODAudioMergeTranscodingJobHandler().create(jobPayload)

27
server/core/lib/transcoding/web-transcoding.ts
View File

@ -1,22 +1,22 @@
import { Job } from 'bullmq'
import { move, remove } from 'fs-extra/esm'
import { copyFile } from 'fs/promises'
import { basename, join } from 'path'
import { buildAspectRatio } from '@peertube/peertube-core-utils'
import { TranscodeVODOptionsType, getVideoStreamDuration } from '@peertube/peertube-ffmpeg'
import { computeOutputFPS } from '@server/helpers/ffmpeg/index.js'
import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent.js'
import { VideoModel } from '@server/models/video/video.js'
import { MVideoFile, MVideoFullLight } from '@server/types/models/index.js'
import { getVideoStreamDuration, TranscodeVODOptionsType } from '@peertube/peertube-ffmpeg'
import { Job } from 'bullmq'
import { move, remove } from 'fs-extra/esm'
import { copyFile } from 'fs/promises'
import { basename, join } from 'path'
import { CONFIG } from '../../initializers/config.js'
import { VideoFileModel } from '../../models/video/video-file.js'
import { JobQueue } from '../job-queue/index.js'
import { generateWebVideoFilename } from '../paths.js'
import { buildNewFile } from '../video-file.js'
import { buildNewFile, saveNewOriginalFileIfNeeded } from '../video-file.js'
import { buildStoryboardJobIfNeeded } from '../video-jobs.js'
import { VideoPathManager } from '../video-path-manager.js'
import { buildFFmpegVOD } from './shared/index.js'
import { buildOriginalFileResolution } from './transcoding-resolutions.js'
import { buildStoryboardJobIfNeeded } from '../video-jobs.js'
import { buildAspectRatio } from '@peertube/peertube-core-utils'
// Optimize the original video file and replace it. The resolution is not changed.
export async function optimizeOriginalVideofile (options: {
@ -73,7 +73,7 @@ export async function optimizeOriginalVideofile (options: {
}
}
// Transcode the original video file to a lower resolution compatible with web browsers
// Transcode the original/old/source video file to a lower resolution compatible with web browsers
export async function transcodeNewWebVideoResolution (options: {
video: MVideoFullLight
resolution: number
@ -162,7 +162,6 @@ export async function mergeAudioVideofile (options: {
try {
await buildFFmpegVOD(job).transcode(transcodeOptions)
await remove(audioInputPath)
await remove(tmpPreviewPath)
} catch (err) {
await remove(tmpPreviewPath)
@ -213,14 +212,16 @@ export async function onWebVideoFileTranscoding (options: {
await createTorrentAndSetInfoHash(video, videoFile)
const oldFile = await VideoFileModel.loadWebVideoFile({ videoId: video.id, fps: videoFile.fps, resolution: videoFile.resolution })
if (oldFile) await video.removeWebVideoFile(oldFile)
if (deleteWebInputVideoFile) {
await saveNewOriginalFileIfNeeded(video, deleteWebInputVideoFile)
await video.removeWebVideoFile(deleteWebInputVideoFile)
await deleteWebInputVideoFile.destroy()
}
const existingFile = await VideoFileModel.loadWebVideoFile({ videoId: video.id, fps: videoFile.fps, resolution: videoFile.resolution })
if (existingFile) await video.removeWebVideoFile(existingFile)
await VideoFileModel.customUpsert(videoFile, 'video', undefined)
video.VideoFiles = await video.$get('VideoFiles')

92
server/core/lib/user-import-export/exporters/videos-exporter.ts
View File

@ -1,8 +1,19 @@
import { VideoModel } from '@server/models/video/video.js'
import { pick } from '@peertube/peertube-core-utils'
import { ActivityCreate, FileStorage, VideoExportJSON, VideoObject, VideoPrivacy } from '@peertube/peertube-models'
import { logger } from '@server/helpers/logger.js'
import { USER_EXPORT_MAX_ITEMS } from '@server/initializers/constants.js'
import { audiencify, getAudience } from '@server/lib/activitypub/audience.js'
import { buildCreateActivity } from '@server/lib/activitypub/send/send-create.js'
import { buildChaptersAPHasPart } from '@server/lib/activitypub/video-chapters.js'
import { getHLSFileReadStream, getOriginalFileReadStream, getWebVideoFileReadStream } from '@server/lib/object-storage/videos.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { VideoCaptionModel } from '@server/models/video/video-caption.js'
import { VideoChannelModel } from '@server/models/video/video-channel.js'
import { VideoChapterModel } from '@server/models/video/video-chapter.js'
import { VideoLiveModel } from '@server/models/video/video-live.js'
import { ExportResult, AbstractUserExporter } from './abstract-user-exporter.js'
import { VideoPasswordModel } from '@server/models/video/video-password.js'
import { VideoSourceModel } from '@server/models/video/video-source.js'
import { VideoModel } from '@server/models/video/video.js'
import {
MStreamingPlaylistFiles,
MThumbnail, MVideo, MVideoAP, MVideoCaption,
@ -12,23 +23,12 @@ import {
MVideoFullLight, MVideoLiveWithSetting,
MVideoPassword
} from '@server/types/models/index.js'
import { logger } from '@server/helpers/logger.js'
import { ActivityCreate, VideoExportJSON, VideoObject, VideoPrivacy, FileStorage } from '@peertube/peertube-models'
import { MVideoSource } from '@server/types/models/video/video-source.js'
import Bluebird from 'bluebird'
import { getHLSFileReadStream, getWebVideoFileReadStream } from '@server/lib/object-storage/videos.js'
import { createReadStream } from 'fs'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { extname, join } from 'path'
import { Readable } from 'stream'
import { getAudience, audiencify } from '@server/lib/activitypub/audience.js'
import { buildCreateActivity } from '@server/lib/activitypub/send/send-create.js'
import { pick } from '@peertube/peertube-core-utils'
import { VideoPasswordModel } from '@server/models/video/video-password.js'
import { MVideoSource } from '@server/types/models/video/video-source.js'
import { VideoSourceModel } from '@server/models/video/video-source.js'
import { VideoChapterModel } from '@server/models/video/video-chapter.js'
import { buildChaptersAPHasPart } from '@server/lib/activitypub/video-chapters.js'
import { USER_EXPORT_MAX_ITEMS } from '@server/initializers/constants.js'
import { AbstractUserExporter, ExportResult } from './abstract-user-exporter.js'
export class VideosExporter extends AbstractUserExporter <VideoExportJSON> {
@ -89,7 +89,7 @@ export class VideosExporter extends AbstractUserExporter <VideoExportJSON> {
// Then fetch more attributes for AP serialization
const videoAP = await video.lightAPToFullAP(undefined)
const { relativePathsFromJSON, staticFiles } = this.exportVideoFiles({ video, captions })
const { relativePathsFromJSON, staticFiles } = await this.exportVideoFiles({ video, captions })
return {
json: this.exportVideoJSON({ video, captions, live, passwords, source, chapters, archiveFiles: relativePathsFromJSON }),
@ -168,9 +168,7 @@ export class VideosExporter extends AbstractUserExporter <VideoExportJSON> {
streamingPlaylists: this.exportStreamingPlaylistsJSON(video, video.VideoStreamingPlaylists),
source: source
? { filename: source.filename }
: null,
source: this.exportVideoSourceJSON(source),
archiveFiles
}
@ -228,6 +226,24 @@ export class VideosExporter extends AbstractUserExporter <VideoExportJSON> {
}))
}
private exportVideoSourceJSON (source: MVideoSource) {
if (!source) return null
return {
inputFilename: source.inputFilename,
resolution: source.resolution,
size: source.size,
width: source.width,
height: source.height,
fps: source.fps,
metadata: source.metadata
}
}
// ---------------------------------------------------------------------------
private async exportVideoAP (video: MVideoAP, chapters: MVideoChapter[]): Promise<ActivityCreate<VideoObject>> {
@ -271,7 +287,7 @@ export class VideosExporter extends AbstractUserExporter <VideoExportJSON> {
// ---------------------------------------------------------------------------
private exportVideoFiles (options: {
private async exportVideoFiles (options: {
video: MVideoFullLight
captions: MVideoCaption[]
}) {
@ -284,15 +300,27 @@ export class VideosExporter extends AbstractUserExporter <VideoExportJSON> {
captions: {} as { [ lang: string ]: string }
}
const videoFile = video.getMaxQualityFile()
if (this.options.withVideoFiles) {
const source = await VideoSourceModel.loadLatest(video.id)
const maxQualityFile = video.getMaxQualityFile()
// Prefer using original file if possible
const file = source?.keptOriginalFilename
? source
: maxQualityFile
if (file) {
const videoPath = this.getArchiveVideoFilePath(video, file)
if (this.options.withVideoFiles && videoFile) {
staticFiles.push({
archivePath: this.getArchiveVideoFilePath(video, videoFile),
createrReadStream: () => this.generateVideoFileReadStream(video, videoFile)
archivePath: videoPath,
createrReadStream: () => file === source
? this.generateVideoSourceReadStream(source)
: this.generateVideoFileReadStream(video, maxQualityFile)
})
relativePathsFromJSON.videoFile = join(this.relativeStaticDirPath, this.getArchiveVideoFilePath(video, videoFile))
relativePathsFromJSON.videoFile = join(this.relativeStaticDirPath, videoPath)
}
}
for (const caption of captions) {
@ -317,6 +345,16 @@ export class VideosExporter extends AbstractUserExporter <VideoExportJSON> {
return { staticFiles, relativePathsFromJSON }
}
private async generateVideoSourceReadStream (source: MVideoSource): Promise<Readable> {
if (source.storage === FileStorage.FILE_SYSTEM) {
return createReadStream(VideoPathManager.Instance.getFSOriginalVideoFilePath(source.keptOriginalFilename))
}
const { stream } = await getOriginalFileReadStream({ keptOriginalFilename: source.keptOriginalFilename, rangeHeader: undefined })
return stream
}
private async generateVideoFileReadStream (video: MVideoFullLight, videoFile: MVideoFile): Promise<Readable> {
if (videoFile.storage === FileStorage.FILE_SYSTEM) {
return createReadStream(VideoPathManager.Instance.getFSVideoFileOutputPath(video, videoFile))
@ -329,8 +367,8 @@ export class VideosExporter extends AbstractUserExporter <VideoExportJSON> {
return stream
}
private getArchiveVideoFilePath (video: MVideo, videoFile: MVideoFile) {
return join('video-files', video.uuid + extname(videoFile.filename))
private getArchiveVideoFilePath (video: MVideo, file: { filename?: string, keptOriginalFilename?: string }) {
return join('video-files', video.uuid + extname(file.filename || file.keptOriginalFilename))
}
private getArchiveCaptionFilePath (video: MVideo, caption: MVideoCaptionLanguageUrl) {

52
server/core/lib/user-import-export/importers/videos-importer.ts
View File

@ -1,15 +1,12 @@
import { LiveVideoLatencyMode, ThumbnailType, VideoExportJSON, VideoPrivacy, VideoState } from '@peertube/peertube-models'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { Hooks } from '@server/lib/plugins/hooks.js'
import { buildNextVideoState } from '@server/lib/video-state.js'
import { VideoModel } from '@server/models/video/video.js'
import { pick } from '@peertube/peertube-core-utils'
import { buildUUID, getFileSize } from '@peertube/peertube-node-utils'
import { MChannelId, MVideoFullLight } from '@server/types/models/index.js'
import { ffprobePromise, getVideoStreamDuration } from '@peertube/peertube-ffmpeg'
import { VideoChannelModel } from '@server/models/video/video-channel.js'
import { AbstractUserImporter } from './abstract-user-importer.js'
import { isUserQuotaValid } from '@server/lib/user.js'
import { LiveVideoLatencyMode, ThumbnailType, VideoExportJSON, VideoPrivacy, VideoState } from '@peertube/peertube-models'
import { buildUUID, getFileSize } from '@peertube/peertube-node-utils'
import { isArray, isBooleanValid, isUUIDValid } from '@server/helpers/custom-validators/misc.js'
import { isVideoCaptionLanguageValid } from '@server/helpers/custom-validators/video-captions.js'
import { isVideoChannelUsernameValid } from '@server/helpers/custom-validators/video-channels.js'
import { isVideoChapterTimecodeValid, isVideoChapterTitleValid } from '@server/helpers/custom-validators/video-chapters.js'
import { isLiveLatencyModeValid } from '@server/helpers/custom-validators/video-lives.js'
import {
isPasswordValid,
isVideoCategoryValid,
@ -25,17 +22,21 @@ import {
isVideoSupportValid,
isVideoTagValid
} from '@server/helpers/custom-validators/videos.js'
import { isVideoChannelUsernameValid } from '@server/helpers/custom-validators/video-channels.js'
import { CONSTRAINTS_FIELDS } from '@server/initializers/constants.js'
import { isArray, isBooleanValid, isUUIDValid } from '@server/helpers/custom-validators/misc.js'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { CONFIG } from '@server/initializers/config.js'
import { isVideoCaptionLanguageValid } from '@server/helpers/custom-validators/video-captions.js'
import { isLiveLatencyModeValid } from '@server/helpers/custom-validators/video-lives.js'
import { parse } from 'path'
import { isLocalVideoFileAccepted } from '@server/lib/moderation.js'
import { CONSTRAINTS_FIELDS } from '@server/initializers/constants.js'
import { LocalVideoCreator, ThumbnailOptions } from '@server/lib/local-video-creator.js'
import { isVideoChapterTimecodeValid, isVideoChapterTitleValid } from '@server/helpers/custom-validators/video-chapters.js'
import { isLocalVideoFileAccepted } from '@server/lib/moderation.js'
import { Hooks } from '@server/lib/plugins/hooks.js'
import { isUserQuotaValid } from '@server/lib/user.js'
import { createLocalCaption } from '@server/lib/video-captions.js'
import { buildNextVideoState } from '@server/lib/video-state.js'
import { VideoChannelModel } from '@server/models/video/video-channel.js'
import { VideoModel } from '@server/models/video/video.js'
import { MChannelId, MVideoFullLight } from '@server/types/models/index.js'
import { FfprobeData } from 'fluent-ffmpeg'
import { parse } from 'path'
import { AbstractUserImporter } from './abstract-user-importer.js'
const lTags = loggerTagsFactory('user-import')
@ -69,7 +70,7 @@ export class VideosImporter extends AbstractUserImporter <VideoExportJSON, Impor
if (!isBooleanValid(o.downloadEnabled)) o.downloadEnabled = CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED
if (!isBooleanValid(o.waitTranscoding)) o.waitTranscoding = true
if (!isVideoSourceFilenameValid(o.source?.filename)) o.source = undefined
if (!isVideoSourceFilenameValid(o.source?.inputFilename)) o.source = undefined
if (!isVideoOriginallyPublishedAtValid(o.originallyPublishedAt)) o.originallyPublishedAt = null
@ -149,6 +150,7 @@ export class VideosImporter extends AbstractUserImporter <VideoExportJSON, Impor
let duration = 0
let ffprobe: FfprobeData
if (videoFilePath) {
if (await isUserQuotaValid({ userId: this.user.id, uploadSize: videoSize, checkDaily: false }) === false) {
throw new Error(`Cannot import video ${videoImportData.name} for user ${this.user.username} because of exceeded quota`)
@ -156,7 +158,7 @@ export class VideosImporter extends AbstractUserImporter <VideoExportJSON, Impor
await this.checkVideoFileIsAcceptedOrThrow({ videoFilePath, size: videoSize, channel: videoChannel, videoImportData })
const ffprobe = await ffprobePromise(videoFilePath)
ffprobe = await ffprobePromise(videoFilePath)
duration = await getVideoStreamDuration(videoFilePath, ffprobe)
}
@ -176,7 +178,11 @@ export class VideosImporter extends AbstractUserImporter <VideoExportJSON, Impor
const localVideoCreator = new LocalVideoCreator({
lTags,
videoFilePath,
videoFile: videoFilePath
? { path: videoFilePath, probe: ffprobe }
: undefined,
user: this.user,
channel: videoChannel,
@ -206,7 +212,9 @@ export class VideosImporter extends AbstractUserImporter <VideoExportJSON, Impor
videoPasswords: videoImportData.passwords,
duration,
filename: videoImportData.source?.filename,
inputFilename: videoImportData.source?.inputFilename,
state: videoImportData.isLive
? VideoState.WAITING_FOR_LIVE
: buildNextVideoState()

14
server/core/lib/user-import-export/user-exporter.ts
View File

@ -48,7 +48,7 @@ export class UserExporter {
if (exportModel.storage === FileStorage.FILE_SYSTEM) {
output = createWriteStream(getFSUserExportFilePath(exportModel))
endPromise = new Promise<void>(res => output.on('close', () => res()))
endPromise = new Promise<string>(res => output.on('close', () => res('')))
} else {
output = new PassThrough()
endPromise = storeUserExportFile(output as PassThrough, exportModel)
@ -56,12 +56,16 @@ export class UserExporter {
await this.createZip({ exportModel, user, output })
await endPromise
const fileUrl = await endPromise
if (exportModel.storage === FileStorage.OBJECT_STORAGE) {
exportModel.fileUrl = fileUrl
exportModel.size = await getUserExportFileObjectStorageSize(exportModel)
} else if (exportModel.storage === FileStorage.FILE_SYSTEM) {
exportModel.size = await getFileSize(getFSUserExportFilePath(exportModel))
}
exportModel.state = UserExportState.COMPLETED
exportModel.size = exportModel.storage === FileStorage.FILE_SYSTEM
? await getFileSize(getFSUserExportFilePath(exportModel))
: await getUserExportFileObjectStorageSize(exportModel)
await saveInTransactionWithRetries(exportModel)
} catch (err) {

118
server/core/lib/video-file.ts
View File

@ -1,16 +1,20 @@
import { FfprobeData } from 'fluent-ffmpeg'
import { VideoFileMetadata, VideoResolution } from '@peertube/peertube-models'
import { logger } from '@server/helpers/logger.js'
import { VideoFileModel } from '@server/models/video/video-file.js'
import { MVideoWithAllFiles } from '@server/types/models/index.js'
import { getFileSize, getLowercaseExtension } from '@peertube/peertube-node-utils'
import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS, isAudioFile } from '@peertube/peertube-ffmpeg'
import { FileStorage, VideoFileMetadata, VideoResolution } from '@peertube/peertube-models'
import { getFileSize, getLowercaseExtension } from '@peertube/peertube-node-utils'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { CONFIG } from '@server/initializers/config.js'
import { MIMETYPES } from '@server/initializers/constants.js'
import { VideoFileModel } from '@server/models/video/video-file.js'
import { VideoSourceModel } from '@server/models/video/video-source.js'
import { MVideo, MVideoFile, MVideoId, MVideoWithAllFiles } from '@server/types/models/index.js'
import { FfprobeData } from 'fluent-ffmpeg'
import { move, remove } from 'fs-extra'
import { lTags } from './object-storage/shared/index.js'
import { storeOriginalVideoFile } from './object-storage/videos.js'
import { generateHLSVideoFilename, generateWebVideoFilename } from './paths.js'
import { VideoPathManager } from './video-path-manager.js'
import { MIMETYPES } from '@server/initializers/constants.js'
async function buildNewFile (options: {
export async function buildNewFile (options: {
path: string
mode: 'web-video' | 'hls'
ffprobe?: FfprobeData
@ -48,7 +52,7 @@ async function buildNewFile (options: {
// ---------------------------------------------------------------------------
async function removeHLSPlaylist (video: MVideoWithAllFiles) {
export async function removeHLSPlaylist (video: MVideoWithAllFiles) {
const hls = video.getHLSPlaylist()
if (!hls) return
@ -64,7 +68,7 @@ async function removeHLSPlaylist (video: MVideoWithAllFiles) {
}
}
async function removeHLSFile (video: MVideoWithAllFiles, fileToDeleteId: number) {
export async function removeHLSFile (video: MVideoWithAllFiles, fileToDeleteId: number) {
logger.info('Deleting HLS file %d of %s.', fileToDeleteId, video.url, lTags(video.uuid))
const hls = video.getHLSPlaylist()
@ -92,7 +96,7 @@ async function removeHLSFile (video: MVideoWithAllFiles, fileToDeleteId: number)
// ---------------------------------------------------------------------------
async function removeAllWebVideoFiles (video: MVideoWithAllFiles) {
export async function removeAllWebVideoFiles (video: MVideoWithAllFiles) {
const videoFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
try {
@ -109,7 +113,7 @@ async function removeAllWebVideoFiles (video: MVideoWithAllFiles) {
return video
}
async function removeWebVideoFile (video: MVideoWithAllFiles, fileToDeleteId: number) {
export async function removeWebVideoFile (video: MVideoWithAllFiles, fileToDeleteId: number) {
const files = video.VideoFiles
if (files.length === 1) {
@ -132,13 +136,13 @@ async function removeWebVideoFile (video: MVideoWithAllFiles, fileToDeleteId: nu
// ---------------------------------------------------------------------------
async function buildFileMetadata (path: string, existingProbe?: FfprobeData) {
export async function buildFileMetadata (path: string, existingProbe?: FfprobeData) {
const metadata = existingProbe || await ffprobePromise(path)
return new VideoFileMetadata(metadata)
}
function getVideoFileMimeType (extname: string, isAudio: boolean) {
export function getVideoFileMimeType (extname: string, isAudio: boolean) {
return isAudio && extname === '.mp4' // We use .mp4 even for audio file only
? MIMETYPES.AUDIO.EXT_MIMETYPE['.m4a']
: MIMETYPES.VIDEO.EXT_MIMETYPE[extname]
@ -146,14 +150,84 @@ function getVideoFileMimeType (extname: string, isAudio: boolean) {
// ---------------------------------------------------------------------------
export {
buildNewFile,
export async function createVideoSource (options: {
inputFilename: string
inputProbe: FfprobeData
inputPath: string
video: MVideoId
createdAt?: Date
}) {
const { inputFilename, inputPath, inputProbe, video, createdAt } = options
removeHLSPlaylist,
removeHLSFile,
removeAllWebVideoFiles,
removeWebVideoFile,
const videoSource = new VideoSourceModel({
inputFilename,
videoId: video.id,
createdAt
})
buildFileMetadata,
getVideoFileMimeType
if (inputPath) {
const probe = inputProbe ?? await ffprobePromise(inputPath)
if (await isAudioFile(inputPath, probe)) {
videoSource.fps = 0
videoSource.resolution = VideoResolution.H_NOVIDEO
videoSource.width = 0
videoSource.height = 0
} else {
const dimensions = await getVideoStreamDimensionsInfo(inputPath, probe)
videoSource.fps = await getVideoStreamFPS(inputPath, probe)
videoSource.resolution = dimensions.resolution
videoSource.width = dimensions.width
videoSource.height = dimensions.height
}
videoSource.metadata = await buildFileMetadata(inputPath, probe)
videoSource.size = await getFileSize(inputPath)
}
return videoSource.save()
}
export async function saveNewOriginalFileIfNeeded (video: MVideo, videoFile: MVideoFile) {
if (!CONFIG.TRANSCODING.ORIGINAL_FILE.KEEP) return
const videoSource = await VideoSourceModel.loadLatest(video.id)
// Already have saved an original file
if (!videoSource || videoSource.keptOriginalFilename) return
videoSource.keptOriginalFilename = videoFile.filename
const lTags = loggerTagsFactory(video.uuid)
logger.info(`Storing original video file ${videoSource.keptOriginalFilename} of video ${video.name}`, lTags())
const sourcePath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, videoFile)
if (CONFIG.OBJECT_STORAGE.ENABLED) {
const fileUrl = await storeOriginalVideoFile(sourcePath, videoSource.keptOriginalFilename)
await remove(sourcePath)
videoSource.storage = FileStorage.OBJECT_STORAGE
videoSource.fileUrl = fileUrl
} else {
const destinationPath = VideoPathManager.Instance.getFSOriginalVideoFilePath(videoSource.keptOriginalFilename)
await move(sourcePath, destinationPath)
videoSource.storage = FileStorage.FILE_SYSTEM
}
await videoSource.save()
// Delete previously kept video files
const allSources = await VideoSourceModel.listAll(video.id)
for (const oldSource of allSources) {
if (!oldSource.keptOriginalFilename) continue
if (oldSource.id === videoSource.id) continue
try {
await video.removeOriginalFile(oldSource)
} catch (err) {
logger.error('Cannot delete old original file ' + oldSource.keptOriginalFilename, { err, ...lTags() })
}
}
}

16
server/core/lib/video-path-manager.ts
View File

@ -1,7 +1,5 @@
import { Mutex } from 'async-mutex'
import { remove } from 'fs-extra/esm'
import { extname, join } from 'path'
import { FileStorage } from '@peertube/peertube-models'
import { buildUUID } from '@peertube/peertube-node-utils'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { extractVideo } from '@server/helpers/video.js'
import { CONFIG } from '@server/initializers/config.js'
@ -13,7 +11,9 @@ import {
MVideoFileStreamingPlaylistVideo,
MVideoFileVideo
} from '@server/types/models/index.js'
import { buildUUID } from '@peertube/peertube-node-utils'
import { Mutex } from 'async-mutex'
import { remove } from 'fs-extra/esm'
import { extname, join } from 'path'
import { makeHLSFileAvailable, makeWebVideoFileAvailable } from './object-storage/index.js'
import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from './paths.js'
import { isVideoInPrivateDirectory } from './video-privacy.js'
@ -56,10 +56,14 @@ class VideoPathManager {
}
if (isVideoInPrivateDirectory(video.privacy)) {
return join(DIRECTORIES.VIDEOS.PRIVATE, videoFile.filename)
return join(DIRECTORIES.WEB_VIDEOS.PRIVATE, videoFile.filename)
}
return join(DIRECTORIES.VIDEOS.PUBLIC, videoFile.filename)
return join(DIRECTORIES.WEB_VIDEOS.PUBLIC, videoFile.filename)
}
getFSOriginalVideoFilePath (filename: string) {
return join(DIRECTORIES.ORIGINAL_VIDEOS, filename)
}
async makeAvailableVideoFile <T> (videoFile: MVideoFileVideo | MVideoFileStreamingPlaylistVideo, cb: MakeAvailableCB<T>) {

4
server/core/lib/video-privacy.ts
View File

@ -101,10 +101,10 @@ async function moveWebVideoFileOnFS (type: MoveType, video: MVideo, file: MVideo
function getWebVideoDirectories (moveType: MoveType) {
if (moveType === 'private-to-public') {
return { old: DIRECTORIES.VIDEOS.PRIVATE, new: DIRECTORIES.VIDEOS.PUBLIC }
return { old: DIRECTORIES.WEB_VIDEOS.PRIVATE, new: DIRECTORIES.WEB_VIDEOS.PUBLIC }
}
return { old: DIRECTORIES.VIDEOS.PUBLIC, new: DIRECTORIES.VIDEOS.PRIVATE }
return { old: DIRECTORIES.WEB_VIDEOS.PUBLIC, new: DIRECTORIES.WEB_VIDEOS.PRIVATE }
}
// ---------------------------------------------------------------------------

1
server/core/middlewares/validators/config.ts
View File

@ -41,6 +41,7 @@ const customConfigUpdateValidator = [
body('videoChannels.maxPerUser').isInt(),
body('transcoding.enabled').isBoolean(),
body('transcoding.originalFile.keep').isBoolean(),
body('transcoding.allowAdditionalExtensions').isBoolean(),
body('transcoding.threads').isInt(),
body('transcoding.concurrency').isInt({ min: 1 }),

82
server/core/middlewares/validators/shared/videos.ts
View File

@ -1,7 +1,6 @@
import { Request, Response } from 'express'
import { HttpStatusCode, ServerErrorCode, UserRight, UserRightType, VideoPrivacy } from '@peertube/peertube-models'
import { exists } from '@server/helpers/custom-validators/misc.js'
import { loadVideo, VideoLoadType } from '@server/lib/model-loaders/index.js'
import { VideoLoadType, loadVideo } from '@server/lib/model-loaders/index.js'
import { isUserQuotaValid } from '@server/lib/user.js'
import { VideoTokensManager } from '@server/lib/video-tokens-manager.js'
import { authenticatePromise } from '@server/middlewares/auth.js'
@ -20,10 +19,12 @@ import {
MVideoId,
MVideoImmutable,
MVideoThumbnail,
MVideoUUID,
MVideoWithRights
} from '@server/types/models/index.js'
import { Request, Response } from 'express'
async function doesVideoExist (id: number | string, res: Response, fetchType: VideoLoadType = 'all') {
export async function doesVideoExist (id: number | string, res: Response, fetchType: VideoLoadType = 'all') {
const userId = res.locals.oauth ? res.locals.oauth.token.User.id : undefined
const video = await loadVideo(id, fetchType, userId)
@ -64,7 +65,7 @@ async function doesVideoExist (id: number | string, res: Response, fetchType: Vi
// ---------------------------------------------------------------------------
async function doesVideoFileOfVideoExist (id: number, videoIdOrUUID: number | string, res: Response) {
export async function doesVideoFileOfVideoExist (id: number, videoIdOrUUID: number | string, res: Response) {
if (!await VideoFileModel.doesVideoExistForVideoFile(id, videoIdOrUUID)) {
res.fail({
status: HttpStatusCode.NOT_FOUND_404,
@ -78,7 +79,7 @@ async function doesVideoFileOfVideoExist (id: number, videoIdOrUUID: number | st
// ---------------------------------------------------------------------------
async function doesVideoChannelOfAccountExist (channelId: number, user: MUserAccountId, res: Response) {
export async function doesVideoChannelOfAccountExist (channelId: number, user: MUserAccountId, res: Response) {
const videoChannel = await VideoChannelModel.loadAndPopulateAccount(channelId)
if (videoChannel === null) {
@ -105,7 +106,7 @@ async function doesVideoChannelOfAccountExist (channelId: number, user: MUserAcc
// ---------------------------------------------------------------------------
async function checkCanSeeVideo (options: {
export async function checkCanSeeVideo (options: {
req: Request
res: Response
paramId: string
@ -128,7 +129,7 @@ async function checkCanSeeVideo (options: {
throw new Error('Unknown video privacy when checking video right ' + video.url)
}
async function checkCanSeeUserAuthVideo (options: {
export async function checkCanSeeUserAuthVideo (options: {
req: Request
res: Response
video: MVideoId | MVideoWithRights
@ -174,7 +175,7 @@ async function checkCanSeeUserAuthVideo (options: {
return fail()
}
async function checkCanSeePasswordProtectedVideo (options: {
export async function checkCanSeePasswordProtectedVideo (options: {
req: Request
res: Response
video: MVideo
@ -215,13 +216,13 @@ async function checkCanSeePasswordProtectedVideo (options: {
return false
}
function canUserAccessVideo (user: MUser, video: MVideoWithRights | MVideoAccountLight, right: UserRightType) {
export function canUserAccessVideo (user: MUser, video: MVideoWithRights | MVideoAccountLight, right: UserRightType) {
const isOwnedByUser = video.VideoChannel.Account.userId === user.id
return isOwnedByUser || user.hasRight(right)
}
async function getVideoWithRights (video: MVideoWithRights): Promise<MVideoWithRights> {
export async function getVideoWithRights (video: MVideoWithRights): Promise<MVideoWithRights> {
return video.VideoChannel?.Account?.userId
? video
: VideoModel.loadFull(video.id)
@ -229,7 +230,7 @@ async function getVideoWithRights (video: MVideoWithRights): Promise<MVideoWithR
// ---------------------------------------------------------------------------
async function checkCanAccessVideoStaticFiles (options: {
export async function checkCanAccessVideoStaticFiles (options: {
video: MVideo
req: Request
res: Response
@ -241,23 +242,51 @@ async function checkCanAccessVideoStaticFiles (options: {
return checkCanSeeVideo(options)
}
const videoFileToken = req.query.videoFileToken
if (videoFileToken && VideoTokensManager.Instance.hasToken({ token: videoFileToken, videoUUID: video.uuid })) {
const user = VideoTokensManager.Instance.getUserFromToken({ token: videoFileToken })
res.locals.videoFileToken = { user }
return true
}
assignVideoTokenIfNeeded(req, res, video)
if (res.locals.videoFileToken) return true
if (!video.hasPrivateStaticPath()) return true
res.sendStatus(HttpStatusCode.FORBIDDEN_403)
return false
}
export async function checkCanAccessVideoSourceFile (options: {
videoId: number
req: Request
res: Response
}) {
const { req, res, videoId } = options
const video = await VideoModel.loadFull(videoId)
if (res.locals.oauth?.token.User) {
if (canUserAccessVideo(res.locals.oauth.token.User, video, UserRight.SEE_ALL_VIDEOS) === true) return true
res.sendStatus(HttpStatusCode.FORBIDDEN_403)
return false
}
assignVideoTokenIfNeeded(req, res, video)
if (res.locals.videoFileToken) return true
res.sendStatus(HttpStatusCode.FORBIDDEN_403)
return false
}
function assignVideoTokenIfNeeded (req: Request, res: Response, video: MVideoUUID) {
const videoFileToken = req.query.videoFileToken
if (videoFileToken && VideoTokensManager.Instance.hasToken({ token: videoFileToken, videoUUID: video.uuid })) {
const user = VideoTokensManager.Instance.getUserFromToken({ token: videoFileToken })
res.locals.videoFileToken = { user }
}
}
// ---------------------------------------------------------------------------
function checkUserCanManageVideo (user: MUser, video: MVideoAccountLight, right: UserRightType, res: Response, onlyOwned = true) {
export function checkUserCanManageVideo (user: MUser, video: MVideoAccountLight, right: UserRightType, res: Response, onlyOwned = true) {
// Retrieve the user who did the request
if (onlyOwned && video.isOwned() === false) {
res.fail({
@ -284,7 +313,7 @@ function checkUserCanManageVideo (user: MUser, video: MVideoAccountLight, right:
// ---------------------------------------------------------------------------
async function checkUserQuota (user: MUserId, videoFileSize: number, res: Response) {
export async function checkUserQuota (user: MUserId, videoFileSize: number, res: Response) {
if (await isUserQuotaValid({ userId: user.id, uploadSize: videoFileSize }) === false) {
res.fail({
status: HttpStatusCode.PAYLOAD_TOO_LARGE_413,
@ -296,16 +325,3 @@ async function checkUserQuota (user: MUserId, videoFileSize: number, res: Respon
return true
}
// ---------------------------------------------------------------------------
export {
doesVideoChannelOfAccountExist,
doesVideoExist,
doesVideoFileOfVideoExist,
checkCanAccessVideoStaticFiles,
checkUserCanManageVideo,
checkCanSeeVideo,
checkUserQuota
}

11
server/core/middlewares/validators/videos/shared/upload.ts
View File

@ -1,6 +1,6 @@
import express from 'express'
import { logger } from '@server/helpers/logger.js'
import { getVideoStreamDuration } from '@peertube/peertube-ffmpeg'
import { ffprobePromise, getVideoStreamDuration } from '@peertube/peertube-ffmpeg'
import { HttpStatusCode } from '@peertube/peertube-models'
export async function addDurationToVideoFileIfNeeded (options: {
@ -11,7 +11,7 @@ export async function addDurationToVideoFileIfNeeded (options: {
const { res, middlewareName, videoFile } = options
try {
if (!videoFile.duration) await addDurationToVideo(videoFile)
if (!videoFile.duration) await addDurationToVideo(res, videoFile)
} catch (err) {
logger.error('Invalid input file in ' + middlewareName, { err })
@ -29,8 +29,11 @@ export async function addDurationToVideoFileIfNeeded (options: {
// Private
// ---------------------------------------------------------------------------
async function addDurationToVideo (videoFile: { path: string, duration?: number }) {
const duration = await getVideoStreamDuration(videoFile.path)
async function addDurationToVideo (res: express.Response, videoFile: { path: string, duration?: number }) {
const probe = await ffprobePromise(videoFile.path)
res.locals.ffprobe = probe
const duration = await getVideoStreamDuration(videoFile.path, probe)
// FFmpeg may not be able to guess video duration
// For example with m2v files: https://trac.ffmpeg.org/ticket/9726#comment:2

35
server/core/middlewares/validators/videos/video-source.ts
View File

@ -1,12 +1,19 @@
import express from 'express'
import { HttpStatusCode, UserRight } from '@peertube/peertube-models'
import { getVideoWithAttributes } from '@server/helpers/video.js'
import { CONFIG } from '@server/initializers/config.js'
import { buildUploadXFile, safeUploadXCleanup } from '@server/lib/uploadx.js'
import { VideoSourceModel } from '@server/models/video/video-source.js'
import { MVideoFullLight } from '@server/types/models/index.js'
import { HttpStatusCode, UserRight } from '@peertube/peertube-models'
import { Metadata as UploadXMetadata } from '@uploadx/core'
import { areValidationErrors, checkUserCanManageVideo, doesVideoExist, isValidVideoIdParam } from '../shared/index.js'
import express from 'express'
import { param } from 'express-validator'
import {
areValidationErrors,
checkCanAccessVideoSourceFile,
checkUserCanManageVideo,
doesVideoExist,
isValidVideoIdParam
} from '../shared/index.js'
import { addDurationToVideoFileIfNeeded, checkVideoFileCanBeEdited, commonVideoFileChecks, isVideoFileAccepted } from './shared/index.js'
export const videoSourceGetLatestValidator = [
@ -71,6 +78,28 @@ export const replaceVideoSourceResumableInitValidator = [
}
]
export const originalVideoFileDownloadValidator = [
param('filename').exists(),
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
if (areValidationErrors(req, res)) return
const videoSource = await VideoSourceModel.loadByKeptOriginalFilename(req.params.filename)
if (!videoSource) {
return res.fail({
status: HttpStatusCode.NOT_FOUND_404,
message: 'Original video file not found'
})
}
if (!await checkCanAccessVideoSourceFile({ req, res, videoId: videoSource.videoId })) return
res.locals.videoSource = videoSource
return next()
}
]
// ---------------------------------------------------------------------------
// Private
// ---------------------------------------------------------------------------

33
server/core/middlewares/validators/videos/videos.ts
View File

@ -1,5 +1,3 @@
import express from 'express'
import { body, param, query, ValidationChain } from 'express-validator'
import { arrayify } from '@peertube/peertube-core-utils'
import { HttpStatusCode, ServerErrorCode, UserRight, VideoState } from '@peertube/peertube-models'
import { Redis } from '@server/lib/redis.js'
@ -7,6 +5,8 @@ import { buildUploadXFile, safeUploadXCleanup } from '@server/lib/uploadx.js'
import { getServerActor } from '@server/models/application/application.js'
import { ExpressPromiseHandler } from '@server/types/express-handler.js'
import { MUserAccountId, MVideoFullLight } from '@server/types/models/index.js'
import express from 'express'
import { ValidationChain, body, param, query } from 'express-validator'
import {
exists,
isBooleanValid,
@ -41,8 +41,7 @@ import { CONFIG } from '../../../initializers/config.js'
import { CONSTRAINTS_FIELDS, OVERVIEWS } from '../../../initializers/constants.js'
import { VideoModel } from '../../../models/video/video.js'
import {
areValidationErrors,
checkCanAccessVideoStaticFiles,
areValidationErrors, checkCanAccessVideoStaticFiles,
checkCanSeeVideo,
checkUserCanManageVideo,
doesVideoChannelOfAccountExist,
@ -501,23 +500,19 @@ const commonVideosFiltersValidator = [
// ---------------------------------------------------------------------------
export {
videosAddLegacyValidator,
videosAddResumableValidator,
videosAddResumableInitValidator,
videosUpdateValidator,
videosGetValidator,
videoFileMetadataGetValidator,
videosDownloadValidator,
checkVideoFollowConstraints,
videosCustomGetValidator,
videosRemoveValidator,
getCommonVideoEditAttributes,
commonVideosFiltersValidator,
videosOverviewValidator
getCommonVideoEditAttributes,
videoFileMetadataGetValidator,
videosAddLegacyValidator,
videosAddResumableInitValidator,
videosAddResumableValidator,
videosCustomGetValidator,
videosDownloadValidator,
videosGetValidator,
videosOverviewValidator,
videosRemoveValidator,
videosUpdateValidator
}
// ---------------------------------------------------------------------------

6
server/core/models/user/user-export.ts
View File

@ -62,6 +62,10 @@ export class UserExportModel extends SequelizeModel<UserExportModel> {
@Column
storage: FileStorageType
@AllowNull(true)
@Column
fileUrl: string
@ForeignKey(() => UserModel)
@Column
userId: number
@ -188,7 +192,7 @@ export class UserExportModel extends SequelizeModel<UserExportModel> {
getFileDownloadUrl () {
if (this.state !== UserExportState.COMPLETED) return null
return WEBSERVER.URL + join(STATIC_DOWNLOAD_PATHS.USER_EXPORT, this.filename) + '?jwt=' + this.generateJWT()
return WEBSERVER.URL + join(STATIC_DOWNLOAD_PATHS.USER_EXPORTS, this.filename) + '?jwt=' + this.generateJWT()
}
// ---------------------------------------------------------------------------

20
server/core/models/video/formatter/video-api-format.ts
View File

@ -1,8 +1,3 @@
import { generateMagnetUri } from '@server/helpers/webtorrent.js'
import { tracer } from '@server/lib/opentelemetry/tracing.js'
import { getLocalVideoFileMetadataUrl } from '@server/lib/video-urls.js'
import { VideoViewsManager } from '@server/lib/views/video-views-manager.js'
import { uuidToShort } from '@peertube/peertube-node-utils'
import {
Video,
VideoAdditionalAttributes,
@ -12,6 +7,11 @@ import {
VideosCommonQueryAfterSanitize,
VideoStreamingPlaylist
} from '@peertube/peertube-models'
import { uuidToShort } from '@peertube/peertube-node-utils'
import { generateMagnetUri } from '@server/helpers/webtorrent.js'
import { tracer } from '@server/lib/opentelemetry/tracing.js'
import { getLocalVideoFileMetadataUrl } from '@server/lib/video-urls.js'
import { VideoViewsManager } from '@server/lib/views/video-views-manager.js'
import { isArray } from '../../../helpers/custom-validators/misc.js'
import { VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES, VIDEO_PRIVACIES, VIDEO_STATES } from '../../../initializers/constants.js'
import { MServer, MStreamingPlaylistRedundanciesOpt, MVideoFormattable, MVideoFormattableDetails } from '../../../types/models/index.js'
@ -211,9 +211,7 @@ export function videoFilesModelToFormattedJSON (
resolution: {
id: videoFile.resolution,
label: videoFile.resolution === 0
? 'Audio'
: `${videoFile.resolution}p`
label: getResolutionLabel(videoFile.resolution)
},
width: videoFile.width,
@ -259,6 +257,12 @@ export function getStateLabel (id: number) {
return VIDEO_STATES[id] || 'Unknown'
}
export function getResolutionLabel (resolution: number) {
if (resolution === 0) return 'Audio'
return `${resolution}p`
}
// ---------------------------------------------------------------------------
// Private
// ---------------------------------------------------------------------------

13
server/core/models/video/video-file.ts
View File

@ -1,15 +1,15 @@
import { ActivityVideoUrlObject, VideoResolution, FileStorage, type FileStorageType } from '@peertube/peertube-models'
import { ActivityVideoUrlObject, FileStorage, VideoResolution, type FileStorageType } from '@peertube/peertube-models'
import { logger } from '@server/helpers/logger.js'
import { extractVideo } from '@server/helpers/video.js'
import { CONFIG } from '@server/initializers/config.js'
import { buildRemoteUrl } from '@server/lib/activitypub/url.js'
import {
getHLSPrivateFileUrl,
getHLSPublicFileUrl,
getWebVideoPrivateFileUrl,
getWebVideoPublicFileUrl
getObjectStoragePublicFileUrl,
getWebVideoPrivateFileUrl
} from '@server/lib/object-storage/index.js'
import { getFSTorrentFilePath } from '@server/lib/paths.js'
import { getVideoFileMimeType } from '@server/lib/video-file.js'
import { isVideoInPrivateDirectory } from '@server/lib/video-privacy.js'
import { MStreamingPlaylistVideo, MVideo, MVideoWithHost, isStreamingPlaylist } from '@server/types/models/index.js'
import { remove } from 'fs-extra/esm'
@ -51,7 +51,6 @@ import { VideoRedundancyModel } from '../redundancy/video-redundancy.js'
import { SequelizeModel, doesExist, parseAggregateResult, throwIfNotValid } from '../shared/index.js'
import { VideoStreamingPlaylistModel } from './video-streaming-playlist.js'
import { VideoModel } from './video.js'
import { getVideoFileMimeType } from '@server/lib/video-file.js'
export enum ScopeNames {
WITH_VIDEO = 'WITH_VIDEO',
@ -534,10 +533,10 @@ export class VideoFileModel extends SequelizeModel<VideoFileModel> {
private getPublicObjectStorageUrl () {
if (this.isHLS()) {
return getHLSPublicFileUrl(this.fileUrl)
return getObjectStoragePublicFileUrl(this.fileUrl, CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
}
return getWebVideoPublicFileUrl(this.fileUrl)
return getObjectStoragePublicFileUrl(this.fileUrl, CONFIG.OBJECT_STORAGE.WEB_VIDEOS)
}
// ---------------------------------------------------------------------------

89
server/core/models/video/video-source.ts
View File

@ -1,8 +1,12 @@
import type { FileStorageType, VideoSource } from '@peertube/peertube-models'
import { STATIC_DOWNLOAD_PATHS, WEBSERVER } from '@server/initializers/constants.js'
import { join } from 'path'
import { Transaction } from 'sequelize'
import { AllowNull, BelongsTo, Column, CreatedAt, ForeignKey, Table, UpdatedAt } from 'sequelize-typescript'
import { VideoSource } from '@peertube/peertube-models'
import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Table, UpdatedAt } from 'sequelize-typescript'
import { SequelizeModel, getSort } from '../shared/index.js'
import { getResolutionLabel } from './formatter/video-api-format.js'
import { VideoModel } from './video.js'
import { MVideoSource } from '@server/types/models/video/video-source.js'
@Table({
tableName: 'videoSource',
@ -12,6 +16,10 @@ import { VideoModel } from './video.js'
},
{
fields: [ { name: 'createdAt', order: 'DESC' } ]
},
{
fields: [ 'keptOriginalFilename' ],
unique: true
}
]
})
@ -24,7 +32,43 @@ export class VideoSourceModel extends SequelizeModel<VideoSourceModel> {
@AllowNull(false)
@Column
filename: string
inputFilename: string
@AllowNull(true)
@Column
keptOriginalFilename: string
@AllowNull(true)
@Column
resolution: number
@AllowNull(true)
@Column
width: number
@AllowNull(true)
@Column
height: number
@AllowNull(true)
@Column
fps: number
@AllowNull(true)
@Column(DataType.BIGINT)
size: number
@AllowNull(true)
@Column(DataType.JSONB)
metadata: any
@AllowNull(true)
@Column
storage: FileStorageType
@AllowNull(true)
@Column
fileUrl: string
@ForeignKey(() => VideoModel)
@Column
@ -39,16 +83,51 @@ export class VideoSourceModel extends SequelizeModel<VideoSourceModel> {
Video: Awaited<VideoModel>
static loadLatest (videoId: number, transaction?: Transaction) {
return VideoSourceModel.findOne({
return VideoSourceModel.findOne<MVideoSource>({
where: { videoId },
order: getSort('-createdAt'),
transaction
})
}
static loadByKeptOriginalFilename (keptOriginalFilename: string) {
return VideoSourceModel.findOne<MVideoSource>({
where: { keptOriginalFilename }
})
}
static listAll (videoId: number, transaction?: Transaction) {
return VideoSourceModel.findAll<MVideoSource>({
where: { videoId },
transaction
})
}
getFileDownloadUrl () {
if (!this.keptOriginalFilename) return null
return WEBSERVER.URL + join(STATIC_DOWNLOAD_PATHS.ORIGINAL_VIDEO_FILE, this.keptOriginalFilename)
}
toFormattedJSON (): VideoSource {
return {
filename: this.filename,
filename: this.inputFilename,
inputFilename: this.inputFilename,
fileDownloadUrl: this.getFileDownloadUrl(),
resolution: {
id: this.resolution,
label: getResolutionLabel(this.resolution)
},
size: this.size,
width: this.width,
height: this.height,
fps: this.fps,
metadata: this.metadata,
createdAt: this.createdAt.toISOString()
}
}

6
server/core/models/video/video-streaming-playlist.ts
View File

@ -6,7 +6,7 @@ import {
} from '@peertube/peertube-models'
import { sha1 } from '@peertube/peertube-node-utils'
import { CONFIG } from '@server/initializers/config.js'
import { getHLSPrivateFileUrl, getHLSPublicFileUrl } from '@server/lib/object-storage/index.js'
import { getHLSPrivateFileUrl, getObjectStoragePublicFileUrl } from '@server/lib/object-storage/index.js'
import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename } from '@server/lib/paths.js'
import { isVideoInPrivateDirectory } from '@server/lib/video-privacy.js'
import { VideoFileModel } from '@server/models/video/video-file.js'
@ -266,7 +266,7 @@ export class VideoStreamingPlaylistModel extends SequelizeModel<VideoStreamingPl
return getHLSPrivateFileUrl(video, this.playlistFilename)
}
return getHLSPublicFileUrl(this.playlistUrl)
return getObjectStoragePublicFileUrl(this.playlistUrl, CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
}
// ---------------------------------------------------------------------------
@ -288,7 +288,7 @@ export class VideoStreamingPlaylistModel extends SequelizeModel<VideoStreamingPl
return getHLSPrivateFileUrl(video, this.segmentsSha256Filename)
}
return getHLSPublicFileUrl(this.segmentsSha256Url)
return getObjectStoragePublicFileUrl(this.segmentsSha256Url, CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
}
// ---------------------------------------------------------------------------

21
server/core/models/video/video.ts
View File

@ -1,6 +1,7 @@
import { buildVideoEmbedPath, buildVideoWatchPath, pick, wait } from '@peertube/peertube-core-utils'
import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo, getVideoStreamFPS, hasAudioStream } from '@peertube/peertube-ffmpeg'
import {
FileStorage,
ResultList,
ThumbnailType,
UserRight,
@ -13,7 +14,6 @@ import {
VideoPrivacy,
VideoRateType,
VideoState,
FileStorage,
VideoStreamingPlaylistType,
type VideoPrivacyType,
type VideoStateType
@ -25,6 +25,7 @@ import { LiveManager } from '@server/lib/live/live-manager.js'
import {
removeHLSFileObjectStorageByFilename,
removeHLSObjectStorage,
removeOriginalFileObjectStorage,
removeWebVideoObjectStorage
} from '@server/lib/object-storage/index.js'
import { tracer } from '@server/lib/opentelemetry/tracing.js'
@ -34,6 +35,7 @@ import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { isVideoInPrivateDirectory } from '@server/lib/video-privacy.js'
import { getServerActor } from '@server/models/application/application.js'
import { ModelCache } from '@server/models/shared/model-cache.js'
import { MVideoSource } from '@server/types/models/video/video-source.js'
import Bluebird from 'bluebird'
import { remove } from 'fs-extra/esm'
import maxBy from 'lodash-es/maxBy.js'
@ -867,6 +869,12 @@ export class VideoModel extends SequelizeModel<VideoModel> {
for (const p of instance.VideoStreamingPlaylists) {
tasks.push(instance.removeStreamingPlaylistFiles(p))
}
// Remove source files
const promiseRemoveSources = VideoSourceModel.listAll(instance.id, options.transaction)
.then(sources => Promise.all(sources.map(s => instance.removeOriginalFile(s))))
tasks.push(promiseRemoveSources)
}
// Do not wait video deletion because we could be in a transaction
@ -2022,6 +2030,17 @@ export class VideoModel extends SequelizeModel<VideoModel> {
}
}
async removeOriginalFile (videoSource: MVideoSource) {
if (!videoSource.keptOriginalFilename) return
const filePath = VideoPathManager.Instance.getFSOriginalVideoFilePath(videoSource.keptOriginalFilename)
await remove(filePath)
if (videoSource.storage === FileStorage.OBJECT_STORAGE) {
await removeOriginalFileObjectStorage(videoSource)
}
}
isOutdated () {
if (this.isOwned()) return false

9
server/core/types/express.d.ts vendored
View File

@ -1,5 +1,3 @@
import { OutgoingHttpHeaders } from 'http'
import { Writable } from 'stream'
import { HttpMethodType, PeerTubeProblemDocumentData, VideoCreate } from '@peertube/peertube-models'
import { RegisterServerAuthExternalOptions } from '@server/types/index.js'
import {
@ -29,7 +27,10 @@ import { MPlugin, MServer, MServerBlocklist } from '@server/types/models/server.
import { MVideoImportDefault } from '@server/types/models/video/video-import.js'
import { MVideoPlaylistElement, MVideoPlaylistElementVideoUrlPlaylistPrivacy } from '@server/types/models/video/video-playlist-element.js'
import { MAccountVideoRateAccountVideo } from '@server/types/models/video/video-rate.js'
import { File as UploadXFile, Metadata } from '@uploadx/core'
import { Metadata, File as UploadXFile } from '@uploadx/core'
import { FfprobeData } from 'fluent-ffmpeg'
import { OutgoingHttpHeaders } from 'http'
import { Writable } from 'stream'
import { RegisteredPlugin } from '../../lib/plugins/plugin-manager.js'
import {
MAccountDefault,
@ -127,6 +128,8 @@ declare module 'express' {
docUrl?: string
ffprobe?: FfprobeData
videoAPI?: MVideoFormattableDetails
videoAll?: MVideoFullLight
onlyImmutableVideo?: MVideoImmutable

2
server/scripts/migrations/peertube-5.0.ts
View File

@ -19,7 +19,7 @@ async function run () {
console.log('Moving private video files in dedicated folders.')
await ensureDir(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE)
await ensureDir(DIRECTORIES.VIDEOS.PRIVATE)
await ensureDir(DIRECTORIES.WEB_VIDEOS.PRIVATE)
await initDatabaseModels(true)

6
server/scripts/prune-storage.ts
View File

@ -38,8 +38,8 @@ async function run () {
console.log('Detecting files to remove, it could take a while...')
toDelete = toDelete.concat(
await pruneDirectory(DIRECTORIES.VIDEOS.PUBLIC, doesWebVideoFileExist()),
await pruneDirectory(DIRECTORIES.VIDEOS.PRIVATE, doesWebVideoFileExist()),
await pruneDirectory(DIRECTORIES.WEB_VIDEOS.PUBLIC, doesWebVideoFileExist()),
await pruneDirectory(DIRECTORIES.WEB_VIDEOS.PRIVATE, doesWebVideoFileExist()),
await pruneDirectory(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, doesHLSPlaylistExist()),
await pruneDirectory(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, doesHLSPlaylistExist()),
@ -97,7 +97,7 @@ async function pruneDirectory (directory: string, existFun: ExistFun) {
function doesWebVideoFileExist () {
return (filePath: string) => {
// Don't delete private directory
if (filePath === DIRECTORIES.VIDEOS.PRIVATE) return true
if (filePath === DIRECTORIES.WEB_VIDEOS.PRIVATE) return true
return VideoFileModel.doesOwnedWebVideoFileExist(basename(filePath))
}

28
support/doc/api/openapi.yaml
View File

@ -8136,6 +8136,29 @@ components:
properties:
filename:
type: string
deprecated: true
description: 'Deprecated in 6.1, use inputFilename instead'
inputFilename:
type: string
description: 'Uploaded/imported filename'
fileDownloadUrl:
type: string
description: "**PeerTube >= 6.1** If enabled by the admin, the video source file is kept on the server and can be downloaded by the owner"
resolution:
$ref: '#/components/schemas/VideoResolutionConstant'
description: "**PeerTube >= 6.1**"
size:
type: integer
description: "**PeerTube >= 6.1** Video file size in bytes"
fps:
type: number
description: "**PeerTube >= 6.1** Frames per second of the video file"
width:
type: number
description: "**PeerTube >= 6.1** Video stream width"
height:
type: number
description: "**PeerTube >= 6.1** Video stream height"
createdAt:
type: string
format: date-time
@ -8792,6 +8815,11 @@ components:
properties:
enabled:
type: boolean
originalFile:
type: object
properties:
keep:
type: boolean
allowAdditionalExtensions:
type: boolean
description: Allow your users to upload .mkv, .mov, .avi, .wmv, .flv, .f4v, .3g2, .3gp, .mts, m2ts, .mxf, .nut videos