diff --git a/client/package.json b/client/package.json index 140fc3095..8486ace22 100644 --- a/client/package.json +++ b/client/package.json @@ -96,6 +96,7 @@ "lodash-es": "^4.17.4", "markdown-it": "12.0.4", "mini-css-extract-plugin": "^1.3.1", + "ngx-uploadx": "^4.1.0", "p2p-media-loader-hlsjs": "^0.6.2", "path-browserify": "^1.0.0", "primeng": "^11.0.0-rc.1", diff --git a/client/src/app/+my-account/my-account-settings/my-account-settings.component.ts b/client/src/app/+my-account/my-account-settings/my-account-settings.component.ts index c16368952..a0f2f28f8 100644 --- a/client/src/app/+my-account/my-account-settings/my-account-settings.component.ts +++ b/client/src/app/+my-account/my-account-settings/my-account-settings.component.ts @@ -2,7 +2,7 @@ import { ViewportScroller } from '@angular/common' import { HttpErrorResponse } from '@angular/common/http' import { AfterViewChecked, Component, OnInit } from '@angular/core' import { AuthService, Notifier, User, UserService } from '@app/core' -import { uploadErrorHandler } from '@app/helpers' +import { genericUploadErrorHandler } from '@app/helpers' @Component({ selector: 'my-account-settings', @@ -46,7 +46,7 @@ export class MyAccountSettingsComponent implements OnInit, AfterViewChecked { this.user.updateAccountAvatar(data.avatar) }, - (err: HttpErrorResponse) => uploadErrorHandler({ + (err: HttpErrorResponse) => genericUploadErrorHandler({ err, name: $localize`avatar`, notifier: this.notifier diff --git a/client/src/app/+my-library/+my-video-channels/my-video-channel-update.component.ts b/client/src/app/+my-library/+my-video-channels/my-video-channel-update.component.ts index a29af176c..c9173039a 100644 --- a/client/src/app/+my-library/+my-video-channels/my-video-channel-update.component.ts +++ b/client/src/app/+my-library/+my-video-channels/my-video-channel-update.component.ts @@ -3,7 +3,7 @@ import { HttpErrorResponse } from '@angular/common/http' import { Component, OnDestroy, OnInit } from '@angular/core' import { ActivatedRoute, Router } from '@angular/router' import { AuthService, Notifier, ServerService } from '@app/core' -import { uploadErrorHandler } from '@app/helpers' +import { genericUploadErrorHandler } from '@app/helpers' import { VIDEO_CHANNEL_DESCRIPTION_VALIDATOR, VIDEO_CHANNEL_DISPLAY_NAME_VALIDATOR, @@ -109,7 +109,7 @@ export class MyVideoChannelUpdateComponent extends MyVideoChannelEdit implements this.videoChannel.updateAvatar(data.avatar) }, - (err: HttpErrorResponse) => uploadErrorHandler({ + (err: HttpErrorResponse) => genericUploadErrorHandler({ err, name: $localize`avatar`, notifier: this.notifier @@ -139,7 +139,7 @@ export class MyVideoChannelUpdateComponent extends MyVideoChannelEdit implements this.videoChannel.updateBanner(data.banner) }, - (err: HttpErrorResponse) => uploadErrorHandler({ + (err: HttpErrorResponse) => genericUploadErrorHandler({ err, name: $localize`banner`, notifier: this.notifier diff --git a/client/src/app/+videos/+video-edit/video-add-components/uploaderx-form-data.ts b/client/src/app/+videos/+video-edit/video-add-components/uploaderx-form-data.ts new file mode 100644 index 000000000..3392a0d8a --- /dev/null +++ b/client/src/app/+videos/+video-edit/video-add-components/uploaderx-form-data.ts @@ -0,0 +1,48 @@ +import { objectToFormData } from '@app/helpers' +import { resolveUrl, UploaderX } from 'ngx-uploadx' + +/** + * multipart/form-data uploader extending the UploaderX implementation of Google Resumable + * for use with multer + * + * @see https://github.com/kukhariev/ngx-uploadx/blob/637e258fe366b8095203f387a6101a230ee4f8e6/src/uploadx/lib/uploaderx.ts + * @example + * + * options: UploadxOptions = { + * uploaderClass: UploaderXFormData + * }; + */ +export class UploaderXFormData extends UploaderX { + + async getFileUrl (): Promise { + const headers = { + 'X-Upload-Content-Length': this.size.toString(), + 'X-Upload-Content-Type': this.file.type || 'application/octet-stream' + } + + const previewfile = this.metadata.previewfile as any as File + delete this.metadata.previewfile + + const data = objectToFormData(this.metadata) + if (previewfile !== undefined) { + data.append('previewfile', previewfile, previewfile.name) + data.append('thumbnailfile', previewfile, previewfile.name) + } + + await this.request({ + method: 'POST', + body: data, + url: this.endpoint, + headers + }) + + const location = this.getValueFromResponse('location') + if (!location) { + throw new Error('Invalid or missing Location header') + } + + this.offset = this.responseStatus === 201 ? 0 : undefined + + return resolveUrl(location, this.endpoint) + } +} diff --git a/client/src/app/+videos/+video-edit/video-add-components/video-upload.component.html b/client/src/app/+videos/+video-edit/video-add-components/video-upload.component.html index 4c0b09894..86a779f8a 100644 --- a/client/src/app/+videos/+video-edit/video-add-components/video-upload.component.html +++ b/client/src/app/+videos/+video-edit/video-add-components/video-upload.component.html @@ -1,12 +1,17 @@ -
+
Select the file to upload
@@ -41,7 +46,13 @@
- + +
@@ -64,6 +75,7 @@ {{ error }}
+
diff --git a/client/src/app/+videos/+video-edit/video-add-components/video-upload.component.scss b/client/src/app/+videos/+video-edit/video-add-components/video-upload.component.scss index 9549257f6..d9f348a70 100644 --- a/client/src/app/+videos/+video-edit/video-add-components/video-upload.component.scss +++ b/client/src/app/+videos/+video-edit/video-add-components/video-upload.component.scss @@ -47,8 +47,4 @@ margin-left: 10px; } - - .btn-group > input:not(:first-child) { - margin-left: 0; - } } diff --git a/client/src/app/+videos/+video-edit/video-add-components/video-upload.component.ts b/client/src/app/+videos/+video-edit/video-add-components/video-upload.component.ts index effb37077..2d3fc3578 100644 --- a/client/src/app/+videos/+video-edit/video-add-components/video-upload.component.ts +++ b/client/src/app/+videos/+video-edit/video-add-components/video-upload.component.ts @@ -1,15 +1,16 @@ -import { Subscription } from 'rxjs' -import { HttpErrorResponse, HttpEventType, HttpResponse } from '@angular/common/http' import { AfterViewInit, Component, ElementRef, EventEmitter, OnDestroy, OnInit, Output, ViewChild } from '@angular/core' import { Router } from '@angular/router' +import { UploadxOptions, UploadState, UploadxService } from 'ngx-uploadx' +import { UploaderXFormData } from './uploaderx-form-data' import { AuthService, CanComponentDeactivate, HooksService, Notifier, ServerService, UserService } from '@app/core' -import { scrollToTop, uploadErrorHandler } from '@app/helpers' +import { scrollToTop, genericUploadErrorHandler } from '@app/helpers' import { FormValidatorService } from '@app/shared/shared-forms' import { BytesPipe, VideoCaptionService, VideoEdit, VideoService } from '@app/shared/shared-main' import { LoadingBarService } from '@ngx-loading-bar/core' import { HttpStatusCode } from '@shared/core-utils/miscs/http-error-codes' import { VideoPrivacy } from '@shared/models' import { VideoSend } from './video-send' +import { HttpErrorResponse, HttpEventType, HttpHeaders } from '@angular/common/http' @Component({ selector: 'my-video-upload', @@ -20,23 +21,18 @@ import { VideoSend } from './video-send' './video-send.scss' ] }) -export class VideoUploadComponent extends VideoSend implements OnInit, AfterViewInit, OnDestroy, CanComponentDeactivate { +export class VideoUploadComponent extends VideoSend implements OnInit, OnDestroy, AfterViewInit, CanComponentDeactivate { @Output() firstStepDone = new EventEmitter() @Output() firstStepError = new EventEmitter() @ViewChild('videofileInput') videofileInput: ElementRef - // So that it can be accessed in the template - readonly SPECIAL_SCHEDULED_PRIVACY = VideoEdit.SPECIAL_SCHEDULED_PRIVACY - userVideoQuotaUsed = 0 userVideoQuotaUsedDaily = 0 isUploadingAudioFile = false isUploadingVideo = false - isUpdatingVideo = false videoUploaded = false - videoUploadObservable: Subscription = null videoUploadPercents = 0 videoUploadedIds = { id: 0, @@ -49,7 +45,13 @@ export class VideoUploadComponent extends VideoSend implements OnInit, AfterView error: string enableRetryAfterError: boolean + // So that it can be accessed in the template protected readonly DEFAULT_VIDEO_PRIVACY = VideoPrivacy.PUBLIC + protected readonly BASE_VIDEO_UPLOAD_URL = VideoService.BASE_VIDEO_URL + 'upload-resumable' + + private uploadxOptions: UploadxOptions + private isUpdatingVideo = false + private fileToUpload: File constructor ( protected formValidatorService: FormValidatorService, @@ -61,15 +63,77 @@ export class VideoUploadComponent extends VideoSend implements OnInit, AfterView protected videoCaptionService: VideoCaptionService, private userService: UserService, private router: Router, - private hooks: HooksService - ) { + private hooks: HooksService, + private resumableUploadService: UploadxService + ) { super() + + this.uploadxOptions = { + endpoint: this.BASE_VIDEO_UPLOAD_URL, + multiple: false, + token: this.authService.getAccessToken(), + uploaderClass: UploaderXFormData, + retryConfig: { + maxAttempts: 6, + shouldRetry: (code: number) => { + return code < 400 || code >= 501 + } + } + } } get videoExtensions () { return this.serverConfig.video.file.extensions.join(', ') } + onUploadVideoOngoing (state: UploadState) { + switch (state.status) { + case 'error': + const error = state.response?.error || 'Unknow error' + + this.handleUploadError({ + error: new Error(error), + name: 'HttpErrorResponse', + message: error, + ok: false, + headers: new HttpHeaders(state.responseHeaders), + status: +state.responseStatus, + statusText: error, + type: HttpEventType.Response, + url: state.url + }) + break + + case 'cancelled': + this.isUploadingVideo = false + this.videoUploadPercents = 0 + + this.firstStepError.emit() + this.enableRetryAfterError = false + this.error = '' + break + + case 'queue': + this.closeFirstStep(state.name) + break + + case 'uploading': + this.videoUploadPercents = state.progress + break + + case 'paused': + this.notifier.info($localize`Upload cancelled`) + break + + case 'complete': + this.videoUploaded = true + this.videoUploadPercents = 100 + + this.videoUploadedIds = state?.response.video + break + } + } + ngOnInit () { super.ngOnInit() @@ -78,6 +142,9 @@ export class VideoUploadComponent extends VideoSend implements OnInit, AfterView this.userVideoQuotaUsed = data.videoQuotaUsed this.userVideoQuotaUsedDaily = data.videoQuotaUsedDaily }) + + this.resumableUploadService.events + .subscribe(state => this.onUploadVideoOngoing(state)) } ngAfterViewInit () { @@ -85,7 +152,7 @@ export class VideoUploadComponent extends VideoSend implements OnInit, AfterView } ngOnDestroy () { - if (this.videoUploadObservable) this.videoUploadObservable.unsubscribe() + this.cancelUpload() } canDeactivate () { @@ -105,137 +172,43 @@ export class VideoUploadComponent extends VideoSend implements OnInit, AfterView } } - getVideoFile () { - return this.videofileInput.nativeElement.files[0] - } - - setVideoFile (files: FileList) { + onFileDropped (files: FileList) { this.videofileInput.nativeElement.files = files - this.fileChange() + + this.onFileChange({ target: this.videofileInput.nativeElement }) } - getAudioUploadLabel () { - const videofile = this.getVideoFile() - if (!videofile) return $localize`Upload` + onFileChange (event: Event | { target: HTMLInputElement }) { + const file = (event.target as HTMLInputElement).files[0] - return $localize`Upload ${videofile.name}` + if (!file) return + + if (!this.checkGlobalUserQuota(file)) return + if (!this.checkDailyUserQuota(file)) return + + if (this.isAudioFile(file.name)) { + this.isUploadingAudioFile = true + return + } + + this.isUploadingVideo = true + this.fileToUpload = file + + this.uploadFile(file) } - fileChange () { - this.uploadFirstStep() + uploadAudio () { + this.uploadFile(this.getInputVideoFile(), this.previewfileUpload) } retryUpload () { this.enableRetryAfterError = false this.error = '' - this.uploadVideo() + this.uploadFile(this.fileToUpload) } cancelUpload () { - if (this.videoUploadObservable !== null) { - this.videoUploadObservable.unsubscribe() - } - - this.isUploadingVideo = false - this.videoUploadPercents = 0 - this.videoUploadObservable = null - - this.firstStepError.emit() - this.enableRetryAfterError = false - this.error = '' - - this.notifier.info($localize`Upload cancelled`) - } - - uploadFirstStep (clickedOnButton = false) { - const videofile = this.getVideoFile() - if (!videofile) return - - if (!this.checkGlobalUserQuota(videofile)) return - if (!this.checkDailyUserQuota(videofile)) return - - if (clickedOnButton === false && this.isAudioFile(videofile.name)) { - this.isUploadingAudioFile = true - return - } - - // Build name field - const nameWithoutExtension = videofile.name.replace(/\.[^/.]+$/, '') - let name: string - - // If the name of the file is very small, keep the extension - if (nameWithoutExtension.length < 3) name = videofile.name - else name = nameWithoutExtension - - const nsfw = this.serverConfig.instance.isNSFW - const waitTranscoding = true - const commentsEnabled = true - const downloadEnabled = true - const channelId = this.firstStepChannelId.toString() - - this.formData = new FormData() - this.formData.append('name', name) - // Put the video "private" -> we are waiting the user validation of the second step - this.formData.append('privacy', VideoPrivacy.PRIVATE.toString()) - this.formData.append('nsfw', '' + nsfw) - this.formData.append('commentsEnabled', '' + commentsEnabled) - this.formData.append('downloadEnabled', '' + downloadEnabled) - this.formData.append('waitTranscoding', '' + waitTranscoding) - this.formData.append('channelId', '' + channelId) - this.formData.append('videofile', videofile) - - if (this.previewfileUpload) { - this.formData.append('previewfile', this.previewfileUpload) - this.formData.append('thumbnailfile', this.previewfileUpload) - } - - this.isUploadingVideo = true - this.firstStepDone.emit(name) - - this.form.patchValue({ - name, - privacy: this.firstStepPrivacyId, - nsfw, - channelId: this.firstStepChannelId, - previewfile: this.previewfileUpload - }) - - this.uploadVideo() - } - - uploadVideo () { - this.videoUploadObservable = this.videoService.uploadVideo(this.formData).subscribe( - event => { - if (event.type === HttpEventType.UploadProgress) { - this.videoUploadPercents = Math.round(100 * event.loaded / event.total) - } else if (event instanceof HttpResponse) { - this.videoUploaded = true - - this.videoUploadedIds = event.body.video - - this.videoUploadObservable = null - } - }, - - (err: HttpErrorResponse) => { - // Reset progress (but keep isUploadingVideo true) - this.videoUploadPercents = 0 - this.videoUploadObservable = null - this.enableRetryAfterError = true - - this.error = uploadErrorHandler({ - err, - name: $localize`video`, - notifier: this.notifier, - sticky: false - }) - - if (err.status === HttpStatusCode.PAYLOAD_TOO_LARGE_413 || - err.status === HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415) { - this.cancelUpload() - } - } - ) + this.resumableUploadService.control({ action: 'cancel' }) } isPublishingButtonDisabled () { @@ -245,6 +218,13 @@ export class VideoUploadComponent extends VideoSend implements OnInit, AfterView !this.videoUploadedIds.id } + getAudioUploadLabel () { + const videofile = this.getInputVideoFile() + if (!videofile) return $localize`Upload` + + return $localize`Upload ${videofile.name}` + } + updateSecondStep () { if (this.isPublishingButtonDisabled() || !this.checkForm()) { return @@ -275,6 +255,62 @@ export class VideoUploadComponent extends VideoSend implements OnInit, AfterView ) } + private getInputVideoFile () { + return this.videofileInput.nativeElement.files[0] + } + + private uploadFile (file: File, previewfile?: File) { + const metadata = { + waitTranscoding: true, + commentsEnabled: true, + downloadEnabled: true, + channelId: this.firstStepChannelId, + nsfw: this.serverConfig.instance.isNSFW, + privacy: VideoPrivacy.PRIVATE.toString(), + filename: file.name, + previewfile: previewfile as any + } + + this.resumableUploadService.handleFiles(file, { + ...this.uploadxOptions, + metadata + }) + + this.isUploadingVideo = true + } + + private handleUploadError (err: HttpErrorResponse) { + // Reset progress (but keep isUploadingVideo true) + this.videoUploadPercents = 0 + this.enableRetryAfterError = true + + this.error = genericUploadErrorHandler({ + err, + name: $localize`video`, + notifier: this.notifier, + sticky: false + }) + + if (err.status === HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415) { + this.cancelUpload() + } + } + + private closeFirstStep (filename: string) { + const nameWithoutExtension = filename.replace(/\.[^/.]+$/, '') + const name = nameWithoutExtension.length < 3 ? filename : nameWithoutExtension + + this.form.patchValue({ + name, + privacy: this.firstStepPrivacyId, + nsfw: this.serverConfig.instance.isNSFW, + channelId: this.firstStepChannelId, + previewfile: this.previewfileUpload + }) + + this.firstStepDone.emit(name) + } + private checkGlobalUserQuota (videofile: File) { const bytePipes = new BytesPipe() @@ -285,8 +321,7 @@ export class VideoUploadComponent extends VideoSend implements OnInit, AfterView const videoQuotaUsedBytes = bytePipes.transform(this.userVideoQuotaUsed, 0) const videoQuotaBytes = bytePipes.transform(videoQuota, 0) - const msg = $localize`Your video quota is exceeded with this video ( -video size: ${videoSizeBytes}, used: ${videoQuotaUsedBytes}, quota: ${videoQuotaBytes})` + const msg = $localize`Your video quota is exceeded with this video (video size: ${videoSizeBytes}, used: ${videoQuotaUsedBytes}, quota: ${videoQuotaBytes})` this.notifier.error(msg) return false @@ -304,9 +339,7 @@ video size: ${videoSizeBytes}, used: ${videoQuotaUsedBytes}, quota: ${videoQuota const videoSizeBytes = bytePipes.transform(videofile.size, 0) const quotaUsedDailyBytes = bytePipes.transform(this.userVideoQuotaUsedDaily, 0) const quotaDailyBytes = bytePipes.transform(videoQuotaDaily, 0) - - const msg = $localize`Your daily video quota is exceeded with this video ( -video size: ${videoSizeBytes}, used: ${quotaUsedDailyBytes}, quota: ${quotaDailyBytes})` + const msg = $localize`Your daily video quota is exceeded with this video (video size: ${videoSizeBytes}, used: ${quotaUsedDailyBytes}, quota: ${quotaDailyBytes})` this.notifier.error(msg) return false diff --git a/client/src/app/+videos/+video-edit/video-add.module.ts b/client/src/app/+videos/+video-edit/video-add.module.ts index da651119b..e836cf81e 100644 --- a/client/src/app/+videos/+video-edit/video-add.module.ts +++ b/client/src/app/+videos/+video-edit/video-add.module.ts @@ -1,5 +1,6 @@ import { NgModule } from '@angular/core' import { CanDeactivateGuard } from '@app/core' +import { UploadxModule } from 'ngx-uploadx' import { VideoEditModule } from './shared/video-edit.module' import { DragDropDirective } from './video-add-components/drag-drop.directive' import { VideoImportTorrentComponent } from './video-add-components/video-import-torrent.component' @@ -13,7 +14,9 @@ import { VideoAddComponent } from './video-add.component' imports: [ VideoAddRoutingModule, - VideoEditModule + VideoEditModule, + + UploadxModule ], declarations: [ diff --git a/client/src/app/helpers/utils.ts b/client/src/app/helpers/utils.ts index 17eb5effc..d6ac5b9b4 100644 --- a/client/src/app/helpers/utils.ts +++ b/client/src/app/helpers/utils.ts @@ -173,8 +173,8 @@ function isXPercentInViewport (el: HTMLElement, percentVisible: number) { ) } -function uploadErrorHandler (parameters: { - err: HttpErrorResponse +function genericUploadErrorHandler (parameters: { + err: Pick name: string notifier: Notifier sticky?: boolean @@ -186,6 +186,9 @@ function uploadErrorHandler (parameters: { if (err instanceof ErrorEvent) { // network error message = $localize`The connection was interrupted` notifier.error(message, title, null, sticky) + } else if (err.status === HttpStatusCode.INTERNAL_SERVER_ERROR_500) { + message = $localize`The server encountered an error` + notifier.error(message, title, null, sticky) } else if (err.status === HttpStatusCode.REQUEST_TIMEOUT_408) { message = $localize`Your ${name} file couldn't be transferred before the set timeout (usually 10min)` notifier.error(message, title, null, sticky) @@ -216,5 +219,5 @@ export { isInViewport, isXPercentInViewport, listUserChannels, - uploadErrorHandler + genericUploadErrorHandler } diff --git a/client/yarn.lock b/client/yarn.lock index 571314f22..1b1455cc8 100644 --- a/client/yarn.lock +++ b/client/yarn.lock @@ -7793,6 +7793,13 @@ next-tick@~1.0.0: resolved "https://registry.yarnpkg.com/next-tick/-/next-tick-1.0.0.tgz#ca86d1fe8828169b0120208e3dc8424b9db8342c" integrity sha1-yobR/ogoFpsBICCOPchCS524NCw= +ngx-uploadx@^4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/ngx-uploadx/-/ngx-uploadx-4.1.0.tgz#b3ed4566a2505239026bbdc10c2345aae28d67df" + integrity sha512-KCG0NT4SBc/5MRl8aR6joHHg+WeTdrkhLeC1DrNgVxrTBuuenlEwOVDpkLJMPX/8HE6Bq33rx1U2NNZYVl9NMQ== + dependencies: + tslib "^1.9.0" + nice-try@^1.0.4: version "1.0.5" resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366" diff --git a/package.json b/package.json index e1508c65f..d3375c7d4 100644 --- a/package.json +++ b/package.json @@ -73,6 +73,7 @@ "swagger-cli": "swagger-cli" }, "dependencies": { + "@uploadx/core": "^4.4.0", "apicache": "1.6.2", "async": "^3.0.1", "async-lru": "^1.1.1", diff --git a/server.ts b/server.ts index 2531080a3..97dffe756 100644 --- a/server.ts +++ b/server.ts @@ -116,6 +116,7 @@ import { YoutubeDlUpdateScheduler } from './server/lib/schedulers/youtube-dl-upd import { VideosRedundancyScheduler } from './server/lib/schedulers/videos-redundancy-scheduler' import { RemoveOldHistoryScheduler } from './server/lib/schedulers/remove-old-history-scheduler' import { AutoFollowIndexInstances } from './server/lib/schedulers/auto-follow-index-instances' +import { RemoveDanglingResumableUploadsScheduler } from './server/lib/schedulers/remove-dangling-resumable-uploads-scheduler' import { isHTTPSignatureDigestValid } from './server/helpers/peertube-crypto' import { PeerTubeSocket } from './server/lib/peertube-socket' import { updateStreamingPlaylistsInfohashesIfNeeded } from './server/lib/hls' @@ -280,6 +281,7 @@ async function startApplication () { PluginsCheckScheduler.Instance.enable() PeerTubeVersionCheckScheduler.Instance.enable() AutoFollowIndexInstances.Instance.enable() + RemoveDanglingResumableUploadsScheduler.Instance.enable() // Redis initialization Redis.Instance.init() diff --git a/server/controllers/api/server/debug.ts b/server/controllers/api/server/debug.ts index 7787186be..ff0d9ca3c 100644 --- a/server/controllers/api/server/debug.ts +++ b/server/controllers/api/server/debug.ts @@ -1,4 +1,6 @@ import { InboxManager } from '@server/lib/activitypub/inbox-manager' +import { RemoveDanglingResumableUploadsScheduler } from '@server/lib/schedulers/remove-dangling-resumable-uploads-scheduler' +import { SendDebugCommand } from '@shared/models' import * as express from 'express' import { UserRight } from '../../../../shared/models/users' import { authenticate, ensureUserHasRight } from '../../../middlewares' @@ -11,6 +13,12 @@ debugRouter.get('/debug', getDebug ) +debugRouter.post('/debug/run-command', + authenticate, + ensureUserHasRight(UserRight.MANAGE_DEBUG), + runCommand +) + // --------------------------------------------------------------------------- export { @@ -25,3 +33,13 @@ function getDebug (req: express.Request, res: express.Response) { activityPubMessagesWaiting: InboxManager.Instance.getActivityPubMessagesWaiting() }) } + +async function runCommand (req: express.Request, res: express.Response) { + const body: SendDebugCommand = req.body + + if (body.command === 'remove-dandling-resumable-uploads') { + await RemoveDanglingResumableUploadsScheduler.Instance.execute() + } + + return res.sendStatus(204) +} diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts index fbdb0f776..c32626d30 100644 --- a/server/controllers/api/videos/index.ts +++ b/server/controllers/api/videos/index.ts @@ -2,6 +2,7 @@ import * as express from 'express' import { move } from 'fs-extra' import { extname } from 'path' import toInt from 'validator/lib/toInt' +import { deleteResumableUploadMetaFile, getResumableUploadPath } from '@server/helpers/upload' import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' import { changeVideoChannelShare } from '@server/lib/activitypub/share' import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' @@ -10,8 +11,9 @@ import { addOptimizeOrMergeAudioJob, buildLocalVideoFromReq, buildVideoThumbnail import { generateVideoFilename, getVideoFilePath } from '@server/lib/video-paths' import { getServerActor } from '@server/models/application/application' import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' +import { uploadx } from '@uploadx/core' import { VideoCreate, VideosCommonQuery, VideoState, VideoUpdate } from '../../../../shared' -import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes' +import { HttpStatusCode } from '../../../../shared/core-utils/miscs' import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger' import { resetSequelizeInstance, retryTransactionWrapper } from '../../../helpers/database-utils' import { buildNSFWFilter, createReqFiles, getCountVideos } from '../../../helpers/express-utils' @@ -47,7 +49,9 @@ import { setDefaultPagination, setDefaultVideosSort, videoFileMetadataGetValidator, - videosAddValidator, + videosAddLegacyValidator, + videosAddResumableInitValidator, + videosAddResumableValidator, videosCustomGetValidator, videosGetValidator, videosRemoveValidator, @@ -69,6 +73,7 @@ import { watchingRouter } from './watching' const lTags = loggerTagsFactory('api', 'video') const auditLogger = auditLoggerFactory('videos') const videosRouter = express.Router() +const uploadxMiddleware = uploadx.upload({ directory: getResumableUploadPath() }) const reqVideoFileAdd = createReqFiles( [ 'videofile', 'thumbnailfile', 'previewfile' ], @@ -79,6 +84,16 @@ const reqVideoFileAdd = createReqFiles( previewfile: CONFIG.STORAGE.TMP_DIR } ) + +const reqVideoFileAddResumable = createReqFiles( + [ 'thumbnailfile', 'previewfile' ], + MIMETYPES.IMAGE.MIMETYPE_EXT, + { + thumbnailfile: getResumableUploadPath(), + previewfile: getResumableUploadPath() + } +) + const reqVideoFileUpdate = createReqFiles( [ 'thumbnailfile', 'previewfile' ], MIMETYPES.IMAGE.MIMETYPE_EXT, @@ -111,18 +126,39 @@ videosRouter.get('/', commonVideosFiltersValidator, asyncMiddleware(listVideos) ) + +videosRouter.post('/upload', + authenticate, + reqVideoFileAdd, + asyncMiddleware(videosAddLegacyValidator), + asyncRetryTransactionMiddleware(addVideoLegacy) +) + +videosRouter.post('/upload-resumable', + authenticate, + reqVideoFileAddResumable, + asyncMiddleware(videosAddResumableInitValidator), + uploadxMiddleware +) + +videosRouter.delete('/upload-resumable', + authenticate, + uploadxMiddleware +) + +videosRouter.put('/upload-resumable', + authenticate, + uploadxMiddleware, // uploadx doesn't use call next() before the file upload completes + asyncMiddleware(videosAddResumableValidator), + asyncMiddleware(addVideoResumable) +) + videosRouter.put('/:id', authenticate, reqVideoFileUpdate, asyncMiddleware(videosUpdateValidator), asyncRetryTransactionMiddleware(updateVideo) ) -videosRouter.post('/upload', - authenticate, - reqVideoFileAdd, - asyncMiddleware(videosAddValidator), - asyncRetryTransactionMiddleware(addVideo) -) videosRouter.get('/:id/description', asyncMiddleware(videosGetValidator), @@ -157,23 +193,23 @@ export { // --------------------------------------------------------------------------- -function listVideoCategories (req: express.Request, res: express.Response) { +function listVideoCategories (_req: express.Request, res: express.Response) { res.json(VIDEO_CATEGORIES) } -function listVideoLicences (req: express.Request, res: express.Response) { +function listVideoLicences (_req: express.Request, res: express.Response) { res.json(VIDEO_LICENCES) } -function listVideoLanguages (req: express.Request, res: express.Response) { +function listVideoLanguages (_req: express.Request, res: express.Response) { res.json(VIDEO_LANGUAGES) } -function listVideoPrivacies (req: express.Request, res: express.Response) { +function listVideoPrivacies (_req: express.Request, res: express.Response) { res.json(VIDEO_PRIVACIES) } -async function addVideo (req: express.Request, res: express.Response) { +async function addVideoLegacy (req: express.Request, res: express.Response) { // Uploading the video could be long // Set timeout to 10 minutes, as Express's default is 2 minutes req.setTimeout(1000 * 60 * 10, () => { @@ -183,13 +219,42 @@ async function addVideo (req: express.Request, res: express.Response) { const videoPhysicalFile = req.files['videofile'][0] const videoInfo: VideoCreate = req.body + const files = req.files - const videoData = buildLocalVideoFromReq(videoInfo, res.locals.videoChannel.id) - videoData.state = CONFIG.TRANSCODING.ENABLED ? VideoState.TO_TRANSCODE : VideoState.PUBLISHED - videoData.duration = videoPhysicalFile['duration'] // duration was added by a previous middleware + return addVideo({ res, videoPhysicalFile, videoInfo, files }) +} + +async function addVideoResumable (_req: express.Request, res: express.Response) { + const videoPhysicalFile = res.locals.videoFileResumable + const videoInfo = videoPhysicalFile.metadata + const files = { previewfile: videoInfo.previewfile } + + // Don't need the meta file anymore + await deleteResumableUploadMetaFile(videoPhysicalFile.path) + + return addVideo({ res, videoPhysicalFile, videoInfo, files }) +} + +async function addVideo (options: { + res: express.Response + videoPhysicalFile: express.VideoUploadFile + videoInfo: VideoCreate + files: express.UploadFiles +}) { + const { res, videoPhysicalFile, videoInfo, files } = options + const videoChannel = res.locals.videoChannel + const user = res.locals.oauth.token.User + + const videoData = buildLocalVideoFromReq(videoInfo, videoChannel.id) + + videoData.state = CONFIG.TRANSCODING.ENABLED + ? VideoState.TO_TRANSCODE + : VideoState.PUBLISHED + + videoData.duration = videoPhysicalFile.duration // duration was added by a previous middleware const video = new VideoModel(videoData) as MVideoFullLight - video.VideoChannel = res.locals.videoChannel + video.VideoChannel = videoChannel video.url = getLocalVideoActivityPubUrl(video) // We use the UUID, so set the URL after building the object const videoFile = new VideoFileModel({ @@ -217,7 +282,7 @@ async function addVideo (req: express.Request, res: express.Response) { const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({ video, - files: req.files, + files, fallback: type => generateVideoMiniature({ video, videoFile, type }) }) @@ -253,7 +318,7 @@ async function addVideo (req: express.Request, res: express.Response) { await autoBlacklistVideoIfNeeded({ video, - user: res.locals.oauth.token.User, + user, isRemote: false, isNew: true, transaction: t @@ -282,7 +347,7 @@ async function addVideo (req: express.Request, res: express.Response) { .catch(err => logger.error('Cannot federate or notify video creation %s', video.url, { err, ...lTags(video.uuid) })) if (video.state === VideoState.TO_TRANSCODE) { - await addOptimizeOrMergeAudioJob(videoCreated, videoFile, res.locals.oauth.token.User) + await addOptimizeOrMergeAudioJob(videoCreated, videoFile, user) } Hooks.runAction('action:api.video.uploaded', { video: videoCreated }) diff --git a/server/helpers/custom-validators/misc.ts b/server/helpers/custom-validators/misc.ts index effdd98cb..fd3b45804 100644 --- a/server/helpers/custom-validators/misc.ts +++ b/server/helpers/custom-validators/misc.ts @@ -1,6 +1,7 @@ import 'multer' -import validator from 'validator' +import { UploadFilesForCheck } from 'express' import { sep } from 'path' +import validator from 'validator' function exists (value: any) { return value !== undefined && value !== null @@ -108,7 +109,7 @@ function isFileFieldValid ( } function isFileMimeTypeValid ( - files: { [ fieldname: string ]: Express.Multer.File[] } | Express.Multer.File[], + files: UploadFilesForCheck, mimeTypeRegex: string, field: string, optional = false diff --git a/server/helpers/custom-validators/videos.ts b/server/helpers/custom-validators/videos.ts index 87966798f..b33e088eb 100644 --- a/server/helpers/custom-validators/videos.ts +++ b/server/helpers/custom-validators/videos.ts @@ -1,4 +1,6 @@ +import { UploadFilesForCheck } from 'express' import { values } from 'lodash' +import * as magnetUtil from 'magnet-uri' import validator from 'validator' import { VideoFilter, VideoPrivacy, VideoRateType } from '../../../shared' import { @@ -6,13 +8,12 @@ import { MIMETYPES, VIDEO_CATEGORIES, VIDEO_LICENCES, + VIDEO_LIVE, VIDEO_PRIVACIES, VIDEO_RATE_TYPES, - VIDEO_STATES, - VIDEO_LIVE + VIDEO_STATES } from '../../initializers/constants' import { exists, isArray, isDateValid, isFileMimeTypeValid, isFileValid } from './misc' -import * as magnetUtil from 'magnet-uri' const VIDEOS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEOS @@ -81,7 +82,7 @@ function isVideoFileExtnameValid (value: string) { return exists(value) && (value === VIDEO_LIVE.EXTENSION || MIMETYPES.VIDEO.EXT_MIMETYPE[value] !== undefined) } -function isVideoFileMimeTypeValid (files: { [ fieldname: string ]: Express.Multer.File[] } | Express.Multer.File[]) { +function isVideoFileMimeTypeValid (files: UploadFilesForCheck) { return isFileMimeTypeValid(files, MIMETYPES.VIDEO.MIMETYPES_REGEX, 'videofile') } diff --git a/server/helpers/express-utils.ts b/server/helpers/express-utils.ts index c0d3f8f32..ede22a3cc 100644 --- a/server/helpers/express-utils.ts +++ b/server/helpers/express-utils.ts @@ -2,7 +2,7 @@ import * as express from 'express' import * as multer from 'multer' import { REMOTE_SCHEME } from '../initializers/constants' import { logger } from './logger' -import { deleteFileAsync, generateRandomString } from './utils' +import { deleteFileAndCatch, generateRandomString } from './utils' import { extname } from 'path' import { isArray } from './custom-validators/misc' import { CONFIG } from '../initializers/config' @@ -36,15 +36,15 @@ function cleanUpReqFiles (req: { files: { [fieldname: string]: Express.Multer.Fi if (!files) return if (isArray(files)) { - (files as Express.Multer.File[]).forEach(f => deleteFileAsync(f.path)) + (files as Express.Multer.File[]).forEach(f => deleteFileAndCatch(f.path)) return } for (const key of Object.keys(files)) { const file = files[key] - if (isArray(file)) file.forEach(f => deleteFileAsync(f.path)) - else deleteFileAsync(file.path) + if (isArray(file)) file.forEach(f => deleteFileAndCatch(f.path)) + else deleteFileAndCatch(file.path) } } diff --git a/server/helpers/upload.ts b/server/helpers/upload.ts new file mode 100644 index 000000000..030a6b7d5 --- /dev/null +++ b/server/helpers/upload.ts @@ -0,0 +1,21 @@ +import { METAFILE_EXTNAME } from '@uploadx/core' +import { remove } from 'fs-extra' +import { join } from 'path' +import { RESUMABLE_UPLOAD_DIRECTORY } from '../initializers/constants' + +function getResumableUploadPath (filename?: string) { + if (filename) return join(RESUMABLE_UPLOAD_DIRECTORY, filename) + + return RESUMABLE_UPLOAD_DIRECTORY +} + +function deleteResumableUploadMetaFile (filepath: string) { + return remove(filepath + METAFILE_EXTNAME) +} + +// --------------------------------------------------------------------------- + +export { + getResumableUploadPath, + deleteResumableUploadMetaFile +} diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts index 0545e8996..6c95a43b6 100644 --- a/server/helpers/utils.ts +++ b/server/helpers/utils.ts @@ -6,7 +6,7 @@ import { CONFIG } from '../initializers/config' import { execPromise, execPromise2, randomBytesPromise, sha256 } from './core-utils' import { logger } from './logger' -function deleteFileAsync (path: string) { +function deleteFileAndCatch (path: string) { remove(path) .catch(err => logger.error('Cannot delete the file %s asynchronously.', path, { err })) } @@ -83,7 +83,7 @@ function getUUIDFromFilename (filename: string) { // --------------------------------------------------------------------------- export { - deleteFileAsync, + deleteFileAndCatch, generateRandomString, getFormattedObjects, getSecureTorrentName, diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index f807a1e58..6f388420e 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -208,7 +208,8 @@ const SCHEDULER_INTERVALS_MS = { autoFollowIndexInstances: 60000 * 60 * 24, // 1 day removeOldViews: 60000 * 60 * 24, // 1 day removeOldHistory: 60000 * 60 * 24, // 1 day - updateInboxStats: 1000 * 60// 1 minute + updateInboxStats: 1000 * 60, // 1 minute + removeDanglingResumableUploads: 60000 * 60 * 16 // 16 hours } // --------------------------------------------------------------------------- @@ -285,6 +286,7 @@ const CONSTRAINTS_FIELDS = { LIKES: { min: 0 }, DISLIKES: { min: 0 }, FILE_SIZE: { min: -1 }, + PARTIAL_UPLOAD_SIZE: { max: 50 * 1024 * 1024 * 1024 }, // 50GB URL: { min: 3, max: 2000 } // Length }, VIDEO_PLAYLISTS: { @@ -645,6 +647,7 @@ const LRU_CACHE = { } } +const RESUMABLE_UPLOAD_DIRECTORY = join(CONFIG.STORAGE.TMP_DIR, 'resumable-uploads') const HLS_STREAMING_PLAYLIST_DIRECTORY = join(CONFIG.STORAGE.STREAMING_PLAYLISTS_DIR, 'hls') const HLS_REDUNDANCY_DIRECTORY = join(CONFIG.STORAGE.REDUNDANCY_DIR, 'hls') @@ -819,6 +822,7 @@ export { PEERTUBE_VERSION, LAZY_STATIC_PATHS, SEARCH_INDEX, + RESUMABLE_UPLOAD_DIRECTORY, HLS_REDUNDANCY_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION, ACTOR_IMAGES_SIZE, diff --git a/server/initializers/installer.ts b/server/initializers/installer.ts index cb58454cb..8dcff64e2 100644 --- a/server/initializers/installer.ts +++ b/server/initializers/installer.ts @@ -6,7 +6,7 @@ import { UserModel } from '../models/account/user' import { ApplicationModel } from '../models/application/application' import { OAuthClientModel } from '../models/oauth/oauth-client' import { applicationExist, clientsExist, usersExist } from './checker-after-init' -import { FILES_CACHE, HLS_STREAMING_PLAYLIST_DIRECTORY, LAST_MIGRATION_VERSION } from './constants' +import { FILES_CACHE, HLS_STREAMING_PLAYLIST_DIRECTORY, LAST_MIGRATION_VERSION, RESUMABLE_UPLOAD_DIRECTORY } from './constants' import { sequelizeTypescript } from './database' import { ensureDir, remove } from 'fs-extra' import { CONFIG } from './config' @@ -79,6 +79,9 @@ function createDirectoriesIfNotExist () { // Playlist directories tasks.push(ensureDir(HLS_STREAMING_PLAYLIST_DIRECTORY)) + // Resumable upload directory + tasks.push(ensureDir(RESUMABLE_UPLOAD_DIRECTORY)) + return Promise.all(tasks) } diff --git a/server/lib/moderation.ts b/server/lib/moderation.ts index 5180b3299..925d64902 100644 --- a/server/lib/moderation.ts +++ b/server/lib/moderation.ts @@ -1,6 +1,8 @@ +import { VideoUploadFile } from 'express' import { PathLike } from 'fs-extra' import { Transaction } from 'sequelize/types' import { AbuseAuditView, auditLoggerFactory } from '@server/helpers/audit-logger' +import { afterCommitIfTransaction } from '@server/helpers/database-utils' import { logger } from '@server/helpers/logger' import { AbuseModel } from '@server/models/abuse/abuse' import { VideoAbuseModel } from '@server/models/abuse/video-abuse' @@ -28,7 +30,6 @@ import { VideoModel } from '../models/video/video' import { VideoCommentModel } from '../models/video/video-comment' import { sendAbuse } from './activitypub/send/send-flag' import { Notifier } from './notifier' -import { afterCommitIfTransaction } from '@server/helpers/database-utils' export type AcceptResult = { accepted: boolean @@ -38,7 +39,7 @@ export type AcceptResult = { // Can be filtered by plugins function isLocalVideoAccepted (object: { videoBody: VideoCreate - videoFile: Express.Multer.File & { duration?: number } + videoFile: VideoUploadFile user: UserModel }): AcceptResult { return { accepted: true } diff --git a/server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.ts b/server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.ts new file mode 100644 index 000000000..1acea7998 --- /dev/null +++ b/server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.ts @@ -0,0 +1,61 @@ +import * as bluebird from 'bluebird' +import { readdir, remove, stat } from 'fs-extra' +import { logger, loggerTagsFactory } from '@server/helpers/logger' +import { getResumableUploadPath } from '@server/helpers/upload' +import { SCHEDULER_INTERVALS_MS } from '@server/initializers/constants' +import { METAFILE_EXTNAME } from '@uploadx/core' +import { AbstractScheduler } from './abstract-scheduler' + +const lTags = loggerTagsFactory('scheduler', 'resumable-upload', 'cleaner') + +export class RemoveDanglingResumableUploadsScheduler extends AbstractScheduler { + + private static instance: AbstractScheduler + private lastExecutionTimeMs: number + + protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.removeDanglingResumableUploads + + private constructor () { + super() + + this.lastExecutionTimeMs = new Date().getTime() + } + + protected async internalExecute () { + const path = getResumableUploadPath() + const files = await readdir(path) + + const metafiles = files.filter(f => f.endsWith(METAFILE_EXTNAME)) + + if (metafiles.length === 0) return + + logger.debug('Reading resumable video upload folder %s with %d files', path, metafiles.length, lTags()) + + try { + await bluebird.map(metafiles, metafile => { + return this.deleteIfOlderThan(metafile, this.lastExecutionTimeMs) + }, { concurrency: 5 }) + } catch (error) { + logger.error('Failed to handle file during resumable video upload folder cleanup', { error, ...lTags() }) + } finally { + this.lastExecutionTimeMs = new Date().getTime() + } + } + + private async deleteIfOlderThan (metafile: string, olderThan: number) { + const metafilePath = getResumableUploadPath(metafile) + const statResult = await stat(metafilePath) + + // Delete uploads that started since a long time + if (statResult.ctimeMs < olderThan) { + await remove(metafilePath) + + const datafile = metafilePath.replace(new RegExp(`${METAFILE_EXTNAME}$`), '') + await remove(datafile) + } + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} diff --git a/server/lib/video.ts b/server/lib/video.ts index 9469b8178..21e4b7ff2 100644 --- a/server/lib/video.ts +++ b/server/lib/video.ts @@ -1,3 +1,4 @@ +import { UploadFiles } from 'express' import { Transaction } from 'sequelize/types' import { DEFAULT_AUDIO_RESOLUTION, JOB_PRIORITY } from '@server/initializers/constants' import { sequelizeTypescript } from '@server/initializers/database' @@ -32,7 +33,7 @@ function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): Fil async function buildVideoThumbnailsFromReq (options: { video: MVideoThumbnail - files: { [fieldname: string]: Express.Multer.File[] } | Express.Multer.File[] + files: UploadFiles fallback: (type: ThumbnailType) => Promise automaticallyGenerated?: boolean }) { diff --git a/server/middlewares/async.ts b/server/middlewares/async.ts index 3d6e38809..0faa4fb8c 100644 --- a/server/middlewares/async.ts +++ b/server/middlewares/async.ts @@ -3,6 +3,7 @@ import { NextFunction, Request, RequestHandler, Response } from 'express' import { ValidationChain } from 'express-validator' import { ExpressPromiseHandler } from '@server/types/express' import { retryTransactionWrapper } from '../helpers/database-utils' +import { HttpMethod, HttpStatusCode } from '@shared/core-utils' // Syntactic sugar to avoid try/catch in express controllers // Thanks: https://medium.com/@Abazhenov/using-async-await-in-express-with-node-8-b8af872c0016 diff --git a/server/middlewares/validators/videos/videos.ts b/server/middlewares/validators/videos/videos.ts index bb617d77c..d26bcd4a6 100644 --- a/server/middlewares/validators/videos/videos.ts +++ b/server/middlewares/validators/videos/videos.ts @@ -1,9 +1,10 @@ import * as express from 'express' -import { body, param, query, ValidationChain } from 'express-validator' +import { body, header, param, query, ValidationChain } from 'express-validator' +import { getResumableUploadPath } from '@server/helpers/upload' import { isAbleToUploadVideo } from '@server/lib/user' import { getServerActor } from '@server/models/application/application' import { ExpressPromiseHandler } from '@server/types/express' -import { MVideoWithRights } from '@server/types/models' +import { MUserAccountId, MVideoWithRights } from '@server/types/models' import { ServerErrorCode, UserRight, VideoChangeOwnershipStatus, VideoPrivacy } from '../../../../shared' import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes' import { VideoChangeOwnershipAccept } from '../../../../shared/models/videos/video-change-ownership-accept.model' @@ -47,6 +48,7 @@ import { doesVideoExist, doesVideoFileOfVideoExist } from '../../../helpers/middlewares' +import { deleteFileAndCatch } from '../../../helpers/utils' import { getVideoWithAttributes } from '../../../helpers/video' import { CONFIG } from '../../../initializers/config' import { CONSTRAINTS_FIELDS, OVERVIEWS } from '../../../initializers/constants' @@ -57,7 +59,7 @@ import { VideoModel } from '../../../models/video/video' import { authenticatePromiseIfNeeded } from '../../auth' import { areValidationErrors } from '../utils' -const videosAddValidator = getCommonVideoEditAttributes().concat([ +const videosAddLegacyValidator = getCommonVideoEditAttributes().concat([ body('videofile') .custom((value, { req }) => isFileFieldValid(req.files, 'videofile')) .withMessage('Should have a file'), @@ -73,59 +75,122 @@ const videosAddValidator = getCommonVideoEditAttributes().concat([ logger.debug('Checking videosAdd parameters', { parameters: req.body, files: req.files }) if (areValidationErrors(req, res)) return cleanUpReqFiles(req) - if (areErrorsInScheduleUpdate(req, res)) return cleanUpReqFiles(req) - const videoFile: Express.Multer.File & { duration?: number } = req.files['videofile'][0] + const videoFile: express.VideoUploadFile = req.files['videofile'][0] const user = res.locals.oauth.token.User - if (!await doesVideoChannelOfAccountExist(req.body.channelId, user, res)) return cleanUpReqFiles(req) - - if (!isVideoFileMimeTypeValid(req.files)) { - res.status(HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415) - .json({ - error: 'This file is not supported. Please, make sure it is of the following type: ' + - CONSTRAINTS_FIELDS.VIDEOS.EXTNAME.join(', ') - }) - + if (!await commonVideoChecksPass({ req, res, user, videoFileSize: videoFile.size, files: req.files })) { return cleanUpReqFiles(req) } - if (!isVideoFileSizeValid(videoFile.size.toString())) { - res.status(HttpStatusCode.PAYLOAD_TOO_LARGE_413) - .json({ - error: 'This file is too large.' - }) - - return cleanUpReqFiles(req) - } - - if (await isAbleToUploadVideo(user.id, videoFile.size) === false) { - res.status(HttpStatusCode.PAYLOAD_TOO_LARGE_413) - .json({ error: 'The user video quota is exceeded with this video.' }) - - return cleanUpReqFiles(req) - } - - let duration: number - try { - duration = await getDurationFromVideoFile(videoFile.path) + if (!videoFile.duration) await addDurationToVideo(videoFile) } catch (err) { - logger.error('Invalid input file in videosAddValidator.', { err }) + logger.error('Invalid input file in videosAddLegacyValidator.', { err }) res.status(HttpStatusCode.UNPROCESSABLE_ENTITY_422) .json({ error: 'Video file unreadable.' }) return cleanUpReqFiles(req) } - videoFile.duration = duration - if (!await isVideoAccepted(req, res, videoFile)) return cleanUpReqFiles(req) return next() } ]) +/** + * Gets called after the last PUT request + */ +const videosAddResumableValidator = [ + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + const user = res.locals.oauth.token.User + + const body: express.CustomUploadXFile = req.body + const file = { ...body, duration: undefined, path: getResumableUploadPath(body.id), filename: body.metadata.filename } + + const cleanup = () => deleteFileAndCatch(file.path) + + if (!await doesVideoChannelOfAccountExist(file.metadata.channelId, user, res)) return cleanup() + + try { + if (!file.duration) await addDurationToVideo(file) + } catch (err) { + logger.error('Invalid input file in videosAddResumableValidator.', { err }) + res.status(HttpStatusCode.UNPROCESSABLE_ENTITY_422) + .json({ error: 'Video file unreadable.' }) + + return cleanup() + } + + if (!await isVideoAccepted(req, res, file)) return cleanup() + + res.locals.videoFileResumable = file + + return next() + } +] + +/** + * File is created in POST initialisation, and its body is saved as a 'metadata' field is saved by uploadx for later use. + * see https://github.com/kukhariev/node-uploadx/blob/dc9fb4a8ac5a6f481902588e93062f591ec6ef03/packages/core/src/handlers/uploadx.ts + * + * Uploadx doesn't use next() until the upload completes, so this middleware has to be placed before uploadx + * see https://github.com/kukhariev/node-uploadx/blob/dc9fb4a8ac5a6f481902588e93062f591ec6ef03/packages/core/src/handlers/base-handler.ts + * + */ +const videosAddResumableInitValidator = getCommonVideoEditAttributes().concat([ + body('filename') + .isString() + .exists() + .withMessage('Should have a valid filename'), + body('name') + .trim() + .custom(isVideoNameValid) + .withMessage('Should have a valid name'), + body('channelId') + .customSanitizer(toIntOrNull) + .custom(isIdValid).withMessage('Should have correct video channel id'), + + header('x-upload-content-length') + .isNumeric() + .exists() + .withMessage('Should specify the file length'), + header('x-upload-content-type') + .isString() + .exists() + .withMessage('Should specify the file mimetype'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + const videoFileMetadata = { + mimetype: req.headers['x-upload-content-type'] as string, + size: +req.headers['x-upload-content-length'], + originalname: req.body.name + } + + const user = res.locals.oauth.token.User + const cleanup = () => cleanUpReqFiles(req) + + logger.debug('Checking videosAddResumableInitValidator parameters and headers', { + parameters: req.body, + headers: req.headers, + files: req.files + }) + + if (areValidationErrors(req, res)) return cleanup() + + const files = { videofile: [ videoFileMetadata ] } + if (!await commonVideoChecksPass({ req, res, user, videoFileSize: videoFileMetadata.size, files })) return cleanup() + + // multer required unsetting the Content-Type, now we can set it for node-uploadx + req.headers['content-type'] = 'application/json; charset=utf-8' + // place previewfile in metadata so that uploadx saves it in .META + if (req.files['previewfile']) req.body.previewfile = req.files['previewfile'] + + return next() + } +]) + const videosUpdateValidator = getCommonVideoEditAttributes().concat([ param('id').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid id'), body('name') @@ -478,7 +543,10 @@ const commonVideosFiltersValidator = [ // --------------------------------------------------------------------------- export { - videosAddValidator, + videosAddLegacyValidator, + videosAddResumableValidator, + videosAddResumableInitValidator, + videosUpdateValidator, videosGetValidator, videoFileMetadataGetValidator, @@ -515,7 +583,51 @@ function areErrorsInScheduleUpdate (req: express.Request, res: express.Response) return false } -async function isVideoAccepted (req: express.Request, res: express.Response, videoFile: Express.Multer.File & { duration?: number }) { +async function commonVideoChecksPass (parameters: { + req: express.Request + res: express.Response + user: MUserAccountId + videoFileSize: number + files: express.UploadFilesForCheck +}): Promise { + const { req, res, user, videoFileSize, files } = parameters + + if (areErrorsInScheduleUpdate(req, res)) return false + + if (!await doesVideoChannelOfAccountExist(req.body.channelId, user, res)) return false + + if (!isVideoFileMimeTypeValid(files)) { + res.status(HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415) + .json({ + error: 'This file is not supported. Please, make sure it is of the following type: ' + + CONSTRAINTS_FIELDS.VIDEOS.EXTNAME.join(', ') + }) + + return false + } + + if (!isVideoFileSizeValid(videoFileSize.toString())) { + res.status(HttpStatusCode.PAYLOAD_TOO_LARGE_413) + .json({ error: 'This file is too large. It exceeds the maximum file size authorized.' }) + + return false + } + + if (await isAbleToUploadVideo(user.id, videoFileSize) === false) { + res.status(HttpStatusCode.PAYLOAD_TOO_LARGE_413) + .json({ error: 'The user video quota is exceeded with this video.' }) + + return false + } + + return true +} + +export async function isVideoAccepted ( + req: express.Request, + res: express.Response, + videoFile: express.VideoUploadFile +) { // Check we accept this video const acceptParameters = { videoBody: req.body, @@ -538,3 +650,11 @@ async function isVideoAccepted (req: express.Request, res: express.Response, vid return true } + +async function addDurationToVideo (videoFile: { path: string, duration?: number }) { + const duration: number = await getDurationFromVideoFile(videoFile.path) + + if (isNaN(duration)) throw new Error(`Couldn't get video duration`) + + videoFile.duration = duration +} diff --git a/server/tests/api/check-params/index.ts b/server/tests/api/check-params/index.ts index d0b0b9c21..143515838 100644 --- a/server/tests/api/check-params/index.ts +++ b/server/tests/api/check-params/index.ts @@ -13,6 +13,7 @@ import './plugins' import './redundancy' import './search' import './services' +import './upload-quota' import './user-notifications' import './user-subscriptions' import './users' diff --git a/server/tests/api/check-params/upload-quota.ts b/server/tests/api/check-params/upload-quota.ts new file mode 100644 index 000000000..d0fbec415 --- /dev/null +++ b/server/tests/api/check-params/upload-quota.ts @@ -0,0 +1,152 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import 'mocha' +import { expect } from 'chai' +import { HttpStatusCode, randomInt } from '@shared/core-utils' +import { getGoodVideoUrl, getMagnetURI, getMyVideoImports, importVideo } from '@shared/extra-utils/videos/video-imports' +import { MyUser, VideoImport, VideoImportState, VideoPrivacy } from '@shared/models' +import { + cleanupTests, + flushAndRunServer, + getMyUserInformation, + immutableAssign, + registerUser, + ServerInfo, + setAccessTokensToServers, + setDefaultVideoChannel, + updateUser, + uploadVideo, + userLogin, + waitJobs +} from '../../../../shared/extra-utils' + +describe('Test upload quota', function () { + let server: ServerInfo + let rootId: number + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(30000) + + server = await flushAndRunServer(1) + await setAccessTokensToServers([ server ]) + await setDefaultVideoChannel([ server ]) + + const res = await getMyUserInformation(server.url, server.accessToken) + rootId = (res.body as MyUser).id + + await updateUser({ + url: server.url, + userId: rootId, + accessToken: server.accessToken, + videoQuota: 42 + }) + }) + + describe('When having a video quota', function () { + + it('Should fail with a registered user having too many videos with legacy upload', async function () { + this.timeout(30000) + + const user = { username: 'registered' + randomInt(1, 1500), password: 'password' } + await registerUser(server.url, user.username, user.password) + const userAccessToken = await userLogin(server, user) + + const videoAttributes = { fixture: 'video_short2.webm' } + for (let i = 0; i < 5; i++) { + await uploadVideo(server.url, userAccessToken, videoAttributes) + } + + await uploadVideo(server.url, userAccessToken, videoAttributes, HttpStatusCode.PAYLOAD_TOO_LARGE_413, 'legacy') + }) + + it('Should fail with a registered user having too many videos with resumable upload', async function () { + this.timeout(30000) + + const user = { username: 'registered' + randomInt(1, 1500), password: 'password' } + await registerUser(server.url, user.username, user.password) + const userAccessToken = await userLogin(server, user) + + const videoAttributes = { fixture: 'video_short2.webm' } + for (let i = 0; i < 5; i++) { + await uploadVideo(server.url, userAccessToken, videoAttributes) + } + + await uploadVideo(server.url, userAccessToken, videoAttributes, HttpStatusCode.PAYLOAD_TOO_LARGE_413, 'resumable') + }) + + it('Should fail to import with HTTP/Torrent/magnet', async function () { + this.timeout(120000) + + const baseAttributes = { + channelId: server.videoChannel.id, + privacy: VideoPrivacy.PUBLIC + } + await importVideo(server.url, server.accessToken, immutableAssign(baseAttributes, { targetUrl: getGoodVideoUrl() })) + await importVideo(server.url, server.accessToken, immutableAssign(baseAttributes, { magnetUri: getMagnetURI() })) + await importVideo(server.url, server.accessToken, immutableAssign(baseAttributes, { torrentfile: 'video-720p.torrent' as any })) + + await waitJobs([ server ]) + + const res = await getMyVideoImports(server.url, server.accessToken) + + expect(res.body.total).to.equal(3) + const videoImports: VideoImport[] = res.body.data + expect(videoImports).to.have.lengthOf(3) + + for (const videoImport of videoImports) { + expect(videoImport.state.id).to.equal(VideoImportState.FAILED) + expect(videoImport.error).not.to.be.undefined + expect(videoImport.error).to.contain('user video quota is exceeded') + } + }) + }) + + describe('When having a daily video quota', function () { + + it('Should fail with a user having too many videos daily', async function () { + await updateUser({ + url: server.url, + userId: rootId, + accessToken: server.accessToken, + videoQuotaDaily: 42 + }) + + await uploadVideo(server.url, server.accessToken, {}, HttpStatusCode.PAYLOAD_TOO_LARGE_413, 'legacy') + await uploadVideo(server.url, server.accessToken, {}, HttpStatusCode.PAYLOAD_TOO_LARGE_413, 'resumable') + }) + }) + + describe('When having an absolute and daily video quota', function () { + it('Should fail if exceeding total quota', async function () { + await updateUser({ + url: server.url, + userId: rootId, + accessToken: server.accessToken, + videoQuota: 42, + videoQuotaDaily: 1024 * 1024 * 1024 + }) + + await uploadVideo(server.url, server.accessToken, {}, HttpStatusCode.PAYLOAD_TOO_LARGE_413, 'legacy') + await uploadVideo(server.url, server.accessToken, {}, HttpStatusCode.PAYLOAD_TOO_LARGE_413, 'resumable') + }) + + it('Should fail if exceeding daily quota', async function () { + await updateUser({ + url: server.url, + userId: rootId, + accessToken: server.accessToken, + videoQuota: 1024 * 1024 * 1024, + videoQuotaDaily: 42 + }) + + await uploadVideo(server.url, server.accessToken, {}, HttpStatusCode.PAYLOAD_TOO_LARGE_413, 'legacy') + await uploadVideo(server.url, server.accessToken, {}, HttpStatusCode.PAYLOAD_TOO_LARGE_413, 'resumable') + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/server/tests/api/check-params/users.ts b/server/tests/api/check-params/users.ts index 2b03fde2d..dcff0d52b 100644 --- a/server/tests/api/check-params/users.ts +++ b/server/tests/api/check-params/users.ts @@ -1,10 +1,10 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ import 'mocha' -import { expect } from 'chai' import { omit } from 'lodash' import { join } from 'path' -import { User, UserRole, VideoImport, VideoImportState } from '../../../../shared' +import { User, UserRole } from '../../../../shared' +import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes' import { addVideoChannel, blockUser, @@ -29,7 +29,6 @@ import { ServerInfo, setAccessTokensToServers, unblockUser, - updateUser, uploadVideo, userLogin } from '../../../../shared/extra-utils' @@ -39,11 +38,7 @@ import { checkBadSortPagination, checkBadStartPagination } from '../../../../shared/extra-utils/requests/check-api-params' -import { waitJobs } from '../../../../shared/extra-utils/server/jobs' -import { getGoodVideoUrl, getMagnetURI, getMyVideoImports, importVideo } from '../../../../shared/extra-utils/videos/video-imports' import { UserAdminFlag } from '../../../../shared/models/users/user-flag.model' -import { VideoPrivacy } from '../../../../shared/models/videos' -import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes' describe('Test users API validators', function () { const path = '/api/v1/users/' @@ -1093,102 +1088,6 @@ describe('Test users API validators', function () { }) }) - describe('When having a video quota', function () { - it('Should fail with a user having too many videos', async function () { - await updateUser({ - url: server.url, - userId: rootId, - accessToken: server.accessToken, - videoQuota: 42 - }) - - await uploadVideo(server.url, server.accessToken, {}, HttpStatusCode.PAYLOAD_TOO_LARGE_413) - }) - - it('Should fail with a registered user having too many videos', async function () { - this.timeout(30000) - - const user = { - username: 'user3', - password: 'my super password' - } - userAccessToken = await userLogin(server, user) - - const videoAttributes = { fixture: 'video_short2.webm' } - await uploadVideo(server.url, userAccessToken, videoAttributes) - await uploadVideo(server.url, userAccessToken, videoAttributes) - await uploadVideo(server.url, userAccessToken, videoAttributes) - await uploadVideo(server.url, userAccessToken, videoAttributes) - await uploadVideo(server.url, userAccessToken, videoAttributes) - await uploadVideo(server.url, userAccessToken, videoAttributes, HttpStatusCode.PAYLOAD_TOO_LARGE_413) - }) - - it('Should fail to import with HTTP/Torrent/magnet', async function () { - this.timeout(120000) - - const baseAttributes = { - channelId: 1, - privacy: VideoPrivacy.PUBLIC - } - await importVideo(server.url, server.accessToken, immutableAssign(baseAttributes, { targetUrl: getGoodVideoUrl() })) - await importVideo(server.url, server.accessToken, immutableAssign(baseAttributes, { magnetUri: getMagnetURI() })) - await importVideo(server.url, server.accessToken, immutableAssign(baseAttributes, { torrentfile: 'video-720p.torrent' as any })) - - await waitJobs([ server ]) - - const res = await getMyVideoImports(server.url, server.accessToken) - - expect(res.body.total).to.equal(3) - const videoImports: VideoImport[] = res.body.data - expect(videoImports).to.have.lengthOf(3) - - for (const videoImport of videoImports) { - expect(videoImport.state.id).to.equal(VideoImportState.FAILED) - expect(videoImport.error).not.to.be.undefined - expect(videoImport.error).to.contain('user video quota is exceeded') - } - }) - }) - - describe('When having a daily video quota', function () { - it('Should fail with a user having too many videos daily', async function () { - await updateUser({ - url: server.url, - userId: rootId, - accessToken: server.accessToken, - videoQuotaDaily: 42 - }) - - await uploadVideo(server.url, server.accessToken, {}, HttpStatusCode.PAYLOAD_TOO_LARGE_413) - }) - }) - - describe('When having an absolute and daily video quota', function () { - it('Should fail if exceeding total quota', async function () { - await updateUser({ - url: server.url, - userId: rootId, - accessToken: server.accessToken, - videoQuota: 42, - videoQuotaDaily: 1024 * 1024 * 1024 - }) - - await uploadVideo(server.url, server.accessToken, {}, HttpStatusCode.PAYLOAD_TOO_LARGE_413) - }) - - it('Should fail if exceeding daily quota', async function () { - await updateUser({ - url: server.url, - userId: rootId, - accessToken: server.accessToken, - videoQuota: 1024 * 1024 * 1024, - videoQuotaDaily: 42 - }) - - await uploadVideo(server.url, server.accessToken, {}, HttpStatusCode.PAYLOAD_TOO_LARGE_413) - }) - }) - describe('When asking a password reset', function () { const path = '/api/v1/users/ask-reset-password' diff --git a/server/tests/api/check-params/videos.ts b/server/tests/api/check-params/videos.ts index 188d1835c..c970c4a15 100644 --- a/server/tests/api/check-params/videos.ts +++ b/server/tests/api/check-params/videos.ts @@ -1,11 +1,12 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ +import 'mocha' import * as chai from 'chai' import { omit } from 'lodash' -import 'mocha' import { join } from 'path' -import { VideoPrivacy } from '../../../../shared/models/videos/video-privacy.enum' +import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes' import { + checkUploadVideoParam, cleanupTests, createUser, flushAndRunServer, @@ -18,17 +19,18 @@ import { makePutBodyRequest, makeUploadRequest, removeVideo, + root, ServerInfo, setAccessTokensToServers, - userLogin, - root + userLogin } from '../../../../shared/extra-utils' import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '../../../../shared/extra-utils/requests/check-api-params' -import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes' +import { VideoPrivacy } from '../../../../shared/models/videos/video-privacy.enum' +import { randomInt } from '@shared/core-utils' const expect = chai.expect @@ -183,7 +185,7 @@ describe('Test videos API validator', function () { describe('When adding a video', function () { let baseCorrectParams const baseCorrectAttaches = { - videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.webm') + fixture: join(root(), 'server', 'tests', 'fixtures', 'video_short.webm') } before(function () { @@ -206,256 +208,243 @@ describe('Test videos API validator', function () { } }) - it('Should fail with nothing', async function () { - const fields = {} - const attaches = {} - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + function runSuite (mode: 'legacy' | 'resumable') { - it('Should fail without name', async function () { - const fields = omit(baseCorrectParams, 'name') - const attaches = baseCorrectAttaches - - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) - - it('Should fail with a long name', async function () { - const fields = immutableAssign(baseCorrectParams, { name: 'super'.repeat(65) }) - const attaches = baseCorrectAttaches - - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) - - it('Should fail with a bad category', async function () { - const fields = immutableAssign(baseCorrectParams, { category: 125 }) - const attaches = baseCorrectAttaches - - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) - - it('Should fail with a bad licence', async function () { - const fields = immutableAssign(baseCorrectParams, { licence: 125 }) - const attaches = baseCorrectAttaches - - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) - - it('Should fail with a bad language', async function () { - const fields = immutableAssign(baseCorrectParams, { language: 'a'.repeat(15) }) - const attaches = baseCorrectAttaches - - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) - - it('Should fail with a long description', async function () { - const fields = immutableAssign(baseCorrectParams, { description: 'super'.repeat(2500) }) - const attaches = baseCorrectAttaches - - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) - - it('Should fail with a long support text', async function () { - const fields = immutableAssign(baseCorrectParams, { support: 'super'.repeat(201) }) - const attaches = baseCorrectAttaches - - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) - - it('Should fail without a channel', async function () { - const fields = omit(baseCorrectParams, 'channelId') - const attaches = baseCorrectAttaches - - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) - - it('Should fail with a bad channel', async function () { - const fields = immutableAssign(baseCorrectParams, { channelId: 545454 }) - const attaches = baseCorrectAttaches - - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) - - it('Should fail with another user channel', async function () { - const user = { - username: 'fake', - password: 'fake_password' - } - await createUser({ url: server.url, accessToken: server.accessToken, username: user.username, password: user.password }) - - const accessTokenUser = await userLogin(server, user) - const res = await getMyUserInformation(server.url, accessTokenUser) - const customChannelId = res.body.videoChannels[0].id - - const fields = immutableAssign(baseCorrectParams, { channelId: customChannelId }) - const attaches = baseCorrectAttaches - - await makeUploadRequest({ url: server.url, path: path + '/upload', token: userAccessToken, fields, attaches }) - }) - - it('Should fail with too many tags', async function () { - const fields = immutableAssign(baseCorrectParams, { tags: [ 'tag1', 'tag2', 'tag3', 'tag4', 'tag5', 'tag6' ] }) - const attaches = baseCorrectAttaches - - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) - - it('Should fail with a tag length too low', async function () { - const fields = immutableAssign(baseCorrectParams, { tags: [ 'tag1', 't' ] }) - const attaches = baseCorrectAttaches - - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) - - it('Should fail with a tag length too big', async function () { - const fields = immutableAssign(baseCorrectParams, { tags: [ 'tag1', 'my_super_tag_too_long_long_long_long_long_long' ] }) - const attaches = baseCorrectAttaches - - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) - - it('Should fail with a bad schedule update (miss updateAt)', async function () { - const fields = immutableAssign(baseCorrectParams, { 'scheduleUpdate[privacy]': VideoPrivacy.PUBLIC }) - const attaches = baseCorrectAttaches - - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) - - it('Should fail with a bad schedule update (wrong updateAt)', async function () { - const fields = immutableAssign(baseCorrectParams, { - 'scheduleUpdate[privacy]': VideoPrivacy.PUBLIC, - 'scheduleUpdate[updateAt]': 'toto' - }) - const attaches = baseCorrectAttaches - - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) - - it('Should fail with a bad originally published at attribute', async function () { - const fields = immutableAssign(baseCorrectParams, { originallyPublishedAt: 'toto' }) - const attaches = baseCorrectAttaches - - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) - - it('Should fail without an input file', async function () { - const fields = baseCorrectParams - const attaches = {} - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) - - it('Should fail with an incorrect input file', async function () { - const fields = baseCorrectParams - let attaches = { - videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short_fake.webm') - } - await makeUploadRequest({ - url: server.url, - path: path + '/upload', - token: server.accessToken, - fields, - attaches, - statusCodeExpected: HttpStatusCode.UNPROCESSABLE_ENTITY_422 + it('Should fail with nothing', async function () { + const fields = {} + const attaches = {} + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) }) - attaches = { - videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mkv') - } - await makeUploadRequest({ - url: server.url, - path: path + '/upload', - token: server.accessToken, - fields, - attaches, - statusCodeExpected: HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415 - }) - }) - - it('Should fail with an incorrect thumbnail file', async function () { - const fields = baseCorrectParams - const attaches = { - thumbnailfile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4'), - videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') - } - - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) - - it('Should fail with a big thumbnail file', async function () { - const fields = baseCorrectParams - const attaches = { - thumbnailfile: join(root(), 'server', 'tests', 'fixtures', 'preview-big.png'), - videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') - } - - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) - - it('Should fail with an incorrect preview file', async function () { - const fields = baseCorrectParams - const attaches = { - previewfile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4'), - videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') - } - - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) - - it('Should fail with a big preview file', async function () { - const fields = baseCorrectParams - const attaches = { - previewfile: join(root(), 'server', 'tests', 'fixtures', 'preview-big.png'), - videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') - } - - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) - - it('Should succeed with the correct parameters', async function () { - this.timeout(10000) - - const fields = baseCorrectParams - - { + it('Should fail without name', async function () { + const fields = omit(baseCorrectParams, 'name') const attaches = baseCorrectAttaches - await makeUploadRequest({ - url: server.url, - path: path + '/upload', - token: server.accessToken, - fields, - attaches, - statusCodeExpected: HttpStatusCode.OK_200 - }) - } - { - const attaches = immutableAssign(baseCorrectAttaches, { - videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') - }) + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - await makeUploadRequest({ - url: server.url, - path: path + '/upload', - token: server.accessToken, - fields, - attaches, - statusCodeExpected: HttpStatusCode.OK_200 - }) - } + it('Should fail with a long name', async function () { + const fields = immutableAssign(baseCorrectParams, { name: 'super'.repeat(65) }) + const attaches = baseCorrectAttaches - { - const attaches = immutableAssign(baseCorrectAttaches, { - videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.ogv') - }) + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - await makeUploadRequest({ - url: server.url, - path: path + '/upload', - token: server.accessToken, - fields, - attaches, - statusCodeExpected: HttpStatusCode.OK_200 + it('Should fail with a bad category', async function () { + const fields = immutableAssign(baseCorrectParams, { category: 125 }) + const attaches = baseCorrectAttaches + + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) + + it('Should fail with a bad licence', async function () { + const fields = immutableAssign(baseCorrectParams, { licence: 125 }) + const attaches = baseCorrectAttaches + + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) + + it('Should fail with a bad language', async function () { + const fields = immutableAssign(baseCorrectParams, { language: 'a'.repeat(15) }) + const attaches = baseCorrectAttaches + + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) + + it('Should fail with a long description', async function () { + const fields = immutableAssign(baseCorrectParams, { description: 'super'.repeat(2500) }) + const attaches = baseCorrectAttaches + + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) + + it('Should fail with a long support text', async function () { + const fields = immutableAssign(baseCorrectParams, { support: 'super'.repeat(201) }) + const attaches = baseCorrectAttaches + + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) + + it('Should fail without a channel', async function () { + const fields = omit(baseCorrectParams, 'channelId') + const attaches = baseCorrectAttaches + + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) + + it('Should fail with a bad channel', async function () { + const fields = immutableAssign(baseCorrectParams, { channelId: 545454 }) + const attaches = baseCorrectAttaches + + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) + + it('Should fail with another user channel', async function () { + const user = { + username: 'fake' + randomInt(0, 1500), + password: 'fake_password' + } + await createUser({ url: server.url, accessToken: server.accessToken, username: user.username, password: user.password }) + + const accessTokenUser = await userLogin(server, user) + const res = await getMyUserInformation(server.url, accessTokenUser) + const customChannelId = res.body.videoChannels[0].id + + const fields = immutableAssign(baseCorrectParams, { channelId: customChannelId }) + const attaches = baseCorrectAttaches + + await checkUploadVideoParam(server.url, userAccessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) + + it('Should fail with too many tags', async function () { + const fields = immutableAssign(baseCorrectParams, { tags: [ 'tag1', 'tag2', 'tag3', 'tag4', 'tag5', 'tag6' ] }) + const attaches = baseCorrectAttaches + + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) + + it('Should fail with a tag length too low', async function () { + const fields = immutableAssign(baseCorrectParams, { tags: [ 'tag1', 't' ] }) + const attaches = baseCorrectAttaches + + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) + + it('Should fail with a tag length too big', async function () { + const fields = immutableAssign(baseCorrectParams, { tags: [ 'tag1', 'my_super_tag_too_long_long_long_long_long_long' ] }) + const attaches = baseCorrectAttaches + + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) + + it('Should fail with a bad schedule update (miss updateAt)', async function () { + const fields = immutableAssign(baseCorrectParams, { scheduleUpdate: { privacy: VideoPrivacy.PUBLIC } }) + const attaches = baseCorrectAttaches + + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) + + it('Should fail with a bad schedule update (wrong updateAt)', async function () { + const fields = immutableAssign(baseCorrectParams, { + scheduleUpdate: { + privacy: VideoPrivacy.PUBLIC, + updateAt: 'toto' + } }) - } + const attaches = baseCorrectAttaches + + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) + + it('Should fail with a bad originally published at attribute', async function () { + const fields = immutableAssign(baseCorrectParams, { originallyPublishedAt: 'toto' }) + const attaches = baseCorrectAttaches + + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) + + it('Should fail without an input file', async function () { + const fields = baseCorrectParams + const attaches = {} + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) + + it('Should fail with an incorrect input file', async function () { + const fields = baseCorrectParams + let attaches = { fixture: join(root(), 'server', 'tests', 'fixtures', 'video_short_fake.webm') } + + await checkUploadVideoParam( + server.url, + server.accessToken, + { ...fields, ...attaches }, + HttpStatusCode.UNPROCESSABLE_ENTITY_422, + mode + ) + + attaches = { fixture: join(root(), 'server', 'tests', 'fixtures', 'video_short.mkv') } + await checkUploadVideoParam( + server.url, + server.accessToken, + { ...fields, ...attaches }, + HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415, + mode + ) + }) + + it('Should fail with an incorrect thumbnail file', async function () { + const fields = baseCorrectParams + const attaches = { + thumbnailfile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4'), + fixture: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') + } + + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) + + it('Should fail with a big thumbnail file', async function () { + const fields = baseCorrectParams + const attaches = { + thumbnailfile: join(root(), 'server', 'tests', 'fixtures', 'preview-big.png'), + fixture: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') + } + + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) + + it('Should fail with an incorrect preview file', async function () { + const fields = baseCorrectParams + const attaches = { + previewfile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4'), + fixture: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') + } + + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) + + it('Should fail with a big preview file', async function () { + const fields = baseCorrectParams + const attaches = { + previewfile: join(root(), 'server', 'tests', 'fixtures', 'preview-big.png'), + fixture: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') + } + + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) + + it('Should succeed with the correct parameters', async function () { + this.timeout(10000) + + const fields = baseCorrectParams + + { + const attaches = baseCorrectAttaches + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.OK_200, mode) + } + + { + const attaches = immutableAssign(baseCorrectAttaches, { + videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') + }) + + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.OK_200, mode) + } + + { + const attaches = immutableAssign(baseCorrectAttaches, { + videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.ogv') + }) + + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.OK_200, mode) + } + }) + } + + describe('Resumable upload', function () { + runSuite('resumable') + }) + + describe('Legacy upload', function () { + runSuite('legacy') }) }) @@ -678,7 +667,7 @@ describe('Test videos API validator', function () { }) expect(res.body.data).to.be.an('array') - expect(res.body.data.length).to.equal(3) + expect(res.body.data.length).to.equal(6) }) it('Should fail without a correct uuid', async function () { diff --git a/server/tests/api/videos/index.ts b/server/tests/api/videos/index.ts index fc8b447b7..5c07f8926 100644 --- a/server/tests/api/videos/index.ts +++ b/server/tests/api/videos/index.ts @@ -1,5 +1,6 @@ import './audio-only' import './multiple-servers' +import './resumable-upload' import './single-server' import './video-captions' import './video-change-ownership' diff --git a/server/tests/api/videos/multiple-servers.ts b/server/tests/api/videos/multiple-servers.ts index 55e280e9f..41cd814e0 100644 --- a/server/tests/api/videos/multiple-servers.ts +++ b/server/tests/api/videos/multiple-servers.ts @@ -181,7 +181,7 @@ describe('Test multiple servers', function () { thumbnailfile: 'thumbnail.jpg', previewfile: 'preview.jpg' } - await uploadVideo(servers[1].url, userAccessToken, videoAttributes) + await uploadVideo(servers[1].url, userAccessToken, videoAttributes, HttpStatusCode.OK_200, 'resumable') // Transcoding await waitJobs(servers) diff --git a/server/tests/api/videos/resumable-upload.ts b/server/tests/api/videos/resumable-upload.ts new file mode 100644 index 000000000..af9221c43 --- /dev/null +++ b/server/tests/api/videos/resumable-upload.ts @@ -0,0 +1,187 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import 'mocha' +import * as chai from 'chai' +import { pathExists, readdir, stat } from 'fs-extra' +import { join } from 'path' +import { HttpStatusCode } from '@shared/core-utils' +import { + buildAbsoluteFixturePath, + buildServerDirectory, + flushAndRunServer, + getMyUserInformation, + prepareResumableUpload, + sendDebugCommand, + sendResumableChunks, + ServerInfo, + setAccessTokensToServers, + setDefaultVideoChannel, + updateUser +} from '@shared/extra-utils' +import { MyUser, VideoPrivacy } from '@shared/models' + +const expect = chai.expect + +// Most classic resumable upload tests are done in other test suites + +describe('Test resumable upload', function () { + const defaultFixture = 'video_short.mp4' + let server: ServerInfo + let rootId: number + + async function buildSize (fixture: string, size?: number) { + if (size !== undefined) return size + + const baseFixture = buildAbsoluteFixturePath(fixture) + return (await stat(baseFixture)).size + } + + async function prepareUpload (sizeArg?: number) { + const size = await buildSize(defaultFixture, sizeArg) + + const attributes = { + name: 'video', + channelId: server.videoChannel.id, + privacy: VideoPrivacy.PUBLIC, + fixture: defaultFixture + } + + const mimetype = 'video/mp4' + + const res = await prepareResumableUpload({ url: server.url, token: server.accessToken, attributes, size, mimetype }) + + return res.header['location'].split('?')[1] + } + + async function sendChunks (options: { + pathUploadId: string + size?: number + expectedStatus?: HttpStatusCode + contentLength?: number + contentRange?: string + contentRangeBuilder?: (start: number, chunk: any) => string + }) { + const { pathUploadId, expectedStatus, contentLength, contentRangeBuilder } = options + + const size = await buildSize(defaultFixture, options.size) + const absoluteFilePath = buildAbsoluteFixturePath(defaultFixture) + + return sendResumableChunks({ + url: server.url, + token: server.accessToken, + pathUploadId, + videoFilePath: absoluteFilePath, + size, + contentLength, + contentRangeBuilder, + specialStatus: expectedStatus + }) + } + + async function checkFileSize (uploadIdArg: string, expectedSize: number | null) { + const uploadId = uploadIdArg.replace(/^upload_id=/, '') + + const subPath = join('tmp', 'resumable-uploads', uploadId) + const filePath = buildServerDirectory(server, subPath) + const exists = await pathExists(filePath) + + if (expectedSize === null) { + expect(exists).to.be.false + return + } + + expect(exists).to.be.true + + expect((await stat(filePath)).size).to.equal(expectedSize) + } + + async function countResumableUploads () { + const subPath = join('tmp', 'resumable-uploads') + const filePath = buildServerDirectory(server, subPath) + + const files = await readdir(filePath) + return files.length + } + + before(async function () { + this.timeout(30000) + + server = await flushAndRunServer(1) + await setAccessTokensToServers([ server ]) + await setDefaultVideoChannel([ server ]) + + const res = await getMyUserInformation(server.url, server.accessToken) + rootId = (res.body as MyUser).id + + await updateUser({ + url: server.url, + userId: rootId, + accessToken: server.accessToken, + videoQuota: 10_000_000 + }) + }) + + describe('Directory cleaning', function () { + + it('Should correctly delete files after an upload', async function () { + const uploadId = await prepareUpload() + await sendChunks({ pathUploadId: uploadId }) + + expect(await countResumableUploads()).to.equal(0) + }) + + it('Should not delete files after an unfinished upload', async function () { + await prepareUpload() + + expect(await countResumableUploads()).to.equal(2) + }) + + it('Should not delete recent uploads', async function () { + await sendDebugCommand(server.url, server.accessToken, { command: 'remove-dandling-resumable-uploads' }) + + expect(await countResumableUploads()).to.equal(2) + }) + + it('Should delete old uploads', async function () { + await sendDebugCommand(server.url, server.accessToken, { command: 'remove-dandling-resumable-uploads' }) + + expect(await countResumableUploads()).to.equal(0) + }) + }) + + describe('Resumable upload and chunks', function () { + + it('Should accept the same amount of chunks', async function () { + const uploadId = await prepareUpload() + await sendChunks({ pathUploadId: uploadId }) + + await checkFileSize(uploadId, null) + }) + + it('Should not accept more chunks than expected', async function () { + const size = 100 + const uploadId = await prepareUpload(size) + + await sendChunks({ pathUploadId: uploadId, expectedStatus: HttpStatusCode.CONFLICT_409 }) + await checkFileSize(uploadId, 0) + }) + + it('Should not accept more chunks than expected with an invalid content length/content range', async function () { + const uploadId = await prepareUpload(1500) + + await sendChunks({ pathUploadId: uploadId, expectedStatus: HttpStatusCode.BAD_REQUEST_400, contentLength: 1000 }) + await checkFileSize(uploadId, 0) + }) + + it('Should not accept more chunks than expected with an invalid content length', async function () { + const uploadId = await prepareUpload(500) + + const size = 1000 + + const contentRangeBuilder = start => `bytes ${start}-${start + size - 1}/${size}` + await sendChunks({ pathUploadId: uploadId, expectedStatus: HttpStatusCode.BAD_REQUEST_400, contentRangeBuilder, contentLength: size }) + await checkFileSize(uploadId, 0) + }) + }) + +}) diff --git a/server/tests/api/videos/single-server.ts b/server/tests/api/videos/single-server.ts index a79648bf7..1058a1e9c 100644 --- a/server/tests/api/videos/single-server.ts +++ b/server/tests/api/videos/single-server.ts @@ -1,9 +1,9 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ +import 'mocha' import * as chai from 'chai' import { keyBy } from 'lodash' -import 'mocha' -import { VideoPrivacy } from '../../../../shared/models/videos' + import { checkVideoFilesWereRemoved, cleanupTests, @@ -28,430 +28,432 @@ import { viewVideo, wait } from '../../../../shared/extra-utils' +import { VideoPrivacy } from '../../../../shared/models/videos' +import { HttpStatusCode } from '@shared/core-utils' const expect = chai.expect describe('Test a single server', function () { - let server: ServerInfo = null - let videoId = -1 - let videoId2 = -1 - let videoUUID = '' - let videosListBase: any[] = null - const getCheckAttributes = () => ({ - name: 'my super name', - category: 2, - licence: 6, - language: 'zh', - nsfw: true, - description: 'my super description', - support: 'my super support text', - account: { - name: 'root', - host: 'localhost:' + server.port - }, - isLocal: true, - duration: 5, - tags: [ 'tag1', 'tag2', 'tag3' ], - privacy: VideoPrivacy.PUBLIC, - commentsEnabled: true, - downloadEnabled: true, - channel: { - displayName: 'Main root channel', - name: 'root_channel', - description: '', - isLocal: true - }, - fixture: 'video_short.webm', - files: [ - { - resolution: 720, - size: 218910 - } - ] - }) + function runSuite (mode: 'legacy' | 'resumable') { + let server: ServerInfo = null + let videoId = -1 + let videoId2 = -1 + let videoUUID = '' + let videosListBase: any[] = null - const updateCheckAttributes = () => ({ - name: 'my super video updated', - category: 4, - licence: 2, - language: 'ar', - nsfw: false, - description: 'my super description updated', - support: 'my super support text updated', - account: { - name: 'root', - host: 'localhost:' + server.port - }, - isLocal: true, - tags: [ 'tagup1', 'tagup2' ], - privacy: VideoPrivacy.PUBLIC, - duration: 5, - commentsEnabled: false, - downloadEnabled: false, - channel: { - name: 'root_channel', - displayName: 'Main root channel', - description: '', - isLocal: true - }, - fixture: 'video_short3.webm', - files: [ - { - resolution: 720, - size: 292677 - } - ] - }) - - before(async function () { - this.timeout(30000) - - server = await flushAndRunServer(1) - - await setAccessTokensToServers([ server ]) - }) - - it('Should list video categories', async function () { - const res = await getVideoCategories(server.url) - - const categories = res.body - expect(Object.keys(categories)).to.have.length.above(10) - - expect(categories[11]).to.equal('News & Politics') - }) - - it('Should list video licences', async function () { - const res = await getVideoLicences(server.url) - - const licences = res.body - expect(Object.keys(licences)).to.have.length.above(5) - - expect(licences[3]).to.equal('Attribution - No Derivatives') - }) - - it('Should list video languages', async function () { - const res = await getVideoLanguages(server.url) - - const languages = res.body - expect(Object.keys(languages)).to.have.length.above(5) - - expect(languages['ru']).to.equal('Russian') - }) - - it('Should list video privacies', async function () { - const res = await getVideoPrivacies(server.url) - - const privacies = res.body - expect(Object.keys(privacies)).to.have.length.at.least(3) - - expect(privacies[3]).to.equal('Private') - }) - - it('Should not have videos', async function () { - const res = await getVideosList(server.url) - - expect(res.body.total).to.equal(0) - expect(res.body.data).to.be.an('array') - expect(res.body.data.length).to.equal(0) - }) - - it('Should upload the video', async function () { - this.timeout(10000) - - const videoAttributes = { + const getCheckAttributes = () => ({ name: 'my super name', category: 2, - nsfw: true, licence: 6, - tags: [ 'tag1', 'tag2', 'tag3' ] - } - const res = await uploadVideo(server.url, server.accessToken, videoAttributes) - expect(res.body.video).to.not.be.undefined - expect(res.body.video.id).to.equal(1) - expect(res.body.video.uuid).to.have.length.above(5) + language: 'zh', + nsfw: true, + description: 'my super description', + support: 'my super support text', + account: { + name: 'root', + host: 'localhost:' + server.port + }, + isLocal: true, + duration: 5, + tags: [ 'tag1', 'tag2', 'tag3' ], + privacy: VideoPrivacy.PUBLIC, + commentsEnabled: true, + downloadEnabled: true, + channel: { + displayName: 'Main root channel', + name: 'root_channel', + description: '', + isLocal: true + }, + fixture: 'video_short.webm', + files: [ + { + resolution: 720, + size: 218910 + } + ] + }) - videoId = res.body.video.id - videoUUID = res.body.video.uuid - }) - - it('Should get and seed the uploaded video', async function () { - this.timeout(5000) - - const res = await getVideosList(server.url) - - expect(res.body.total).to.equal(1) - expect(res.body.data).to.be.an('array') - expect(res.body.data.length).to.equal(1) - - const video = res.body.data[0] - await completeVideoCheck(server.url, video, getCheckAttributes()) - }) - - it('Should get the video by UUID', async function () { - this.timeout(5000) - - const res = await getVideo(server.url, videoUUID) - - const video = res.body - await completeVideoCheck(server.url, video, getCheckAttributes()) - }) - - it('Should have the views updated', async function () { - this.timeout(20000) - - await viewVideo(server.url, videoId) - await viewVideo(server.url, videoId) - await viewVideo(server.url, videoId) - - await wait(1500) - - await viewVideo(server.url, videoId) - await viewVideo(server.url, videoId) - - await wait(1500) - - await viewVideo(server.url, videoId) - await viewVideo(server.url, videoId) - - // Wait the repeatable job - await wait(8000) - - const res = await getVideo(server.url, videoId) - - const video = res.body - expect(video.views).to.equal(3) - }) - - it('Should remove the video', async function () { - await removeVideo(server.url, server.accessToken, videoId) - - await checkVideoFilesWereRemoved(videoUUID, 1) - }) - - it('Should not have videos', async function () { - const res = await getVideosList(server.url) - - expect(res.body.total).to.equal(0) - expect(res.body.data).to.be.an('array') - expect(res.body.data).to.have.lengthOf(0) - }) - - it('Should upload 6 videos', async function () { - this.timeout(25000) - - const videos = new Set([ - 'video_short.mp4', 'video_short.ogv', 'video_short.webm', - 'video_short1.webm', 'video_short2.webm', 'video_short3.webm' - ]) - - for (const video of videos) { - const videoAttributes = { - name: video + ' name', - description: video + ' description', - category: 2, - licence: 1, - language: 'en', - nsfw: true, - tags: [ 'tag1', 'tag2', 'tag3' ], - fixture: video - } - - await uploadVideo(server.url, server.accessToken, videoAttributes) - } - }) - - it('Should have the correct durations', async function () { - const res = await getVideosList(server.url) - - expect(res.body.total).to.equal(6) - const videos = res.body.data - expect(videos).to.be.an('array') - expect(videos).to.have.lengthOf(6) - - const videosByName = keyBy<{ duration: number }>(videos, 'name') - expect(videosByName['video_short.mp4 name'].duration).to.equal(5) - expect(videosByName['video_short.ogv name'].duration).to.equal(5) - expect(videosByName['video_short.webm name'].duration).to.equal(5) - expect(videosByName['video_short1.webm name'].duration).to.equal(10) - expect(videosByName['video_short2.webm name'].duration).to.equal(5) - expect(videosByName['video_short3.webm name'].duration).to.equal(5) - }) - - it('Should have the correct thumbnails', async function () { - const res = await getVideosList(server.url) - - const videos = res.body.data - // For the next test - videosListBase = videos - - for (const video of videos) { - const videoName = video.name.replace(' name', '') - await testImage(server.url, videoName, video.thumbnailPath) - } - }) - - it('Should list only the two first videos', async function () { - const res = await getVideosListPagination(server.url, 0, 2, 'name') - - const videos = res.body.data - expect(res.body.total).to.equal(6) - expect(videos.length).to.equal(2) - expect(videos[0].name).to.equal(videosListBase[0].name) - expect(videos[1].name).to.equal(videosListBase[1].name) - }) - - it('Should list only the next three videos', async function () { - const res = await getVideosListPagination(server.url, 2, 3, 'name') - - const videos = res.body.data - expect(res.body.total).to.equal(6) - expect(videos.length).to.equal(3) - expect(videos[0].name).to.equal(videosListBase[2].name) - expect(videos[1].name).to.equal(videosListBase[3].name) - expect(videos[2].name).to.equal(videosListBase[4].name) - }) - - it('Should list the last video', async function () { - const res = await getVideosListPagination(server.url, 5, 6, 'name') - - const videos = res.body.data - expect(res.body.total).to.equal(6) - expect(videos.length).to.equal(1) - expect(videos[0].name).to.equal(videosListBase[5].name) - }) - - it('Should not have the total field', async function () { - const res = await getVideosListPagination(server.url, 5, 6, 'name', true) - - const videos = res.body.data - expect(res.body.total).to.not.exist - expect(videos.length).to.equal(1) - expect(videos[0].name).to.equal(videosListBase[5].name) - }) - - it('Should list and sort by name in descending order', async function () { - const res = await getVideosListSort(server.url, '-name') - - const videos = res.body.data - expect(res.body.total).to.equal(6) - expect(videos.length).to.equal(6) - expect(videos[0].name).to.equal('video_short.webm name') - expect(videos[1].name).to.equal('video_short.ogv name') - expect(videos[2].name).to.equal('video_short.mp4 name') - expect(videos[3].name).to.equal('video_short3.webm name') - expect(videos[4].name).to.equal('video_short2.webm name') - expect(videos[5].name).to.equal('video_short1.webm name') - - videoId = videos[3].uuid - videoId2 = videos[5].uuid - }) - - it('Should list and sort by trending in descending order', async function () { - const res = await getVideosListPagination(server.url, 0, 2, '-trending') - - const videos = res.body.data - expect(res.body.total).to.equal(6) - expect(videos.length).to.equal(2) - }) - - it('Should list and sort by hotness in descending order', async function () { - const res = await getVideosListPagination(server.url, 0, 2, '-hot') - - const videos = res.body.data - expect(res.body.total).to.equal(6) - expect(videos.length).to.equal(2) - }) - - it('Should list and sort by best in descending order', async function () { - const res = await getVideosListPagination(server.url, 0, 2, '-best') - - const videos = res.body.data - expect(res.body.total).to.equal(6) - expect(videos.length).to.equal(2) - }) - - it('Should update a video', async function () { - const attributes = { + const updateCheckAttributes = () => ({ name: 'my super video updated', category: 4, licence: 2, language: 'ar', nsfw: false, description: 'my super description updated', + support: 'my super support text updated', + account: { + name: 'root', + host: 'localhost:' + server.port + }, + isLocal: true, + tags: [ 'tagup1', 'tagup2' ], + privacy: VideoPrivacy.PUBLIC, + duration: 5, commentsEnabled: false, downloadEnabled: false, - tags: [ 'tagup1', 'tagup2' ] - } - await updateVideo(server.url, server.accessToken, videoId, attributes) - }) + channel: { + name: 'root_channel', + displayName: 'Main root channel', + description: '', + isLocal: true + }, + fixture: 'video_short3.webm', + files: [ + { + resolution: 720, + size: 292677 + } + ] + }) - it('Should filter by tags and category', async function () { - const res1 = await getVideosWithFilters(server.url, { tagsAllOf: [ 'tagup1', 'tagup2' ], categoryOneOf: [ 4 ] }) - expect(res1.body.total).to.equal(1) - expect(res1.body.data[0].name).to.equal('my super video updated') + before(async function () { + this.timeout(30000) - const res2 = await getVideosWithFilters(server.url, { tagsAllOf: [ 'tagup1', 'tagup2' ], categoryOneOf: [ 3 ] }) - expect(res2.body.total).to.equal(0) - }) + server = await flushAndRunServer(1) - it('Should have the video updated', async function () { - this.timeout(60000) + await setAccessTokensToServers([ server ]) + }) - const res = await getVideo(server.url, videoId) - const video = res.body + it('Should list video categories', async function () { + const res = await getVideoCategories(server.url) - await completeVideoCheck(server.url, video, updateCheckAttributes()) - }) + const categories = res.body + expect(Object.keys(categories)).to.have.length.above(10) - it('Should update only the tags of a video', async function () { - const attributes = { - tags: [ 'supertag', 'tag1', 'tag2' ] - } - await updateVideo(server.url, server.accessToken, videoId, attributes) + expect(categories[11]).to.equal('News & Politics') + }) - const res = await getVideo(server.url, videoId) - const video = res.body + it('Should list video licences', async function () { + const res = await getVideoLicences(server.url) - await completeVideoCheck(server.url, video, Object.assign(updateCheckAttributes(), attributes)) - }) + const licences = res.body + expect(Object.keys(licences)).to.have.length.above(5) - it('Should update only the description of a video', async function () { - const attributes = { - description: 'hello everybody' - } - await updateVideo(server.url, server.accessToken, videoId, attributes) + expect(licences[3]).to.equal('Attribution - No Derivatives') + }) - const res = await getVideo(server.url, videoId) - const video = res.body + it('Should list video languages', async function () { + const res = await getVideoLanguages(server.url) - const expectedAttributes = Object.assign(updateCheckAttributes(), { tags: [ 'supertag', 'tag1', 'tag2' ] }, attributes) - await completeVideoCheck(server.url, video, expectedAttributes) - }) + const languages = res.body + expect(Object.keys(languages)).to.have.length.above(5) - it('Should like a video', async function () { - await rateVideo(server.url, server.accessToken, videoId, 'like') + expect(languages['ru']).to.equal('Russian') + }) - const res = await getVideo(server.url, videoId) - const video = res.body + it('Should list video privacies', async function () { + const res = await getVideoPrivacies(server.url) - expect(video.likes).to.equal(1) - expect(video.dislikes).to.equal(0) - }) + const privacies = res.body + expect(Object.keys(privacies)).to.have.length.at.least(3) - it('Should dislike the same video', async function () { - await rateVideo(server.url, server.accessToken, videoId, 'dislike') + expect(privacies[3]).to.equal('Private') + }) - const res = await getVideo(server.url, videoId) - const video = res.body + it('Should not have videos', async function () { + const res = await getVideosList(server.url) - expect(video.likes).to.equal(0) - expect(video.dislikes).to.equal(1) - }) + expect(res.body.total).to.equal(0) + expect(res.body.data).to.be.an('array') + expect(res.body.data.length).to.equal(0) + }) - it('Should sort by originallyPublishedAt', async function () { - { + it('Should upload the video', async function () { + this.timeout(10000) + const videoAttributes = { + name: 'my super name', + category: 2, + nsfw: true, + licence: 6, + tags: [ 'tag1', 'tag2', 'tag3' ] + } + const res = await uploadVideo(server.url, server.accessToken, videoAttributes, HttpStatusCode.OK_200, mode) + expect(res.body.video).to.not.be.undefined + expect(res.body.video.id).to.equal(1) + expect(res.body.video.uuid).to.have.length.above(5) + + videoId = res.body.video.id + videoUUID = res.body.video.uuid + }) + + it('Should get and seed the uploaded video', async function () { + this.timeout(5000) + + const res = await getVideosList(server.url) + + expect(res.body.total).to.equal(1) + expect(res.body.data).to.be.an('array') + expect(res.body.data.length).to.equal(1) + + const video = res.body.data[0] + await completeVideoCheck(server.url, video, getCheckAttributes()) + }) + + it('Should get the video by UUID', async function () { + this.timeout(5000) + + const res = await getVideo(server.url, videoUUID) + + const video = res.body + await completeVideoCheck(server.url, video, getCheckAttributes()) + }) + + it('Should have the views updated', async function () { + this.timeout(20000) + + await viewVideo(server.url, videoId) + await viewVideo(server.url, videoId) + await viewVideo(server.url, videoId) + + await wait(1500) + + await viewVideo(server.url, videoId) + await viewVideo(server.url, videoId) + + await wait(1500) + + await viewVideo(server.url, videoId) + await viewVideo(server.url, videoId) + + // Wait the repeatable job + await wait(8000) + + const res = await getVideo(server.url, videoId) + + const video = res.body + expect(video.views).to.equal(3) + }) + + it('Should remove the video', async function () { + await removeVideo(server.url, server.accessToken, videoId) + + await checkVideoFilesWereRemoved(videoUUID, 1) + }) + + it('Should not have videos', async function () { + const res = await getVideosList(server.url) + + expect(res.body.total).to.equal(0) + expect(res.body.data).to.be.an('array') + expect(res.body.data).to.have.lengthOf(0) + }) + + it('Should upload 6 videos', async function () { + this.timeout(25000) + + const videos = new Set([ + 'video_short.mp4', 'video_short.ogv', 'video_short.webm', + 'video_short1.webm', 'video_short2.webm', 'video_short3.webm' + ]) + + for (const video of videos) { + const videoAttributes = { + name: video + ' name', + description: video + ' description', + category: 2, + licence: 1, + language: 'en', + nsfw: true, + tags: [ 'tag1', 'tag2', 'tag3' ], + fixture: video + } + + await uploadVideo(server.url, server.accessToken, videoAttributes, HttpStatusCode.OK_200, mode) + } + }) + + it('Should have the correct durations', async function () { + const res = await getVideosList(server.url) + + expect(res.body.total).to.equal(6) + const videos = res.body.data + expect(videos).to.be.an('array') + expect(videos).to.have.lengthOf(6) + + const videosByName = keyBy<{ duration: number }>(videos, 'name') + expect(videosByName['video_short.mp4 name'].duration).to.equal(5) + expect(videosByName['video_short.ogv name'].duration).to.equal(5) + expect(videosByName['video_short.webm name'].duration).to.equal(5) + expect(videosByName['video_short1.webm name'].duration).to.equal(10) + expect(videosByName['video_short2.webm name'].duration).to.equal(5) + expect(videosByName['video_short3.webm name'].duration).to.equal(5) + }) + + it('Should have the correct thumbnails', async function () { + const res = await getVideosList(server.url) + + const videos = res.body.data + // For the next test + videosListBase = videos + + for (const video of videos) { + const videoName = video.name.replace(' name', '') + await testImage(server.url, videoName, video.thumbnailPath) + } + }) + + it('Should list only the two first videos', async function () { + const res = await getVideosListPagination(server.url, 0, 2, 'name') + + const videos = res.body.data + expect(res.body.total).to.equal(6) + expect(videos.length).to.equal(2) + expect(videos[0].name).to.equal(videosListBase[0].name) + expect(videos[1].name).to.equal(videosListBase[1].name) + }) + + it('Should list only the next three videos', async function () { + const res = await getVideosListPagination(server.url, 2, 3, 'name') + + const videos = res.body.data + expect(res.body.total).to.equal(6) + expect(videos.length).to.equal(3) + expect(videos[0].name).to.equal(videosListBase[2].name) + expect(videos[1].name).to.equal(videosListBase[3].name) + expect(videos[2].name).to.equal(videosListBase[4].name) + }) + + it('Should list the last video', async function () { + const res = await getVideosListPagination(server.url, 5, 6, 'name') + + const videos = res.body.data + expect(res.body.total).to.equal(6) + expect(videos.length).to.equal(1) + expect(videos[0].name).to.equal(videosListBase[5].name) + }) + + it('Should not have the total field', async function () { + const res = await getVideosListPagination(server.url, 5, 6, 'name', true) + + const videos = res.body.data + expect(res.body.total).to.not.exist + expect(videos.length).to.equal(1) + expect(videos[0].name).to.equal(videosListBase[5].name) + }) + + it('Should list and sort by name in descending order', async function () { + const res = await getVideosListSort(server.url, '-name') + + const videos = res.body.data + expect(res.body.total).to.equal(6) + expect(videos.length).to.equal(6) + expect(videos[0].name).to.equal('video_short.webm name') + expect(videos[1].name).to.equal('video_short.ogv name') + expect(videos[2].name).to.equal('video_short.mp4 name') + expect(videos[3].name).to.equal('video_short3.webm name') + expect(videos[4].name).to.equal('video_short2.webm name') + expect(videos[5].name).to.equal('video_short1.webm name') + + videoId = videos[3].uuid + videoId2 = videos[5].uuid + }) + + it('Should list and sort by trending in descending order', async function () { + const res = await getVideosListPagination(server.url, 0, 2, '-trending') + + const videos = res.body.data + expect(res.body.total).to.equal(6) + expect(videos.length).to.equal(2) + }) + + it('Should list and sort by hotness in descending order', async function () { + const res = await getVideosListPagination(server.url, 0, 2, '-hot') + + const videos = res.body.data + expect(res.body.total).to.equal(6) + expect(videos.length).to.equal(2) + }) + + it('Should list and sort by best in descending order', async function () { + const res = await getVideosListPagination(server.url, 0, 2, '-best') + + const videos = res.body.data + expect(res.body.total).to.equal(6) + expect(videos.length).to.equal(2) + }) + + it('Should update a video', async function () { + const attributes = { + name: 'my super video updated', + category: 4, + licence: 2, + language: 'ar', + nsfw: false, + description: 'my super description updated', + commentsEnabled: false, + downloadEnabled: false, + tags: [ 'tagup1', 'tagup2' ] + } + await updateVideo(server.url, server.accessToken, videoId, attributes) + }) + + it('Should filter by tags and category', async function () { + const res1 = await getVideosWithFilters(server.url, { tagsAllOf: [ 'tagup1', 'tagup2' ], categoryOneOf: [ 4 ] }) + expect(res1.body.total).to.equal(1) + expect(res1.body.data[0].name).to.equal('my super video updated') + + const res2 = await getVideosWithFilters(server.url, { tagsAllOf: [ 'tagup1', 'tagup2' ], categoryOneOf: [ 3 ] }) + expect(res2.body.total).to.equal(0) + }) + + it('Should have the video updated', async function () { + this.timeout(60000) + + const res = await getVideo(server.url, videoId) + const video = res.body + + await completeVideoCheck(server.url, video, updateCheckAttributes()) + }) + + it('Should update only the tags of a video', async function () { + const attributes = { + tags: [ 'supertag', 'tag1', 'tag2' ] + } + await updateVideo(server.url, server.accessToken, videoId, attributes) + + const res = await getVideo(server.url, videoId) + const video = res.body + + await completeVideoCheck(server.url, video, Object.assign(updateCheckAttributes(), attributes)) + }) + + it('Should update only the description of a video', async function () { + const attributes = { + description: 'hello everybody' + } + await updateVideo(server.url, server.accessToken, videoId, attributes) + + const res = await getVideo(server.url, videoId) + const video = res.body + + const expectedAttributes = Object.assign(updateCheckAttributes(), { tags: [ 'supertag', 'tag1', 'tag2' ] }, attributes) + await completeVideoCheck(server.url, video, expectedAttributes) + }) + + it('Should like a video', async function () { + await rateVideo(server.url, server.accessToken, videoId, 'like') + + const res = await getVideo(server.url, videoId) + const video = res.body + + expect(video.likes).to.equal(1) + expect(video.dislikes).to.equal(0) + }) + + it('Should dislike the same video', async function () { + await rateVideo(server.url, server.accessToken, videoId, 'dislike') + + const res = await getVideo(server.url, videoId) + const video = res.body + + expect(video.likes).to.equal(0) + expect(video.dislikes).to.equal(1) + }) + + it('Should sort by originallyPublishedAt', async function () { { const now = new Date() const attributes = { originallyPublishedAt: now.toISOString() } @@ -483,10 +485,18 @@ describe('Test a single server', function () { expect(names[4]).to.equal('video_short.ogv name') expect(names[5]).to.equal('video_short.mp4 name') } - } + }) + + after(async function () { + await cleanupTests([ server ]) + }) + } + + describe('Legacy upload', function () { + runSuite('legacy') }) - after(async function () { - await cleanupTests([ server ]) + describe('Resumable upload', function () { + runSuite('resumable') }) }) diff --git a/server/tests/api/videos/video-transcoder.ts b/server/tests/api/videos/video-transcoder.ts index 1c99f26df..ea5ffd239 100644 --- a/server/tests/api/videos/video-transcoder.ts +++ b/server/tests/api/videos/video-transcoder.ts @@ -361,106 +361,117 @@ describe('Test video transcoding', function () { describe('Audio upload', function () { - before(async function () { - await updateCustomSubConfig(servers[1].url, servers[1].accessToken, { - transcoding: { - hls: { enabled: true }, - webtorrent: { enabled: true }, - resolutions: { - '0p': false, - '240p': false, - '360p': false, - '480p': false, - '720p': false, - '1080p': false, - '1440p': false, - '2160p': false + function runSuite (mode: 'legacy' | 'resumable') { + + before(async function () { + await updateCustomSubConfig(servers[1].url, servers[1].accessToken, { + transcoding: { + hls: { enabled: true }, + webtorrent: { enabled: true }, + resolutions: { + '0p': false, + '240p': false, + '360p': false, + '480p': false, + '720p': false, + '1080p': false, + '1440p': false, + '2160p': false + } } - } + }) }) - }) - it('Should merge an audio file with the preview file', async function () { - this.timeout(60_000) + it('Should merge an audio file with the preview file', async function () { + this.timeout(60_000) - const videoAttributesArg = { name: 'audio_with_preview', previewfile: 'preview.jpg', fixture: 'sample.ogg' } - await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributesArg) + const videoAttributesArg = { name: 'audio_with_preview', previewfile: 'preview.jpg', fixture: 'sample.ogg' } + await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributesArg, HttpStatusCode.OK_200, mode) - await waitJobs(servers) + await waitJobs(servers) - for (const server of servers) { - const res = await getVideosList(server.url) + for (const server of servers) { + const res = await getVideosList(server.url) - const video = res.body.data.find(v => v.name === 'audio_with_preview') - const res2 = await getVideo(server.url, video.id) - const videoDetails: VideoDetails = res2.body + const video = res.body.data.find(v => v.name === 'audio_with_preview') + const res2 = await getVideo(server.url, video.id) + const videoDetails: VideoDetails = res2.body - expect(videoDetails.files).to.have.lengthOf(1) + expect(videoDetails.files).to.have.lengthOf(1) - await makeGetRequest({ url: server.url, path: videoDetails.thumbnailPath, statusCodeExpected: HttpStatusCode.OK_200 }) - await makeGetRequest({ url: server.url, path: videoDetails.previewPath, statusCodeExpected: HttpStatusCode.OK_200 }) + await makeGetRequest({ url: server.url, path: videoDetails.thumbnailPath, statusCodeExpected: HttpStatusCode.OK_200 }) + await makeGetRequest({ url: server.url, path: videoDetails.previewPath, statusCodeExpected: HttpStatusCode.OK_200 }) - const magnetUri = videoDetails.files[0].magnetUri - expect(magnetUri).to.contain('.mp4') - } - }) - - it('Should upload an audio file and choose a default background image', async function () { - this.timeout(60_000) - - const videoAttributesArg = { name: 'audio_without_preview', fixture: 'sample.ogg' } - await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributesArg) - - await waitJobs(servers) - - for (const server of servers) { - const res = await getVideosList(server.url) - - const video = res.body.data.find(v => v.name === 'audio_without_preview') - const res2 = await getVideo(server.url, video.id) - const videoDetails = res2.body - - expect(videoDetails.files).to.have.lengthOf(1) - - await makeGetRequest({ url: server.url, path: videoDetails.thumbnailPath, statusCodeExpected: HttpStatusCode.OK_200 }) - await makeGetRequest({ url: server.url, path: videoDetails.previewPath, statusCodeExpected: HttpStatusCode.OK_200 }) - - const magnetUri = videoDetails.files[0].magnetUri - expect(magnetUri).to.contain('.mp4') - } - }) - - it('Should upload an audio file and create an audio version only', async function () { - this.timeout(60_000) - - await updateCustomSubConfig(servers[1].url, servers[1].accessToken, { - transcoding: { - hls: { enabled: true }, - webtorrent: { enabled: true }, - resolutions: { - '0p': true, - '240p': false, - '360p': false - } + const magnetUri = videoDetails.files[0].magnetUri + expect(magnetUri).to.contain('.mp4') } }) - const videoAttributesArg = { name: 'audio_with_preview', previewfile: 'preview.jpg', fixture: 'sample.ogg' } - const resVideo = await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributesArg) + it('Should upload an audio file and choose a default background image', async function () { + this.timeout(60_000) - await waitJobs(servers) + const videoAttributesArg = { name: 'audio_without_preview', fixture: 'sample.ogg' } + await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributesArg, HttpStatusCode.OK_200, mode) - for (const server of servers) { - const res2 = await getVideo(server.url, resVideo.body.video.id) - const videoDetails: VideoDetails = res2.body + await waitJobs(servers) - for (const files of [ videoDetails.files, videoDetails.streamingPlaylists[0].files ]) { - expect(files).to.have.lengthOf(2) - expect(files.find(f => f.resolution.id === 0)).to.not.be.undefined + for (const server of servers) { + const res = await getVideosList(server.url) + + const video = res.body.data.find(v => v.name === 'audio_without_preview') + const res2 = await getVideo(server.url, video.id) + const videoDetails = res2.body + + expect(videoDetails.files).to.have.lengthOf(1) + + await makeGetRequest({ url: server.url, path: videoDetails.thumbnailPath, statusCodeExpected: HttpStatusCode.OK_200 }) + await makeGetRequest({ url: server.url, path: videoDetails.previewPath, statusCodeExpected: HttpStatusCode.OK_200 }) + + const magnetUri = videoDetails.files[0].magnetUri + expect(magnetUri).to.contain('.mp4') } - } + }) - await updateConfigForTranscoding(servers[1]) + it('Should upload an audio file and create an audio version only', async function () { + this.timeout(60_000) + + await updateCustomSubConfig(servers[1].url, servers[1].accessToken, { + transcoding: { + hls: { enabled: true }, + webtorrent: { enabled: true }, + resolutions: { + '0p': true, + '240p': false, + '360p': false + } + } + }) + + const videoAttributesArg = { name: 'audio_with_preview', previewfile: 'preview.jpg', fixture: 'sample.ogg' } + const resVideo = await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributesArg, HttpStatusCode.OK_200, mode) + + await waitJobs(servers) + + for (const server of servers) { + const res2 = await getVideo(server.url, resVideo.body.video.id) + const videoDetails: VideoDetails = res2.body + + for (const files of [ videoDetails.files, videoDetails.streamingPlaylists[0].files ]) { + expect(files).to.have.lengthOf(2) + expect(files.find(f => f.resolution.id === 0)).to.not.be.undefined + } + } + + await updateConfigForTranscoding(servers[1]) + }) + } + + describe('Legacy upload', function () { + runSuite('legacy') + }) + + describe('Resumable upload', function () { + runSuite('resumable') }) }) diff --git a/server/typings/express/index.d.ts b/server/typings/express/index.d.ts index cf3e7ae34..55b6e0039 100644 --- a/server/typings/express/index.d.ts +++ b/server/typings/express/index.d.ts @@ -19,6 +19,9 @@ import { MPlugin, MServer, MServerBlocklist } from '@server/types/models/server' import { MVideoImportDefault } from '@server/types/models/video/video-import' import { MVideoPlaylistElement, MVideoPlaylistElementVideoUrlPlaylistPrivacy } from '@server/types/models/video/video-playlist-element' import { MAccountVideoRateAccountVideo } from '@server/types/models/video/video-rate' +import { HttpMethod } from '@shared/core-utils/miscs/http-methods' +import { VideoCreate } from '@shared/models' +import { File as UploadXFile, Metadata } from '@uploadx/core' import { RegisteredPlugin } from '../../lib/plugins/plugin-manager' import { MAccountDefault, @@ -37,86 +40,125 @@ import { MVideoThumbnail, MVideoWithRights } from '../../types/models' - declare module 'express' { export interface Request { query: any + method: HttpMethod } + + // Upload using multer or uploadx middleware + export type MulterOrUploadXFile = UploadXFile | Express.Multer.File + + export type UploadFiles = { + [fieldname: string]: MulterOrUploadXFile[] + } | MulterOrUploadXFile[] + + // Partial object used by some functions to check the file mimetype/extension + export type UploadFileForCheck = { + originalname: string + mimetype: string + } + + export type UploadFilesForCheck = { + [fieldname: string]: UploadFileForCheck[] + } | UploadFileForCheck[] + + // Upload file with a duration added by our middleware + export type VideoUploadFile = Pick & { + duration: number + } + + // Extends Metadata property of UploadX object + export type UploadXFileMetadata = Metadata & VideoCreate & { + previewfile: Express.Multer.File[] + thumbnailfile: Express.Multer.File[] + } + + // Our custom UploadXFile object using our custom metadata + export type CustomUploadXFile = UploadXFile & { metadata: T } + + export type EnhancedUploadXFile = CustomUploadXFile & { + duration: number + path: string + filename: string + } + + // Extends locals property from Response interface Response { - locals: PeerTubeLocals + locals: { + videoAll?: MVideoFullLight + onlyImmutableVideo?: MVideoImmutable + onlyVideo?: MVideoThumbnail + onlyVideoWithRights?: MVideoWithRights + videoId?: MVideoIdThumbnail + + videoLive?: MVideoLive + + videoShare?: MVideoShareActor + + videoFile?: MVideoFile + + videoFileResumable?: EnhancedUploadXFile + + videoImport?: MVideoImportDefault + + videoBlacklist?: MVideoBlacklist + + videoCaption?: MVideoCaptionVideo + + abuse?: MAbuseReporter + abuseMessage?: MAbuseMessage + + videoStreamingPlaylist?: MStreamingPlaylist + + videoChannel?: MChannelBannerAccountDefault + + videoPlaylistFull?: MVideoPlaylistFull + videoPlaylistSummary?: MVideoPlaylistFullSummary + + videoPlaylistElement?: MVideoPlaylistElement + videoPlaylistElementAP?: MVideoPlaylistElementVideoUrlPlaylistPrivacy + + accountVideoRate?: MAccountVideoRateAccountVideo + + videoCommentFull?: MCommentOwnerVideoReply + videoCommentThread?: MComment + + follow?: MActorFollowActorsDefault + subscription?: MActorFollowActorsDefaultSubscription + + nextOwner?: MAccountDefault + videoChangeOwnership?: MVideoChangeOwnershipFull + + account?: MAccountDefault + + actorUrl?: MActorUrl + actorFull?: MActorFull + + user?: MUserDefault + + server?: MServer + + videoRedundancy?: MVideoRedundancyVideo + + accountBlock?: MAccountBlocklist + serverBlock?: MServerBlocklist + + oauth?: { + token: MOAuthTokenUser + } + + signature?: { + actor: MActorAccountChannelId + } + + authenticated?: boolean + + registeredPlugin?: RegisteredPlugin + + externalAuth?: RegisterServerAuthExternalOptions + + plugin?: MPlugin + } } } - -interface PeerTubeLocals { - videoAll?: MVideoFullLight - onlyImmutableVideo?: MVideoImmutable - onlyVideo?: MVideoThumbnail - onlyVideoWithRights?: MVideoWithRights - videoId?: MVideoIdThumbnail - - videoLive?: MVideoLive - - videoShare?: MVideoShareActor - - videoFile?: MVideoFile - - videoImport?: MVideoImportDefault - - videoBlacklist?: MVideoBlacklist - - videoCaption?: MVideoCaptionVideo - - abuse?: MAbuseReporter - abuseMessage?: MAbuseMessage - - videoStreamingPlaylist?: MStreamingPlaylist - - videoChannel?: MChannelBannerAccountDefault - - videoPlaylistFull?: MVideoPlaylistFull - videoPlaylistSummary?: MVideoPlaylistFullSummary - - videoPlaylistElement?: MVideoPlaylistElement - videoPlaylistElementAP?: MVideoPlaylistElementVideoUrlPlaylistPrivacy - - accountVideoRate?: MAccountVideoRateAccountVideo - - videoCommentFull?: MCommentOwnerVideoReply - videoCommentThread?: MComment - - follow?: MActorFollowActorsDefault - subscription?: MActorFollowActorsDefaultSubscription - - nextOwner?: MAccountDefault - videoChangeOwnership?: MVideoChangeOwnershipFull - - account?: MAccountDefault - - actorUrl?: MActorUrl - actorFull?: MActorFull - - user?: MUserDefault - - server?: MServer - - videoRedundancy?: MVideoRedundancyVideo - - accountBlock?: MAccountBlocklist - serverBlock?: MServerBlocklist - - oauth?: { - token: MOAuthTokenUser - } - - signature?: { - actor: MActorAccountChannelId - } - - authenticated?: boolean - - registeredPlugin?: RegisteredPlugin - - externalAuth?: RegisterServerAuthExternalOptions - - plugin?: MPlugin -} diff --git a/shared/core-utils/miscs/http-methods.ts b/shared/core-utils/miscs/http-methods.ts new file mode 100644 index 000000000..1cfa458b9 --- /dev/null +++ b/shared/core-utils/miscs/http-methods.ts @@ -0,0 +1,21 @@ +/** HTTP request method to indicate the desired action to be performed for a given resource. */ +export enum HttpMethod { + /** The CONNECT method establishes a tunnel to the server identified by the target resource. */ + CONNECT = 'CONNECT', + /** The DELETE method deletes the specified resource. */ + DELETE = 'DELETE', + /** The GET method requests a representation of the specified resource. Requests using GET should only retrieve data. */ + GET = 'GET', + /** The HEAD method asks for a response identical to that of a GET request, but without the response body. */ + HEAD = 'HEAD', + /** The OPTIONS method is used to describe the communication options for the target resource. */ + OPTIONS = 'OPTIONS', + /** The PATCH method is used to apply partial modifications to a resource. */ + PATCH = 'PATCH', + /** The POST method is used to submit an entity to the specified resource */ + POST = 'POST', + /** The PUT method replaces all current representations of the target resource with the request payload. */ + PUT = 'PUT', + /** The TRACE method performs a message loop-back test along the path to the target resource. */ + TRACE = 'TRACE' +} diff --git a/shared/core-utils/miscs/index.ts b/shared/core-utils/miscs/index.ts index 898fd4791..251df1de2 100644 --- a/shared/core-utils/miscs/index.ts +++ b/shared/core-utils/miscs/index.ts @@ -2,3 +2,4 @@ export * from './date' export * from './miscs' export * from './types' export * from './http-error-codes' +export * from './http-methods' diff --git a/shared/extra-utils/server/debug.ts b/shared/extra-utils/server/debug.ts index 5cf80a5fb..f196812b7 100644 --- a/shared/extra-utils/server/debug.ts +++ b/shared/extra-utils/server/debug.ts @@ -1,5 +1,6 @@ -import { makeGetRequest } from '../requests/requests' +import { makeGetRequest, makePostBodyRequest } from '../requests/requests' import { HttpStatusCode } from '../../core-utils/miscs/http-error-codes' +import { SendDebugCommand } from '@shared/models' function getDebug (url: string, token: string) { const path = '/api/v1/server/debug' @@ -12,8 +13,21 @@ function getDebug (url: string, token: string) { }) } +function sendDebugCommand (url: string, token: string, body: SendDebugCommand) { + const path = '/api/v1/server/debug/run-command' + + return makePostBodyRequest({ + url, + path, + token, + fields: body, + statusCodeExpected: HttpStatusCode.NO_CONTENT_204 + }) +} + // --------------------------------------------------------------------------- export { - getDebug + getDebug, + sendDebugCommand } diff --git a/shared/extra-utils/server/servers.ts b/shared/extra-utils/server/servers.ts index 779a3cc36..479f08e12 100644 --- a/shared/extra-utils/server/servers.ts +++ b/shared/extra-utils/server/servers.ts @@ -274,7 +274,7 @@ async function reRunServer (server: ServerInfo, configOverride?: any) { } async function checkTmpIsEmpty (server: ServerInfo) { - await checkDirectoryIsEmpty(server, 'tmp', [ 'plugins-global.css', 'hls' ]) + await checkDirectoryIsEmpty(server, 'tmp', [ 'plugins-global.css', 'hls', 'resumable-uploads' ]) if (await pathExists(join('test' + server.internalServerNumber, 'tmp', 'hls'))) { await checkDirectoryIsEmpty(server, 'tmp/hls') diff --git a/shared/extra-utils/videos/video-channels.ts b/shared/extra-utils/videos/video-channels.ts index d0dfb5856..0aab93e52 100644 --- a/shared/extra-utils/videos/video-channels.ts +++ b/shared/extra-utils/videos/video-channels.ts @@ -5,7 +5,7 @@ import { VideoChannelUpdate } from '../../models/videos/channel/video-channel-up import { VideoChannelCreate } from '../../models/videos/channel/video-channel-create.model' import { makeDeleteRequest, makeGetRequest, updateImageRequest } from '../requests/requests' import { ServerInfo } from '../server/servers' -import { User } from '../../models/users/user.model' +import { MyUser, User } from '../../models/users/user.model' import { getMyUserInformation } from '../users/users' import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes' @@ -170,6 +170,12 @@ function setDefaultVideoChannel (servers: ServerInfo[]) { return Promise.all(tasks) } +async function getDefaultVideoChannel (url: string, token: string) { + const res = await getMyUserInformation(url, token) + + return (res.body as MyUser).videoChannels[0].id +} + // --------------------------------------------------------------------------- export { @@ -181,5 +187,6 @@ export { deleteVideoChannel, getVideoChannel, setDefaultVideoChannel, - deleteVideoChannelImage + deleteVideoChannelImage, + getDefaultVideoChannel } diff --git a/shared/extra-utils/videos/videos.ts b/shared/extra-utils/videos/videos.ts index a0143b0ef..e88256ac0 100644 --- a/shared/extra-utils/videos/videos.ts +++ b/shared/extra-utils/videos/videos.ts @@ -1,7 +1,8 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/no-floating-promises */ import { expect } from 'chai' -import { pathExists, readdir, readFile } from 'fs-extra' +import { createReadStream, pathExists, readdir, readFile, stat } from 'fs-extra' +import got, { Response as GotResponse } from 'got/dist/source' import * as parseTorrent from 'parse-torrent' import { extname, join } from 'path' import * as request from 'supertest' @@ -42,6 +43,7 @@ type VideoAttributes = { channelId?: number privacy?: VideoPrivacy fixture?: string + support?: string thumbnailfile?: string previewfile?: string scheduleUpdate?: { @@ -364,8 +366,13 @@ async function checkVideoFilesWereRemoved ( } } -async function uploadVideo (url: string, accessToken: string, videoAttributesArg: VideoAttributes, specialStatus = HttpStatusCode.OK_200) { - const path = '/api/v1/videos/upload' +async function uploadVideo ( + url: string, + accessToken: string, + videoAttributesArg: VideoAttributes, + specialStatus = HttpStatusCode.OK_200, + mode: 'legacy' | 'resumable' = 'legacy' +) { let defaultChannelId = '1' try { @@ -391,61 +398,9 @@ async function uploadVideo (url: string, accessToken: string, videoAttributesArg fixture: 'video_short.webm' }, videoAttributesArg) - const req = request(url) - .post(path) - .set('Accept', 'application/json') - .set('Authorization', 'Bearer ' + accessToken) - .field('name', attributes.name) - .field('nsfw', JSON.stringify(attributes.nsfw)) - .field('commentsEnabled', JSON.stringify(attributes.commentsEnabled)) - .field('downloadEnabled', JSON.stringify(attributes.downloadEnabled)) - .field('waitTranscoding', JSON.stringify(attributes.waitTranscoding)) - .field('privacy', attributes.privacy.toString()) - .field('channelId', attributes.channelId) - - if (attributes.support !== undefined) { - req.field('support', attributes.support) - } - - if (attributes.description !== undefined) { - req.field('description', attributes.description) - } - if (attributes.language !== undefined) { - req.field('language', attributes.language.toString()) - } - if (attributes.category !== undefined) { - req.field('category', attributes.category.toString()) - } - if (attributes.licence !== undefined) { - req.field('licence', attributes.licence.toString()) - } - - const tags = attributes.tags || [] - for (let i = 0; i < tags.length; i++) { - req.field('tags[' + i + ']', attributes.tags[i]) - } - - if (attributes.thumbnailfile !== undefined) { - req.attach('thumbnailfile', buildAbsoluteFixturePath(attributes.thumbnailfile)) - } - if (attributes.previewfile !== undefined) { - req.attach('previewfile', buildAbsoluteFixturePath(attributes.previewfile)) - } - - if (attributes.scheduleUpdate) { - req.field('scheduleUpdate[updateAt]', attributes.scheduleUpdate.updateAt) - - if (attributes.scheduleUpdate.privacy) { - req.field('scheduleUpdate[privacy]', attributes.scheduleUpdate.privacy) - } - } - - if (attributes.originallyPublishedAt !== undefined) { - req.field('originallyPublishedAt', attributes.originallyPublishedAt) - } - - const res = await req.attach('videofile', buildAbsoluteFixturePath(attributes.fixture)) - .expect(specialStatus) + const res = mode === 'legacy' + ? await buildLegacyUpload(url, accessToken, attributes, specialStatus) + : await buildResumeUpload(url, accessToken, attributes, specialStatus) // Wait torrent generation if (specialStatus === HttpStatusCode.OK_200) { @@ -461,6 +416,154 @@ async function uploadVideo (url: string, accessToken: string, videoAttributesArg return res } +function checkUploadVideoParam ( + url: string, + token: string, + attributes: Partial, + specialStatus = HttpStatusCode.OK_200, + mode: 'legacy' | 'resumable' = 'legacy' +) { + return mode === 'legacy' + ? buildLegacyUpload(url, token, attributes, specialStatus) + : buildResumeUpload(url, token, attributes, specialStatus) +} + +async function buildLegacyUpload (url: string, token: string, attributes: VideoAttributes, specialStatus = HttpStatusCode.OK_200) { + const path = '/api/v1/videos/upload' + const req = request(url) + .post(path) + .set('Accept', 'application/json') + .set('Authorization', 'Bearer ' + token) + + buildUploadReq(req, attributes) + + if (attributes.fixture !== undefined) { + req.attach('videofile', buildAbsoluteFixturePath(attributes.fixture)) + } + + return req.expect(specialStatus) +} + +async function buildResumeUpload (url: string, token: string, attributes: VideoAttributes, specialStatus = HttpStatusCode.OK_200) { + let size = 0 + let videoFilePath: string + let mimetype = 'video/mp4' + + if (attributes.fixture) { + videoFilePath = buildAbsoluteFixturePath(attributes.fixture) + size = (await stat(videoFilePath)).size + + if (videoFilePath.endsWith('.mkv')) { + mimetype = 'video/x-matroska' + } else if (videoFilePath.endsWith('.webm')) { + mimetype = 'video/webm' + } + } + + const initializeSessionRes = await prepareResumableUpload({ url, token, attributes, size, mimetype }) + const initStatus = initializeSessionRes.status + + if (videoFilePath && initStatus === HttpStatusCode.CREATED_201) { + const locationHeader = initializeSessionRes.header['location'] + expect(locationHeader).to.not.be.undefined + + const pathUploadId = locationHeader.split('?')[1] + + return sendResumableChunks({ url, token, pathUploadId, videoFilePath, size, specialStatus }) + } + + const expectedInitStatus = specialStatus === HttpStatusCode.OK_200 + ? HttpStatusCode.CREATED_201 + : specialStatus + + expect(initStatus).to.equal(expectedInitStatus) + + return initializeSessionRes +} + +async function prepareResumableUpload (options: { + url: string + token: string + attributes: VideoAttributes + size: number + mimetype: string +}) { + const { url, token, attributes, size, mimetype } = options + + const path = '/api/v1/videos/upload-resumable' + + const req = request(url) + .post(path) + .set('Authorization', 'Bearer ' + token) + .set('X-Upload-Content-Type', mimetype) + .set('X-Upload-Content-Length', size.toString()) + + buildUploadReq(req, attributes) + + if (attributes.fixture) { + req.field('filename', attributes.fixture) + } + + return req +} + +function sendResumableChunks (options: { + url: string + token: string + pathUploadId: string + videoFilePath: string + size: number + specialStatus?: HttpStatusCode + contentLength?: number + contentRangeBuilder?: (start: number, chunk: any) => string +}) { + const { url, token, pathUploadId, videoFilePath, size, specialStatus, contentLength, contentRangeBuilder } = options + + const expectedStatus = specialStatus || HttpStatusCode.OK_200 + + const path = '/api/v1/videos/upload-resumable' + let start = 0 + + const readable = createReadStream(videoFilePath, { highWaterMark: 8 * 1024 }) + return new Promise((resolve, reject) => { + readable.on('data', async function onData (chunk) { + readable.pause() + + const headers = { + 'Authorization': 'Bearer ' + token, + 'Content-Type': 'application/octet-stream', + 'Content-Range': contentRangeBuilder + ? contentRangeBuilder(start, chunk) + : `bytes ${start}-${start + chunk.length - 1}/${size}`, + 'Content-Length': contentLength ? contentLength + '' : chunk.length + '' + } + + const res = await got({ + url, + method: 'put', + headers, + path: path + '?' + pathUploadId, + body: chunk, + responseType: 'json', + throwHttpErrors: false + }) + + start += chunk.length + + if (res.statusCode === expectedStatus) { + return resolve(res) + } + + if (res.statusCode !== HttpStatusCode.PERMANENT_REDIRECT_308) { + readable.off('data', onData) + return reject(new Error('Incorrect transient behaviour sending intermediary chunks')) + } + + readable.resume() + }) + }) +} + function updateVideo ( url: string, accessToken: string, @@ -749,11 +852,13 @@ export { getVideoWithToken, getVideosList, removeAllVideos, + checkUploadVideoParam, getVideosListPagination, getVideosListSort, removeVideo, getVideosListWithToken, uploadVideo, + sendResumableChunks, getVideosWithFilters, uploadRandomVideoOnServers, updateVideo, @@ -767,5 +872,50 @@ export { getMyVideosWithFilter, uploadVideoAndGetId, getLocalIdByUUID, - getVideoIdFromUUID + getVideoIdFromUUID, + prepareResumableUpload +} + +// --------------------------------------------------------------------------- + +function buildUploadReq (req: request.Test, attributes: VideoAttributes) { + + for (const key of [ 'name', 'support', 'channelId', 'description', 'originallyPublishedAt' ]) { + if (attributes[key] !== undefined) { + req.field(key, attributes[key]) + } + } + + for (const key of [ 'nsfw', 'commentsEnabled', 'downloadEnabled', 'waitTranscoding' ]) { + if (attributes[key] !== undefined) { + req.field(key, JSON.stringify(attributes[key])) + } + } + + for (const key of [ 'language', 'privacy', 'category', 'licence' ]) { + if (attributes[key] !== undefined) { + req.field(key, attributes[key].toString()) + } + } + + const tags = attributes.tags || [] + for (let i = 0; i < tags.length; i++) { + req.field('tags[' + i + ']', attributes.tags[i]) + } + + for (const key of [ 'thumbnailfile', 'previewfile' ]) { + if (attributes[key] !== undefined) { + req.attach(key, buildAbsoluteFixturePath(attributes[key])) + } + } + + if (attributes.scheduleUpdate) { + if (attributes.scheduleUpdate.updateAt) { + req.field('scheduleUpdate[updateAt]', attributes.scheduleUpdate.updateAt) + } + + if (attributes.scheduleUpdate.privacy) { + req.field('scheduleUpdate[privacy]', attributes.scheduleUpdate.privacy) + } + } } diff --git a/shared/models/server/debug.model.ts b/shared/models/server/debug.model.ts index 61cba6518..7ceff9137 100644 --- a/shared/models/server/debug.model.ts +++ b/shared/models/server/debug.model.ts @@ -1,3 +1,7 @@ export interface Debug { ip: string } + +export interface SendDebugCommand { + command: 'remove-dandling-resumable-uploads' +} diff --git a/support/doc/api/openapi.yaml b/support/doc/api/openapi.yaml index 90e30545f..050ab82f8 100644 --- a/support/doc/api/openapi.yaml +++ b/support/doc/api/openapi.yaml @@ -125,7 +125,7 @@ tags: Redundancy is part of the inter-server solidarity that PeerTube fosters. Manage the list of instances you wish to help by seeding their videos according to the policy of video selection of your choice. Note that you have a similar functionality - to mirror individual videos, see `Video Mirroring`. + to mirror individual videos, see [video mirroring](#tag/Video-Mirroring). externalDocs: url: https://docs.joinpeertube.org/admin-following-instances?id=instances-redundancy - name: Plugins @@ -139,6 +139,50 @@ tags: - name: Video description: | Operations dealing with listing, uploading, fetching or modifying videos. + - name: Video Upload + description: | + Operations dealing with adding video or audio. PeerTube supports two upload modes, and three import modes. + + ### Upload + + - [_legacy_](#tag/Video-Upload/paths/~1videos~1upload/post), where the video file is sent in a single request + - [_resumable_](#tag/Video-Upload/paths/~1videos~1upload-resumable/post), where the video file is sent in chunks + + You can upload videos more reliably by using the resumable variant. Its protocol lets + you resume an upload operation after a network interruption or other transmission failure, + saving time and bandwidth in the event of network failures. + + Favor using resumable uploads in any of the following cases: + - You are transferring large files + - The likelihood of a network interruption is high + - Uploads are originating from a device with a low-bandwidth or unstable Internet connection, + such as a mobile device + + ### Import + + - _URL_-based: where the URL points to any service supported by [youtube-dl](https://ytdl-org.github.io/youtube-dl/) + - _magnet_-based: where the URI resolves to a BitTorrent ressource containing a single supported video file + - _torrent_-based: where the metainfo file resolves to a BitTorrent ressource containing a single supported video file + + The import function is practical when the desired video/audio is available online. It makes PeerTube + download it for you, saving you as much bandwidth and avoiding any instability or limitation your network might have. + - name: Video Captions + description: Operations dealing with listing, adding and removing closed captions of a video. + - name: Video Channels + description: Operations dealing with the creation, modification and listing of videos within a channel. + - name: Video Comments + description: > + Operations dealing with comments to a video. Comments are organized in threads: adding a + comment in response to the video starts a thread, adding a reply to a comment adds it to + its root comment thread. + - name: Video Blocks + description: Operations dealing with blocking videos (removing them from view and preventing interactions). + - name: Video Rates + description: Like/dislike a video. + - name: Video Playlists + description: Operations dealing with playlists of videos. Playlists are bound to users and/or channels. + - name: Feeds + description: Server syndication feeds - name: Search description: | The search helps to find _videos_ or _channels_ from within the instance and beyond. @@ -148,27 +192,11 @@ tags: Administrators can also enable the use of a remote search system, indexing videos and channels not could be not federated by the instance. - - name: Video Comments - description: > - Operations dealing with comments to a video. Comments are organized in - threads. - - name: Video Playlists - description: > - Operations dealing with playlists of videos. Playlists are bound to users - and/or channels. - - name: Video Channels - description: > - Operations dealing with the creation, modification and listing of videos within a channel. - - name: Video Blocks - description: > - Operations dealing with blocking videos (removing them from view and - preventing interactions). - - name: Video Rates - description: > - Like/dislike a video. - - name: Feeds - description: > - Server syndication feeds + - name: Video Mirroring + description: | + PeerTube instances can mirror videos from one another, and help distribute some videos. + + For importing videos as your own, refer to [video imports](#tag/Video-Upload/paths/~1videos~1imports/post). x-tagGroups: - name: Accounts tags: @@ -181,6 +209,7 @@ x-tagGroups: - name: Videos tags: - Video + - Video Upload - Video Captions - Video Channels - Video Comments @@ -1347,10 +1376,12 @@ paths: /videos/upload: post: summary: Upload a video + description: Uses a single request to upload a video. security: - OAuth2: [] tags: - Video + - Video Upload responses: '200': description: successful operation @@ -1380,80 +1411,7 @@ paths: content: multipart/form-data: schema: - type: object - properties: - videofile: - description: Video file - type: string - format: binary - channelId: - description: Channel id that will contain this video - type: integer - thumbnailfile: - description: Video thumbnail file - type: string - format: binary - previewfile: - description: Video preview file - type: string - format: binary - privacy: - $ref: '#/components/schemas/VideoPrivacySet' - category: - description: Video category - type: integer - example: 4 - licence: - description: Video licence - type: integer - example: 2 - language: - description: Video language - type: string - description: - description: Video description - type: string - waitTranscoding: - description: Whether or not we wait transcoding before publish the video - type: boolean - support: - description: A text tell the audience how to support the video creator - example: Please support my work on ! <3 - type: string - nsfw: - description: Whether or not this video contains sensitive content - type: boolean - name: - description: Video name - type: string - minLength: 3 - maxLength: 120 - tags: - description: Video tags (maximum 5 tags each between 2 and 30 characters) - type: array - minItems: 1 - maxItems: 5 - uniqueItems: true - items: - type: string - minLength: 2 - maxLength: 30 - commentsEnabled: - description: Enable or disable comments for this video - type: boolean - downloadEnabled: - description: Enable or disable downloading for this video - type: boolean - originallyPublishedAt: - description: Date when the content was originally published - type: string - format: date-time - scheduleUpdate: - $ref: '#/components/schemas/VideoScheduledUpdate' - required: - - videofile - - channelId - - name + $ref: '#/components/schemas/VideoUploadRequestLegacy' encoding: videofile: contentType: video/mp4, video/webm, video/ogg, video/avi, video/quicktime, video/x-msvideo, video/x-flv, video/x-matroska, application/octet-stream @@ -1490,6 +1448,164 @@ paths: --form videofile=@"$FILE_PATH" \ --form channelId=$CHANNEL_ID \ --form name="$NAME" + /videos/upload-resumable: + post: + summary: Initialize the resumable upload of a video + description: Uses [a resumable protocol](https://github.com/kukhariev/node-uploadx/blob/master/proto.md) to initialize the upload of a video + security: + - OAuth2: [] + tags: + - Video + - Video Upload + parameters: + - name: X-Upload-Content-Length + in: header + schema: + type: number + example: 2469036 + required: true + description: Number of bytes that will be uploaded in subsequent requests. Set this value to the size of the file you are uploading. + - name: X-Upload-Content-Type + in: header + schema: + type: string + format: mimetype + example: video/mp4 + required: true + description: MIME type of the file that you are uploading. Depending on your instance settings, acceptable values might vary. + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/VideoUploadRequestResumable' + responses: + '200': + description: file already exists, send a [`resume`](https://github.com/kukhariev/node-uploadx/blob/master/proto.md) request instead + '201': + description: created + headers: + Location: + schema: + type: string + format: url + example: /api/v1/videos/upload-resumable?upload_id=471e97554f21dec3b8bb5d4602939c51 + Content-Length: + schema: + type: number + example: 0 + '400': + description: invalid file field, schedule date or parameter + '413': + description: video file too large, due to quota, absolute max file size or concurrent partial upload limit + '415': + description: video type unsupported + put: + summary: Send chunk for the resumable upload of a video + description: Uses [a resumable protocol](https://github.com/kukhariev/node-uploadx/blob/master/proto.md) to continue, pause or resume the upload of a video + security: + - OAuth2: [] + tags: + - Video + - Video Upload + parameters: + - name: upload_id + in: path + required: true + description: | + Created session id to proceed with. If you didn't send chunks in the last 12 hours, it is + not valid anymore and you need to initialize a new upload. + schema: + type: string + - name: Content-Range + in: header + schema: + type: string + example: bytes 0-262143/2469036 + required: true + description: | + Specifies the bytes in the file that the request is uploading. + + For example, a value of `bytes 0-262143/1000000` shows that the request is sending the first + 262144 bytes (256 x 1024) in a 2,469,036 byte file. + - name: Content-Length + in: header + schema: + type: number + example: 262144 + required: true + description: | + Size of the chunk that the request is sending. + + The chunk size __must be a multiple of 256 KB__, and unlike [Google Resumable](https://developers.google.com/youtube/v3/guides/using_resumable_upload_protocol) + doesn't mandate for chunks to have the same size throughout the upload sequence. + + Remember that larger chunks are more efficient. PeerTube's web client uses chunks varying from + 1048576 bytes (~1MB) and increases or reduces size depending on connection health. + requestBody: + content: + application/octet-stream: + schema: + type: string + format: binary + responses: + '200': + description: last chunk received + headers: + Content-Length: + schema: + type: number + content: + application/json: + schema: + $ref: '#/components/schemas/VideoUploadResponse' + '308': + description: resume incomplete + headers: + Range: + schema: + type: string + example: bytes=0-262143 + Content-Length: + schema: + type: number + example: 0 + '403': + description: video didn't pass upload filter + '413': + description: video file too large, due to quota or max body size limit set by the reverse-proxy + '422': + description: video unreadable + delete: + summary: Cancel the resumable upload of a video, deleting any data uploaded so far + description: Uses [a resumable protocol](https://github.com/kukhariev/node-uploadx/blob/master/proto.md) to cancel the upload of a video + security: + - OAuth2: [] + tags: + - Video + - Video Upload + parameters: + - name: upload_id + in: path + required: true + description: | + Created session id to proceed with. If you didn't send chunks in the last 12 hours, it is + not valid anymore and the upload session has already been deleted with its data ;-) + schema: + type: string + - name: Content-Length + in: header + required: true + schema: + type: number + example: 0 + responses: + '204': + description: upload cancelled + headers: + Content-Length: + schema: + type: number + example: 0 /videos/imports: post: summary: Import a video @@ -1498,6 +1614,7 @@ paths: - OAuth2: [] tags: - Video + - Video Upload requestBody: content: multipart/form-data: @@ -1688,7 +1805,7 @@ paths: /videos/live/{id}: get: - summary: Get a live information + summary: Get information about a live security: - OAuth2: [] tags: @@ -1704,7 +1821,7 @@ paths: schema: $ref: '#/components/schemas/LiveVideoResponse' put: - summary: Update a live information + summary: Update information about a live security: - OAuth2: [] tags: @@ -3940,6 +4057,7 @@ components: oneOf: - type: string - type: array + maxItems: 5 items: type: string style: form @@ -4636,7 +4754,7 @@ components: message: type: string minLength: 2 - maxLength: 3000 + maxLength: 3000 byModerator: type: boolean createdAt: @@ -5229,6 +5347,7 @@ components: PredefinedAbuseReasons: description: Reason categories that help triage reports type: array + maxItems: 8 items: type: string enum: @@ -5298,6 +5417,103 @@ components: id: type: integer example: 37 + VideoUploadRequestCommon: + properties: + name: + description: Video name + type: string + channelId: + description: Channel id that will contain this video + type: integer + privacy: + $ref: '#/components/schemas/VideoPrivacySet' + category: + description: Video category + type: integer + example: 4 + licence: + description: Video licence + type: integer + example: 2 + language: + description: Video language + type: string + description: + description: Video description + type: string + waitTranscoding: + description: Whether or not we wait transcoding before publish the video + type: boolean + support: + description: A text tell the audience how to support the video creator + example: Please support my work on ! <3 + type: string + nsfw: + description: Whether or not this video contains sensitive content + type: boolean + tags: + description: Video tags (maximum 5 tags each between 2 and 30 characters) + type: array + minItems: 1 + maxItems: 5 + uniqueItems: true + items: + type: string + minLength: 2 + maxLength: 30 + commentsEnabled: + description: Enable or disable comments for this video + type: boolean + downloadEnabled: + description: Enable or disable downloading for this video + type: boolean + originallyPublishedAt: + description: Date when the content was originally published + type: string + format: date-time + scheduleUpdate: + $ref: '#/components/schemas/VideoScheduledUpdate' + thumbnailfile: + description: Video thumbnail file + type: string + format: binary + previewfile: + description: Video preview file + type: string + format: binary + required: + - channelId + - name + VideoUploadRequestLegacy: + allOf: + - $ref: '#/components/schemas/VideoUploadRequestCommon' + - type: object + required: + - videofile + properties: + videofile: + description: Video file + type: string + format: binary + VideoUploadRequestResumable: + allOf: + - $ref: '#/components/schemas/VideoUploadRequestCommon' + - type: object + required: + - filename + properties: + filename: + description: Video filename including extension + type: string + format: filename + thumbnailfile: + description: Video thumbnail file + type: string + format: binary + previewfile: + description: Video preview file + type: string + format: binary VideoUploadResponse: properties: video: diff --git a/support/nginx/peertube b/support/nginx/peertube index 00ce1d0dc..385acac24 100644 --- a/support/nginx/peertube +++ b/support/nginx/peertube @@ -78,6 +78,13 @@ server { try_files /dev/null @api; } + location = /api/v1/videos/upload-resumable { + client_max_body_size 0; + proxy_request_buffering off; + + try_files /dev/null @api; + } + location = /api/v1/videos/upload { limit_except POST HEAD { deny all; } diff --git a/yarn.lock b/yarn.lock index b61589fa5..adfb8c912 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1061,6 +1061,15 @@ resolved "https://registry.yarnpkg.com/@ungap/promise-all-settled/-/promise-all-settled-1.1.2.tgz#aa58042711d6e3275dd37dc597e5d31e8c290a44" integrity sha512-sL/cEvJWAnClXw0wHk85/2L0G6Sj8UB0Ctc1TEMbKSsmpRosqhwj9gWgFRZSrBr2f9tiXISwNhCPmlfqUqyb9Q== +"@uploadx/core@^4.4.0": + version "4.4.0" + resolved "https://registry.yarnpkg.com/@uploadx/core/-/core-4.4.0.tgz#27ea2b0d28125e81a6bdd65637dc5c7829306cc7" + integrity sha512-dU0oDURYR5RvuAzf63EL9e/fCY4OOQKOs237UTbZDulbRbiyxwEZR+IpRYYr3hKRjjij03EF/Y5j54VGkebAKg== + dependencies: + bytes "^3.1.0" + debug "^4.3.1" + multiparty "^4.2.2" + abbrev@1: version "1.1.1" resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8" @@ -1794,7 +1803,7 @@ busboy@^0.2.11: dicer "0.2.5" readable-stream "1.1.x" -bytes@3.1.0, bytes@^3.0.0: +bytes@3.1.0, bytes@^3.0.0, bytes@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.0.tgz#f6cf7933a360e0588fa9fde85651cdc7f805d1f6" integrity sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg== @@ -4098,6 +4107,17 @@ http-errors@~1.7.2: statuses ">= 1.5.0 < 2" toidentifier "1.0.0" +http-errors@~1.8.0: + version "1.8.0" + resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.8.0.tgz#75d1bbe497e1044f51e4ee9e704a62f28d336507" + integrity sha512-4I8r0C5JDhT5VkvI47QktDW75rNlGVsUf/8hzjCC/wkWI/jdTRmBb9aI7erSG82r1bjKY3F6k28WnsVxB1C73A== + dependencies: + depd "~1.1.2" + inherits "2.0.4" + setprototypeof "1.2.0" + statuses ">= 1.5.0 < 2" + toidentifier "1.0.0" + "http-node@github:feross/http-node#webtorrent": version "1.2.0" resolved "https://codeload.github.com/feross/http-node/tar.gz/342ef8624495343ffd050bd0808b3750cf0e3974" @@ -5567,6 +5587,15 @@ multimatch@^5.0.0: arrify "^2.0.1" minimatch "^3.0.4" +multiparty@^4.2.2: + version "4.2.2" + resolved "https://registry.yarnpkg.com/multiparty/-/multiparty-4.2.2.tgz#bee5fb5737247628d39dab4979ffd6d57bf60ef6" + integrity sha512-NtZLjlvsjcoGrzojtwQwn/Tm90aWJ6XXtPppYF4WmOk/6ncdwMMKggFY2NlRRN9yiCEIVxpOfPWahVEG2HAG8Q== + dependencies: + http-errors "~1.8.0" + safe-buffer "5.2.1" + uid-safe "2.1.5" + multistream@^4.0.1, multistream@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/multistream/-/multistream-4.1.0.tgz#7bf00dfd119556fbc153cff3de4c6d477909f5a8" @@ -6656,6 +6685,11 @@ random-access-storage@^1.1.1: dependencies: inherits "^2.0.3" +random-bytes@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/random-bytes/-/random-bytes-1.0.0.tgz#4f68a1dc0ae58bd3fb95848c30324db75d64360b" + integrity sha1-T2ih3Arli9P7lYSMMDJNt11kNgs= + random-iterate@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/random-iterate/-/random-iterate-1.0.1.tgz#f7d97d92dee6665ec5f6da08c7f963cad4b2ac99" @@ -7040,7 +7074,7 @@ safe-buffer@5.1.2, safe-buffer@~5.1.0, safe-buffer@~5.1.1: resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== -safe-buffer@^5.1.0, safe-buffer@^5.1.1, safe-buffer@~5.2.0: +safe-buffer@5.2.1, safe-buffer@^5.1.0, safe-buffer@^5.1.1, safe-buffer@~5.2.0: version "5.2.1" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== @@ -7186,6 +7220,11 @@ setprototypeof@1.1.1: resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.1.1.tgz#7e95acb24aa92f5885e0abef5ba131330d4ae683" integrity sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw== +setprototypeof@1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.2.0.tgz#66c9a24a73f9fc28cbe66b09fed3d33dcaf1b424" + integrity sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw== + shebang-command@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/shebang-command/-/shebang-command-1.2.0.tgz#44aac65b695b03398968c39f363fee5deafdf1ea" @@ -8139,6 +8178,13 @@ uc.micro@^1.0.1, uc.micro@^1.0.5: resolved "https://registry.yarnpkg.com/uc.micro/-/uc.micro-1.0.6.tgz#9c411a802a409a91fc6cf74081baba34b24499ac" integrity sha512-8Y75pvTYkLJW2hWQHXxoqRgV7qb9B+9vFEtidML+7koHUFapnVJAZ6cKs+Qjz5Aw3aZWHMC6u0wJE3At+nSGwA== +uid-safe@2.1.5: + version "2.1.5" + resolved "https://registry.yarnpkg.com/uid-safe/-/uid-safe-2.1.5.tgz#2b3d5c7240e8fc2e58f8aa269e5ee49c0857bd3a" + integrity sha512-KPHm4VL5dDXKz01UuEd88Df+KzynaohSL9fBh096KWAxSKZQDI2uBrVqtvRM4rwrIrRRKsdLNML/lnaaVSRioA== + dependencies: + random-bytes "~1.0.0" + uint64be@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/uint64be/-/uint64be-2.0.2.tgz#ef4a179752fe8f9ddaa29544ecfc13490031e8e5"