PeerTube/server/core/lib/video.ts

192 lines
5.6 KiB
TypeScript
Raw Normal View History

Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent <par@rigelk.eu> Co-authored-by: Rigel Kent <sendmemail@rigelk.eu> Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-05-10 04:13:41 -05:00
import { UploadFiles } from 'express'
2022-08-08 08:48:17 -05:00
import memoizee from 'memoizee'
import { Transaction } from 'sequelize'
import {
ManageVideoTorrentPayload,
ThumbnailType,
ThumbnailType_Type,
VideoCreate,
VideoPrivacy,
VideoPrivacyType,
VideoStateType
} from '@peertube/peertube-models'
import { CONFIG } from '@server/initializers/config.js'
import { MEMOIZE_LENGTH, MEMOIZE_TTL } from '@server/initializers/constants.js'
import { TagModel } from '@server/models/video/tag.js'
import { VideoJobInfoModel } from '@server/models/video/video-job-info.js'
import { VideoModel } from '@server/models/video/video.js'
import { FilteredModelAttributes } from '@server/types/index.js'
import { MThumbnail, MVideoFullLight, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models/index.js'
import { CreateJobArgument, JobQueue } from './job-queue/job-queue.js'
import { updateLocalVideoMiniatureFromExisting } from './thumbnail.js'
import { moveFilesIfPrivacyChanged } from './video-privacy.js'
export function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> {
return {
name: videoInfo.name,
remote: false,
category: videoInfo.category,
licence: videoInfo.licence ?? CONFIG.DEFAULTS.PUBLISH.LICENCE,
language: videoInfo.language,
commentsEnabled: videoInfo.commentsEnabled ?? CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED,
downloadEnabled: videoInfo.downloadEnabled ?? CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED,
waitTranscoding: videoInfo.waitTranscoding || false,
nsfw: videoInfo.nsfw || false,
description: videoInfo.description,
support: videoInfo.support,
privacy: videoInfo.privacy || VideoPrivacy.PRIVATE,
2022-07-13 04:58:01 -05:00
channelId,
originallyPublishedAt: videoInfo.originallyPublishedAt
2021-05-12 07:09:04 -05:00
? new Date(videoInfo.originallyPublishedAt)
: null
}
}
export async function buildVideoThumbnailsFromReq (options: {
2020-09-17 03:00:46 -05:00
video: MVideoThumbnail
Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent <par@rigelk.eu> Co-authored-by: Rigel Kent <sendmemail@rigelk.eu> Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-05-10 04:13:41 -05:00
files: UploadFiles
fallback: (type: ThumbnailType_Type) => Promise<MThumbnail>
2020-09-17 03:00:46 -05:00
automaticallyGenerated?: boolean
}) {
const { video, files, fallback, automaticallyGenerated } = options
const promises = [
{
type: ThumbnailType.MINIATURE,
fieldName: 'thumbnailfile'
},
{
type: ThumbnailType.PREVIEW,
fieldName: 'previewfile'
}
].map(p => {
const fields = files?.[p.fieldName]
if (fields) {
2023-06-06 08:59:51 -05:00
return updateLocalVideoMiniatureFromExisting({
2020-09-17 03:00:46 -05:00
inputPath: fields[0].path,
video,
type: p.type,
automaticallyGenerated: automaticallyGenerated || false
})
}
return fallback(p.type)
})
return Promise.all(promises)
}
// ---------------------------------------------------------------------------
export async function setVideoTags (options: {
2020-09-17 03:00:46 -05:00
video: MVideoTag
tags: string[]
transaction?: Transaction
}) {
2021-03-03 04:03:30 -06:00
const { video, tags, transaction } = options
2020-09-17 03:00:46 -05:00
2021-03-03 04:03:30 -06:00
const internalTags = tags || []
const tagInstances = await TagModel.findOrCreateTags(internalTags, transaction)
await video.$set('Tags', tagInstances, { transaction })
video.Tags = tagInstances
2020-09-17 03:00:46 -05:00
}
// ---------------------------------------------------------------------------
export async function buildMoveJob (options: {
video: MVideoUUID
previousVideoState: VideoStateType
type: 'move-to-object-storage' | 'move-to-file-system'
isNewVideo?: boolean // Default true
}) {
const { video, previousVideoState, isNewVideo = true, type } = options
await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingMove')
2022-08-08 08:48:17 -05:00
return {
type,
2022-08-08 08:48:17 -05:00
payload: {
videoUUID: video.uuid,
isNewVideo,
previousVideoState
}
}
}
// ---------------------------------------------------------------------------
export async function getVideoDuration (videoId: number | string) {
2022-06-17 07:34:37 -05:00
const video = await VideoModel.load(videoId)
const duration = video.isLive
? undefined
: video.duration
return { duration, isLive: video.isLive }
}
export const getCachedVideoDuration = memoizee(getVideoDuration, {
2022-06-17 07:34:37 -05:00
promise: true,
max: MEMOIZE_LENGTH.VIDEO_DURATION,
maxAge: MEMOIZE_TTL.VIDEO_DURATION
})
// ---------------------------------------------------------------------------
export async function addVideoJobsAfterUpdate (options: {
video: MVideoFullLight
isNewVideoForFederation: boolean
nameChanged: boolean
oldPrivacy: VideoPrivacyType
}) {
const { video, nameChanged, oldPrivacy, isNewVideoForFederation } = options
const jobs: CreateJobArgument[] = []
const filePathChanged = await moveFilesIfPrivacyChanged(video, oldPrivacy)
if (!video.isLive && (nameChanged || filePathChanged)) {
for (const file of (video.VideoFiles || [])) {
const payload: ManageVideoTorrentPayload = { action: 'update-metadata', videoId: video.id, videoFileId: file.id }
jobs.push({ type: 'manage-video-torrent', payload })
}
const hls = video.getHLSPlaylist()
for (const file of (hls?.VideoFiles || [])) {
const payload: ManageVideoTorrentPayload = { action: 'update-metadata', streamingPlaylistId: hls.id, videoFileId: file.id }
jobs.push({ type: 'manage-video-torrent', payload })
}
}
jobs.push({
type: 'federate-video',
payload: {
videoUUID: video.uuid,
isNewVideoForFederation
}
})
const wasConfidentialVideo = new Set<VideoPrivacyType>([
VideoPrivacy.PRIVATE,
VideoPrivacy.UNLISTED,
VideoPrivacy.INTERNAL
]).has(oldPrivacy)
if (wasConfidentialVideo) {
jobs.push({
type: 'notify',
payload: {
action: 'new-video',
videoUUID: video.uuid
}
})
}
return JobQueue.Instance.createSequentialJobFlow(...jobs)
}