PeerTube/server/helpers/express-utils.ts

139 lines
3.7 KiB
TypeScript
Raw Normal View History

2018-04-24 08:10:54 -05:00
import * as express from 'express'
import * as multer from 'multer'
import { REMOTE_SCHEME } from '../initializers/constants'
2018-04-24 08:10:54 -05:00
import { logger } from './logger'
Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent <par@rigelk.eu> Co-authored-by: Rigel Kent <sendmemail@rigelk.eu> Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-05-10 04:13:41 -05:00
import { deleteFileAndCatch, generateRandomString } from './utils'
2018-08-06 04:45:24 -05:00
import { extname } from 'path'
2018-08-14 08:28:30 -05:00
import { isArray } from './custom-validators/misc'
2019-04-11 04:33:44 -05:00
import { CONFIG } from '../initializers/config'
import { getExtFromMimetype } from './video'
import { HttpStatusCode } from '../../shared/core-utils/miscs/http-error-codes'
2018-04-24 08:10:54 -05:00
2018-12-05 10:27:24 -06:00
function buildNSFWFilter (res?: express.Response, paramNSFW?: string) {
2018-07-20 11:31:49 -05:00
if (paramNSFW === 'true') return true
if (paramNSFW === 'false') return false
if (paramNSFW === 'both') return undefined
2018-07-20 07:35:18 -05:00
2020-01-31 09:56:52 -06:00
if (res?.locals.oauth) {
2019-03-19 04:35:15 -05:00
const user = res.locals.oauth.token.User
2018-09-04 04:19:19 -05:00
2018-07-20 07:35:18 -05:00
// User does not want NSFW videos
2018-09-04 04:19:19 -05:00
if (user.nsfwPolicy === 'do_not_list') return false
// Both
return undefined
2018-04-24 08:10:54 -05:00
}
2018-07-20 07:35:18 -05:00
if (CONFIG.INSTANCE.DEFAULT_NSFW_POLICY === 'do_not_list') return false
// Display all
return null
2018-04-24 08:10:54 -05:00
}
2020-01-31 09:56:52 -06:00
function cleanUpReqFiles (req: { files: { [fieldname: string]: Express.Multer.File[] } | Express.Multer.File[] }) {
2018-08-14 08:28:30 -05:00
const files = req.files
if (!files) return
if (isArray(files)) {
Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent <par@rigelk.eu> Co-authored-by: Rigel Kent <sendmemail@rigelk.eu> Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-05-10 04:13:41 -05:00
(files as Express.Multer.File[]).forEach(f => deleteFileAndCatch(f.path))
2018-08-14 08:28:30 -05:00
return
}
for (const key of Object.keys(files)) {
2020-01-31 09:56:52 -06:00
const file = files[key]
2018-08-14 08:28:30 -05:00
Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent <par@rigelk.eu> Co-authored-by: Rigel Kent <sendmemail@rigelk.eu> Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-05-10 04:13:41 -05:00
if (isArray(file)) file.forEach(f => deleteFileAndCatch(f.path))
else deleteFileAndCatch(file.path)
2018-08-14 08:28:30 -05:00
}
}
2018-04-24 08:10:54 -05:00
function getHostWithPort (host: string) {
const splitted = host.split(':')
// The port was not specified
if (splitted.length === 1) {
if (REMOTE_SCHEME.HTTP === 'https') return host + ':443'
return host + ':80'
}
return host
}
2019-04-11 04:33:44 -05:00
function badRequest (req: express.Request, res: express.Response) {
return res.type('json')
.status(HttpStatusCode.BAD_REQUEST_400)
.end()
2018-04-24 08:10:54 -05:00
}
function createReqFiles (
fieldNames: string[],
mimeTypes: { [id: string]: string | string[] },
2020-01-31 09:56:52 -06:00
destinations: { [fieldName: string]: string }
2018-04-24 08:10:54 -05:00
) {
const storage = multer.diskStorage({
destination: (req, file, cb) => {
2020-01-31 09:56:52 -06:00
cb(null, destinations[file.fieldname])
2018-04-24 08:10:54 -05:00
},
filename: async (req, file, cb) => {
2019-05-21 02:53:03 -05:00
let extension: string
const fileExtension = extname(file.originalname)
const extensionFromMimetype = getExtFromMimetype(mimeTypes, file.mimetype)
2019-05-21 02:53:03 -05:00
// Take the file extension if we don't understand the mime type
if (!extensionFromMimetype) {
2019-05-21 02:53:03 -05:00
extension = fileExtension
} else {
// Take the first available extension for this mimetype
2019-05-21 02:53:03 -05:00
extension = extensionFromMimetype
}
2018-04-24 08:10:54 -05:00
let randomString = ''
try {
randomString = await generateRandomString(16)
} catch (err) {
logger.error('Cannot generate random string for file name.', { err })
randomString = 'fake-random-string'
}
cb(null, randomString + extension)
}
})
2020-01-31 09:56:52 -06:00
const fields: { name: string, maxCount: number }[] = []
2018-04-24 08:10:54 -05:00
for (const fieldName of fieldNames) {
fields.push({
name: fieldName,
maxCount: 1
})
}
return multer({ storage }).fields(fields)
}
2018-08-24 08:36:50 -05:00
function isUserAbleToSearchRemoteURI (res: express.Response) {
2019-03-19 04:35:15 -05:00
const user = res.locals.oauth ? res.locals.oauth.token.User : undefined
2018-08-24 08:36:50 -05:00
return CONFIG.SEARCH.REMOTE_URI.ANONYMOUS === true ||
(CONFIG.SEARCH.REMOTE_URI.USERS === true && user !== undefined)
}
2020-01-08 07:15:16 -06:00
function getCountVideos (req: express.Request) {
return req.query.skipCount !== true
}
2018-04-24 08:10:54 -05:00
// ---------------------------------------------------------------------------
export {
2018-07-20 07:35:18 -05:00
buildNSFWFilter,
2018-04-24 08:10:54 -05:00
getHostWithPort,
2018-08-24 08:36:50 -05:00
isUserAbleToSearchRemoteURI,
2018-04-24 08:10:54 -05:00
badRequest,
2018-08-14 08:28:30 -05:00
createReqFiles,
2020-01-08 07:15:16 -06:00
cleanUpReqFiles,
getCountVideos
2018-04-24 08:10:54 -05:00
}