PeerTube/server/initializers/installer.ts

181 lines
5.0 KiB
TypeScript
Raw Normal View History

2022-05-03 01:18:48 -05:00
import { ensureDir, readdir, remove } from 'fs-extra'
2021-08-27 07:32:44 -05:00
import passwordGenerator from 'password-generator'
2021-12-24 03:14:47 -06:00
import { UserRole } from '@shared/models'
2017-12-28 04:16:08 -06:00
import { logger } from '../helpers/logger'
import { buildUser, createApplicationActor, createUserAccountAndChannelAndPlaylist } from '../lib/user'
2017-12-12 10:53:50 -06:00
import { ApplicationModel } from '../models/application/application'
import { OAuthClientModel } from '../models/oauth/oauth-client'
import { applicationExist, clientsExist, usersExist } from './checker-after-init'
2021-08-27 07:32:44 -05:00
import { CONFIG } from './config'
Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent <par@rigelk.eu> Co-authored-by: Rigel Kent <sendmemail@rigelk.eu> Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-05-10 04:13:41 -05:00
import { FILES_CACHE, HLS_STREAMING_PLAYLIST_DIRECTORY, LAST_MIGRATION_VERSION, RESUMABLE_UPLOAD_DIRECTORY } from './constants'
2017-12-12 10:53:50 -06:00
import { sequelizeTypescript } from './database'
async function installApplication () {
2017-11-14 03:57:56 -06:00
try {
2018-11-19 08:21:09 -06:00
await Promise.all([
// Database related
sequelizeTypescript.sync()
.then(() => {
return Promise.all([
createApplicationIfNotExist(),
createOAuthClientIfNotExist(),
createOAuthAdminIfNotExist()
])
}),
// Directories
2019-03-19 04:53:53 -05:00
removeCacheAndTmpDirectories()
2018-11-19 08:21:09 -06:00
.then(() => createDirectoriesIfNotExist())
])
2017-11-14 03:57:56 -06:00
} catch (err) {
2018-03-26 08:54:13 -05:00
logger.error('Cannot install application.', { err })
2018-01-10 10:18:12 -06:00
process.exit(-1)
2017-11-14 03:57:56 -06:00
}
}
// ---------------------------------------------------------------------------
2017-05-15 15:22:03 -05:00
export {
installApplication
}
// ---------------------------------------------------------------------------
2019-03-19 04:53:53 -05:00
function removeCacheAndTmpDirectories () {
2019-03-19 08:23:17 -05:00
const cacheDirectories = Object.keys(FILES_CACHE)
.map(k => FILES_CACHE[k].DIRECTORY)
2017-07-12 04:56:02 -05:00
2017-11-10 10:27:49 -06:00
const tasks: Promise<any>[] = []
2017-07-12 04:56:02 -05:00
// Cache directories
for (const key of Object.keys(cacheDirectories)) {
2017-07-12 04:56:02 -05:00
const dir = cacheDirectories[key]
2022-05-03 01:18:48 -05:00
tasks.push(removeDirectoryOrContent(dir))
}
2017-07-12 04:56:02 -05:00
2022-05-03 01:18:48 -05:00
tasks.push(removeDirectoryOrContent(CONFIG.STORAGE.TMP_DIR))
2019-03-19 04:53:53 -05:00
2017-07-12 04:56:02 -05:00
return Promise.all(tasks)
}
2022-05-03 01:18:48 -05:00
async function removeDirectoryOrContent (dir: string) {
try {
await remove(dir)
} catch (err) {
logger.debug('Cannot remove directory %s. Removing content instead.', dir, { err })
const files = await readdir(dir)
for (const file of files) {
await remove(file)
}
}
}
function createDirectoriesIfNotExist () {
2017-09-04 13:07:54 -05:00
const storage = CONFIG.STORAGE
2019-03-19 08:23:17 -05:00
const cacheDirectories = Object.keys(FILES_CACHE)
.map(k => FILES_CACHE[k].DIRECTORY)
2018-08-27 09:23:34 -05:00
const tasks: Promise<void>[] = []
for (const key of Object.keys(storage)) {
2017-09-04 13:07:54 -05:00
const dir = storage[key]
2018-08-27 09:23:34 -05:00
tasks.push(ensureDir(dir))
}
2017-07-12 04:56:02 -05:00
// Cache directories
for (const key of Object.keys(cacheDirectories)) {
2017-07-12 04:56:02 -05:00
const dir = cacheDirectories[key]
2018-08-27 09:23:34 -05:00
tasks.push(ensureDir(dir))
}
2019-01-29 01:37:25 -06:00
// Playlist directories
tasks.push(ensureDir(HLS_STREAMING_PLAYLIST_DIRECTORY))
2019-01-29 01:37:25 -06:00
Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent <par@rigelk.eu> Co-authored-by: Rigel Kent <sendmemail@rigelk.eu> Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-05-10 04:13:41 -05:00
// Resumable upload directory
tasks.push(ensureDir(RESUMABLE_UPLOAD_DIRECTORY))
return Promise.all(tasks)
}
async function createOAuthClientIfNotExist () {
2017-12-12 10:53:50 -06:00
const exist = await clientsExist()
// Nothing to do, clients already exist
if (exist === true) return undefined
logger.info('Creating a default OAuth Client.')
const id = passwordGenerator(32, false, /[a-z0-9]/)
const secret = passwordGenerator(32, false, /[a-zA-Z0-9]/)
2017-12-12 10:53:50 -06:00
const client = new OAuthClientModel({
clientId: id,
clientSecret: secret,
grants: [ 'password', 'refresh_token' ],
redirectUris: null
})
const createdClient = await client.save()
logger.info('Client id: ' + createdClient.clientId)
logger.info('Client secret: ' + createdClient.clientSecret)
return undefined
}
async function createOAuthAdminIfNotExist () {
2017-12-12 10:53:50 -06:00
const exist = await usersExist()
// Nothing to do, users already exist
if (exist === true) return undefined
logger.info('Creating the administrator.')
const username = 'root'
const role = UserRole.ADMINISTRATOR
const email = CONFIG.ADMIN.EMAIL
let validatePassword = true
let password = ''
2016-12-28 08:49:23 -06:00
// Do not generate a random password for tests
if (process.env.NODE_ENV === 'test') {
password = 'test'
if (process.env.NODE_APP_INSTANCE) {
password += process.env.NODE_APP_INSTANCE
2016-12-28 08:49:23 -06:00
}
// Our password is weak so do not validate it
validatePassword = false
} else if (process.env.PT_INITIAL_ROOT_PASSWORD) {
password = process.env.PT_INITIAL_ROOT_PASSWORD
} else {
2018-03-29 03:58:24 -05:00
password = passwordGenerator(16, true)
}
const user = buildUser({
username,
email,
password,
role,
emailVerified: true,
videoQuota: -1,
videoQuotaDaily: -1
})
await createUserAccountAndChannelAndPlaylist({ userToCreate: user, channelNames: undefined, validateUser: validatePassword })
logger.info('Username: ' + username)
logger.info('User password: ' + password)
2017-11-10 10:27:49 -06:00
}
2017-11-10 10:27:49 -06:00
async function createApplicationIfNotExist () {
2017-12-12 10:53:50 -06:00
const exist = await applicationExist()
2017-11-14 10:31:26 -06:00
// Nothing to do, application already exist
if (exist === true) return undefined
2017-11-10 10:27:49 -06:00
logger.info('Creating application account.')
2017-11-16 11:40:50 -06:00
2017-12-14 10:38:41 -06:00
const application = await ApplicationModel.create({
migrationVersion: LAST_MIGRATION_VERSION
})
2017-11-17 02:12:03 -06:00
2017-12-14 10:38:41 -06:00
return createApplicationActor(application.id)
}