Use async/await in lib and initializers

This commit is contained in:
Chocobozzz 2017-10-25 16:03:33 +02:00
parent eb08047657
commit f5028693a8
No known key found for this signature in database
GPG Key ID: 583A612D890159BE
21 changed files with 721 additions and 751 deletions

View File

@ -1,5 +1,4 @@
import * as crypto from 'crypto' import * as crypto from 'crypto'
import * as Promise from 'bluebird'
import { join } from 'path' import { join } from 'path'
import { import {
@ -41,7 +40,7 @@ function checkSignature (publicKey: string, data: string, hexSignature: string)
return isValid return isValid
} }
function sign (data: string|Object) { async function sign (data: string|Object) {
const sign = crypto.createSign(SIGNATURE_ALGORITHM) const sign = crypto.createSign(SIGNATURE_ALGORITHM)
let dataString: string let dataString: string
@ -52,33 +51,33 @@ function sign (data: string|Object) {
dataString = JSON.stringify(data) dataString = JSON.stringify(data)
} catch (err) { } catch (err) {
logger.error('Cannot sign data.', err) logger.error('Cannot sign data.', err)
return Promise.resolve('') return ''
} }
} }
sign.update(dataString, 'utf8') sign.update(dataString, 'utf8')
return getMyPrivateCert().then(myKey => { const myKey = await getMyPrivateCert()
return sign.sign(myKey, SIGNATURE_ENCODING) return await sign.sign(myKey, SIGNATURE_ENCODING)
})
} }
function comparePassword (plainPassword: string, hashPassword: string) { function comparePassword (plainPassword: string, hashPassword: string) {
return bcryptComparePromise(plainPassword, hashPassword) return bcryptComparePromise(plainPassword, hashPassword)
} }
function createCertsIfNotExist () { async function createCertsIfNotExist () {
return certsExist().then(exist => { const exist = await certsExist()
if (exist === true) { if (exist === true) {
return undefined return undefined
} }
return createCerts() return await createCerts()
})
} }
function cryptPassword (password: string) { async function cryptPassword (password: string) {
return bcryptGenSaltPromise(BCRYPT_SALT_SIZE).then(salt => bcryptHashPromise(password, salt)) const salt = await bcryptGenSaltPromise(BCRYPT_SALT_SIZE)
return await bcryptHashPromise(password, salt)
} }
function getMyPrivateCert () { function getMyPrivateCert () {
@ -105,17 +104,21 @@ export {
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
function certsExist () { async function certsExist () {
const certPath = join(CONFIG.STORAGE.CERT_DIR, PRIVATE_CERT_NAME) const certPath = join(CONFIG.STORAGE.CERT_DIR, PRIVATE_CERT_NAME)
// If there is an error the certificates do not exist // If there is an error the certificates do not exist
return accessPromise(certPath) try {
.then(() => true) await accessPromise(certPath)
.catch(() => false)
return true
} catch {
return false
}
} }
function createCerts () { async function createCerts () {
return certsExist().then(exist => { const exist = await certsExist()
if (exist === true) { if (exist === true) {
const errorMessage = 'Certs already exist.' const errorMessage = 'Certs already exist.'
logger.warning(errorMessage) logger.warning(errorMessage)
@ -129,8 +132,8 @@ function createCerts () {
'out': privateCertPath, 'out': privateCertPath,
'2048': false '2048': false
} }
return opensslExecPromise('genrsa', genRsaOptions)
.then(() => { await opensslExecPromise('genrsa', genRsaOptions)
logger.info('RSA key generated.') logger.info('RSA key generated.')
logger.info('Managing public key...') logger.info('Managing public key...')
@ -140,16 +143,6 @@ function createCerts () {
'pubout': true, 'pubout': true,
'out': publicCertPath 'out': publicCertPath
} }
return opensslExecPromise('rsa', rsaOptions)
.then(() => logger.info('Public key managed.')) await opensslExecPromise('rsa', rsaOptions)
.catch(err => {
logger.error('Cannot create public key on this pod.')
throw err
})
})
.catch(err => {
logger.error('Cannot create private key on this pod.')
throw err
})
})
} }

View File

@ -73,7 +73,7 @@ function makeSecureRequest (params: MakeSecureRequestParams) {
signature signature
} }
// If there are data informations // If there are data information
if (params.data) { if (params.data) {
requestParams.json.data = params.data requestParams.json.data = params.data
} }

View File

@ -8,11 +8,13 @@ import { ResultList } from '../../shared'
import { VideoResolution } from '../../shared/models/videos/video-resolution.enum' import { VideoResolution } from '../../shared/models/videos/video-resolution.enum'
function badRequest (req: express.Request, res: express.Response, next: express.NextFunction) { function badRequest (req: express.Request, res: express.Response, next: express.NextFunction) {
res.type('json').status(400).end() return res.type('json').status(400).end()
} }
function generateRandomString (size: number) { async function generateRandomString (size: number) {
return pseudoRandomBytesPromise(size).then(raw => raw.toString('hex')) const raw = await pseudoRandomBytesPromise(size)
return raw.toString('hex')
} }
interface FormattableToJSON { interface FormattableToJSON {
@ -34,19 +36,19 @@ function getFormattedObjects<U, T extends FormattableToJSON> (objects: T[], obje
return res return res
} }
function isSignupAllowed () { async function isSignupAllowed () {
if (CONFIG.SIGNUP.ENABLED === false) { if (CONFIG.SIGNUP.ENABLED === false) {
return Promise.resolve(false) return false
} }
// No limit and signup is enabled // No limit and signup is enabled
if (CONFIG.SIGNUP.LIMIT === -1) { if (CONFIG.SIGNUP.LIMIT === -1) {
return Promise.resolve(true) return true
} }
return db.User.countTotal().then(totalUsers => { const totalUsers = await db.User.countTotal()
return totalUsers < CONFIG.SIGNUP.LIMIT return totalUsers < CONFIG.SIGNUP.LIMIT
})
} }
function computeResolutionsToTranscode (videoFileHeight: number) { function computeResolutionsToTranscode (videoFileHeight: number) {

View File

@ -37,16 +37,15 @@ function checkMissedConfig () {
// Check the available codecs // Check the available codecs
// We get CONFIG by param to not import it in this file (import orders) // We get CONFIG by param to not import it in this file (import orders)
function checkFFmpeg (CONFIG: { TRANSCODING: { ENABLED: boolean } }) { async function checkFFmpeg (CONFIG: { TRANSCODING: { ENABLED: boolean } }) {
const Ffmpeg = require('fluent-ffmpeg') const Ffmpeg = require('fluent-ffmpeg')
const getAvailableCodecsPromise = promisify0(Ffmpeg.getAvailableCodecs) const getAvailableCodecsPromise = promisify0(Ffmpeg.getAvailableCodecs)
getAvailableCodecsPromise() const codecs = await getAvailableCodecsPromise()
.then(codecs => {
if (CONFIG.TRANSCODING.ENABLED === false) return undefined if (CONFIG.TRANSCODING.ENABLED === false) return undefined
const canEncode = [ 'libx264' ] const canEncode = [ 'libx264' ]
canEncode.forEach(codec => { for (const codec of canEncode) {
if (codecs[codec] === undefined) { if (codecs[codec] === undefined) {
throw new Error('Unknown codec ' + codec + ' in FFmpeg.') throw new Error('Unknown codec ' + codec + ' in FFmpeg.')
} }
@ -54,22 +53,21 @@ function checkFFmpeg (CONFIG: { TRANSCODING: { ENABLED: boolean } }) {
if (codecs[codec].canEncode !== true) { if (codecs[codec].canEncode !== true) {
throw new Error('Unavailable encode codec ' + codec + ' in FFmpeg') throw new Error('Unavailable encode codec ' + codec + ' in FFmpeg')
} }
}) }
})
} }
// We get db by param to not import it in this file (import orders) // We get db by param to not import it in this file (import orders)
function clientsExist (OAuthClient: OAuthClientModel) { async function clientsExist (OAuthClient: OAuthClientModel) {
return OAuthClient.countTotal().then(totalClients => { const totalClients = await OAuthClient.countTotal()
return totalClients !== 0 return totalClients !== 0
})
} }
// We get db by param to not import it in this file (import orders) // We get db by param to not import it in this file (import orders)
function usersExist (User: UserModel) { async function usersExist (User: UserModel) {
return User.countTotal().then(totalUsers => { const totalUsers = await User.countTotal()
return totalUsers !== 0 return totalUsers !== 0
})
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------

View File

@ -2,7 +2,7 @@ import { join } from 'path'
import { flattenDepth } from 'lodash' import { flattenDepth } from 'lodash'
require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string
import * as Sequelize from 'sequelize' import * as Sequelize from 'sequelize'
import * as Promise from 'bluebird' import * as Bluebird from 'bluebird'
import { CONFIG } from './constants' import { CONFIG } from './constants'
// Do not use barrel, we need to load database first // Do not use barrel, we need to load database first
@ -77,26 +77,26 @@ const sequelize = new Sequelize(dbname, username, password, {
database.sequelize = sequelize database.sequelize = sequelize
database.init = (silent: boolean) => { database.init = async (silent: boolean) => {
const modelDirectory = join(__dirname, '..', 'models') const modelDirectory = join(__dirname, '..', 'models')
return getModelFiles(modelDirectory).then(filePaths => { const filePaths = await getModelFiles(modelDirectory)
filePaths.forEach(filePath => {
for (const filePath of filePaths) {
const model = sequelize.import(filePath) const model = sequelize.import(filePath)
database[model['name']] = model database[model['name']] = model
}) }
Object.keys(database).forEach(modelName => { for (const modelName of Object.keys(database)) {
if ('associate' in database[modelName]) { if ('associate' in database[modelName]) {
database[modelName].associate(database) database[modelName].associate(database)
} }
}) }
if (!silent) logger.info('Database %s is ready.', dbname) if (!silent) logger.info('Database %s is ready.', dbname)
return undefined return undefined
})
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@ -107,10 +107,9 @@ export {
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
function getModelFiles (modelDirectory: string) { async function getModelFiles (modelDirectory: string) {
return readdirPromise(modelDirectory) const files = await readdirPromise(modelDirectory)
.then(files => { const directories = files.filter(directory => {
const directories: string[] = files.filter(directory => {
// Find directories // Find directories
if ( if (
directory.endsWith('.js.map') || directory.endsWith('.js.map') ||
@ -121,17 +120,16 @@ function getModelFiles (modelDirectory: string) {
return true return true
}) })
return directories const tasks: Bluebird<any>[] = []
})
.then(directories => {
const tasks = []
// For each directory we read it and append model in the modelFilePaths array // For each directory we read it and append model in the modelFilePaths array
directories.forEach(directory => { for (const directory of directories) {
const modelDirectoryPath = join(modelDirectory, directory) const modelDirectoryPath = join(modelDirectory, directory)
const promise = readdirPromise(modelDirectoryPath).then(files => { const promise = readdirPromise(modelDirectoryPath)
const filteredFiles = files.filter(file => { .then(files => {
const filteredFiles = files
.filter(file => {
if ( if (
file === 'index.js' || file === 'index.ts' || file === 'index.js' || file === 'index.ts' ||
file === 'utils.js' || file === 'utils.ts' || file === 'utils.js' || file === 'utils.ts' ||
@ -140,17 +138,15 @@ function getModelFiles (modelDirectory: string) {
) return false ) return false
return true return true
}).map(file => join(modelDirectoryPath, file)) })
.map(file => join(modelDirectoryPath, file))
return filteredFiles return filteredFiles
}) })
tasks.push(promise) tasks.push(promise)
}) }
return Promise.all(tasks) const filteredFilesArray: string[][] = await Promise.all(tasks)
}) return flattenDepth<string>(filteredFilesArray, 1)
.then((filteredFiles: string[][]) => {
return flattenDepth<string>(filteredFiles, 1)
})
} }

View File

@ -1,4 +1,4 @@
// Constants first, databse in second! // Constants first, database in second!
export * from './constants' export * from './constants'
export * from './database' export * from './database'
export * from './checker' export * from './checker'

View File

@ -1,5 +1,5 @@
import * as passwordGenerator from 'password-generator' import * as passwordGenerator from 'password-generator'
import * as Promise from 'bluebird' import * as Bluebird from 'bluebird'
import { database as db } from './database' import { database as db } from './database'
import { USER_ROLES, CONFIG, LAST_MIGRATION_VERSION, CACHE } from './constants' import { USER_ROLES, CONFIG, LAST_MIGRATION_VERSION, CACHE } from './constants'
@ -7,13 +7,13 @@ import { clientsExist, usersExist } from './checker'
import { logger, createCertsIfNotExist, mkdirpPromise, rimrafPromise } from '../helpers' import { logger, createCertsIfNotExist, mkdirpPromise, rimrafPromise } from '../helpers'
import { createUserAuthorAndChannel } from '../lib' import { createUserAuthorAndChannel } from '../lib'
function installApplication () { async function installApplication () {
return db.sequelize.sync() await db.sequelize.sync()
.then(() => removeCacheDirectories()) await removeCacheDirectories()
.then(() => createDirectoriesIfNotExist()) await createDirectoriesIfNotExist()
.then(() => createCertsIfNotExist()) await createCertsIfNotExist()
.then(() => createOAuthClientIfNotExist()) await createOAuthClientIfNotExist()
.then(() => createOAuthAdminIfNotExist()) await createOAuthAdminIfNotExist()
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@ -27,13 +27,13 @@ export {
function removeCacheDirectories () { function removeCacheDirectories () {
const cacheDirectories = CACHE.DIRECTORIES const cacheDirectories = CACHE.DIRECTORIES
const tasks = [] const tasks: Bluebird<any>[] = []
// Cache directories // Cache directories
Object.keys(cacheDirectories).forEach(key => { for (const key of Object.keys(cacheDirectories)) {
const dir = cacheDirectories[key] const dir = cacheDirectories[key]
tasks.push(rimrafPromise(dir)) tasks.push(rimrafPromise(dir))
}) }
return Promise.all(tasks) return Promise.all(tasks)
} }
@ -43,22 +43,22 @@ function createDirectoriesIfNotExist () {
const cacheDirectories = CACHE.DIRECTORIES const cacheDirectories = CACHE.DIRECTORIES
const tasks = [] const tasks = []
Object.keys(storage).forEach(key => { for (const key of Object.keys(storage)) {
const dir = storage[key] const dir = storage[key]
tasks.push(mkdirpPromise(dir)) tasks.push(mkdirpPromise(dir))
}) }
// Cache directories // Cache directories
Object.keys(cacheDirectories).forEach(key => { for (const key of Object.keys(cacheDirectories)) {
const dir = cacheDirectories[key] const dir = cacheDirectories[key]
tasks.push(mkdirpPromise(dir)) tasks.push(mkdirpPromise(dir))
}) }
return Promise.all(tasks) return Promise.all(tasks)
} }
function createOAuthClientIfNotExist () { async function createOAuthClientIfNotExist () {
return clientsExist(db.OAuthClient).then(exist => { const exist = await clientsExist(db.OAuthClient)
// Nothing to do, clients already exist // Nothing to do, clients already exist
if (exist === true) return undefined if (exist === true) return undefined
@ -73,17 +73,15 @@ function createOAuthClientIfNotExist () {
redirectUris: null redirectUris: null
}) })
return client.save().then(createdClient => { const createdClient = await client.save()
logger.info('Client id: ' + createdClient.clientId) logger.info('Client id: ' + createdClient.clientId)
logger.info('Client secret: ' + createdClient.clientSecret) logger.info('Client secret: ' + createdClient.clientSecret)
return undefined return undefined
})
})
} }
function createOAuthAdminIfNotExist () { async function createOAuthAdminIfNotExist () {
return usersExist(db.User).then(exist => { const exist = await usersExist(db.User)
// Nothing to do, users already exist // Nothing to do, users already exist
if (exist === true) return undefined if (exist === true) return undefined
@ -118,13 +116,10 @@ function createOAuthAdminIfNotExist () {
} }
const user = db.User.build(userData) const user = db.User.build(userData)
return createUserAuthorAndChannel(user, validatePassword) await createUserAuthorAndChannel(user, validatePassword)
.then(({ user }) => {
logger.info('Username: ' + username) logger.info('Username: ' + username)
logger.info('User password: ' + password) logger.info('User password: ' + password)
logger.info('Creating Application table.') logger.info('Creating Application table.')
return db.Application.create({ migrationVersion: LAST_MIGRATION_VERSION }) await db.Application.create({ migrationVersion: LAST_MIGRATION_VERSION })
})
})
} }

View File

@ -1,52 +1,35 @@
import * as path from 'path' import * as path from 'path'
import * as Promise from 'bluebird'
import { database as db } from './database' import { database as db } from './database'
import { LAST_MIGRATION_VERSION } from './constants' import { LAST_MIGRATION_VERSION } from './constants'
import { logger, readdirPromise } from '../helpers' import { logger, readdirPromise } from '../helpers'
function migrate () { async function migrate () {
const p = db.sequelize.getQueryInterface().showAllTables() const tables = await db.sequelize.getQueryInterface().showAllTables()
.then(tables => {
// No tables, we don't need to migrate anything // No tables, we don't need to migrate anything
// The installer will do that // The installer will do that
if (tables.length === 0) throw null if (tables.length === 0) return
})
.then(() => { let actualVersion = await db.Application.loadMigrationVersion()
return db.Application.loadMigrationVersion()
})
.then(actualVersion => {
if (actualVersion === null) { if (actualVersion === null) {
return db.Application.create({ migrationVersion: 0 }).then(() => 0) await db.Application.create({ migrationVersion: 0 })
actualVersion = 0
} }
return actualVersion
})
.then(actualVersion => {
// No need migrations, abort // No need migrations, abort
if (actualVersion >= LAST_MIGRATION_VERSION) throw null if (actualVersion >= LAST_MIGRATION_VERSION) return
return actualVersion
})
.then(actualVersion => {
// If there are a new migration scripts // If there are a new migration scripts
logger.info('Begin migrations.') logger.info('Begin migrations.')
return getMigrationScripts().then(migrationScripts => ({ actualVersion, migrationScripts })) const migrationScripts = await getMigrationScripts()
})
.then(({ actualVersion, migrationScripts }) => { for (const migrationScript of migrationScripts) {
return Promise.each(migrationScripts, entity => executeMigration(actualVersion, entity)) await executeMigration(actualVersion, migrationScript)
}) }
.then(() => {
logger.info('Migrations finished. New migration version schema: %s', LAST_MIGRATION_VERSION) logger.info('Migrations finished. New migration version schema: %s', LAST_MIGRATION_VERSION)
})
.catch(err => {
if (err === null) return undefined
throw err
})
return p
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@ -57,8 +40,8 @@ export {
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
function getMigrationScripts () { async function getMigrationScripts () {
return readdirPromise(path.join(__dirname, 'migrations')).then(files => { const files = await readdirPromise(path.join(__dirname, 'migrations'))
const filesToMigrate: { const filesToMigrate: {
version: string, version: string,
script: string script: string
@ -76,10 +59,9 @@ function getMigrationScripts () {
}) })
return filesToMigrate return filesToMigrate
})
} }
function executeMigration (actualVersion: number, entity: { version: string, script: string }) { async function executeMigration (actualVersion: number, entity: { version: string, script: string }) {
const versionScript = parseInt(entity.version, 10) const versionScript = parseInt(entity.version, 10)
// Do not execute old migration scripts // Do not execute old migration scripts
@ -91,7 +73,7 @@ function executeMigration (actualVersion: number, entity: { version: string, scr
const migrationScript = require(path.join(__dirname, 'migrations', migrationScriptName)) const migrationScript = require(path.join(__dirname, 'migrations', migrationScriptName))
return db.sequelize.transaction(t => { await db.sequelize.transaction(async t => {
const options = { const options = {
transaction: t, transaction: t,
queryInterface: db.sequelize.getQueryInterface(), queryInterface: db.sequelize.getQueryInterface(),
@ -99,10 +81,9 @@ function executeMigration (actualVersion: number, entity: { version: string, scr
db db
} }
return migrationScript.up(options) await migrationScript.up(options)
.then(() => {
// Update the new migration version // Update the new migration version
return db.Application.updateMigrationVersion(versionScript, t) await db.Application.updateMigrationVersion(versionScript, t)
})
}) })
} }

View File

@ -1,7 +1,6 @@
import * as asyncLRU from 'async-lru' import * as asyncLRU from 'async-lru'
import { join } from 'path' import { join } from 'path'
import { createWriteStream } from 'fs' import { createWriteStream } from 'fs'
import * as Promise from 'bluebird'
import { database as db, CONFIG, CACHE } from '../../initializers' import { database as db, CONFIG, CACHE } from '../../initializers'
import { logger, unlinkPromise } from '../../helpers' import { logger, unlinkPromise } from '../../helpers'
@ -43,15 +42,15 @@ class VideosPreviewCache {
}) })
} }
private loadPreviews (key: string) { private async loadPreviews (key: string) {
return db.Video.loadByUUIDAndPopulateAuthorAndPodAndTags(key) const video = await db.Video.loadByUUIDAndPopulateAuthorAndPodAndTags(key)
.then(video => {
if (!video) return undefined if (!video) return undefined
if (video.isOwned()) return join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName()) if (video.isOwned()) return join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName())
return this.saveRemotePreviewAndReturnPath(video) const res = await this.saveRemotePreviewAndReturnPath(video)
})
return res
} }
private saveRemotePreviewAndReturnPath (video: VideoInstance) { private saveRemotePreviewAndReturnPath (video: VideoInstance) {

View File

@ -1,6 +1,6 @@
import * as request from 'request' import * as request from 'request'
import * as Sequelize from 'sequelize' import * as Sequelize from 'sequelize'
import * as Promise from 'bluebird' import * as Bluebird from 'bluebird'
import { join } from 'path' import { join } from 'path'
import { database as db } from '../initializers/database' import { database as db } from '../initializers/database'
@ -188,46 +188,47 @@ function addEventToRemoteVideo (eventParam: EventParam, transaction?: Sequelize.
function addEventsToRemoteVideo (eventsParams: EventParam[], transaction: Sequelize.Transaction) { function addEventsToRemoteVideo (eventsParams: EventParam[], transaction: Sequelize.Transaction) {
const tasks = [] const tasks = []
eventsParams.forEach(eventParams => { for (const eventParams of eventsParams) {
tasks.push(addEventToRemoteVideo(eventParams, transaction)) tasks.push(addEventToRemoteVideo(eventParams, transaction))
}) }
return Promise.all(tasks) return Promise.all(tasks)
} }
function hasFriends () { async function hasFriends () {
return db.Pod.countAll().then(count => count !== 0) const count = await db.Pod.countAll()
return count !== 0
} }
function makeFriends (hosts: string[]) { async function makeFriends (hosts: string[]) {
const podsScore = {} const podsScore = {}
logger.info('Make friends!') logger.info('Make friends!')
return getMyPublicCert() const cert = await getMyPublicCert()
.then(cert => {
return Promise.each(hosts, host => computeForeignPodsList(host, podsScore)).then(() => cert) for (const host of hosts) {
}) await computeForeignPodsList(host, podsScore)
.then(cert => { }
logger.debug('Pods scores computed.', { podsScore: podsScore }) logger.debug('Pods scores computed.', { podsScore: podsScore })
const podsList = computeWinningPods(hosts, podsScore) const podsList = computeWinningPods(hosts, podsScore)
logger.debug('Pods that we keep.', { podsToKeep: podsList }) logger.debug('Pods that we keep.', { podsToKeep: podsList })
return makeRequestsToWinningPods(cert, podsList) return makeRequestsToWinningPods(cert, podsList)
})
} }
function quitFriends () { async function quitFriends () {
// Stop pool requests // Stop pool requests
requestScheduler.deactivate() requestScheduler.deactivate()
return requestScheduler.flush() try {
.then(() => { await requestScheduler.flush()
return requestVideoQaduScheduler.flush()
}) await requestVideoQaduScheduler.flush()
.then(() => {
return db.Pod.list() const pods = await db.Pod.list()
})
.then(pods => {
const requestParams = { const requestParams = {
method: 'POST' as 'POST', method: 'POST' as 'POST',
path: '/api/' + API_VERSION + '/remote/pods/remove', path: '/api/' + API_VERSION + '/remote/pods/remove',
@ -237,44 +238,45 @@ function quitFriends () {
// Announce we quit them // Announce we quit them
// We don't care if the request fails // We don't care if the request fails
// The other pod will exclude us automatically after a while // The other pod will exclude us automatically after a while
return Promise.map(pods, pod => { try {
await Bluebird.map(pods, pod => {
requestParams.toPod = pod requestParams.toPod = pod
return makeSecureRequest(requestParams) return makeSecureRequest(requestParams)
}, { concurrency: REQUESTS_IN_PARALLEL }) }, { concurrency: REQUESTS_IN_PARALLEL })
.then(() => pods) } catch (err) { // Don't stop the process
.catch(err => {
logger.error('Some errors while quitting friends.', err) logger.error('Some errors while quitting friends.', err)
// Don't stop the process }
return pods
}) const tasks = []
}) for (const pod of pods) {
.then(pods => { tasks.push(pod.destroy())
const tasks = [] }
pods.forEach(pod => tasks.push(pod.destroy())) await Promise.all(pods)
return Promise.all(pods)
})
.then(() => {
logger.info('Removed all remote videos.') logger.info('Removed all remote videos.')
requestScheduler.activate()
} catch (err) {
// Don't forget to re activate the scheduler, even if there was an error // Don't forget to re activate the scheduler, even if there was an error
return requestScheduler.activate() requestScheduler.activate()
})
.finally(() => requestScheduler.activate()) throw err
}
} }
function sendOwnedDataToPod (podId: number) { async function sendOwnedDataToPod (podId: number) {
// First send authors // First send authors
return sendOwnedAuthorsToPod(podId) await sendOwnedAuthorsToPod(podId)
.then(() => sendOwnedChannelsToPod(podId)) await sendOwnedChannelsToPod(podId)
.then(() => sendOwnedVideosToPod(podId)) await sendOwnedVideosToPod(podId)
} }
function sendOwnedChannelsToPod (podId: number) { async function sendOwnedChannelsToPod (podId: number) {
return db.VideoChannel.listOwned() const videoChannels = await db.VideoChannel.listOwned()
.then(videoChannels => {
const tasks = [] const tasks: Promise<any>[] = []
videoChannels.forEach(videoChannel => { for (const videoChannel of videoChannels) {
const remoteVideoChannel = videoChannel.toAddRemoteJSON() const remoteVideoChannel = videoChannel.toAddRemoteJSON()
const options = { const options = {
type: 'add-channel' as 'add-channel', type: 'add-channel' as 'add-channel',
@ -286,17 +288,16 @@ function sendOwnedChannelsToPod (podId: number) {
const p = createRequest(options) const p = createRequest(options)
tasks.push(p) tasks.push(p)
})
return Promise.all(tasks)
})
} }
function sendOwnedAuthorsToPod (podId: number) { await Promise.all(tasks)
return db.Author.listOwned() }
.then(authors => {
const tasks = [] async function sendOwnedAuthorsToPod (podId: number) {
authors.forEach(author => { const authors = await db.Author.listOwned()
const tasks: Promise<any>[] = []
for (const author of authors) {
const remoteAuthor = author.toAddRemoteJSON() const remoteAuthor = author.toAddRemoteJSON()
const options = { const options = {
type: 'add-author' as 'add-author', type: 'add-author' as 'add-author',
@ -308,17 +309,16 @@ function sendOwnedAuthorsToPod (podId: number) {
const p = createRequest(options) const p = createRequest(options)
tasks.push(p) tasks.push(p)
})
return Promise.all(tasks)
})
} }
function sendOwnedVideosToPod (podId: number) { await Promise.all(tasks)
return db.Video.listOwnedAndPopulateAuthorAndTags() }
.then(videosList => {
const tasks = [] async function sendOwnedVideosToPod (podId: number) {
videosList.forEach(video => { const videosList = await db.Video.listOwnedAndPopulateAuthorAndTags()
const tasks: Bluebird<any>[] = []
for (const video of videosList) {
const promise = video.toAddRemoteJSON() const promise = video.toAddRemoteJSON()
.then(remoteVideo => { .then(remoteVideo => {
const options = { const options = {
@ -337,10 +337,9 @@ function sendOwnedVideosToPod (podId: number) {
}) })
tasks.push(promise) tasks.push(promise)
}) }
return Promise.all(tasks) await Promise.all(tasks)
})
} }
function fetchRemotePreview (video: VideoInstance) { function fetchRemotePreview (video: VideoInstance) {
@ -350,18 +349,26 @@ function fetchRemotePreview (video: VideoInstance) {
return request.get(REMOTE_SCHEME.HTTP + '://' + host + path) return request.get(REMOTE_SCHEME.HTTP + '://' + host + path)
} }
function removeFriend (pod: PodInstance) { async function removeFriend (pod: PodInstance) {
const requestParams = { const requestParams = {
method: 'POST' as 'POST', method: 'POST' as 'POST',
path: '/api/' + API_VERSION + '/remote/pods/remove', path: '/api/' + API_VERSION + '/remote/pods/remove',
toPod: pod toPod: pod
} }
return makeSecureRequest(requestParams) try {
.catch(err => logger.warn('Cannot notify friends %s we are quitting him.', pod.host, err)) await makeSecureRequest(requestParams)
.then(() => pod.destroy()) } catch (err) {
.then(() => logger.info('Removed friend %s.', pod.host)) logger.warn('Cannot notify friends %s we are quitting him.', pod.host, err)
.catch(err => logger.error('Cannot destroy friend %s.', pod.host, err)) }
try {
await pod.destroy()
logger.info('Removed friend %s.', pod.host)
} catch (err) {
logger.error('Cannot destroy friend %s.', pod.host, err)
}
} }
function getRequestScheduler () { function getRequestScheduler () {
@ -406,23 +413,21 @@ export {
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
function computeForeignPodsList (host: string, podsScore: { [ host: string ]: number }) { async function computeForeignPodsList (host: string, podsScore: { [ host: string ]: number }) {
// TODO: type res const result = await getForeignPodsList(host)
return getForeignPodsList(host).then(res => { const foreignPodsList: { host: string }[] = result.data
const foreignPodsList: { host: string }[] = res.data
// Let's give 1 point to the pod we ask the friends list // Let's give 1 point to the pod we ask the friends list
foreignPodsList.push({ host }) foreignPodsList.push({ host })
foreignPodsList.forEach(foreignPod => { for (const foreignPod of foreignPodsList) {
const foreignPodHost = foreignPod.host const foreignPodHost = foreignPod.host
if (podsScore[foreignPodHost]) podsScore[foreignPodHost]++ if (podsScore[foreignPodHost]) podsScore[foreignPodHost]++
else podsScore[foreignPodHost] = 1 else podsScore[foreignPodHost] = 1
}) }
return undefined return undefined
})
} }
function computeWinningPods (hosts: string[], podsScore: { [ host: string ]: number }) { function computeWinningPods (hosts: string[], podsScore: { [ host: string ]: number }) {
@ -431,12 +436,12 @@ function computeWinningPods (hosts: string[], podsScore: { [ host: string ]: num
const podsList = [] const podsList = []
const baseScore = hosts.length / 2 const baseScore = hosts.length / 2
Object.keys(podsScore).forEach(podHost => { for (const podHost of Object.keys(podsScore)) {
// If the pod is not me and with a good score we add it // If the pod is not me and with a good score we add it
if (isMe(podHost) === false && podsScore[podHost] > baseScore) { if (isMe(podHost) === false && podsScore[podHost] > baseScore) {
podsList.push({ host: podHost }) podsList.push({ host: podHost })
} }
}) }
return podsList return podsList
} }
@ -449,7 +454,7 @@ function getForeignPodsList (host: string) {
if (err) return rej(err) if (err) return rej(err)
try { try {
const json = JSON.parse(body) const json: ResultList<FormattedPod> = JSON.parse(body)
return res(json) return res(json)
} catch (err) { } catch (err) {
return rej(err) return rej(err)
@ -458,13 +463,14 @@ function getForeignPodsList (host: string) {
}) })
} }
function makeRequestsToWinningPods (cert: string, podsList: PodInstance[]) { async function makeRequestsToWinningPods (cert: string, podsList: PodInstance[]) {
// Stop pool requests // Stop pool requests
requestScheduler.deactivate() requestScheduler.deactivate()
// Flush pool requests // Flush pool requests
requestScheduler.forceSend() requestScheduler.forceSend()
return Promise.map(podsList, pod => { try {
await Bluebird.map(podsList, async pod => {
const params = { const params = {
url: REMOTE_SCHEME.HTTP + '://' + pod.host + '/api/' + API_VERSION + '/remote/pods/add', url: REMOTE_SCHEME.HTTP + '://' + pod.host + '/api/' + API_VERSION + '/remote/pods/add',
method: 'POST' as 'POST', method: 'POST' as 'POST',
@ -475,36 +481,35 @@ function makeRequestsToWinningPods (cert: string, podsList: PodInstance[]) {
} }
} }
return makeRetryRequest(params) const { response, body } = await makeRetryRequest(params)
.then(({ response, body }) => { const typedBody = body as { cert: string, email: string }
body = body as { cert: string, email: string }
if (response.statusCode === 200) { if (response.statusCode === 200) {
const podObj = db.Pod.build({ host: pod.host, publicKey: body.cert, email: body.email }) const podObj = db.Pod.build({ host: pod.host, publicKey: typedBody.cert, email: typedBody.email })
return podObj.save()
.then(podCreated => { let podCreated: PodInstance
try {
podCreated = await podObj.save()
} catch (err) {
logger.error('Cannot add friend %s pod.', pod.host, err)
}
// Add our videos to the request scheduler // Add our videos to the request scheduler
sendOwnedDataToPod(podCreated.id) sendOwnedDataToPod(podCreated.id)
}) .catch(err => logger.warn('Cannot send owned data to pod %d.', podCreated.id, err))
.catch(err => {
logger.error('Cannot add friend %s pod.', pod.host, err)
})
} else { } else {
logger.error('Status not 200 for %s pod.', pod.host) logger.error('Status not 200 for %s pod.', pod.host)
} }
})
.catch(err => {
logger.error('Error with adding %s pod.', pod.host, { error: err.stack })
// Don't break the process
})
}, { concurrency: REQUESTS_IN_PARALLEL }) }, { concurrency: REQUESTS_IN_PARALLEL })
.then(() => logger.debug('makeRequestsToWinningPods finished.'))
.finally(() => { logger.debug('makeRequestsToWinningPods finished.')
requestScheduler.activate()
} catch (err) {
// Final callback, we've ended all the requests // Final callback, we've ended all the requests
// Now we made new friends, we can re activate the pool of requests // Now we made new friends, we can re activate the pool of requests
requestScheduler.activate() requestScheduler.activate()
}) }
} }
// Wrapper that populate "toIds" argument with all our friends if it is not specified // Wrapper that populate "toIds" argument with all our friends if it is not specified
@ -515,14 +520,19 @@ type CreateRequestOptions = {
toIds?: number[] toIds?: number[]
transaction: Sequelize.Transaction transaction: Sequelize.Transaction
} }
function createRequest (options: CreateRequestOptions) { async function createRequest (options: CreateRequestOptions) {
if (options.toIds !== undefined) return requestScheduler.createRequest(options as RequestSchedulerOptions) if (options.toIds !== undefined) {
await requestScheduler.createRequest(options as RequestSchedulerOptions)
return undefined
}
// If the "toIds" pods is not specified, we send the request to all our friends // If the "toIds" pods is not specified, we send the request to all our friends
return db.Pod.listAllIds(options.transaction).then(podIds => { const podIds = await db.Pod.listAllIds(options.transaction)
const newOptions = Object.assign(options, { toIds: podIds }) const newOptions = Object.assign(options, { toIds: podIds })
return requestScheduler.createRequest(newOptions) await requestScheduler.createRequest(newOptions)
})
return undefined
} }
function createVideoQaduRequest (options: RequestVideoQaduSchedulerOptions) { function createVideoQaduRequest (options: RequestVideoQaduSchedulerOptions) {

View File

@ -1,4 +1,4 @@
import * as Promise from 'bluebird' import * as Bluebird from 'bluebird'
import { database as db } from '../../../initializers/database' import { database as db } from '../../../initializers/database'
import { logger, computeResolutionsToTranscode } from '../../../helpers' import { logger, computeResolutionsToTranscode } from '../../../helpers'
@ -6,16 +6,17 @@ import { VideoInstance } from '../../../models'
import { addVideoToFriends } from '../../friends' import { addVideoToFriends } from '../../friends'
import { JobScheduler } from '../job-scheduler' import { JobScheduler } from '../job-scheduler'
function process (data: { videoUUID: string }, jobId: number) { async function process (data: { videoUUID: string }, jobId: number) {
return db.Video.loadByUUIDAndPopulateAuthorAndPodAndTags(data.videoUUID).then(video => { const video = await db.Video.loadByUUIDAndPopulateAuthorAndPodAndTags(data.videoUUID)
// No video, maybe deleted? // No video, maybe deleted?
if (!video) { if (!video) {
logger.info('Do not process job %d, video does not exist.', jobId, { videoUUID: video.uuid }) logger.info('Do not process job %d, video does not exist.', jobId, { videoUUID: video.uuid })
return undefined return undefined
} }
return video.optimizeOriginalVideofile().then(() => video) await video.optimizeOriginalVideofile()
})
return video
} }
function onError (err: Error, jobId: number) { function onError (err: Error, jobId: number) {
@ -23,33 +24,31 @@ function onError (err: Error, jobId: number) {
return Promise.resolve() return Promise.resolve()
} }
function onSuccess (jobId: number, video: VideoInstance) { async function onSuccess (jobId: number, video: VideoInstance) {
if (video === undefined) return undefined if (video === undefined) return undefined
logger.info('Job %d is a success.', jobId) logger.info('Job %d is a success.', jobId)
video.toAddRemoteJSON() const remoteVideo = await video.toAddRemoteJSON()
.then(remoteVideo => {
// Now we'll add the video's meta data to our friends // Now we'll add the video's meta data to our friends
return addVideoToFriends(remoteVideo, null) await addVideoToFriends(remoteVideo, null)
})
.then(() => { const originalFileHeight = await video.getOriginalFileHeight()
return video.getOriginalFileHeight()
})
.then(originalFileHeight => {
// Create transcoding jobs if there are enabled resolutions // Create transcoding jobs if there are enabled resolutions
const resolutionsEnabled = computeResolutionsToTranscode(originalFileHeight) const resolutionsEnabled = computeResolutionsToTranscode(originalFileHeight)
logger.info( logger.info(
'Resolutions computed for video %s and origin file height of %d.', video.uuid, originalFileHeight, 'Resolutions computed for video %s and origin file height of %d.', video.uuid, originalFileHeight,
{ resolutions: resolutionsEnabled } { resolutions: resolutionsEnabled }
) )
if (resolutionsEnabled.length === 0) return undefined if (resolutionsEnabled.length !== 0) {
try {
await db.sequelize.transaction(async t => {
const tasks: Bluebird<any>[] = []
return db.sequelize.transaction(t => { for (const resolution of resolutionsEnabled) {
const tasks: Promise<any>[] = []
resolutionsEnabled.forEach(resolution => {
const dataInput = { const dataInput = {
videoUUID: video.uuid, videoUUID: video.uuid,
resolution resolution
@ -57,24 +56,19 @@ function onSuccess (jobId: number, video: VideoInstance) {
const p = JobScheduler.Instance.createJob(t, 'videoFileTranscoder', dataInput) const p = JobScheduler.Instance.createJob(t, 'videoFileTranscoder', dataInput)
tasks.push(p) tasks.push(p)
}
await Promise.all(tasks)
}) })
return Promise.all(tasks).then(() => resolutionsEnabled) logger.info('Transcoding jobs created for uuid %s.', video.uuid, { resolutionsEnabled })
}) } catch (err) {
}) logger.warn('Cannot transcode the video.', err)
.then(resolutionsEnabled => { }
if (resolutionsEnabled === undefined) { } else {
logger.info('No transcoding jobs created for video %s (no resolutions enabled).') logger.info('No transcoding jobs created for video %s (no resolutions enabled).')
return undefined return undefined
} }
logger.info('Transcoding jobs created for uuid %s.', video.uuid, { resolutionsEnabled })
})
.catch((err: Error) => {
logger.debug('Cannot transcode the video.', err)
throw err
})
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------

View File

@ -4,16 +4,17 @@ import { logger } from '../../../helpers'
import { VideoInstance } from '../../../models' import { VideoInstance } from '../../../models'
import { VideoResolution } from '../../../../shared' import { VideoResolution } from '../../../../shared'
function process (data: { videoUUID: string, resolution: VideoResolution }, jobId: number) { async function process (data: { videoUUID: string, resolution: VideoResolution }, jobId: number) {
return db.Video.loadByUUIDAndPopulateAuthorAndPodAndTags(data.videoUUID).then(video => { const video = await db.Video.loadByUUIDAndPopulateAuthorAndPodAndTags(data.videoUUID)
// No video, maybe deleted? // No video, maybe deleted?
if (!video) { if (!video) {
logger.info('Do not process job %d, video does not exist.', jobId, { videoUUID: video.uuid }) logger.info('Do not process job %d, video does not exist.', jobId, { videoUUID: video.uuid })
return undefined return undefined
} }
return video.transcodeOriginalVideofile(data.resolution).then(() => video) await video.transcodeOriginalVideofile(data.resolution)
})
return video
} }
function onError (err: Error, jobId: number) { function onError (err: Error, jobId: number) {

View File

@ -23,7 +23,7 @@ class JobScheduler {
return this.instance || (this.instance = new this()) return this.instance || (this.instance = new this())
} }
activate () { async activate () {
const limit = JOBS_FETCH_LIMIT_PER_CYCLE const limit = JOBS_FETCH_LIMIT_PER_CYCLE
logger.info('Jobs scheduler activated.') logger.info('Jobs scheduler activated.')
@ -32,32 +32,36 @@ class JobScheduler {
// Finish processing jobs from a previous start // Finish processing jobs from a previous start
const state = JOB_STATES.PROCESSING const state = JOB_STATES.PROCESSING
db.Job.listWithLimit(limit, state) try {
.then(jobs => { const jobs = await db.Job.listWithLimit(limit, state)
this.enqueueJobs(jobsQueue, jobs) this.enqueueJobs(jobsQueue, jobs)
} catch (err) {
logger.error('Cannot list pending jobs.', err)
}
forever( forever(
next => { async next => {
if (jobsQueue.length() !== 0) { if (jobsQueue.length() !== 0) {
// Finish processing the queue first // Finish processing the queue first
return setTimeout(next, JOBS_FETCHING_INTERVAL) return setTimeout(next, JOBS_FETCHING_INTERVAL)
} }
const state = JOB_STATES.PENDING const state = JOB_STATES.PENDING
db.Job.listWithLimit(limit, state) try {
.then(jobs => { const jobs = await db.Job.listWithLimit(limit, state)
this.enqueueJobs(jobsQueue, jobs) this.enqueueJobs(jobsQueue, jobs)
} catch (err) {
logger.error('Cannot list pending jobs.', err)
}
// Optimization: we could use "drain" from queue object // Optimization: we could use "drain" from queue object
return setTimeout(next, JOBS_FETCHING_INTERVAL) return setTimeout(next, JOBS_FETCHING_INTERVAL)
})
.catch(err => logger.error('Cannot list pending jobs.', err))
}, },
err => logger.error('Error in job scheduler queue.', err) err => logger.error('Error in job scheduler queue.', err)
) )
})
.catch(err => logger.error('Cannot list pending jobs.', err))
} }
createJob (transaction: Sequelize.Transaction, handlerName: string, handlerInputData: object) { createJob (transaction: Sequelize.Transaction, handlerName: string, handlerInputData: object) {
@ -75,7 +79,7 @@ class JobScheduler {
jobs.forEach(job => jobsQueue.push(job)) jobs.forEach(job => jobsQueue.push(job))
} }
private processJob (job: JobInstance, callback: (err: Error) => void) { private async processJob (job: JobInstance, callback: (err: Error) => void) {
const jobHandler = jobHandlers[job.handlerName] const jobHandler = jobHandlers[job.handlerName]
if (jobHandler === undefined) { if (jobHandler === undefined) {
logger.error('Unknown job handler for job %s.', job.handlerName) logger.error('Unknown job handler for job %s.', job.handlerName)
@ -85,41 +89,45 @@ class JobScheduler {
logger.info('Processing job %d with handler %s.', job.id, job.handlerName) logger.info('Processing job %d with handler %s.', job.id, job.handlerName)
job.state = JOB_STATES.PROCESSING job.state = JOB_STATES.PROCESSING
return job.save() await job.save()
.then(() => {
return jobHandler.process(job.handlerInputData, job.id)
})
.then(
result => {
return this.onJobSuccess(jobHandler, job, result)
},
err => { try {
const result = await jobHandler.process(job.handlerInputData, job.id)
await this.onJobSuccess(jobHandler, job, result)
} catch (err) {
logger.error('Error in job handler %s.', job.handlerName, err) logger.error('Error in job handler %s.', job.handlerName, err)
return this.onJobError(jobHandler, job, err)
try {
await this.onJobError(jobHandler, job, err)
} catch (innerErr) {
this.cannotSaveJobError(innerErr)
return callback(innerErr)
} }
)
.then(() => callback(null))
.catch(err => {
this.cannotSaveJobError(err)
return callback(err)
})
} }
private onJobError (jobHandler: JobHandler<any>, job: JobInstance, err: Error) { callback(null)
}
private async onJobError (jobHandler: JobHandler<any>, job: JobInstance, err: Error) {
job.state = JOB_STATES.ERROR job.state = JOB_STATES.ERROR
return job.save() try {
.then(() => jobHandler.onError(err, job.id)) await job.save()
.catch(err => this.cannotSaveJobError(err)) await jobHandler.onError(err, job.id)
} catch (err) {
this.cannotSaveJobError(err)
}
} }
private onJobSuccess (jobHandler: JobHandler<any>, job: JobInstance, jobResult: any) { private async onJobSuccess (jobHandler: JobHandler<any>, job: JobInstance, jobResult: any) {
job.state = JOB_STATES.SUCCESS job.state = JOB_STATES.SUCCESS
return job.save() try {
.then(() => jobHandler.onSuccess(job.id, jobResult)) await job.save()
.catch(err => this.cannotSaveJobError(err)) jobHandler.onSuccess(job.id, jobResult)
} catch (err) {
this.cannotSaveJobError(err)
}
} }
private cannotSaveJobError (err: Error) { private cannotSaveJobError (err: Error) {

View File

@ -24,23 +24,21 @@ function getRefreshToken (refreshToken: string) {
return db.OAuthToken.getByRefreshTokenAndPopulateClient(refreshToken) return db.OAuthToken.getByRefreshTokenAndPopulateClient(refreshToken)
} }
function getUser (username: string, password: string) { async function getUser (username: string, password: string) {
logger.debug('Getting User (username: ' + username + ', password: ******).') logger.debug('Getting User (username: ' + username + ', password: ******).')
return db.User.getByUsername(username).then(user => { const user = await db.User.getByUsername(username)
if (!user) return null if (!user) return null
return user.isPasswordMatch(password).then(passwordMatch => { const passwordMatch = await user.isPasswordMatch(password)
if (passwordMatch === false) return null if (passwordMatch === false) return null
return user return user
})
})
} }
function revokeToken (token: TokenInfo) { async function revokeToken (tokenInfo: TokenInfo) {
return db.OAuthToken.getByRefreshTokenAndPopulateUser(token.refreshToken).then(tokenDB => { const token = await db.OAuthToken.getByRefreshTokenAndPopulateUser(tokenInfo.refreshToken)
if (tokenDB) tokenDB.destroy() if (token) token.destroy()
/* /*
* Thanks to https://github.com/manjeshpv/node-oauth2-server-implementation/blob/master/components/oauth/mongo-models.js * Thanks to https://github.com/manjeshpv/node-oauth2-server-implementation/blob/master/components/oauth/mongo-models.js
@ -49,14 +47,13 @@ function revokeToken (token: TokenInfo) {
* https://github.com/oauthjs/node-oauth2-server/pull/274 * https://github.com/oauthjs/node-oauth2-server/pull/274
* https://github.com/oauthjs/node-oauth2-server/issues/290" * https://github.com/oauthjs/node-oauth2-server/issues/290"
*/ */
const expiredToken = tokenDB const expiredToken = token
expiredToken.refreshTokenExpiresAt = new Date('2015-05-28T06:59:53.000Z') expiredToken.refreshTokenExpiresAt = new Date('2015-05-28T06:59:53.000Z')
return expiredToken return expiredToken
})
} }
function saveToken (token: TokenInfo, client: OAuthClientInstance, user: UserInstance) { async function saveToken (token: TokenInfo, client: OAuthClientInstance, user: UserInstance) {
logger.debug('Saving token ' + token.accessToken + ' for client ' + client.id + ' and user ' + user.id + '.') logger.debug('Saving token ' + token.accessToken + ' for client ' + client.id + ' and user ' + user.id + '.')
const tokenToCreate = { const tokenToCreate = {
@ -68,11 +65,10 @@ function saveToken (token: TokenInfo, client: OAuthClientInstance, user: UserIns
userId: user.id userId: user.id
} }
return db.OAuthToken.create(tokenToCreate).then(tokenCreated => { const tokenCreated = await db.OAuthToken.create(tokenToCreate)
const tokenToReturn = Object.assign(tokenCreated, { client, user }) const tokenToReturn = Object.assign(tokenCreated, { client, user })
return tokenToReturn return tokenToReturn
})
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------

View File

@ -1,5 +1,5 @@
import { isEmpty } from 'lodash' import { isEmpty } from 'lodash'
import * as Promise from 'bluebird' import * as Bluebird from 'bluebird'
import { database as db } from '../../initializers/database' import { database as db } from '../../initializers/database'
import { logger, makeSecureRequest } from '../../helpers' import { logger, makeSecureRequest } from '../../helpers'
@ -76,7 +76,7 @@ abstract class AbstractRequestScheduler <T> {
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Make a requests to friends of a certain type // Make a requests to friends of a certain type
protected makeRequest (toPod: PodInstance, requestEndpoint: string, requestsToMake: any) { protected async makeRequest (toPod: PodInstance, requestEndpoint: string, requestsToMake: any) {
const params = { const params = {
toPod: toPod, toPod: toPod,
method: 'POST' as 'POST', method: 'POST' as 'POST',
@ -86,23 +86,29 @@ abstract class AbstractRequestScheduler <T> {
// Make multiple retry requests to all of pods // Make multiple retry requests to all of pods
// The function fire some useful callbacks // The function fire some useful callbacks
return makeSecureRequest(params) try {
.then(({ response, body }) => { const { response } = await makeSecureRequest(params)
if (response.statusCode !== 200 && response.statusCode !== 201 && response.statusCode !== 204) { if (response.statusCode !== 200 && response.statusCode !== 201 && response.statusCode !== 204) {
throw new Error('Status code not 20x : ' + response.statusCode) throw new Error('Status code not 20x : ' + response.statusCode)
} }
}) } catch (err) {
.catch(err => {
logger.error('Error sending secure request to %s pod.', toPod.host, err) logger.error('Error sending secure request to %s pod.', toPod.host, err)
throw err throw err
}) }
} }
// Make all the requests of the scheduler // Make all the requests of the scheduler
protected makeRequests () { protected async makeRequests () {
return this.getRequestModel().listWithLimitAndRandom(this.limitPods, this.limitPerPod) let requestsGrouped: T
.then((requestsGrouped: T) => {
try {
requestsGrouped = await this.getRequestModel().listWithLimitAndRandom(this.limitPods, this.limitPerPod)
} catch (err) {
logger.error('Cannot get the list of "%s".', this.description, { error: err.stack })
throw err
}
// We want to group requests by destinations pod and endpoint // We want to group requests by destinations pod and endpoint
const requestsToMake = this.buildRequestsObjects(requestsGrouped) const requestsToMake = this.buildRequestsObjects(requestsGrouped)
@ -117,41 +123,38 @@ abstract class AbstractRequestScheduler <T> {
const goodPods: number[] = [] const goodPods: number[] = []
const badPods: number[] = [] const badPods: number[] = []
return Promise.map(Object.keys(requestsToMake), hashKey => { await Bluebird.map(Object.keys(requestsToMake), async hashKey => {
const requestToMake = requestsToMake[hashKey] const requestToMake = requestsToMake[hashKey]
const toPod: PodInstance = requestToMake.toPod const toPod: PodInstance = requestToMake.toPod
return this.makeRequest(toPod, requestToMake.endpoint, requestToMake.datas) try {
.then(() => { await this.makeRequest(toPod, requestToMake.endpoint, requestToMake.datas)
logger.debug('Removing requests for pod %s.', requestToMake.toPod.id, { requestsIds: requestToMake.ids }) logger.debug('Removing requests for pod %s.', requestToMake.toPod.id, { requestsIds: requestToMake.ids })
goodPods.push(requestToMake.toPod.id) goodPods.push(requestToMake.toPod.id)
this.afterRequestHook() this.afterRequestHook()
// Remove the pod id of these request ids // Remove the pod id of these request ids
return this.getRequestToPodModel().removeByRequestIdsAndPod(requestToMake.ids, requestToMake.toPod.id) await this.getRequestToPodModel()
}) .removeByRequestIdsAndPod(requestToMake.ids, requestToMake.toPod.id)
.catch(err => { } catch (err) {
badPods.push(requestToMake.toPod.id) badPods.push(requestToMake.toPod.id)
logger.info('Cannot make request to %s.', toPod.host, err) logger.info('Cannot make request to %s.', toPod.host, err)
}) }
}, { concurrency: REQUESTS_IN_PARALLEL }).then(() => ({ goodPods, badPods })) }, { concurrency: REQUESTS_IN_PARALLEL })
})
.then(({ goodPods, badPods }) => {
this.afterRequestsHook() this.afterRequestsHook()
// All the requests were made, we update the pods score // All the requests were made, we update the pods score
return db.Pod.updatePodsScore(goodPods, badPods) await db.Pod.updatePodsScore(goodPods, badPods)
})
.catch(err => logger.error('Cannot get the list of "%s".', this.description, { error: err.stack }))
} }
protected afterRequestHook () { protected afterRequestHook () {
// Nothing to do, let children reimplement it // Nothing to do, let children re-implement it
} }
protected afterRequestsHook () { protected afterRequestsHook () {
// Nothing to do, let children reimplement it // Nothing to do, let children re-implement it
} }
} }

View File

@ -37,8 +37,8 @@ class RequestScheduler extends AbstractRequestScheduler<RequestsGrouped> {
buildRequestsObjects (requestsGrouped: RequestsGrouped) { buildRequestsObjects (requestsGrouped: RequestsGrouped) {
const requestsToMakeGrouped: RequestsObjects<RemoteVideoRequest> = {} const requestsToMakeGrouped: RequestsObjects<RemoteVideoRequest> = {}
Object.keys(requestsGrouped).forEach(toPodId => { for (const toPodId of Object.keys(requestsGrouped)) {
requestsGrouped[toPodId].forEach(data => { for (const data of requestsGrouped[toPodId]) {
const request = data.request const request = data.request
const pod = data.pod const pod = data.pod
const hashKey = toPodId + request.endpoint const hashKey = toPodId + request.endpoint
@ -54,13 +54,13 @@ class RequestScheduler extends AbstractRequestScheduler<RequestsGrouped> {
requestsToMakeGrouped[hashKey].ids.push(request.id) requestsToMakeGrouped[hashKey].ids.push(request.id)
requestsToMakeGrouped[hashKey].datas.push(request.request) requestsToMakeGrouped[hashKey].datas.push(request.request)
}) }
}) }
return requestsToMakeGrouped return requestsToMakeGrouped
} }
createRequest ({ type, endpoint, data, toIds, transaction }: RequestSchedulerOptions) { async createRequest ({ type, endpoint, data, toIds, transaction }: RequestSchedulerOptions) {
// If there are no destination pods abort // If there are no destination pods abort
if (toIds.length === 0) return undefined if (toIds.length === 0) return undefined
@ -76,10 +76,8 @@ class RequestScheduler extends AbstractRequestScheduler<RequestsGrouped> {
transaction transaction
} }
return db.Request.create(createQuery, dbRequestOptions) const request = await db.Request.create(createQuery, dbRequestOptions)
.then(request => { await request.setPods(toIds, dbRequestOptions)
return request.setPods(toIds, dbRequestOptions)
})
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------

View File

@ -59,8 +59,8 @@ class RequestVideoEventScheduler extends AbstractRequestScheduler<RequestsVideoE
// We group video events per video and per pod // We group video events per video and per pod
// We add the counts of the same event types // We add the counts of the same event types
Object.keys(eventRequests).forEach(toPodId => { for (const toPodId of Object.keys(eventRequests)) {
eventRequests[toPodId].forEach(eventToProcess => { for (const eventToProcess of eventRequests[toPodId]) {
if (!eventsPerVideoPerPod[toPodId]) eventsPerVideoPerPod[toPodId] = {} if (!eventsPerVideoPerPod[toPodId]) eventsPerVideoPerPod[toPodId] = {}
if (!requestsToMakeGrouped[toPodId]) { if (!requestsToMakeGrouped[toPodId]) {
@ -81,17 +81,17 @@ class RequestVideoEventScheduler extends AbstractRequestScheduler<RequestsVideoE
if (!events[eventToProcess.type]) events[eventToProcess.type] = 0 if (!events[eventToProcess.type]) events[eventToProcess.type] = 0
events[eventToProcess.type] += eventToProcess.count events[eventToProcess.type] += eventToProcess.count
}) }
}) }
// Now we build our requests array per pod // Now we build our requests array per pod
Object.keys(eventsPerVideoPerPod).forEach(toPodId => { for (const toPodId of Object.keys(eventsPerVideoPerPod)) {
const eventsForPod = eventsPerVideoPerPod[toPodId] const eventsForPod = eventsPerVideoPerPod[toPodId]
Object.keys(eventsForPod).forEach(uuid => { for (const uuid of Object.keys(eventsForPod)) {
const eventsForVideo = eventsForPod[uuid] const eventsForVideo = eventsForPod[uuid]
Object.keys(eventsForVideo).forEach(eventType => { for (const eventType of Object.keys(eventsForVideo)) {
requestsToMakeGrouped[toPodId].datas.push({ requestsToMakeGrouped[toPodId].datas.push({
data: { data: {
uuid, uuid,
@ -99,9 +99,9 @@ class RequestVideoEventScheduler extends AbstractRequestScheduler<RequestsVideoE
count: +eventsForVideo[eventType] count: +eventsForVideo[eventType]
} }
}) })
}) }
}) }
}) }
return requestsToMakeGrouped return requestsToMakeGrouped
} }

View File

@ -59,8 +59,8 @@ class RequestVideoQaduScheduler extends AbstractRequestScheduler<RequestsVideoQa
buildRequestsObjects (requests: RequestsVideoQaduGrouped) { buildRequestsObjects (requests: RequestsVideoQaduGrouped) {
const requestsToMakeGrouped: RequestsObjectsCustom<RemoteQaduVideoRequest> = {} const requestsToMakeGrouped: RequestsObjectsCustom<RemoteQaduVideoRequest> = {}
Object.keys(requests).forEach(toPodId => { for (const toPodId of Object.keys(requests)) {
requests[toPodId].forEach(data => { for (const data of requests[toPodId]) {
const request = data.request const request = data.request
const video = data.video const video = data.video
const pod = data.pod const pod = data.pod
@ -105,39 +105,39 @@ class RequestVideoQaduScheduler extends AbstractRequestScheduler<RequestsVideoQa
// Maybe there are multiple quick and dirty update for the same video // Maybe there are multiple quick and dirty update for the same video
// We use this hash map to dedupe them // We use this hash map to dedupe them
requestsToMakeGrouped[hashKey].videos[video.id] = videoData requestsToMakeGrouped[hashKey].videos[video.id] = videoData
}) }
}) }
// Now we deduped similar quick and dirty updates, we can build our requests data // Now we deduped similar quick and dirty updates, we can build our requests data
Object.keys(requestsToMakeGrouped).forEach(hashKey => { for (const hashKey of Object.keys(requestsToMakeGrouped)) {
Object.keys(requestsToMakeGrouped[hashKey].videos).forEach(videoUUID => { for (const videoUUID of Object.keys(requestsToMakeGrouped[hashKey].videos)) {
const videoData = requestsToMakeGrouped[hashKey].videos[videoUUID] const videoData = requestsToMakeGrouped[hashKey].videos[videoUUID]
requestsToMakeGrouped[hashKey].datas.push({ requestsToMakeGrouped[hashKey].datas.push({
data: videoData data: videoData
}) })
}) }
// We don't need it anymore, it was just to build our data array // We don't need it anymore, it was just to build our data array
delete requestsToMakeGrouped[hashKey].videos delete requestsToMakeGrouped[hashKey].videos
}) }
return requestsToMakeGrouped return requestsToMakeGrouped
} }
createRequest ({ type, videoId, transaction }: RequestVideoQaduSchedulerOptions) { async createRequest ({ type, videoId, transaction }: RequestVideoQaduSchedulerOptions) {
const dbRequestOptions: Sequelize.BulkCreateOptions = {} const dbRequestOptions: Sequelize.BulkCreateOptions = {}
if (transaction) dbRequestOptions.transaction = transaction if (transaction) dbRequestOptions.transaction = transaction
// Send the update to all our friends // Send the update to all our friends
return db.Pod.listAllIds(transaction).then(podIds => { const podIds = await db.Pod.listAllIds(transaction)
const queries = [] const queries = []
podIds.forEach(podId => { for (const podId of podIds) {
queries.push({ type, videoId, podId }) queries.push({ type, videoId, podId })
}) }
return db.RequestVideoQadu.bulkCreate(queries, dbRequestOptions) await db.RequestVideoQadu.bulkCreate(queries, dbRequestOptions)
}) return undefined
} }
} }

View File

@ -3,40 +3,36 @@ import { UserInstance } from '../models'
import { addVideoAuthorToFriends } from './friends' import { addVideoAuthorToFriends } from './friends'
import { createVideoChannel } from './video-channel' import { createVideoChannel } from './video-channel'
function createUserAuthorAndChannel (user: UserInstance, validateUser = true) { async function createUserAuthorAndChannel (user: UserInstance, validateUser = true) {
return db.sequelize.transaction(t => { const res = await db.sequelize.transaction(async t => {
const userOptions = { const userOptions = {
transaction: t, transaction: t,
validate: validateUser validate: validateUser
} }
return user.save(userOptions) const userCreated = await user.save(userOptions)
.then(user => { const authorInstance = db.Author.build({
const author = db.Author.build({ name: userCreated.username,
name: user.username,
podId: null, // It is our pod podId: null, // It is our pod
userId: user.id userId: userCreated.id
}) })
return author.save({ transaction: t }) const authorCreated = await authorInstance.save({ transaction: t })
.then(author => ({ author, user }))
}) const remoteVideoAuthor = authorCreated.toAddRemoteJSON()
.then(({ author, user }) => {
const remoteVideoAuthor = author.toAddRemoteJSON()
// Now we'll add the video channel's meta data to our friends // Now we'll add the video channel's meta data to our friends
return addVideoAuthorToFriends(remoteVideoAuthor, t) const author = await addVideoAuthorToFriends(remoteVideoAuthor, t)
.then(() => ({ author, user }))
})
.then(({ author, user }) => {
const videoChannelInfo = {
name: `Default ${user.username} channel`
}
return createVideoChannel(videoChannelInfo, author, t) const videoChannelInfo = {
.then(videoChannel => ({ author, user, videoChannel })) name: `Default ${userCreated.username} channel`
}) }
const videoChannel = await createVideoChannel(videoChannelInfo, authorCreated, t)
return { author, videoChannel }
}) })
return res
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------

View File

@ -8,12 +8,12 @@ import {
} from '../helpers' } from '../helpers'
import { PodSignature } from '../../shared' import { PodSignature } from '../../shared'
function checkSignature (req: express.Request, res: express.Response, next: express.NextFunction) { async function checkSignature (req: express.Request, res: express.Response, next: express.NextFunction) {
const signatureObject: PodSignature = req.body.signature const signatureObject: PodSignature = req.body.signature
const host = signatureObject.host const host = signatureObject.host
db.Pod.loadByHost(host) try {
.then(pod => { const pod = await db.Pod.loadByHost(host)
if (pod === null) { if (pod === null) {
logger.error('Unknown pod %s.', host) logger.error('Unknown pod %s.', host)
return res.sendStatus(403) return res.sendStatus(403)
@ -42,11 +42,10 @@ function checkSignature (req: express.Request, res: express.Response, next: expr
logger.error('Signature is not okay in body for %s.', signatureObject.host) logger.error('Signature is not okay in body for %s.', signatureObject.host)
return res.sendStatus(403) return res.sendStatus(403)
}) } catch (err) {
.catch(err => {
logger.error('Cannot get signed host in body.', { error: err.stack, signature: signatureObject.signature }) logger.error('Cannot get signed host in body.', { error: err.stack, signature: signatureObject.signature })
return res.sendStatus(500) return res.sendStatus(500)
}) }
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------

View File

@ -247,20 +247,21 @@ updatePodsScore = function (goodPods: number[], badPods: number[]) {
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Remove pods with a score of 0 (too many requests where they were unreachable) // Remove pods with a score of 0 (too many requests where they were unreachable)
function removeBadPods () { async function removeBadPods () {
return listBadPods() try {
.then(pods => { const pods = await listBadPods()
const podsRemovePromises = pods.map(pod => pod.destroy()) const podsRemovePromises = pods.map(pod => pod.destroy())
return Promise.all(podsRemovePromises).then(() => pods.length) await Promise.all(podsRemovePromises)
})
.then(numberOfPodsRemoved => { const numberOfPodsRemoved = pods.length
if (numberOfPodsRemoved) { if (numberOfPodsRemoved) {
logger.info('Removed %d pods.', numberOfPodsRemoved) logger.info('Removed %d pods.', numberOfPodsRemoved)
} else { } else {
logger.info('No need to remove bad pods.') logger.info('No need to remove bad pods.')
} }
}) } catch (err) {
.catch(err => {
logger.error('Cannot remove bad pods.', err) logger.error('Cannot remove bad pods.', err)
}) }
} }