diff --git a/README.md b/README.md index 491b09a36..956403552 100644 --- a/README.md +++ b/README.md @@ -131,6 +131,7 @@ BitTorrent) inside the web browser, as of today. * nginx * PostgreSQL + * Redis * **NodeJS >= 8.x** * yarn * OpenSSL (cli) diff --git a/client/src/app/+admin/jobs/jobs-list/jobs-list.component.html b/client/src/app/+admin/jobs/jobs-list/jobs-list.component.html index 809155338..f1b14e5e3 100644 --- a/client/src/app/+admin/jobs/jobs-list/jobs-list.component.html +++ b/client/src/app/+admin/jobs/jobs-list/jobs-list.component.html @@ -1,20 +1,27 @@
Jobs list
+ +
+ +
+ + - - - + + + - 
{{ job.handlerInputData }}
+ 
{{ job.data }}
 - diff --git a/client/src/app/+admin/jobs/jobs-list/jobs-list.component.scss b/client/src/app/+admin/jobs/jobs-list/jobs-list.component.scss index 47c01f64a..5c2ad21f2 100644 --- a/client/src/app/+admin/jobs/jobs-list/jobs-list.component.scss +++ b/client/src/app/+admin/jobs/jobs-list/jobs-list.component.scss @@ -1,3 +1,10 @@ +@import '_variables'; +@import '_mixins'; + +.peertube-select-container { + @include peertube-select-container(auto); +} + pre { font-size: 11px; } diff --git a/client/src/app/+admin/jobs/jobs-list/jobs-list.component.ts b/client/src/app/+admin/jobs/jobs-list/jobs-list.component.ts index f93847f29..7de6f70d2 100644 --- a/client/src/app/+admin/jobs/jobs-list/jobs-list.component.ts +++ b/client/src/app/+admin/jobs/jobs-list/jobs-list.component.ts @@ -2,6 +2,7 @@ import { Component, OnInit } from '@angular/core' import { NotificationsService } from 'angular2-notifications' import { SortMeta } from 'primeng/primeng' import { Job } from '../../../../../../shared/index' +import { JobState } from '../../../../../../shared/models' import { RestPagination, RestTable } from '../../../shared' import { viewportHeight } from '../../../shared/misc/utils' import { JobService } from '../shared' @@ -13,10 +14,12 @@ import { RestExtractor } from '../../../shared/rest/rest-extractor.service' styleUrls: [ './jobs-list.component.scss' ] }) export class JobsListComponent extends RestTable implements OnInit { + jobState: JobState = 'inactive' + jobStates: JobState[] = [ 'active', 'complete', 'failed', 'inactive', 'delayed' ] jobs: Job[] = [] totalRecords = 0 rowsPerPage = 20 - sort: SortMeta = { field: 'createdAt', order: 1 } + sort: SortMeta = { field: 'createdAt', order: -1 } pagination: RestPagination = { count: this.rowsPerPage, start: 0 } scrollHeight = '' @@ -33,9 +36,13 @@ export class JobsListComponent extends RestTable implements OnInit { this.scrollHeight = (viewportHeight() - 380) + 'px' } + onJobStateChanged () { + this.loadData() + } + protected loadData () { this.jobsService - .getJobs(this.pagination, this.sort) + .getJobs(this.jobState, this.pagination, this.sort) .subscribe( resultList => { this.jobs = resultList.data diff --git a/client/src/app/+admin/jobs/shared/job.service.ts b/client/src/app/+admin/jobs/shared/job.service.ts index 61ee16077..a19d278c9 100644 --- a/client/src/app/+admin/jobs/shared/job.service.ts +++ b/client/src/app/+admin/jobs/shared/job.service.ts @@ -5,6 +5,7 @@ import 'rxjs/add/operator/catch' import 'rxjs/add/operator/map' import { Observable } from 'rxjs/Observable' import { ResultList } from '../../../../../../shared' +import { JobState } from '../../../../../../shared/models' import { Job } from '../../../../../../shared/models/job.model' import { environment } from '../../../../environments/environment' import { RestExtractor, RestPagination, RestService } from '../../../shared' @@ -19,19 +20,19 @@ export class JobService { private restExtractor: RestExtractor ) {} - getJobs (pagination: RestPagination, sort: SortMeta): Observable> { + getJobs (state: JobState, pagination: RestPagination, sort: SortMeta): Observable> { let params = new HttpParams() params = this.restService.addRestGetParams(params, pagination, sort) - return this.authHttp.get>(JobService.BASE_JOB_URL, { params }) + return this.authHttp.get>(JobService.BASE_JOB_URL + '/' + state, { params }) .map(res => this.restExtractor.convertResultListDateToHuman(res)) .map(res => this.restExtractor.applyToResultListData(res, this.prettyPrintData)) .catch(err => this.restExtractor.handleError(err)) } private prettyPrintData (obj: Job) { - const handlerInputData = JSON.stringify(obj.handlerInputData, null, 2) + const data = JSON.stringify(obj.data, null, 2) - return Object.assign(obj, { handlerInputData }) + return Object.assign(obj, { data }) } } diff --git a/config/default.yaml b/config/default.yaml index 3d0732069..fd04b5ce6 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -14,6 +14,11 @@ database: username: 'peertube' password: 'peertube' +redis: + hostname: 'localhost' + port: 6379 + auth: null + # From the project root directory storage: avatars: 'storage/avatars/' diff --git a/config/production.yaml.example b/config/production.yaml.example index 1288eeaa2..a2b332983 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example @@ -15,6 +15,11 @@ database: username: 'peertube' password: 'peertube' +redis: + hostname: 'localhost' + port: 6379 + auth: null + # From the project root directory storage: avatars: '/var/www/peertube/storage/avatars/' diff --git a/package.json b/package.json index 45651a1e5..db1bfe5d8 100644 --- a/package.json +++ b/package.json @@ -70,6 +70,7 @@ "js-yaml": "^3.5.4", "jsonld": "^0.5.12", "jsonld-signatures": "https://github.com/Chocobozzz/jsonld-signatures#rsa2017", + "kue": "^0.11.6", "lodash": "^4.11.1", "magnet-uri": "^5.1.4", "mkdirp": "^0.5.1", @@ -103,6 +104,7 @@ "@types/chai": "^4.0.4", "@types/config": "^0.0.33", "@types/express": "^4.0.35", + "@types/kue": "^0.11.8", "@types/lodash": "^4.14.64", "@types/magnet-uri": "^5.1.1", "@types/mkdirp": "^0.5.1", diff --git a/scripts/clean/server/test.sh b/scripts/clean/server/test.sh index 2ceb71244..b56021874 100755 --- a/scripts/clean/server/test.sh +++ b/scripts/clean/server/test.sh @@ -6,4 +6,5 @@ for i in $(seq 1 6); do rm -f "./config/local-test.json" rm -f "./config/local-test-$i.json" createdb "peertube_test$i" + redis-cli KEYS "q-localhost:900$i*" | grep -v empty | xargs --no-run-if-empty redis-cli DEL done diff --git a/scripts/parse-log.ts b/scripts/parse-log.ts index 7e804b3f9..9429512b7 100755 --- a/scripts/parse-log.ts +++ b/scripts/parse-log.ts @@ -2,16 +2,34 @@ import { createReadStream } from 'fs' import { join } from 'path' import { createInterface } from 'readline' import * as winston from 'winston' -import { labelFormatter, loggerFormat, timestampFormatter } from '../server/helpers/logger' +import { labelFormatter } from '../server/helpers/logger' import { CONFIG } from '../server/initializers/constants' +const excludedKeys = { + level: true, + message: true, + splat: true, + timestamp: true, + label: true +} +function keysExcluder (key, value) { + return excludedKeys[key] === true ? undefined : value +} + +const loggerFormat = winston.format.printf((info) => { + let additionalInfos = JSON.stringify(info, keysExcluder, 2) + if (additionalInfos === '{}') additionalInfos = '' + else additionalInfos = ' ' + additionalInfos + + return `[${info.label}] ${new Date(info.timestamp).toISOString()} ${info.level}: ${info.message}${additionalInfos}` +}) + const logger = new winston.createLogger({ transports: [ new winston.transports.Console({ level: 'debug', stderrLevels: [], format: winston.format.combine( - timestampFormatter, winston.format.splat(), labelFormatter, winston.format.colorize(), diff --git a/server.ts b/server.ts index a138b6359..d0b351c62 100644 --- a/server.ts +++ b/server.ts @@ -53,10 +53,11 @@ migrate() // ----------- PeerTube modules ----------- import { installApplication } from './server/initializers' -import { activitypubHttpJobScheduler, transcodingJobScheduler } from './server/lib/jobs' +import { JobQueue } from './server/lib/job-queue' import { VideosPreviewCache } from './server/lib/cache' import { apiRouter, clientsRouter, staticRouter, servicesRouter, webfingerRouter, activityPubRouter } from './server/controllers' import { BadActorFollowScheduler } from './server/lib/schedulers/bad-actor-follow-scheduler' +import { RemoveOldJobsScheduler } from './server/lib/schedulers/remove-old-jobs-scheduler' // ----------- Command line ----------- @@ -170,9 +171,8 @@ function onDatabaseInitDone () { server.listen(port, () => { VideosPreviewCache.Instance.init(CONFIG.CACHE.PREVIEWS.SIZE) BadActorFollowScheduler.Instance.enable() - - activitypubHttpJobScheduler.activate() - transcodingJobScheduler.activate() + RemoveOldJobsScheduler.Instance.enable() + JobQueue.Instance.init() logger.info('Server listening on port %d', port) logger.info('Web server: %s', CONFIG.WEBSERVER.URL) diff --git a/server/controllers/api/jobs.ts b/server/controllers/api/jobs.ts index de37dea39..132d110ad 100644 --- a/server/controllers/api/jobs.ts +++ b/server/controllers/api/jobs.ts @@ -1,22 +1,29 @@ import * as express from 'express' +import { ResultList } from '../../../shared' +import { Job, JobType, JobState } from '../../../shared/models' import { UserRight } from '../../../shared/models/users' -import { getFormattedObjects } from '../../helpers/utils' +import { JobQueue } from '../../lib/job-queue' import { - asyncMiddleware, authenticate, ensureUserHasRight, jobsSortValidator, setDefaultPagination, + asyncMiddleware, + authenticate, + ensureUserHasRight, + jobsSortValidator, + setDefaultPagination, setDefaultSort } from '../../middlewares' import { paginationValidator } from '../../middlewares/validators' -import { JobModel } from '../../models/job/job' +import { listJobsValidator } from '../../middlewares/validators/jobs' const jobsRouter = express.Router() -jobsRouter.get('/', +jobsRouter.get('/:state', authenticate, ensureUserHasRight(UserRight.MANAGE_JOBS), paginationValidator, jobsSortValidator, setDefaultSort, setDefaultPagination, + asyncMiddleware(listJobsValidator), asyncMiddleware(listJobs) ) @@ -29,7 +36,26 @@ export { // --------------------------------------------------------------------------- async function listJobs (req: express.Request, res: express.Response, next: express.NextFunction) { - const resultList = await JobModel.listForApi(req.query.start, req.query.count, req.query.sort) + const sort = req.query.sort === 'createdAt' ? 'asc' : 'desc' - return res.json(getFormattedObjects(resultList.data, resultList.total)) + const jobs = await JobQueue.Instance.listForApi(req.params.state, req.query.start, req.query.count, sort) + const total = await JobQueue.Instance.count(req.params.state) + + const result: ResultList = { + total, + data: jobs.map(j => formatJob(j.toJSON())) + } + return res.json(result) +} + +function formatJob (job: any): Job { + return { + id: job.id, + state: job.state as JobState, + type: job.type as JobType, + data: job.data, + error: job.error, + createdAt: new Date(parseInt(job.created_at, 10)), + updatedAt: new Date(parseInt(job.updated_at, 10)) + } } diff --git a/server/controllers/api/server/follows.ts b/server/controllers/api/server/follows.ts index 506b9668e..bb8713e7a 100644 --- a/server/controllers/api/server/follows.ts +++ b/server/controllers/api/server/follows.ts @@ -123,7 +123,7 @@ function follow (fromActor: ActorModel, targetActor: ActorModel) { actorFollow.ActorFollower = fromActor // Send a notification to remote server - await sendFollow(actorFollow, t) + await sendFollow(actorFollow) }) } diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts index c2fdb4f95..459795141 100644 --- a/server/controllers/api/videos/index.ts +++ b/server/controllers/api/videos/index.ts @@ -12,7 +12,7 @@ import { } from '../../../initializers' import { fetchRemoteVideoDescription, getVideoActivityPubUrl, shareVideoByServerAndChannel } from '../../../lib/activitypub' import { sendCreateVideo, sendCreateViewToOrigin, sendCreateViewToVideoFollowers, sendUpdateVideo } from '../../../lib/activitypub/send' -import { transcodingJobScheduler } from '../../../lib/jobs/transcoding-job-scheduler' +import { JobQueue } from '../../../lib/job-queue' import { asyncMiddleware, authenticate, paginationValidator, setDefaultSort, setDefaultPagination, videosAddValidator, videosGetValidator, videosRemoveValidator, videosSearchValidator, videosSortValidator, videosUpdateValidator @@ -176,18 +176,9 @@ async function addVideo (req: express.Request, res: express.Response, videoPhysi ) await Promise.all(tasks) - return sequelizeTypescript.transaction(async t => { + const videoCreated = await sequelizeTypescript.transaction(async t => { const sequelizeOptions = { transaction: t } - if (CONFIG.TRANSCODING.ENABLED === true) { - // Put uuid because we don't have id auto incremented for now - const dataInput = { - videoUUID: video.uuid - } - - await transcodingJobScheduler.createJob(t, 'videoFileOptimizer', dataInput) - } - const videoCreated = await video.save(sequelizeOptions) // Do not forget to add video channel information to the created video videoCreated.VideoChannel = res.locals.videoChannel @@ -216,6 +207,17 @@ async function addVideo (req: express.Request, res: express.Response, videoPhysi return videoCreated }) + + if (CONFIG.TRANSCODING.ENABLED === true) { + // Put uuid because we don't have id auto incremented for now + const dataInput = { + videoUUID: videoCreated.uuid + } + + await JobQueue.Instance.createJob({ type: 'video-file', payload: dataInput }) + } + + return videoCreated } async function updateVideoRetryWrapper (req: express.Request, res: express.Response, next: express.NextFunction) { diff --git a/server/helpers/custom-validators/jobs.ts b/server/helpers/custom-validators/jobs.ts new file mode 100644 index 000000000..9700fbd12 --- /dev/null +++ b/server/helpers/custom-validators/jobs.ts @@ -0,0 +1,14 @@ +import { JobState } from '../../../shared/models' +import { exists } from './misc' + +const jobStates: JobState[] = [ 'active', 'complete', 'failed', 'inactive', 'delayed' ] + +function isValidJobState (value: JobState) { + return exists(value) && jobStates.indexOf(value) !== -1 +} + +// --------------------------------------------------------------------------- + +export { + isValidJobState +} diff --git a/server/helpers/database-utils.ts b/server/helpers/database-utils.ts index 78ca768b9..b4adaf9cc 100644 --- a/server/helpers/database-utils.ts +++ b/server/helpers/database-utils.ts @@ -16,6 +16,7 @@ function retryTransactionWrapper ( .catch(err => callback(err)) }) .catch(err => { + console.error(err) logger.error(options.errorMessage, err) throw err }) diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index cb043251a..329d0ffe8 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -1,6 +1,6 @@ import { IConfig } from 'config' import { dirname, join } from 'path' -import { JobCategory, JobState, VideoRateType } from '../../shared/models' +import { JobType, VideoRateType } from '../../shared/models' import { ActivityPubActorType } from '../../shared/models/activitypub' import { FollowState } from '../../shared/models/actors' import { VideoPrivacy } from '../../shared/models/videos' @@ -12,7 +12,7 @@ let config: IConfig = require('config') // --------------------------------------------------------------------------- -const LAST_MIGRATION_VERSION = 175 +const LAST_MIGRATION_VERSION = 180 // --------------------------------------------------------------------------- @@ -26,7 +26,7 @@ const PAGINATION_COUNT_DEFAULT = 15 const SORTABLE_COLUMNS = { USERS: [ 'id', 'username', 'createdAt' ], ACCOUNTS: [ 'createdAt' ], - JOBS: [ 'id', 'createdAt' ], + JOBS: [ 'createdAt' ], VIDEO_ABUSES: [ 'id', 'createdAt' ], VIDEO_CHANNELS: [ 'id', 'name', 'updatedAt', 'createdAt' ], VIDEOS: [ 'name', 'duration', 'createdAt', 'views', 'likes' ], @@ -61,23 +61,20 @@ const REMOTE_SCHEME = { WS: 'wss' } -const JOB_STATES: { [ id: string ]: JobState } = { - PENDING: 'pending', - PROCESSING: 'processing', - ERROR: 'error', - SUCCESS: 'success' +const JOB_ATTEMPTS: { [ id in JobType ]: number } = { + 'activitypub-http-broadcast': 5, + 'activitypub-http-unicast': 5, + 'activitypub-http-fetcher': 5, + 'video-file': 1 } -const JOB_CATEGORIES: { [ id: string ]: JobCategory } = { - TRANSCODING: 'transcoding', - ACTIVITYPUB_HTTP: 'activitypub-http' +const JOB_CONCURRENCY: { [ id in JobType ]: number } = { + 'activitypub-http-broadcast': 1, + 'activitypub-http-unicast': 5, + 'activitypub-http-fetcher': 1, + 'video-file': 1 } -// How many maximum jobs we fetch from the database per cycle -const JOBS_FETCH_LIMIT_PER_CYCLE = { - transcoding: 10, - httpRequest: 20 -} -// 1 minutes -let JOBS_FETCHING_INTERVAL = 60000 +// 2 days +const JOB_COMPLETED_LIFETIME = 60000 * 60 * 24 * 2 // 1 hour let SCHEDULER_INTERVAL = 60000 * 60 @@ -96,6 +93,11 @@ const CONFIG = { USERNAME: config.get('database.username'), PASSWORD: config.get('database.password') }, + REDIS: { + HOSTNAME: config.get('redis.hostname'), + PORT: config.get('redis.port'), + AUTH: config.get('redis.auth') + }, STORAGE: { AVATARS_DIR: buildPath(config.get('storage.avatars')), LOG_DIR: buildPath(config.get('storage.logs')), @@ -284,7 +286,6 @@ const ACTIVITY_PUB = { PUBLIC: 'https://www.w3.org/ns/activitystreams#Public', COLLECTION_ITEMS_PER_PAGE: 10, FETCH_PAGE_LIMIT: 100, - MAX_HTTP_ATTEMPT: 5, URL_MIME_TYPES: { VIDEO: Object.keys(VIDEO_MIMETYPE_EXT), TORRENT: [ 'application/x-bittorrent' ], @@ -358,7 +359,6 @@ const OPENGRAPH_AND_OEMBED_COMMENT = '' // Special constants for a test instance if (isTestInstance() === true) { ACTOR_FOLLOW_SCORE.BASE = 20 - JOBS_FETCHING_INTERVAL = 1000 REMOTE_SCHEME.HTTP = 'http' REMOTE_SCHEME.WS = 'ws' STATIC_MAX_AGE = '0' @@ -381,10 +381,8 @@ export { CONFIG, CONSTRAINTS_FIELDS, EMBED_SIZE, - JOB_STATES, - JOBS_FETCH_LIMIT_PER_CYCLE, - JOBS_FETCHING_INTERVAL, - JOB_CATEGORIES, + JOB_CONCURRENCY, + JOB_ATTEMPTS, LAST_MIGRATION_VERSION, OAUTH_LIFETIME, OPENGRAPH_AND_OEMBED_COMMENT, @@ -408,7 +406,8 @@ export { VIDEO_RATE_TYPES, VIDEO_MIMETYPE_EXT, AVATAR_MIMETYPE_EXT, - SCHEDULER_INTERVAL + SCHEDULER_INTERVAL, + JOB_COMPLETED_LIFETIME } // --------------------------------------------------------------------------- diff --git a/server/initializers/database.ts b/server/initializers/database.ts index 852db68a0..b537ee59a 100644 --- a/server/initializers/database.ts +++ b/server/initializers/database.ts @@ -9,7 +9,6 @@ import { ActorModel } from '../models/activitypub/actor' import { ActorFollowModel } from '../models/activitypub/actor-follow' import { ApplicationModel } from '../models/application/application' import { AvatarModel } from '../models/avatar/avatar' -import { JobModel } from '../models/job/job' import { OAuthClientModel } from '../models/oauth/oauth-client' import { OAuthTokenModel } from '../models/oauth/oauth-token' import { ServerModel } from '../models/server/server' @@ -61,7 +60,6 @@ async function initDatabaseModels (silent: boolean) { ActorFollowModel, AvatarModel, AccountModel, - JobModel, OAuthClientModel, OAuthTokenModel, ServerModel, diff --git a/server/initializers/migrations/0100-activitypub.ts b/server/initializers/migrations/0100-activitypub.ts index 8c5198f85..a7ebd804c 100644 --- a/server/initializers/migrations/0100-activitypub.ts +++ b/server/initializers/migrations/0100-activitypub.ts @@ -1,11 +1,10 @@ -import { values } from 'lodash' import * as Sequelize from 'sequelize' import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto' import { shareVideoByServerAndChannel } from '../../lib/activitypub/share' import { getVideoActivityPubUrl, getVideoChannelActivityPubUrl } from '../../lib/activitypub/url' import { createLocalAccountWithoutKeys } from '../../lib/user' import { ApplicationModel } from '../../models/application/application' -import { JOB_CATEGORIES, SERVER_ACTOR_NAME } from '../constants' +import { SERVER_ACTOR_NAME } from '../constants' async function up (utils: { transaction: Sequelize.Transaction, @@ -161,7 +160,7 @@ async function up (utils: { { const data = { - type: Sequelize.ENUM(values(JOB_CATEGORIES)), + type: Sequelize.ENUM('transcoding', 'activitypub-http'), defaultValue: 'transcoding', allowNull: false } diff --git a/server/initializers/migrations/0180-job-table-delete.ts b/server/initializers/migrations/0180-job-table-delete.ts new file mode 100644 index 000000000..df29145d0 --- /dev/null +++ b/server/initializers/migrations/0180-job-table-delete.ts @@ -0,0 +1,18 @@ +import * as Sequelize from 'sequelize' + +async function up (utils: { + transaction: Sequelize.Transaction, + queryInterface: Sequelize.QueryInterface, + sequelize: Sequelize.Sequelize +}): Promise { + await utils.queryInterface.dropTable('job') +} + +function down (options) { + throw new Error('Not implemented.') +} + +export { + up, + down +} diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts index c708b38ba..712de7d0d 100644 --- a/server/lib/activitypub/actor.ts +++ b/server/lib/activitypub/actor.ts @@ -64,7 +64,11 @@ async function getOrCreateActorAndServerAndModel (actorUrl: string, recurseIfNee actor = await retryTransactionWrapper(saveActorAndServerAndModelIfNotExist, options) } - return refreshActorIfNeeded(actor) + const options = { + arguments: [ actor ], + errorMessage: 'Cannot refresh actor if needed with many retries.' + } + return retryTransactionWrapper(refreshActorIfNeeded, options) } function buildActorInstance (type: ActivityPubActorType, url: string, preferredUsername: string, uuid?: string) { @@ -325,38 +329,43 @@ async function saveVideoChannel (actor: ActorModel, result: FetchRemoteActorResu async function refreshActorIfNeeded (actor: ActorModel) { if (!actor.isOutdated()) return actor - const actorUrl = await getUrlFromWebfinger(actor.preferredUsername, actor.getHost()) - const result = await fetchRemoteActor(actorUrl) - if (result === undefined) { - logger.warn('Cannot fetch remote actor in refresh actor.') + try { + const actorUrl = await getUrlFromWebfinger(actor.preferredUsername, actor.getHost()) + const result = await fetchRemoteActor(actorUrl) + if (result === undefined) { + logger.warn('Cannot fetch remote actor in refresh actor.') + return actor + } + + return sequelizeTypescript.transaction(async t => { + updateInstanceWithAnother(actor, result.actor) + + if (result.avatarName !== undefined) { + await updateActorAvatarInstance(actor, result.avatarName, t) + } + + // Force update + actor.setDataValue('updatedAt', new Date()) + await actor.save({ transaction: t }) + + if (actor.Account) { + await actor.save({ transaction: t }) + + actor.Account.set('name', result.name) + await actor.Account.save({ transaction: t }) + } else if (actor.VideoChannel) { + await actor.save({ transaction: t }) + + actor.VideoChannel.set('name', result.name) + await actor.VideoChannel.save({ transaction: t }) + } + + return actor + }) + } catch (err) { + logger.warn('Cannot refresh actor.', err) return actor } - - return sequelizeTypescript.transaction(async t => { - updateInstanceWithAnother(actor, result.actor) - - if (result.avatarName !== undefined) { - await updateActorAvatarInstance(actor, result.avatarName, t) - } - - // Force update - actor.setDataValue('updatedAt', new Date()) - await actor.save({ transaction: t }) - - if (actor.Account) { - await actor.save({ transaction: t }) - - actor.Account.set('name', result.name) - await actor.Account.save({ transaction: t }) - } else if (actor.VideoChannel) { - await actor.save({ transaction: t }) - - actor.VideoChannel.set('name', result.name) - await actor.VideoChannel.save({ transaction: t }) - } - - return actor - }) } function normalizeActor (actor: any) { diff --git a/server/lib/activitypub/fetch.ts b/server/lib/activitypub/fetch.ts index 4fc97cc38..b1b370a1a 100644 --- a/server/lib/activitypub/fetch.ts +++ b/server/lib/activitypub/fetch.ts @@ -1,13 +1,12 @@ -import { Transaction } from 'sequelize' import { ActorModel } from '../../models/activitypub/actor' -import { activitypubHttpJobScheduler, ActivityPubHttpPayload } from '../jobs/activitypub-http-job-scheduler' +import { JobQueue } from '../job-queue' -async function addFetchOutboxJob (actor: ActorModel, t: Transaction) { - const jobPayload: ActivityPubHttpPayload = { +async function addFetchOutboxJob (actor: ActorModel) { + const payload = { uris: [ actor.outboxUrl ] } - return activitypubHttpJobScheduler.createJob(t, 'activitypubHttpFetcherHandler', jobPayload) + return JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }) } export { diff --git a/server/lib/activitypub/process/process-accept.ts b/server/lib/activitypub/process/process-accept.ts index 551f09ea7..7db2f8ff0 100644 --- a/server/lib/activitypub/process/process-accept.ts +++ b/server/lib/activitypub/process/process-accept.ts @@ -26,6 +26,6 @@ async function processAccept (actor: ActorModel, targetActor: ActorModel) { if (follow.state !== 'accepted') { follow.set('state', 'accepted') await follow.save() - await addFetchOutboxJob(targetActor, undefined) + await addFetchOutboxJob(targetActor) } } diff --git a/server/lib/activitypub/process/process-follow.ts b/server/lib/activitypub/process/process-follow.ts index 69f5c51b5..dc1d542b5 100644 --- a/server/lib/activitypub/process/process-follow.ts +++ b/server/lib/activitypub/process/process-follow.ts @@ -63,7 +63,7 @@ async function follow (actor: ActorModel, targetActorURL: string) { actorFollow.ActorFollowing = targetActor // Target sends to actor he accepted the follow request - return sendAccept(actorFollow, t) + return sendAccept(actorFollow) }) logger.info('Actor %s is followed by actor %s.', targetActorURL, actor.url) diff --git a/server/lib/activitypub/send/misc.ts b/server/lib/activitypub/send/misc.ts index dc0d3de57..7a21f0c94 100644 --- a/server/lib/activitypub/send/misc.ts +++ b/server/lib/activitypub/send/misc.ts @@ -7,7 +7,7 @@ import { ActorFollowModel } from '../../../models/activitypub/actor-follow' import { VideoModel } from '../../../models/video/video' import { VideoCommentModel } from '../../../models/video/video-comment' import { VideoShareModel } from '../../../models/video/video-share' -import { activitypubHttpJobScheduler, ActivityPubHttpPayload } from '../../jobs/activitypub-http-job-scheduler' +import { JobQueue } from '../../job-queue' async function forwardActivity ( activity: Activity, @@ -35,12 +35,11 @@ async function forwardActivity ( logger.debug('Creating forwarding job.', { uris }) - const jobPayload: ActivityPubHttpPayload = { + const payload = { uris, body: activity } - - return activitypubHttpJobScheduler.createJob(t, 'activitypubHttpBroadcastHandler', jobPayload) + return JobQueue.Instance.createJob({ type: 'activitypub-http-broadcast', payload }) } async function broadcastToFollowers ( @@ -51,44 +50,43 @@ async function broadcastToFollowers ( actorsException: ActorModel[] = [] ) { const uris = await computeFollowerUris(toActorFollowers, actorsException, t) - return broadcastTo(uris, data, byActor, t) + return broadcastTo(uris, data, byActor) } async function broadcastToActors ( data: any, byActor: ActorModel, toActors: ActorModel[], - t: Transaction, actorsException: ActorModel[] = [] ) { const uris = await computeUris(toActors, actorsException) - return broadcastTo(uris, data, byActor, t) + return broadcastTo(uris, data, byActor) } -async function broadcastTo (uris: string[], data: any, byActor: ActorModel, t: Transaction) { +async function broadcastTo (uris: string[], data: any, byActor: ActorModel) { if (uris.length === 0) return undefined logger.debug('Creating broadcast job.', { uris }) - const jobPayload: ActivityPubHttpPayload = { + const payload = { uris, signatureActorId: byActor.id, body: data } - return activitypubHttpJobScheduler.createJob(t, 'activitypubHttpBroadcastHandler', jobPayload) + return JobQueue.Instance.createJob({ type: 'activitypub-http-broadcast', payload }) } -async function unicastTo (data: any, byActor: ActorModel, toActorUrl: string, t: Transaction) { +async function unicastTo (data: any, byActor: ActorModel, toActorUrl: string) { logger.debug('Creating unicast job.', { uri: toActorUrl }) - const jobPayload: ActivityPubHttpPayload = { - uris: [ toActorUrl ], + const payload = { + uri: toActorUrl, signatureActorId: byActor.id, body: data } - return activitypubHttpJobScheduler.createJob(t, 'activitypubHttpUnicastHandler', jobPayload) + return JobQueue.Instance.createJob({ type: 'activitypub-http-unicast', payload }) } function getOriginVideoAudience (video: VideoModel, actorsInvolvedInVideo: ActorModel[]) { diff --git a/server/lib/activitypub/send/send-accept.ts b/server/lib/activitypub/send/send-accept.ts index 4eaa329d9..064fd88d2 100644 --- a/server/lib/activitypub/send/send-accept.ts +++ b/server/lib/activitypub/send/send-accept.ts @@ -1,4 +1,3 @@ -import { Transaction } from 'sequelize' import { ActivityAccept, ActivityFollow } from '../../../../shared/models/activitypub' import { ActorModel } from '../../../models/activitypub/actor' import { ActorFollowModel } from '../../../models/activitypub/actor-follow' @@ -6,7 +5,7 @@ import { getActorFollowAcceptActivityPubUrl, getActorFollowActivityPubUrl } from import { unicastTo } from './misc' import { followActivityData } from './send-follow' -async function sendAccept (actorFollow: ActorFollowModel, t: Transaction) { +async function sendAccept (actorFollow: ActorFollowModel) { const follower = actorFollow.ActorFollower const me = actorFollow.ActorFollowing @@ -16,7 +15,7 @@ async function sendAccept (actorFollow: ActorFollowModel, t: Transaction) { const url = getActorFollowAcceptActivityPubUrl(actorFollow) const data = acceptActivityData(url, me, followData) - return unicastTo(data, me, follower.inboxUrl, t) + return unicastTo(data, me, follower.inboxUrl) } // --------------------------------------------------------------------------- diff --git a/server/lib/activitypub/send/send-announce.ts b/server/lib/activitypub/send/send-announce.ts index 578fbc630..93b5668d2 100644 --- a/server/lib/activitypub/send/send-announce.ts +++ b/server/lib/activitypub/send/send-announce.ts @@ -42,7 +42,7 @@ async function sendVideoAnnounceToOrigin (byActor: ActorModel, video: VideoModel const audience = getOriginVideoAudience(video, actorsInvolvedInVideo) const data = await createActivityData(url, byActor, announcedActivity, t, audience) - return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl, t) + return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) } async function announceActivityData ( diff --git a/server/lib/activitypub/send/send-create.ts b/server/lib/activitypub/send/send-create.ts index 9db663be1..b92615e9b 100644 --- a/server/lib/activitypub/send/send-create.ts +++ b/server/lib/activitypub/send/send-create.ts @@ -8,8 +8,14 @@ import { VideoAbuseModel } from '../../../models/video/video-abuse' import { VideoCommentModel } from '../../../models/video/video-comment' import { getVideoAbuseActivityPubUrl, getVideoDislikeActivityPubUrl, getVideoViewActivityPubUrl } from '../url' import { - audiencify, broadcastToActors, broadcastToFollowers, getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience, - getOriginVideoAudience, getOriginVideoCommentAudience, + audiencify, + broadcastToActors, + broadcastToFollowers, + getActorsInvolvedInVideo, + getAudience, + getObjectFollowersAudience, + getOriginVideoAudience, + getOriginVideoCommentAudience, unicastTo } from './misc' @@ -31,7 +37,7 @@ async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, const audience = { to: [ video.VideoChannel.Account.Actor.url ], cc: [] } const data = await createActivityData(url, byActor, videoAbuse.toActivityPubObject(), t, audience) - return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl, t) + return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) } async function sendCreateVideoCommentToOrigin (comment: VideoCommentModel, t: Transaction) { @@ -47,13 +53,13 @@ async function sendCreateVideoCommentToOrigin (comment: VideoCommentModel, t: Tr // This was a reply, send it to the parent actors const actorsException = [ byActor ] - await broadcastToActors(data, byActor, threadParentComments.map(c => c.Account.Actor), t, actorsException) + await broadcastToActors(data, byActor, threadParentComments.map(c => c.Account.Actor), actorsException) // Broadcast to our followers await broadcastToFollowers(data, byActor, [ byActor ], t) // Send to origin - return unicastTo(data, byActor, comment.Video.VideoChannel.Account.Actor.sharedInboxUrl, t) + return unicastTo(data, byActor, comment.Video.VideoChannel.Account.Actor.sharedInboxUrl) } async function sendCreateVideoCommentToVideoFollowers (comment: VideoCommentModel, t: Transaction) { @@ -69,7 +75,7 @@ async function sendCreateVideoCommentToVideoFollowers (comment: VideoCommentMode // This was a reply, send it to the parent actors const actorsException = [ byActor ] - await broadcastToActors(data, byActor, threadParentComments.map(c => c.Account.Actor), t, actorsException) + await broadcastToActors(data, byActor, threadParentComments.map(c => c.Account.Actor), actorsException) // Broadcast to our followers await broadcastToFollowers(data, byActor, [ byActor ], t) @@ -86,7 +92,7 @@ async function sendCreateViewToOrigin (byActor: ActorModel, video: VideoModel, t const audience = getOriginVideoAudience(video, actorsInvolvedInVideo) const data = await createActivityData(url, byActor, viewActivityData, t, audience) - return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl, t) + return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) } async function sendCreateViewToVideoFollowers (byActor: ActorModel, video: VideoModel, t: Transaction) { @@ -111,7 +117,7 @@ async function sendCreateDislikeToOrigin (byActor: ActorModel, video: VideoModel const audience = getOriginVideoAudience(video, actorsInvolvedInVideo) const data = await createActivityData(url, byActor, dislikeActivityData, t, audience) - return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl, t) + return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) } async function sendCreateDislikeToVideoFollowers (byActor: ActorModel, video: VideoModel, t: Transaction) { diff --git a/server/lib/activitypub/send/send-follow.ts b/server/lib/activitypub/send/send-follow.ts index eac60e94f..4e9865af4 100644 --- a/server/lib/activitypub/send/send-follow.ts +++ b/server/lib/activitypub/send/send-follow.ts @@ -1,18 +1,17 @@ -import { Transaction } from 'sequelize' import { ActivityFollow } from '../../../../shared/models/activitypub' import { ActorModel } from '../../../models/activitypub/actor' import { ActorFollowModel } from '../../../models/activitypub/actor-follow' import { getActorFollowActivityPubUrl } from '../url' import { unicastTo } from './misc' -function sendFollow (actorFollow: ActorFollowModel, t: Transaction) { +function sendFollow (actorFollow: ActorFollowModel) { const me = actorFollow.ActorFollower const following = actorFollow.ActorFollowing const url = getActorFollowActivityPubUrl(actorFollow) const data = followActivityData(url, me, following) - return unicastTo(data, me, following.inboxUrl, t) + return unicastTo(data, me, following.inboxUrl) } function followActivityData (url: string, byActor: ActorModel, targetActor: ActorModel): ActivityFollow { diff --git a/server/lib/activitypub/send/send-like.ts b/server/lib/activitypub/send/send-like.ts index 743646455..78ed1aaf2 100644 --- a/server/lib/activitypub/send/send-like.ts +++ b/server/lib/activitypub/send/send-like.ts @@ -20,7 +20,7 @@ async function sendLikeToOrigin (byActor: ActorModel, video: VideoModel, t: Tran const audience = getOriginVideoAudience(video, accountsInvolvedInVideo) const data = await likeActivityData(url, byActor, video, t, audience) - return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl, t) + return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) } async function sendLikeToVideoFollowers (byActor: ActorModel, video: VideoModel, t: Transaction) { diff --git a/server/lib/activitypub/send/send-undo.ts b/server/lib/activitypub/send/send-undo.ts index 3a0597fba..4a08b5ca1 100644 --- a/server/lib/activitypub/send/send-undo.ts +++ b/server/lib/activitypub/send/send-undo.ts @@ -1,11 +1,5 @@ import { Transaction } from 'sequelize' -import { - ActivityAudience, - ActivityCreate, - ActivityFollow, - ActivityLike, - ActivityUndo -} from '../../../../shared/models/activitypub' +import { ActivityAudience, ActivityCreate, ActivityFollow, ActivityLike, ActivityUndo } from '../../../../shared/models/activitypub' import { ActorModel } from '../../../models/activitypub/actor' import { ActorFollowModel } from '../../../models/activitypub/actor-follow' import { VideoModel } from '../../../models/video/video' @@ -33,7 +27,7 @@ async function sendUndoFollow (actorFollow: ActorFollowModel, t: Transaction) { const object = followActivityData(followUrl, me, following) const data = await undoActivityData(undoUrl, me, object, t) - return unicastTo(data, me, following.inboxUrl, t) + return unicastTo(data, me, following.inboxUrl) } async function sendUndoLikeToOrigin (byActor: ActorModel, video: VideoModel, t: Transaction) { @@ -45,7 +39,7 @@ async function sendUndoLikeToOrigin (byActor: ActorModel, video: VideoModel, t: const object = await likeActivityData(likeUrl, byActor, video, t) const data = await undoActivityData(undoUrl, byActor, object, t, audience) - return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl, t) + return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) } async function sendUndoLikeToVideoFollowers (byActor: ActorModel, video: VideoModel, t: Transaction) { @@ -72,7 +66,7 @@ async function sendUndoDislikeToOrigin (byActor: ActorModel, video: VideoModel, const data = await undoActivityData(undoUrl, byActor, object, t, audience) - return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl, t) + return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) } async function sendUndoDislikeToVideoFollowers (byActor: ActorModel, video: VideoModel, t: Transaction) { diff --git a/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-broadcast-handler.ts b/server/lib/job-queue/handlers/activitypub-http-broadcast.ts similarity index 57% rename from server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-broadcast-handler.ts rename to server/lib/job-queue/handlers/activitypub-http-broadcast.ts index 3f780e319..159856cda 100644 --- a/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-broadcast-handler.ts +++ b/server/lib/job-queue/handlers/activitypub-http-broadcast.ts @@ -1,10 +1,19 @@ +import * as kue from 'kue' import { logger } from '../../../helpers/logger' import { doRequest } from '../../../helpers/requests' import { ActorFollowModel } from '../../../models/activitypub/actor-follow' -import { ActivityPubHttpPayload, buildSignedRequestOptions, computeBody, maybeRetryRequestLater } from './activitypub-http-job-scheduler' +import { buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils' -async function process (payload: ActivityPubHttpPayload, jobId: number) { - logger.info('Processing ActivityPub broadcast in job %d.', jobId) +export type ActivitypubHttpBroadcastPayload = { + uris: string[] + signatureActorId?: number + body: any +} + +async function processActivityPubHttpBroadcast (job: kue.Job) { + logger.info('Processing ActivityPub broadcast in job %d.', job.id) + + const payload = job.data as ActivitypubHttpBroadcastPayload const body = await computeBody(payload) const httpSignatureOptions = await buildSignedRequestOptions(payload) @@ -26,28 +35,15 @@ async function process (payload: ActivityPubHttpPayload, jobId: number) { await doRequest(options) goodUrls.push(uri) } catch (err) { - const isRetryingLater = await maybeRetryRequestLater(err, payload, uri) - if (isRetryingLater === false) badUrls.push(uri) + badUrls.push(uri) } } return ActorFollowModel.updateActorFollowsScoreAndRemoveBadOnes(goodUrls, badUrls, undefined) } -function onError (err: Error, jobId: number) { - logger.error('Error when broadcasting ActivityPub request in job %d.', jobId, err) - return Promise.resolve() -} - -function onSuccess (jobId: number) { - logger.info('Job %d is a success.', jobId) - return Promise.resolve() -} - // --------------------------------------------------------------------------- export { - process, - onError, - onSuccess + processActivityPubHttpBroadcast } diff --git a/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-fetcher-handler.ts b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts similarity index 73% rename from server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-fetcher-handler.ts rename to server/lib/job-queue/handlers/activitypub-http-fetcher.ts index a7b5aabd0..062211c85 100644 --- a/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-fetcher-handler.ts +++ b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts @@ -1,11 +1,18 @@ +import * as kue from 'kue' import { logger } from '../../../helpers/logger' import { doRequest } from '../../../helpers/requests' import { ACTIVITY_PUB } from '../../../initializers' import { processActivities } from '../../activitypub/process' -import { ActivityPubHttpPayload } from './activitypub-http-job-scheduler' +import { ActivitypubHttpBroadcastPayload } from './activitypub-http-broadcast' -async function process (payload: ActivityPubHttpPayload, jobId: number) { - logger.info('Processing ActivityPub fetcher in job %d.', jobId) +export type ActivitypubHttpFetcherPayload = { + uris: string[] +} + +async function processActivityPubHttpFetcher (job: kue.Job) { + logger.info('Processing ActivityPub fetcher in job %d.', job.id) + + const payload = job.data as ActivitypubHttpBroadcastPayload const options = { method: 'GET', @@ -49,20 +56,8 @@ async function process (payload: ActivityPubHttpPayload, jobId: number) { } } -function onError (err: Error, jobId: number) { - logger.error('Error when fetcher ActivityPub request in job %d.', jobId, err) - return Promise.resolve() -} - -function onSuccess (jobId: number) { - logger.info('Job %d is a success.', jobId) - return Promise.resolve() -} - // --------------------------------------------------------------------------- export { - process, - onError, - onSuccess + processActivityPubHttpFetcher } diff --git a/server/lib/job-queue/handlers/activitypub-http-unicast.ts b/server/lib/job-queue/handlers/activitypub-http-unicast.ts new file mode 100644 index 000000000..9b4188c50 --- /dev/null +++ b/server/lib/job-queue/handlers/activitypub-http-unicast.ts @@ -0,0 +1,43 @@ +import * as kue from 'kue' +import { logger } from '../../../helpers/logger' +import { doRequest } from '../../../helpers/requests' +import { ActorFollowModel } from '../../../models/activitypub/actor-follow' +import { buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils' + +export type ActivitypubHttpUnicastPayload = { + uri: string + signatureActorId?: number + body: any +} + +async function processActivityPubHttpUnicast (job: kue.Job) { + logger.info('Processing ActivityPub unicast in job %d.', job.id) + + const payload = job.data as ActivitypubHttpUnicastPayload + const uri = payload.uri + + const body = await computeBody(payload) + const httpSignatureOptions = await buildSignedRequestOptions(payload) + + const options = { + method: 'POST', + uri, + json: body, + httpSignature: httpSignatureOptions + } + + try { + await doRequest(options) + ActorFollowModel.updateActorFollowsScoreAndRemoveBadOnes([ uri ], [], undefined) + } catch (err) { + ActorFollowModel.updateActorFollowsScoreAndRemoveBadOnes([], [ uri ], undefined) + + throw err + } +} + +// --------------------------------------------------------------------------- + +export { + processActivityPubHttpUnicast +} diff --git a/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts b/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts new file mode 100644 index 000000000..c087371c6 --- /dev/null +++ b/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts @@ -0,0 +1,39 @@ +import { buildSignedActivity } from '../../../../helpers/activitypub' +import { getServerActor } from '../../../../helpers/utils' +import { ActorModel } from '../../../../models/activitypub/actor' + +async function computeBody (payload: { body: any, signatureActorId?: number }) { + let body = payload.body + + if (payload.signatureActorId) { + const actorSignature = await ActorModel.load(payload.signatureActorId) + if (!actorSignature) throw new Error('Unknown signature actor id.') + body = await buildSignedActivity(actorSignature, payload.body) + } + + return body +} + +async function buildSignedRequestOptions (payload: { signatureActorId?: number }) { + let actor: ActorModel + if (payload.signatureActorId) { + actor = await ActorModel.load(payload.signatureActorId) + if (!actor) throw new Error('Unknown signature actor id.') + } else { + // We need to sign the request, so use the server + actor = await getServerActor() + } + + const keyId = actor.getWebfingerUrl() + return { + algorithm: 'rsa-sha256', + authorizationHeaderName: 'Signature', + keyId, + key: actor.privateKey + } +} + +export { + computeBody, + buildSignedRequestOptions +} diff --git a/server/lib/jobs/transcoding-job-scheduler/video-file-optimizer-handler.ts b/server/lib/job-queue/handlers/video-file.ts similarity index 60% rename from server/lib/jobs/transcoding-job-scheduler/video-file-optimizer-handler.ts rename to server/lib/job-queue/handlers/video-file.ts index f224a31b4..5294483bd 100644 --- a/server/lib/jobs/transcoding-job-scheduler/video-file-optimizer-handler.ts +++ b/server/lib/job-queue/handlers/video-file.ts @@ -1,37 +1,59 @@ -import * as Bluebird from 'bluebird' +import * as kue from 'kue' +import { VideoResolution } from '../../../../shared' import { VideoPrivacy } from '../../../../shared/models/videos' import { logger } from '../../../helpers/logger' import { computeResolutionsToTranscode } from '../../../helpers/utils' import { sequelizeTypescript } from '../../../initializers' -import { JobModel } from '../../../models/job/job' import { VideoModel } from '../../../models/video/video' import { shareVideoByServerAndChannel } from '../../activitypub' -import { sendCreateVideo } from '../../activitypub/send' -import { JobScheduler } from '../job-scheduler' -import { TranscodingJobPayload } from './transcoding-job-scheduler' +import { sendCreateVideo, sendUpdateVideo } from '../../activitypub/send' +import { JobQueue } from '../job-queue' -async function process (data: TranscodingJobPayload, jobId: number) { - const video = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(data.videoUUID) +export type VideoFilePayload = { + videoUUID: string + resolution?: VideoResolution +} + +async function processVideoFile (job: kue.Job) { + const payload = job.data as VideoFilePayload + logger.info('Processing video file in job %d.', job.id) + + const video = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(payload.videoUUID) // No video, maybe deleted? if (!video) { - logger.info('Do not process job %d, video does not exist.', jobId, { videoUUID: video.uuid }) + logger.info('Do not process job %d, video does not exist.', job.id, { videoUUID: video.uuid }) return undefined } - await video.optimizeOriginalVideofile() + // Transcoding in other resolution + if (payload.resolution) { + await video.transcodeOriginalVideofile(payload.resolution) + await onVideoFileTranscoderSuccess(video) + } else { + await video.optimizeOriginalVideofile() + await onVideoFileOptimizerSuccess(video) + } return video } -function onError (err: Error, jobId: number) { - logger.error('Error when optimized video file in job %d.', jobId, err) - return Promise.resolve() -} - -async function onSuccess (jobId: number, video: VideoModel, jobScheduler: JobScheduler) { +async function onVideoFileTranscoderSuccess (video: VideoModel) { if (video === undefined) return undefined - logger.info('Job %d is a success.', jobId) + // Maybe the video changed in database, refresh it + const videoDatabase = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(video.uuid) + // Video does not exist anymore + if (!videoDatabase) return undefined + + if (video.privacy !== VideoPrivacy.PRIVATE) { + await sendUpdateVideo(video, undefined) + } + + return undefined +} + +async function onVideoFileOptimizerSuccess (video: VideoModel) { + if (video === undefined) return undefined // Maybe the video changed in database, refresh it const videoDatabase = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(video.uuid) @@ -56,7 +78,7 @@ async function onSuccess (jobId: number, video: VideoModel, jobScheduler: JobSch if (resolutionsEnabled.length !== 0) { try { await sequelizeTypescript.transaction(async t => { - const tasks: Bluebird[] = [] + const tasks: Promise[] = [] for (const resolution of resolutionsEnabled) { const dataInput = { @@ -64,7 +86,7 @@ async function onSuccess (jobId: number, video: VideoModel, jobScheduler: JobSch resolution } - const p = jobScheduler.createJob(t, 'videoFileTranscoder', dataInput) + const p = JobQueue.Instance.createJob({ type: 'video-file', payload: dataInput }) tasks.push(p) } @@ -84,7 +106,5 @@ async function onSuccess (jobId: number, video: VideoModel, jobScheduler: JobSch // --------------------------------------------------------------------------- export { - process, - onError, - onSuccess + processVideoFile } diff --git a/server/lib/job-queue/index.ts b/server/lib/job-queue/index.ts new file mode 100644 index 000000000..57231e649 --- /dev/null +++ b/server/lib/job-queue/index.ts @@ -0,0 +1 @@ +export * from './job-queue' diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts new file mode 100644 index 000000000..7a2b6c78d --- /dev/null +++ b/server/lib/job-queue/job-queue.ts @@ -0,0 +1,124 @@ +import * as kue from 'kue' +import { JobType, JobState } from '../../../shared/models' +import { logger } from '../../helpers/logger' +import { CONFIG, JOB_ATTEMPTS, JOB_COMPLETED_LIFETIME, JOB_CONCURRENCY } from '../../initializers' +import { ActivitypubHttpBroadcastPayload, processActivityPubHttpBroadcast } from './handlers/activitypub-http-broadcast' +import { ActivitypubHttpFetcherPayload, processActivityPubHttpFetcher } from './handlers/activitypub-http-fetcher' +import { ActivitypubHttpUnicastPayload, processActivityPubHttpUnicast } from './handlers/activitypub-http-unicast' +import { processVideoFile, VideoFilePayload } from './handlers/video-file' + +type CreateJobArgument = + { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | + { type: 'activitypub-http-unicast', payload: ActivitypubHttpUnicastPayload } | + { type: 'activitypub-http-fetcher', payload: ActivitypubHttpFetcherPayload } | + { type: 'video-file', payload: VideoFilePayload } + +const handlers: { [ id in JobType ]: (job: kue.Job) => Promise} = { + 'activitypub-http-broadcast': processActivityPubHttpBroadcast, + 'activitypub-http-unicast': processActivityPubHttpUnicast, + 'activitypub-http-fetcher': processActivityPubHttpFetcher, + 'video-file': processVideoFile +} + +class JobQueue { + + private static instance: JobQueue + + private jobQueue: kue.Queue + private initialized = false + + private constructor () {} + + init () { + // Already initialized + if (this.initialized === true) return + this.initialized = true + + this.jobQueue = kue.createQueue({ + prefix: 'q-' + CONFIG.WEBSERVER.HOST, + redis: { + host: CONFIG.REDIS.HOSTNAME, + port: CONFIG.REDIS.PORT, + auth: CONFIG.REDIS.AUTH + } + }) + + this.jobQueue.on('error', err => { + logger.error('Error in job queue.', err) + process.exit(-1) + }) + this.jobQueue.watchStuckJobs(5000) + + for (const handlerName of Object.keys(handlers)) { + this.jobQueue.process(handlerName, JOB_CONCURRENCY[handlerName], async (job, done) => { + try { + const res = await handlers[ handlerName ](job) + return done(null, res) + } catch (err) { + return done(err) + } + }) + } + } + + createJob (obj: CreateJobArgument, priority = 'normal') { + return new Promise((res, rej) => { + this.jobQueue + .create(obj.type, obj.payload) + .priority(priority) + .attempts(JOB_ATTEMPTS[obj.type]) + .backoff({ type: 'exponential' }) + .save(err => { + if (err) return rej(err) + + return res() + }) + }) + } + + listForApi (state: JobState, start: number, count: number, sort: string) { + return new Promise((res, rej) => { + kue.Job.rangeByState(state, start, count, sort, (err, jobs) => { + if (err) return rej(err) + + return res(jobs) + }) + }) + } + + count (state: JobState) { + return new Promise((res, rej) => { + this.jobQueue[state + 'Count']((err, total) => { + if (err) return rej(err) + + return res(total) + }) + }) + } + + removeOldJobs () { + const now = new Date().getTime() + kue.Job.rangeByState('complete', 0, -1, 'asc', (err, jobs) => { + if (err) { + logger.error('Cannot get jobs when removing old jobs.', err) + return + } + + for (const job of jobs) { + if (now - job.created_at > JOB_COMPLETED_LIFETIME) { + job.remove() + } + } + }) + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} + +// --------------------------------------------------------------------------- + +export { + JobQueue +} diff --git a/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-job-scheduler.ts b/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-job-scheduler.ts deleted file mode 100644 index 4459152db..000000000 --- a/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-job-scheduler.ts +++ /dev/null @@ -1,94 +0,0 @@ -import { JobCategory } from '../../../../shared' -import { buildSignedActivity } from '../../../helpers/activitypub' -import { logger } from '../../../helpers/logger' -import { getServerActor } from '../../../helpers/utils' -import { ACTIVITY_PUB } from '../../../initializers' -import { ActorModel } from '../../../models/activitypub/actor' -import { ActorFollowModel } from '../../../models/activitypub/actor-follow' -import { JobHandler, JobScheduler } from '../job-scheduler' - -import * as activitypubHttpBroadcastHandler from './activitypub-http-broadcast-handler' -import * as activitypubHttpFetcherHandler from './activitypub-http-fetcher-handler' -import * as activitypubHttpUnicastHandler from './activitypub-http-unicast-handler' - -type ActivityPubHttpPayload = { - uris: string[] - signatureActorId?: number - body?: any - attemptNumber?: number -} - -const jobHandlers: { [ handlerName: string ]: JobHandler } = { - activitypubHttpBroadcastHandler, - activitypubHttpUnicastHandler, - activitypubHttpFetcherHandler -} -const jobCategory: JobCategory = 'activitypub-http' - -const activitypubHttpJobScheduler = new JobScheduler(jobCategory, jobHandlers) - -async function maybeRetryRequestLater (err: Error, payload: ActivityPubHttpPayload, uri: string) { - logger.warn('Cannot make request to %s.', uri, err) - - let attemptNumber = payload.attemptNumber || 1 - attemptNumber += 1 - - if (attemptNumber < ACTIVITY_PUB.MAX_HTTP_ATTEMPT) { - logger.debug('Retrying request to %s (attempt %d/%d).', uri, attemptNumber, ACTIVITY_PUB.MAX_HTTP_ATTEMPT, err) - - const actor = await ActorFollowModel.loadByFollowerInbox(uri, undefined) - if (!actor) { - logger.debug('Actor %s is not a follower, do not retry the request.', uri) - return false - } - - const newPayload = Object.assign(payload, { - uris: [ uri ], - attemptNumber - }) - await activitypubHttpJobScheduler.createJob(undefined, 'activitypubHttpUnicastHandler', newPayload) - - return true - } - - return false -} - -async function computeBody (payload: ActivityPubHttpPayload) { - let body = payload.body - - if (payload.signatureActorId) { - const actorSignature = await ActorModel.load(payload.signatureActorId) - if (!actorSignature) throw new Error('Unknown signature actor id.') - body = await buildSignedActivity(actorSignature, payload.body) - } - - return body -} - -async function buildSignedRequestOptions (payload: ActivityPubHttpPayload) { - let actor: ActorModel - if (payload.signatureActorId) { - actor = await ActorModel.load(payload.signatureActorId) - if (!actor) throw new Error('Unknown signature actor id.') - } else { - // We need to sign the request, so use the server - actor = await getServerActor() - } - - const keyId = actor.getWebfingerUrl() - return { - algorithm: 'rsa-sha256', - authorizationHeaderName: 'Signature', - keyId, - key: actor.privateKey - } -} - -export { - ActivityPubHttpPayload, - activitypubHttpJobScheduler, - maybeRetryRequestLater, - computeBody, - buildSignedRequestOptions -} diff --git a/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-unicast-handler.ts b/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-unicast-handler.ts deleted file mode 100644 index 54a7504e8..000000000 --- a/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-unicast-handler.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { logger } from '../../../helpers/logger' -import { doRequest } from '../../../helpers/requests' -import { ActorFollowModel } from '../../../models/activitypub/actor-follow' -import { ActivityPubHttpPayload, buildSignedRequestOptions, computeBody, maybeRetryRequestLater } from './activitypub-http-job-scheduler' - -async function process (payload: ActivityPubHttpPayload, jobId: number) { - logger.info('Processing ActivityPub unicast in job %d.', jobId) - - const uri = payload.uris[0] - - const body = await computeBody(payload) - const httpSignatureOptions = await buildSignedRequestOptions(payload) - - const options = { - method: 'POST', - uri, - json: body, - httpSignature: httpSignatureOptions - } - - try { - await doRequest(options) - ActorFollowModel.updateActorFollowsScoreAndRemoveBadOnes([ uri ], [], undefined) - } catch (err) { - const isRetryingLater = await maybeRetryRequestLater(err, payload, uri) - if (isRetryingLater === false) { - ActorFollowModel.updateActorFollowsScoreAndRemoveBadOnes([], [ uri ], undefined) - } - - throw err - } -} - -function onError (err: Error, jobId: number) { - logger.error('Error when sending ActivityPub request in job %d.', jobId, err) - return Promise.resolve() -} - -function onSuccess (jobId: number) { - logger.info('Job %d is a success.', jobId) - return Promise.resolve() -} - -// --------------------------------------------------------------------------- - -export { - process, - onError, - onSuccess -} diff --git a/server/lib/jobs/activitypub-http-job-scheduler/index.ts b/server/lib/jobs/activitypub-http-job-scheduler/index.ts deleted file mode 100644 index ad8f527b4..000000000 --- a/server/lib/jobs/activitypub-http-job-scheduler/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './activitypub-http-job-scheduler' diff --git a/server/lib/jobs/index.ts b/server/lib/jobs/index.ts deleted file mode 100644 index 394264ec1..000000000 --- a/server/lib/jobs/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './activitypub-http-job-scheduler' -export * from './transcoding-job-scheduler' diff --git a/server/lib/jobs/job-scheduler.ts b/server/lib/jobs/job-scheduler.ts deleted file mode 100644 index 9d55880e6..000000000 --- a/server/lib/jobs/job-scheduler.ts +++ /dev/null @@ -1,144 +0,0 @@ -import { AsyncQueue, forever, queue } from 'async' -import * as Sequelize from 'sequelize' -import { JobCategory } from '../../../shared' -import { logger } from '../../helpers/logger' -import { JOB_STATES, JOBS_FETCH_LIMIT_PER_CYCLE, JOBS_FETCHING_INTERVAL } from '../../initializers' -import { JobModel } from '../../models/job/job' - -export interface JobHandler { - process (data: object, jobId: number): Promise - onError (err: Error, jobId: number) - onSuccess (jobId: number, jobResult: T, jobScheduler: JobScheduler): Promise -} -type JobQueueCallback = (err: Error) => void - -class JobScheduler { - - constructor ( - private jobCategory: JobCategory, - private jobHandlers: { [ id: string ]: JobHandler } - ) {} - - async activate () { - const limit = JOBS_FETCH_LIMIT_PER_CYCLE[this.jobCategory] - - logger.info('Jobs scheduler %s activated.', this.jobCategory) - - const jobsQueue = queue(this.processJob.bind(this)) - - // Finish processing jobs from a previous start - const state = JOB_STATES.PROCESSING - try { - const jobs = await JobModel.listWithLimitByCategory(limit, state, this.jobCategory) - - this.enqueueJobs(jobsQueue, jobs) - } catch (err) { - logger.error('Cannot list pending jobs.', err) - } - - forever( - async next => { - if (jobsQueue.length() !== 0) { - // Finish processing the queue first - return setTimeout(next, JOBS_FETCHING_INTERVAL) - } - - const state = JOB_STATES.PENDING - try { - const jobs = await JobModel.listWithLimitByCategory(limit, state, this.jobCategory) - - this.enqueueJobs(jobsQueue, jobs) - } catch (err) { - logger.error('Cannot list pending jobs.', err) - } - - // Optimization: we could use "drain" from queue object - return setTimeout(next, JOBS_FETCHING_INTERVAL) - }, - - err => logger.error('Error in job scheduler queue.', err) - ) - } - - createJob (transaction: Sequelize.Transaction, handlerName: string, handlerInputData: P) { - const createQuery = { - state: JOB_STATES.PENDING, - category: this.jobCategory, - handlerName, - handlerInputData - } - - const options = { transaction } - - return JobModel.create(createQuery, options) - } - - private enqueueJobs (jobsQueue: AsyncQueue, jobs: JobModel[]) { - jobs.forEach(job => jobsQueue.push(job)) - } - - private async processJob (job: JobModel, callback: (err: Error) => void) { - const jobHandler = this.jobHandlers[job.handlerName] - if (jobHandler === undefined) { - const errorString = 'Unknown job handler ' + job.handlerName + ' for job ' + job.id - logger.error(errorString) - - const error = new Error(errorString) - await this.onJobError(jobHandler, job, error) - return callback(error) - } - - logger.info('Processing job %d with handler %s.', job.id, job.handlerName) - - job.state = JOB_STATES.PROCESSING - await job.save() - - try { - const result: T = await jobHandler.process(job.handlerInputData, job.id) - await this.onJobSuccess(jobHandler, job, result) - } catch (err) { - logger.error('Error in job handler %s.', job.handlerName, err) - - try { - await this.onJobError(jobHandler, job, err) - } catch (innerErr) { - this.cannotSaveJobError(innerErr) - return callback(innerErr) - } - } - - return callback(null) - } - - private async onJobError (jobHandler: JobHandler, job: JobModel, err: Error) { - job.state = JOB_STATES.ERROR - - try { - await job.save() - if (jobHandler) await jobHandler.onError(err, job.id) - } catch (err) { - this.cannotSaveJobError(err) - } - } - - private async onJobSuccess (jobHandler: JobHandler, job: JobModel, jobResult: T) { - job.state = JOB_STATES.SUCCESS - - try { - await job.save() - await jobHandler.onSuccess(job.id, jobResult, this) - } catch (err) { - this.cannotSaveJobError(err) - } - } - - private cannotSaveJobError (err: Error) { - logger.error('Cannot save new job state.', err) - } -} - -// --------------------------------------------------------------------------- - -export { - JobScheduler -} diff --git a/server/lib/jobs/transcoding-job-scheduler/index.ts b/server/lib/jobs/transcoding-job-scheduler/index.ts deleted file mode 100644 index 73152a1be..000000000 --- a/server/lib/jobs/transcoding-job-scheduler/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './transcoding-job-scheduler' diff --git a/server/lib/jobs/transcoding-job-scheduler/transcoding-job-scheduler.ts b/server/lib/jobs/transcoding-job-scheduler/transcoding-job-scheduler.ts deleted file mode 100644 index e5530a73c..000000000 --- a/server/lib/jobs/transcoding-job-scheduler/transcoding-job-scheduler.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { JobCategory } from '../../../../shared' -import { VideoModel } from '../../../models/video/video' -import { JobHandler, JobScheduler } from '../job-scheduler' - -import * as videoFileOptimizer from './video-file-optimizer-handler' -import * as videoFileTranscoder from './video-file-transcoder-handler' - -type TranscodingJobPayload = { - videoUUID: string - resolution?: number -} -const jobHandlers: { [ handlerName: string ]: JobHandler } = { - videoFileOptimizer, - videoFileTranscoder -} -const jobCategory: JobCategory = 'transcoding' - -const transcodingJobScheduler = new JobScheduler(jobCategory, jobHandlers) - -export { - TranscodingJobPayload, - transcodingJobScheduler -} diff --git a/server/lib/jobs/transcoding-job-scheduler/video-file-transcoder-handler.ts b/server/lib/jobs/transcoding-job-scheduler/video-file-transcoder-handler.ts deleted file mode 100644 index 883d3eba8..000000000 --- a/server/lib/jobs/transcoding-job-scheduler/video-file-transcoder-handler.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { VideoResolution } from '../../../../shared' -import { VideoPrivacy } from '../../../../shared/models/videos' -import { logger } from '../../../helpers/logger' -import { VideoModel } from '../../../models/video/video' -import { sendUpdateVideo } from '../../activitypub/send' - -async function process (data: { videoUUID: string, resolution: VideoResolution }, jobId: number) { - const video = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(data.videoUUID) - // No video, maybe deleted? - if (!video) { - logger.info('Do not process job %d, video does not exist.', jobId, { videoUUID: video.uuid }) - return undefined - } - - await video.transcodeOriginalVideofile(data.resolution) - - return video -} - -function onError (err: Error, jobId: number) { - logger.error('Error when transcoding video file in job %d.', jobId, err) - return Promise.resolve() -} - -async function onSuccess (jobId: number, video: VideoModel) { - if (video === undefined) return undefined - - logger.info('Job %d is a success.', jobId) - - // Maybe the video changed in database, refresh it - const videoDatabase = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(video.uuid) - // Video does not exist anymore - if (!videoDatabase) return undefined - - if (video.privacy !== VideoPrivacy.PRIVATE) { - await sendUpdateVideo(video, undefined) - } - - return undefined -} - -// --------------------------------------------------------------------------- - -export { - process, - onError, - onSuccess -} diff --git a/server/lib/schedulers/remove-old-jobs-scheduler.ts b/server/lib/schedulers/remove-old-jobs-scheduler.ts new file mode 100644 index 000000000..add5677ac --- /dev/null +++ b/server/lib/schedulers/remove-old-jobs-scheduler.ts @@ -0,0 +1,19 @@ +import { JobQueue } from '../job-queue' +import { AbstractScheduler } from './abstract-scheduler' + +export class RemoveOldJobsScheduler extends AbstractScheduler { + + private static instance: AbstractScheduler + + private constructor () { + super() + } + + async execute () { + JobQueue.Instance.removeOldJobs() + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} diff --git a/server/middlewares/validators/jobs.ts b/server/middlewares/validators/jobs.ts new file mode 100644 index 000000000..2f8b1738c --- /dev/null +++ b/server/middlewares/validators/jobs.ts @@ -0,0 +1,23 @@ +import * as express from 'express' +import { param } from 'express-validator/check' +import { isValidJobState } from '../../helpers/custom-validators/jobs' +import { logger } from '../../helpers/logger' +import { areValidationErrors } from './utils' + +const listJobsValidator = [ + param('state').custom(isValidJobState).not().isEmpty().withMessage('Should have a valid job state'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking listJobsValidator parameters.', { parameters: req.params }) + + if (areValidationErrors(req, res)) return + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + listJobsValidator +} diff --git a/server/models/job/job.ts b/server/models/job/job.ts deleted file mode 100644 index ba1c6737e..000000000 --- a/server/models/job/job.ts +++ /dev/null @@ -1,80 +0,0 @@ -import { values } from 'lodash' -import { AllowNull, Column, CreatedAt, DataType, Model, Table, UpdatedAt } from 'sequelize-typescript' -import { JobCategory, JobState } from '../../../shared/models' -import { JOB_CATEGORIES, JOB_STATES } from '../../initializers' -import { getSort } from '../utils' - -@Table({ - tableName: 'job', - indexes: [ - { - fields: [ 'state', 'category' ] - } - ] -}) -export class JobModel extends Model { - @AllowNull(false) - @Column(DataType.ENUM(values(JOB_STATES))) - state: JobState - - @AllowNull(false) - @Column(DataType.ENUM(values(JOB_CATEGORIES))) - category: JobCategory - - @AllowNull(false) - @Column - handlerName: string - - @AllowNull(true) - @Column(DataType.JSON) - handlerInputData: any - - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - static listWithLimitByCategory (limit: number, state: JobState, jobCategory: JobCategory) { - const query = { - order: [ - [ 'id', 'ASC' ] - ], - limit: limit, - where: { - state, - category: jobCategory - }, - logging: false - } - - return JobModel.findAll(query) - } - - static listForApi (start: number, count: number, sort: string) { - const query = { - offset: start, - limit: count, - order: [ getSort(sort) ] - } - - return JobModel.findAndCountAll(query).then(({ rows, count }) => { - return { - data: rows, - total: count - } - }) - } - - toFormattedJSON () { - return { - id: this.id, - state: this.state, - category: this.category, - handlerName: this.handlerName, - handlerInputData: this.handlerInputData, - createdAt: this.createdAt, - updatedAt: this.updatedAt - } - } -} diff --git a/server/tests/api/check-params/jobs.ts b/server/tests/api/check-params/jobs.ts index b12818bb1..ce3ac8809 100644 --- a/server/tests/api/check-params/jobs.ts +++ b/server/tests/api/check-params/jobs.ts @@ -7,7 +7,7 @@ import { checkBadCountPagination, checkBadSortPagination, checkBadStartPaginatio import { makeGetRequest } from '../../utils/requests/requests' describe('Test jobs API validators', function () { - const path = '/api/v1/jobs/' + const path = '/api/v1/jobs/failed' let server: ServerInfo let userAccessToken = '' @@ -31,6 +31,15 @@ describe('Test jobs API validators', function () { }) describe('When listing jobs', function () { + + it('Should fail with a bad state', async function () { + await makeGetRequest({ + url: server.url, + token: server.accessToken, + path: path + 'ade' + }) + }) + it('Should fail with a bad start pagination', async function () { await checkBadStartPagination(server.url, path, server.accessToken) }) diff --git a/server/tests/api/server/handle-down.ts b/server/tests/api/server/handle-down.ts index de4e77b2f..4cedeb89e 100644 --- a/server/tests/api/server/handle-down.ts +++ b/server/tests/api/server/handle-down.ts @@ -2,6 +2,7 @@ import * as chai from 'chai' import 'mocha' +import { JobState } from '../../../../shared/models' import { VideoPrivacy } from '../../../../shared/models/videos' import { VideoCommentThreadTree } from '../../../../shared/models/videos/video-comment.model' import { completeVideoCheck, getVideo, immutableAssign, reRunServer, viewVideo } from '../../utils' @@ -139,12 +140,11 @@ describe('Test handle downs', function () { }) it('Should not have pending/processing jobs anymore', async function () { - const res = await getJobsListPaginationAndSort(servers[0].url, servers[0].accessToken, 0, 50, '-createdAt') - const jobs = res.body.data + const states: JobState[] = [ 'inactive', 'active' ] - for (const job of jobs) { - expect(job.state).not.to.equal('pending') - expect(job.state).not.to.equal('processing') + for (const state of states) { + const res = await getJobsListPaginationAndSort(servers[ 0 ].url, servers[ 0 ].accessToken, state,0, 50, '-createdAt') + expect(res.body.data).to.have.length(0) } }) diff --git a/server/tests/api/server/jobs.ts b/server/tests/api/server/jobs.ts index 2e17e71a4..671498769 100644 --- a/server/tests/api/server/jobs.ts +++ b/server/tests/api/server/jobs.ts @@ -35,20 +35,20 @@ describe('Test jobs', function () { }) it('Should list jobs', async function () { - const res = await getJobsList(servers[1].url, servers[1].accessToken) + const res = await getJobsList(servers[1].url, servers[1].accessToken, 'complete') expect(res.body.total).to.be.above(2) expect(res.body.data).to.have.length.above(2) }) it('Should list jobs with sort and pagination', async function () { - const res = await getJobsListPaginationAndSort(servers[1].url, servers[1].accessToken, 4, 1, 'createdAt') + const res = await getJobsListPaginationAndSort(servers[1].url, servers[1].accessToken, 'complete', 1, 1, 'createdAt') expect(res.body.total).to.be.above(2) expect(res.body.data).to.have.lengthOf(1) const job = res.body.data[0] - expect(job.state).to.equal('success') - expect(job.category).to.equal('transcoding') - expect(job.handlerName).to.have.length.above(3) + + expect(job.state).to.equal('complete') + expect(job.type).to.equal('activitypub-http-unicast') expect(dateIsValid(job.createdAt)).to.be.true expect(dateIsValid(job.updatedAt)).to.be.true }) diff --git a/server/tests/api/videos/multiple-servers.ts b/server/tests/api/videos/multiple-servers.ts index 4c4b5123d..0215b3011 100644 --- a/server/tests/api/videos/multiple-servers.ts +++ b/server/tests/api/videos/multiple-servers.ts @@ -475,16 +475,17 @@ describe('Test multiple servers', function () { it('Should like and dislikes videos on different services', async function () { this.timeout(20000) - const tasks: Promise[] = [] - tasks.push(rateVideo(servers[0].url, servers[0].accessToken, remoteVideosServer1[0], 'like')) - tasks.push(rateVideo(servers[0].url, servers[0].accessToken, remoteVideosServer1[0], 'dislike')) - tasks.push(rateVideo(servers[0].url, servers[0].accessToken, remoteVideosServer1[0], 'like')) - tasks.push(rateVideo(servers[2].url, servers[2].accessToken, localVideosServer3[1], 'like')) - tasks.push(rateVideo(servers[2].url, servers[2].accessToken, localVideosServer3[1], 'dislike')) - tasks.push(rateVideo(servers[2].url, servers[2].accessToken, remoteVideosServer3[1], 'dislike')) - tasks.push(rateVideo(servers[2].url, servers[2].accessToken, remoteVideosServer3[0], 'like')) - - await Promise.all(tasks) + await rateVideo(servers[0].url, servers[0].accessToken, remoteVideosServer1[0], 'like') + await wait(200) + await rateVideo(servers[0].url, servers[0].accessToken, remoteVideosServer1[0], 'dislike') + await wait(200) + await rateVideo(servers[0].url, servers[0].accessToken, remoteVideosServer1[0], 'like') + await rateVideo(servers[2].url, servers[2].accessToken, localVideosServer3[1], 'like') + await wait(200) + await rateVideo(servers[2].url, servers[2].accessToken, localVideosServer3[1], 'dislike') + await rateVideo(servers[2].url, servers[2].accessToken, remoteVideosServer3[1], 'dislike') + await wait(200) + await rateVideo(servers[2].url, servers[2].accessToken, remoteVideosServer3[0], 'like') await wait(10000) diff --git a/server/tests/real-world/real-world.ts b/server/tests/real-world/real-world.ts index e41203351..7f67525ed 100644 --- a/server/tests/real-world/real-world.ts +++ b/server/tests/real-world/real-world.ts @@ -3,6 +3,7 @@ process.env.NODE_ENV = 'test' import * as program from 'commander' import { Video, VideoFile, VideoRateType } from '../../../shared' +import { JobState } from '../../../shared/models' import { flushAndRunMultipleServers, flushTests, follow, @@ -346,23 +347,19 @@ function goodbye () { } async function isTherePendingRequests (servers: ServerInfo[]) { + const states: JobState[] = [ 'inactive', 'active' ] const tasks: Promise[] = [] let pendingRequests = false // Check if each server has pending request for (const server of servers) { - const p = getJobsListPaginationAndSort(server.url, server.accessToken, 0, 10, '-createdAt') - .then(res => { - const jobs = res.body.data - - for (const job of jobs) { - if (job.state === 'pending' || job.state === 'processing') { - pendingRequests = true - } - } - }) - - tasks.push(p) + for (const state of states) { + const p = getJobsListPaginationAndSort(server.url, server.accessToken, state, 0, 10, '-createdAt') + .then(res => { + if (res.body.total > 0) pendingRequests = true + }) + tasks.push(p) + } } await Promise.all(tasks) diff --git a/server/tests/utils/server/jobs.ts b/server/tests/utils/server/jobs.ts index 0a8c51575..4053dd40b 100644 --- a/server/tests/utils/server/jobs.ts +++ b/server/tests/utils/server/jobs.ts @@ -1,7 +1,8 @@ import * as request from 'supertest' +import { JobState } from '../../../../shared/models' -function getJobsList (url: string, accessToken: string) { - const path = '/api/v1/jobs' +function getJobsList (url: string, accessToken: string, state: JobState) { + const path = '/api/v1/jobs/' + state return request(url) .get(path) @@ -11,8 +12,8 @@ function getJobsList (url: string, accessToken: string) { .expect('Content-Type', /json/) } -function getJobsListPaginationAndSort (url: string, accessToken: string, start: number, count: number, sort: string) { - const path = '/api/v1/jobs' +function getJobsListPaginationAndSort (url: string, accessToken: string, state: JobState, start: number, count: number, sort: string) { + const path = '/api/v1/jobs/' + state return request(url) .get(path) diff --git a/shared/models/job.model.ts b/shared/models/job.model.ts index 1c46a7900..1a25600f3 100644 --- a/shared/models/job.model.ts +++ b/shared/models/job.model.ts @@ -1,12 +1,16 @@ -export type JobState = 'pending' | 'processing' | 'error' | 'success' -export type JobCategory = 'transcoding' | 'activitypub-http' +export type JobState = 'active' | 'complete' | 'failed' | 'inactive' | 'delayed' + +export type JobType = 'activitypub-http-unicast' | + 'activitypub-http-broadcast' | + 'activitypub-http-fetcher' | + 'video-file' export interface Job { id: number state: JobState - category: JobCategory - handlerName: string - handlerInputData: any + type: JobType + data: any, + error: any, createdAt: Date updatedAt: Date } diff --git a/support/doc/dependencies.md b/support/doc/dependencies.md index c950b357f..7017976e5 100644 --- a/support/doc/dependencies.md +++ b/support/doc/dependencies.md @@ -10,7 +10,7 @@ ``` $ sudo apt update -$ sudo apt install nginx ffmpeg postgresql openssl g++ make +$ sudo apt install nginx ffmpeg postgresql openssl g++ make redis-server ``` ## Arch Linux @@ -18,7 +18,7 @@ $ sudo apt install nginx ffmpeg postgresql openssl g++ make 1. Run: ``` -$ sudo pacman -S nodejs yarn ffmpeg postgresql openssl +$ sudo pacman -S nodejs yarn ffmpeg postgresql openssl redis ``` ## Other distributions diff --git a/support/doc/development/server/code.md b/support/doc/development/server/code.md index 953ccdbfe..e9ab7373c 100644 --- a/support/doc/development/server/code.md +++ b/support/doc/development/server/code.md @@ -7,6 +7,7 @@ The server is a web server developed with [TypeScript](https://www.typescriptlan * [TypeScript](https://www.typescriptlang.org/) -> Language * [PostgreSQL](https://www.postgresql.org/) -> Database + * [Redis](https://redis.io/) -> Job queue/cache * [Express](http://expressjs.com) -> Web server framework * [Sequelize](http://docs.sequelizejs.com/en/v3/) -> SQL ORM * [WebTorrent](https://webtorrent.io/) -> BitTorrent tracker and torrent creation diff --git a/tsconfig.json b/tsconfig.json index 71674e165..4e6816430 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -19,6 +19,12 @@ }, "exclude": [ "node_modules", - "client" + "client", + "text1", + "text2", + "text3", + "text4", + "text5", + "text6" ] } diff --git a/yarn.lock b/yarn.lock index a0f6da289..a3f6fce8a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -82,6 +82,14 @@ version "1.0.6" resolved "https://registry.yarnpkg.com/@types/geojson/-/geojson-1.0.6.tgz#3e02972728c69248c2af08d60a48cbb8680fffdf" +"@types/kue@^0.11.8": + version "0.11.8" + resolved "https://registry.yarnpkg.com/@types/kue/-/kue-0.11.8.tgz#820f5e3db6025f0411e0942cd3ccab461a060c90" + dependencies: + "@types/express" "*" + "@types/node" "*" + "@types/redis" "*" + "@types/lodash@*", "@types/lodash@^4.14.64": version "4.14.95" resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.95.tgz#02c170690719bbaca8293d9c8cdcccb565728081" @@ -144,6 +152,13 @@ version "1.9.3" resolved "https://registry.yarnpkg.com/@types/pem/-/pem-1.9.3.tgz#0c864c8b79e43fef6367db895f60fd1edd10e86c" +"@types/redis@*": + version "2.8.5" + resolved "https://registry.yarnpkg.com/@types/redis/-/redis-2.8.5.tgz#c4a31a63e95434202eb84908290528ad8510b149" + dependencies: + "@types/events" "*" + "@types/node" "*" + "@types/reflect-metadata@0.0.4": version "0.0.4" resolved "https://registry.yarnpkg.com/@types/reflect-metadata/-/reflect-metadata-0.0.4.tgz#b6477ca9a97e5265f2ac67f9ea704eae5e0eaf4d" @@ -240,16 +255,26 @@ accepts@~1.3.4: mime-types "~2.1.16" negotiator "0.6.1" +acorn-globals@^3.0.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/acorn-globals/-/acorn-globals-3.1.0.tgz#fd8270f71fbb4996b004fa880ee5d46573a731bf" + dependencies: + acorn "^4.0.4" + acorn-jsx@^3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/acorn-jsx/-/acorn-jsx-3.0.1.tgz#afdf9488fb1ecefc8348f6fb22f464e32a58b36b" dependencies: acorn "^3.0.4" -acorn@^3.0.4: +acorn@^3.0.4, acorn@^3.1.0, acorn@~3.3.0: version "3.3.0" resolved "https://registry.yarnpkg.com/acorn/-/acorn-3.3.0.tgz#45e37fb39e8da3f25baee3ff5369e2bb5f22017a" +acorn@^4.0.4, acorn@~4.0.2: + version "4.0.13" + resolved "https://registry.yarnpkg.com/acorn/-/acorn-4.0.13.tgz#105495ae5361d697bd195c825192e1ad7f253787" + acorn@^5.2.1: version "5.3.0" resolved "https://registry.yarnpkg.com/acorn/-/acorn-5.3.0.tgz#7446d39459c54fb49a80e6ee6478149b940ec822" @@ -340,6 +365,12 @@ anymatch@^2.0.0: micromatch "^3.1.4" normalize-path "^2.1.1" +"apparatus@>= 0.0.9": + version "0.0.9" + resolved "https://registry.yarnpkg.com/apparatus/-/apparatus-0.0.9.tgz#37dcd25834ad0b651076596291db823eeb1908bd" + dependencies: + sylvester ">= 0.0.8" + append-field@^0.1.0: version "0.1.0" resolved "https://registry.yarnpkg.com/append-field/-/append-field-0.1.0.tgz#6ddc58fa083c7bc545d3c5995b2830cc2366d44a" @@ -717,7 +748,7 @@ bn.js@^4.4.0: version "4.11.8" resolved "https://registry.yarnpkg.com/bn.js/-/bn.js-4.11.8.tgz#2cde09eb5ee341f484746bb0309b3253b1b1442f" -body-parser@1.18.2, body-parser@^1.12.4: +body-parser@1.18.2, body-parser@^1.12.2, body-parser@^1.12.4: version "1.18.2" resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.18.2.tgz#87678a19d84b47d859b83199bd59bce222b10454" dependencies: @@ -959,6 +990,12 @@ chalk@^2.0.0, chalk@^2.0.1, chalk@^2.3.0: escape-string-regexp "^1.0.5" supports-color "^4.0.0" +character-parser@^2.1.1: + version "2.2.0" + resolved "https://registry.yarnpkg.com/character-parser/-/character-parser-2.2.0.tgz#c7ce28f36d4bcd9744e5ffc2c5fcde1c73261fc0" + dependencies: + is-regex "^1.0.3" + charenc@~0.0.1: version "0.0.2" resolved "https://registry.yarnpkg.com/charenc/-/charenc-0.0.2.tgz#c0a1d2f3a7092e03774bfa83f14c0fc5790a8667" @@ -1025,7 +1062,7 @@ class-utils@^0.3.5: isobject "^3.0.0" static-extend "^0.1.1" -clean-css@~3.4.2: +clean-css@^3.3.0, clean-css@~3.4.2: version "3.4.28" resolved "https://registry.yarnpkg.com/clean-css/-/clean-css-3.4.28.tgz#bf1945e82fc808f55695e6ddeaec01400efd03ff" dependencies: @@ -1268,6 +1305,13 @@ console-control-strings@^1.0.0, console-control-strings@~1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/console-control-strings/-/console-control-strings-1.1.0.tgz#3d7cf4464db6446ea644bf4b39507f9851008e8e" +constantinople@^3.0.1: + version "3.1.0" + resolved "https://registry.yarnpkg.com/constantinople/-/constantinople-3.1.0.tgz#7569caa8aa3f8d5935d62e1fa96f9f702cd81c79" + dependencies: + acorn "^3.1.0" + is-expression "^2.0.1" + contains-path@^0.1.0: version "0.1.0" resolved "https://registry.yarnpkg.com/contains-path/-/contains-path-0.1.0.tgz#fe8cf184ff6670b6baef01a9d4861a5cbec4120a" @@ -1367,6 +1411,10 @@ crypto-random-string@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/crypto-random-string/-/crypto-random-string-1.0.0.tgz#a230f64f568310e1498009940790ec99545bca7e" +css-parse@1.7.x: + version "1.7.0" + resolved "https://registry.yarnpkg.com/css-parse/-/css-parse-1.7.0.tgz#321f6cf73782a6ff751111390fc05e2c657d8c9b" + css-select@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/css-select/-/css-select-1.0.0.tgz#b1121ca51848dd264e2244d058cee254deeb44b0" @@ -1410,15 +1458,15 @@ debug-log@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/debug-log/-/debug-log-1.0.1.tgz#2307632d4c04382b8df8a32f70b895046d52745f" -debug@2.6.9, debug@^2.0.0, debug@^2.1.0, debug@^2.1.1, debug@^2.2.0, debug@^2.3.3, debug@^2.5.2, debug@^2.6.8, debug@^2.6.9: - version "2.6.9" - resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" +debug@*, debug@3.1.0, debug@^3.0.0, debug@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261" dependencies: ms "2.0.0" -debug@3.1.0, debug@^3.0.0, debug@^3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261" +debug@2.6.9, debug@^2.0.0, debug@^2.1.0, debug@^2.1.1, debug@^2.2.0, debug@^2.3.3, debug@^2.5.2, debug@^2.6.8, debug@^2.6.9: + version "2.6.9" + resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" dependencies: ms "2.0.0" @@ -1567,6 +1615,10 @@ doctrine@^2.0.0: dependencies: esutils "^2.0.2" +doctypes@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/doctypes/-/doctypes-1.1.0.tgz#ea80b106a87538774e8a3a4a5afe293de489e0a9" + dom-serializer@0, dom-serializer@~0.1.0: version "0.1.0" resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-0.1.0.tgz#073c697546ce0780ce23be4a28e293e40bc30c82" @@ -1611,6 +1663,10 @@ dottie@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/dottie/-/dottie-2.0.0.tgz#da191981c8b8d713ca0115d5898cf397c2f0ddd0" +double-ended-queue@^2.1.0-0: + version "2.1.0-0" + resolved "https://registry.yarnpkg.com/double-ended-queue/-/double-ended-queue-2.1.0-0.tgz#103d3527fd31528f40188130c841efdd78264e5c" + duplexer3@^0.1.4: version "0.1.4" resolved "https://registry.yarnpkg.com/duplexer3/-/duplexer3-0.1.4.tgz#ee01dd1cac0ed3cbc7fdbea37dc0a8f1ce002ce2" @@ -2010,7 +2066,7 @@ express-validator@^4.1.1: lodash "^4.16.0" validator "~8.2.0" -express@^4.12.4, express@^4.13.3: +express@^4.12.2, express@^4.12.4, express@^4.13.3: version "4.16.2" resolved "https://registry.yarnpkg.com/express/-/express-4.16.2.tgz#e35c6dfe2d64b7dca0a5cd4f21781be3299e076c" dependencies: @@ -2058,6 +2114,10 @@ extend-shallow@^3.0.0: assign-symbols "^1.0.0" is-extendable "^1.0.1" +extend@^1.2.1: + version "1.3.0" + resolved "https://registry.yarnpkg.com/extend/-/extend-1.3.0.tgz#d1516fb0ff5624d2ebf9123ea1dac5a1994004f8" + extend@^3.0.0, extend@~3.0.0, extend@~3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.1.tgz#a755ea7bc1adfcc5a31ce7e762dbaadc5e636444" @@ -2409,6 +2469,17 @@ glob-parent@^3.1.0: is-glob "^3.1.0" path-dirname "^1.0.0" +glob@7.0.x: + version "7.0.6" + resolved "https://registry.yarnpkg.com/glob/-/glob-7.0.6.tgz#211bafaf49e525b8cd93260d14ab136152b3f57a" + dependencies: + fs.realpath "^1.0.0" + inflight "^1.0.4" + inherits "2" + minimatch "^3.0.2" + once "^1.3.0" + path-is-absolute "^1.0.0" + glob@7.1.2, glob@^7.0.0, glob@^7.0.3, glob@^7.0.5, glob@^7.1.1, glob@~7.1.1: version "7.1.2" resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15" @@ -3094,6 +3165,20 @@ is-descriptor@^1.0.0: is-data-descriptor "^1.0.0" kind-of "^6.0.2" +is-expression@^2.0.1: + version "2.1.0" + resolved "https://registry.yarnpkg.com/is-expression/-/is-expression-2.1.0.tgz#91be9d47debcfef077977e9722be6dcfb4465ef0" + dependencies: + acorn "~3.3.0" + object-assign "^4.0.1" + +is-expression@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/is-expression/-/is-expression-3.0.0.tgz#39acaa6be7fd1f3471dc42c7416e61c24317ac9f" + dependencies: + acorn "~4.0.2" + object-assign "^4.0.1" + is-extendable@^0.1.0, is-extendable@^0.1.1: version "0.1.1" resolved "https://registry.yarnpkg.com/is-extendable/-/is-extendable-0.1.1.tgz#62b110e289a471418e3ec36a617d472e301dfc89" @@ -3202,6 +3287,10 @@ is-plain-object@^2.0.1, is-plain-object@^2.0.3, is-plain-object@^2.0.4: dependencies: isobject "^3.0.1" +is-promise@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/is-promise/-/is-promise-2.1.0.tgz#79a2a9ece7f096e80f36d2b2f3bc16c1ff4bf3fa" + is-property@^1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/is-property/-/is-property-1.0.2.tgz#57fe1c4e48474edd65b09911f26b1cd4095dda84" @@ -3210,7 +3299,7 @@ is-redirect@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/is-redirect/-/is-redirect-1.0.0.tgz#1d03dded53bd8db0f30c26e4f95d36fc7c87dc24" -is-regex@^1.0.4: +is-regex@^1.0.3, is-regex@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/is-regex/-/is-regex-1.0.4.tgz#5517489b547091b0930e095654ced25ee97e9491" dependencies: @@ -3286,6 +3375,10 @@ js-string-escape@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/js-string-escape/-/js-string-escape-1.0.1.tgz#e2625badbc0d67c7533e9edc1068c587ae4137ef" +js-stringify@^1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/js-stringify/-/js-stringify-1.0.2.tgz#1736fddfd9724f28a3682adc6230ae7e4e9679db" + js-tokens@^3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-3.0.2.tgz#9866df395102130e38f7f996bceb65443209c25b" @@ -3382,6 +3475,13 @@ jsprim@^1.2.2: json-schema "0.2.3" verror "1.10.0" +jstransformer@1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/jstransformer/-/jstransformer-1.0.0.tgz#ed8bf0921e2f3f1ed4d5c1a44f68709ed24722c3" + dependencies: + is-promise "^2.0.0" + promise "^7.0.1" + jsx-ast-utils@^1.3.4: version "1.4.1" resolved "https://registry.yarnpkg.com/jsx-ast-utils/-/jsx-ast-utils-1.4.1.tgz#3867213e8dd79bf1e8f2300c0cfc1efb182c0df1" @@ -3453,6 +3553,22 @@ kind-of@^6.0.0, kind-of@^6.0.2: version "6.0.2" resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-6.0.2.tgz#01146b36a6218e64e58f3a8d66de5d7fc6f6d051" +kue@^0.11.6: + version "0.11.6" + resolved "https://registry.yarnpkg.com/kue/-/kue-0.11.6.tgz#5b76916bcedd56636a107861471c63c94611860a" + dependencies: + body-parser "^1.12.2" + express "^4.12.2" + lodash "^4.0.0" + nib "~1.1.2" + node-redis-warlock "~0.2.0" + pug "^2.0.0-beta3" + redis "~2.6.0-2" + stylus "~0.54.5" + yargs "^4.0.0" + optionalDependencies: + reds "^0.2.5" + kuler@0.0.x: version "0.0.0" resolved "https://registry.yarnpkg.com/kuler/-/kuler-0.0.0.tgz#b66bb46b934e550f59d818848e0abba4f7f5553c" @@ -3564,7 +3680,7 @@ lodash._isiterateecall@^3.0.0: version "3.0.9" resolved "https://registry.yarnpkg.com/lodash._isiterateecall/-/lodash._isiterateecall-3.0.9.tgz#5203ad7ba425fae842460e696db9cf3e6aac057c" -lodash.assign@^4.2.0: +lodash.assign@^4.0.3, lodash.assign@^4.0.6, lodash.assign@^4.2.0: version "4.2.0" resolved "https://registry.yarnpkg.com/lodash.assign/-/lodash.assign-4.2.0.tgz#0d99f3ccd7a6d261d19bdaeb9245005d285808e7" @@ -3692,7 +3808,7 @@ lowercase-keys@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/lowercase-keys/-/lowercase-keys-1.0.0.tgz#4e3366b39e7f5457e35f1324bdf6f88d0bfc7306" -lru-cache@2: +lru-cache@2, lru-cache@^2.5.0: version "2.7.3" resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-2.7.3.tgz#6d4524e8b955f95d4f5b58851ce21dd72fb4e952" @@ -3916,7 +4032,7 @@ mixin-deep@^1.2.0: for-in "^1.0.2" is-extendable "^1.0.1" -mkdirp@0.5.1, "mkdirp@>=0.5 0", mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@~0.5.0: +mkdirp@0.5.1, mkdirp@0.5.x, "mkdirp@>=0.5 0", mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@~0.5.0: version "0.5.1" resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.1.tgz#30057438eac6cf7f8c4767f38648d6697d75c903" dependencies: @@ -4036,6 +4152,14 @@ natural-compare@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7" +natural@^0.2.0: + version "0.2.1" + resolved "https://registry.yarnpkg.com/natural/-/natural-0.2.1.tgz#1eb5156a9d90b4591949e20e94ebc77bb2339eda" + dependencies: + apparatus ">= 0.0.9" + sylvester ">= 0.0.12" + underscore ">=1.3.1" + negotiator@0.6.1: version "0.6.1" resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.1.tgz#2b327184e8992101177b28563fb5e7102acd0ca9" @@ -4048,6 +4172,12 @@ next-event@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/next-event/-/next-event-1.0.0.tgz#e7778acde2e55802e0ad1879c39cf6f75eda61d8" +nib@~1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/nib/-/nib-1.1.2.tgz#6a69ede4081b95c0def8be024a4c8ae0c2cbb6c7" + dependencies: + stylus "0.54.5" + node-abi@^2.1.1: version "2.1.2" resolved "https://registry.yarnpkg.com/node-abi/-/node-abi-2.1.2.tgz#4da6caceb6685fcd31e7dd1994ef6bb7d0a9c0b2" @@ -4106,6 +4236,20 @@ node-pre-gyp@^0.6.39: tar "^2.2.1" tar-pack "^3.4.0" +node-redis-scripty@0.0.5: + version "0.0.5" + resolved "https://registry.yarnpkg.com/node-redis-scripty/-/node-redis-scripty-0.0.5.tgz#4bf2d365ab6dab202cc08b7ac63f8f55aadc9625" + dependencies: + extend "^1.2.1" + lru-cache "^2.5.0" + +node-redis-warlock@~0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/node-redis-warlock/-/node-redis-warlock-0.2.0.tgz#56395b994c828e8e32f6aae53b93b6edfcd97990" + dependencies: + node-redis-scripty "0.0.5" + uuid "^2.0.1" + node-sass@^4.0.0: version "4.7.2" resolved "https://registry.yarnpkg.com/node-sass/-/node-sass-4.7.2.tgz#9366778ba1469eb01438a9e8592f4262bcb6794e" @@ -4703,7 +4847,7 @@ progress@^1.1.8: version "1.1.8" resolved "https://registry.yarnpkg.com/progress/-/progress-1.1.8.tgz#e260c78f6161cdd9b0e56cc3e0a85de17c7a57be" -promise@^7.1.1: +promise@^7.0.1, promise@^7.1.1: version "7.3.1" resolved "https://registry.yarnpkg.com/promise/-/promise-7.3.1.tgz#064b72602b18f90f29192b8b1bc418ffd1ebd3bf" dependencies: @@ -4744,6 +4888,99 @@ pstree.remy@^1.1.0: dependencies: ps-tree "^1.1.0" +pug-attrs@^2.0.2: + version "2.0.2" + resolved "https://registry.yarnpkg.com/pug-attrs/-/pug-attrs-2.0.2.tgz#8be2b2225568ffa75d1b866982bff9f4111affcb" + dependencies: + constantinople "^3.0.1" + js-stringify "^1.0.1" + pug-runtime "^2.0.3" + +pug-code-gen@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/pug-code-gen/-/pug-code-gen-2.0.0.tgz#96aea39a9e62f1ec5d2b6a5b42a29d528c70b43d" + dependencies: + constantinople "^3.0.1" + doctypes "^1.1.0" + js-stringify "^1.0.1" + pug-attrs "^2.0.2" + pug-error "^1.3.2" + pug-runtime "^2.0.3" + void-elements "^2.0.1" + with "^5.0.0" + +pug-error@^1.3.2: + version "1.3.2" + resolved "https://registry.yarnpkg.com/pug-error/-/pug-error-1.3.2.tgz#53ae7d9d29bb03cf564493a026109f54c47f5f26" + +pug-filters@^2.1.5: + version "2.1.5" + resolved "https://registry.yarnpkg.com/pug-filters/-/pug-filters-2.1.5.tgz#66bf6e80d97fbef829bab0aa35eddff33fc964f3" + dependencies: + clean-css "^3.3.0" + constantinople "^3.0.1" + jstransformer "1.0.0" + pug-error "^1.3.2" + pug-walk "^1.1.5" + resolve "^1.1.6" + uglify-js "^2.6.1" + +pug-lexer@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/pug-lexer/-/pug-lexer-3.1.0.tgz#fd087376d4a675b4f59f8fef422883434e9581a2" + dependencies: + character-parser "^2.1.1" + is-expression "^3.0.0" + pug-error "^1.3.2" + +pug-linker@^3.0.3: + version "3.0.3" + resolved "https://registry.yarnpkg.com/pug-linker/-/pug-linker-3.0.3.tgz#25f59eb750237f0368e59c3379764229c0189c41" + dependencies: + pug-error "^1.3.2" + pug-walk "^1.1.5" + +pug-load@^2.0.9: + version "2.0.9" + resolved "https://registry.yarnpkg.com/pug-load/-/pug-load-2.0.9.tgz#ee217c914cc1d9324d44b86c32d1df241d36de7a" + dependencies: + object-assign "^4.1.0" + pug-walk "^1.1.5" + +pug-parser@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/pug-parser/-/pug-parser-4.0.0.tgz#c9f52322e4eabe4bf5beeba64ed18373bb627801" + dependencies: + pug-error "^1.3.2" + token-stream "0.0.1" + +pug-runtime@^2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/pug-runtime/-/pug-runtime-2.0.3.tgz#98162607b0fce9e254d427f33987a5aee7168bda" + +pug-strip-comments@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/pug-strip-comments/-/pug-strip-comments-1.0.2.tgz#d313afa01bcc374980e1399e23ebf2eb9bdc8513" + dependencies: + pug-error "^1.3.2" + +pug-walk@^1.1.5: + version "1.1.5" + resolved "https://registry.yarnpkg.com/pug-walk/-/pug-walk-1.1.5.tgz#90e943acbcf7021e6454cf1b32245891cba6f851" + +pug@^2.0.0-beta3: + version "2.0.0-rc.4" + resolved "https://registry.yarnpkg.com/pug/-/pug-2.0.0-rc.4.tgz#b7b08f6599bd5302568042b7436984fb28c80a13" + dependencies: + pug-code-gen "^2.0.0" + pug-filters "^2.1.5" + pug-lexer "^3.1.0" + pug-linker "^3.0.3" + pug-load "^2.0.9" + pug-parser "^4.0.0" + pug-runtime "^2.0.3" + pug-strip-comments "^1.0.2" + pump@^1.0.0, pump@^1.0.1: version "1.0.3" resolved "https://registry.yarnpkg.com/pump/-/pump-1.0.3.tgz#5dfe8311c33bbf6fc18261f9f34702c47c08a954" @@ -4908,6 +5145,33 @@ redent@^1.0.0: indent-string "^2.1.0" strip-indent "^1.0.1" +redis-commands@^1.2.0: + version "1.3.1" + resolved "https://registry.yarnpkg.com/redis-commands/-/redis-commands-1.3.1.tgz#81d826f45fa9c8b2011f4cd7a0fe597d241d442b" + +redis-parser@^2.0.0: + version "2.6.0" + resolved "https://registry.yarnpkg.com/redis-parser/-/redis-parser-2.6.0.tgz#52ed09dacac108f1a631c07e9b69941e7a19504b" + +redis@^0.12.1: + version "0.12.1" + resolved "https://registry.yarnpkg.com/redis/-/redis-0.12.1.tgz#64df76ad0fc8acebaebd2a0645e8a48fac49185e" + +redis@~2.6.0-2: + version "2.6.5" + resolved "https://registry.yarnpkg.com/redis/-/redis-2.6.5.tgz#87c1eff4a489f94b70871f3d08b6988f23a95687" + dependencies: + double-ended-queue "^2.1.0-0" + redis-commands "^1.2.0" + redis-parser "^2.0.0" + +reds@^0.2.5: + version "0.2.5" + resolved "https://registry.yarnpkg.com/reds/-/reds-0.2.5.tgz#38a767f7663cd749036848697d82c74fd29bc01f" + dependencies: + natural "^0.2.0" + redis "^0.12.1" + reflect-metadata@^0.1.10: version "0.1.12" resolved "https://registry.yarnpkg.com/reflect-metadata/-/reflect-metadata-0.1.12.tgz#311bf0c6b63cd782f228a81abe146a2bfa9c56f2" @@ -5140,6 +5404,10 @@ sass-graph@^2.2.4: scss-tokenizer "^0.2.3" yargs "^7.0.0" +sax@0.5.x: + version "0.5.8" + resolved "https://registry.yarnpkg.com/sax/-/sax-0.5.8.tgz#d472db228eb331c2506b0e8c15524adb939d12c1" + scripty@^1.5.0: version "1.7.2" resolved "https://registry.yarnpkg.com/scripty/-/scripty-1.7.2.tgz#92367b724cb77b086729691f7b01aa57f3ddd356" @@ -5453,6 +5721,12 @@ source-map-url@^0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/source-map-url/-/source-map-url-0.4.0.tgz#3e935d7ddd73631b97659956d55128e87b5084a3" +source-map@0.1.x: + version "0.1.43" + resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.1.43.tgz#c24bc146ca517c1471f5dacbe2571b2b7f9e3346" + dependencies: + amdefine ">=0.0.4" + source-map@0.4.x, source-map@^0.4.2, source-map@^0.4.4: version "0.4.4" resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.4.4.tgz#eba4f5da9c0dc999de68032d8b4f76173652036b" @@ -5721,6 +5995,17 @@ strip-json-comments@^2.0.0, strip-json-comments@~2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-2.0.1.tgz#3c531942e908c2697c0ec344858c286c7ca0a60a" +stylus@0.54.5, stylus@~0.54.5: + version "0.54.5" + resolved "https://registry.yarnpkg.com/stylus/-/stylus-0.54.5.tgz#42b9560931ca7090ce8515a798ba9e6aa3d6dc79" + dependencies: + css-parse "1.7.x" + debug "*" + glob "7.0.x" + mkdirp "0.5.x" + sax "0.5.x" + source-map "0.1.x" + superagent@^3.0.0, superagent@^3.6.3: version "3.8.2" resolved "https://registry.yarnpkg.com/superagent/-/superagent-3.8.2.tgz#e4a11b9d047f7d3efeb3bbe536d9ec0021d16403" @@ -5769,6 +6054,10 @@ supports-color@^4.0.0: dependencies: has-flag "^2.0.0" +"sylvester@>= 0.0.12", "sylvester@>= 0.0.8": + version "0.0.21" + resolved "https://registry.yarnpkg.com/sylvester/-/sylvester-0.0.21.tgz#2987b1ce2bd2f38b0dce2a34388884bfa4400ea7" + sync-request@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/sync-request/-/sync-request-4.1.0.tgz#324b4e506fb994d2afd2a0021a455f800725f07a" @@ -5933,6 +6222,10 @@ to-regex@^3.0.1: extend-shallow "^2.0.1" regex-not "^1.0.0" +token-stream@0.0.1: + version "0.0.1" + resolved "https://registry.yarnpkg.com/token-stream/-/token-stream-0.0.1.tgz#ceeefc717a76c4316f126d0b9dbaa55d7e7df01a" + toposort-class@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/toposort-class/-/toposort-class-1.0.1.tgz#7ffd1f78c8be28c3ba45cd4e1a3f5ee193bd9988" @@ -6100,7 +6393,7 @@ typescript@^2.5.2: version "2.6.2" resolved "https://registry.yarnpkg.com/typescript/-/typescript-2.6.2.tgz#3c5b6fd7f6de0914269027f03c0946758f7673a4" -uglify-js@^2.6: +uglify-js@^2.6, uglify-js@^2.6.1: version "2.8.29" resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-2.8.29.tgz#29c5733148057bb4e1f75df35b7a9cb72e6a59dd" dependencies: @@ -6152,7 +6445,7 @@ underscore.string@~2.4.0: version "2.4.0" resolved "https://registry.yarnpkg.com/underscore.string/-/underscore.string-2.4.0.tgz#8cdd8fbac4e2d2ea1e7e2e8097c42f442280f85b" -underscore@^1.7.0: +underscore@>=1.3.1, underscore@^1.7.0: version "1.8.3" resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.8.3.tgz#4f3fb53b106e6097fcf9cb4109f2a5e9bdfa5022" @@ -6278,6 +6571,10 @@ utils-merge@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713" +uuid@^2.0.1: + version "2.0.3" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-2.0.3.tgz#67e2e863797215530dff318e5bf9dcebfd47b21a" + uuid@^3.0.0, uuid@^3.1.0, uuid@^3.2.1: version "3.2.1" resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.2.1.tgz#12c528bb9d58d0b9265d9a2f6f0fe8be17ff1f14" @@ -6328,6 +6625,10 @@ videostream@^2.3.0: pump "^1.0.1" range-slice-stream "^1.2.0" +void-elements@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/void-elements/-/void-elements-2.0.1.tgz#c066afb582bb1cb4128d60ea92392e94d5e9dbec" + webfinger.js@^2.6.6: version "2.6.6" resolved "https://registry.yarnpkg.com/webfinger.js/-/webfinger.js-2.6.6.tgz#52ebdc85da8c8fb6beb690e8e32594c99d2ff4ae" @@ -6415,6 +6716,10 @@ window-size@0.1.0: version "0.1.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.1.0.tgz#5438cd2ea93b202efa3a19fe8887aee7c94f9c9d" +window-size@^0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" + winston-transport@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/winston-transport/-/winston-transport-3.0.1.tgz#8008b15eef5660c4fb3fa094d58ccbd08528c58d" @@ -6432,6 +6737,13 @@ winston@3.0.0-rc1: triple-beam "^1.0.1" winston-transport "^3.0.1" +with@^5.0.0: + version "5.1.1" + resolved "https://registry.yarnpkg.com/with/-/with-5.1.1.tgz#fa4daa92daf32c4ea94ed453c81f04686b575dfe" + dependencies: + acorn "^3.1.0" + acorn-globals "^3.0.0" + wkx@^0.4.1: version "0.4.2" resolved "https://registry.yarnpkg.com/wkx/-/wkx-0.4.2.tgz#776d35a634a5c22e656e4744bdeb54f83fd2ce8d" @@ -6519,6 +6831,13 @@ yallist@^3.0.0, yallist@^3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.0.2.tgz#8452b4bb7e83c7c188d8041c1a837c773d6d8bb9" +yargs-parser@^2.4.1: + version "2.4.1" + resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-2.4.1.tgz#85568de3cf150ff49fa51825f03a8c880ddcc5c4" + dependencies: + camelcase "^3.0.0" + lodash.assign "^4.0.6" + yargs-parser@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-5.0.0.tgz#275ecf0d7ffe05c77e64e7c86e4cd94bf0e1228a" @@ -6531,6 +6850,25 @@ yargs-parser@^8.0.0: dependencies: camelcase "^4.1.0" +yargs@^4.0.0: + version "4.8.1" + resolved "https://registry.yarnpkg.com/yargs/-/yargs-4.8.1.tgz#c0c42924ca4aaa6b0e6da1739dfb216439f9ddc0" + dependencies: + cliui "^3.2.0" + decamelize "^1.1.1" + get-caller-file "^1.0.1" + lodash.assign "^4.0.3" + os-locale "^1.4.0" + read-pkg-up "^1.0.1" + require-directory "^2.1.1" + require-main-filename "^1.0.1" + set-blocking "^2.0.0" + string-width "^1.0.1" + which-module "^1.0.0" + window-size "^0.2.0" + y18n "^3.2.1" + yargs-parser "^2.4.1" + yargs@^7.0.0: version "7.1.0" resolved "https://registry.yarnpkg.com/yargs/-/yargs-7.1.0.tgz#6ba318eb16961727f5d284f8ea003e8d6154d0c8"