From feb4bdfd9b46e87aadfa7c0d5338cde887d1f58c Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Sun, 11 Dec 2016 21:50:51 +0100 Subject: [PATCH 01/47] First version with PostgreSQL --- .../friend-list/friend-list.component.html | 2 +- .../app/admin/friends/shared/friend.model.ts | 2 +- .../request-stats.component.html | 2 +- .../request-stats/request-stats.component.ts | 2 +- .../requests/shared/request-stats.model.ts | 6 +- .../users/user-list/user-list.component.html | 2 +- client/src/app/shared/auth/auth-user.model.ts | 9 +- .../app/shared/search/search-field.type.ts | 2 +- .../src/app/shared/search/search.component.ts | 4 +- client/src/app/shared/users/user.model.ts | 10 +- .../src/app/videos/shared/sort-field.type.ts | 2 +- client/src/app/videos/shared/video.model.ts | 6 +- .../videos/video-list/video-list.component.ts | 2 +- .../video-list/video-miniature.component.html | 2 +- .../video-list/video-miniature.component.scss | 2 +- .../videos/video-list/video-sort.component.ts | 4 +- .../video-watch/video-watch.component.html | 2 +- config/default.yaml | 4 +- config/production.yaml.example | 2 +- config/test-1.yaml | 2 +- config/test-2.yaml | 2 +- config/test-3.yaml | 2 +- config/test-4.yaml | 2 +- config/test-5.yaml | 2 +- config/test-6.yaml | 2 +- config/test.yaml | 2 +- package.json | 4 +- scripts/clean/server/test.sh | 3 +- server.js | 9 +- server/controllers/api/clients.js | 8 +- server/controllers/api/pods.js | 15 +- server/controllers/api/remote.js | 95 +++- server/controllers/api/requests.js | 10 +- server/controllers/api/users.js | 28 +- server/controllers/api/videos.js | 63 ++- server/controllers/client.js | 11 +- server/helpers/custom-validators/videos.js | 19 +- server/helpers/logger.js | 6 +- server/initializers/checker.js | 8 +- server/initializers/constants.js | 19 +- server/initializers/database.js | 54 ++- server/initializers/installer.js | 29 +- server/initializers/migrator.js | 12 +- server/lib/friends.js | 73 ++- server/lib/oauth-model.js | 32 +- server/middlewares/pods.js | 1 - server/middlewares/secure.js | 6 +- server/middlewares/sort.js | 4 +- server/middlewares/validators/users.js | 13 +- server/middlewares/validators/videos.js | 17 +- server/models/application.js | 45 +- server/models/author.js | 28 ++ server/models/oauth-client.js | 68 ++- server/models/oauth-token.js | 105 ++++- server/models/pods.js | 152 ++++--- server/models/request.js | 187 ++++---- server/models/requestToPod.js | 30 ++ server/models/user.js | 128 ++++-- server/models/utils.js | 31 +- server/models/video.js | 430 ++++++++++++------ server/tests/api/check-params.js | 6 +- server/tests/api/friends-basic.js | 4 +- server/tests/api/multiple-pods.js | 8 +- server/tests/api/requests.js | 29 +- server/tests/api/single-pod.js | 16 +- server/tests/api/users.js | 8 +- server/tests/utils/servers.js | 4 +- server/tests/utils/videos.js | 2 +- 68 files changed, 1171 insertions(+), 730 deletions(-) create mode 100644 server/models/author.js create mode 100644 server/models/requestToPod.js diff --git a/client/src/app/admin/friends/friend-list/friend-list.component.html b/client/src/app/admin/friends/friend-list/friend-list.component.html index 4236fc5f6..06258f8c8 100644 --- a/client/src/app/admin/friends/friend-list/friend-list.component.html +++ b/client/src/app/admin/friends/friend-list/friend-list.component.html @@ -15,7 +15,7 @@ {{ friend.id }} {{ friend.host }} {{ friend.score }} - {{ friend.createdDate | date: 'medium' }} + {{ friend.createdAt | date: 'medium' }} diff --git a/client/src/app/admin/friends/shared/friend.model.ts b/client/src/app/admin/friends/shared/friend.model.ts index 3c23feebc..462cc82ed 100644 --- a/client/src/app/admin/friends/shared/friend.model.ts +++ b/client/src/app/admin/friends/shared/friend.model.ts @@ -2,5 +2,5 @@ export interface Friend { id: string; host: string; score: number; - createdDate: Date; + createdAt: Date; } diff --git a/client/src/app/admin/requests/request-stats/request-stats.component.html b/client/src/app/admin/requests/request-stats/request-stats.component.html index b5ac59a9a..6698eac48 100644 --- a/client/src/app/admin/requests/request-stats/request-stats.component.html +++ b/client/src/app/admin/requests/request-stats/request-stats.component.html @@ -18,6 +18,6 @@
Remaining requests: - {{ stats.requests.length }} + {{ stats.totalRequests }}
diff --git a/client/src/app/admin/requests/request-stats/request-stats.component.ts b/client/src/app/admin/requests/request-stats/request-stats.component.ts index d20b12199..9e2af219c 100644 --- a/client/src/app/admin/requests/request-stats/request-stats.component.ts +++ b/client/src/app/admin/requests/request-stats/request-stats.component.ts @@ -19,7 +19,7 @@ export class RequestStatsComponent implements OnInit, OnDestroy { } ngOnDestroy() { - if (this.stats.secondsInterval !== null) { + if (this.stats !== null && this.stats.secondsInterval !== null) { clearInterval(this.interval); } } diff --git a/client/src/app/admin/requests/shared/request-stats.model.ts b/client/src/app/admin/requests/shared/request-stats.model.ts index 766e80836..49ecbc79e 100644 --- a/client/src/app/admin/requests/shared/request-stats.model.ts +++ b/client/src/app/admin/requests/shared/request-stats.model.ts @@ -7,18 +7,18 @@ export class RequestStats { maxRequestsInParallel: number; milliSecondsInterval: number; remainingMilliSeconds: number; - requests: Request[]; + totalRequests: number; constructor(hash: { maxRequestsInParallel: number, milliSecondsInterval: number, remainingMilliSeconds: number, - requests: Request[]; + totalRequests: number; }) { this.maxRequestsInParallel = hash.maxRequestsInParallel; this.milliSecondsInterval = hash.milliSecondsInterval; this.remainingMilliSeconds = hash.remainingMilliSeconds; - this.requests = hash.requests; + this.totalRequests = hash.totalRequests; } get remainingSeconds() { diff --git a/client/src/app/admin/users/user-list/user-list.component.html b/client/src/app/admin/users/user-list/user-list.component.html index 328b1be77..36193d119 100644 --- a/client/src/app/admin/users/user-list/user-list.component.html +++ b/client/src/app/admin/users/user-list/user-list.component.html @@ -14,7 +14,7 @@ {{ user.id }} {{ user.username }} - {{ user.createdDate | date: 'medium' }} + {{ user.createdAt | date: 'medium' }} diff --git a/client/src/app/shared/auth/auth-user.model.ts b/client/src/app/shared/auth/auth-user.model.ts index bdd5ea5a9..f560351f4 100644 --- a/client/src/app/shared/auth/auth-user.model.ts +++ b/client/src/app/shared/auth/auth-user.model.ts @@ -7,9 +7,6 @@ export class AuthUser extends User { USERNAME: 'username' }; - id: string; - role: string; - username: string; tokens: Tokens; static load() { @@ -17,7 +14,7 @@ export class AuthUser extends User { if (usernameLocalStorage) { return new AuthUser( { - id: localStorage.getItem(this.KEYS.ID), + id: parseInt(localStorage.getItem(this.KEYS.ID)), username: localStorage.getItem(this.KEYS.USERNAME), role: localStorage.getItem(this.KEYS.ROLE) }, @@ -35,7 +32,7 @@ export class AuthUser extends User { Tokens.flush(); } - constructor(userHash: { id: string, username: string, role: string }, hashTokens: any) { + constructor(userHash: { id: number, username: string, role: string }, hashTokens: any) { super(userHash); this.tokens = new Tokens(hashTokens); } @@ -58,7 +55,7 @@ export class AuthUser extends User { } save() { - localStorage.setItem(AuthUser.KEYS.ID, this.id); + localStorage.setItem(AuthUser.KEYS.ID, this.id.toString()); localStorage.setItem(AuthUser.KEYS.USERNAME, this.username); localStorage.setItem(AuthUser.KEYS.ROLE, this.role); this.tokens.save(); diff --git a/client/src/app/shared/search/search-field.type.ts b/client/src/app/shared/search/search-field.type.ts index 5228ee68a..6be584ed1 100644 --- a/client/src/app/shared/search/search-field.type.ts +++ b/client/src/app/shared/search/search-field.type.ts @@ -1 +1 @@ -export type SearchField = "name" | "author" | "podUrl" | "magnetUri" | "tags"; +export type SearchField = "name" | "author" | "host" | "magnetUri" | "tags"; diff --git a/client/src/app/shared/search/search.component.ts b/client/src/app/shared/search/search.component.ts index b6237469b..9f7e156ec 100644 --- a/client/src/app/shared/search/search.component.ts +++ b/client/src/app/shared/search/search.component.ts @@ -14,8 +14,8 @@ export class SearchComponent implements OnInit { fieldChoices = { name: 'Name', author: 'Author', - podUrl: 'Pod Url', - magnetUri: 'Magnet Uri', + host: 'Pod Host', + magnetUri: 'Magnet URI', tags: 'Tags' }; searchCriterias: Search = { diff --git a/client/src/app/shared/users/user.model.ts b/client/src/app/shared/users/user.model.ts index 726495d11..52d89e004 100644 --- a/client/src/app/shared/users/user.model.ts +++ b/client/src/app/shared/users/user.model.ts @@ -1,16 +1,16 @@ export class User { - id: string; + id: number; username: string; role: string; - createdDate: Date; + createdAt: Date; - constructor(hash: { id: string, username: string, role: string, createdDate?: Date }) { + constructor(hash: { id: number, username: string, role: string, createdAt?: Date }) { this.id = hash.id; this.username = hash.username; this.role = hash.role; - if (hash.createdDate) { - this.createdDate = hash.createdDate; + if (hash.createdAt) { + this.createdAt = hash.createdAt; } } diff --git a/client/src/app/videos/shared/sort-field.type.ts b/client/src/app/videos/shared/sort-field.type.ts index 6e8cc7936..74908e344 100644 --- a/client/src/app/videos/shared/sort-field.type.ts +++ b/client/src/app/videos/shared/sort-field.type.ts @@ -1,3 +1,3 @@ export type SortField = "name" | "-name" | "duration" | "-duration" - | "createdDate" | "-createdDate"; + | "createdAt" | "-createdAt"; diff --git a/client/src/app/videos/shared/video.model.ts b/client/src/app/videos/shared/video.model.ts index b51a0e9de..fae001d78 100644 --- a/client/src/app/videos/shared/video.model.ts +++ b/client/src/app/videos/shared/video.model.ts @@ -1,7 +1,7 @@ export class Video { author: string; by: string; - createdDate: Date; + createdAt: Date; description: string; duration: string; id: string; @@ -27,7 +27,7 @@ export class Video { constructor(hash: { author: string, - createdDate: string, + createdAt: string, description: string, duration: number; id: string, @@ -39,7 +39,7 @@ export class Video { thumbnailPath: string }) { this.author = hash.author; - this.createdDate = new Date(hash.createdDate); + this.createdAt = new Date(hash.createdAt); this.description = hash.description; this.duration = Video.createDurationString(hash.duration); this.id = hash.id; diff --git a/client/src/app/videos/video-list/video-list.component.ts b/client/src/app/videos/video-list/video-list.component.ts index a8b92480b..6c42ba5be 100644 --- a/client/src/app/videos/video-list/video-list.component.ts +++ b/client/src/app/videos/video-list/video-list.component.ts @@ -145,7 +145,7 @@ export class VideoListComponent implements OnInit, OnDestroy { }; } - this.sort = routeParams['sort'] || '-createdDate'; + this.sort = routeParams['sort'] || '-createdAt'; if (routeParams['page'] !== undefined) { this.pagination.currentPage = parseInt(routeParams['page']); diff --git a/client/src/app/videos/video-list/video-miniature.component.html b/client/src/app/videos/video-list/video-miniature.component.html index 16513902b..f2f4a53a9 100644 --- a/client/src/app/videos/video-list/video-miniature.component.html +++ b/client/src/app/videos/video-list/video-miniature.component.html @@ -23,6 +23,6 @@ {{ video.by }} - {{ video.createdDate | date:'short' }} + {{ video.createdAt | date:'short' }} diff --git a/client/src/app/videos/video-list/video-miniature.component.scss b/client/src/app/videos/video-list/video-miniature.component.scss index 6b3fa3bf0..d70b1b50d 100644 --- a/client/src/app/videos/video-list/video-miniature.component.scss +++ b/client/src/app/videos/video-list/video-miniature.component.scss @@ -79,7 +79,7 @@ } } - .video-miniature-author, .video-miniature-created-date { + .video-miniature-author, .video-miniature-created-at { display: block; margin-left: 1px; font-size: 12px; diff --git a/client/src/app/videos/video-list/video-sort.component.ts b/client/src/app/videos/video-list/video-sort.component.ts index ca94b07c2..53951deb4 100644 --- a/client/src/app/videos/video-list/video-sort.component.ts +++ b/client/src/app/videos/video-list/video-sort.component.ts @@ -17,8 +17,8 @@ export class VideoSortComponent { '-name': 'Name - Desc', 'duration': 'Duration - Asc', '-duration': 'Duration - Desc', - 'createdDate': 'Created Date - Asc', - '-createdDate': 'Created Date - Desc' + 'createdAt': 'Created Date - Asc', + '-createdAt': 'Created Date - Desc' }; get choiceKeys() { diff --git a/client/src/app/videos/video-watch/video-watch.component.html b/client/src/app/videos/video-watch/video-watch.component.html index 0f0fa68cc..a726ef3ff 100644 --- a/client/src/app/videos/video-watch/video-watch.component.html +++ b/client/src/app/videos/video-watch/video-watch.component.html @@ -47,7 +47,7 @@ {{ video.by }} - on {{ video.createdDate | date:'short' }} + on {{ video.createdAt | date:'short' }} diff --git a/config/default.yaml b/config/default.yaml index 90f4b9466..631400f7d 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -8,8 +8,8 @@ webserver: database: hostname: 'localhost' - port: 27017 - suffix: '-dev' + port: 5432 + suffix: '_dev' # From the project root directory storage: diff --git a/config/production.yaml.example b/config/production.yaml.example index 056ace434..743f972de 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example @@ -5,4 +5,4 @@ webserver: port: 80 database: - suffix: '-prod' + suffix: '_prod' diff --git a/config/test-1.yaml b/config/test-1.yaml index 6dcc7f294..b2a0a5422 100644 --- a/config/test-1.yaml +++ b/config/test-1.yaml @@ -6,7 +6,7 @@ webserver: port: 9001 database: - suffix: '-test1' + suffix: '_test1' # From the project root directory storage: diff --git a/config/test-2.yaml b/config/test-2.yaml index 209525963..7285f3394 100644 --- a/config/test-2.yaml +++ b/config/test-2.yaml @@ -6,7 +6,7 @@ webserver: port: 9002 database: - suffix: '-test2' + suffix: '_test2' # From the project root directory storage: diff --git a/config/test-3.yaml b/config/test-3.yaml index 15719d107..138a2cd53 100644 --- a/config/test-3.yaml +++ b/config/test-3.yaml @@ -6,7 +6,7 @@ webserver: port: 9003 database: - suffix: '-test3' + suffix: '_test3' # From the project root directory storage: diff --git a/config/test-4.yaml b/config/test-4.yaml index e6f34d013..7425f4af7 100644 --- a/config/test-4.yaml +++ b/config/test-4.yaml @@ -6,7 +6,7 @@ webserver: port: 9004 database: - suffix: '-test4' + suffix: '_test4' # From the project root directory storage: diff --git a/config/test-5.yaml b/config/test-5.yaml index fdeec76d4..1bf0de658 100644 --- a/config/test-5.yaml +++ b/config/test-5.yaml @@ -6,7 +6,7 @@ webserver: port: 9005 database: - suffix: '-test5' + suffix: '_test5' # From the project root directory storage: diff --git a/config/test-6.yaml b/config/test-6.yaml index 0c9630c89..7303a08fc 100644 --- a/config/test-6.yaml +++ b/config/test-6.yaml @@ -6,7 +6,7 @@ webserver: port: 9006 database: - suffix: '-test6' + suffix: '_test6' # From the project root directory storage: diff --git a/config/test.yaml b/config/test.yaml index 06705a987..493a23076 100644 --- a/config/test.yaml +++ b/config/test.yaml @@ -6,4 +6,4 @@ webserver: database: hostname: 'localhost' - port: 27017 + port: 5432 diff --git a/package.json b/package.json index 300af4867..bff21082f 100644 --- a/package.json +++ b/package.json @@ -56,16 +56,18 @@ "lodash": "^4.11.1", "magnet-uri": "^5.1.4", "mkdirp": "^0.5.1", - "mongoose": "^4.0.5", "morgan": "^1.5.3", "multer": "^1.1.0", "openssl-wrapper": "^0.3.4", "parse-torrent": "^5.8.0", "password-generator": "^2.0.2", + "pg": "^6.1.0", + "pg-hstore": "^2.3.2", "request": "^2.57.0", "request-replay": "^1.0.2", "rimraf": "^2.5.4", "scripty": "^1.5.0", + "sequelize": "^3.27.0", "ursa": "^0.9.1", "winston": "^2.1.1", "ws": "^1.1.1" diff --git a/scripts/clean/server/test.sh b/scripts/clean/server/test.sh index 927671dd4..35d3ad50f 100755 --- a/scripts/clean/server/test.sh +++ b/scripts/clean/server/test.sh @@ -1,6 +1,7 @@ #!/usr/bin/env sh for i in $(seq 1 6); do - printf "use peertube-test%s;\ndb.dropDatabase();" "$i" | mongo + dropdb "peertube_test$i" rm -rf "./test$i" + createdb "peertube_test$i" done diff --git a/server.js b/server.js index 6eb022000..e54ffe69f 100644 --- a/server.js +++ b/server.js @@ -17,10 +17,9 @@ const app = express() // ----------- Database ----------- const constants = require('./server/initializers/constants') -const database = require('./server/initializers/database') const logger = require('./server/helpers/logger') - -database.connect() +// Initialize database and models +const db = require('./server/initializers/database') // ----------- Checker ----------- const checker = require('./server/initializers/checker') @@ -39,9 +38,7 @@ if (errorMessage !== null) { const customValidators = require('./server/helpers/custom-validators') const installer = require('./server/initializers/installer') const migrator = require('./server/initializers/migrator') -const mongoose = require('mongoose') const routes = require('./server/controllers') -const Request = mongoose.model('Request') // ----------- Command line ----------- @@ -130,7 +127,7 @@ installer.installApplication(function (err) { // ----------- Make the server listening ----------- server.listen(port, function () { // Activate the pool requests - Request.activate() + db.Request.activate() logger.info('Server listening on port %d', port) logger.info('Webserver: %s', constants.CONFIG.WEBSERVER.URL) diff --git a/server/controllers/api/clients.js b/server/controllers/api/clients.js index 7755f6c2b..cf83cb835 100644 --- a/server/controllers/api/clients.js +++ b/server/controllers/api/clients.js @@ -1,13 +1,11 @@ 'use strict' const express = require('express') -const mongoose = require('mongoose') const constants = require('../../initializers/constants') +const db = require('../../initializers/database') const logger = require('../../helpers/logger') -const Client = mongoose.model('OAuthClient') - const router = express.Router() router.get('/local', getLocalClient) @@ -27,12 +25,12 @@ function getLocalClient (req, res, next) { return res.type('json').status(403).end() } - Client.loadFirstClient(function (err, client) { + db.OAuthClient.loadFirstClient(function (err, client) { if (err) return next(err) if (!client) return next(new Error('No client available.')) res.json({ - client_id: client._id, + client_id: client.clientId, client_secret: client.clientSecret }) }) diff --git a/server/controllers/api/pods.js b/server/controllers/api/pods.js index 7857fcee0..79f3f9d8d 100644 --- a/server/controllers/api/pods.js +++ b/server/controllers/api/pods.js @@ -1,9 +1,9 @@ 'use strict' const express = require('express') -const mongoose = require('mongoose') const waterfall = require('async/waterfall') +const db = require('../../initializers/database') const logger = require('../../helpers/logger') const friends = require('../../lib/friends') const middlewares = require('../../middlewares') @@ -15,7 +15,6 @@ const validators = middlewares.validators.pods const signatureValidator = middlewares.validators.remote.signature const router = express.Router() -const Pod = mongoose.model('Pod') router.get('/', listPods) router.post('/', @@ -53,15 +52,15 @@ function addPods (req, res, next) { waterfall([ function addPod (callback) { - const pod = new Pod(informations) - pod.save(function (err, podCreated) { + const pod = db.Pod.build(informations) + pod.save().asCallback(function (err, podCreated) { // Be sure about the number of parameters for the callback return callback(err, podCreated) }) }, function sendMyVideos (podCreated, callback) { - friends.sendOwnedVideosToPod(podCreated._id) + friends.sendOwnedVideosToPod(podCreated.id) callback(null) }, @@ -84,7 +83,7 @@ function addPods (req, res, next) { } function listPods (req, res, next) { - Pod.list(function (err, podsList) { + db.Pod.list(function (err, podsList) { if (err) return next(err) res.json(getFormatedPods(podsList)) @@ -111,11 +110,11 @@ function removePods (req, res, next) { waterfall([ function loadPod (callback) { - Pod.loadByHost(host, callback) + db.Pod.loadByHost(host, callback) }, function removePod (pod, callback) { - pod.remove(callback) + pod.destroy().asCallback(callback) } ], function (err) { if (err) return next(err) diff --git a/server/controllers/api/remote.js b/server/controllers/api/remote.js index f1046c534..d856576a9 100644 --- a/server/controllers/api/remote.js +++ b/server/controllers/api/remote.js @@ -3,15 +3,15 @@ const each = require('async/each') const eachSeries = require('async/eachSeries') const express = require('express') -const mongoose = require('mongoose') +const waterfall = require('async/waterfall') +const db = require('../../initializers/database') const middlewares = require('../../middlewares') const secureMiddleware = middlewares.secure const validators = middlewares.validators.remote const logger = require('../../helpers/logger') const router = express.Router() -const Video = mongoose.model('Video') router.post('/videos', validators.signature, @@ -53,34 +53,99 @@ function remoteVideos (req, res, next) { function addRemoteVideo (videoToCreateData, fromHost, callback) { logger.debug('Adding remote video "%s".', videoToCreateData.name) - const video = new Video(videoToCreateData) - video.podHost = fromHost - Video.generateThumbnailFromBase64(video, videoToCreateData.thumbnailBase64, function (err) { - if (err) { - logger.error('Cannot generate thumbnail from base 64 data.', { error: err }) - return callback(err) + waterfall([ + + function findOrCreatePod (callback) { + fromHost + + const query = { + where: { + host: fromHost + }, + defaults: { + host: fromHost + } + } + + db.Pod.findOrCreate(query).asCallback(function (err, result) { + // [ instance, wasCreated ] + return callback(err, result[0]) + }) + }, + + function findOrCreateAuthor (pod, callback) { + const username = videoToCreateData.author + + const query = { + where: { + name: username, + podId: pod.id + }, + defaults: { + name: username, + podId: pod.id + } + } + + db.Author.findOrCreate(query).asCallback(function (err, result) { + // [ instance, wasCreated ] + return callback(err, result[0]) + }) + }, + + function createVideoObject (author, callback) { + const videoData = { + name: videoToCreateData.name, + remoteId: videoToCreateData.remoteId, + extname: videoToCreateData.extname, + infoHash: videoToCreateData.infoHash, + description: videoToCreateData.description, + authorId: author.id, + duration: videoToCreateData.duration, + tags: videoToCreateData.tags + } + + const video = db.Video.build(videoData) + + return callback(null, video) + }, + + function generateThumbnail (video, callback) { + db.Video.generateThumbnailFromBase64(video, videoToCreateData.thumbnailBase64, function (err) { + if (err) { + logger.error('Cannot generate thumbnail from base 64 data.', { error: err }) + return callback(err) + } + + video.save().asCallback(callback) + }) + }, + + function insertIntoDB (video, callback) { + video.save().asCallback(callback) } - video.save(callback) - }) + ], callback) } function removeRemoteVideo (videoToRemoveData, fromHost, callback) { + // TODO: use bulkDestroy? + // We need the list because we have to remove some other stuffs (thumbnail etc) - Video.listByHostAndRemoteId(fromHost, videoToRemoveData.remoteId, function (err, videosList) { + db.Video.listByHostAndRemoteId(fromHost, videoToRemoveData.remoteId, function (err, videosList) { if (err) { - logger.error('Cannot list videos from host and magnets.', { error: err }) + logger.error('Cannot list videos from host and remote id.', { error: err.message }) return callback(err) } if (videosList.length === 0) { - logger.error('No remote video was found for this pod.', { magnetUri: videoToRemoveData.magnetUri, podHost: fromHost }) + logger.error('No remote video was found for this pod.', { remoteId: videoToRemoveData.remoteId, podHost: fromHost }) } each(videosList, function (video, callbackEach) { - logger.debug('Removing remote video %s.', video.magnetUri) + logger.debug('Removing remote video %s.', video.remoteId) - video.remove(callbackEach) + video.destroy().asCallback(callbackEach) }, callback) }) } diff --git a/server/controllers/api/requests.js b/server/controllers/api/requests.js index 52aad6997..1f9193fc8 100644 --- a/server/controllers/api/requests.js +++ b/server/controllers/api/requests.js @@ -1,15 +1,13 @@ 'use strict' const express = require('express') -const mongoose = require('mongoose') const constants = require('../../initializers/constants') +const db = require('../../initializers/database') const middlewares = require('../../middlewares') const admin = middlewares.admin const oAuth = middlewares.oauth -const Request = mongoose.model('Request') - const router = express.Router() router.get('/stats', @@ -25,13 +23,13 @@ module.exports = router // --------------------------------------------------------------------------- function getStatsRequests (req, res, next) { - Request.list(function (err, requests) { + db.Request.countTotalRequests(function (err, totalRequests) { if (err) return next(err) return res.json({ - requests: requests, + totalRequests: totalRequests, maxRequestsInParallel: constants.REQUESTS_IN_PARALLEL, - remainingMilliSeconds: Request.remainingMilliSeconds(), + remainingMilliSeconds: db.Request.remainingMilliSeconds(), milliSecondsInterval: constants.REQUESTS_INTERVAL }) }) diff --git a/server/controllers/api/users.js b/server/controllers/api/users.js index b4d687312..890028b36 100644 --- a/server/controllers/api/users.js +++ b/server/controllers/api/users.js @@ -2,10 +2,10 @@ const each = require('async/each') const express = require('express') -const mongoose = require('mongoose') const waterfall = require('async/waterfall') const constants = require('../../initializers/constants') +const db = require('../../initializers/database') const friends = require('../../lib/friends') const logger = require('../../helpers/logger') const middlewares = require('../../middlewares') @@ -17,9 +17,6 @@ const validatorsPagination = middlewares.validators.pagination const validatorsSort = middlewares.validators.sort const validatorsUsers = middlewares.validators.users -const User = mongoose.model('User') -const Video = mongoose.model('Video') - const router = express.Router() router.get('/me', oAuth.authenticate, getUserInformation) @@ -62,13 +59,13 @@ module.exports = router // --------------------------------------------------------------------------- function createUser (req, res, next) { - const user = new User({ + const user = db.User.build({ username: req.body.username, password: req.body.password, role: constants.USER_ROLES.USER }) - user.save(function (err, createdUser) { + user.save().asCallback(function (err, createdUser) { if (err) return next(err) return res.type('json').status(204).end() @@ -76,7 +73,7 @@ function createUser (req, res, next) { } function getUserInformation (req, res, next) { - User.loadByUsername(res.locals.oauth.token.user.username, function (err, user) { + db.User.loadByUsername(res.locals.oauth.token.user.username, function (err, user) { if (err) return next(err) return res.json(user.toFormatedJSON()) @@ -84,7 +81,7 @@ function getUserInformation (req, res, next) { } function listUsers (req, res, next) { - User.listForApi(req.query.start, req.query.count, req.query.sort, function (err, usersList, usersTotal) { + db.User.listForApi(req.query.start, req.query.count, req.query.sort, function (err, usersList, usersTotal) { if (err) return next(err) res.json(getFormatedUsers(usersList, usersTotal)) @@ -94,18 +91,19 @@ function listUsers (req, res, next) { function removeUser (req, res, next) { waterfall([ function getUser (callback) { - User.loadById(req.params.id, callback) + db.User.loadById(req.params.id, callback) }, + // TODO: use foreignkey? function getVideos (user, callback) { - Video.listOwnedByAuthor(user.username, function (err, videos) { + db.Video.listOwnedByAuthor(user.username, function (err, videos) { return callback(err, user, videos) }) }, function removeVideosFromDB (user, videos, callback) { each(videos, function (video, callbackEach) { - video.remove(callbackEach) + video.destroy().asCallback(callbackEach) }, function (err) { return callback(err, user, videos) }) @@ -115,7 +113,7 @@ function removeUser (req, res, next) { videos.forEach(function (video) { const params = { name: video.name, - magnetUri: video.magnetUri + remoteId: video.id } friends.removeVideoToFriends(params) @@ -125,7 +123,7 @@ function removeUser (req, res, next) { }, function removeUserFromDB (user, callback) { - user.remove(callback) + user.destroy().asCallback(callback) } ], function andFinally (err) { if (err) { @@ -138,11 +136,11 @@ function removeUser (req, res, next) { } function updateUser (req, res, next) { - User.loadByUsername(res.locals.oauth.token.user.username, function (err, user) { + db.User.loadByUsername(res.locals.oauth.token.user.username, function (err, user) { if (err) return next(err) user.password = req.body.password - user.save(function (err) { + user.save().asCallback(function (err) { if (err) return next(err) return res.sendStatus(204) diff --git a/server/controllers/api/videos.js b/server/controllers/api/videos.js index daf452573..a61f2b2c9 100644 --- a/server/controllers/api/videos.js +++ b/server/controllers/api/videos.js @@ -2,12 +2,12 @@ const express = require('express') const fs = require('fs') -const mongoose = require('mongoose') const multer = require('multer') const path = require('path') const waterfall = require('async/waterfall') const constants = require('../../initializers/constants') +const db = require('../../initializers/database') const logger = require('../../helpers/logger') const friends = require('../../lib/friends') const middlewares = require('../../middlewares') @@ -22,7 +22,6 @@ const sort = middlewares.sort const utils = require('../../helpers/utils') const router = express.Router() -const Video = mongoose.model('Video') // multer configuration const storage = multer.diskStorage({ @@ -87,40 +86,60 @@ function addVideo (req, res, next) { const videoInfos = req.body waterfall([ - function createVideoObject (callback) { - const id = mongoose.Types.ObjectId() + function findOrCreateAuthor (callback) { + const username = res.locals.oauth.token.user.username + + const query = { + where: { + name: username, + podId: null + }, + defaults: { + name: username, + podId: null // null because it is OUR pod + } + } + + db.Author.findOrCreate(query).asCallback(function (err, result) { + // [ instance, wasCreated ] + return callback(err, result[0]) + }) + }, + + function createVideoObject (author, callback) { const videoData = { - _id: id, name: videoInfos.name, remoteId: null, extname: path.extname(videoFile.filename), description: videoInfos.description, - author: res.locals.oauth.token.user.username, duration: videoFile.duration, - tags: videoInfos.tags + tags: videoInfos.tags, + authorId: author.id } - const video = new Video(videoData) + const video = db.Video.build(videoData) - return callback(null, video) + return callback(null, author, video) }, - // Set the videoname the same as the MongoDB id - function renameVideoFile (video, callback) { + // Set the videoname the same as the id + function renameVideoFile (author, video, callback) { const videoDir = constants.CONFIG.STORAGE.VIDEOS_DIR const source = path.join(videoDir, videoFile.filename) const destination = path.join(videoDir, video.getVideoFilename()) fs.rename(source, destination, function (err) { - return callback(err, video) + return callback(err, author, video) }) }, - function insertIntoDB (video, callback) { - video.save(function (err, video) { - // Assert there are only one argument sent to the next function (video) - return callback(err, video) + function insertIntoDB (author, video, callback) { + video.save().asCallback(function (err, videoCreated) { + // Do not forget to add Author informations to the created video + videoCreated.Author = author + + return callback(err, videoCreated) }) }, @@ -147,7 +166,7 @@ function addVideo (req, res, next) { } function getVideo (req, res, next) { - Video.load(req.params.id, function (err, video) { + db.Video.loadAndPopulateAuthorAndPod(req.params.id, function (err, video) { if (err) return next(err) if (!video) { @@ -159,7 +178,7 @@ function getVideo (req, res, next) { } function listVideos (req, res, next) { - Video.listForApi(req.query.start, req.query.count, req.query.sort, function (err, videosList, videosTotal) { + db.Video.listForApi(req.query.start, req.query.count, req.query.sort, function (err, videosList, videosTotal) { if (err) return next(err) res.json(getFormatedVideos(videosList, videosTotal)) @@ -171,11 +190,11 @@ function removeVideo (req, res, next) { waterfall([ function getVideo (callback) { - Video.load(videoId, callback) + db.Video.load(videoId, callback) }, function removeFromDB (video, callback) { - video.remove(function (err) { + video.destroy().asCallback(function (err) { if (err) return callback(err) return callback(null, video) @@ -185,7 +204,7 @@ function removeVideo (req, res, next) { function sendInformationToFriends (video, callback) { const params = { name: video.name, - remoteId: video._id + remoteId: video.id } friends.removeVideoToFriends(params) @@ -203,7 +222,7 @@ function removeVideo (req, res, next) { } function searchVideos (req, res, next) { - Video.search(req.params.value, req.query.field, req.query.start, req.query.count, req.query.sort, + db.Video.searchAndPopulateAuthorAndPod(req.params.value, req.query.field, req.query.start, req.query.count, req.query.sort, function (err, videosList, videosTotal) { if (err) return next(err) diff --git a/server/controllers/client.js b/server/controllers/client.js index 572db6133..a5fac5626 100644 --- a/server/controllers/client.js +++ b/server/controllers/client.js @@ -3,13 +3,12 @@ const parallel = require('async/parallel') const express = require('express') const fs = require('fs') -const mongoose = require('mongoose') const path = require('path') const validator = require('express-validator').validator const constants = require('../initializers/constants') +const db = require('../initializers/database') -const Video = mongoose.model('Video') const router = express.Router() const opengraphComment = '' @@ -45,14 +44,14 @@ function addOpenGraphTags (htmlStringPage, video) { if (video.isOwned()) { basePreviewUrlHttp = constants.CONFIG.WEBSERVER.URL } else { - basePreviewUrlHttp = constants.REMOTE_SCHEME.HTTP + '://' + video.podHost + basePreviewUrlHttp = constants.REMOTE_SCHEME.HTTP + '://' + video.Author.Pod.host } // We fetch the remote preview (bigger than the thumbnail) // This should not overhead the remote server since social websites put in a cache the OpenGraph tags // We can't use the thumbnail because these social websites want bigger images (> 200x200 for Facebook for example) const previewUrl = basePreviewUrlHttp + constants.STATIC_PATHS.PREVIEWS + video.getPreviewName() - const videoUrl = constants.CONFIG.WEBSERVER.URL + '/videos/watch/' + video._id + const videoUrl = constants.CONFIG.WEBSERVER.URL + '/videos/watch/' + video.id const metaTags = { 'og:type': 'video', @@ -86,7 +85,7 @@ function generateWatchHtmlPage (req, res, next) { const videoId = req.params.id // Let Angular application handle errors - if (!validator.isMongoId(videoId)) return res.sendFile(indexPath) + if (!validator.isUUID(videoId, 4)) return res.sendFile(indexPath) parallel({ file: function (callback) { @@ -94,7 +93,7 @@ function generateWatchHtmlPage (req, res, next) { }, video: function (callback) { - Video.load(videoId, callback) + db.Video.loadAndPopulateAuthorAndPod(videoId, callback) } }, function (err, results) { if (err) return next(err) diff --git a/server/helpers/custom-validators/videos.js b/server/helpers/custom-validators/videos.js index 1a7753265..be8256a80 100644 --- a/server/helpers/custom-validators/videos.js +++ b/server/helpers/custom-validators/videos.js @@ -13,7 +13,7 @@ const videosValidators = { isVideoDateValid, isVideoDescriptionValid, isVideoDurationValid, - isVideoMagnetValid, + isVideoInfoHashValid, isVideoNameValid, isVideoPodHostValid, isVideoTagsValid, @@ -28,14 +28,15 @@ function isEachRemoteVideosValid (requests) { return ( isRequestTypeAddValid(request.type) && isVideoAuthorValid(video.author) && - isVideoDateValid(video.createdDate) && + isVideoDateValid(video.createdAt) && isVideoDescriptionValid(video.description) && isVideoDurationValid(video.duration) && - isVideoMagnetValid(video.magnet) && + isVideoInfoHashValid(video.infoHash) && isVideoNameValid(video.name) && isVideoTagsValid(video.tags) && isVideoThumbnail64Valid(video.thumbnailBase64) && - isVideoRemoteIdValid(video.remoteId) + isVideoRemoteIdValid(video.remoteId) && + isVideoExtnameValid(video.extname) ) || ( isRequestTypeRemoveValid(request.type) && @@ -61,8 +62,12 @@ function isVideoDurationValid (value) { return validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.DURATION) } -function isVideoMagnetValid (value) { - return validator.isLength(value.infoHash, VIDEOS_CONSTRAINTS_FIELDS.MAGNET.INFO_HASH) +function isVideoExtnameValid (value) { + return VIDEOS_CONSTRAINTS_FIELDS.EXTNAME.indexOf(value) !== -1 +} + +function isVideoInfoHashValid (value) { + return validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.INFO_HASH) } function isVideoNameValid (value) { @@ -93,7 +98,7 @@ function isVideoThumbnail64Valid (value) { } function isVideoRemoteIdValid (value) { - return validator.isMongoId(value) + return validator.isUUID(value, 4) } // --------------------------------------------------------------------------- diff --git a/server/helpers/logger.js b/server/helpers/logger.js index fcc1789fd..281acedb8 100644 --- a/server/helpers/logger.js +++ b/server/helpers/logger.js @@ -22,7 +22,8 @@ const logger = new winston.Logger({ json: true, maxsize: 5242880, maxFiles: 5, - colorize: false + colorize: false, + prettyPrint: true }), new winston.transports.Console({ level: 'debug', @@ -30,7 +31,8 @@ const logger = new winston.Logger({ handleExceptions: true, humanReadableUnhandledException: true, json: false, - colorize: true + colorize: true, + prettyPrint: true }) ], exitOnError: true diff --git a/server/initializers/checker.js b/server/initializers/checker.js index aea013fa9..7b402de82 100644 --- a/server/initializers/checker.js +++ b/server/initializers/checker.js @@ -1,10 +1,8 @@ 'use strict' const config = require('config') -const mongoose = require('mongoose') -const Client = mongoose.model('OAuthClient') -const User = mongoose.model('User') +const db = require('./database') const checker = { checkConfig, @@ -44,7 +42,7 @@ function checkMissedConfig () { } function clientsExist (callback) { - Client.list(function (err, clients) { + db.OAuthClient.list(function (err, clients) { if (err) return callback(err) return callback(null, clients.length !== 0) @@ -52,7 +50,7 @@ function clientsExist (callback) { } function usersExist (callback) { - User.countTotal(function (err, totalUsers) { + db.User.countTotal(function (err, totalUsers) { if (err) return callback(err) return callback(null, totalUsers !== 0) diff --git a/server/initializers/constants.js b/server/initializers/constants.js index 3ddf87454..1ad0c82a0 100644 --- a/server/initializers/constants.js +++ b/server/initializers/constants.js @@ -14,13 +14,13 @@ const PAGINATION_COUNT_DEFAULT = 15 // Sortable columns per schema const SEARCHABLE_COLUMNS = { - VIDEOS: [ 'name', 'magnetUri', 'podHost', 'author', 'tags' ] + VIDEOS: [ 'name', 'magnetUri', 'host', 'author', 'tags' ] } // Sortable columns per schema const SORTABLE_COLUMNS = { - USERS: [ 'username', '-username', 'createdDate', '-createdDate' ], - VIDEOS: [ 'name', '-name', 'duration', '-duration', 'createdDate', '-createdDate' ] + USERS: [ 'username', '-username', 'createdAt', '-createdAt' ], + VIDEOS: [ 'name', '-name', 'duration', '-duration', 'createdAt', '-createdAt' ] } const OAUTH_LIFETIME = { @@ -67,9 +67,8 @@ const CONSTRAINTS_FIELDS = { VIDEOS: { NAME: { min: 3, max: 50 }, // Length DESCRIPTION: { min: 3, max: 250 }, // Length - MAGNET: { - INFO_HASH: { min: 10, max: 50 } // Length - }, + EXTNAME: [ '.mp4', '.ogv', '.webm' ], + INFO_HASH: { min: 10, max: 50 }, // Length DURATION: { min: 1, max: 7200 }, // Number TAGS: { min: 1, max: 3 }, // Number of total tags TAG: { min: 2, max: 10 }, // Length @@ -88,7 +87,7 @@ const FRIEND_SCORE = { // --------------------------------------------------------------------------- -const MONGO_MIGRATION_SCRIPTS = [ +const MIGRATION_SCRIPTS = [ { script: '0005-create-application', version: 5 @@ -122,7 +121,7 @@ const MONGO_MIGRATION_SCRIPTS = [ version: 40 } ] -const LAST_MONGO_SCHEMA_VERSION = (maxBy(MONGO_MIGRATION_SCRIPTS, 'version'))['version'] +const LAST_SQL_SCHEMA_VERSION = (maxBy(MIGRATION_SCRIPTS, 'version'))['version'] // --------------------------------------------------------------------------- @@ -198,8 +197,8 @@ module.exports = { CONFIG, CONSTRAINTS_FIELDS, FRIEND_SCORE, - LAST_MONGO_SCHEMA_VERSION, - MONGO_MIGRATION_SCRIPTS, + LAST_SQL_SCHEMA_VERSION, + MIGRATION_SCRIPTS, OAUTH_LIFETIME, PAGINATION_COUNT_DEFAULT, PODS_SCORE, diff --git a/server/initializers/database.js b/server/initializers/database.js index 0564e4e77..cc6f59b63 100644 --- a/server/initializers/database.js +++ b/server/initializers/database.js @@ -1,36 +1,46 @@ 'use strict' -const mongoose = require('mongoose') +const fs = require('fs') +const path = require('path') +const Sequelize = require('sequelize') const constants = require('../initializers/constants') const logger = require('../helpers/logger') -// Bootstrap models -require('../models/application') -require('../models/oauth-token') -require('../models/user') -require('../models/oauth-client') -require('../models/video') -// Request model needs Video model -require('../models/pods') -// Request model needs Pod model -require('../models/request') +const database = {} -const database = { - connect: connect -} +const sequelize = new Sequelize(constants.CONFIG.DATABASE.DBNAME, 'peertube', 'peertube', { + dialect: 'postgres', + host: constants.CONFIG.DATABASE.HOSTNAME, + port: constants.CONFIG.DATABASE.PORT +}) -function connect () { - mongoose.Promise = global.Promise - mongoose.connect('mongodb://' + constants.CONFIG.DATABASE.HOSTNAME + ':' + constants.CONFIG.DATABASE.PORT + '/' + constants.CONFIG.DATABASE.DBNAME) - mongoose.connection.on('error', function () { - throw new Error('Mongodb connection error.') +const modelDirectory = path.join(__dirname, '..', 'models') +fs.readdir(modelDirectory, function (err, files) { + if (err) throw err + + files.filter(function (file) { + if (file === 'utils.js') return false + + return true + }) + .forEach(function (file) { + const model = sequelize.import(path.join(modelDirectory, file)) + + database[model.name] = model }) - mongoose.connection.on('open', function () { - logger.info('Connected to mongodb.') + Object.keys(database).forEach(function (modelName) { + if ('associate' in database[modelName]) { + database[modelName].associate(database) + } }) -} + + logger.info('Database is ready.') +}) + +database.sequelize = sequelize +database.Sequelize = Sequelize // --------------------------------------------------------------------------- diff --git a/server/initializers/installer.js b/server/initializers/installer.js index 1df300ba8..4823bc8c8 100644 --- a/server/initializers/installer.js +++ b/server/initializers/installer.js @@ -3,26 +3,27 @@ const config = require('config') const each = require('async/each') const mkdirp = require('mkdirp') -const mongoose = require('mongoose') const passwordGenerator = require('password-generator') const path = require('path') const series = require('async/series') const checker = require('./checker') const constants = require('./constants') +const db = require('./database') const logger = require('../helpers/logger') const peertubeCrypto = require('../helpers/peertube-crypto') -const Application = mongoose.model('Application') -const Client = mongoose.model('OAuthClient') -const User = mongoose.model('User') - const installer = { installApplication } function installApplication (callback) { series([ + function createDatabase (callbackAsync) { + db.sequelize.sync().asCallback(callbackAsync) + // db.sequelize.sync({ force: true }).asCallback(callbackAsync) + }, + function createDirectories (callbackAsync) { createDirectoriesIfNotExist(callbackAsync) }, @@ -65,16 +66,18 @@ function createOAuthClientIfNotExist (callback) { logger.info('Creating a default OAuth Client.') - const secret = passwordGenerator(32, false) - const client = new Client({ + const id = passwordGenerator(32, false, /[a-z0-9]/) + const secret = passwordGenerator(32, false, /[a-zA-Z0-9]/) + const client = db.OAuthClient.build({ + clientId: id, clientSecret: secret, grants: [ 'password', 'refresh_token' ] }) - client.save(function (err, createdClient) { + client.save().asCallback(function (err, createdClient) { if (err) return callback(err) - logger.info('Client id: ' + createdClient._id) + logger.info('Client id: ' + createdClient.clientId) logger.info('Client secret: ' + createdClient.clientSecret) return callback(null) @@ -106,21 +109,21 @@ function createOAuthAdminIfNotExist (callback) { password = passwordGenerator(8, true) } - const user = new User({ + const user = db.User.build({ username, password, role }) - user.save(function (err, createdUser) { + user.save().asCallback(function (err, createdUser) { if (err) return callback(err) logger.info('Username: ' + username) logger.info('User password: ' + password) logger.info('Creating Application collection.') - const application = new Application({ mongoSchemaVersion: constants.LAST_MONGO_SCHEMA_VERSION }) - application.save(callback) + const application = db.Application.build({ sqlSchemaVersion: constants.LAST_SQL_SCHEMA_VERSION }) + application.save().asCallback(callback) }) }) } diff --git a/server/initializers/migrator.js b/server/initializers/migrator.js index 6b31d994f..9e5350e60 100644 --- a/server/initializers/migrator.js +++ b/server/initializers/migrator.js @@ -1,24 +1,22 @@ 'use strict' const eachSeries = require('async/eachSeries') -const mongoose = require('mongoose') const path = require('path') const constants = require('./constants') +const db = require('./database') const logger = require('../helpers/logger') -const Application = mongoose.model('Application') - const migrator = { migrate: migrate } function migrate (callback) { - Application.loadMongoSchemaVersion(function (err, actualVersion) { + db.Application.loadSqlSchemaVersion(function (err, actualVersion) { if (err) return callback(err) // If there are a new mongo schemas - if (!actualVersion || actualVersion < constants.LAST_MONGO_SCHEMA_VERSION) { + if (!actualVersion || actualVersion < constants.LAST_SQL_SCHEMA_VERSION) { logger.info('Begin migrations.') eachSeries(constants.MONGO_MIGRATION_SCRIPTS, function (entity, callbackEach) { @@ -36,12 +34,12 @@ function migrate (callback) { if (err) return callbackEach(err) // Update the new mongo version schema - Application.updateMongoSchemaVersion(versionScript, callbackEach) + db.Application.updateSqlSchemaVersion(versionScript, callbackEach) }) }, function (err) { if (err) return callback(err) - logger.info('Migrations finished. New mongo version schema: %s', constants.LAST_MONGO_SCHEMA_VERSION) + logger.info('Migrations finished. New SQL version schema: %s', constants.LAST_SQL_SCHEMA_VERSION) return callback(null) }) } else { diff --git a/server/lib/friends.js b/server/lib/friends.js index eaea040ca..3ed29f651 100644 --- a/server/lib/friends.js +++ b/server/lib/friends.js @@ -4,18 +4,14 @@ const each = require('async/each') const eachLimit = require('async/eachLimit') const eachSeries = require('async/eachSeries') const fs = require('fs') -const mongoose = require('mongoose') const request = require('request') const waterfall = require('async/waterfall') const constants = require('../initializers/constants') +const db = require('../initializers/database') const logger = require('../helpers/logger') const requests = require('../helpers/requests') -const Pod = mongoose.model('Pod') -const Request = mongoose.model('Request') -const Video = mongoose.model('Video') - const friends = { addVideoToFriends, hasFriends, @@ -31,7 +27,7 @@ function addVideoToFriends (video) { } function hasFriends (callback) { - Pod.countAll(function (err, count) { + db.Pod.countAll(function (err, count) { if (err) return callback(err) const hasFriends = (count !== 0) @@ -69,13 +65,13 @@ function makeFriends (hosts, callback) { function quitFriends (callback) { // Stop pool requests - Request.deactivate() + db.Request.deactivate() // Flush pool requests - Request.flush() + db.Request.flush() waterfall([ function getPodsList (callbackAsync) { - return Pod.list(callbackAsync) + return db.Pod.list(callbackAsync) }, function announceIQuitMyFriends (pods, callbackAsync) { @@ -103,12 +99,12 @@ function quitFriends (callback) { function removePodsFromDB (pods, callbackAsync) { each(pods, function (pod, callbackEach) { - pod.remove(callbackEach) + pod.destroy().asCallback(callbackEach) }, callbackAsync) } ], function (err) { // Don't forget to re activate the scheduler, even if there was an error - Request.activate() + db.Request.activate() if (err) return callback(err) @@ -122,7 +118,7 @@ function removeVideoToFriends (videoParams) { } function sendOwnedVideosToPod (podId) { - Video.listOwned(function (err, videosList) { + db.Video.listOwnedAndPopulateAuthor(function (err, videosList) { if (err) { logger.error('Cannot get the list of videos we own.') return @@ -200,9 +196,9 @@ function getForeignPodsList (host, callback) { function makeRequestsToWinningPods (cert, podsList, callback) { // Stop pool requests - Request.deactivate() + db.Request.deactivate() // Flush pool requests - Request.forceSend() + db.Request.forceSend() eachLimit(podsList, constants.REQUESTS_IN_PARALLEL, function (pod, callbackEach) { const params = { @@ -222,8 +218,8 @@ function makeRequestsToWinningPods (cert, podsList, callback) { } if (res.statusCode === 200) { - const podObj = new Pod({ host: pod.host, publicKey: body.cert }) - podObj.save(function (err, podCreated) { + const podObj = db.Pod.build({ host: pod.host, publicKey: body.cert }) + podObj.save().asCallback(function (err, podCreated) { if (err) { logger.error('Cannot add friend %s pod.', pod.host, { error: err }) return callbackEach() @@ -242,28 +238,57 @@ function makeRequestsToWinningPods (cert, podsList, callback) { }, function endRequests () { // Final callback, we've ended all the requests // Now we made new friends, we can re activate the pool of requests - Request.activate() + db.Request.activate() logger.debug('makeRequestsToWinningPods finished.') return callback() }) } +// Wrapper that populate "to" argument with all our friends if it is not specified function createRequest (type, endpoint, data, to) { - const req = new Request({ + if (to) return _createRequest(type, endpoint, data, to) + + // If the "to" pods is not specified, we send the request to all our friends + db.Pod.listAllIds(function (err, podIds) { + if (err) { + logger.error('Cannot get pod ids', { error: err }) + return + } + + return _createRequest(type, endpoint, data, podIds) + }) +} + +function _createRequest (type, endpoint, data, to) { + const pods = [] + + // If there are no destination pods abort + if (to.length === 0) return + + to.forEach(function (toPod) { + pods.push(db.Pod.build({ id: toPod })) + }) + + const createQuery = { endpoint, request: { type: type, data: data } - }) - - if (to) { - req.to = to } - req.save(function (err) { - if (err) logger.error('Cannot save the request.', { error: err }) + // We run in transaction to keep coherency between Request and RequestToPod tables + db.sequelize.transaction(function (t) { + const dbRequestOptions = { + transaction: t + } + + return db.Request.create(createQuery, dbRequestOptions).then(function (request) { + return request.setPods(pods, dbRequestOptions) + }) + }).asCallback(function (err) { + if (err) logger.error('Error in createRequest transaction.', { error: err }) }) } diff --git a/server/lib/oauth-model.js b/server/lib/oauth-model.js index d011c4b72..1c12f1b14 100644 --- a/server/lib/oauth-model.js +++ b/server/lib/oauth-model.js @@ -1,11 +1,6 @@ -const mongoose = require('mongoose') - +const db = require('../initializers/database') const logger = require('../helpers/logger') -const OAuthClient = mongoose.model('OAuthClient') -const OAuthToken = mongoose.model('OAuthToken') -const User = mongoose.model('User') - // See https://github.com/oauthjs/node-oauth2-server/wiki/Model-specification for the model specifications const OAuthModel = { getAccessToken, @@ -21,27 +16,25 @@ const OAuthModel = { function getAccessToken (bearerToken) { logger.debug('Getting access token (bearerToken: ' + bearerToken + ').') - return OAuthToken.getByTokenAndPopulateUser(bearerToken) + return db.OAuthToken.getByTokenAndPopulateUser(bearerToken) } function getClient (clientId, clientSecret) { logger.debug('Getting Client (clientId: ' + clientId + ', clientSecret: ' + clientSecret + ').') - // TODO req validator - const mongoId = new mongoose.mongo.ObjectID(clientId) - return OAuthClient.getByIdAndSecret(mongoId, clientSecret) + return db.OAuthClient.getByIdAndSecret(clientId, clientSecret) } function getRefreshToken (refreshToken) { logger.debug('Getting RefreshToken (refreshToken: ' + refreshToken + ').') - return OAuthToken.getByRefreshTokenAndPopulateClient(refreshToken) + return db.OAuthToken.getByRefreshTokenAndPopulateClient(refreshToken) } function getUser (username, password) { logger.debug('Getting User (username: ' + username + ', password: ' + password + ').') - return User.getByUsername(username).then(function (user) { + return db.User.getByUsername(username).then(function (user) { if (!user) return null // We need to return a promise @@ -60,8 +53,8 @@ function getUser (username, password) { } function revokeToken (token) { - return OAuthToken.getByRefreshTokenAndPopulateUser(token.refreshToken).then(function (tokenDB) { - if (tokenDB) tokenDB.remove() + return db.OAuthToken.getByRefreshTokenAndPopulateUser(token.refreshToken).then(function (tokenDB) { + if (tokenDB) tokenDB.destroy() /* * Thanks to https://github.com/manjeshpv/node-oauth2-server-implementation/blob/master/components/oauth/mongo-models.js @@ -80,18 +73,19 @@ function revokeToken (token) { function saveToken (token, client, user) { logger.debug('Saving token ' + token.accessToken + ' for client ' + client.id + ' and user ' + user.id + '.') - const tokenObj = new OAuthToken({ + const tokenToCreate = { accessToken: token.accessToken, accessTokenExpiresAt: token.accessTokenExpiresAt, - client: client.id, refreshToken: token.refreshToken, refreshTokenExpiresAt: token.refreshTokenExpiresAt, - user: user.id - }) + oAuthClientId: client.id, + userId: user.id + } - return tokenObj.save().then(function (tokenCreated) { + return db.OAuthToken.create(tokenToCreate).then(function (tokenCreated) { tokenCreated.client = client tokenCreated.user = user + return tokenCreated }).catch(function (err) { throw err diff --git a/server/middlewares/pods.js b/server/middlewares/pods.js index 487ea1259..e38fb341d 100644 --- a/server/middlewares/pods.js +++ b/server/middlewares/pods.js @@ -44,7 +44,6 @@ module.exports = podsMiddleware function getHostWithPort (host) { const splitted = host.split(':') - console.log(splitted) // The port was not specified if (splitted.length === 1) { if (constants.REMOTE_SCHEME.HTTP === 'https') return host + ':443' diff --git a/server/middlewares/secure.js b/server/middlewares/secure.js index ee836beed..b7b4cdfb4 100644 --- a/server/middlewares/secure.js +++ b/server/middlewares/secure.js @@ -1,18 +1,16 @@ 'use strict' +const db = require('../initializers/database') const logger = require('../helpers/logger') -const mongoose = require('mongoose') const peertubeCrypto = require('../helpers/peertube-crypto') -const Pod = mongoose.model('Pod') - const secureMiddleware = { checkSignature } function checkSignature (req, res, next) { const host = req.body.signature.host - Pod.loadByHost(host, function (err, pod) { + db.Pod.loadByHost(host, function (err, pod) { if (err) { logger.error('Cannot get signed host in body.', { error: err }) return res.sendStatus(500) diff --git a/server/middlewares/sort.js b/server/middlewares/sort.js index f0b7274eb..477e10571 100644 --- a/server/middlewares/sort.js +++ b/server/middlewares/sort.js @@ -6,13 +6,13 @@ const sortMiddleware = { } function setUsersSort (req, res, next) { - if (!req.query.sort) req.query.sort = '-createdDate' + if (!req.query.sort) req.query.sort = '-createdAt' return next() } function setVideosSort (req, res, next) { - if (!req.query.sort) req.query.sort = '-createdDate' + if (!req.query.sort) req.query.sort = '-createdAt' return next() } diff --git a/server/middlewares/validators/users.js b/server/middlewares/validators/users.js index 02e4f34cb..0629550bc 100644 --- a/server/middlewares/validators/users.js +++ b/server/middlewares/validators/users.js @@ -1,12 +1,9 @@ 'use strict' -const mongoose = require('mongoose') - const checkErrors = require('./utils').checkErrors +const db = require('../../initializers/database') const logger = require('../../helpers/logger') -const User = mongoose.model('User') - const validatorsUsers = { usersAdd, usersRemove, @@ -20,7 +17,7 @@ function usersAdd (req, res, next) { logger.debug('Checking usersAdd parameters', { parameters: req.body }) checkErrors(req, res, function () { - User.loadByUsername(req.body.username, function (err, user) { + db.User.loadByUsername(req.body.username, function (err, user) { if (err) { logger.error('Error in usersAdd request validator.', { error: err }) return res.sendStatus(500) @@ -34,12 +31,12 @@ function usersAdd (req, res, next) { } function usersRemove (req, res, next) { - req.checkParams('id', 'Should have a valid id').notEmpty().isMongoId() + req.checkParams('id', 'Should have a valid id').notEmpty().isInt() logger.debug('Checking usersRemove parameters', { parameters: req.params }) checkErrors(req, res, function () { - User.loadById(req.params.id, function (err, user) { + db.User.loadById(req.params.id, function (err, user) { if (err) { logger.error('Error in usersRemove request validator.', { error: err }) return res.sendStatus(500) @@ -55,7 +52,7 @@ function usersRemove (req, res, next) { } function usersUpdate (req, res, next) { - req.checkParams('id', 'Should have a valid id').notEmpty().isMongoId() + req.checkParams('id', 'Should have a valid id').notEmpty().isInt() // Add old password verification req.checkBody('password', 'Should have a valid password').isUserPasswordValid() diff --git a/server/middlewares/validators/videos.js b/server/middlewares/validators/videos.js index 76e943e77..7e90ca047 100644 --- a/server/middlewares/validators/videos.js +++ b/server/middlewares/validators/videos.js @@ -1,14 +1,11 @@ 'use strict' -const mongoose = require('mongoose') - const checkErrors = require('./utils').checkErrors const constants = require('../../initializers/constants') const customVideosValidators = require('../../helpers/custom-validators').videos +const db = require('../../initializers/database') const logger = require('../../helpers/logger') -const Video = mongoose.model('Video') - const validatorsVideos = { videosAdd, videosGet, @@ -29,7 +26,7 @@ function videosAdd (req, res, next) { checkErrors(req, res, function () { const videoFile = req.files.videofile[0] - Video.getDurationFromFile(videoFile.path, function (err, duration) { + db.Video.getDurationFromFile(videoFile.path, function (err, duration) { if (err) { return res.status(400).send('Cannot retrieve metadata of the file.') } @@ -45,12 +42,12 @@ function videosAdd (req, res, next) { } function videosGet (req, res, next) { - req.checkParams('id', 'Should have a valid id').notEmpty().isMongoId() + req.checkParams('id', 'Should have a valid id').notEmpty().isUUID(4) logger.debug('Checking videosGet parameters', { parameters: req.params }) checkErrors(req, res, function () { - Video.load(req.params.id, function (err, video) { + db.Video.load(req.params.id, function (err, video) { if (err) { logger.error('Error in videosGet request validator.', { error: err }) return res.sendStatus(500) @@ -64,12 +61,12 @@ function videosGet (req, res, next) { } function videosRemove (req, res, next) { - req.checkParams('id', 'Should have a valid id').notEmpty().isMongoId() + req.checkParams('id', 'Should have a valid id').notEmpty().isUUID(4) logger.debug('Checking videosRemove parameters', { parameters: req.params }) checkErrors(req, res, function () { - Video.load(req.params.id, function (err, video) { + db.Video.loadAndPopulateAuthor(req.params.id, function (err, video) { if (err) { logger.error('Error in videosRemove request validator.', { error: err }) return res.sendStatus(500) @@ -77,7 +74,7 @@ function videosRemove (req, res, next) { if (!video) return res.status(404).send('Video not found') else if (video.isOwned() === false) return res.status(403).send('Cannot remove video of another pod') - else if (video.author !== res.locals.oauth.token.user.username) return res.status(403).send('Cannot remove video of another user') + else if (video.Author.name !== res.locals.oauth.token.user.username) return res.status(403).send('Cannot remove video of another user') next() }) diff --git a/server/models/application.js b/server/models/application.js index 452ac4283..ec1d7b122 100644 --- a/server/models/application.js +++ b/server/models/application.js @@ -1,31 +1,36 @@ -const mongoose = require('mongoose') +module.exports = function (sequelize, DataTypes) { + const Application = sequelize.define('Application', + { + sqlSchemaVersion: { + type: DataTypes.INTEGER, + defaultValue: 0 + } + }, + { + classMethods: { + loadSqlSchemaVersion, + updateSqlSchemaVersion + } + } + ) -// --------------------------------------------------------------------------- - -const ApplicationSchema = mongoose.Schema({ - mongoSchemaVersion: { - type: Number, - default: 0 - } -}) - -ApplicationSchema.statics = { - loadMongoSchemaVersion, - updateMongoSchemaVersion + return Application } -mongoose.model('Application', ApplicationSchema) - // --------------------------------------------------------------------------- -function loadMongoSchemaVersion (callback) { - return this.findOne({}, { mongoSchemaVersion: 1 }, function (err, data) { - const version = data ? data.mongoSchemaVersion : 0 +function loadSqlSchemaVersion (callback) { + const query = { + attributes: [ 'sqlSchemaVersion' ] + } + + return this.findOne(query).asCallback(function (err, data) { + const version = data ? data.sqlSchemaVersion : 0 return callback(err, version) }) } -function updateMongoSchemaVersion (newVersion, callback) { - return this.update({}, { mongoSchemaVersion: newVersion }, callback) +function updateSqlSchemaVersion (newVersion, callback) { + return this.update({ sqlSchemaVersion: newVersion }).asCallback(callback) } diff --git a/server/models/author.js b/server/models/author.js new file mode 100644 index 000000000..493c2ca11 --- /dev/null +++ b/server/models/author.js @@ -0,0 +1,28 @@ +module.exports = function (sequelize, DataTypes) { + const Author = sequelize.define('Author', + { + name: { + type: DataTypes.STRING + } + }, + { + classMethods: { + associate + } + } + ) + + return Author +} + +// --------------------------------------------------------------------------- + +function associate (models) { + this.belongsTo(models.Pod, { + foreignKey: { + name: 'podId', + allowNull: true + }, + onDelete: 'cascade' + }) +} diff --git a/server/models/oauth-client.js b/server/models/oauth-client.js index a1aefa985..15118591a 100644 --- a/server/models/oauth-client.js +++ b/server/models/oauth-client.js @@ -1,33 +1,63 @@ -const mongoose = require('mongoose') +module.exports = function (sequelize, DataTypes) { + const OAuthClient = sequelize.define('OAuthClient', + { + clientId: { + type: DataTypes.STRING + }, + clientSecret: { + type: DataTypes.STRING + }, + grants: { + type: DataTypes.ARRAY(DataTypes.STRING) + }, + redirectUris: { + type: DataTypes.ARRAY(DataTypes.STRING) + } + }, + { + classMethods: { + associate, -// --------------------------------------------------------------------------- + getByIdAndSecret, + list, + loadFirstClient + } + } + ) -const OAuthClientSchema = mongoose.Schema({ - clientSecret: String, - grants: Array, - redirectUris: Array -}) - -OAuthClientSchema.path('clientSecret').required(true) - -OAuthClientSchema.statics = { - getByIdAndSecret, - list, - loadFirstClient + return OAuthClient } -mongoose.model('OAuthClient', OAuthClientSchema) +// TODO: validation +// OAuthClientSchema.path('clientSecret').required(true) // --------------------------------------------------------------------------- +function associate (models) { + this.hasMany(models.OAuthToken, { + foreignKey: { + name: 'oAuthClientId', + allowNull: false + }, + onDelete: 'cascade' + }) +} + function list (callback) { - return this.find(callback) + return this.findAll().asCallback(callback) } function loadFirstClient (callback) { - return this.findOne({}, callback) + return this.findOne().asCallback(callback) } -function getByIdAndSecret (id, clientSecret) { - return this.findOne({ _id: id, clientSecret: clientSecret }).exec() +function getByIdAndSecret (clientId, clientSecret) { + const query = { + where: { + clientId: clientId, + clientSecret: clientSecret + } + } + + return this.findOne(query) } diff --git a/server/models/oauth-token.js b/server/models/oauth-token.js index aff73bfb1..c9108bf95 100644 --- a/server/models/oauth-token.js +++ b/server/models/oauth-token.js @@ -1,42 +1,71 @@ -const mongoose = require('mongoose') - const logger = require('../helpers/logger') // --------------------------------------------------------------------------- -const OAuthTokenSchema = mongoose.Schema({ - accessToken: String, - accessTokenExpiresAt: Date, - client: { type: mongoose.Schema.Types.ObjectId, ref: 'OAuthClient' }, - refreshToken: String, - refreshTokenExpiresAt: Date, - user: { type: mongoose.Schema.Types.ObjectId, ref: 'User' } -}) +module.exports = function (sequelize, DataTypes) { + const OAuthToken = sequelize.define('OAuthToken', + { + accessToken: { + type: DataTypes.STRING + }, + accessTokenExpiresAt: { + type: DataTypes.DATE + }, + refreshToken: { + type: DataTypes.STRING + }, + refreshTokenExpiresAt: { + type: DataTypes.DATE + } + }, + { + classMethods: { + associate, -OAuthTokenSchema.path('accessToken').required(true) -OAuthTokenSchema.path('client').required(true) -OAuthTokenSchema.path('user').required(true) + getByRefreshTokenAndPopulateClient, + getByTokenAndPopulateUser, + getByRefreshTokenAndPopulateUser, + removeByUserId + } + } + ) -OAuthTokenSchema.statics = { - getByRefreshTokenAndPopulateClient, - getByTokenAndPopulateUser, - getByRefreshTokenAndPopulateUser, - removeByUserId + return OAuthToken } -mongoose.model('OAuthToken', OAuthTokenSchema) +// TODO: validation +// OAuthTokenSchema.path('accessToken').required(true) +// OAuthTokenSchema.path('client').required(true) +// OAuthTokenSchema.path('user').required(true) // --------------------------------------------------------------------------- +function associate (models) { + this.belongsTo(models.User, { + foreignKey: { + name: 'userId', + allowNull: false + }, + onDelete: 'cascade' + }) +} + function getByRefreshTokenAndPopulateClient (refreshToken) { - return this.findOne({ refreshToken: refreshToken }).populate('client').exec().then(function (token) { + const query = { + where: { + refreshToken: refreshToken + }, + include: [ this.associations.OAuthClient ] + } + + return this.findOne(query).then(function (token) { if (!token) return token const tokenInfos = { refreshToken: token.refreshToken, refreshTokenExpiresAt: token.refreshTokenExpiresAt, client: { - id: token.client._id.toString() + id: token.client.id }, user: { id: token.user @@ -50,13 +79,41 @@ function getByRefreshTokenAndPopulateClient (refreshToken) { } function getByTokenAndPopulateUser (bearerToken) { - return this.findOne({ accessToken: bearerToken }).populate('user').exec() + const query = { + where: { + accessToken: bearerToken + }, + include: [ this.sequelize.models.User ] + } + + return this.findOne(query).then(function (token) { + if (token) token.user = token.User + + return token + }) } function getByRefreshTokenAndPopulateUser (refreshToken) { - return this.findOne({ refreshToken: refreshToken }).populate('user').exec() + const query = { + where: { + refreshToken: refreshToken + }, + include: [ this.sequelize.models.User ] + } + + return this.findOne(query).then(function (token) { + token.user = token.User + + return token + }) } function removeByUserId (userId, callback) { - return this.remove({ user: userId }, callback) + const query = { + where: { + userId: userId + } + } + + return this.destroy(query).asCallback(callback) } diff --git a/server/models/pods.js b/server/models/pods.js index 49c73472a..2c1f56203 100644 --- a/server/models/pods.js +++ b/server/models/pods.js @@ -1,79 +1,62 @@ 'use strict' -const each = require('async/each') -const mongoose = require('mongoose') const map = require('lodash/map') -const validator = require('express-validator').validator const constants = require('../initializers/constants') -const Video = mongoose.model('Video') - // --------------------------------------------------------------------------- -const PodSchema = mongoose.Schema({ - host: String, - publicKey: String, - score: { type: Number, max: constants.FRIEND_SCORE.MAX }, - createdDate: { - type: Date, - default: Date.now - } -}) +module.exports = function (sequelize, DataTypes) { + const Pod = sequelize.define('Pod', + { + host: { + type: DataTypes.STRING + }, + publicKey: { + type: DataTypes.STRING(5000) + }, + score: { + type: DataTypes.INTEGER, + defaultValue: constants.FRIEND_SCORE.BASE + } + // Check createdAt + }, + { + classMethods: { + associate, -PodSchema.path('host').validate(validator.isURL) -PodSchema.path('publicKey').required(true) -PodSchema.path('score').validate(function (value) { return !isNaN(value) }) + countAll, + incrementScores, + list, + listAllIds, + listBadPods, + load, + loadByHost, + removeAll + }, + instanceMethods: { + toFormatedJSON + } + } + ) -PodSchema.methods = { - toFormatedJSON + return Pod } -PodSchema.statics = { - countAll, - incrementScores, - list, - listAllIds, - listBadPods, - load, - loadByHost, - removeAll -} - -PodSchema.pre('save', function (next) { - const self = this - - Pod.loadByHost(this.host, function (err, pod) { - if (err) return next(err) - - if (pod) return next(new Error('Pod already exists.')) - - self.score = constants.FRIEND_SCORE.BASE - return next() - }) -}) - -PodSchema.pre('remove', function (next) { - // Remove the videos owned by this pod too - Video.listByHost(this.host, function (err, videos) { - if (err) return next(err) - - each(videos, function (video, callbackEach) { - video.remove(callbackEach) - }, next) - }) -}) - -const Pod = mongoose.model('Pod', PodSchema) +// TODO: max score -> constants.FRIENDS_SCORE.MAX +// TODO: validation +// PodSchema.path('host').validate(validator.isURL) +// PodSchema.path('publicKey').required(true) +// PodSchema.path('score').validate(function (value) { return !isNaN(value) }) // ------------------------------ METHODS ------------------------------ function toFormatedJSON () { const json = { - id: this._id, + id: this.id, host: this.host, score: this.score, - createdDate: this.createdDate + createdAt: this.createdAt } return json @@ -81,39 +64,76 @@ function toFormatedJSON () { // ------------------------------ Statics ------------------------------ +function associate (models) { + this.belongsToMany(models.Request, { + foreignKey: 'podId', + through: models.RequestToPod, + onDelete: 'CASCADE' + }) +} + function countAll (callback) { - return this.count(callback) + return this.count().asCallback(callback) } function incrementScores (ids, value, callback) { if (!callback) callback = function () {} - return this.update({ _id: { $in: ids } }, { $inc: { score: value } }, { multi: true }, callback) + + const update = { + score: this.sequelize.literal('score +' + value) + } + + const query = { + where: { + id: { + $in: ids + } + } + } + + return this.update(update, query).asCallback(callback) } function list (callback) { - return this.find(callback) + return this.findAll().asCallback(callback) } function listAllIds (callback) { - return this.find({}, { _id: 1 }, function (err, pods) { + const query = { + attributes: [ 'id' ] + } + + return this.findAll(query).asCallback(function (err, pods) { if (err) return callback(err) - return callback(null, map(pods, '_id')) + return callback(null, map(pods, 'id')) }) } function listBadPods (callback) { - return this.find({ score: 0 }, callback) + const query = { + where: { + score: { $lte: 0 } + } + } + + return this.findAll(query).asCallback(callback) } function load (id, callback) { - return this.findById(id, callback) + return this.findById(id).asCallback(callback) } function loadByHost (host, callback) { - return this.findOne({ host }, callback) + const query = { + where: { + host: host + } + } + + return this.findOne(query).asCallback(callback) } function removeAll (callback) { - return this.remove({}, callback) + return this.destroy().asCallback(callback) } diff --git a/server/models/request.js b/server/models/request.js index c2cfe83ce..882f747b7 100644 --- a/server/models/request.js +++ b/server/models/request.js @@ -2,66 +2,58 @@ const each = require('async/each') const eachLimit = require('async/eachLimit') -const values = require('lodash/values') -const mongoose = require('mongoose') const waterfall = require('async/waterfall') const constants = require('../initializers/constants') const logger = require('../helpers/logger') const requests = require('../helpers/requests') -const Pod = mongoose.model('Pod') - let timer = null let lastRequestTimestamp = 0 // --------------------------------------------------------------------------- -const RequestSchema = mongoose.Schema({ - request: mongoose.Schema.Types.Mixed, - endpoint: { - type: String, - enum: [ values(constants.REQUEST_ENDPOINTS) ] - }, - to: [ +module.exports = function (sequelize, DataTypes) { + const Request = sequelize.define('Request', { - type: mongoose.Schema.Types.ObjectId, - ref: 'Pod' - } - ] -}) + request: { + type: DataTypes.JSON + }, + endpoint: { + // TODO: enum? + type: DataTypes.STRING + } + }, + { + classMethods: { + associate, -RequestSchema.statics = { - activate, - deactivate, - flush, - forceSend, - list, - remainingMilliSeconds + activate, + countTotalRequests, + deactivate, + flush, + forceSend, + remainingMilliSeconds + } + } + ) + + return Request } -RequestSchema.pre('save', function (next) { - const self = this - - if (self.to.length === 0) { - Pod.listAllIds(function (err, podIds) { - if (err) return next(err) - - // No friends - if (podIds.length === 0) return - - self.to = podIds - return next() - }) - } else { - return next() - } -}) - -mongoose.model('Request', RequestSchema) - // ------------------------------ STATICS ------------------------------ +function associate (models) { + this.belongsToMany(models.Pod, { + foreignKey: { + name: 'requestId', + allowNull: false + }, + through: models.RequestToPod, + onDelete: 'CASCADE' + }) +} + function activate () { logger.info('Requests scheduler activated.') lastRequestTimestamp = Date.now() @@ -73,6 +65,14 @@ function activate () { }, constants.REQUESTS_INTERVAL) } +function countTotalRequests (callback) { + const query = { + include: [ this.sequelize.models.Pod ] + } + + return this.count(query).asCallback(callback) +} + function deactivate () { logger.info('Requests scheduler deactivated.') clearInterval(timer) @@ -90,10 +90,6 @@ function forceSend () { makeRequests.call(this) } -function list (callback) { - this.find({ }, callback) -} - function remainingMilliSeconds () { if (timer === null) return -1 @@ -136,6 +132,7 @@ function makeRequest (toPod, requestEndpoint, requestsToMake, callback) { // Make all the requests of the scheduler function makeRequests () { const self = this + const RequestToPod = this.sequelize.models.RequestToPod // We limit the size of the requests (REQUESTS_LIMIT) // We don't want to stuck with the same failing requests so we get a random list @@ -156,20 +153,20 @@ function makeRequests () { // We want to group requests by destinations pod and endpoint const requestsToMakeGrouped = {} - requests.forEach(function (poolRequest) { - poolRequest.to.forEach(function (toPodId) { - const hashKey = toPodId + poolRequest.endpoint + requests.forEach(function (request) { + request.Pods.forEach(function (toPod) { + const hashKey = toPod.id + request.endpoint if (!requestsToMakeGrouped[hashKey]) { requestsToMakeGrouped[hashKey] = { - toPodId, - endpoint: poolRequest.endpoint, - ids: [], // pool request ids, to delete them from the DB in the future + toPodId: toPod.id, + endpoint: request.endpoint, + ids: [], // request ids, to delete them from the DB in the future datas: [] // requests data, } } - requestsToMakeGrouped[hashKey].ids.push(poolRequest._id) - requestsToMakeGrouped[hashKey].datas.push(poolRequest.request) + requestsToMakeGrouped[hashKey].ids.push(request.id) + requestsToMakeGrouped[hashKey].datas.push(request.request) }) }) @@ -179,8 +176,8 @@ function makeRequests () { eachLimit(Object.keys(requestsToMakeGrouped), constants.REQUESTS_IN_PARALLEL, function (hashKey, callbackEach) { const requestToMake = requestsToMakeGrouped[hashKey] - // FIXME: mongodb request inside a loop :/ - Pod.load(requestToMake.toPodId, function (err, toPod) { + // FIXME: SQL request inside a loop :/ + self.sequelize.models.Pod.load(requestToMake.toPodId, function (err, toPod) { if (err) { logger.error('Error finding pod by id.', { err: err }) return callbackEach() @@ -191,7 +188,7 @@ function makeRequests () { const requestIdsToDelete = requestToMake.ids logger.info('Removing %d requests of unexisting pod %s.', requestIdsToDelete.length, requestToMake.toPodId) - removePodOf.call(self, requestIdsToDelete, requestToMake.toPodId) + RequestToPod.removePodOf.call(self, requestIdsToDelete, requestToMake.toPodId) return callbackEach() } @@ -202,7 +199,7 @@ function makeRequests () { goodPods.push(requestToMake.toPodId) // Remove the pod id of these request ids - removePodOf.call(self, requestToMake.ids, requestToMake.toPodId, callbackEach) + RequestToPod.removePodOf(requestToMake.ids, requestToMake.toPodId, callbackEach) } else { badPods.push(requestToMake.toPodId) callbackEach() @@ -211,18 +208,22 @@ function makeRequests () { }) }, function () { // All the requests were made, we update the pods score - updatePodsScore(goodPods, badPods) + updatePodsScore.call(self, goodPods, badPods) // Flush requests with no pod - removeWithEmptyTo.call(self) + removeWithEmptyTo.call(self, function (err) { + if (err) logger.error('Error when removing requests with no pods.', { error: err }) + }) }) }) } // Remove pods with a score of 0 (too many requests where they were unreachable) function removeBadPods () { + const self = this + waterfall([ function findBadPods (callback) { - Pod.listBadPods(function (err, pods) { + self.sequelize.models.Pod.listBadPods(function (err, pods) { if (err) { logger.error('Cannot find bad pods.', { error: err }) return callback(err) @@ -233,10 +234,8 @@ function removeBadPods () { }, function removeTheseBadPods (pods, callback) { - if (pods.length === 0) return callback(null, 0) - each(pods, function (pod, callbackEach) { - pod.remove(callbackEach) + pod.destroy().asCallback(callbackEach) }, function (err) { return callback(err, pods.length) }) @@ -253,43 +252,67 @@ function removeBadPods () { } function updatePodsScore (goodPods, badPods) { + const self = this + const Pod = this.sequelize.models.Pod + logger.info('Updating %d good pods and %d bad pods scores.', goodPods.length, badPods.length) - Pod.incrementScores(goodPods, constants.PODS_SCORE.BONUS, function (err) { - if (err) logger.error('Cannot increment scores of good pods.') - }) + if (goodPods.length !== 0) { + Pod.incrementScores(goodPods, constants.PODS_SCORE.BONUS, function (err) { + if (err) logger.error('Cannot increment scores of good pods.') + }) + } - Pod.incrementScores(badPods, constants.PODS_SCORE.MALUS, function (err) { - if (err) logger.error('Cannot decrement scores of bad pods.') - removeBadPods() - }) + if (badPods.length !== 0) { + Pod.incrementScores(badPods, constants.PODS_SCORE.MALUS, function (err) { + if (err) logger.error('Cannot decrement scores of bad pods.') + removeBadPods.call(self) + }) + } } function listWithLimitAndRandom (limit, callback) { const self = this - self.count(function (err, count) { + self.count().asCallback(function (err, count) { if (err) return callback(err) + // Optimization... + if (count === 0) return callback(null, []) + let start = Math.floor(Math.random() * count) - limit if (start < 0) start = 0 - self.find().sort({ _id: 1 }).skip(start).limit(limit).exec(callback) + const query = { + order: [ + [ 'id', 'ASC' ] + ], + offset: start, + limit: limit, + include: [ this.sequelize.models.Pod ] + } + + self.findAll(query).asCallback(callback) }) } function removeAll (callback) { - this.remove({ }, callback) -} - -function removePodOf (requestsIds, podId, callback) { - if (!callback) callback = function () {} - - this.update({ _id: { $in: requestsIds } }, { $pull: { to: podId } }, { multi: true }, callback) + // Delete all requests + this.destroy({ truncate: true }).asCallback(callback) } function removeWithEmptyTo (callback) { if (!callback) callback = function () {} - this.remove({ to: { $size: 0 } }, callback) + const query = { + where: { + id: { + $notIn: [ + this.sequelize.literal('SELECT "requestId" FROM "RequestToPods"') + ] + } + } + } + + this.destroy(query).asCallback(callback) } diff --git a/server/models/requestToPod.js b/server/models/requestToPod.js new file mode 100644 index 000000000..378c2bdcf --- /dev/null +++ b/server/models/requestToPod.js @@ -0,0 +1,30 @@ +'use strict' + +// --------------------------------------------------------------------------- + +module.exports = function (sequelize, DataTypes) { + const RequestToPod = sequelize.define('RequestToPod', {}, { + classMethods: { + removePodOf + } + }) + + return RequestToPod +} + +// --------------------------------------------------------------------------- + +function removePodOf (requestsIds, podId, callback) { + if (!callback) callback = function () {} + + const query = { + where: { + requestId: { + $in: requestsIds + }, + podId: podId + } + } + + this.destroy(query).asCallback(callback) +} diff --git a/server/models/user.js b/server/models/user.js index a19de7072..e50eb96ea 100644 --- a/server/models/user.js +++ b/server/models/user.js @@ -1,60 +1,60 @@ -const mongoose = require('mongoose') - -const customUsersValidators = require('../helpers/custom-validators').users const modelUtils = require('./utils') const peertubeCrypto = require('../helpers/peertube-crypto') -const OAuthToken = mongoose.model('OAuthToken') - // --------------------------------------------------------------------------- -const UserSchema = mongoose.Schema({ - createdDate: { - type: Date, - default: Date.now - }, - password: String, - username: String, - role: String -}) +module.exports = function (sequelize, DataTypes) { + const User = sequelize.define('User', + { + password: { + type: DataTypes.STRING + }, + username: { + type: DataTypes.STRING + }, + role: { + type: DataTypes.STRING + } + }, + { + classMethods: { + associate, -UserSchema.path('password').required(customUsersValidators.isUserPasswordValid) -UserSchema.path('username').required(customUsersValidators.isUserUsernameValid) -UserSchema.path('role').validate(customUsersValidators.isUserRoleValid) + countTotal, + getByUsername, + list, + listForApi, + loadById, + loadByUsername + }, + instanceMethods: { + isPasswordMatch, + toFormatedJSON + }, + hooks: { + beforeCreate: beforeCreateOrUpdate, + beforeUpdate: beforeCreateOrUpdate + } + } + ) -UserSchema.methods = { - isPasswordMatch, - toFormatedJSON + return User } -UserSchema.statics = { - countTotal, - getByUsername, - list, - listForApi, - loadById, - loadByUsername -} +// TODO: Validation +// UserSchema.path('password').required(customUsersValidators.isUserPasswordValid) +// UserSchema.path('username').required(customUsersValidators.isUserUsernameValid) +// UserSchema.path('role').validate(customUsersValidators.isUserRoleValid) -UserSchema.pre('save', function (next) { - const user = this - - peertubeCrypto.cryptPassword(this.password, function (err, hash) { +function beforeCreateOrUpdate (user, options, next) { + peertubeCrypto.cryptPassword(user.password, function (err, hash) { if (err) return next(err) user.password = hash return next() }) -}) - -UserSchema.pre('remove', function (next) { - const user = this - - OAuthToken.removeByUserId(user._id, next) -}) - -mongoose.model('User', UserSchema) +} // ------------------------------ METHODS ------------------------------ @@ -64,35 +64,63 @@ function isPasswordMatch (password, callback) { function toFormatedJSON () { return { - id: this._id, + id: this.id, username: this.username, role: this.role, - createdDate: this.createdDate + createdAt: this.createdAt } } // ------------------------------ STATICS ------------------------------ +function associate (models) { + this.hasMany(models.OAuthToken, { + foreignKey: 'userId', + onDelete: 'cascade' + }) +} + function countTotal (callback) { - return this.count(callback) + return this.count().asCallback(callback) } function getByUsername (username) { - return this.findOne({ username: username }) + const query = { + where: { + username: username + } + } + + return this.findOne(query) } function list (callback) { - return this.find(callback) + return this.find().asCallback(callback) } function listForApi (start, count, sort, callback) { - const query = {} - return modelUtils.listForApiWithCount.call(this, query, start, count, sort, callback) + const query = { + offset: start, + limit: count, + order: [ modelUtils.getSort(sort) ] + } + + return this.findAndCountAll(query).asCallback(function (err, result) { + if (err) return callback(err) + + return callback(null, result.rows, result.count) + }) } function loadById (id, callback) { - return this.findById(id, callback) + return this.findById(id).asCallback(callback) } function loadByUsername (username, callback) { - return this.findOne({ username: username }, callback) + const query = { + where: { + username: username + } + } + + return this.findOne(query).asCallback(callback) } diff --git a/server/models/utils.js b/server/models/utils.js index e798aabe6..49636b3d8 100644 --- a/server/models/utils.js +++ b/server/models/utils.js @@ -1,28 +1,23 @@ 'use strict' -const parallel = require('async/parallel') - const utils = { - listForApiWithCount + getSort } -function listForApiWithCount (query, start, count, sort, callback) { - const self = this +// Translate for example "-name" to [ 'name', 'DESC' ] +function getSort (value) { + let field + let direction - parallel([ - function (asyncCallback) { - self.find(query).skip(start).limit(count).sort(sort).exec(asyncCallback) - }, - function (asyncCallback) { - self.count(query, asyncCallback) - } - ], function (err, results) { - if (err) return callback(err) + if (value.substring(0, 1) === '-') { + direction = 'DESC' + field = value.substring(1) + } else { + direction = 'ASC' + field = value + } - const data = results[0] - const total = results[1] - return callback(null, data, total) - }) + return [ field, direction ] } // --------------------------------------------------------------------------- diff --git a/server/models/video.js b/server/models/video.js index 330067cdf..8ef07c9e6 100644 --- a/server/models/video.js +++ b/server/models/video.js @@ -7,70 +7,134 @@ const magnetUtil = require('magnet-uri') const parallel = require('async/parallel') const parseTorrent = require('parse-torrent') const pathUtils = require('path') -const mongoose = require('mongoose') const constants = require('../initializers/constants') -const customVideosValidators = require('../helpers/custom-validators').videos const logger = require('../helpers/logger') const modelUtils = require('./utils') // --------------------------------------------------------------------------- +module.exports = function (sequelize, DataTypes) { // TODO: add indexes on searchable columns -const VideoSchema = mongoose.Schema({ - name: String, - extname: { - type: String, - enum: [ '.mp4', '.webm', '.ogv' ] - }, - remoteId: mongoose.Schema.Types.ObjectId, - description: String, - magnet: { - infoHash: String - }, - podHost: String, - author: String, - duration: Number, - tags: [ String ], - createdDate: { - type: Date, - default: Date.now + const Video = sequelize.define('Video', + { + id: { + type: DataTypes.UUID, + defaultValue: DataTypes.UUIDV4, + primaryKey: true + }, + name: { + type: DataTypes.STRING + }, + extname: { + // TODO: enum? + type: DataTypes.STRING + }, + remoteId: { + type: DataTypes.UUID + }, + description: { + type: DataTypes.STRING + }, + infoHash: { + type: DataTypes.STRING + }, + duration: { + type: DataTypes.INTEGER + }, + tags: { + type: DataTypes.ARRAY(DataTypes.STRING) + } + }, + { + classMethods: { + associate, + + generateThumbnailFromBase64, + getDurationFromFile, + listForApi, + listByHostAndRemoteId, + listOwnedAndPopulateAuthor, + listOwnedByAuthor, + load, + loadAndPopulateAuthor, + loadAndPopulateAuthorAndPod, + searchAndPopulateAuthorAndPod + }, + instanceMethods: { + generateMagnetUri, + getVideoFilename, + getThumbnailName, + getPreviewName, + getTorrentName, + isOwned, + toFormatedJSON, + toRemoteJSON + }, + hooks: { + beforeCreate, + afterDestroy + } + } + ) + + return Video +} + +// TODO: Validation +// VideoSchema.path('name').validate(customVideosValidators.isVideoNameValid) +// VideoSchema.path('description').validate(customVideosValidators.isVideoDescriptionValid) +// VideoSchema.path('podHost').validate(customVideosValidators.isVideoPodHostValid) +// VideoSchema.path('author').validate(customVideosValidators.isVideoAuthorValid) +// VideoSchema.path('duration').validate(customVideosValidators.isVideoDurationValid) +// VideoSchema.path('tags').validate(customVideosValidators.isVideoTagsValid) + +function beforeCreate (video, options, next) { + const tasks = [] + + if (video.isOwned()) { + const videoPath = pathUtils.join(constants.CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename()) + + tasks.push( + // TODO: refractoring + function (callback) { + const options = { + announceList: [ + [ constants.CONFIG.WEBSERVER.WS + '://' + constants.CONFIG.WEBSERVER.HOSTNAME + ':' + constants.CONFIG.WEBSERVER.PORT + '/tracker/socket' ] + ], + urlList: [ + constants.CONFIG.WEBSERVER.URL + constants.STATIC_PATHS.WEBSEED + video.getVideoFilename() + ] + } + + createTorrent(videoPath, options, function (err, torrent) { + if (err) return callback(err) + + fs.writeFile(constants.CONFIG.STORAGE.TORRENTS_DIR + video.getTorrentName(), torrent, function (err) { + if (err) return callback(err) + + const parsedTorrent = parseTorrent(torrent) + video.infoHash = parsedTorrent.infoHash + + callback(null) + }) + }) + }, + function (callback) { + createThumbnail(video, videoPath, callback) + }, + function (callback) { + createPreview(video, videoPath, callback) + } + ) + + return parallel(tasks, next) } -}) -VideoSchema.path('name').validate(customVideosValidators.isVideoNameValid) -VideoSchema.path('description').validate(customVideosValidators.isVideoDescriptionValid) -VideoSchema.path('podHost').validate(customVideosValidators.isVideoPodHostValid) -VideoSchema.path('author').validate(customVideosValidators.isVideoAuthorValid) -VideoSchema.path('duration').validate(customVideosValidators.isVideoDurationValid) -VideoSchema.path('tags').validate(customVideosValidators.isVideoTagsValid) - -VideoSchema.methods = { - generateMagnetUri, - getVideoFilename, - getThumbnailName, - getPreviewName, - getTorrentName, - isOwned, - toFormatedJSON, - toRemoteJSON + return next() } -VideoSchema.statics = { - generateThumbnailFromBase64, - getDurationFromFile, - listForApi, - listByHostAndRemoteId, - listByHost, - listOwned, - listOwnedByAuthor, - listRemotes, - load, - search -} - -VideoSchema.pre('remove', function (next) { - const video = this +function afterDestroy (video, options, next) { const tasks = [] tasks.push( @@ -94,59 +158,20 @@ VideoSchema.pre('remove', function (next) { } parallel(tasks, next) -}) - -VideoSchema.pre('save', function (next) { - const video = this - const tasks = [] - - if (video.isOwned()) { - const videoPath = pathUtils.join(constants.CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename()) - this.podHost = constants.CONFIG.WEBSERVER.HOST - - tasks.push( - // TODO: refractoring - function (callback) { - const options = { - announceList: [ - [ constants.CONFIG.WEBSERVER.WS + '://' + constants.CONFIG.WEBSERVER.HOSTNAME + ':' + constants.CONFIG.WEBSERVER.PORT + '/tracker/socket' ] - ], - urlList: [ - constants.CONFIG.WEBSERVER.URL + constants.STATIC_PATHS.WEBSEED + video.getVideoFilename() - ] - } - - createTorrent(videoPath, options, function (err, torrent) { - if (err) return callback(err) - - fs.writeFile(constants.CONFIG.STORAGE.TORRENTS_DIR + video.getTorrentName(), torrent, function (err) { - if (err) return callback(err) - - const parsedTorrent = parseTorrent(torrent) - video.magnet.infoHash = parsedTorrent.infoHash - - callback(null) - }) - }) - }, - function (callback) { - createThumbnail(video, videoPath, callback) - }, - function (callback) { - createPreview(video, videoPath, callback) - } - ) - - return parallel(tasks, next) - } - - return next() -}) - -mongoose.model('Video', VideoSchema) +} // ------------------------------ METHODS ------------------------------ +function associate (models) { + this.belongsTo(models.Author, { + foreignKey: { + name: 'authorId', + allowNull: false + }, + onDelete: 'cascade' + }) +} + function generateMagnetUri () { let baseUrlHttp, baseUrlWs @@ -154,8 +179,8 @@ function generateMagnetUri () { baseUrlHttp = constants.CONFIG.WEBSERVER.URL baseUrlWs = constants.CONFIG.WEBSERVER.WS + '://' + constants.CONFIG.WEBSERVER.HOSTNAME + ':' + constants.CONFIG.WEBSERVER.PORT } else { - baseUrlHttp = constants.REMOTE_SCHEME.HTTP + '://' + this.podHost - baseUrlWs = constants.REMOTE_SCHEME.WS + '://' + this.podHost + baseUrlHttp = constants.REMOTE_SCHEME.HTTP + '://' + this.Author.Pod.host + baseUrlWs = constants.REMOTE_SCHEME.WS + '://' + this.Author.Pod.host } const xs = baseUrlHttp + constants.STATIC_PATHS.TORRENTS + this.getTorrentName() @@ -166,7 +191,7 @@ function generateMagnetUri () { xs, announce, urlList, - infoHash: this.magnet.infoHash, + infoHash: this.infoHash, name: this.name } @@ -174,20 +199,20 @@ function generateMagnetUri () { } function getVideoFilename () { - if (this.isOwned()) return this._id + this.extname + if (this.isOwned()) return this.id + this.extname return this.remoteId + this.extname } function getThumbnailName () { // We always have a copy of the thumbnail - return this._id + '.jpg' + return this.id + '.jpg' } function getPreviewName () { const extension = '.jpg' - if (this.isOwned()) return this._id + extension + if (this.isOwned()) return this.id + extension return this.remoteId + extension } @@ -195,7 +220,7 @@ function getPreviewName () { function getTorrentName () { const extension = '.torrent' - if (this.isOwned()) return this._id + extension + if (this.isOwned()) return this.id + extension return this.remoteId + extension } @@ -205,18 +230,27 @@ function isOwned () { } function toFormatedJSON () { + let podHost + + if (this.Author.Pod) { + podHost = this.Author.Pod.host + } else { + // It means it's our video + podHost = constants.CONFIG.WEBSERVER.HOST + } + const json = { - id: this._id, + id: this.id, name: this.name, description: this.description, - podHost: this.podHost, + podHost, isLocal: this.isOwned(), magnetUri: this.generateMagnetUri(), - author: this.author, + author: this.Author.name, duration: this.duration, tags: this.tags, thumbnailPath: constants.STATIC_PATHS.THUMBNAILS + '/' + this.getThumbnailName(), - createdDate: this.createdDate + createdAt: this.createdAt } return json @@ -236,13 +270,13 @@ function toRemoteJSON (callback) { const remoteVideo = { name: self.name, description: self.description, - magnet: self.magnet, - remoteId: self._id, - author: self.author, + infoHash: self.infoHash, + remoteId: self.id, + author: self.Author.name, duration: self.duration, thumbnailBase64: new Buffer(thumbnailData).toString('base64'), tags: self.tags, - createdDate: self.createdDate, + createdAt: self.createdAt, extname: self.extname } @@ -273,50 +307,168 @@ function getDurationFromFile (videoPath, callback) { } function listForApi (start, count, sort, callback) { - const query = {} - return modelUtils.listForApiWithCount.call(this, query, start, count, sort, callback) + const query = { + offset: start, + limit: count, + order: [ modelUtils.getSort(sort) ], + include: [ + { + model: this.sequelize.models.Author, + include: [ this.sequelize.models.Pod ] + } + ] + } + + return this.findAndCountAll(query).asCallback(function (err, result) { + if (err) return callback(err) + + return callback(null, result.rows, result.count) + }) } function listByHostAndRemoteId (fromHost, remoteId, callback) { - this.find({ podHost: fromHost, remoteId: remoteId }, callback) + const query = { + where: { + remoteId: remoteId + }, + include: [ + { + model: this.sequelize.models.Author, + include: [ + { + model: this.sequelize.models.Pod, + where: { + host: fromHost + } + } + ] + } + ] + } + + return this.findAll(query).asCallback(callback) } -function listByHost (fromHost, callback) { - this.find({ podHost: fromHost }, callback) -} - -function listOwned (callback) { +function listOwnedAndPopulateAuthor (callback) { // If remoteId is null this is *our* video - this.find({ remoteId: null }, callback) + const query = { + where: { + remoteId: null + }, + include: [ this.sequelize.models.Author ] + } + + return this.findAll(query).asCallback(callback) } function listOwnedByAuthor (author, callback) { - this.find({ remoteId: null, author: author }, callback) -} + const query = { + where: { + remoteId: null + }, + include: [ + { + model: this.sequelize.models.Author, + where: { + name: author + } + } + ] + } -function listRemotes (callback) { - this.find({ remoteId: { $ne: null } }, callback) + return this.findAll(query).asCallback(callback) } function load (id, callback) { - this.findById(id, callback) + return this.findById(id).asCallback(callback) } -function search (value, field, start, count, sort, callback) { - const query = {} +function loadAndPopulateAuthor (id, callback) { + const options = { + include: [ this.sequelize.models.Author ] + } + + return this.findById(id, options).asCallback(callback) +} + +function loadAndPopulateAuthorAndPod (id, callback) { + const options = { + include: [ + { + model: this.sequelize.models.Author, + include: [ this.sequelize.models.Pod ] + } + ] + } + + return this.findById(id, options).asCallback(callback) +} + +function searchAndPopulateAuthorAndPod (value, field, start, count, sort, callback) { + const podInclude = { + model: this.sequelize.models.Pod + } + const authorInclude = { + model: this.sequelize.models.Author, + include: [ + podInclude + ] + } + + const query = { + where: {}, + include: [ + authorInclude + ], + offset: start, + limit: count, + order: [ modelUtils.getSort(sort) ] + } + + // TODO: include our pod for podHost searches (we are not stored in the database) // Make an exact search with the magnet if (field === 'magnetUri') { const infoHash = magnetUtil.decode(value).infoHash - query.magnet = { - infoHash - } + query.where.infoHash = infoHash } else if (field === 'tags') { - query[field] = value + query.where[field] = value + } else if (field === 'host') { + const whereQuery = { + '$Author.Pod.host$': { + $like: '%' + value + '%' + } + } + + // Include our pod? (not stored in the database) + if (constants.CONFIG.WEBSERVER.HOST.indexOf(value) !== -1) { + query.where = { + $or: [ + whereQuery, + { + remoteId: null + } + ] + } + } else { + query.where = whereQuery + } + } else if (field === 'author') { + query.where = { + '$Author.name$': { + $like: '%' + value + '%' + } + } } else { - query[field] = new RegExp(value, 'i') + query.where[field] = { + $like: '%' + value + '%' + } } - modelUtils.listForApiWithCount.call(this, query, start, count, sort, callback) + return this.findAndCountAll(query).asCallback(function (err, result) { + if (err) return callback(err) + + return callback(null, result.rows, result.count) + }) } // --------------------------------------------------------------------------- diff --git a/server/tests/api/check-params.js b/server/tests/api/check-params.js index 444c2fc55..d9e51770c 100644 --- a/server/tests/api/check-params.js +++ b/server/tests/api/check-params.js @@ -465,7 +465,7 @@ describe('Test parameters validator', function () { it('Should return 404 with an incorrect video', function (done) { request(server.url) - .get(path + '123456789012345678901234') + .get(path + '4da6fde3-88f7-4d16-b119-108df5630b06') .set('Accept', 'application/json') .expect(404, done) }) @@ -490,7 +490,7 @@ describe('Test parameters validator', function () { it('Should fail with a video which does not exist', function (done) { request(server.url) - .delete(path + '123456789012345678901234') + .delete(path + '4da6fde3-88f7-4d16-b119-108df5630b06') .set('Authorization', 'Bearer ' + server.accessToken) .expect(404, done) }) @@ -711,7 +711,7 @@ describe('Test parameters validator', function () { it('Should return 404 with a non existing id', function (done) { request(server.url) - .delete(path + '579f982228c99c221d8092b8') + .delete(path + '45') .set('Authorization', 'Bearer ' + server.accessToken) .expect(404, done) }) diff --git a/server/tests/api/friends-basic.js b/server/tests/api/friends-basic.js index a871f9838..3a904dbd7 100644 --- a/server/tests/api/friends-basic.js +++ b/server/tests/api/friends-basic.js @@ -97,7 +97,7 @@ describe('Test basic friends', function () { const pod = result[0] expect(pod.host).to.equal(servers[2].host) expect(pod.score).to.equal(20) - expect(miscsUtils.dateIsValid(pod.createdDate)).to.be.true + expect(miscsUtils.dateIsValid(pod.createdAt)).to.be.true next() }) @@ -114,7 +114,7 @@ describe('Test basic friends', function () { const pod = result[0] expect(pod.host).to.equal(servers[1].host) expect(pod.score).to.equal(20) - expect(miscsUtils.dateIsValid(pod.createdDate)).to.be.true + expect(miscsUtils.dateIsValid(pod.createdAt)).to.be.true next() }) diff --git a/server/tests/api/multiple-pods.js b/server/tests/api/multiple-pods.js index be278d7c5..f0fe59c5f 100644 --- a/server/tests/api/multiple-pods.js +++ b/server/tests/api/multiple-pods.js @@ -104,7 +104,7 @@ describe('Test multiple pods', function () { expect(video.magnetUri).to.exist expect(video.duration).to.equal(10) expect(video.tags).to.deep.equal([ 'tag1p1', 'tag2p1' ]) - expect(miscsUtils.dateIsValid(video.createdDate)).to.be.true + expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true expect(video.author).to.equal('root') if (server.url !== 'http://localhost:9001') { @@ -166,7 +166,7 @@ describe('Test multiple pods', function () { expect(video.magnetUri).to.exist expect(video.duration).to.equal(5) expect(video.tags).to.deep.equal([ 'tag1p2', 'tag2p2', 'tag3p2' ]) - expect(miscsUtils.dateIsValid(video.createdDate)).to.be.true + expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true expect(video.author).to.equal('root') if (server.url !== 'http://localhost:9002') { @@ -246,7 +246,7 @@ describe('Test multiple pods', function () { expect(video1.duration).to.equal(5) expect(video1.tags).to.deep.equal([ 'tag1p3' ]) expect(video1.author).to.equal('root') - expect(miscsUtils.dateIsValid(video1.createdDate)).to.be.true + expect(miscsUtils.dateIsValid(video1.createdAt)).to.be.true expect(video2.name).to.equal('my super name for pod 3-2') expect(video2.description).to.equal('my super description for pod 3-2') @@ -255,7 +255,7 @@ describe('Test multiple pods', function () { expect(video2.duration).to.equal(5) expect(video2.tags).to.deep.equal([ 'tag2p3', 'tag3p3', 'tag4p3' ]) expect(video2.author).to.equal('root') - expect(miscsUtils.dateIsValid(video2.createdDate)).to.be.true + expect(miscsUtils.dateIsValid(video2.createdAt)).to.be.true if (server.url !== 'http://localhost:9003') { expect(video1.isLocal).to.be.false diff --git a/server/tests/api/requests.js b/server/tests/api/requests.js index af36f6e34..7e790b54b 100644 --- a/server/tests/api/requests.js +++ b/server/tests/api/requests.js @@ -69,7 +69,7 @@ describe('Test requests stats', function () { }) }) - it('Should have the correct request', function (done) { + it('Should have the correct total request', function (done) { this.timeout(15000) const server = servers[0] @@ -83,11 +83,7 @@ describe('Test requests stats', function () { if (err) throw err const body = res.body - expect(body.requests).to.have.lengthOf(1) - - const request = body.requests[0] - expect(request.to).to.have.lengthOf(1) - expect(request.request.type).to.equal('add') + expect(body.totalRequests).to.equal(1) // Wait one cycle setTimeout(done, 10000) @@ -95,27 +91,6 @@ describe('Test requests stats', function () { }) }) - it('Should have the correct requests', function (done) { - const server = servers[0] - - uploadVideo(server, function (err) { - if (err) throw err - - getRequestsStats(server, function (err, res) { - if (err) throw err - - const body = res.body - expect(body.requests).to.have.lengthOf(2) - - const request = body.requests[1] - expect(request.to).to.have.lengthOf(1) - expect(request.request.type).to.equal('add') - - done() - }) - }) - }) - after(function (done) { process.kill(-servers[0].app.pid) diff --git a/server/tests/api/single-pod.js b/server/tests/api/single-pod.js index 65d1a7a65..aedecacf3 100644 --- a/server/tests/api/single-pod.js +++ b/server/tests/api/single-pod.js @@ -82,7 +82,7 @@ describe('Test a single pod', function () { expect(video.author).to.equal('root') expect(video.isLocal).to.be.true expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'tag3' ]) - expect(miscsUtils.dateIsValid(video.createdDate)).to.be.true + expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true videosUtils.testVideoImage(server.url, 'video_short.webm', video.thumbnailPath, function (err, test) { if (err) throw err @@ -116,7 +116,7 @@ describe('Test a single pod', function () { expect(video.author).to.equal('root') expect(video.isLocal).to.be.true expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'tag3' ]) - expect(miscsUtils.dateIsValid(video.createdDate)).to.be.true + expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true videosUtils.testVideoImage(server.url, 'video_short.webm', video.thumbnailPath, function (err, test) { if (err) throw err @@ -142,7 +142,7 @@ describe('Test a single pod', function () { expect(video.author).to.equal('root') expect(video.isLocal).to.be.true expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'tag3' ]) - expect(miscsUtils.dateIsValid(video.createdDate)).to.be.true + expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true videosUtils.testVideoImage(server.url, 'video_short.webm', video.thumbnailPath, function (err, test) { if (err) throw err @@ -154,7 +154,7 @@ describe('Test a single pod', function () { }) it('Should search the video by podHost', function (done) { - videosUtils.searchVideo(server.url, '9001', 'podHost', function (err, res) { + videosUtils.searchVideo(server.url, '9001', 'host', function (err, res) { if (err) throw err expect(res.body.total).to.equal(1) @@ -168,7 +168,7 @@ describe('Test a single pod', function () { expect(video.author).to.equal('root') expect(video.isLocal).to.be.true expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'tag3' ]) - expect(miscsUtils.dateIsValid(video.createdDate)).to.be.true + expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true videosUtils.testVideoImage(server.url, 'video_short.webm', video.thumbnailPath, function (err, test) { if (err) throw err @@ -194,7 +194,7 @@ describe('Test a single pod', function () { expect(video.author).to.equal('root') expect(video.isLocal).to.be.true expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'tag3' ]) - expect(miscsUtils.dateIsValid(video.createdDate)).to.be.true + expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true videosUtils.testVideoImage(server.url, 'video_short.webm', video.thumbnailPath, function (err, test) { if (err) throw err @@ -425,7 +425,7 @@ describe('Test a single pod', function () { }) it('Should search all the 9001 port videos', function (done) { - videosUtils.searchVideoWithPagination(server.url, '9001', 'podHost', 0, 15, function (err, res) { + videosUtils.searchVideoWithPagination(server.url, '9001', 'host', 0, 15, function (err, res) { if (err) throw err const videos = res.body.data @@ -437,7 +437,7 @@ describe('Test a single pod', function () { }) it('Should search all the localhost videos', function (done) { - videosUtils.searchVideoWithPagination(server.url, 'localhost', 'podHost', 0, 15, function (err, res) { + videosUtils.searchVideoWithPagination(server.url, 'localhost', 'host', 0, 15, function (err, res) { if (err) throw err const videos = res.body.data diff --git a/server/tests/api/users.js b/server/tests/api/users.js index 94267f104..e6d937eb0 100644 --- a/server/tests/api/users.js +++ b/server/tests/api/users.js @@ -261,8 +261,8 @@ describe('Test users', function () { }) }) - it('Should list only the second user by createdDate desc', function (done) { - usersUtils.getUsersListPaginationAndSort(server.url, 0, 1, '-createdDate', function (err, res) { + it('Should list only the second user by createdAt desc', function (done) { + usersUtils.getUsersListPaginationAndSort(server.url, 0, 1, '-createdAt', function (err, res) { if (err) throw err const result = res.body @@ -279,8 +279,8 @@ describe('Test users', function () { }) }) - it('Should list all the users by createdDate asc', function (done) { - usersUtils.getUsersListPaginationAndSort(server.url, 0, 2, 'createdDate', function (err, res) { + it('Should list all the users by createdAt asc', function (done) { + usersUtils.getUsersListPaginationAndSort(server.url, 0, 2, 'createdAt', function (err, res) { if (err) throw err const result = res.body diff --git a/server/tests/utils/servers.js b/server/tests/utils/servers.js index 01c9a2f39..4e55f8f5c 100644 --- a/server/tests/utils/servers.js +++ b/server/tests/utils/servers.js @@ -60,12 +60,12 @@ function runServer (number, callback) { // These actions are async so we need to be sure that they have both been done const serverRunString = { - 'Connected to mongodb': false, + 'Database is ready': false, 'Server listening on port': false } const regexps = { - client_id: 'Client id: ([a-f0-9]+)', + client_id: 'Client id: (.+)', client_secret: 'Client secret: (.+)', user_username: 'Username: (.+)', user_password: 'User password: (.+)' diff --git a/server/tests/utils/videos.js b/server/tests/utils/videos.js index 536093db1..5c120597f 100644 --- a/server/tests/utils/videos.js +++ b/server/tests/utils/videos.js @@ -25,7 +25,7 @@ function getAllVideosListBy (url, end) { request(url) .get(path) - .query({ sort: 'createdDate' }) + .query({ sort: 'createdAt' }) .query({ start: 0 }) .query({ count: 10000 }) .set('Accept', 'application/json') From 3897209f46f4c4581be2b8963bf9acc28ca5032b Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 19 Dec 2016 21:50:20 +0100 Subject: [PATCH 02/47] Server: rename Pods -> Pod --- server/controllers/api/remote.js | 2 -- server/models/{pods.js => pod.js} | 0 2 files changed, 2 deletions(-) rename server/models/{pods.js => pod.js} (100%) diff --git a/server/controllers/api/remote.js b/server/controllers/api/remote.js index d856576a9..a6753a2b0 100644 --- a/server/controllers/api/remote.js +++ b/server/controllers/api/remote.js @@ -56,8 +56,6 @@ function addRemoteVideo (videoToCreateData, fromHost, callback) { waterfall([ function findOrCreatePod (callback) { - fromHost - const query = { where: { host: fromHost diff --git a/server/models/pods.js b/server/models/pod.js similarity index 100% rename from server/models/pods.js rename to server/models/pod.js From 7920c273a204e2469416a30b752b12ccd3160102 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Sat, 24 Dec 2016 16:59:17 +0100 Subject: [PATCH 03/47] Move tags in another table --- server/controllers/api/remote.js | 95 +++++++++++++++++++++++----- server/controllers/api/videos.js | 99 +++++++++++++++++++++++------ server/controllers/client.js | 2 +- server/initializers/database.js | 13 +++- server/lib/friends.js | 10 +-- server/models/pod.js | 3 +- server/models/request.js | 6 +- server/models/tag.js | 30 +++++++++ server/models/video.js | 103 ++++++++++++++++++------------- server/models/videoTag.js | 9 +++ server/tests/api/check-params.js | 4 +- server/tests/api/requests.js | 15 ++--- server/tests/api/single-pod.js | 82 ++++++++++++------------ 13 files changed, 333 insertions(+), 138 deletions(-) create mode 100644 server/models/tag.js create mode 100644 server/models/videoTag.js diff --git a/server/controllers/api/remote.js b/server/controllers/api/remote.js index a6753a2b0..c7a5658e8 100644 --- a/server/controllers/api/remote.js +++ b/server/controllers/api/remote.js @@ -50,28 +50,35 @@ function remoteVideos (req, res, next) { return res.type('json').status(204).end() } -function addRemoteVideo (videoToCreateData, fromHost, callback) { +function addRemoteVideo (videoToCreateData, fromHost, finalCallback) { logger.debug('Adding remote video "%s".', videoToCreateData.name) waterfall([ - function findOrCreatePod (callback) { + function startTransaction (callback) { + db.sequelize.transaction().asCallback(function (err, t) { + return callback(err, t) + }) + }, + + function findOrCreatePod (t, callback) { const query = { where: { host: fromHost }, defaults: { host: fromHost - } + }, + transaction: t } db.Pod.findOrCreate(query).asCallback(function (err, result) { // [ instance, wasCreated ] - return callback(err, result[0]) + return callback(err, t, result[0]) }) }, - function findOrCreateAuthor (pod, callback) { + function findOrCreateAuthor (t, pod, callback) { const username = videoToCreateData.author const query = { @@ -82,16 +89,45 @@ function addRemoteVideo (videoToCreateData, fromHost, callback) { defaults: { name: username, podId: pod.id - } + }, + transaction: t } db.Author.findOrCreate(query).asCallback(function (err, result) { // [ instance, wasCreated ] - return callback(err, result[0]) + return callback(err, t, result[0]) }) }, - function createVideoObject (author, callback) { + function findOrCreateTags (t, author, callback) { + const tags = videoToCreateData.tags + const tagInstances = [] + + each(tags, function (tag, callbackEach) { + const query = { + where: { + name: tag + }, + defaults: { + name: tag + }, + transaction: t + } + + db.Tag.findOrCreate(query).asCallback(function (err, res) { + if (err) return callbackEach(err) + + // res = [ tag, isCreated ] + const tag = res[0] + tagInstances.push(tag) + return callbackEach() + }) + }, function (err) { + return callback(err, t, author, tagInstances) + }) + }, + + function createVideoObject (t, author, tagInstances, callback) { const videoData = { name: videoToCreateData.name, remoteId: videoToCreateData.remoteId, @@ -99,31 +135,58 @@ function addRemoteVideo (videoToCreateData, fromHost, callback) { infoHash: videoToCreateData.infoHash, description: videoToCreateData.description, authorId: author.id, - duration: videoToCreateData.duration, - tags: videoToCreateData.tags + duration: videoToCreateData.duration } const video = db.Video.build(videoData) - return callback(null, video) + return callback(null, t, tagInstances, video) }, - function generateThumbnail (video, callback) { + function generateThumbnail (t, tagInstances, video, callback) { db.Video.generateThumbnailFromBase64(video, videoToCreateData.thumbnailBase64, function (err) { if (err) { logger.error('Cannot generate thumbnail from base 64 data.', { error: err }) return callback(err) } - video.save().asCallback(callback) + return callback(err, t, tagInstances, video) }) }, - function insertIntoDB (video, callback) { - video.save().asCallback(callback) + function insertVideoIntoDB (t, tagInstances, video, callback) { + const options = { + transaction: t + } + + video.save(options).asCallback(function (err, videoCreated) { + return callback(err, t, tagInstances, videoCreated) + }) + }, + + function associateTagsToVideo (t, tagInstances, video, callback) { + const options = { transaction: t } + + video.setTags(tagInstances, options).asCallback(function (err) { + return callback(err, t) + }) } - ], callback) + ], function (err, t) { + if (err) { + logger.error('Cannot insert the remote video.') + + // Abort transaction? + if (t) t.rollback() + + return finalCallback(err) + } + + // Commit transaction + t.commit() + + return finalCallback() + }) } function removeRemoteVideo (videoToRemoveData, fromHost, callback) { diff --git a/server/controllers/api/videos.js b/server/controllers/api/videos.js index a61f2b2c9..992f03db0 100644 --- a/server/controllers/api/videos.js +++ b/server/controllers/api/videos.js @@ -1,5 +1,6 @@ 'use strict' +const each = require('async/each') const express = require('express') const fs = require('fs') const multer = require('multer') @@ -87,7 +88,13 @@ function addVideo (req, res, next) { waterfall([ - function findOrCreateAuthor (callback) { + function startTransaction (callback) { + db.sequelize.transaction().asCallback(function (err, t) { + return callback(err, t) + }) + }, + + function findOrCreateAuthor (t, callback) { const username = res.locals.oauth.token.user.username const query = { @@ -98,75 +105,125 @@ function addVideo (req, res, next) { defaults: { name: username, podId: null // null because it is OUR pod - } + }, + transaction: t } db.Author.findOrCreate(query).asCallback(function (err, result) { // [ instance, wasCreated ] - return callback(err, result[0]) + return callback(err, t, result[0]) }) }, - function createVideoObject (author, callback) { + function findOrCreateTags (t, author, callback) { + const tags = videoInfos.tags + const tagInstances = [] + + each(tags, function (tag, callbackEach) { + const query = { + where: { + name: tag + }, + defaults: { + name: tag + }, + transaction: t + } + + db.Tag.findOrCreate(query).asCallback(function (err, res) { + if (err) return callbackEach(err) + + // res = [ tag, isCreated ] + const tag = res[0] + tagInstances.push(tag) + return callbackEach() + }) + }, function (err) { + return callback(err, t, author, tagInstances) + }) + }, + + function createVideoObject (t, author, tagInstances, callback) { const videoData = { name: videoInfos.name, remoteId: null, extname: path.extname(videoFile.filename), description: videoInfos.description, duration: videoFile.duration, - tags: videoInfos.tags, authorId: author.id } const video = db.Video.build(videoData) - return callback(null, author, video) + return callback(null, t, author, tagInstances, video) }, // Set the videoname the same as the id - function renameVideoFile (author, video, callback) { + function renameVideoFile (t, author, tagInstances, video, callback) { const videoDir = constants.CONFIG.STORAGE.VIDEOS_DIR const source = path.join(videoDir, videoFile.filename) const destination = path.join(videoDir, video.getVideoFilename()) fs.rename(source, destination, function (err) { - return callback(err, author, video) + return callback(err, t, author, tagInstances, video) }) }, - function insertIntoDB (author, video, callback) { - video.save().asCallback(function (err, videoCreated) { + function insertVideoIntoDB (t, author, tagInstances, video, callback) { + const options = { transaction: t } + + // Add tags association + video.save(options).asCallback(function (err, videoCreated) { + if (err) return callback(err) + // Do not forget to add Author informations to the created video videoCreated.Author = author - return callback(err, videoCreated) + return callback(err, t, tagInstances, videoCreated) }) }, - function sendToFriends (video, callback) { + function associateTagsToVideo (t, tagInstances, video, callback) { + const options = { transaction: t } + + video.setTags(tagInstances, options).asCallback(function (err) { + video.Tags = tagInstances + + return callback(err, t, video) + }) + }, + + function sendToFriends (t, video, callback) { video.toRemoteJSON(function (err, remoteVideo) { if (err) return callback(err) // Now we'll add the video's meta data to our friends friends.addVideoToFriends(remoteVideo) - return callback(null) + return callback(null, t) }) } - ], function andFinally (err) { + ], function andFinally (err, t) { if (err) { logger.error('Cannot insert the video.') + + // Abort transaction? + if (t) t.rollback() + return next(err) } + // Commit transaction + t.commit() + // TODO : include Location of the new video -> 201 return res.type('json').status(204).end() }) } function getVideo (req, res, next) { - db.Video.loadAndPopulateAuthorAndPod(req.params.id, function (err, video) { + db.Video.loadAndPopulateAuthorAndPodAndTags(req.params.id, function (err, video) { if (err) return next(err) if (!video) { @@ -222,12 +279,14 @@ function removeVideo (req, res, next) { } function searchVideos (req, res, next) { - db.Video.searchAndPopulateAuthorAndPod(req.params.value, req.query.field, req.query.start, req.query.count, req.query.sort, - function (err, videosList, videosTotal) { - if (err) return next(err) + db.Video.searchAndPopulateAuthorAndPodAndTags( + req.params.value, req.query.field, req.query.start, req.query.count, req.query.sort, + function (err, videosList, videosTotal) { + if (err) return next(err) - res.json(getFormatedVideos(videosList, videosTotal)) - }) + res.json(getFormatedVideos(videosList, videosTotal)) + } + ) } // --------------------------------------------------------------------------- diff --git a/server/controllers/client.js b/server/controllers/client.js index a5fac5626..8c242af07 100644 --- a/server/controllers/client.js +++ b/server/controllers/client.js @@ -93,7 +93,7 @@ function generateWatchHtmlPage (req, res, next) { }, video: function (callback) { - db.Video.loadAndPopulateAuthorAndPod(videoId, callback) + db.Video.loadAndPopulateAuthorAndPodAndTags(videoId, callback) } }, function (err, results) { if (err) return next(err) diff --git a/server/initializers/database.js b/server/initializers/database.js index cc6f59b63..9642231b9 100644 --- a/server/initializers/database.js +++ b/server/initializers/database.js @@ -6,13 +6,24 @@ const Sequelize = require('sequelize') const constants = require('../initializers/constants') const logger = require('../helpers/logger') +const utils = require('../helpers/utils') const database = {} const sequelize = new Sequelize(constants.CONFIG.DATABASE.DBNAME, 'peertube', 'peertube', { dialect: 'postgres', host: constants.CONFIG.DATABASE.HOSTNAME, - port: constants.CONFIG.DATABASE.PORT + port: constants.CONFIG.DATABASE.PORT, + benchmark: utils.isTestInstance(), + + logging: function (message, benchmark) { + let newMessage = message + if (benchmark !== undefined) { + newMessage += ' | ' + benchmark + 'ms' + } + + logger.debug(newMessage) + } }) const modelDirectory = path.join(__dirname, '..', 'models') diff --git a/server/lib/friends.js b/server/lib/friends.js index 3ed29f651..ad9e4fdae 100644 --- a/server/lib/friends.js +++ b/server/lib/friends.js @@ -66,10 +66,12 @@ function makeFriends (hosts, callback) { function quitFriends (callback) { // Stop pool requests db.Request.deactivate() - // Flush pool requests - db.Request.flush() waterfall([ + function flushRequests (callbackAsync) { + db.Request.flush(callbackAsync) + }, + function getPodsList (callbackAsync) { return db.Pod.list(callbackAsync) }, @@ -118,7 +120,7 @@ function removeVideoToFriends (videoParams) { } function sendOwnedVideosToPod (podId) { - db.Video.listOwnedAndPopulateAuthor(function (err, videosList) { + db.Video.listOwnedAndPopulateAuthorAndTags(function (err, videosList) { if (err) { logger.error('Cannot get the list of videos we own.') return @@ -226,7 +228,7 @@ function makeRequestsToWinningPods (cert, podsList, callback) { } // Add our videos to the request scheduler - sendOwnedVideosToPod(podCreated._id) + sendOwnedVideosToPod(podCreated.id) return callbackEach() }) diff --git a/server/models/pod.js b/server/models/pod.js index 2c1f56203..fff6970a7 100644 --- a/server/models/pod.js +++ b/server/models/pod.js @@ -19,7 +19,6 @@ module.exports = function (sequelize, DataTypes) { type: DataTypes.INTEGER, defaultValue: constants.FRIEND_SCORE.BASE } - // Check createdAt }, { classMethods: { @@ -68,7 +67,7 @@ function associate (models) { this.belongsToMany(models.Request, { foreignKey: 'podId', through: models.RequestToPod, - onDelete: 'CASCADE' + onDelete: 'cascade' }) } diff --git a/server/models/request.js b/server/models/request.js index 882f747b7..70aa32610 100644 --- a/server/models/request.js +++ b/server/models/request.js @@ -79,9 +79,11 @@ function deactivate () { timer = null } -function flush () { +function flush (callback) { removeAll.call(this, function (err) { if (err) logger.error('Cannot flush the requests.', { error: err }) + + return callback(err) }) } @@ -298,7 +300,7 @@ function listWithLimitAndRandom (limit, callback) { function removeAll (callback) { // Delete all requests - this.destroy({ truncate: true }).asCallback(callback) + this.truncate({ cascade: true }).asCallback(callback) } function removeWithEmptyTo (callback) { diff --git a/server/models/tag.js b/server/models/tag.js new file mode 100644 index 000000000..874e88842 --- /dev/null +++ b/server/models/tag.js @@ -0,0 +1,30 @@ +'use strict' + +// --------------------------------------------------------------------------- + +module.exports = function (sequelize, DataTypes) { + const Tag = sequelize.define('Tag', + { + name: { + type: DataTypes.STRING + } + }, + { + classMethods: { + associate + } + } + ) + + return Tag +} + +// --------------------------------------------------------------------------- + +function associate (models) { + this.belongsToMany(models.Video, { + foreignKey: 'tagId', + through: models.VideoTag, + onDelete: 'cascade' + }) +} diff --git a/server/models/video.js b/server/models/video.js index 8ef07c9e6..0023a24e1 100644 --- a/server/models/video.js +++ b/server/models/video.js @@ -4,6 +4,7 @@ const createTorrent = require('create-torrent') const ffmpeg = require('fluent-ffmpeg') const fs = require('fs') const magnetUtil = require('magnet-uri') +const map = require('lodash/map') const parallel = require('async/parallel') const parseTorrent = require('parse-torrent') const pathUtils = require('path') @@ -41,9 +42,6 @@ module.exports = function (sequelize, DataTypes) { }, duration: { type: DataTypes.INTEGER - }, - tags: { - type: DataTypes.ARRAY(DataTypes.STRING) } }, { @@ -54,12 +52,12 @@ module.exports = function (sequelize, DataTypes) { getDurationFromFile, listForApi, listByHostAndRemoteId, - listOwnedAndPopulateAuthor, + listOwnedAndPopulateAuthorAndTags, listOwnedByAuthor, load, loadAndPopulateAuthor, - loadAndPopulateAuthorAndPod, - searchAndPopulateAuthorAndPod + loadAndPopulateAuthorAndPodAndTags, + searchAndPopulateAuthorAndPodAndTags }, instanceMethods: { generateMagnetUri, @@ -170,6 +168,12 @@ function associate (models) { }, onDelete: 'cascade' }) + + this.belongsToMany(models.Tag, { + foreignKey: 'videoId', + through: models.VideoTag, + onDelete: 'cascade' + }) } function generateMagnetUri () { @@ -248,7 +252,7 @@ function toFormatedJSON () { magnetUri: this.generateMagnetUri(), author: this.Author.name, duration: this.duration, - tags: this.tags, + tags: map(this.Tags, 'name'), thumbnailPath: constants.STATIC_PATHS.THUMBNAILS + '/' + this.getThumbnailName(), createdAt: this.createdAt } @@ -275,7 +279,7 @@ function toRemoteJSON (callback) { author: self.Author.name, duration: self.duration, thumbnailBase64: new Buffer(thumbnailData).toString('base64'), - tags: self.tags, + tags: map(self.Tags, 'name'), createdAt: self.createdAt, extname: self.extname } @@ -310,12 +314,15 @@ function listForApi (start, count, sort, callback) { const query = { offset: start, limit: count, + distinct: true, // For the count, a video can have many tags order: [ modelUtils.getSort(sort) ], include: [ { model: this.sequelize.models.Author, - include: [ this.sequelize.models.Pod ] - } + include: [ { model: this.sequelize.models.Pod, required: false } ] + }, + + this.sequelize.models.Tag ] } @@ -337,6 +344,7 @@ function listByHostAndRemoteId (fromHost, remoteId, callback) { include: [ { model: this.sequelize.models.Pod, + required: true, where: { host: fromHost } @@ -349,13 +357,13 @@ function listByHostAndRemoteId (fromHost, remoteId, callback) { return this.findAll(query).asCallback(callback) } -function listOwnedAndPopulateAuthor (callback) { +function listOwnedAndPopulateAuthorAndTags (callback) { // If remoteId is null this is *our* video const query = { where: { remoteId: null }, - include: [ this.sequelize.models.Author ] + include: [ this.sequelize.models.Author, this.sequelize.models.Tag ] } return this.findAll(query).asCallback(callback) @@ -391,23 +399,26 @@ function loadAndPopulateAuthor (id, callback) { return this.findById(id, options).asCallback(callback) } -function loadAndPopulateAuthorAndPod (id, callback) { +function loadAndPopulateAuthorAndPodAndTags (id, callback) { const options = { include: [ { model: this.sequelize.models.Author, - include: [ this.sequelize.models.Pod ] - } + include: [ { model: this.sequelize.models.Pod, required: false } ] + }, + this.sequelize.models.Tag ] } return this.findById(id, options).asCallback(callback) } -function searchAndPopulateAuthorAndPod (value, field, start, count, sort, callback) { +function searchAndPopulateAuthorAndPodAndTags (value, field, start, count, sort, callback) { const podInclude = { - model: this.sequelize.models.Pod + model: this.sequelize.models.Pod, + required: false } + const authorInclude = { model: this.sequelize.models.Author, include: [ @@ -415,55 +426,61 @@ function searchAndPopulateAuthorAndPod (value, field, start, count, sort, callba ] } + const tagInclude = { + model: this.sequelize.models.Tag + } + const query = { where: {}, - include: [ - authorInclude - ], offset: start, limit: count, + distinct: true, // For the count, a video can have many tags order: [ modelUtils.getSort(sort) ] } - // TODO: include our pod for podHost searches (we are not stored in the database) // Make an exact search with the magnet if (field === 'magnetUri') { const infoHash = magnetUtil.decode(value).infoHash query.where.infoHash = infoHash } else if (field === 'tags') { - query.where[field] = value + const escapedValue = this.sequelize.escape('%' + value + '%') + query.where = { + id: { + $in: this.sequelize.literal( + '(SELECT "VideoTags"."videoId" FROM "Tags" INNER JOIN "VideoTags" ON "Tags"."id" = "VideoTags"."tagId" WHERE name LIKE ' + escapedValue + ')' + ) + } + } } else if (field === 'host') { - const whereQuery = { - '$Author.Pod.host$': { + // FIXME: Include our pod? (not stored in the database) + podInclude.where = { + host: { + $like: '%' + value + '%' + } + } + podInclude.required = true + } else if (field === 'author') { + authorInclude.where = { + name: { $like: '%' + value + '%' } } - // Include our pod? (not stored in the database) - if (constants.CONFIG.WEBSERVER.HOST.indexOf(value) !== -1) { - query.where = { - $or: [ - whereQuery, - { - remoteId: null - } - ] - } - } else { - query.where = whereQuery - } - } else if (field === 'author') { - query.where = { - '$Author.name$': { - $like: '%' + value + '%' - } - } + // authorInclude.or = true } else { query.where[field] = { $like: '%' + value + '%' } } + query.include = [ + authorInclude, tagInclude + ] + + if (tagInclude.where) { + // query.include.push([ this.sequelize.models.Tag ]) + } + return this.findAndCountAll(query).asCallback(function (err, result) { if (err) return callback(err) diff --git a/server/models/videoTag.js b/server/models/videoTag.js new file mode 100644 index 000000000..0f2b20838 --- /dev/null +++ b/server/models/videoTag.js @@ -0,0 +1,9 @@ +'use strict' + +// --------------------------------------------------------------------------- + +module.exports = function (sequelize, DataTypes) { + const VideoTag = sequelize.define('VideoTag', {}, {}) + + return VideoTag +} diff --git a/server/tests/api/check-params.js b/server/tests/api/check-params.js index d9e51770c..9aecc3720 100644 --- a/server/tests/api/check-params.js +++ b/server/tests/api/check-params.js @@ -456,7 +456,7 @@ describe('Test parameters validator', function () { }) }) - it('Should fail without a mongodb id', function (done) { + it('Should fail without a correct uuid', function (done) { request(server.url) .get(path + 'coucou') .set('Accept', 'application/json') @@ -481,7 +481,7 @@ describe('Test parameters validator', function () { .expect(400, done) }) - it('Should fail without a mongodb id', function (done) { + it('Should fail without a correct uuid', function (done) { request(server.url) .delete(path + 'hello') .set('Authorization', 'Bearer ' + server.accessToken) diff --git a/server/tests/api/requests.js b/server/tests/api/requests.js index 7e790b54b..933ed29b4 100644 --- a/server/tests/api/requests.js +++ b/server/tests/api/requests.js @@ -79,15 +79,16 @@ describe('Test requests stats', function () { uploadVideo(server, function (err) { if (err) throw err - getRequestsStats(server, function (err, res) { - if (err) throw err + setTimeout(function () { + getRequestsStats(server, function (err, res) { + if (err) throw err - const body = res.body - expect(body.totalRequests).to.equal(1) + const body = res.body + expect(body.totalRequests).to.equal(1) - // Wait one cycle - setTimeout(done, 10000) - }) + done() + }) + }, 1000) }) }) diff --git a/server/tests/api/single-pod.js b/server/tests/api/single-pod.js index aedecacf3..66b762f82 100644 --- a/server/tests/api/single-pod.js +++ b/server/tests/api/single-pod.js @@ -153,31 +153,32 @@ describe('Test a single pod', function () { }) }) - it('Should search the video by podHost', function (done) { - videosUtils.searchVideo(server.url, '9001', 'host', function (err, res) { - if (err) throw err + // Not implemented yet + // it('Should search the video by podHost', function (done) { + // videosUtils.searchVideo(server.url, '9001', 'host', function (err, res) { + // if (err) throw err - expect(res.body.total).to.equal(1) - expect(res.body.data).to.be.an('array') - expect(res.body.data.length).to.equal(1) + // expect(res.body.total).to.equal(1) + // expect(res.body.data).to.be.an('array') + // expect(res.body.data.length).to.equal(1) - const video = res.body.data[0] - expect(video.name).to.equal('my super name') - expect(video.description).to.equal('my super description') - expect(video.podHost).to.equal('localhost:9001') - expect(video.author).to.equal('root') - expect(video.isLocal).to.be.true - expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'tag3' ]) - expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true + // const video = res.body.data[0] + // expect(video.name).to.equal('my super name') + // expect(video.description).to.equal('my super description') + // expect(video.podHost).to.equal('localhost:9001') + // expect(video.author).to.equal('root') + // expect(video.isLocal).to.be.true + // expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'tag3' ]) + // expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true - videosUtils.testVideoImage(server.url, 'video_short.webm', video.thumbnailPath, function (err, test) { - if (err) throw err - expect(test).to.equal(true) + // videosUtils.testVideoImage(server.url, 'video_short.webm', video.thumbnailPath, function (err, test) { + // if (err) throw err + // expect(test).to.equal(true) - done() - }) - }) - }) + // done() + // }) + // }) + // }) it('Should search the video by tag', function (done) { videosUtils.searchVideo(server.url, 'tag1', 'tags', function (err, res) { @@ -230,7 +231,7 @@ describe('Test a single pod', function () { }) it('Should not find a search by tag', function (done) { - videosUtils.searchVideo(server.url, 'tag', 'tags', function (err, res) { + videosUtils.searchVideo(server.url, 'hello', 'tags', function (err, res) { if (err) throw err expect(res.body.total).to.equal(0) @@ -424,29 +425,30 @@ describe('Test a single pod', function () { }) }) - it('Should search all the 9001 port videos', function (done) { - videosUtils.searchVideoWithPagination(server.url, '9001', 'host', 0, 15, function (err, res) { - if (err) throw err + // Not implemented yet + // it('Should search all the 9001 port videos', function (done) { + // videosUtils.searchVideoWithPagination(server.url, '9001', 'host', 0, 15, function (err, res) { + // if (err) throw err - const videos = res.body.data - expect(res.body.total).to.equal(6) - expect(videos.length).to.equal(6) + // const videos = res.body.data + // expect(res.body.total).to.equal(6) + // expect(videos.length).to.equal(6) - done() - }) - }) + // done() + // }) + // }) - it('Should search all the localhost videos', function (done) { - videosUtils.searchVideoWithPagination(server.url, 'localhost', 'host', 0, 15, function (err, res) { - if (err) throw err + // it('Should search all the localhost videos', function (done) { + // videosUtils.searchVideoWithPagination(server.url, 'localhost', 'host', 0, 15, function (err, res) { + // if (err) throw err - const videos = res.body.data - expect(res.body.total).to.equal(6) - expect(videos.length).to.equal(6) + // const videos = res.body.data + // expect(res.body.total).to.equal(6) + // expect(videos.length).to.equal(6) - done() - }) - }) + // done() + // }) + // }) it('Should search the good magnetUri video', function (done) { const video = videosListBase[0] From 01735eb0170ba421c1a979f3c724608ade176c6e Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Sat, 24 Dec 2016 17:12:40 +0100 Subject: [PATCH 04/47] Fix readme image closing tag --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ceba2a3f7..3f16697ac 100644 --- a/README.md +++ b/README.md @@ -234,7 +234,7 @@ Here are some simple schemes: Join a network -Many networks

From 55e0591c8e773b27cecec39d18da6ade898c89ed Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Sun, 25 Dec 2016 09:42:01 +0100 Subject: [PATCH 05/47] Update architecture document with postgresql measurements --- ARCHITECTURE.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index cc6864fac..ebcffd6cb 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -54,7 +54,16 @@ * A pod is a websocket tracker which is responsible for all the video uploaded in it * A pod has an administrator that can add/remove users, make friends and quit friends * A pod has different user accounts that can upload videos - * All pods have an index of all videos of the network (name, origin pod url, small description, uploader username, magnet Uri, thumbnail name, created date and the thumbnail file). For example, a test with 1000000 videos with alphanum characters and the following lengths: name = 50, author = 50, url = 25, description = 250, magnerUri = 200, thumbnail name = 50 has a mongodb size of ~ 4GB. To this, we add 1 000 000 thumbnails of 5-15 KB so 15GB maximum + * All pods have an index of all videos of the network (name, origin pod url, small description, uploader username, magnet Uri, thumbnail name, created date and the thumbnail file). For example, a test with 1000000 videos (3 tags each) with alphanum characters and the following lengths: name = 50, author = 50, podHost = 25, description = 250, videoExtension = 4, remoteId = 50, infoHash = 50 and tag = 10 has a PostgreSQL size of ~ 2GB with all the useful indexes. To this, we add 1 000 000 thumbnails of 5-15 KB so 15GB maximum + + table_name | row_estimate | index | toast | table + pod | 983416 | 140 MB | 83 MB | 57 MB + author | 1e+06 | 229 MB | 140 MB | 89 MB + tag | 2.96758e+06 | 309 MB | 182 MB | 127 MB + video | 1e+06 | 723 MB | 263 MB | 460 MB + video_tag | 3e+06 | 316 MB | 212 MB | 104 MB + + * After having uploaded a video, the server seeds it (WebSeed protocol), adds the meta data in its database and makes a secure request to all of its friends * If a user wants to watch a video, he asks its pod the magnetUri and the frontend adds the torrent (with WebTorrent), creates the HTML5 video tag and streams the file into it * A user watching a video seeds it too (BitTorrent) so another user who is watching the same video can get the data from the origin server and the user 1 (etc) From 8c67719c80cef7319afecdf1078d5447149ad8e7 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Sun, 25 Dec 2016 09:43:08 +0100 Subject: [PATCH 06/47] Update README with postgresql --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3f16697ac..68c2f907b 100644 --- a/README.md +++ b/README.md @@ -117,7 +117,7 @@ Thanks to [WebTorrent](https://github.com/feross/webtorrent), we can make P2P (t * **NodeJS >= 4.x** * **npm >= 3.x** * OpenSSL (cli) - * MongoDB + * PostgreSQL * ffmpeg #### Debian @@ -127,7 +127,7 @@ Thanks to [WebTorrent](https://github.com/feross/webtorrent), we can make P2P (t * Run: # apt-get update - # apt-get install ffmpeg mongodb openssl + # apt-get install ffmpeg postgresql-9.4 openssl # npm install -g npm@3 #### Other distribution... (PR welcome) From dd6019932efd6ae3b790bf024bc0cd74162e4517 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Sun, 25 Dec 2016 09:44:13 +0100 Subject: [PATCH 07/47] Update scripts with postgresql --- scripts/danger/clean/cleaner.js | 31 +++++++++++++++---------------- scripts/danger/clean/dev.sh | 3 ++- scripts/danger/clean/prod.sh | 3 ++- scripts/update-host.js | 25 +++++++++---------------- 4 files changed, 28 insertions(+), 34 deletions(-) diff --git a/scripts/danger/clean/cleaner.js b/scripts/danger/clean/cleaner.js index 10e91e700..d1e218145 100644 --- a/scripts/danger/clean/cleaner.js +++ b/scripts/danger/clean/cleaner.js @@ -1,24 +1,23 @@ const rimraf = require('rimraf') -const mongoose = require('mongoose') -mongoose.Promise = global.Promise const constants = require('../../../server/initializers/constants') +const db = require('../../../server/initializers/database') -const mongodbUrl = 'mongodb://' + constants.CONFIG.DATABASE.HOSTNAME + ':' + constants.CONFIG.DATABASE.PORT + '/' + constants.CONFIG.DATABASE.DBNAME -mongoose.connect(mongodbUrl, function () { - console.info('Deleting MongoDB %s database.', constants.CONFIG.DATABASE.DBNAME) - mongoose.connection.dropDatabase(function () { - mongoose.connection.close() - }) -}) - -const STORAGE = constants.CONFIG.STORAGE -Object.keys(STORAGE).forEach(function (storage) { - const storageDir = STORAGE[storage] - - rimraf(storageDir, function (err) { +db.init(true, function () { + db.sequelize.drop().asCallback(function (err) { if (err) throw err - console.info('Deleting %s.', storageDir) + console.info('Tables of %s deleted.', db.sequelize.config.database) + + const STORAGE = constants.CONFIG.STORAGE + Object.keys(STORAGE).forEach(function (storage) { + const storageDir = STORAGE[storage] + + rimraf(storageDir, function (err) { + if (err) throw err + + console.info('Deleting %s.', storageDir) + }) + }) }) }) diff --git a/scripts/danger/clean/dev.sh b/scripts/danger/clean/dev.sh index 5f09565cf..f4a5d1e8a 100755 --- a/scripts/danger/clean/dev.sh +++ b/scripts/danger/clean/dev.sh @@ -1,6 +1,7 @@ #!/bin/bash -read -p "This will remove all directories and Mongo database. Are you sure? " -n 1 -r +read -p "This will remove all directories and SQL tables. Are you sure? (y/*) " -n 1 -r +echo if [[ "$REPLY" =~ ^[Yy]$ ]]; then NODE_ENV=test node "./scripts/danger/clean/cleaner" diff --git a/scripts/danger/clean/prod.sh b/scripts/danger/clean/prod.sh index e6c92671d..7ee13ba83 100755 --- a/scripts/danger/clean/prod.sh +++ b/scripts/danger/clean/prod.sh @@ -1,6 +1,7 @@ #!/bin/bash -read -p "This will remove all directories and Mongo database. Are you sure? " -n 1 -r +read -p "This will remove all directories and SQL tables. Are you sure? (y/*) " -n 1 -r +echo if [[ "$REPLY" =~ ^[Yy]$ ]]; then NODE_ENV=production node "./scripts/danger/clean/cleaner" diff --git a/scripts/update-host.js b/scripts/update-host.js index 8a17f2402..91051e68b 100755 --- a/scripts/update-host.js +++ b/scripts/update-host.js @@ -5,31 +5,24 @@ // TODO: document this script const fs = require('fs') -const mongoose = require('mongoose') const parseTorrent = require('parse-torrent') const constants = require('../server/initializers/constants') -const database = require('../server/initializers/database') - -database.connect() +const db = require('../server/initializers/database') const friends = require('../server/lib/friends') -const Video = mongoose.model('Video') -friends.hasFriends(function (err, hasFriends) { - if (err) throw err - - if (hasFriends === true) { - console.log('Cannot update host because you have friends!') - process.exit(-1) - } - - console.log('Updating videos host in database.') - Video.update({ }, { podHost: constants.CONFIG.WEBSERVER.HOST }, { multi: true }, function (err) { +db.init(true, function () { + friends.hasFriends(function (err, hasFriends) { if (err) throw err + if (hasFriends === true) { + console.log('Cannot update host because you have friends!') + process.exit(-1) + } + console.log('Updating torrent files.') - Video.find().lean().exec(function (err, videos) { + db.Video.list(function (err, videos) { if (err) throw err videos.forEach(function (video) { From b769007f733769d3afe2d29a3eb23e2e7693f301 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Sun, 25 Dec 2016 09:44:57 +0100 Subject: [PATCH 08/47] Update migrations code --- config/default.yaml | 2 + server.js | 1 + server/initializers/checker.js | 2 +- server/initializers/constants.js | 43 ++------- server/initializers/database.js | 69 ++++++++------ server/initializers/installer.js | 5 +- .../migrations/0005-create-application.js | 17 ---- .../initializers/migrations/0005-example.js | 14 +++ .../migrations/0010-users-password.js | 22 ----- .../migrations/0015-admin-role.js | 16 ---- .../migrations/0020-requests-endpoint.js | 15 ---- .../migrations/0025-video-filenames.js | 57 ------------ .../migrations/0030-video-magnet.js | 32 ------- .../migrations/0035-url-to-host.js | 30 ------- .../migrations/0040-video-remote-id.js | 59 ------------ server/initializers/migrator.js | 90 ++++++++++++++----- server/models/application.js | 26 ++++-- server/models/video.js | 7 +- server/tests/utils/servers.js | 2 +- 19 files changed, 161 insertions(+), 348 deletions(-) delete mode 100644 server/initializers/migrations/0005-create-application.js create mode 100644 server/initializers/migrations/0005-example.js delete mode 100644 server/initializers/migrations/0010-users-password.js delete mode 100644 server/initializers/migrations/0015-admin-role.js delete mode 100644 server/initializers/migrations/0020-requests-endpoint.js delete mode 100644 server/initializers/migrations/0025-video-filenames.js delete mode 100644 server/initializers/migrations/0030-video-magnet.js delete mode 100644 server/initializers/migrations/0035-url-to-host.js delete mode 100644 server/initializers/migrations/0040-video-remote-id.js diff --git a/config/default.yaml b/config/default.yaml index 631400f7d..2dd5e05f9 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -10,6 +10,8 @@ database: hostname: 'localhost' port: 5432 suffix: '_dev' + username: peertube + password: peertube # From the project root directory storage: diff --git a/server.js b/server.js index e54ffe69f..f4ca53907 100644 --- a/server.js +++ b/server.js @@ -20,6 +20,7 @@ const constants = require('./server/initializers/constants') const logger = require('./server/helpers/logger') // Initialize database and models const db = require('./server/initializers/database') +db.init() // ----------- Checker ----------- const checker = require('./server/initializers/checker') diff --git a/server/initializers/checker.js b/server/initializers/checker.js index 7b402de82..2753604dc 100644 --- a/server/initializers/checker.js +++ b/server/initializers/checker.js @@ -27,7 +27,7 @@ function checkConfig () { function checkMissedConfig () { const required = [ 'listen.port', 'webserver.https', 'webserver.hostname', 'webserver.port', - 'database.hostname', 'database.port', 'database.suffix', + 'database.hostname', 'database.port', 'database.suffix', 'database.username', 'database.password', 'storage.certs', 'storage.videos', 'storage.logs', 'storage.thumbnails', 'storage.previews' ] const miss = [] diff --git a/server/initializers/constants.js b/server/initializers/constants.js index 1ad0c82a0..6f39b65da 100644 --- a/server/initializers/constants.js +++ b/server/initializers/constants.js @@ -37,7 +37,9 @@ const CONFIG = { DATABASE: { DBNAME: 'peertube' + config.get('database.suffix'), HOSTNAME: config.get('database.hostname'), - PORT: config.get('database.port') + PORT: config.get('database.port'), + USERNAME: config.get('database.username'), + PASSWORD: config.get('database.password') }, STORAGE: { CERT_DIR: path.join(__dirname, '..', '..', config.get('storage.certs')), @@ -87,41 +89,7 @@ const FRIEND_SCORE = { // --------------------------------------------------------------------------- -const MIGRATION_SCRIPTS = [ - { - script: '0005-create-application', - version: 5 - }, - { - script: '0010-users-password', - version: 10 - }, - { - script: '0015-admin-role', - version: 15 - }, - { - script: '0020-requests-endpoint', - version: 20 - }, - { - script: '0025-video-filenames', - version: 25 - }, - { - script: '0030-video-magnet', - version: 30 - }, - { - script: '0035-url-to-host', - version: 35 - }, - { - script: '0040-video-remote-id', - version: 40 - } -] -const LAST_SQL_SCHEMA_VERSION = (maxBy(MIGRATION_SCRIPTS, 'version'))['version'] +const LAST_MIGRATION_VERSION = 0 // --------------------------------------------------------------------------- @@ -197,8 +165,7 @@ module.exports = { CONFIG, CONSTRAINTS_FIELDS, FRIEND_SCORE, - LAST_SQL_SCHEMA_VERSION, - MIGRATION_SCRIPTS, + LAST_MIGRATION_VERSION, OAUTH_LIFETIME, PAGINATION_COUNT_DEFAULT, PODS_SCORE, diff --git a/server/initializers/database.js b/server/initializers/database.js index 9642231b9..f8f68adeb 100644 --- a/server/initializers/database.js +++ b/server/initializers/database.js @@ -10,7 +10,11 @@ const utils = require('../helpers/utils') const database = {} -const sequelize = new Sequelize(constants.CONFIG.DATABASE.DBNAME, 'peertube', 'peertube', { +const dbname = constants.CONFIG.DATABASE.DBNAME +const username = constants.CONFIG.DATABASE.USERNAME +const password = constants.CONFIG.DATABASE.PASSWORD + +const sequelize = new Sequelize(dbname, username, password, { dialect: 'postgres', host: constants.CONFIG.DATABASE.HOSTNAME, port: constants.CONFIG.DATABASE.PORT, @@ -26,33 +30,48 @@ const sequelize = new Sequelize(constants.CONFIG.DATABASE.DBNAME, 'peertube', 'p } }) -const modelDirectory = path.join(__dirname, '..', 'models') -fs.readdir(modelDirectory, function (err, files) { - if (err) throw err - - files.filter(function (file) { - if (file === 'utils.js') return false - - return true - }) - .forEach(function (file) { - const model = sequelize.import(path.join(modelDirectory, file)) - - database[model.name] = model - }) - - Object.keys(database).forEach(function (modelName) { - if ('associate' in database[modelName]) { - database[modelName].associate(database) - } - }) - - logger.info('Database is ready.') -}) - database.sequelize = sequelize database.Sequelize = Sequelize +database.init = init // --------------------------------------------------------------------------- module.exports = database + +// --------------------------------------------------------------------------- + +function init (silent, callback) { + if (!callback) { + callback = silent + silent = false + } + + if (!callback) callback = function () {} + + const modelDirectory = path.join(__dirname, '..', 'models') + fs.readdir(modelDirectory, function (err, files) { + if (err) throw err + + files.filter(function (file) { + // For all models but not utils.js + if (file === 'utils.js') return false + + return true + }) + .forEach(function (file) { + const model = sequelize.import(path.join(modelDirectory, file)) + + database[model.name] = model + }) + + Object.keys(database).forEach(function (modelName) { + if ('associate' in database[modelName]) { + database[modelName].associate(database) + } + }) + + if (!silent) logger.info('Database is ready.') + + return callback(null) + }) +} diff --git a/server/initializers/installer.js b/server/initializers/installer.js index 4823bc8c8..d5382364e 100644 --- a/server/initializers/installer.js +++ b/server/initializers/installer.js @@ -121,9 +121,8 @@ function createOAuthAdminIfNotExist (callback) { logger.info('Username: ' + username) logger.info('User password: ' + password) - logger.info('Creating Application collection.') - const application = db.Application.build({ sqlSchemaVersion: constants.LAST_SQL_SCHEMA_VERSION }) - application.save().asCallback(callback) + logger.info('Creating Application table.') + db.Application.create({ migrationVersion: constants.LAST_MIGRATION_VERSION }).asCallback(callback) }) }) } diff --git a/server/initializers/migrations/0005-create-application.js b/server/initializers/migrations/0005-create-application.js deleted file mode 100644 index e99dec019..000000000 --- a/server/initializers/migrations/0005-create-application.js +++ /dev/null @@ -1,17 +0,0 @@ -/* - Create the application collection in MongoDB. - Used to store the actual MongoDB scheme version. -*/ - -const mongoose = require('mongoose') - -const Application = mongoose.model('Application') - -exports.up = function (callback) { - const application = new Application() - application.save(callback) -} - -exports.down = function (callback) { - throw new Error('Not implemented.') -} diff --git a/server/initializers/migrations/0005-example.js b/server/initializers/migrations/0005-example.js new file mode 100644 index 000000000..481c2c4dd --- /dev/null +++ b/server/initializers/migrations/0005-example.js @@ -0,0 +1,14 @@ +/* + This is just an example. +*/ + +const db = require('../database') + +// options contains the transaction +exports.up = function (options, callback) { + // db.Application.create({ migrationVersion: 42 }, { transaction: options.transaction }).asCallback(callback) +} + +exports.down = function (options, callback) { + throw new Error('Not implemented.') +} diff --git a/server/initializers/migrations/0010-users-password.js b/server/initializers/migrations/0010-users-password.js deleted file mode 100644 index a0616a269..000000000 --- a/server/initializers/migrations/0010-users-password.js +++ /dev/null @@ -1,22 +0,0 @@ -/* - Convert plain user password to encrypted user password. -*/ - -const eachSeries = require('async/eachSeries') -const mongoose = require('mongoose') - -const User = mongoose.model('User') - -exports.up = function (callback) { - User.list(function (err, users) { - if (err) return callback(err) - - eachSeries(users, function (user, callbackEach) { - user.save(callbackEach) - }, callback) - }) -} - -exports.down = function (callback) { - throw new Error('Not implemented.') -} diff --git a/server/initializers/migrations/0015-admin-role.js b/server/initializers/migrations/0015-admin-role.js deleted file mode 100644 index af06dca9e..000000000 --- a/server/initializers/migrations/0015-admin-role.js +++ /dev/null @@ -1,16 +0,0 @@ -/* - Set the admin role to the root user. -*/ - -const constants = require('../constants') -const mongoose = require('mongoose') - -const User = mongoose.model('User') - -exports.up = function (callback) { - User.update({ username: 'root' }, { role: constants.USER_ROLES.ADMIN }, callback) -} - -exports.down = function (callback) { - throw new Error('Not implemented.') -} diff --git a/server/initializers/migrations/0020-requests-endpoint.js b/server/initializers/migrations/0020-requests-endpoint.js deleted file mode 100644 index 55feec571..000000000 --- a/server/initializers/migrations/0020-requests-endpoint.js +++ /dev/null @@ -1,15 +0,0 @@ -/* - Set the endpoint videos for requests. -*/ - -const mongoose = require('mongoose') - -const Request = mongoose.model('Request') - -exports.up = function (callback) { - Request.update({ }, { endpoint: 'videos' }, callback) -} - -exports.down = function (callback) { - throw new Error('Not implemented.') -} diff --git a/server/initializers/migrations/0025-video-filenames.js b/server/initializers/migrations/0025-video-filenames.js deleted file mode 100644 index df21494d7..000000000 --- a/server/initializers/migrations/0025-video-filenames.js +++ /dev/null @@ -1,57 +0,0 @@ -/* - Rename thumbnails and video filenames to _id.extension -*/ - -const each = require('async/each') -const fs = require('fs') -const path = require('path') -const mongoose = require('mongoose') - -const constants = require('../constants') -const logger = require('../../helpers/logger') - -const Video = mongoose.model('Video') - -exports.up = function (callback) { - // Use of lean because the new Video scheme does not have filename field - Video.find({ filename: { $ne: null } }).lean().exec(function (err, videos) { - if (err) throw err - - each(videos, function (video, callbackEach) { - const torrentName = video.filename + '.torrent' - const thumbnailName = video.thumbnail - const thumbnailExtension = path.extname(thumbnailName) - const videoName = video.filename - const videoExtension = path.extname(videoName) - - const newTorrentName = video._id + '.torrent' - const newThumbnailName = video._id + thumbnailExtension - const newVideoName = video._id + videoExtension - - const torrentsDir = constants.CONFIG.STORAGE.TORRENTS_DIR - const thumbnailsDir = constants.CONFIG.STORAGE.THUMBNAILS_DIR - const videosDir = constants.CONFIG.STORAGE.VIDEOS_DIR - - logger.info('Renaming %s to %s.', torrentsDir + torrentName, torrentsDir + newTorrentName) - fs.renameSync(torrentsDir + torrentName, torrentsDir + newTorrentName) - - logger.info('Renaming %s to %s.', thumbnailsDir + thumbnailName, thumbnailsDir + newThumbnailName) - fs.renameSync(thumbnailsDir + thumbnailName, thumbnailsDir + newThumbnailName) - - logger.info('Renaming %s to %s.', videosDir + videoName, videosDir + newVideoName) - fs.renameSync(videosDir + videoName, videosDir + newVideoName) - - Video.load(video._id, function (err, videoObj) { - if (err) return callbackEach(err) - - videoObj.extname = videoExtension - videoObj.remoteId = null - videoObj.save(callbackEach) - }) - }, callback) - }) -} - -exports.down = function (callback) { - throw new Error('Not implemented.') -} diff --git a/server/initializers/migrations/0030-video-magnet.js b/server/initializers/migrations/0030-video-magnet.js deleted file mode 100644 index b9119d61c..000000000 --- a/server/initializers/migrations/0030-video-magnet.js +++ /dev/null @@ -1,32 +0,0 @@ -/* - Change video magnet structures -*/ - -const each = require('async/each') -const magnet = require('magnet-uri') -const mongoose = require('mongoose') - -const Video = mongoose.model('Video') - -exports.up = function (callback) { - // Use of lean because the new Video scheme does not have magnetUri field - Video.find({ }).lean().exec(function (err, videos) { - if (err) throw err - - each(videos, function (video, callbackEach) { - const parsed = magnet.decode(video.magnetUri) - const infoHash = parsed.infoHash - - Video.load(video._id, function (err, videoObj) { - if (err) return callbackEach(err) - - videoObj.magnet.infoHash = infoHash - videoObj.save(callbackEach) - }) - }, callback) - }) -} - -exports.down = function (callback) { - throw new Error('Not implemented.') -} diff --git a/server/initializers/migrations/0035-url-to-host.js b/server/initializers/migrations/0035-url-to-host.js deleted file mode 100644 index 6243304d5..000000000 --- a/server/initializers/migrations/0035-url-to-host.js +++ /dev/null @@ -1,30 +0,0 @@ -/* - Change video magnet structures -*/ - -const each = require('async/each') -const mongoose = require('mongoose') - -const Video = mongoose.model('Video') - -exports.up = function (callback) { - // Use of lean because the new Video scheme does not have podUrl field - Video.find({ }).lean().exec(function (err, videos) { - if (err) throw err - - each(videos, function (video, callbackEach) { - Video.load(video._id, function (err, videoObj) { - if (err) return callbackEach(err) - - const host = video.podUrl.split('://')[1] - - videoObj.podHost = host - videoObj.save(callbackEach) - }) - }, callback) - }) -} - -exports.down = function (callback) { - throw new Error('Not implemented.') -} diff --git a/server/initializers/migrations/0040-video-remote-id.js b/server/initializers/migrations/0040-video-remote-id.js deleted file mode 100644 index 46a14a689..000000000 --- a/server/initializers/migrations/0040-video-remote-id.js +++ /dev/null @@ -1,59 +0,0 @@ -/* - Use remote id as identifier -*/ - -const map = require('lodash/map') -const mongoose = require('mongoose') -const readline = require('readline') - -const rl = readline.createInterface({ - input: process.stdin, - output: process.stdout -}) - -const logger = require('../../helpers/logger') -const friends = require('../../lib/friends') - -const Pod = mongoose.model('Pod') -const Video = mongoose.model('Video') - -exports.up = function (callback) { - Pod.find({}).lean().exec(function (err, pods) { - if (err) return callback(err) - - // We need to quit friends first - if (pods.length === 0) { - return setVideosRemoteId(callback) - } - - const timeout = setTimeout(function () { - throw new Error('You need to enter a value!') - }, 10000) - - rl.question('I am sorry but I need to quit friends for upgrading. Do you want to continue? (yes/*)', function (answer) { - if (answer !== 'yes') throw new Error('I cannot continue.') - - clearTimeout(timeout) - rl.close() - - const urls = map(pods, 'url') - logger.info('Saying goodbye to: ' + urls.join(', ')) - - setVideosRemoteId(function () { - friends.quitFriends(callback) - }) - }) - }) -} - -exports.down = function (callback) { - throw new Error('Not implemented.') -} - -function setVideosRemoteId (callback) { - Video.update({ filename: { $ne: null } }, { remoteId: null }, function (err) { - if (err) throw err - - Video.update({ filename: null }, { remoteId: mongoose.Types.ObjectId() }, callback) - }) -} diff --git a/server/initializers/migrator.js b/server/initializers/migrator.js index 9e5350e60..eaecb4936 100644 --- a/server/initializers/migrator.js +++ b/server/initializers/migrator.js @@ -1,6 +1,7 @@ 'use strict' const eachSeries = require('async/eachSeries') +const fs = require('fs') const path = require('path') const constants = require('./constants') @@ -12,35 +13,24 @@ const migrator = { } function migrate (callback) { - db.Application.loadSqlSchemaVersion(function (err, actualVersion) { + db.Application.loadMigrationVersion(function (err, actualVersion) { if (err) return callback(err) - // If there are a new mongo schemas - if (!actualVersion || actualVersion < constants.LAST_SQL_SCHEMA_VERSION) { + // If there are a new migration scripts + if (actualVersion < constants.LAST_MIGRATION_VERSION) { logger.info('Begin migrations.') - eachSeries(constants.MONGO_MIGRATION_SCRIPTS, function (entity, callbackEach) { - const versionScript = entity.version - - // Do not execute old migration scripts - if (versionScript <= actualVersion) return callbackEach(null) - - // Load the migration module and run it - const migrationScriptName = entity.script - logger.info('Executing %s migration script.', migrationScriptName) - - const migrationScript = require(path.join(__dirname, 'migrations', migrationScriptName)) - migrationScript.up(function (err) { - if (err) return callbackEach(err) - - // Update the new mongo version schema - db.Application.updateSqlSchemaVersion(versionScript, callbackEach) - }) - }, function (err) { + getMigrationScripts(function (err, migrationScripts) { if (err) return callback(err) - logger.info('Migrations finished. New SQL version schema: %s', constants.LAST_SQL_SCHEMA_VERSION) - return callback(null) + eachSeries(migrationScripts, function (entity, callbackEach) { + executeMigration(actualVersion, entity, callbackEach) + }, function (err) { + if (err) return callback(err) + + logger.info('Migrations finished. New migration version schema: %s', constants.LAST_MIGRATION_VERSION) + return callback(null) + }) }) } else { return callback(null) @@ -52,3 +42,57 @@ function migrate (callback) { module.exports = migrator +// --------------------------------------------------------------------------- + +function getMigrationScripts (callback) { + fs.readdir(path.join(__dirname, 'migrations'), function (err, files) { + if (err) return callback(err) + + const filesToMigrate = [] + + files.forEach(function (file) { + // Filename is something like 'version-blabla.js' + const version = file.split('-')[0] + filesToMigrate.push({ + version, + script: file + }) + }) + + return callback(err, filesToMigrate) + }) +} + +function executeMigration (actualVersion, entity, callback) { + const versionScript = entity.version + + // Do not execute old migration scripts + if (versionScript <= actualVersion) return callback(null) + + // Load the migration module and run it + const migrationScriptName = entity.script + logger.info('Executing %s migration script.', migrationScriptName) + + const migrationScript = require(path.join(__dirname, 'migrations', migrationScriptName)) + + db.sequelize.transaction().asCallback(function (err, t) { + if (err) return callback(err) + + migrationScript.up({ transaction: t }, function (err) { + if (err) { + t.rollback() + return callback(err) + } + + // Update the new migration version + db.Application.updateMigrationVersion(versionScript, t, function (err) { + if (err) { + t.rollback() + return callback(err) + } + + t.commit() + }) + }) + }) +} diff --git a/server/models/application.js b/server/models/application.js index ec1d7b122..4114ed76d 100644 --- a/server/models/application.js +++ b/server/models/application.js @@ -1,15 +1,15 @@ module.exports = function (sequelize, DataTypes) { const Application = sequelize.define('Application', { - sqlSchemaVersion: { + migrationVersion: { type: DataTypes.INTEGER, defaultValue: 0 } }, { classMethods: { - loadSqlSchemaVersion, - updateSqlSchemaVersion + loadMigrationVersion, + updateMigrationVersion } } ) @@ -19,18 +19,28 @@ module.exports = function (sequelize, DataTypes) { // --------------------------------------------------------------------------- -function loadSqlSchemaVersion (callback) { +function loadMigrationVersion (callback) { const query = { - attributes: [ 'sqlSchemaVersion' ] + attributes: [ 'migrationVersion' ] } return this.findOne(query).asCallback(function (err, data) { - const version = data ? data.sqlSchemaVersion : 0 + const version = data ? data.migrationVersion : 0 return callback(err, version) }) } -function updateSqlSchemaVersion (newVersion, callback) { - return this.update({ sqlSchemaVersion: newVersion }).asCallback(callback) +function updateMigrationVersion (newVersion, transaction, callback) { + const options = { + where: {} + } + + if (!callback) { + transaction = callback + } else { + options.transaction = transaction + } + + return this.update({ migrationVersion: newVersion }, options).asCallback(callback) } diff --git a/server/models/video.js b/server/models/video.js index 0023a24e1..af05a861f 100644 --- a/server/models/video.js +++ b/server/models/video.js @@ -16,7 +16,7 @@ const modelUtils = require('./utils') // --------------------------------------------------------------------------- module.exports = function (sequelize, DataTypes) { -// TODO: add indexes on searchable columns + // TODO: add indexes on searchable columns const Video = sequelize.define('Video', { id: { @@ -50,6 +50,7 @@ module.exports = function (sequelize, DataTypes) { generateThumbnailFromBase64, getDurationFromFile, + list, listForApi, listByHostAndRemoteId, listOwnedAndPopulateAuthorAndTags, @@ -310,6 +311,10 @@ function getDurationFromFile (videoPath, callback) { }) } +function list (callback) { + return this.find().asCallback() +} + function listForApi (start, count, sort, callback) { const query = { offset: start, diff --git a/server/tests/utils/servers.js b/server/tests/utils/servers.js index 4e55f8f5c..e7c756499 100644 --- a/server/tests/utils/servers.js +++ b/server/tests/utils/servers.js @@ -103,7 +103,7 @@ function runServer (number, callback) { if (serverRunString[key] === false) dontContinue = true } - // If no, there is maybe one thing not already initialized (mongodb...) + // If no, there is maybe one thing not already initialized (client/user credentials generation...) if (dontContinue === true) return server.app.stdout.removeListener('data', onStdout) From f83e27958109b829ba6326efda0679cc032003e5 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Sun, 25 Dec 2016 09:47:49 +0100 Subject: [PATCH 09/47] Fix standard lint --- server/helpers/peertube-crypto.js | 10 --------- server/initializers/constants.js | 1 - .../initializers/migrations/0005-example.js | 22 +++++++++---------- 3 files changed, 11 insertions(+), 22 deletions(-) diff --git a/server/helpers/peertube-crypto.js b/server/helpers/peertube-crypto.js index 2e07df00e..302ddca58 100644 --- a/server/helpers/peertube-crypto.js +++ b/server/helpers/peertube-crypto.js @@ -9,8 +9,6 @@ const ursa = require('ursa') const constants = require('../initializers/constants') const logger = require('./logger') -const algorithm = 'aes-256-ctr' - const peertubeCrypto = { checkSignature, comparePassword, @@ -113,11 +111,3 @@ function createCerts (callback) { }) }) } - -function generatePassword (callback) { - crypto.randomBytes(32, function (err, buf) { - if (err) return callback(err) - - callback(null, buf.toString('utf8')) - }) -} diff --git a/server/initializers/constants.js b/server/initializers/constants.js index 6f39b65da..fc501845a 100644 --- a/server/initializers/constants.js +++ b/server/initializers/constants.js @@ -1,7 +1,6 @@ 'use strict' const config = require('config') -const maxBy = require('lodash/maxBy') const path = require('path') // --------------------------------------------------------------------------- diff --git a/server/initializers/migrations/0005-example.js b/server/initializers/migrations/0005-example.js index 481c2c4dd..cedc42919 100644 --- a/server/initializers/migrations/0005-example.js +++ b/server/initializers/migrations/0005-example.js @@ -1,14 +1,14 @@ -/* - This is just an example. -*/ +// /* +// This is just an example. +// */ -const db = require('../database') +// const db = require('../database') -// options contains the transaction -exports.up = function (options, callback) { - // db.Application.create({ migrationVersion: 42 }, { transaction: options.transaction }).asCallback(callback) -} +// // options contains the transaction +// exports.up = function (options, callback) { +// db.Application.create({ migrationVersion: 42 }, { transaction: options.transaction }).asCallback(callback) +// } -exports.down = function (options, callback) { - throw new Error('Not implemented.') -} +// exports.down = function (options, callback) { +// throw new Error('Not implemented.') +// } From 7e6afafd6a43a337b77c4a6c1e93e12bda994255 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Sun, 25 Dec 2016 09:53:50 +0100 Subject: [PATCH 10/47] Update support documentations with postgresql --- support/doc/client/code.md | 2 +- support/doc/server/code.md | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/support/doc/client/code.md b/support/doc/client/code.md index 3c2029847..f629af32f 100644 --- a/support/doc/client/code.md +++ b/support/doc/client/code.md @@ -54,7 +54,7 @@ Uses [TSLint](https://palantir.github.io/tslint/) for TypeScript linting and [An * Install [the dependencies](https://github.com/Chocobozzz/PeerTube#dependencies) * Run `npm install` at the root directory to install all the dependencies - * Run MongoDB + * Run PostgreSQL and create the database `peertube_dev`. * Run `npm run dev` to compile the client and automatically run the server. Then the server will watch and compile the client files automatically. You just need to refresh the browser to see your modifications. In a Angular 2 application, we create components that we put together. Each component is defined by an HTML structure, a TypeScript file and optionnaly a SASS file. diff --git a/support/doc/server/code.md b/support/doc/server/code.md index 652bd53df..c15885c8c 100644 --- a/support/doc/server/code.md +++ b/support/doc/server/code.md @@ -6,9 +6,9 @@ The server is a web server developed with [NodeJS](https://nodejs.org)/[Express] ## Technologies * [NodeJS](https://nodejs.org) -> Language - * [MongoDB](https://www.mongodb.com/) -> Database + * [PostgreSQL](https://www.postgresql.org/) -> Database * [Express](http://expressjs.com) -> Web server framework - * [Mongoose](http://mongoosejs.com/) -> MongoDB object modeling + * [Sequelize](http://docs.sequelizejs.com/en/v3/) -> SQL ORM * [WebTorrent](https://webtorrent.io/) -> BitTorrent tracker and torrent creation * [Mocha](https://mochajs.org/) -> Test framework @@ -28,7 +28,7 @@ All other server files are in the [server](https://github.com/Chocobozzz/PeerTub |__ initializers -> functions used at the server startup (installer, database, constants...) |__ lib -> library function (WebTorrent, OAuth2, friends logic...) |__ middlewares -> middlewares for controllers (requests validators, requests pagination...) - |__ models -> Mongoose models for each MongoDB collection (videos, users, pods...) + |__ models -> Sequelize models for each SQL tables (videos, users, pods...) |__ tests -> API tests and real world simulations (to test the decentralized feature...) @@ -41,7 +41,7 @@ Uses [JavaScript Standard Style](http://standardjs.com/). * Install [the dependencies](https://github.com/Chocobozzz/PeerTube#dependencies) * Run `npm install` at the root directory to install all the dependencies - * Run MongoDB + * Run PostgreSQL and create the database `peertube_dev`. * Run `npm run dev` to compile the client and automatically run the server. If the client files are already compiled you can simply run `NODE_ENV=test node server` The `NODE_ENV=test` is set to speed up communications between pods (see [constants.js](https://github.com/Chocobozzz/PeerTube/blob/master/server/initializers/constants.js)). @@ -62,7 +62,7 @@ A video is seeded by the server throught the [WebSeed](http://www.bittorrent.org ![Architecture scheme](https://github.com/Chocobozzz/PeerTube/blob/master/support/doc/server/upload-video.png) -When a user uploads a video, the rest API create the torrent file and then adds it to its Mongo database. +When a user uploads a video, the rest API create the torrent file and then adds it to its database. If a user wants to watch the video, the tracker will indicate all other users that are watching the video + the HTTP url for the WebSeed. From da268a9758ac4af03c8ae5894d6d13fd097ad504 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Sun, 25 Dec 2016 10:41:51 +0100 Subject: [PATCH 11/47] Try to adapt travis to postgresql --- .travis.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9fcd50dee..19fcd53c6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,8 @@ language: node_js node_js: - - "4.6" - - "6.9" + - "4" + - "6" env: - CXX=g++-4.8 @@ -13,11 +13,12 @@ addons: - ubuntu-toolchain-r-test packages: - g++-4.8 + postgresql: "9.4" sudo: false services: - - mongodb + - postgresql before_install: if [[ `npm -v` != 3* ]]; then npm i -g npm@3; fi @@ -29,6 +30,13 @@ before_script: - cp ffmpeg-*-64bit-static/{ffmpeg,ffprobe,ffserver} $HOME/bin - export PATH=$HOME/bin:$PATH - export NODE_TEST_IMAGE=true + - psql -c 'create database peertube_test1;' -U postgres + - psql -c 'create database peertube_test2;' -U postgres + - psql -c 'create database peertube_test3;' -U postgres + - psql -c 'create database peertube_test4;' -U postgres + - psql -c 'create database peertube_test5;' -U postgres + - psql -c 'create database peertube_test6;' -U postgres + - psql -c 'create user peertube with password \'peertube\';' -U postgres after_failure: - cat test1/logs/all-logs.log From a25970cc20ad5d61b7ae4e94850f3e42c0359bcd Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Sun, 25 Dec 2016 10:48:00 +0100 Subject: [PATCH 12/47] Fix travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 19fcd53c6..76efedd50 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,7 +36,7 @@ before_script: - psql -c 'create database peertube_test4;' -U postgres - psql -c 'create database peertube_test5;' -U postgres - psql -c 'create database peertube_test6;' -U postgres - - psql -c 'create user peertube with password \'peertube\';' -U postgres + - psql -c "create user peertube with password 'peertube';" -U postgres after_failure: - cat test1/logs/all-logs.log From 56ac84d0a32844c7b7df4c584dccb6e6c17e35de Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Sun, 25 Dec 2016 12:05:47 +0100 Subject: [PATCH 13/47] Fix real world script --- scripts/play.sh | 4 +++- server/tests/real-world/real-world.js | 11 ++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/scripts/play.sh b/scripts/play.sh index 33dc1a545..ab812025c 100755 --- a/scripts/play.sh +++ b/scripts/play.sh @@ -5,7 +5,9 @@ if [ ! -f server.js ]; then exit -1 fi -for i in 1 2 3; do +max=${1:-3} + +for i in $(seq 1 $max); do NODE_ENV=test NODE_APP_INSTANCE=$i node server.js & sleep 1 done diff --git a/server/tests/real-world/real-world.js b/server/tests/real-world/real-world.js index dba1970c5..2ae3dc15b 100644 --- a/server/tests/real-world/real-world.js +++ b/server/tests/real-world/real-world.js @@ -30,9 +30,9 @@ let integrityInterval = parseInt(program.integrity) || 60000 const numberOfPods = 6 // Wait requests between pods -const requestsMaxPerInterval = constants.INTERVAL / actionInterval +const requestsMaxPerInterval = constants.REQUESTS_INTERVAL / actionInterval const intervalsToMakeAllRequests = Math.ceil(requestsMaxPerInterval / constants.REQUESTS_LIMIT) -const waitForBeforeIntegrityCheck = (intervalsToMakeAllRequests * constants.INTERVAL) + 1000 +const waitForBeforeIntegrityCheck = (intervalsToMakeAllRequests * constants.REQUESTS_INTERVAL) + 1000 integrityInterval += waitForBeforeIntegrityCheck @@ -160,9 +160,9 @@ function exitServers (servers, callback) { function upload (servers, numServer, callback) { if (!callback) callback = function () {} - const name = 'my super name for pod 1' - const description = 'my super description for pod 1' - const tags = [ 'tag1p1', 'tag2p1' ] + const name = Date.now() + ' name' + const description = Date.now() + ' description' + const tags = [ Date.now().toString().substring(0, 5) + 't1', Date.now().toString().substring(0, 5) + 't2' ] const file = 'video_short1.webm' console.log('Upload video to server ' + numServer) @@ -205,6 +205,7 @@ function checkIntegrity (servers, callback) { for (const video of videos) { if (!isEqual(video, videos[0])) { console.error('Integrity not ok!') + process.exit(-1) } } From 124648d7fcfc3c9a91fe702cbc40c317429c05bd Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Sun, 25 Dec 2016 12:06:08 +0100 Subject: [PATCH 14/47] Server: add createdAt from remote video in database --- server/controllers/api/remote.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/controllers/api/remote.js b/server/controllers/api/remote.js index c7a5658e8..2cf916ff3 100644 --- a/server/controllers/api/remote.js +++ b/server/controllers/api/remote.js @@ -135,7 +135,8 @@ function addRemoteVideo (videoToCreateData, fromHost, finalCallback) { infoHash: videoToCreateData.infoHash, description: videoToCreateData.description, authorId: author.id, - duration: videoToCreateData.duration + duration: videoToCreateData.duration, + createdAt: videoToCreateData.createdAt } const video = db.Video.build(videoData) From 178edb20259f90b1c59f40728aaf8073f097f1f5 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 26 Dec 2016 17:44:25 +0100 Subject: [PATCH 15/47] Server: correctly sort tags by name asc --- server/models/video.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/models/video.js b/server/models/video.js index af05a861f..04478c8d7 100644 --- a/server/models/video.js +++ b/server/models/video.js @@ -320,7 +320,7 @@ function listForApi (start, count, sort, callback) { offset: start, limit: count, distinct: true, // For the count, a video can have many tags - order: [ modelUtils.getSort(sort) ], + order: [ modelUtils.getSort(sort), [ this.sequelize.models.Tag, 'name', 'ASC' ] ], include: [ { model: this.sequelize.models.Author, @@ -440,7 +440,7 @@ function searchAndPopulateAuthorAndPodAndTags (value, field, start, count, sort, offset: start, limit: count, distinct: true, // For the count, a video can have many tags - order: [ modelUtils.getSort(sort) ] + order: [ modelUtils.getSort(sort), [ this.sequelize.models.Tag, 'name', 'ASC' ] ] } // Make an exact search with the magnet From 00d6a41e46e4c4948b0d5b4cf21433150a57c067 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 27 Dec 2016 18:33:38 +0100 Subject: [PATCH 16/47] Add script to migrate from mongodb to postgresql Usage: NODE_ENV=production ./scripts/mongo-to-postgre.js --mongo-database peertube-prod --- scripts/danger/clean/cleaner.js | 10 +- scripts/mongo-to-postgre.js | 244 ++++++++++++++++++++++++++++++ server/helpers/peertube-crypto.js | 1 - 3 files changed, 250 insertions(+), 5 deletions(-) create mode 100755 scripts/mongo-to-postgre.js diff --git a/scripts/danger/clean/cleaner.js b/scripts/danger/clean/cleaner.js index d1e218145..1a1e3dee7 100644 --- a/scripts/danger/clean/cleaner.js +++ b/scripts/danger/clean/cleaner.js @@ -1,3 +1,4 @@ +const eachSeries = require('async/eachSeries') const rimraf = require('rimraf') const constants = require('../../../server/initializers/constants') @@ -10,14 +11,15 @@ db.init(true, function () { console.info('Tables of %s deleted.', db.sequelize.config.database) const STORAGE = constants.CONFIG.STORAGE - Object.keys(STORAGE).forEach(function (storage) { + eachSeries(Object.keys(STORAGE), function (storage, callbackEach) { const storageDir = STORAGE[storage] rimraf(storageDir, function (err) { - if (err) throw err - - console.info('Deleting %s.', storageDir) + console.info('%s deleted.', storageDir) + return callbackEach(err) }) + }, function () { + process.exit(0) }) }) }) diff --git a/scripts/mongo-to-postgre.js b/scripts/mongo-to-postgre.js new file mode 100755 index 000000000..4a581b46a --- /dev/null +++ b/scripts/mongo-to-postgre.js @@ -0,0 +1,244 @@ +#!/usr/bin/env node + +'use strict' + +// TODO: document this script + +const program = require('commander') +const eachSeries = require('async/eachSeries') +const series = require('async/series') +const waterfall = require('async/waterfall') +const fs = require('fs') +const path = require('path') +const MongoClient = require('mongodb').MongoClient + +const constants = require('../server/initializers/constants') + +program + .option('-mh, --mongo-host [host]', 'MongoDB host', 'localhost') + .option('-mp, --mongo-port [weight]', 'MongoDB port', '27017') + .option('-md, --mongo-database [dbname]', 'MongoDB database') + .parse(process.argv) + +if (!program.mongoDatabase) { + console.error('The mongodb database is mandatory.') + process.exit(-1) +} + +const mongoUrl = 'mongodb://' + program.mongoHost + ':' + program.mongoPort + '/' + program.mongoDatabase +const dbSequelize = require('../server/initializers/database') + +console.log('Connecting to ' + mongoUrl) +MongoClient.connect(mongoUrl, function (err, dbMongo) { + if (err) throw err + + console.log('Connected to ' + mongoUrl) + + const videoMongo = dbMongo.collection('videos') + const userMongo = dbMongo.collection('users') + const podMongo = dbMongo.collection('pods') + + podMongo.count(function (err, podsLength) { + if (err) throw err + + if (podsLength > 0) { + console.error('You need to quit friends first.') + process.exit(-1) + } + + console.log('Connecting to ' + dbSequelize.sequelize.config.database) + dbSequelize.init(true, function (err) { + if (err) throw err + + console.log('Connected to SQL database %s.', dbSequelize.sequelize.config.database) + + series([ + function (next) { + dbSequelize.sequelize.sync({ force: true }).asCallback(next) + }, + + function (next) { + migrateVideos(videoMongo, dbSequelize, next) + }, + + function (next) { + migrateUsers(userMongo, dbSequelize, next) + } + ], function (err) { + if (err) console.error(err) + + process.exit(0) + }) + }) + }) +}) + +// --------------------------------------------------------------------------- + +function migrateUsers (userMongo, dbSequelize, callback) { + userMongo.find().toArray(function (err, mongoUsers) { + if (err) return callback(err) + + eachSeries(mongoUsers, function (mongoUser, callbackEach) { + console.log('Migrating user %s', mongoUser.username) + + const userData = { + username: mongoUser.username, + password: mongoUser.password, + role: mongoUser.role + } + const options = { + hooks: false + } + + dbSequelize.User.create(userData, options).asCallback(callbackEach) + }, callback) + }) +} + +function migrateVideos (videoMongo, dbSequelize, finalCallback) { + videoMongo.find().toArray(function (err, mongoVideos) { + if (err) return finalCallback(err) + + eachSeries(mongoVideos, function (mongoVideo, callbackEach) { + console.log('Migrating video %s.', mongoVideo.name) + + waterfall([ + + function startTransaction (callback) { + dbSequelize.sequelize.transaction().asCallback(function (err, t) { + return callback(err, t) + }) + }, + + function findOrCreatePod (t, callback) { + if (mongoVideo.remoteId === null) return callback(null, t, null) + + const query = { + where: { + host: mongoVideo.podHost + }, + defaults: { + host: mongoVideo.podHost + }, + transaction: t + } + + dbSequelize.Pod.findOrCreate(query).asCallback(function (err, result) { + // [ instance, wasCreated ] + return callback(err, t, result[0]) + }) + }, + + function findOrCreateAuthor (t, pod, callback) { + const podId = pod ? pod.id : null + const username = mongoVideo.author + + const query = { + where: { + podId, + name: username + }, + defaults: { + podId, + name: username + }, + transaction: t + } + + dbSequelize.Author.findOrCreate(query).asCallback(function (err, result) { + // [ instance, wasCreated ] + return callback(err, t, result[0]) + }) + }, + + function findOrCreateTags (t, author, callback) { + const tags = mongoVideo.tags + const tagInstances = [] + + eachSeries(tags, function (tag, callbackEach) { + const query = { + where: { + name: tag + }, + defaults: { + name: tag + }, + transaction: t + } + + dbSequelize.Tag.findOrCreate(query).asCallback(function (err, res) { + if (err) return callbackEach(err) + + // res = [ tag, isCreated ] + const tag = res[0] + tagInstances.push(tag) + return callbackEach() + }) + }, function (err) { + return callback(err, t, author, tagInstances) + }) + }, + + function createVideoObject (t, author, tagInstances, callback) { + const videoData = { + name: mongoVideo.name, + remoteId: mongoVideo.remoteId, + extname: mongoVideo.extname, + infoHash: mongoVideo.magnet.infoHash, + description: mongoVideo.description, + authorId: author.id, + duration: mongoVideo.duration, + createdAt: mongoVideo.createdDate + } + + const video = dbSequelize.Video.build(videoData) + + return callback(null, t, tagInstances, video) + }, + + function moveVideoFile (t, tagInstances, video, callback) { + const basePath = constants.CONFIG.STORAGE.VIDEOS_DIR + const src = path.join(basePath, mongoVideo._id.toString()) + video.extname + const dst = path.join(basePath, video.id) + video.extname + fs.rename(src, dst, function (err) { + if (err) return callback(err) + + return callback(null, t, tagInstances, video) + }) + }, + + function insertVideoIntoDB (t, tagInstances, video, callback) { + const options = { + transaction: t + } + + video.save(options).asCallback(function (err, videoCreated) { + return callback(err, t, tagInstances, videoCreated) + }) + }, + + function associateTagsToVideo (t, tagInstances, video, callback) { + const options = { transaction: t } + + video.setTags(tagInstances, options).asCallback(function (err) { + return callback(err, t) + }) + } + + ], function (err, t) { + if (err) { + // Abort transaction? + if (t) t.rollback() + + return callbackEach(err) + } + + // Commit transaction + t.commit() + + return callbackEach() + }) + }, finalCallback) + }) +} diff --git a/server/helpers/peertube-crypto.js b/server/helpers/peertube-crypto.js index 302ddca58..610cb16cd 100644 --- a/server/helpers/peertube-crypto.js +++ b/server/helpers/peertube-crypto.js @@ -1,7 +1,6 @@ 'use strict' const bcrypt = require('bcrypt') -const crypto = require('crypto') const fs = require('fs') const openssl = require('openssl-wrapper') const ursa = require('ursa') From 552cc9d646e78edae8b0fe61564d4e49db0b6206 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 27 Dec 2016 18:35:56 +0100 Subject: [PATCH 17/47] Copy all keys in production example so if we chagne the default one production servers won't break --- config/production.yaml.example | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/config/production.yaml.example b/config/production.yaml.example index 743f972de..005444e73 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example @@ -1,3 +1,6 @@ +listen: + port: 9000 + # Correspond to your reverse proxy "listen" configuration webserver: https: false @@ -5,4 +8,17 @@ webserver: port: 80 database: + hostname: 'localhost' + port: 5432 suffix: '_prod' + username: peertube + password: peertube + +# From the project root directory +storage: + certs: 'certs/' + videos: 'videos/' + logs: 'logs/' + previews: 'previews/' + thumbnails: 'thumbnails/' + torrents: 'torrents/' From 67bf9b96bbcd92b069fe86d9223fe0f8b9c6e677 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 28 Dec 2016 15:49:23 +0100 Subject: [PATCH 18/47] Server: add database field validations --- server/helpers/custom-validators/pods.js | 9 ++- server/helpers/custom-validators/videos.js | 6 -- server/initializers/constants.js | 2 +- server/initializers/installer.js | 10 ++- server/models/application.js | 8 ++- server/models/author.js | 13 +++- server/models/oauth-client.js | 11 ++-- server/models/oauth-token.js | 19 +++--- server/models/pod.js | 34 ++++++---- server/models/request.js | 14 ++-- server/models/tag.js | 3 +- server/models/user.js | 32 +++++++--- server/models/video.js | 74 ++++++++++++++++------ 13 files changed, 162 insertions(+), 73 deletions(-) diff --git a/server/helpers/custom-validators/pods.js b/server/helpers/custom-validators/pods.js index 0154a2424..8bb3733ff 100644 --- a/server/helpers/custom-validators/pods.js +++ b/server/helpers/custom-validators/pods.js @@ -5,14 +5,19 @@ const validator = require('express-validator').validator const miscValidators = require('./misc') const podsValidators = { - isEachUniqueHostValid + isEachUniqueHostValid, + isHostValid +} + +function isHostValid (host) { + return validator.isURL(host) && host.split('://').length === 1 } function isEachUniqueHostValid (hosts) { return miscValidators.isArray(hosts) && hosts.length !== 0 && hosts.every(function (host) { - return validator.isURL(host) && host.split('://').length === 1 && hosts.indexOf(host) === hosts.lastIndexOf(host) + return isHostValid(host) && hosts.indexOf(host) === hosts.lastIndexOf(host) }) } diff --git a/server/helpers/custom-validators/videos.js b/server/helpers/custom-validators/videos.js index be8256a80..da857ba5f 100644 --- a/server/helpers/custom-validators/videos.js +++ b/server/helpers/custom-validators/videos.js @@ -15,7 +15,6 @@ const videosValidators = { isVideoDurationValid, isVideoInfoHashValid, isVideoNameValid, - isVideoPodHostValid, isVideoTagsValid, isVideoThumbnailValid, isVideoThumbnail64Valid @@ -74,11 +73,6 @@ function isVideoNameValid (value) { return validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.NAME) } -function isVideoPodHostValid (value) { - // TODO: set options (TLD...) - return validator.isURL(value) -} - function isVideoTagsValid (tags) { return miscValidators.isArray(tags) && validator.isInt(tags.length, VIDEOS_CONSTRAINTS_FIELDS.TAGS) && diff --git a/server/initializers/constants.js b/server/initializers/constants.js index fc501845a..0af7aca3c 100644 --- a/server/initializers/constants.js +++ b/server/initializers/constants.js @@ -69,7 +69,7 @@ const CONSTRAINTS_FIELDS = { NAME: { min: 3, max: 50 }, // Length DESCRIPTION: { min: 3, max: 250 }, // Length EXTNAME: [ '.mp4', '.ogv', '.webm' ], - INFO_HASH: { min: 10, max: 50 }, // Length + INFO_HASH: { min: 40, max: 40 }, // Length, infohash is 20 bytes length but we represent it in hexa so 20 * 2 DURATION: { min: 1, max: 7200 }, // Number TAGS: { min: 1, max: 3 }, // Number of total tags TAG: { min: 2, max: 10 }, // Length diff --git a/server/initializers/installer.js b/server/initializers/installer.js index d5382364e..fb63b81ac 100644 --- a/server/initializers/installer.js +++ b/server/initializers/installer.js @@ -96,6 +96,7 @@ function createOAuthAdminIfNotExist (callback) { const username = 'root' const role = constants.USER_ROLES.ADMIN + const createOptions = {} let password = '' // Do not generate a random password for tests @@ -105,17 +106,20 @@ function createOAuthAdminIfNotExist (callback) { if (process.env.NODE_APP_INSTANCE) { password += process.env.NODE_APP_INSTANCE } + + // Our password is weak so do not validate it + createOptions.validate = false } else { password = passwordGenerator(8, true) } - const user = db.User.build({ + const userData = { username, password, role - }) + } - user.save().asCallback(function (err, createdUser) { + db.User.create(userData, createOptions).asCallback(function (err, createdUser) { if (err) return callback(err) logger.info('Username: ' + username) diff --git a/server/models/application.js b/server/models/application.js index 4114ed76d..46dcfde33 100644 --- a/server/models/application.js +++ b/server/models/application.js @@ -1,9 +1,15 @@ +'use strict' + module.exports = function (sequelize, DataTypes) { const Application = sequelize.define('Application', { migrationVersion: { type: DataTypes.INTEGER, - defaultValue: 0 + defaultValue: 0, + allowNull: false, + validate: { + isInt: true + } } }, { diff --git a/server/models/author.js b/server/models/author.js index 493c2ca11..e0ac868ea 100644 --- a/server/models/author.js +++ b/server/models/author.js @@ -1,8 +1,19 @@ +'use strict' + +const customUsersValidators = require('../helpers/custom-validators').users + module.exports = function (sequelize, DataTypes) { const Author = sequelize.define('Author', { name: { - type: DataTypes.STRING + type: DataTypes.STRING, + allowNull: false, + validate: { + usernameValid: function (value) { + const res = customUsersValidators.isUserUsernameValid(value) + if (res === false) throw new Error('Username is not valid.') + } + } } }, { diff --git a/server/models/oauth-client.js b/server/models/oauth-client.js index 15118591a..b56838d4c 100644 --- a/server/models/oauth-client.js +++ b/server/models/oauth-client.js @@ -1,11 +1,15 @@ +'use strict' + module.exports = function (sequelize, DataTypes) { const OAuthClient = sequelize.define('OAuthClient', { clientId: { - type: DataTypes.STRING + type: DataTypes.STRING, + allowNull: false }, clientSecret: { - type: DataTypes.STRING + type: DataTypes.STRING, + allowNull: false }, grants: { type: DataTypes.ARRAY(DataTypes.STRING) @@ -28,9 +32,6 @@ module.exports = function (sequelize, DataTypes) { return OAuthClient } -// TODO: validation -// OAuthClientSchema.path('clientSecret').required(true) - // --------------------------------------------------------------------------- function associate (models) { diff --git a/server/models/oauth-token.js b/server/models/oauth-token.js index c9108bf95..f8de4e916 100644 --- a/server/models/oauth-token.js +++ b/server/models/oauth-token.js @@ -1,3 +1,5 @@ +'use strict' + const logger = require('../helpers/logger') // --------------------------------------------------------------------------- @@ -6,16 +8,20 @@ module.exports = function (sequelize, DataTypes) { const OAuthToken = sequelize.define('OAuthToken', { accessToken: { - type: DataTypes.STRING + type: DataTypes.STRING, + allowNull: false }, accessTokenExpiresAt: { - type: DataTypes.DATE + type: DataTypes.DATE, + allowNull: false }, refreshToken: { - type: DataTypes.STRING + type: DataTypes.STRING, + allowNull: false }, refreshTokenExpiresAt: { - type: DataTypes.DATE + type: DataTypes.DATE, + allowNull: false } }, { @@ -33,11 +39,6 @@ module.exports = function (sequelize, DataTypes) { return OAuthToken } -// TODO: validation -// OAuthTokenSchema.path('accessToken').required(true) -// OAuthTokenSchema.path('client').required(true) -// OAuthTokenSchema.path('user').required(true) - // --------------------------------------------------------------------------- function associate (models) { diff --git a/server/models/pod.js b/server/models/pod.js index fff6970a7..84f78f200 100644 --- a/server/models/pod.js +++ b/server/models/pod.js @@ -3,6 +3,7 @@ const map = require('lodash/map') const constants = require('../initializers/constants') +const customPodsValidators = require('../helpers/custom-validators').pods // --------------------------------------------------------------------------- @@ -10,14 +11,27 @@ module.exports = function (sequelize, DataTypes) { const Pod = sequelize.define('Pod', { host: { - type: DataTypes.STRING + type: DataTypes.STRING, + allowNull: false, + validate: { + isHost: function (value) { + const res = customPodsValidators.isHostValid(value) + if (res === false) throw new Error('Host not valid.') + } + } }, publicKey: { - type: DataTypes.STRING(5000) + type: DataTypes.STRING(5000), + allowNull: false }, score: { type: DataTypes.INTEGER, - defaultValue: constants.FRIEND_SCORE.BASE + defaultValue: constants.FRIEND_SCORE.BASE, + allowNull: false, + validate: { + isInt: true, + max: constants.FRIEND_SCORE.MAX + } } }, { @@ -42,12 +56,6 @@ module.exports = function (sequelize, DataTypes) { return Pod } -// TODO: max score -> constants.FRIENDS_SCORE.MAX -// TODO: validation -// PodSchema.path('host').validate(validator.isURL) -// PodSchema.path('publicKey').required(true) -// PodSchema.path('score').validate(function (value) { return !isNaN(value) }) - // ------------------------------ METHODS ------------------------------ function toFormatedJSON () { @@ -82,15 +90,17 @@ function incrementScores (ids, value, callback) { score: this.sequelize.literal('score +' + value) } - const query = { + const options = { where: { id: { $in: ids } - } + }, + // In this case score is a literal and not an integer so we do not validate it + validate: false } - return this.update(update, query).asCallback(callback) + return this.update(update, options).asCallback(callback) } function list (callback) { diff --git a/server/models/request.js b/server/models/request.js index 70aa32610..e18f8fe3d 100644 --- a/server/models/request.js +++ b/server/models/request.js @@ -3,6 +3,7 @@ const each = require('async/each') const eachLimit = require('async/eachLimit') const waterfall = require('async/waterfall') +const values = require('lodash/values') const constants = require('../initializers/constants') const logger = require('../helpers/logger') @@ -17,11 +18,12 @@ module.exports = function (sequelize, DataTypes) { const Request = sequelize.define('Request', { request: { - type: DataTypes.JSON + type: DataTypes.JSON, + allowNull: false }, endpoint: { - // TODO: enum? - type: DataTypes.STRING + type: DataTypes.ENUM(values(constants.REQUEST_ENDPOINTS)), + allowNull: false } }, { @@ -196,7 +198,7 @@ function makeRequests () { makeRequest(toPod, requestToMake.endpoint, requestToMake.datas, function (success) { if (success === true) { - logger.debug('Removing requests for %s pod.', requestToMake.toPodId, { requestsIds: requestToMake.ids }) + logger.debug('Removing requests for pod %s.', requestToMake.toPodId, { requestsIds: requestToMake.ids }) goodPods.push(requestToMake.toPodId) @@ -261,13 +263,13 @@ function updatePodsScore (goodPods, badPods) { if (goodPods.length !== 0) { Pod.incrementScores(goodPods, constants.PODS_SCORE.BONUS, function (err) { - if (err) logger.error('Cannot increment scores of good pods.') + if (err) logger.error('Cannot increment scores of good pods.', { error: err }) }) } if (badPods.length !== 0) { Pod.incrementScores(badPods, constants.PODS_SCORE.MALUS, function (err) { - if (err) logger.error('Cannot decrement scores of bad pods.') + if (err) logger.error('Cannot decrement scores of bad pods.', { error: err }) removeBadPods.call(self) }) } diff --git a/server/models/tag.js b/server/models/tag.js index 874e88842..d6c2d3bb1 100644 --- a/server/models/tag.js +++ b/server/models/tag.js @@ -6,7 +6,8 @@ module.exports = function (sequelize, DataTypes) { const Tag = sequelize.define('Tag', { name: { - type: DataTypes.STRING + type: DataTypes.STRING, + allowNull: false } }, { diff --git a/server/models/user.js b/server/models/user.js index e50eb96ea..944986a44 100644 --- a/server/models/user.js +++ b/server/models/user.js @@ -1,5 +1,11 @@ +'use strict' + +const values = require('lodash/values') + const modelUtils = require('./utils') +const constants = require('../initializers/constants') const peertubeCrypto = require('../helpers/peertube-crypto') +const customUsersValidators = require('../helpers/custom-validators').users // --------------------------------------------------------------------------- @@ -7,13 +13,28 @@ module.exports = function (sequelize, DataTypes) { const User = sequelize.define('User', { password: { - type: DataTypes.STRING + type: DataTypes.STRING, + allowNull: false, + validate: { + passwordValid: function (value) { + const res = customUsersValidators.isUserPasswordValid(value) + if (res === false) throw new Error('Password not valid.') + } + } }, username: { - type: DataTypes.STRING + type: DataTypes.STRING, + allowNull: false, + validate: { + usernameValid: function (value) { + const res = customUsersValidators.isUserUsernameValid(value) + if (res === false) throw new Error('Username not valid.') + } + } }, role: { - type: DataTypes.STRING + type: DataTypes.ENUM(values(constants.USER_ROLES)), + allowNull: false } }, { @@ -41,11 +62,6 @@ module.exports = function (sequelize, DataTypes) { return User } -// TODO: Validation -// UserSchema.path('password').required(customUsersValidators.isUserPasswordValid) -// UserSchema.path('username').required(customUsersValidators.isUserUsernameValid) -// UserSchema.path('role').validate(customUsersValidators.isUserRoleValid) - function beforeCreateOrUpdate (user, options, next) { peertubeCrypto.cryptPassword(user.password, function (err, hash) { if (err) return next(err) diff --git a/server/models/video.js b/server/models/video.js index 04478c8d7..3ebc48ad4 100644 --- a/server/models/video.js +++ b/server/models/video.js @@ -8,10 +8,12 @@ const map = require('lodash/map') const parallel = require('async/parallel') const parseTorrent = require('parse-torrent') const pathUtils = require('path') +const values = require('lodash/values') const constants = require('../initializers/constants') const logger = require('../helpers/logger') const modelUtils = require('./utils') +const customVideosValidators = require('../helpers/custom-validators').videos // --------------------------------------------------------------------------- @@ -22,26 +24,61 @@ module.exports = function (sequelize, DataTypes) { id: { type: DataTypes.UUID, defaultValue: DataTypes.UUIDV4, - primaryKey: true + primaryKey: true, + validate: { + isUUID: 4 + } }, name: { - type: DataTypes.STRING + type: DataTypes.STRING, + allowNull: false, + validate: { + nameValid: function (value) { + const res = customVideosValidators.isVideoNameValid(value) + if (res === false) throw new Error('Video name is not valid.') + } + } }, extname: { - // TODO: enum? - type: DataTypes.STRING + type: DataTypes.ENUM(values(constants.CONSTRAINTS_FIELDS.VIDEOS.EXTNAME)), + allowNull: false }, remoteId: { - type: DataTypes.UUID + type: DataTypes.UUID, + allowNull: true, + validate: { + isUUID: 4 + } }, description: { - type: DataTypes.STRING + type: DataTypes.STRING, + allowNull: false, + validate: { + descriptionValid: function (value) { + const res = customVideosValidators.isVideoDescriptionValid(value) + if (res === false) throw new Error('Video description is not valid.') + } + } }, infoHash: { - type: DataTypes.STRING + type: DataTypes.STRING, + allowNull: false, + validate: { + infoHashValid: function (value) { + const res = customVideosValidators.isVideoInfoHashValid(value) + if (res === false) throw new Error('Video info hash is not valid.') + } + } }, duration: { - type: DataTypes.INTEGER + type: DataTypes.INTEGER, + allowNull: false, + validate: { + durationValid: function (value) { + const res = customVideosValidators.isVideoDurationValid(value) + if (res === false) throw new Error('Video duration is not valid.') + } + } } }, { @@ -71,6 +108,7 @@ module.exports = function (sequelize, DataTypes) { toRemoteJSON }, hooks: { + beforeValidate, beforeCreate, afterDestroy } @@ -80,13 +118,14 @@ module.exports = function (sequelize, DataTypes) { return Video } -// TODO: Validation -// VideoSchema.path('name').validate(customVideosValidators.isVideoNameValid) -// VideoSchema.path('description').validate(customVideosValidators.isVideoDescriptionValid) -// VideoSchema.path('podHost').validate(customVideosValidators.isVideoPodHostValid) -// VideoSchema.path('author').validate(customVideosValidators.isVideoAuthorValid) -// VideoSchema.path('duration').validate(customVideosValidators.isVideoDurationValid) -// VideoSchema.path('tags').validate(customVideosValidators.isVideoTagsValid) +function beforeValidate (video, options, next) { + if (video.isOwned()) { + // 40 hexa length + video.infoHash = '0123456789abcdef0123456789abcdef01234567' + } + + return next(null) +} function beforeCreate (video, options, next) { const tasks = [] @@ -113,9 +152,8 @@ function beforeCreate (video, options, next) { if (err) return callback(err) const parsedTorrent = parseTorrent(torrent) - video.infoHash = parsedTorrent.infoHash - - callback(null) + video.set('infoHash', parsedTorrent.infoHash) + video.validate().asCallback(callback) }) }) }, From 319d072e8eb7266cd8d33e0bb2fb5ebe76c487d1 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 29 Dec 2016 09:33:28 +0100 Subject: [PATCH 19/47] Server: Add postgresql indexes --- server/models/author.js | 8 ++++++++ server/models/oauth-client.js | 22 ++++++++++------------ server/models/oauth-token.js | 24 ++++++++++++++++++++++++ server/models/pod.js | 8 ++++++++ server/models/requestToPod.js | 12 ++++++++++++ server/models/tag.js | 7 +++++++ server/models/user.js | 5 +++++ server/models/video.js | 20 ++++++++++++++++++++ server/models/videoTag.js | 11 ++++++++++- 9 files changed, 104 insertions(+), 13 deletions(-) diff --git a/server/models/author.js b/server/models/author.js index e0ac868ea..8f5b598c8 100644 --- a/server/models/author.js +++ b/server/models/author.js @@ -17,6 +17,14 @@ module.exports = function (sequelize, DataTypes) { } }, { + indexes: [ + { + fields: [ 'name' ] + }, + { + fields: [ 'podId' ] + } + ], classMethods: { associate } diff --git a/server/models/oauth-client.js b/server/models/oauth-client.js index b56838d4c..758c4cf2f 100644 --- a/server/models/oauth-client.js +++ b/server/models/oauth-client.js @@ -19,9 +19,17 @@ module.exports = function (sequelize, DataTypes) { } }, { + indexes: [ + { + fields: [ 'clientId' ], + unique: true + }, + { + fields: [ 'clientId', 'clientSecret' ], + unique: true + } + ], classMethods: { - associate, - getByIdAndSecret, list, loadFirstClient @@ -34,16 +42,6 @@ module.exports = function (sequelize, DataTypes) { // --------------------------------------------------------------------------- -function associate (models) { - this.hasMany(models.OAuthToken, { - foreignKey: { - name: 'oAuthClientId', - allowNull: false - }, - onDelete: 'cascade' - }) -} - function list (callback) { return this.findAll().asCallback(callback) } diff --git a/server/models/oauth-token.js b/server/models/oauth-token.js index f8de4e916..68e7c9ff7 100644 --- a/server/models/oauth-token.js +++ b/server/models/oauth-token.js @@ -25,6 +25,22 @@ module.exports = function (sequelize, DataTypes) { } }, { + indexes: [ + { + fields: [ 'refreshToken' ], + unique: true + }, + { + fields: [ 'accessToken' ], + unique: true + }, + { + fields: [ 'userId' ] + }, + { + fields: [ 'oAuthClientId' ] + } + ], classMethods: { associate, @@ -49,6 +65,14 @@ function associate (models) { }, onDelete: 'cascade' }) + + this.belongsTo(models.OAuthClient, { + foreignKey: { + name: 'oAuthClientId', + allowNull: false + }, + onDelete: 'cascade' + }) } function getByRefreshTokenAndPopulateClient (refreshToken) { diff --git a/server/models/pod.js b/server/models/pod.js index 84f78f200..83ecd732e 100644 --- a/server/models/pod.js +++ b/server/models/pod.js @@ -35,6 +35,14 @@ module.exports = function (sequelize, DataTypes) { } }, { + indexes: [ + { + fields: [ 'host' ] + }, + { + fields: [ 'score' ] + } + ], classMethods: { associate, diff --git a/server/models/requestToPod.js b/server/models/requestToPod.js index 378c2bdcf..f42a53458 100644 --- a/server/models/requestToPod.js +++ b/server/models/requestToPod.js @@ -4,6 +4,18 @@ module.exports = function (sequelize, DataTypes) { const RequestToPod = sequelize.define('RequestToPod', {}, { + indexes: [ + { + fields: [ 'requestId' ] + }, + { + fields: [ 'podId' ] + }, + { + fields: [ 'requestId', 'podId' ], + unique: true + } + ], classMethods: { removePodOf } diff --git a/server/models/tag.js b/server/models/tag.js index d6c2d3bb1..27eecdc84 100644 --- a/server/models/tag.js +++ b/server/models/tag.js @@ -11,6 +11,13 @@ module.exports = function (sequelize, DataTypes) { } }, { + timestamps: false, + indexes: [ + { + fields: [ 'name' ], + unique: true + } + ], classMethods: { associate } diff --git a/server/models/user.js b/server/models/user.js index 944986a44..631cd96c9 100644 --- a/server/models/user.js +++ b/server/models/user.js @@ -38,6 +38,11 @@ module.exports = function (sequelize, DataTypes) { } }, { + indexes: [ + { + fields: [ 'username' ] + } + ], classMethods: { associate, diff --git a/server/models/video.js b/server/models/video.js index 3ebc48ad4..d1595ce51 100644 --- a/server/models/video.js +++ b/server/models/video.js @@ -82,6 +82,26 @@ module.exports = function (sequelize, DataTypes) { } }, { + indexes: [ + { + fields: [ 'authorId' ] + }, + { + fields: [ 'remoteId' ] + }, + { + fields: [ 'name' ] + }, + { + fields: [ 'createdAt' ] + }, + { + fields: [ 'duration' ] + }, + { + fields: [ 'infoHash' ] + } + ], classMethods: { associate, diff --git a/server/models/videoTag.js b/server/models/videoTag.js index 0f2b20838..cd9277a6e 100644 --- a/server/models/videoTag.js +++ b/server/models/videoTag.js @@ -3,7 +3,16 @@ // --------------------------------------------------------------------------- module.exports = function (sequelize, DataTypes) { - const VideoTag = sequelize.define('VideoTag', {}, {}) + const VideoTag = sequelize.define('VideoTag', {}, { + indexes: [ + { + fields: [ 'videoId' ] + }, + { + fields: [ 'tagId' ] + } + ] + }) return VideoTag } From 4712081f2a5f48749cf125d729e78b926ab28d6d Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 29 Dec 2016 10:33:36 +0100 Subject: [PATCH 20/47] Server: add association between author and user --- server/controllers/api/remote.js | 6 ++++-- server/controllers/api/videos.js | 17 ++++++++++------- server/initializers/checker.js | 4 ++-- server/models/author.js | 11 +++++++++++ server/models/oauth-client.js | 6 +++--- server/models/user.js | 5 +++++ 6 files changed, 35 insertions(+), 14 deletions(-) diff --git a/server/controllers/api/remote.js b/server/controllers/api/remote.js index 2cf916ff3..94d6e740e 100644 --- a/server/controllers/api/remote.js +++ b/server/controllers/api/remote.js @@ -84,11 +84,13 @@ function addRemoteVideo (videoToCreateData, fromHost, finalCallback) { const query = { where: { name: username, - podId: pod.id + podId: pod.id, + userId: null }, defaults: { name: username, - podId: pod.id + podId: pod.id, + userId: null }, transaction: t } diff --git a/server/controllers/api/videos.js b/server/controllers/api/videos.js index 992f03db0..170224634 100644 --- a/server/controllers/api/videos.js +++ b/server/controllers/api/videos.js @@ -95,23 +95,26 @@ function addVideo (req, res, next) { }, function findOrCreateAuthor (t, callback) { - const username = res.locals.oauth.token.user.username + const user = res.locals.oauth.token.User const query = { where: { - name: username, - podId: null + name: user.username, + podId: null, + userId: user.id }, defaults: { - name: username, - podId: null // null because it is OUR pod + name: user.username, + podId: null, // null because it is OUR pod + userId: user.id }, transaction: t } db.Author.findOrCreate(query).asCallback(function (err, result) { - // [ instance, wasCreated ] - return callback(err, t, result[0]) + const authorInstance = result[0] + + return callback(err, t, authorInstance) }) }, diff --git a/server/initializers/checker.js b/server/initializers/checker.js index 2753604dc..6471bb4f1 100644 --- a/server/initializers/checker.js +++ b/server/initializers/checker.js @@ -42,10 +42,10 @@ function checkMissedConfig () { } function clientsExist (callback) { - db.OAuthClient.list(function (err, clients) { + db.OAuthClient.countTotal(function (err, totalClients) { if (err) return callback(err) - return callback(null, clients.length !== 0) + return callback(null, totalClients !== 0) }) } diff --git a/server/models/author.js b/server/models/author.js index 8f5b598c8..5835ada99 100644 --- a/server/models/author.js +++ b/server/models/author.js @@ -23,6 +23,9 @@ module.exports = function (sequelize, DataTypes) { }, { fields: [ 'podId' ] + }, + { + fields: [ 'userId' ] } ], classMethods: { @@ -44,4 +47,12 @@ function associate (models) { }, onDelete: 'cascade' }) + + this.belongsTo(models.User, { + foreignKey: { + name: 'userId', + allowNull: true + }, + onDelete: 'cascade' + }) } diff --git a/server/models/oauth-client.js b/server/models/oauth-client.js index 758c4cf2f..021a34007 100644 --- a/server/models/oauth-client.js +++ b/server/models/oauth-client.js @@ -30,8 +30,8 @@ module.exports = function (sequelize, DataTypes) { } ], classMethods: { + countTotal, getByIdAndSecret, - list, loadFirstClient } } @@ -42,8 +42,8 @@ module.exports = function (sequelize, DataTypes) { // --------------------------------------------------------------------------- -function list (callback) { - return this.findAll().asCallback(callback) +function countTotal (callback) { + return this.count().asCallback(callback) } function loadFirstClient (callback) { diff --git a/server/models/user.js b/server/models/user.js index 631cd96c9..36ed723cc 100644 --- a/server/models/user.js +++ b/server/models/user.js @@ -94,6 +94,11 @@ function toFormatedJSON () { // ------------------------------ STATICS ------------------------------ function associate (models) { + this.hasOne(models.Author, { + foreignKey: 'userId', + onDelete: 'cascade' + }) + this.hasMany(models.OAuthToken, { foreignKey: 'userId', onDelete: 'cascade' From 98ac898a03ed7bbb4edec74fe823b3f2d6d4904a Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 29 Dec 2016 11:17:11 +0100 Subject: [PATCH 21/47] Server: use video hook to send information to other pods when a video is deleted --- server/controllers/api/pods.js | 2 +- server/controllers/api/users.js | 32 ++------------------------------ server/controllers/api/videos.js | 24 ++++++------------------ server/models/video.js | 14 ++++++++++++++ 4 files changed, 23 insertions(+), 49 deletions(-) diff --git a/server/controllers/api/pods.js b/server/controllers/api/pods.js index 79f3f9d8d..d9279f1d9 100644 --- a/server/controllers/api/pods.js +++ b/server/controllers/api/pods.js @@ -113,7 +113,7 @@ function removePods (req, res, next) { db.Pod.loadByHost(host, callback) }, - function removePod (pod, callback) { + function deletePod (pod, callback) { pod.destroy().asCallback(callback) } ], function (err) { diff --git a/server/controllers/api/users.js b/server/controllers/api/users.js index 890028b36..e4423680c 100644 --- a/server/controllers/api/users.js +++ b/server/controllers/api/users.js @@ -90,39 +90,11 @@ function listUsers (req, res, next) { function removeUser (req, res, next) { waterfall([ - function getUser (callback) { + function loadUser (callback) { db.User.loadById(req.params.id, callback) }, - // TODO: use foreignkey? - function getVideos (user, callback) { - db.Video.listOwnedByAuthor(user.username, function (err, videos) { - return callback(err, user, videos) - }) - }, - - function removeVideosFromDB (user, videos, callback) { - each(videos, function (video, callbackEach) { - video.destroy().asCallback(callbackEach) - }, function (err) { - return callback(err, user, videos) - }) - }, - - function sendInformationToFriends (user, videos, callback) { - videos.forEach(function (video) { - const params = { - name: video.name, - remoteId: video.id - } - - friends.removeVideoToFriends(params) - }) - - return callback(null, user) - }, - - function removeUserFromDB (user, callback) { + function deleteUser (user, callback) { user.destroy().asCallback(callback) } ], function andFinally (err) { diff --git a/server/controllers/api/videos.js b/server/controllers/api/videos.js index 170224634..ddf85d77d 100644 --- a/server/controllers/api/videos.js +++ b/server/controllers/api/videos.js @@ -249,27 +249,15 @@ function removeVideo (req, res, next) { const videoId = req.params.id waterfall([ - function getVideo (callback) { - db.Video.load(videoId, callback) - }, - - function removeFromDB (video, callback) { - video.destroy().asCallback(function (err) { - if (err) return callback(err) - - return callback(null, video) + function loadVideo (callback) { + db.Video.load(videoId, function (err, video) { + return callback(err, video) }) }, - function sendInformationToFriends (video, callback) { - const params = { - name: video.name, - remoteId: video.id - } - - friends.removeVideoToFriends(params) - - return callback(null) + function deleteVideo (video, callback) { + // Informations to other pods will be sent by the afterDestroy video hook + video.destroy().asCallback(callback) } ], function andFinally (err) { if (err) { diff --git a/server/models/video.js b/server/models/video.js index d1595ce51..564e362fd 100644 --- a/server/models/video.js +++ b/server/models/video.js @@ -12,6 +12,7 @@ const values = require('lodash/values') const constants = require('../initializers/constants') const logger = require('../helpers/logger') +const friends = require('../lib/friends') const modelUtils = require('./utils') const customVideosValidators = require('../helpers/custom-validators').videos @@ -205,11 +206,24 @@ function afterDestroy (video, options, next) { function (callback) { removeFile(video, callback) }, + function (callback) { removeTorrent(video, callback) }, + function (callback) { removePreview(video, callback) + }, + + function (callback) { + const params = { + name: video.name, + remoteId: video.id + } + + friends.removeVideoToFriends(params) + + return callback() } ) } From 4d32448895ad29ef694bcf790d59253249ad5939 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 29 Dec 2016 12:13:19 +0100 Subject: [PATCH 22/47] Server: use binary data instead of base64 to send thumbnails --- package.json | 1 + server/controllers/api/remote.js | 4 ++-- server/controllers/api/users.js | 2 -- server/helpers/custom-validators/videos.js | 9 ++++----- server/initializers/constants.js | 2 +- server/models/video.js | 11 ++++++----- 6 files changed, 14 insertions(+), 15 deletions(-) diff --git a/package.json b/package.json index bff21082f..5eadcc363 100644 --- a/package.json +++ b/package.json @@ -66,6 +66,7 @@ "request": "^2.57.0", "request-replay": "^1.0.2", "rimraf": "^2.5.4", + "safe-buffer": "^5.0.1", "scripty": "^1.5.0", "sequelize": "^3.27.0", "ursa": "^0.9.1", diff --git a/server/controllers/api/remote.js b/server/controllers/api/remote.js index 94d6e740e..ac850c2d2 100644 --- a/server/controllers/api/remote.js +++ b/server/controllers/api/remote.js @@ -147,9 +147,9 @@ function addRemoteVideo (videoToCreateData, fromHost, finalCallback) { }, function generateThumbnail (t, tagInstances, video, callback) { - db.Video.generateThumbnailFromBase64(video, videoToCreateData.thumbnailBase64, function (err) { + db.Video.generateThumbnailFromData(video, videoToCreateData.thumbnailData, function (err) { if (err) { - logger.error('Cannot generate thumbnail from base 64 data.', { error: err }) + logger.error('Cannot generate thumbnail from data.', { error: err }) return callback(err) } diff --git a/server/controllers/api/users.js b/server/controllers/api/users.js index e4423680c..53bf56790 100644 --- a/server/controllers/api/users.js +++ b/server/controllers/api/users.js @@ -1,12 +1,10 @@ 'use strict' -const each = require('async/each') const express = require('express') const waterfall = require('async/waterfall') const constants = require('../../initializers/constants') const db = require('../../initializers/database') -const friends = require('../../lib/friends') const logger = require('../../helpers/logger') const middlewares = require('../../middlewares') const admin = middlewares.admin diff --git a/server/helpers/custom-validators/videos.js b/server/helpers/custom-validators/videos.js index da857ba5f..4aaa6aaa9 100644 --- a/server/helpers/custom-validators/videos.js +++ b/server/helpers/custom-validators/videos.js @@ -17,7 +17,7 @@ const videosValidators = { isVideoNameValid, isVideoTagsValid, isVideoThumbnailValid, - isVideoThumbnail64Valid + isVideoThumbnailDataValid } function isEachRemoteVideosValid (requests) { @@ -33,7 +33,7 @@ function isEachRemoteVideosValid (requests) { isVideoInfoHashValid(video.infoHash) && isVideoNameValid(video.name) && isVideoTagsValid(video.tags) && - isVideoThumbnail64Valid(video.thumbnailBase64) && + isVideoThumbnailDataValid(video.thumbnailData) && isVideoRemoteIdValid(video.remoteId) && isVideoExtnameValid(video.extname) ) || @@ -86,9 +86,8 @@ function isVideoThumbnailValid (value) { return validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.THUMBNAIL) } -function isVideoThumbnail64Valid (value) { - return validator.isBase64(value) && - validator.isByteLength(value, VIDEOS_CONSTRAINTS_FIELDS.THUMBNAIL64) +function isVideoThumbnailDataValid (value) { + return validator.isByteLength(value, VIDEOS_CONSTRAINTS_FIELDS.THUMBNAIL_DATA) } function isVideoRemoteIdValid (value) { diff --git a/server/initializers/constants.js b/server/initializers/constants.js index 0af7aca3c..474a37277 100644 --- a/server/initializers/constants.js +++ b/server/initializers/constants.js @@ -74,7 +74,7 @@ const CONSTRAINTS_FIELDS = { TAGS: { min: 1, max: 3 }, // Number of total tags TAG: { min: 2, max: 10 }, // Length THUMBNAIL: { min: 2, max: 30 }, - THUMBNAIL64: { min: 0, max: 20000 } // Bytes + THUMBNAIL_DATA: { min: 0, max: 20000 } // Bytes } } diff --git a/server/models/video.js b/server/models/video.js index 564e362fd..0e84e8986 100644 --- a/server/models/video.js +++ b/server/models/video.js @@ -1,5 +1,6 @@ 'use strict' +const Buffer = require('safe-buffer').Buffer const createTorrent = require('create-torrent') const ffmpeg = require('fluent-ffmpeg') const fs = require('fs') @@ -106,7 +107,7 @@ module.exports = function (sequelize, DataTypes) { classMethods: { associate, - generateThumbnailFromBase64, + generateThumbnailFromData, getDurationFromFile, list, listForApi, @@ -336,7 +337,7 @@ function toFormatedJSON () { function toRemoteJSON (callback) { const self = this - // Convert thumbnail to base64 + // Get thumbnail data to send to the other pod const thumbnailPath = pathUtils.join(constants.CONFIG.STORAGE.THUMBNAILS_DIR, this.getThumbnailName()) fs.readFile(thumbnailPath, function (err, thumbnailData) { if (err) { @@ -351,7 +352,7 @@ function toRemoteJSON (callback) { remoteId: self.id, author: self.Author.name, duration: self.duration, - thumbnailBase64: new Buffer(thumbnailData).toString('base64'), + thumbnailData: thumbnailData.toString('binary'), tags: map(self.Tags, 'name'), createdAt: self.createdAt, extname: self.extname @@ -363,12 +364,12 @@ function toRemoteJSON (callback) { // ------------------------------ STATICS ------------------------------ -function generateThumbnailFromBase64 (video, thumbnailData, callback) { +function generateThumbnailFromData (video, thumbnailData, callback) { // Creating the thumbnail for a remote video const thumbnailName = video.getThumbnailName() const thumbnailPath = constants.CONFIG.STORAGE.THUMBNAILS_DIR + thumbnailName - fs.writeFile(thumbnailPath, thumbnailData, { encoding: 'base64' }, function (err) { + fs.writeFile(thumbnailPath, Buffer.from(thumbnailData, 'binary'), function (err) { if (err) return callback(err) return callback(null, thumbnailName) From d396a937b642d616beb72dde54c0c2d37c7e8c30 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 29 Dec 2016 17:34:29 +0100 Subject: [PATCH 23/47] Client: upgrade angular dep' --- client/package.json | 18 +++++++++--------- client/src/app/shared/shared.module.ts | 19 ++++++++++--------- .../video-watch/video-magnet.component.ts | 2 +- .../video-watch/video-share.component.ts | 2 +- client/src/vendor.ts | 8 ++++---- 5 files changed, 25 insertions(+), 24 deletions(-) diff --git a/client/package.json b/client/package.json index c31f04f7b..e9f41959d 100644 --- a/client/package.json +++ b/client/package.json @@ -18,14 +18,14 @@ }, "license": "GPLv3", "dependencies": { - "@angular/common": "~2.3.0", - "@angular/compiler": "~2.3.0", - "@angular/core": "~2.3.0", - "@angular/forms": "~2.3.0", - "@angular/http": "~2.3.0", - "@angular/platform-browser": "~2.3.0", - "@angular/platform-browser-dynamic": "~2.3.0", - "@angular/router": "~3.3.0", + "@angular/common": "~2.4.1", + "@angular/compiler": "~2.4.1", + "@angular/core": "~2.4.1", + "@angular/forms": "~2.4.1", + "@angular/http": "~2.4.1", + "@angular/platform-browser": "~2.4.1", + "@angular/platform-browser-dynamic": "~2.4.1", + "@angular/router": "~3.4.1", "@angularclass/hmr": "^1.2.0", "@angularclass/hmr-loader": "^3.0.2", "@types/core-js": "^0.9.28", @@ -51,7 +51,7 @@ "ie-shim": "^0.1.0", "intl": "^1.2.4", "json-loader": "^0.5.4", - "ng2-bootstrap": "1.1.16", + "ng2-bootstrap": "1.1.16-10", "ng2-file-upload": "^1.1.0", "ng2-meta": "^2.0.0", "node-sass": "^3.10.0", diff --git a/client/src/app/shared/shared.module.ts b/client/src/app/shared/shared.module.ts index 141922322..748c5d520 100644 --- a/client/src/app/shared/shared.module.ts +++ b/client/src/app/shared/shared.module.ts @@ -5,10 +5,10 @@ import { FormsModule, ReactiveFormsModule } from '@angular/forms'; import { RouterModule } from '@angular/router'; import { BytesPipe } from 'angular-pipes/src/math/bytes.pipe'; -import { DropdownModule } from 'ng2-bootstrap/components/dropdown'; -import { ProgressbarModule } from 'ng2-bootstrap/components/progressbar'; -import { PaginationModule } from 'ng2-bootstrap/components/pagination'; -import { ModalModule } from 'ng2-bootstrap/components/modal'; +import { DropdownModule } from 'ng2-bootstrap/dropdown'; +import { ProgressbarModule } from 'ng2-bootstrap/progressbar'; +import { PaginationModule } from 'ng2-bootstrap/pagination'; +import { ModalModule } from 'ng2-bootstrap/modal'; import { FileUploadModule } from 'ng2-file-upload/ng2-file-upload'; import { AUTH_HTTP_PROVIDERS } from './auth'; @@ -23,11 +23,12 @@ import { SearchComponent, SearchService } from './search'; HttpModule, RouterModule, - DropdownModule, - FileUploadModule, - ModalModule, - PaginationModule, - ProgressbarModule + DropdownModule.forRoot(), + ModalModule.forRoot(), + PaginationModule.forRoot(), + ProgressbarModule.forRoot(), + + FileUploadModule ], declarations: [ diff --git a/client/src/app/videos/video-watch/video-magnet.component.ts b/client/src/app/videos/video-watch/video-magnet.component.ts index 2894e7df6..8bee848a4 100644 --- a/client/src/app/videos/video-watch/video-magnet.component.ts +++ b/client/src/app/videos/video-watch/video-magnet.component.ts @@ -1,6 +1,6 @@ import { Component, Input, ViewChild } from '@angular/core'; -import { ModalDirective } from 'ng2-bootstrap/components/modal'; +import { ModalDirective } from 'ng2-bootstrap/modal'; import { Video } from '../shared'; diff --git a/client/src/app/videos/video-watch/video-share.component.ts b/client/src/app/videos/video-watch/video-share.component.ts index 8e6de1294..0b85052cd 100644 --- a/client/src/app/videos/video-watch/video-share.component.ts +++ b/client/src/app/videos/video-watch/video-share.component.ts @@ -1,6 +1,6 @@ import { Component, Input, ViewChild } from '@angular/core'; -import { ModalDirective } from 'ng2-bootstrap/components/modal'; +import { ModalDirective } from 'ng2-bootstrap/modal'; import { Video } from '../shared'; diff --git a/client/src/vendor.ts b/client/src/vendor.ts index 760fc8257..436c58f48 100644 --- a/client/src/vendor.ts +++ b/client/src/vendor.ts @@ -29,7 +29,7 @@ import 'angular-pipes/src/math/bytes.pipe'; import 'ng2-file-upload'; import 'video.js'; import 'ng2-meta'; -import 'ng2-bootstrap/components/pagination'; -import 'ng2-bootstrap/components/dropdown'; -import 'ng2-bootstrap/components/progressbar'; -import 'ng2-bootstrap/components/modal'; +import 'ng2-bootstrap/pagination'; +import 'ng2-bootstrap/dropdown'; +import 'ng2-bootstrap/progressbar'; +import 'ng2-bootstrap/modal'; From 4ff0d86208dafbdd07beb6286fd93c795db8a95f Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 29 Dec 2016 18:02:03 +0100 Subject: [PATCH 24/47] Server: little refractoring --- server/controllers/api/remote.js | 79 ++++++-------------------------- server/controllers/api/videos.js | 44 +++--------------- server/middlewares/secure.js | 4 ++ server/models/author.js | 29 +++++++++++- server/models/tag.js | 40 +++++++++++++++- 5 files changed, 92 insertions(+), 104 deletions(-) diff --git a/server/controllers/api/remote.js b/server/controllers/api/remote.js index ac850c2d2..929ade555 100644 --- a/server/controllers/api/remote.js +++ b/server/controllers/api/remote.js @@ -28,7 +28,7 @@ module.exports = router function remoteVideos (req, res, next) { const requests = req.body.data - const fromHost = req.body.signature.host + const fromPod = res.locals.secure.pod // We need to process in the same order to keep consistency // TODO: optimization @@ -36,9 +36,9 @@ function remoteVideos (req, res, next) { const videoData = request.data if (request.type === 'add') { - addRemoteVideo(videoData, fromHost, callbackEach) + addRemoteVideo(videoData, fromPod, callbackEach) } else if (request.type === 'remove') { - removeRemoteVideo(videoData, fromHost, callbackEach) + removeRemoteVideo(videoData, fromPod, callbackEach) } else { logger.error('Unkown remote request type %s.', request.type) } @@ -50,7 +50,7 @@ function remoteVideos (req, res, next) { return res.type('json').status(204).end() } -function addRemoteVideo (videoToCreateData, fromHost, finalCallback) { +function addRemoteVideo (videoToCreateData, fromPod, finalCallback) { logger.debug('Adding remote video "%s".', videoToCreateData.name) waterfall([ @@ -61,70 +61,21 @@ function addRemoteVideo (videoToCreateData, fromHost, finalCallback) { }) }, - function findOrCreatePod (t, callback) { - const query = { - where: { - host: fromHost - }, - defaults: { - host: fromHost - }, - transaction: t - } + function findOrCreateAuthor (t, callback) { + const name = videoToCreateData.author + const podId = fromPod.id + // This author is from another pod so we do not associate a user + const userId = null - db.Pod.findOrCreate(query).asCallback(function (err, result) { - // [ instance, wasCreated ] - return callback(err, t, result[0]) - }) - }, - - function findOrCreateAuthor (t, pod, callback) { - const username = videoToCreateData.author - - const query = { - where: { - name: username, - podId: pod.id, - userId: null - }, - defaults: { - name: username, - podId: pod.id, - userId: null - }, - transaction: t - } - - db.Author.findOrCreate(query).asCallback(function (err, result) { - // [ instance, wasCreated ] - return callback(err, t, result[0]) + db.Author.findOrCreateAuthor(name, podId, userId, t, function (err, authorInstance) { + return callback(err, t, authorInstance) }) }, function findOrCreateTags (t, author, callback) { const tags = videoToCreateData.tags - const tagInstances = [] - each(tags, function (tag, callbackEach) { - const query = { - where: { - name: tag - }, - defaults: { - name: tag - }, - transaction: t - } - - db.Tag.findOrCreate(query).asCallback(function (err, res) { - if (err) return callbackEach(err) - - // res = [ tag, isCreated ] - const tag = res[0] - tagInstances.push(tag) - return callbackEach() - }) - }, function (err) { + db.Tag.findOrCreateTags(tags, t, function (err, tagInstances) { return callback(err, t, author, tagInstances) }) }, @@ -192,18 +143,18 @@ function addRemoteVideo (videoToCreateData, fromHost, finalCallback) { }) } -function removeRemoteVideo (videoToRemoveData, fromHost, callback) { +function removeRemoteVideo (videoToRemoveData, fromPod, callback) { // TODO: use bulkDestroy? // We need the list because we have to remove some other stuffs (thumbnail etc) - db.Video.listByHostAndRemoteId(fromHost, videoToRemoveData.remoteId, function (err, videosList) { + db.Video.listByHostAndRemoteId(fromPod.host, videoToRemoveData.remoteId, function (err, videosList) { if (err) { logger.error('Cannot list videos from host and remote id.', { error: err.message }) return callback(err) } if (videosList.length === 0) { - logger.error('No remote video was found for this pod.', { remoteId: videoToRemoveData.remoteId, podHost: fromHost }) + logger.error('No remote video was found for this pod.', { remoteId: videoToRemoveData.remoteId, podHost: fromPod.host }) } each(videosList, function (video, callbackEach) { diff --git a/server/controllers/api/videos.js b/server/controllers/api/videos.js index ddf85d77d..f29edac74 100644 --- a/server/controllers/api/videos.js +++ b/server/controllers/api/videos.js @@ -1,6 +1,5 @@ 'use strict' -const each = require('async/each') const express = require('express') const fs = require('fs') const multer = require('multer') @@ -97,51 +96,20 @@ function addVideo (req, res, next) { function findOrCreateAuthor (t, callback) { const user = res.locals.oauth.token.User - const query = { - where: { - name: user.username, - podId: null, - userId: user.id - }, - defaults: { - name: user.username, - podId: null, // null because it is OUR pod - userId: user.id - }, - transaction: t - } - - db.Author.findOrCreate(query).asCallback(function (err, result) { - const authorInstance = result[0] + const name = user.username + // null because it is OUR pod + const podId = null + const userId = user.id + db.Author.findOrCreateAuthor(name, podId, userId, t, function (err, authorInstance) { return callback(err, t, authorInstance) }) }, function findOrCreateTags (t, author, callback) { const tags = videoInfos.tags - const tagInstances = [] - each(tags, function (tag, callbackEach) { - const query = { - where: { - name: tag - }, - defaults: { - name: tag - }, - transaction: t - } - - db.Tag.findOrCreate(query).asCallback(function (err, res) { - if (err) return callbackEach(err) - - // res = [ tag, isCreated ] - const tag = res[0] - tagInstances.push(tag) - return callbackEach() - }) - }, function (err) { + db.Tag.findOrCreateTags(tags, t, function (err, tagInstances) { return callback(err, t, author, tagInstances) }) }, diff --git a/server/middlewares/secure.js b/server/middlewares/secure.js index b7b4cdfb4..2aae715c4 100644 --- a/server/middlewares/secure.js +++ b/server/middlewares/secure.js @@ -26,6 +26,10 @@ function checkSignature (req, res, next) { const signatureOk = peertubeCrypto.checkSignature(pod.publicKey, host, req.body.signature.signature) if (signatureOk === true) { + res.locals.secure = { + pod + } + return next() } diff --git a/server/models/author.js b/server/models/author.js index 5835ada99..7d15fb6ec 100644 --- a/server/models/author.js +++ b/server/models/author.js @@ -29,7 +29,9 @@ module.exports = function (sequelize, DataTypes) { } ], classMethods: { - associate + associate, + + findOrCreateAuthor } } ) @@ -56,3 +58,28 @@ function associate (models) { onDelete: 'cascade' }) } + +function findOrCreateAuthor (name, podId, userId, transaction, callback) { + if (!callback) { + callback = transaction + transaction = null + } + + const author = { + name, + podId, + userId + } + + const query = { + where: author, + defaults: author + } + + if (transaction) query.transaction = transaction + + this.findOrCreate(query).asCallback(function (err, result) { + // [ instance, wasCreated ] + return callback(err, result[0]) + }) +} diff --git a/server/models/tag.js b/server/models/tag.js index 27eecdc84..145e090c1 100644 --- a/server/models/tag.js +++ b/server/models/tag.js @@ -1,5 +1,7 @@ 'use strict' +const each = require('async/each') + // --------------------------------------------------------------------------- module.exports = function (sequelize, DataTypes) { @@ -19,7 +21,9 @@ module.exports = function (sequelize, DataTypes) { } ], classMethods: { - associate + associate, + + findOrCreateTags } } ) @@ -36,3 +40,37 @@ function associate (models) { onDelete: 'cascade' }) } + +function findOrCreateTags (tags, transaction, callback) { + if (!callback) { + callback = transaction + transaction = null + } + + const self = this + const tagInstances = [] + + each(tags, function (tag, callbackEach) { + const query = { + where: { + name: tag + }, + defaults: { + name: tag + } + } + + if (transaction) query.transaction = transaction + + self.findOrCreate(query).asCallback(function (err, res) { + if (err) return callbackEach(err) + + // res = [ tag, isCreated ] + const tag = res[0] + tagInstances.push(tag) + return callbackEach() + }) + }, function (err) { + return callback(err, tagInstances) + }) +} From 7b1f49de22c40ae121ddb3c399b2540ba56fd414 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 29 Dec 2016 19:07:05 +0100 Subject: [PATCH 25/47] Server: add ability to update a video --- server/controllers/api/videos.js | 85 +++++++++++++++++++- server/lib/friends.js | 7 +- server/middlewares/validators/videos.js | 41 +++++++--- server/models/video.js | 21 ++++- server/tests/api/check-params.js | 101 ++++++++++++++++++++++++ server/tests/api/single-pod.js | 76 ++++++++++++++++++ server/tests/utils/videos.js | 28 ++++++- 7 files changed, 344 insertions(+), 15 deletions(-) diff --git a/server/controllers/api/videos.js b/server/controllers/api/videos.js index f29edac74..1b306d1cf 100644 --- a/server/controllers/api/videos.js +++ b/server/controllers/api/videos.js @@ -50,6 +50,12 @@ router.get('/', pagination.setPagination, listVideos ) +router.put('/:id', + oAuth.authenticate, + reqFiles, + validatorsVideos.videosUpdate, + updateVideo +) router.post('/', oAuth.authenticate, reqFiles, @@ -165,7 +171,7 @@ function addVideo (req, res, next) { }, function sendToFriends (t, video, callback) { - video.toRemoteJSON(function (err, remoteVideo) { + video.toAddRemoteJSON(function (err, remoteVideo) { if (err) return callback(err) // Now we'll add the video's meta data to our friends @@ -193,6 +199,83 @@ function addVideo (req, res, next) { }) } +function updateVideo (req, res, next) { + let videoInstance = res.locals.video + const videoInfosToUpdate = req.body + + waterfall([ + + function startTransaction (callback) { + db.sequelize.transaction().asCallback(function (err, t) { + return callback(err, t) + }) + }, + + function findOrCreateTags (t, callback) { + if (videoInfosToUpdate.tags) { + db.Tag.findOrCreateTags(videoInfosToUpdate.tags, t, function (err, tagInstances) { + return callback(err, t, tagInstances) + }) + } else { + return callback(null, t, null) + } + }, + + function updateVideoIntoDB (t, tagInstances, callback) { + const options = { transaction: t } + + if (videoInfosToUpdate.name) videoInstance.set('name', videoInfosToUpdate.name) + if (videoInfosToUpdate.description) videoInstance.set('description', videoInfosToUpdate.description) + + // Add tags association + videoInstance.save(options).asCallback(function (err) { + if (err) return callback(err) + + return callback(err, t, tagInstances) + }) + }, + + function associateTagsToVideo (t, tagInstances, callback) { + if (tagInstances) { + const options = { transaction: t } + + videoInstance.setTags(tagInstances, options).asCallback(function (err) { + videoInstance.Tags = tagInstances + + return callback(err, t) + }) + } else { + return callback(null, t) + } + }, + + function sendToFriends (t, callback) { + const json = videoInstance.toUpdateRemoteJSON() + + // Now we'll update the video's meta data to our friends + friends.updateVideoToFriends(json) + + return callback(null, t) + } + + ], function andFinally (err, t) { + if (err) { + logger.error('Cannot insert the video.') + + // Abort transaction? + if (t) t.rollback() + + return next(err) + } + + // Commit transaction + t.commit() + + // TODO : include Location of the new video -> 201 + return res.type('json').status(204).end() + }) +} + function getVideo (req, res, next) { db.Video.loadAndPopulateAuthorAndPodAndTags(req.params.id, function (err, video) { if (err) return next(err) diff --git a/server/lib/friends.js b/server/lib/friends.js index ad9e4fdae..589b79660 100644 --- a/server/lib/friends.js +++ b/server/lib/friends.js @@ -14,6 +14,7 @@ const requests = require('../helpers/requests') const friends = { addVideoToFriends, + updateVideoToFriends, hasFriends, getMyCertificate, makeFriends, @@ -26,6 +27,10 @@ function addVideoToFriends (video) { createRequest('add', constants.REQUEST_ENDPOINTS.VIDEOS, video) } +function updateVideoToFriends (video) { + createRequest('update', constants.REQUEST_ENDPOINTS.VIDEOS, video) +} + function hasFriends (callback) { db.Pod.countAll(function (err, count) { if (err) return callback(err) @@ -127,7 +132,7 @@ function sendOwnedVideosToPod (podId) { } videosList.forEach(function (video) { - video.toRemoteJSON(function (err, remoteVideo) { + video.toAddRemoteJSON(function (err, remoteVideo) { if (err) { logger.error('Cannot convert video to remote.', { error: err }) // Don't break the process diff --git a/server/middlewares/validators/videos.js b/server/middlewares/validators/videos.js index 7e90ca047..09a188c76 100644 --- a/server/middlewares/validators/videos.js +++ b/server/middlewares/validators/videos.js @@ -8,6 +8,7 @@ const logger = require('../../helpers/logger') const validatorsVideos = { videosAdd, + videosUpdate, videosGet, videosRemove, videosSearch @@ -41,22 +42,26 @@ function videosAdd (req, res, next) { }) } +function videosUpdate (req, res, next) { + req.checkParams('id', 'Should have a valid id').notEmpty().isUUID(4) + req.checkBody('name', 'Should have a valid name').optional().isVideoNameValid() + req.checkBody('description', 'Should have a valid description').optional().isVideoDescriptionValid() + req.checkBody('tags', 'Should have correct tags').optional().isVideoTagsValid() + + logger.debug('Checking videosUpdate parameters', { parameters: req.body }) + + checkErrors(req, res, function () { + checkVideoExists(req.params.id, res, next) + }) +} + function videosGet (req, res, next) { req.checkParams('id', 'Should have a valid id').notEmpty().isUUID(4) logger.debug('Checking videosGet parameters', { parameters: req.params }) checkErrors(req, res, function () { - db.Video.load(req.params.id, function (err, video) { - if (err) { - logger.error('Error in videosGet request validator.', { error: err }) - return res.sendStatus(500) - } - - if (!video) return res.status(404).send('Video not found') - - next() - }) + checkVideoExists(req.params.id, res, next) }) } @@ -94,3 +99,19 @@ function videosSearch (req, res, next) { // --------------------------------------------------------------------------- module.exports = validatorsVideos + +// --------------------------------------------------------------------------- + +function checkVideoExists (id, res, callback) { + db.Video.loadAndPopulateAuthorAndPodAndTags(id, function (err, video) { + if (err) { + logger.error('Error in video request validator.', { error: err }) + return res.sendStatus(500) + } + + if (!video) return res.status(404).send('Video not found') + + res.locals.video = video + callback() + }) +} diff --git a/server/models/video.js b/server/models/video.js index 0e84e8986..14fbe2f71 100644 --- a/server/models/video.js +++ b/server/models/video.js @@ -127,7 +127,8 @@ module.exports = function (sequelize, DataTypes) { getTorrentName, isOwned, toFormatedJSON, - toRemoteJSON + toAddRemoteJSON, + toUpdateRemoteJSON }, hooks: { beforeValidate, @@ -334,7 +335,7 @@ function toFormatedJSON () { return json } -function toRemoteJSON (callback) { +function toAddRemoteJSON (callback) { const self = this // Get thumbnail data to send to the other pod @@ -362,6 +363,22 @@ function toRemoteJSON (callback) { }) } +function toUpdateRemoteJSON (callback) { + const json = { + name: this.name, + description: this.description, + infoHash: this.infoHash, + remoteId: this.id, + author: this.Author.name, + duration: this.duration, + tags: map(this.Tags, 'name'), + createdAt: this.createdAt, + extname: this.extname + } + + return json +} + // ------------------------------ STATICS ------------------------------ function generateThumbnailFromData (video, thumbnailData, callback) { diff --git a/server/tests/api/check-params.js b/server/tests/api/check-params.js index 9aecc3720..e8f2aa821 100644 --- a/server/tests/api/check-params.js +++ b/server/tests/api/check-params.js @@ -10,6 +10,7 @@ const loginUtils = require('../utils/login') const requestsUtils = require('../utils/requests') const serversUtils = require('../utils/servers') const usersUtils = require('../utils/users') +const videosUtils = require('../utils/videos') describe('Test parameters validator', function () { let server = null @@ -439,6 +440,106 @@ describe('Test parameters validator', function () { }) }) + describe('When updating a video', function () { + let videoId + + before(function (done) { + videosUtils.getVideosList(server.url, function (err, res) { + if (err) throw err + + videoId = res.body.data[0].id + + return done() + }) + }) + + it('Should fail with nothing', function (done) { + const data = {} + requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) + }) + + it('Should fail without a valid uuid', function (done) { + const data = { + description: 'my super description', + tags: [ 'tag1', 'tag2' ] + } + requestsUtils.makePutBodyRequest(server.url, path + 'blabla', server.accessToken, data, done) + }) + + it('Should fail with an unknown id', function (done) { + const data = { + description: 'my super description', + tags: [ 'tag1', 'tag2' ] + } + requestsUtils.makePutBodyRequest(server.url, path + '4da6fde3-88f7-4d16-b119-108df5630b06', server.accessToken, data, done) + }) + + it('Should fail with a long name', function (done) { + const data = { + name: 'My very very very very very very very very very very very very very very very very long name', + description: 'my super description', + tags: [ 'tag1', 'tag2' ] + } + requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) + }) + + it('Should fail with a long description', function (done) { + const data = { + name: 'my super name', + description: 'my super description which is very very very very very very very very very very very very very very' + + 'very very very very very very very very very very very very very very very very very very very very very' + + 'very very very very very very very very very very very very very very very long', + tags: [ 'tag1', 'tag2' ] + } + requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) + }) + + it('Should fail with too many tags', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ 'tag1', 'tag2', 'tag3', 'tag4' ] + } + requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) + }) + + it('Should fail with not enough tags', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ ] + } + requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) + }) + + it('Should fail with a tag length too low', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ 'tag1', 't' ] + } + requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) + }) + + it('Should fail with a tag length too big', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ 'mysupertagtoolong', 'tag1' ] + } + requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) + }) + + it('Should fail with malformed tags', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ 'my tag' ] + } + requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) + }) + }) + describe('When getting a video', function () { it('Should return the list of the videos with nothing', function (done) { request(server.url) diff --git a/server/tests/api/single-pod.js b/server/tests/api/single-pod.js index 66b762f82..57146900d 100644 --- a/server/tests/api/single-pod.js +++ b/server/tests/api/single-pod.js @@ -495,10 +495,86 @@ describe('Test a single pod', function () { expect(videos[2].name === 'video_short2.webm name') expect(videos[3].name === 'video_short3.webm name') + videoId = videos[3].id + done() }) }) + it('Should update a video', function (done) { + const name = 'my super video updated' + const description = 'my super description updated' + const tags = [ 'tagup1', 'tagup2' ] + + videosUtils.updateVideo(server.url, server.accessToken, videoId, name, description, tags, done) + }) + + it('Should have the video updated', function (done) { + videosUtils.getVideo(server.url, videoId, function (err, res) { + if (err) throw err + + const video = res.body + + expect(video.name).to.equal('my super video updated') + expect(video.description).to.equal('my super description updated') + expect(video.podHost).to.equal('localhost:9001') + expect(video.author).to.equal('root') + expect(video.isLocal).to.be.true + expect(video.tags).to.deep.equal([ 'tagup1', 'tagup2' ]) + expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true + + done() + }) + }) + + it('Should update only the tags of a video', function (done) { + const tags = [ 'tag1', 'tag2', 'supertag' ] + + videosUtils.updateVideo(server.url, server.accessToken, videoId, null, null, tags, function (err) { + if (err) throw err + + videosUtils.getVideo(server.url, videoId, function (err, res) { + if (err) throw err + + const video = res.body + + expect(video.name).to.equal('my super video updated') + expect(video.description).to.equal('my super description updated') + expect(video.podHost).to.equal('localhost:9001') + expect(video.author).to.equal('root') + expect(video.isLocal).to.be.true + expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'supertag' ]) + expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true + + done() + }) + }) + }) + + it('Should update only the description of a video', function (done) { + const description = 'hello everybody' + + videosUtils.updateVideo(server.url, server.accessToken, videoId, null, description, null, function (err) { + if (err) throw err + + videosUtils.getVideo(server.url, videoId, function (err, res) { + if (err) throw err + + const video = res.body + + expect(video.name).to.equal('my super video updated') + expect(video.description).to.equal('hello everybody') + expect(video.podHost).to.equal('localhost:9001') + expect(video.author).to.equal('root') + expect(video.isLocal).to.be.true + expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'supertag' ]) + expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true + + done() + }) + }) + }) + after(function (done) { process.kill(-server.app.pid) diff --git a/server/tests/utils/videos.js b/server/tests/utils/videos.js index 5c120597f..beafd3cf5 100644 --- a/server/tests/utils/videos.js +++ b/server/tests/utils/videos.js @@ -15,7 +15,8 @@ const videosUtils = { searchVideoWithPagination, searchVideoWithSort, testVideoImage, - uploadVideo + uploadVideo, + updateVideo } // ---------------------- Export functions -------------------- @@ -194,6 +195,31 @@ function uploadVideo (url, accessToken, name, description, tags, fixture, specia .end(end) } +function updateVideo (url, accessToken, id, name, description, tags, specialStatus, end) { + if (!end) { + end = specialStatus + specialStatus = 204 + } + + const path = '/api/v1/videos/' + id + + const req = request(url) + .put(path) + .set('Accept', 'application/json') + .set('Authorization', 'Bearer ' + accessToken) + + if (name) req.field('name', name) + if (description) req.field('description', description) + + if (tags) { + for (let i = 0; i < tags.length; i++) { + req.field('tags[' + i + ']', tags[i]) + } + } + + req.expect(specialStatus).end(end) +} + // --------------------------------------------------------------------------- module.exports = videosUtils From 3d118fb501f576a298f6bb059167e4c7f4dd8dcc Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 30 Dec 2016 11:27:42 +0100 Subject: [PATCH 26/47] Server: propagate video update to other pods --- server/controllers/api/remote.js | 110 +++++++++++++++++---- server/controllers/api/videos.js | 2 - server/helpers/custom-validators/videos.js | 15 +++ server/models/video.js | 6 +- server/tests/api/multiple-pods.js | 55 +++++++++-- 5 files changed, 153 insertions(+), 35 deletions(-) diff --git a/server/controllers/api/remote.js b/server/controllers/api/remote.js index 929ade555..254ae56d5 100644 --- a/server/controllers/api/remote.js +++ b/server/controllers/api/remote.js @@ -35,12 +35,21 @@ function remoteVideos (req, res, next) { eachSeries(requests, function (request, callbackEach) { const videoData = request.data - if (request.type === 'add') { - addRemoteVideo(videoData, fromPod, callbackEach) - } else if (request.type === 'remove') { - removeRemoteVideo(videoData, fromPod, callbackEach) - } else { - logger.error('Unkown remote request type %s.', request.type) + switch (request.type) { + case 'add': + addRemoteVideo(videoData, fromPod, callbackEach) + break + + case 'update': + updateRemoteVideo(videoData, fromPod, callbackEach) + break + + case 'remove': + removeRemoteVideo(videoData, fromPod, callbackEach) + break + + default: + logger.error('Unkown remote request type %s.', request.type) } }, function (err) { if (err) logger.error('Error managing remote videos.', { error: err }) @@ -143,24 +152,85 @@ function addRemoteVideo (videoToCreateData, fromPod, finalCallback) { }) } -function removeRemoteVideo (videoToRemoveData, fromPod, callback) { - // TODO: use bulkDestroy? +function updateRemoteVideo (videoAttributesToUpdate, fromPod, finalCallback) { + logger.debug('Updating remote video "%s".', videoAttributesToUpdate.name) - // We need the list because we have to remove some other stuffs (thumbnail etc) - db.Video.listByHostAndRemoteId(fromPod.host, videoToRemoveData.remoteId, function (err, videosList) { + waterfall([ + + function startTransaction (callback) { + db.sequelize.transaction().asCallback(function (err, t) { + return callback(err, t) + }) + }, + + function findVideo (t, callback) { + db.Video.loadByHostAndRemoteId(fromPod.host, videoAttributesToUpdate.remoteId, function (err, videoInstance) { + if (err || !videoInstance) { + logger.error('Cannot load video from host and remote id.', { error: err.message }) + return callback(err) + } + + return callback(null, t, videoInstance) + }) + }, + + function findOrCreateTags (t, videoInstance, callback) { + const tags = videoAttributesToUpdate.tags + + db.Tag.findOrCreateTags(tags, t, function (err, tagInstances) { + return callback(err, t, videoInstance, tagInstances) + }) + }, + + function updateVideoIntoDB (t, videoInstance, tagInstances, callback) { + const options = { transaction: t } + + videoInstance.set('name', videoAttributesToUpdate.name) + videoInstance.set('description', videoAttributesToUpdate.description) + videoInstance.set('infoHash', videoAttributesToUpdate.infoHash) + videoInstance.set('duration', videoAttributesToUpdate.duration) + videoInstance.set('createdAt', videoAttributesToUpdate.createdAt) + videoInstance.set('extname', videoAttributesToUpdate.extname) + + videoInstance.save(options).asCallback(function (err) { + return callback(err, t, videoInstance, tagInstances) + }) + }, + + function associateTagsToVideo (t, videoInstance, tagInstances, callback) { + const options = { transaction: t } + + videoInstance.setTags(tagInstances, options).asCallback(function (err) { + return callback(err, t) + }) + } + + ], function (err, t) { if (err) { - logger.error('Cannot list videos from host and remote id.', { error: err.message }) + logger.error('Cannot update the remote video.') + + // Abort transaction? + if (t) t.rollback() + + return finalCallback(err) + } + + // Commit transaction + t.commit() + + return finalCallback() + }) +} + +function removeRemoteVideo (videoToRemoveData, fromPod, callback) { + // We need the instance because we have to remove some other stuffs (thumbnail etc) + db.Video.loadByHostAndRemoteId(fromPod.host, videoToRemoveData.remoteId, function (err, video) { + if (err || !video) { + logger.error('Cannot load video from host and remote id.', { error: err.message }) return callback(err) } - if (videosList.length === 0) { - logger.error('No remote video was found for this pod.', { remoteId: videoToRemoveData.remoteId, podHost: fromPod.host }) - } - - each(videosList, function (video, callbackEach) { - logger.debug('Removing remote video %s.', video.remoteId) - - video.destroy().asCallback(callbackEach) - }, callback) + logger.debug('Removing remote video %s.', video.remoteId) + video.destroy().asCallback(callback) }) } diff --git a/server/controllers/api/videos.js b/server/controllers/api/videos.js index 1b306d1cf..e5c52a87b 100644 --- a/server/controllers/api/videos.js +++ b/server/controllers/api/videos.js @@ -229,8 +229,6 @@ function updateVideo (req, res, next) { // Add tags association videoInstance.save(options).asCallback(function (err) { - if (err) return callback(err) - return callback(err, t, tagInstances) }) }, diff --git a/server/helpers/custom-validators/videos.js b/server/helpers/custom-validators/videos.js index 4aaa6aaa9..b76eec1b5 100644 --- a/server/helpers/custom-validators/videos.js +++ b/server/helpers/custom-validators/videos.js @@ -37,6 +37,17 @@ function isEachRemoteVideosValid (requests) { isVideoRemoteIdValid(video.remoteId) && isVideoExtnameValid(video.extname) ) || + ( + isRequestTypeUpdateValid(request.type) && + isVideoDateValid(video.createdAt) && + isVideoDescriptionValid(video.description) && + isVideoDurationValid(video.duration) && + isVideoInfoHashValid(video.infoHash) && + isVideoNameValid(video.name) && + isVideoTagsValid(video.tags) && + isVideoRemoteIdValid(video.remoteId) && + isVideoExtnameValid(video.extname) + ) || ( isRequestTypeRemoveValid(request.type) && isVideoNameValid(video.name) && @@ -104,6 +115,10 @@ function isRequestTypeAddValid (value) { return value === 'add' } +function isRequestTypeUpdateValid (value) { + return value === 'update' +} + function isRequestTypeRemoveValid (value) { return value === 'remove' } diff --git a/server/models/video.js b/server/models/video.js index 14fbe2f71..f51d08f06 100644 --- a/server/models/video.js +++ b/server/models/video.js @@ -111,10 +111,10 @@ module.exports = function (sequelize, DataTypes) { getDurationFromFile, list, listForApi, - listByHostAndRemoteId, listOwnedAndPopulateAuthorAndTags, listOwnedByAuthor, load, + loadByHostAndRemoteId, loadAndPopulateAuthor, loadAndPopulateAuthorAndPodAndTags, searchAndPopulateAuthorAndPodAndTags @@ -428,7 +428,7 @@ function listForApi (start, count, sort, callback) { }) } -function listByHostAndRemoteId (fromHost, remoteId, callback) { +function loadByHostAndRemoteId (fromHost, remoteId, callback) { const query = { where: { remoteId: remoteId @@ -449,7 +449,7 @@ function listByHostAndRemoteId (fromHost, remoteId, callback) { ] } - return this.findAll(query).asCallback(callback) + return this.findOne(query).asCallback(callback) } function listOwnedAndPopulateAuthorAndTags (callback) { diff --git a/server/tests/api/multiple-pods.js b/server/tests/api/multiple-pods.js index f0fe59c5f..672187068 100644 --- a/server/tests/api/multiple-pods.js +++ b/server/tests/api/multiple-pods.js @@ -299,8 +299,8 @@ describe('Test multiple pods', function () { if (err) throw err const video = res.body.data[0] - toRemove.push(res.body.data[2].id) - toRemove.push(res.body.data[3].id) + toRemove.push(res.body.data[2]) + toRemove.push(res.body.data[3]) webtorrent.add(video.magnetUri, function (torrent) { expect(torrent.files).to.exist @@ -368,16 +368,51 @@ describe('Test multiple pods', function () { }) }) }) + }) - it('Should remove the file 3 and 3-2 by asking pod 3', function (done) { + describe('Should manipulate these videos', function () { + it('Should update the video 3 by asking pod 3', function (done) { + this.timeout(15000) + + const name = 'my super video updated' + const description = 'my super description updated' + const tags = [ 'tagup1', 'tagup2' ] + + videosUtils.updateVideo(servers[2].url, servers[2].accessToken, toRemove[0].id, name, description, tags, function (err) { + if (err) throw err + + setTimeout(done, 11000) + }) + }) + + it('Should have the video 3 updated on each pod', function (done) { + each(servers, function (server, callback) { + videosUtils.getVideosList(server.url, function (err, res) { + if (err) throw err + + const videos = res.body.data + const videoUpdated = videos.find(function (video) { + return video.name === 'my super video updated' + }) + + expect(!!videoUpdated).to.be.true + expect(videoUpdated.description).to.equal('my super description updated') + expect(videoUpdated.tags).to.deep.equal([ 'tagup1', 'tagup2' ]) + + callback() + }) + }, done) + }) + + it('Should remove the videos 3 and 3-2 by asking pod 3', function (done) { this.timeout(15000) series([ function (next) { - videosUtils.removeVideo(servers[2].url, servers[2].accessToken, toRemove[0], next) + videosUtils.removeVideo(servers[2].url, servers[2].accessToken, toRemove[0].id, next) }, function (next) { - videosUtils.removeVideo(servers[2].url, servers[2].accessToken, toRemove[1], next) + videosUtils.removeVideo(servers[2].url, servers[2].accessToken, toRemove[1].id, next) }], function (err) { if (err) throw err @@ -394,11 +429,11 @@ describe('Test multiple pods', function () { const videos = res.body.data expect(videos).to.be.an('array') expect(videos.length).to.equal(2) - expect(videos[0].id).not.to.equal(videos[1].id) - expect(videos[0].id).not.to.equal(toRemove[0]) - expect(videos[1].id).not.to.equal(toRemove[0]) - expect(videos[0].id).not.to.equal(toRemove[1]) - expect(videos[1].id).not.to.equal(toRemove[1]) + expect(videos[0].name).not.to.equal(videos[1].name) + expect(videos[0].name).not.to.equal(toRemove[0].name) + expect(videos[1].name).not.to.equal(toRemove[0].name) + expect(videos[0].name).not.to.equal(toRemove[1].name) + expect(videos[1].name).not.to.equal(toRemove[1].name) callback() }) From 79066fdf33f79d2d41394f10881e2c226ca26b49 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 30 Dec 2016 11:45:00 +0100 Subject: [PATCH 27/47] Server: add updatedAt attribute to videos --- server/controllers/api/remote.js | 4 +++- server/helpers/custom-validators/videos.js | 2 ++ server/models/video.js | 6 ++++-- server/tests/api/multiple-pods.js | 5 +++++ server/tests/api/single-pod.js | 8 ++++++++ server/tests/utils/miscs.js | 6 ++++-- 6 files changed, 26 insertions(+), 5 deletions(-) diff --git a/server/controllers/api/remote.js b/server/controllers/api/remote.js index 254ae56d5..a36c31c38 100644 --- a/server/controllers/api/remote.js +++ b/server/controllers/api/remote.js @@ -98,7 +98,8 @@ function addRemoteVideo (videoToCreateData, fromPod, finalCallback) { description: videoToCreateData.description, authorId: author.id, duration: videoToCreateData.duration, - createdAt: videoToCreateData.createdAt + createdAt: videoToCreateData.createdAt, + updatedAt: videoToCreateData.updatedAt } const video = db.Video.build(videoData) @@ -190,6 +191,7 @@ function updateRemoteVideo (videoAttributesToUpdate, fromPod, finalCallback) { videoInstance.set('infoHash', videoAttributesToUpdate.infoHash) videoInstance.set('duration', videoAttributesToUpdate.duration) videoInstance.set('createdAt', videoAttributesToUpdate.createdAt) + videoInstance.set('updatedAt', videoAttributesToUpdate.updatedAt) videoInstance.set('extname', videoAttributesToUpdate.extname) videoInstance.save(options).asCallback(function (err) { diff --git a/server/helpers/custom-validators/videos.js b/server/helpers/custom-validators/videos.js index b76eec1b5..8448386d9 100644 --- a/server/helpers/custom-validators/videos.js +++ b/server/helpers/custom-validators/videos.js @@ -28,6 +28,7 @@ function isEachRemoteVideosValid (requests) { isRequestTypeAddValid(request.type) && isVideoAuthorValid(video.author) && isVideoDateValid(video.createdAt) && + isVideoDateValid(video.updatedAt) && isVideoDescriptionValid(video.description) && isVideoDurationValid(video.duration) && isVideoInfoHashValid(video.infoHash) && @@ -40,6 +41,7 @@ function isEachRemoteVideosValid (requests) { ( isRequestTypeUpdateValid(request.type) && isVideoDateValid(video.createdAt) && + isVideoDateValid(video.updatedAt) && isVideoDescriptionValid(video.description) && isVideoDurationValid(video.duration) && isVideoInfoHashValid(video.infoHash) && diff --git a/server/models/video.js b/server/models/video.js index f51d08f06..3fe8368c7 100644 --- a/server/models/video.js +++ b/server/models/video.js @@ -20,7 +20,6 @@ const customVideosValidators = require('../helpers/custom-validators').videos // --------------------------------------------------------------------------- module.exports = function (sequelize, DataTypes) { - // TODO: add indexes on searchable columns const Video = sequelize.define('Video', { id: { @@ -329,7 +328,8 @@ function toFormatedJSON () { duration: this.duration, tags: map(this.Tags, 'name'), thumbnailPath: constants.STATIC_PATHS.THUMBNAILS + '/' + this.getThumbnailName(), - createdAt: this.createdAt + createdAt: this.createdAt, + updatedAt: this.updatedAt } return json @@ -356,6 +356,7 @@ function toAddRemoteJSON (callback) { thumbnailData: thumbnailData.toString('binary'), tags: map(self.Tags, 'name'), createdAt: self.createdAt, + updatedAt: self.updatedAt, extname: self.extname } @@ -373,6 +374,7 @@ function toUpdateRemoteJSON (callback) { duration: this.duration, tags: map(this.Tags, 'name'), createdAt: this.createdAt, + updatedAt: this.updatedAt, extname: this.extname } diff --git a/server/tests/api/multiple-pods.js b/server/tests/api/multiple-pods.js index 672187068..4442a7ff7 100644 --- a/server/tests/api/multiple-pods.js +++ b/server/tests/api/multiple-pods.js @@ -105,6 +105,7 @@ describe('Test multiple pods', function () { expect(video.duration).to.equal(10) expect(video.tags).to.deep.equal([ 'tag1p1', 'tag2p1' ]) expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true + expect(miscsUtils.dateIsValid(video.updatedAt)).to.be.true expect(video.author).to.equal('root') if (server.url !== 'http://localhost:9001') { @@ -167,6 +168,7 @@ describe('Test multiple pods', function () { expect(video.duration).to.equal(5) expect(video.tags).to.deep.equal([ 'tag1p2', 'tag2p2', 'tag3p2' ]) expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true + expect(miscsUtils.dateIsValid(video.updatedAt)).to.be.true expect(video.author).to.equal('root') if (server.url !== 'http://localhost:9002') { @@ -247,6 +249,7 @@ describe('Test multiple pods', function () { expect(video1.tags).to.deep.equal([ 'tag1p3' ]) expect(video1.author).to.equal('root') expect(miscsUtils.dateIsValid(video1.createdAt)).to.be.true + expect(miscsUtils.dateIsValid(video1.updatedAt)).to.be.true expect(video2.name).to.equal('my super name for pod 3-2') expect(video2.description).to.equal('my super description for pod 3-2') @@ -256,6 +259,7 @@ describe('Test multiple pods', function () { expect(video2.tags).to.deep.equal([ 'tag2p3', 'tag3p3', 'tag4p3' ]) expect(video2.author).to.equal('root') expect(miscsUtils.dateIsValid(video2.createdAt)).to.be.true + expect(miscsUtils.dateIsValid(video2.updatedAt)).to.be.true if (server.url !== 'http://localhost:9003') { expect(video1.isLocal).to.be.false @@ -398,6 +402,7 @@ describe('Test multiple pods', function () { expect(!!videoUpdated).to.be.true expect(videoUpdated.description).to.equal('my super description updated') expect(videoUpdated.tags).to.deep.equal([ 'tagup1', 'tagup2' ]) + expect(miscsUtils.dateIsValid(videoUpdated.updatedAt, 20000)).to.be.true callback() }) diff --git a/server/tests/api/single-pod.js b/server/tests/api/single-pod.js index 57146900d..29512dfc6 100644 --- a/server/tests/api/single-pod.js +++ b/server/tests/api/single-pod.js @@ -83,6 +83,7 @@ describe('Test a single pod', function () { expect(video.isLocal).to.be.true expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'tag3' ]) expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true + expect(miscsUtils.dateIsValid(video.updatedAt)).to.be.true videosUtils.testVideoImage(server.url, 'video_short.webm', video.thumbnailPath, function (err, test) { if (err) throw err @@ -117,6 +118,7 @@ describe('Test a single pod', function () { expect(video.isLocal).to.be.true expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'tag3' ]) expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true + expect(miscsUtils.dateIsValid(video.updatedAt)).to.be.true videosUtils.testVideoImage(server.url, 'video_short.webm', video.thumbnailPath, function (err, test) { if (err) throw err @@ -143,6 +145,7 @@ describe('Test a single pod', function () { expect(video.isLocal).to.be.true expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'tag3' ]) expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true + expect(miscsUtils.dateIsValid(video.updatedAt)).to.be.true videosUtils.testVideoImage(server.url, 'video_short.webm', video.thumbnailPath, function (err, test) { if (err) throw err @@ -170,6 +173,7 @@ describe('Test a single pod', function () { // expect(video.isLocal).to.be.true // expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'tag3' ]) // expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true + // expect(miscsUtils.dateIsValid(video.updatedAt)).to.be.true // videosUtils.testVideoImage(server.url, 'video_short.webm', video.thumbnailPath, function (err, test) { // if (err) throw err @@ -196,6 +200,7 @@ describe('Test a single pod', function () { expect(video.isLocal).to.be.true expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'tag3' ]) expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true + expect(miscsUtils.dateIsValid(video.updatedAt)).to.be.true videosUtils.testVideoImage(server.url, 'video_short.webm', video.thumbnailPath, function (err, test) { if (err) throw err @@ -522,6 +527,7 @@ describe('Test a single pod', function () { expect(video.isLocal).to.be.true expect(video.tags).to.deep.equal([ 'tagup1', 'tagup2' ]) expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true + expect(miscsUtils.dateIsValid(video.updatedAt)).to.be.true done() }) @@ -545,6 +551,7 @@ describe('Test a single pod', function () { expect(video.isLocal).to.be.true expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'supertag' ]) expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true + expect(miscsUtils.dateIsValid(video.updatedAt)).to.be.true done() }) @@ -569,6 +576,7 @@ describe('Test a single pod', function () { expect(video.isLocal).to.be.true expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'supertag' ]) expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true + expect(miscsUtils.dateIsValid(video.updatedAt)).to.be.true done() }) diff --git a/server/tests/utils/miscs.js b/server/tests/utils/miscs.js index 4ceff65df..c4b661496 100644 --- a/server/tests/utils/miscs.js +++ b/server/tests/utils/miscs.js @@ -6,12 +6,14 @@ const miscsUtils = { // ---------------------- Export functions -------------------- -function dateIsValid (dateString) { +function dateIsValid (dateString, interval) { const dateToCheck = new Date(dateString) const now = new Date() // Check if the interval is more than 2 minutes - if (now - dateToCheck > 120000) return false + if (!interval) interval = 120000 + + if (now - dateToCheck > interval) return false return true } From 818f7987eba27c59793e2103168b26129c9404f2 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 30 Dec 2016 11:51:08 +0100 Subject: [PATCH 28/47] Server: optimization for videoGet and videoRemove --- server/controllers/api/videos.js | 28 +++++-------------------- server/middlewares/validators/videos.js | 15 ++++++------- 2 files changed, 13 insertions(+), 30 deletions(-) diff --git a/server/controllers/api/videos.js b/server/controllers/api/videos.js index e5c52a87b..35d6979e5 100644 --- a/server/controllers/api/videos.js +++ b/server/controllers/api/videos.js @@ -200,7 +200,7 @@ function addVideo (req, res, next) { } function updateVideo (req, res, next) { - let videoInstance = res.locals.video + const videoInstance = res.locals.video const videoInfosToUpdate = req.body waterfall([ @@ -275,15 +275,8 @@ function updateVideo (req, res, next) { } function getVideo (req, res, next) { - db.Video.loadAndPopulateAuthorAndPodAndTags(req.params.id, function (err, video) { - if (err) return next(err) - - if (!video) { - return res.type('json').status(204).end() - } - - res.json(video.toFormatedJSON()) - }) + const videoInstance = res.locals.video + res.json(videoInstance.toFormatedJSON()) } function listVideos (req, res, next) { @@ -295,20 +288,9 @@ function listVideos (req, res, next) { } function removeVideo (req, res, next) { - const videoId = req.params.id + const videoInstance = res.locals.video - waterfall([ - function loadVideo (callback) { - db.Video.load(videoId, function (err, video) { - return callback(err, video) - }) - }, - - function deleteVideo (video, callback) { - // Informations to other pods will be sent by the afterDestroy video hook - video.destroy().asCallback(callback) - } - ], function andFinally (err) { + videoInstance.destroy().asCallback(function (err) { if (err) { logger.error('Errors when removed the video.', { error: err }) return next(err) diff --git a/server/middlewares/validators/videos.js b/server/middlewares/validators/videos.js index 09a188c76..1b6dbccf0 100644 --- a/server/middlewares/validators/videos.js +++ b/server/middlewares/validators/videos.js @@ -71,15 +71,16 @@ function videosRemove (req, res, next) { logger.debug('Checking videosRemove parameters', { parameters: req.params }) checkErrors(req, res, function () { - db.Video.loadAndPopulateAuthor(req.params.id, function (err, video) { - if (err) { - logger.error('Error in videosRemove request validator.', { error: err }) - return res.sendStatus(500) + checkVideoExists(req.params.id, res, function () { + // We need to make additional checks + + if (res.locals.video.isOwned() === false) { + return res.status(403).send('Cannot remove video of another pod') } - if (!video) return res.status(404).send('Video not found') - else if (video.isOwned() === false) return res.status(403).send('Cannot remove video of another pod') - else if (video.Author.name !== res.locals.oauth.token.user.username) return res.status(403).send('Cannot remove video of another user') + if (res.locals.video.authorId !== res.locals.oauth.token.User.id) { + return res.status(403).send('Cannot remove video of another user') + } next() }) From efe923bcdaf15b47593ad8583df09a92c715ac6c Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 30 Dec 2016 12:23:53 +0100 Subject: [PATCH 29/47] Server: split check params tests --- server/controllers/api/remote.js | 1 - server/tests/api/check-params.js | 865 ---------------------- server/tests/api/check-params/index.js | 8 + server/tests/api/check-params/pods.js | 204 +++++ server/tests/api/check-params/remotes.js | 60 ++ server/tests/api/check-params/requests.js | 87 +++ server/tests/api/check-params/users.js | 284 +++++++ server/tests/api/check-params/videos.js | 456 ++++++++++++ server/tests/utils/login.js | 11 +- server/tests/utils/servers.js | 2 +- 10 files changed, 1110 insertions(+), 868 deletions(-) delete mode 100644 server/tests/api/check-params.js create mode 100644 server/tests/api/check-params/index.js create mode 100644 server/tests/api/check-params/pods.js create mode 100644 server/tests/api/check-params/remotes.js create mode 100644 server/tests/api/check-params/requests.js create mode 100644 server/tests/api/check-params/users.js create mode 100644 server/tests/api/check-params/videos.js diff --git a/server/controllers/api/remote.js b/server/controllers/api/remote.js index a36c31c38..be5e6dc98 100644 --- a/server/controllers/api/remote.js +++ b/server/controllers/api/remote.js @@ -1,6 +1,5 @@ 'use strict' -const each = require('async/each') const eachSeries = require('async/eachSeries') const express = require('express') const waterfall = require('async/waterfall') diff --git a/server/tests/api/check-params.js b/server/tests/api/check-params.js deleted file mode 100644 index e8f2aa821..000000000 --- a/server/tests/api/check-params.js +++ /dev/null @@ -1,865 +0,0 @@ -'use strict' - -const chai = require('chai') -const expect = chai.expect -const pathUtils = require('path') -const request = require('supertest') -const series = require('async/series') - -const loginUtils = require('../utils/login') -const requestsUtils = require('../utils/requests') -const serversUtils = require('../utils/servers') -const usersUtils = require('../utils/users') -const videosUtils = require('../utils/videos') - -describe('Test parameters validator', function () { - let server = null - let userAccessToken = null - - // --------------------------------------------------------------- - - before(function (done) { - this.timeout(20000) - - series([ - function (next) { - serversUtils.flushTests(next) - }, - function (next) { - serversUtils.runServer(1, function (server1) { - server = server1 - - next() - }) - }, - function (next) { - loginUtils.loginAndGetAccessToken(server, function (err, token) { - if (err) throw err - server.accessToken = token - - next() - }) - } - ], done) - }) - - describe('Of the pods API', function () { - const path = '/api/v1/pods/' - - describe('When making friends', function () { - let userAccessToken = null - - before(function (done) { - usersUtils.createUser(server.url, server.accessToken, 'user1', 'password', function () { - server.user = { - username: 'user1', - password: 'password' - } - - loginUtils.loginAndGetAccessToken(server, function (err, accessToken) { - if (err) throw err - - userAccessToken = accessToken - - done() - }) - }) - }) - - describe('When making friends', function () { - const body = { - hosts: [ 'localhost:9002' ] - } - - it('Should fail without hosts', function (done) { - request(server.url) - .post(path + '/makefriends') - .set('Authorization', 'Bearer ' + server.accessToken) - .set('Accept', 'application/json') - .expect(400, done) - }) - - it('Should fail if hosts is not an array', function (done) { - request(server.url) - .post(path + '/makefriends') - .send({ hosts: 'localhost:9002' }) - .set('Authorization', 'Bearer ' + server.accessToken) - .set('Accept', 'application/json') - .expect(400, done) - }) - - it('Should fail if the array is not composed by hosts', function (done) { - request(server.url) - .post(path + '/makefriends') - .send({ hosts: [ 'localhost:9002', 'localhost:coucou' ] }) - .set('Authorization', 'Bearer ' + server.accessToken) - .set('Accept', 'application/json') - .expect(400, done) - }) - - it('Should fail if the array is composed with http schemes', function (done) { - request(server.url) - .post(path + '/makefriends') - .send({ hosts: [ 'localhost:9002', 'http://localhost:9003' ] }) - .set('Authorization', 'Bearer ' + server.accessToken) - .set('Accept', 'application/json') - .expect(400, done) - }) - - it('Should fail if hosts are not unique', function (done) { - request(server.url) - .post(path + '/makefriends') - .send({ urls: [ 'localhost:9002', 'localhost:9002' ] }) - .set('Authorization', 'Bearer ' + server.accessToken) - .set('Accept', 'application/json') - .expect(400, done) - }) - - it('Should fail with a invalid token', function (done) { - request(server.url) - .post(path + '/makefriends') - .send(body) - .set('Authorization', 'Bearer faketoken') - .set('Accept', 'application/json') - .expect(401, done) - }) - - it('Should fail if the user is not an administrator', function (done) { - request(server.url) - .post(path + '/makefriends') - .send(body) - .set('Authorization', 'Bearer ' + userAccessToken) - .set('Accept', 'application/json') - .expect(403, done) - }) - }) - - describe('When quitting friends', function () { - it('Should fail with a invalid token', function (done) { - request(server.url) - .get(path + '/quitfriends') - .query({ start: 'hello' }) - .set('Authorization', 'Bearer faketoken') - .set('Accept', 'application/json') - .expect(401, done) - }) - - it('Should fail if the user is not an administrator', function (done) { - request(server.url) - .get(path + '/quitfriends') - .query({ start: 'hello' }) - .set('Authorization', 'Bearer ' + userAccessToken) - .set('Accept', 'application/json') - .expect(403, done) - }) - }) - }) - - describe('When adding a pod', function () { - it('Should fail with nothing', function (done) { - const data = {} - requestsUtils.makePostBodyRequest(server.url, path, null, data, done) - }) - - it('Should fail without public key', function (done) { - const data = { - host: 'coucou.com' - } - requestsUtils.makePostBodyRequest(server.url, path, null, data, done) - }) - - it('Should fail without an host', function (done) { - const data = { - publicKey: 'mysuperpublickey' - } - requestsUtils.makePostBodyRequest(server.url, path, null, data, done) - }) - - it('Should fail with an incorrect host', function (done) { - const data = { - host: 'http://coucou.com', - publicKey: 'mysuperpublickey' - } - requestsUtils.makePostBodyRequest(server.url, path, null, data, function () { - data.host = 'http://coucou' - requestsUtils.makePostBodyRequest(server.url, path, null, data, function () { - data.host = 'coucou' - requestsUtils.makePostBodyRequest(server.url, path, null, data, done) - }) - }) - }) - - it('Should succeed with the correct parameters', function (done) { - const data = { - host: 'coucou.com', - publicKey: 'mysuperpublickey' - } - requestsUtils.makePostBodyRequest(server.url, path, null, data, done, 200) - }) - }) - }) - - describe('Of the videos API', function () { - const path = '/api/v1/videos/' - - describe('When listing a video', function () { - it('Should fail with a bad start pagination', function (done) { - request(server.url) - .get(path) - .query({ start: 'hello' }) - .set('Accept', 'application/json') - .expect(400, done) - }) - - it('Should fail with a bad count pagination', function (done) { - request(server.url) - .get(path) - .query({ count: 'hello' }) - .set('Accept', 'application/json') - .expect(400, done) - }) - - it('Should fail with an incorrect sort', function (done) { - request(server.url) - .get(path) - .query({ sort: 'hello' }) - .set('Accept', 'application/json') - .expect(400, done) - }) - }) - - describe('When searching a video', function () { - it('Should fail with nothing', function (done) { - request(server.url) - .get(pathUtils.join(path, 'search')) - .set('Accept', 'application/json') - .expect(400, done) - }) - - it('Should fail with a bad start pagination', function (done) { - request(server.url) - .get(pathUtils.join(path, 'search', 'test')) - .query({ start: 'hello' }) - .set('Accept', 'application/json') - .expect(400, done) - }) - - it('Should fail with a bad count pagination', function (done) { - request(server.url) - .get(pathUtils.join(path, 'search', 'test')) - .query({ count: 'hello' }) - .set('Accept', 'application/json') - .expect(400, done) - }) - - it('Should fail with an incorrect sort', function (done) { - request(server.url) - .get(pathUtils.join(path, 'search', 'test')) - .query({ sort: 'hello' }) - .set('Accept', 'application/json') - .expect(400, done) - }) - }) - - describe('When adding a video', function () { - it('Should fail with nothing', function (done) { - const data = {} - const attach = {} - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) - }) - - it('Should fail without name', function (done) { - const data = { - description: 'my super description', - tags: [ 'tag1', 'tag2' ] - } - const attach = { - 'videofile': pathUtils.join(__dirname, 'fixtures', 'video_short.webm') - } - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) - }) - - it('Should fail with a long name', function (done) { - const data = { - name: 'My very very very very very very very very very very very very very very very very long name', - description: 'my super description', - tags: [ 'tag1', 'tag2' ] - } - const attach = { - 'videofile': pathUtils.join(__dirname, 'fixtures', 'video_short.webm') - } - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) - }) - - it('Should fail without description', function (done) { - const data = { - name: 'my super name', - tags: [ 'tag1', 'tag2' ] - } - const attach = { - 'videofile': pathUtils.join(__dirname, 'fixtures', 'video_short.webm') - } - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) - }) - - it('Should fail with a long description', function (done) { - const data = { - name: 'my super name', - description: 'my super description which is very very very very very very very very very very very very very very' + - 'very very very very very very very very very very very very very very very very very very very very very' + - 'very very very very very very very very very very very very very very very long', - tags: [ 'tag1', 'tag2' ] - } - const attach = { - 'videofile': pathUtils.join(__dirname, 'fixtures', 'video_short.webm') - } - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) - }) - - it('Should fail without tags', function (done) { - const data = { - name: 'my super name', - description: 'my super description' - } - const attach = { - 'videofile': pathUtils.join(__dirname, 'fixtures', 'video_short.webm') - } - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) - }) - - it('Should fail with too many tags', function (done) { - const data = { - name: 'my super name', - description: 'my super description', - tags: [ 'tag1', 'tag2', 'tag3', 'tag4' ] - } - const attach = { - 'videofile': pathUtils.join(__dirname, 'fixtures', 'video_short.webm') - } - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) - }) - - it('Should fail with not enough tags', function (done) { - const data = { - name: 'my super name', - description: 'my super description', - tags: [ ] - } - const attach = { - 'videofile': pathUtils.join(__dirname, 'fixtures', 'video_short.webm') - } - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) - }) - - it('Should fail with a tag length too low', function (done) { - const data = { - name: 'my super name', - description: 'my super description', - tags: [ 'tag1', 't' ] - } - const attach = { - 'videofile': pathUtils.join(__dirname, 'fixtures', 'video_short.webm') - } - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) - }) - - it('Should fail with a tag length too big', function (done) { - const data = { - name: 'my super name', - description: 'my super description', - tags: [ 'mysupertagtoolong', 'tag1' ] - } - const attach = { - 'videofile': pathUtils.join(__dirname, 'fixtures', 'video_short.webm') - } - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) - }) - - it('Should fail with malformed tags', function (done) { - const data = { - name: 'my super name', - description: 'my super description', - tags: [ 'my tag' ] - } - const attach = { - 'videofile': pathUtils.join(__dirname, 'fixtures', 'video_short.webm') - } - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) - }) - - it('Should fail without an input file', function (done) { - const data = { - name: 'my super name', - description: 'my super description', - tags: [ 'tag1', 'tag2' ] - } - const attach = {} - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) - }) - - it('Should fail without an incorrect input file', function (done) { - const data = { - name: 'my super name', - description: 'my super description', - tags: [ 'tag1', 'tag2' ] - } - const attach = { - 'videofile': pathUtils.join(__dirname, 'fixtures', 'video_short_fake.webm') - } - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) - }) - - it('Should fail with a too big duration', function (done) { - const data = { - name: 'my super name', - description: 'my super description', - tags: [ 'tag1', 'tag2' ] - } - const attach = { - 'videofile': pathUtils.join(__dirname, 'fixtures', 'video_too_long.webm') - } - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) - }) - - it('Should succeed with the correct parameters', function (done) { - const data = { - name: 'my super name', - description: 'my super description', - tags: [ 'tag1', 'tag2' ] - } - const attach = { - 'videofile': pathUtils.join(__dirname, 'fixtures', 'video_short.webm') - } - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, function () { - attach.videofile = pathUtils.join(__dirname, 'fixtures', 'video_short.mp4') - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, function () { - attach.videofile = pathUtils.join(__dirname, 'fixtures', 'video_short.ogv') - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done, 204) - }, false) - }, false) - }) - }) - - describe('When updating a video', function () { - let videoId - - before(function (done) { - videosUtils.getVideosList(server.url, function (err, res) { - if (err) throw err - - videoId = res.body.data[0].id - - return done() - }) - }) - - it('Should fail with nothing', function (done) { - const data = {} - requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) - }) - - it('Should fail without a valid uuid', function (done) { - const data = { - description: 'my super description', - tags: [ 'tag1', 'tag2' ] - } - requestsUtils.makePutBodyRequest(server.url, path + 'blabla', server.accessToken, data, done) - }) - - it('Should fail with an unknown id', function (done) { - const data = { - description: 'my super description', - tags: [ 'tag1', 'tag2' ] - } - requestsUtils.makePutBodyRequest(server.url, path + '4da6fde3-88f7-4d16-b119-108df5630b06', server.accessToken, data, done) - }) - - it('Should fail with a long name', function (done) { - const data = { - name: 'My very very very very very very very very very very very very very very very very long name', - description: 'my super description', - tags: [ 'tag1', 'tag2' ] - } - requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) - }) - - it('Should fail with a long description', function (done) { - const data = { - name: 'my super name', - description: 'my super description which is very very very very very very very very very very very very very very' + - 'very very very very very very very very very very very very very very very very very very very very very' + - 'very very very very very very very very very very very very very very very long', - tags: [ 'tag1', 'tag2' ] - } - requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) - }) - - it('Should fail with too many tags', function (done) { - const data = { - name: 'my super name', - description: 'my super description', - tags: [ 'tag1', 'tag2', 'tag3', 'tag4' ] - } - requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) - }) - - it('Should fail with not enough tags', function (done) { - const data = { - name: 'my super name', - description: 'my super description', - tags: [ ] - } - requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) - }) - - it('Should fail with a tag length too low', function (done) { - const data = { - name: 'my super name', - description: 'my super description', - tags: [ 'tag1', 't' ] - } - requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) - }) - - it('Should fail with a tag length too big', function (done) { - const data = { - name: 'my super name', - description: 'my super description', - tags: [ 'mysupertagtoolong', 'tag1' ] - } - requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) - }) - - it('Should fail with malformed tags', function (done) { - const data = { - name: 'my super name', - description: 'my super description', - tags: [ 'my tag' ] - } - requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) - }) - }) - - describe('When getting a video', function () { - it('Should return the list of the videos with nothing', function (done) { - request(server.url) - .get(path) - .set('Accept', 'application/json') - .expect(200) - .expect('Content-Type', /json/) - .end(function (err, res) { - if (err) throw err - - expect(res.body.data).to.be.an('array') - expect(res.body.data.length).to.equal(3) - - done() - }) - }) - - it('Should fail without a correct uuid', function (done) { - request(server.url) - .get(path + 'coucou') - .set('Accept', 'application/json') - .expect(400, done) - }) - - it('Should return 404 with an incorrect video', function (done) { - request(server.url) - .get(path + '4da6fde3-88f7-4d16-b119-108df5630b06') - .set('Accept', 'application/json') - .expect(404, done) - }) - - it('Should succeed with the correct parameters') - }) - - describe('When removing a video', function () { - it('Should have 404 with nothing', function (done) { - request(server.url) - .delete(path) - .set('Authorization', 'Bearer ' + server.accessToken) - .expect(400, done) - }) - - it('Should fail without a correct uuid', function (done) { - request(server.url) - .delete(path + 'hello') - .set('Authorization', 'Bearer ' + server.accessToken) - .expect(400, done) - }) - - it('Should fail with a video which does not exist', function (done) { - request(server.url) - .delete(path + '4da6fde3-88f7-4d16-b119-108df5630b06') - .set('Authorization', 'Bearer ' + server.accessToken) - .expect(404, done) - }) - - it('Should fail with a video of another user') - - it('Should fail with a video of another pod') - - it('Should succeed with the correct parameters') - }) - }) - - describe('Of the users API', function () { - const path = '/api/v1/users/' - let userId = null - let rootId = null - - describe('When listing users', function () { - it('Should fail with a bad start pagination', function (done) { - request(server.url) - .get(path) - .query({ start: 'hello' }) - .set('Accept', 'application/json') - .expect(400, done) - }) - - it('Should fail with a bad count pagination', function (done) { - request(server.url) - .get(path) - .query({ count: 'hello' }) - .set('Accept', 'application/json') - .expect(400, done) - }) - - it('Should fail with an incorrect sort', function (done) { - request(server.url) - .get(path) - .query({ sort: 'hello' }) - .set('Accept', 'application/json') - .expect(400, done) - }) - }) - - describe('When adding a new user', function () { - it('Should fail with a too small username', function (done) { - const data = { - username: 'ji', - password: 'mysuperpassword' - } - - requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done) - }) - - it('Should fail with a too long username', function (done) { - const data = { - username: 'mysuperusernamewhichisverylong', - password: 'mysuperpassword' - } - - requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done) - }) - - it('Should fail with an incorrect username', function (done) { - const data = { - username: 'my username', - password: 'mysuperpassword' - } - - requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done) - }) - - it('Should fail with a too small password', function (done) { - const data = { - username: 'myusername', - password: 'bla' - } - - requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done) - }) - - it('Should fail with a too long password', function (done) { - const data = { - username: 'myusername', - password: 'my super long password which is very very very very very very very very very very very very very very' + - 'very very very very very very very very very very very very very very very veryv very very very very' + - 'very very very very very very very very very very very very very very very very very very very very long' - } - - requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done) - }) - - it('Should fail with an non authenticated user', function (done) { - const data = { - username: 'myusername', - password: 'my super password' - } - - requestsUtils.makePostBodyRequest(server.url, path, 'super token', data, done, 401) - }) - - it('Should fail if we add a user with the same username', function (done) { - const data = { - username: 'user1', - password: 'my super password' - } - - requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done, 409) - }) - - it('Should succeed with the correct params', function (done) { - const data = { - username: 'user2', - password: 'my super password' - } - - requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done, 204) - }) - - it('Should fail with a non admin user', function (done) { - server.user = { - username: 'user1', - password: 'password' - } - - loginUtils.loginAndGetAccessToken(server, function (err, accessToken) { - if (err) throw err - - userAccessToken = accessToken - - const data = { - username: 'user3', - password: 'my super password' - } - - requestsUtils.makePostBodyRequest(server.url, path, userAccessToken, data, done, 403) - }) - }) - }) - - describe('When updating a user', function () { - before(function (done) { - usersUtils.getUsersList(server.url, function (err, res) { - if (err) throw err - - userId = res.body.data[1].id - rootId = res.body.data[2].id - done() - }) - }) - - it('Should fail with a too small password', function (done) { - const data = { - password: 'bla' - } - - requestsUtils.makePutBodyRequest(server.url, path + userId, userAccessToken, data, done) - }) - - it('Should fail with a too long password', function (done) { - const data = { - password: 'my super long password which is very very very very very very very very very very very very very very' + - 'very very very very very very very very very very very very very very very veryv very very very very' + - 'very very very very very very very very very very very very very very very very very very very very long' - } - - requestsUtils.makePutBodyRequest(server.url, path + userId, userAccessToken, data, done) - }) - - it('Should fail with an non authenticated user', function (done) { - const data = { - password: 'my super password' - } - - requestsUtils.makePutBodyRequest(server.url, path + userId, 'super token', data, done, 401) - }) - - it('Should succeed with the correct params', function (done) { - const data = { - password: 'my super password' - } - - requestsUtils.makePutBodyRequest(server.url, path + userId, userAccessToken, data, done, 204) - }) - }) - - describe('When getting my information', function () { - it('Should fail with a non authenticated user', function (done) { - request(server.url) - .get(path + 'me') - .set('Authorization', 'Bearer faketoken') - .set('Accept', 'application/json') - .expect(401, done) - }) - - it('Should success with the correct parameters', function (done) { - request(server.url) - .get(path + 'me') - .set('Authorization', 'Bearer ' + userAccessToken) - .set('Accept', 'application/json') - .expect(200, done) - }) - }) - - describe('When removing an user', function () { - it('Should fail with an incorrect id', function (done) { - request(server.url) - .delete(path + 'bla-bla') - .set('Authorization', 'Bearer ' + server.accessToken) - .expect(400, done) - }) - - it('Should fail with the root user', function (done) { - request(server.url) - .delete(path + rootId) - .set('Authorization', 'Bearer ' + server.accessToken) - .expect(400, done) - }) - - it('Should return 404 with a non existing id', function (done) { - request(server.url) - .delete(path + '45') - .set('Authorization', 'Bearer ' + server.accessToken) - .expect(404, done) - }) - }) - }) - - describe('Of the remote videos API', function () { - describe('When making a secure request', function () { - it('Should check a secure request') - }) - - describe('When adding a video', function () { - it('Should check when adding a video') - }) - - describe('When removing a video', function () { - it('Should check when removing a video') - }) - }) - - describe('Of the requests API', function () { - const path = '/api/v1/requests/stats' - - it('Should fail with an non authenticated user', function (done) { - request(server.url) - .get(path) - .set('Accept', 'application/json') - .expect(401, done) - }) - - it('Should fail with a non admin user', function (done) { - request(server.url) - .get(path) - .set('Authorization', 'Bearer ' + userAccessToken) - .set('Accept', 'application/json') - .expect(403, done) - }) - }) - - after(function (done) { - process.kill(-server.app.pid) - - // Keep the logs if the test failed - if (this.ok) { - serversUtils.flushTests(done) - } else { - done() - } - }) -}) diff --git a/server/tests/api/check-params/index.js b/server/tests/api/check-params/index.js new file mode 100644 index 000000000..3d6f09267 --- /dev/null +++ b/server/tests/api/check-params/index.js @@ -0,0 +1,8 @@ +'use strict' + +// Order of the tests we want to execute +require('./pods') +require('./remotes') +require('./users') +require('./requests') +require('./videos') diff --git a/server/tests/api/check-params/pods.js b/server/tests/api/check-params/pods.js new file mode 100644 index 000000000..2f85af644 --- /dev/null +++ b/server/tests/api/check-params/pods.js @@ -0,0 +1,204 @@ +'use strict' + +const request = require('supertest') +const series = require('async/series') + +const loginUtils = require('../../utils/login') +const requestsUtils = require('../../utils/requests') +const serversUtils = require('../../utils/servers') +const usersUtils = require('../../utils/users') + +describe('Test pods API validators', function () { + const path = '/api/v1/pods/' + let server = null + + // --------------------------------------------------------------- + + before(function (done) { + this.timeout(20000) + + series([ + function (next) { + serversUtils.flushTests(next) + }, + function (next) { + serversUtils.runServer(1, function (server1) { + server = server1 + + next() + }) + }, + function (next) { + loginUtils.loginAndGetAccessToken(server, function (err, token) { + if (err) throw err + server.accessToken = token + + next() + }) + } + ], done) + }) + + describe('When making friends', function () { + let userAccessToken = null + + before(function (done) { + usersUtils.createUser(server.url, server.accessToken, 'user1', 'password', function () { + server.user = { + username: 'user1', + password: 'password' + } + + loginUtils.loginAndGetAccessToken(server, function (err, accessToken) { + if (err) throw err + + userAccessToken = accessToken + + done() + }) + }) + }) + + describe('When making friends', function () { + const body = { + hosts: [ 'localhost:9002' ] + } + + it('Should fail without hosts', function (done) { + request(server.url) + .post(path + '/makefriends') + .set('Authorization', 'Bearer ' + server.accessToken) + .set('Accept', 'application/json') + .expect(400, done) + }) + + it('Should fail if hosts is not an array', function (done) { + request(server.url) + .post(path + '/makefriends') + .send({ hosts: 'localhost:9002' }) + .set('Authorization', 'Bearer ' + server.accessToken) + .set('Accept', 'application/json') + .expect(400, done) + }) + + it('Should fail if the array is not composed by hosts', function (done) { + request(server.url) + .post(path + '/makefriends') + .send({ hosts: [ 'localhost:9002', 'localhost:coucou' ] }) + .set('Authorization', 'Bearer ' + server.accessToken) + .set('Accept', 'application/json') + .expect(400, done) + }) + + it('Should fail if the array is composed with http schemes', function (done) { + request(server.url) + .post(path + '/makefriends') + .send({ hosts: [ 'localhost:9002', 'http://localhost:9003' ] }) + .set('Authorization', 'Bearer ' + server.accessToken) + .set('Accept', 'application/json') + .expect(400, done) + }) + + it('Should fail if hosts are not unique', function (done) { + request(server.url) + .post(path + '/makefriends') + .send({ urls: [ 'localhost:9002', 'localhost:9002' ] }) + .set('Authorization', 'Bearer ' + server.accessToken) + .set('Accept', 'application/json') + .expect(400, done) + }) + + it('Should fail with a invalid token', function (done) { + request(server.url) + .post(path + '/makefriends') + .send(body) + .set('Authorization', 'Bearer faketoken') + .set('Accept', 'application/json') + .expect(401, done) + }) + + it('Should fail if the user is not an administrator', function (done) { + request(server.url) + .post(path + '/makefriends') + .send(body) + .set('Authorization', 'Bearer ' + userAccessToken) + .set('Accept', 'application/json') + .expect(403, done) + }) + }) + + describe('When quitting friends', function () { + it('Should fail with a invalid token', function (done) { + request(server.url) + .get(path + '/quitfriends') + .query({ start: 'hello' }) + .set('Authorization', 'Bearer faketoken') + .set('Accept', 'application/json') + .expect(401, done) + }) + + it('Should fail if the user is not an administrator', function (done) { + request(server.url) + .get(path + '/quitfriends') + .query({ start: 'hello' }) + .set('Authorization', 'Bearer ' + userAccessToken) + .set('Accept', 'application/json') + .expect(403, done) + }) + }) + }) + + describe('When adding a pod', function () { + it('Should fail with nothing', function (done) { + const data = {} + requestsUtils.makePostBodyRequest(server.url, path, null, data, done) + }) + + it('Should fail without public key', function (done) { + const data = { + host: 'coucou.com' + } + requestsUtils.makePostBodyRequest(server.url, path, null, data, done) + }) + + it('Should fail without an host', function (done) { + const data = { + publicKey: 'mysuperpublickey' + } + requestsUtils.makePostBodyRequest(server.url, path, null, data, done) + }) + + it('Should fail with an incorrect host', function (done) { + const data = { + host: 'http://coucou.com', + publicKey: 'mysuperpublickey' + } + requestsUtils.makePostBodyRequest(server.url, path, null, data, function () { + data.host = 'http://coucou' + requestsUtils.makePostBodyRequest(server.url, path, null, data, function () { + data.host = 'coucou' + requestsUtils.makePostBodyRequest(server.url, path, null, data, done) + }) + }) + }) + + it('Should succeed with the correct parameters', function (done) { + const data = { + host: 'coucou.com', + publicKey: 'mysuperpublickey' + } + requestsUtils.makePostBodyRequest(server.url, path, null, data, done, 200) + }) + }) + + after(function (done) { + process.kill(-server.app.pid) + + // Keep the logs if the test failed + if (this.ok) { + serversUtils.flushTests(done) + } else { + done() + } + }) +}) diff --git a/server/tests/api/check-params/remotes.js b/server/tests/api/check-params/remotes.js new file mode 100644 index 000000000..30ba3b697 --- /dev/null +++ b/server/tests/api/check-params/remotes.js @@ -0,0 +1,60 @@ +'use strict' + +const series = require('async/series') + +const loginUtils = require('../../utils/login') +const serversUtils = require('../../utils/servers') + +describe('Test remote videos API validators', function () { + let server = null + + // --------------------------------------------------------------- + + before(function (done) { + this.timeout(20000) + + series([ + function (next) { + serversUtils.flushTests(next) + }, + function (next) { + serversUtils.runServer(1, function (server1) { + server = server1 + + next() + }) + }, + function (next) { + loginUtils.loginAndGetAccessToken(server, function (err, token) { + if (err) throw err + server.accessToken = token + + next() + }) + } + ], done) + }) + + describe('When making a secure request', function () { + it('Should check a secure request') + }) + + describe('When adding a video', function () { + it('Should check when adding a video') + }) + + describe('When removing a video', function () { + it('Should check when removing a video') + }) + + after(function (done) { + process.kill(-server.app.pid) + + // Keep the logs if the test failed + if (this.ok) { + serversUtils.flushTests(done) + } else { + done() + } + }) +}) diff --git a/server/tests/api/check-params/requests.js b/server/tests/api/check-params/requests.js new file mode 100644 index 000000000..08f58db43 --- /dev/null +++ b/server/tests/api/check-params/requests.js @@ -0,0 +1,87 @@ +'use strict' + +const request = require('supertest') +const series = require('async/series') + +const loginUtils = require('../../utils/login') +const usersUtils = require('../../utils/users') +const serversUtils = require('../../utils/servers') + +describe('Test requests API validators', function () { + const path = '/api/v1/requests/stats' + let server = null + let userAccessToken = null + + // --------------------------------------------------------------- + + before(function (done) { + this.timeout(20000) + + series([ + function (next) { + serversUtils.flushTests(next) + }, + function (next) { + serversUtils.runServer(1, function (server1) { + server = server1 + + next() + }) + }, + function (next) { + loginUtils.loginAndGetAccessToken(server, function (err, token) { + if (err) throw err + server.accessToken = token + + next() + }) + }, + function (next) { + const username = 'user' + const password = 'my super password' + + usersUtils.createUser(server.url, server.accessToken, username, password, next) + }, + function (next) { + const user = { + username: 'user', + password: 'my super password' + } + + loginUtils.getUserAccessToken(server, user, function (err, accessToken) { + if (err) throw err + + userAccessToken = accessToken + + next() + }) + } + ], done) + }) + + it('Should fail with an non authenticated user', function (done) { + request(server.url) + .get(path) + .set('Accept', 'application/json') + .expect(401, done) + }) + + it('Should fail with a non admin user', function (done) { + request(server.url) + .get(path) + .set('Authorization', 'Bearer ' + userAccessToken) + .set('Accept', 'application/json') + .expect(403, done) + }) + + after(function (done) { + process.kill(-server.app.pid) + + // Keep the logs if the test failed + if (this.ok) { + serversUtils.flushTests(done) + } else { + done() + } + }) +}) diff --git a/server/tests/api/check-params/users.js b/server/tests/api/check-params/users.js new file mode 100644 index 000000000..c1fcf34a4 --- /dev/null +++ b/server/tests/api/check-params/users.js @@ -0,0 +1,284 @@ +'use strict' + +const request = require('supertest') +const series = require('async/series') + +const loginUtils = require('../../utils/login') +const requestsUtils = require('../../utils/requests') +const serversUtils = require('../../utils/servers') +const usersUtils = require('../../utils/users') + +describe('Test users API validators', function () { + const path = '/api/v1/users/' + let userId = null + let rootId = null + let server = null + let userAccessToken = null + + // --------------------------------------------------------------- + + before(function (done) { + this.timeout(20000) + + series([ + function (next) { + serversUtils.flushTests(next) + }, + function (next) { + serversUtils.runServer(1, function (server1) { + server = server1 + + next() + }) + }, + function (next) { + loginUtils.loginAndGetAccessToken(server, function (err, token) { + if (err) throw err + server.accessToken = token + + next() + }) + }, + function (next) { + const username = 'user1' + const password = 'my super password' + + usersUtils.createUser(server.url, server.accessToken, username, password, next) + }, + function (next) { + const user = { + username: 'user1', + password: 'my super password' + } + + loginUtils.getUserAccessToken(server, user, function (err, accessToken) { + if (err) throw err + + userAccessToken = accessToken + + next() + }) + } + ], done) + }) + + describe('When listing users', function () { + it('Should fail with a bad start pagination', function (done) { + request(server.url) + .get(path) + .query({ start: 'hello' }) + .set('Accept', 'application/json') + .expect(400, done) + }) + + it('Should fail with a bad count pagination', function (done) { + request(server.url) + .get(path) + .query({ count: 'hello' }) + .set('Accept', 'application/json') + .expect(400, done) + }) + + it('Should fail with an incorrect sort', function (done) { + request(server.url) + .get(path) + .query({ sort: 'hello' }) + .set('Accept', 'application/json') + .expect(400, done) + }) + }) + + describe('When adding a new user', function () { + it('Should fail with a too small username', function (done) { + const data = { + username: 'ji', + password: 'mysuperpassword' + } + + requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done) + }) + + it('Should fail with a too long username', function (done) { + const data = { + username: 'mysuperusernamewhichisverylong', + password: 'mysuperpassword' + } + + requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done) + }) + + it('Should fail with an incorrect username', function (done) { + const data = { + username: 'my username', + password: 'mysuperpassword' + } + + requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done) + }) + + it('Should fail with a too small password', function (done) { + const data = { + username: 'myusername', + password: 'bla' + } + + requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done) + }) + + it('Should fail with a too long password', function (done) { + const data = { + username: 'myusername', + password: 'my super long password which is very very very very very very very very very very very very very very' + + 'very very very very very very very very very very very very very very very veryv very very very very' + + 'very very very very very very very very very very very very very very very very very very very very long' + } + + requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done) + }) + + it('Should fail with an non authenticated user', function (done) { + const data = { + username: 'myusername', + password: 'my super password' + } + + requestsUtils.makePostBodyRequest(server.url, path, 'super token', data, done, 401) + }) + + it('Should fail if we add a user with the same username', function (done) { + const data = { + username: 'user1', + password: 'my super password' + } + + requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done, 409) + }) + + it('Should succeed with the correct params', function (done) { + const data = { + username: 'user2', + password: 'my super password' + } + + requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done, 204) + }) + + it('Should fail with a non admin user', function (done) { + server.user = { + username: 'user1', + password: 'my super password' + } + + loginUtils.loginAndGetAccessToken(server, function (err, accessToken) { + if (err) throw err + + userAccessToken = accessToken + + const data = { + username: 'user3', + password: 'my super password' + } + + requestsUtils.makePostBodyRequest(server.url, path, userAccessToken, data, done, 403) + }) + }) + }) + + describe('When updating a user', function () { + before(function (done) { + usersUtils.getUsersList(server.url, function (err, res) { + if (err) throw err + + userId = res.body.data[1].id + rootId = res.body.data[2].id + done() + }) + }) + + it('Should fail with a too small password', function (done) { + const data = { + password: 'bla' + } + + requestsUtils.makePutBodyRequest(server.url, path + userId, userAccessToken, data, done) + }) + + it('Should fail with a too long password', function (done) { + const data = { + password: 'my super long password which is very very very very very very very very very very very very very very' + + 'very very very very very very very very very very very very very very very veryv very very very very' + + 'very very very very very very very very very very very very very very very very very very very very long' + } + + requestsUtils.makePutBodyRequest(server.url, path + userId, userAccessToken, data, done) + }) + + it('Should fail with an non authenticated user', function (done) { + const data = { + password: 'my super password' + } + + requestsUtils.makePutBodyRequest(server.url, path + userId, 'super token', data, done, 401) + }) + + it('Should succeed with the correct params', function (done) { + const data = { + password: 'my super password' + } + + requestsUtils.makePutBodyRequest(server.url, path + userId, userAccessToken, data, done, 204) + }) + }) + + describe('When getting my information', function () { + it('Should fail with a non authenticated user', function (done) { + request(server.url) + .get(path + 'me') + .set('Authorization', 'Bearer faketoken') + .set('Accept', 'application/json') + .expect(401, done) + }) + + it('Should success with the correct parameters', function (done) { + request(server.url) + .get(path + 'me') + .set('Authorization', 'Bearer ' + userAccessToken) + .set('Accept', 'application/json') + .expect(200, done) + }) + }) + + describe('When removing an user', function () { + it('Should fail with an incorrect id', function (done) { + request(server.url) + .delete(path + 'bla-bla') + .set('Authorization', 'Bearer ' + server.accessToken) + .expect(400, done) + }) + + it('Should fail with the root user', function (done) { + request(server.url) + .delete(path + rootId) + .set('Authorization', 'Bearer ' + server.accessToken) + .expect(400, done) + }) + + it('Should return 404 with a non existing id', function (done) { + request(server.url) + .delete(path + '45') + .set('Authorization', 'Bearer ' + server.accessToken) + .expect(404, done) + }) + }) + + after(function (done) { + process.kill(-server.app.pid) + + // Keep the logs if the test failed + if (this.ok) { + serversUtils.flushTests(done) + } else { + done() + } + }) +}) diff --git a/server/tests/api/check-params/videos.js b/server/tests/api/check-params/videos.js new file mode 100644 index 000000000..d18305291 --- /dev/null +++ b/server/tests/api/check-params/videos.js @@ -0,0 +1,456 @@ +'use strict' + +const chai = require('chai') +const expect = chai.expect +const pathUtils = require('path') +const request = require('supertest') +const series = require('async/series') + +const loginUtils = require('../../utils/login') +const requestsUtils = require('../../utils/requests') +const serversUtils = require('../../utils/servers') +const videosUtils = require('../../utils/videos') + +describe('Test videos API validator', function () { + const path = '/api/v1/videos/' + let server = null + + // --------------------------------------------------------------- + + before(function (done) { + this.timeout(20000) + + series([ + function (next) { + serversUtils.flushTests(next) + }, + function (next) { + serversUtils.runServer(1, function (server1) { + server = server1 + + next() + }) + }, + function (next) { + loginUtils.loginAndGetAccessToken(server, function (err, token) { + if (err) throw err + server.accessToken = token + + next() + }) + } + ], done) + }) + + describe('When listing a video', function () { + it('Should fail with a bad start pagination', function (done) { + request(server.url) + .get(path) + .query({ start: 'hello' }) + .set('Accept', 'application/json') + .expect(400, done) + }) + + it('Should fail with a bad count pagination', function (done) { + request(server.url) + .get(path) + .query({ count: 'hello' }) + .set('Accept', 'application/json') + .expect(400, done) + }) + + it('Should fail with an incorrect sort', function (done) { + request(server.url) + .get(path) + .query({ sort: 'hello' }) + .set('Accept', 'application/json') + .expect(400, done) + }) + }) + + describe('When searching a video', function () { + it('Should fail with nothing', function (done) { + request(server.url) + .get(pathUtils.join(path, 'search')) + .set('Accept', 'application/json') + .expect(400, done) + }) + + it('Should fail with a bad start pagination', function (done) { + request(server.url) + .get(pathUtils.join(path, 'search', 'test')) + .query({ start: 'hello' }) + .set('Accept', 'application/json') + .expect(400, done) + }) + + it('Should fail with a bad count pagination', function (done) { + request(server.url) + .get(pathUtils.join(path, 'search', 'test')) + .query({ count: 'hello' }) + .set('Accept', 'application/json') + .expect(400, done) + }) + + it('Should fail with an incorrect sort', function (done) { + request(server.url) + .get(pathUtils.join(path, 'search', 'test')) + .query({ sort: 'hello' }) + .set('Accept', 'application/json') + .expect(400, done) + }) + }) + + describe('When adding a video', function () { + it('Should fail with nothing', function (done) { + const data = {} + const attach = {} + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) + }) + + it('Should fail without name', function (done) { + const data = { + description: 'my super description', + tags: [ 'tag1', 'tag2' ] + } + const attach = { + 'videofile': pathUtils.join(__dirname, '..', 'fixtures', 'video_short.webm') + } + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) + }) + + it('Should fail with a long name', function (done) { + const data = { + name: 'My very very very very very very very very very very very very very very very very long name', + description: 'my super description', + tags: [ 'tag1', 'tag2' ] + } + const attach = { + 'videofile': pathUtils.join(__dirname, '..', 'fixtures', 'video_short.webm') + } + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) + }) + + it('Should fail without description', function (done) { + const data = { + name: 'my super name', + tags: [ 'tag1', 'tag2' ] + } + const attach = { + 'videofile': pathUtils.join(__dirname, '..', 'fixtures', 'video_short.webm') + } + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) + }) + + it('Should fail with a long description', function (done) { + const data = { + name: 'my super name', + description: 'my super description which is very very very very very very very very very very very very very very' + + 'very very very very very very very very very very very very very very very very very very very very very' + + 'very very very very very very very very very very very very very very very long', + tags: [ 'tag1', 'tag2' ] + } + const attach = { + 'videofile': pathUtils.join(__dirname, '..', 'fixtures', 'video_short.webm') + } + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) + }) + + it('Should fail without tags', function (done) { + const data = { + name: 'my super name', + description: 'my super description' + } + const attach = { + 'videofile': pathUtils.join(__dirname, '..', 'fixtures', 'video_short.webm') + } + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) + }) + + it('Should fail with too many tags', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ 'tag1', 'tag2', 'tag3', 'tag4' ] + } + const attach = { + 'videofile': pathUtils.join(__dirname, '..', 'fixtures', 'video_short.webm') + } + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) + }) + + it('Should fail with not enough tags', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ ] + } + const attach = { + 'videofile': pathUtils.join(__dirname, '..', 'fixtures', 'video_short.webm') + } + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) + }) + + it('Should fail with a tag length too low', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ 'tag1', 't' ] + } + const attach = { + 'videofile': pathUtils.join(__dirname, '..', 'fixtures', 'video_short.webm') + } + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) + }) + + it('Should fail with a tag length too big', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ 'mysupertagtoolong', 'tag1' ] + } + const attach = { + 'videofile': pathUtils.join(__dirname, '..', 'fixtures', 'video_short.webm') + } + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) + }) + + it('Should fail with malformed tags', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ 'my tag' ] + } + const attach = { + 'videofile': pathUtils.join(__dirname, '..', 'fixtures', 'video_short.webm') + } + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) + }) + + it('Should fail without an input file', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ 'tag1', 'tag2' ] + } + const attach = {} + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) + }) + + it('Should fail without an incorrect input file', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ 'tag1', 'tag2' ] + } + const attach = { + 'videofile': pathUtils.join(__dirname, '..', 'fixtures', 'video_short_fake.webm') + } + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) + }) + + it('Should fail with a too big duration', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ 'tag1', 'tag2' ] + } + const attach = { + 'videofile': pathUtils.join(__dirname, '..', 'fixtures', 'video_too_long.webm') + } + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) + }) + + it('Should succeed with the correct parameters', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ 'tag1', 'tag2' ] + } + const attach = { + 'videofile': pathUtils.join(__dirname, '..', 'fixtures', 'video_short.webm') + } + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, function () { + attach.videofile = pathUtils.join(__dirname, '..', 'fixtures', 'video_short.mp4') + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, function () { + attach.videofile = pathUtils.join(__dirname, '..', 'fixtures', 'video_short.ogv') + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done, 204) + }, false) + }, false) + }) + }) + + describe('When updating a video', function () { + let videoId + + before(function (done) { + videosUtils.getVideosList(server.url, function (err, res) { + if (err) throw err + + videoId = res.body.data[0].id + + return done() + }) + }) + + it('Should fail with nothing', function (done) { + const data = {} + requestsUtils.makePutBodyRequest(server.url, path, server.accessToken, data, done) + }) + + it('Should fail without a valid uuid', function (done) { + const data = { + description: 'my super description', + tags: [ 'tag1', 'tag2' ] + } + requestsUtils.makePutBodyRequest(server.url, path + 'blabla', server.accessToken, data, done) + }) + + it('Should fail with an unknown id', function (done) { + const data = { + description: 'my super description', + tags: [ 'tag1', 'tag2' ] + } + requestsUtils.makePutBodyRequest(server.url, path + '4da6fde3-88f7-4d16-b119-108df5630b06', server.accessToken, data, done, 404) + }) + + it('Should fail with a long name', function (done) { + const data = { + name: 'My very very very very very very very very very very very very very very very very long name', + description: 'my super description', + tags: [ 'tag1', 'tag2' ] + } + requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) + }) + + it('Should fail with a long description', function (done) { + const data = { + name: 'my super name', + description: 'my super description which is very very very very very very very very very very very very very very' + + 'very very very very very very very very very very very very very very very very very very very very very' + + 'very very very very very very very very very very very very very very very long', + tags: [ 'tag1', 'tag2' ] + } + requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) + }) + + it('Should fail with too many tags', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ 'tag1', 'tag2', 'tag3', 'tag4' ] + } + requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) + }) + + it('Should fail with not enough tags', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ ] + } + requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) + }) + + it('Should fail with a tag length too low', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ 'tag1', 't' ] + } + requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) + }) + + it('Should fail with a tag length too big', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ 'mysupertagtoolong', 'tag1' ] + } + requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) + }) + + it('Should fail with malformed tags', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ 'my tag' ] + } + requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) + }) + }) + + describe('When getting a video', function () { + it('Should return the list of the videos with nothing', function (done) { + request(server.url) + .get(path) + .set('Accept', 'application/json') + .expect(200) + .expect('Content-Type', /json/) + .end(function (err, res) { + if (err) throw err + + expect(res.body.data).to.be.an('array') + expect(res.body.data.length).to.equal(3) + + done() + }) + }) + + it('Should fail without a correct uuid', function (done) { + request(server.url) + .get(path + 'coucou') + .set('Accept', 'application/json') + .expect(400, done) + }) + + it('Should return 404 with an incorrect video', function (done) { + request(server.url) + .get(path + '4da6fde3-88f7-4d16-b119-108df5630b06') + .set('Accept', 'application/json') + .expect(404, done) + }) + + it('Should succeed with the correct parameters') + }) + + describe('When removing a video', function () { + it('Should have 404 with nothing', function (done) { + request(server.url) + .delete(path) + .set('Authorization', 'Bearer ' + server.accessToken) + .expect(400, done) + }) + + it('Should fail without a correct uuid', function (done) { + request(server.url) + .delete(path + 'hello') + .set('Authorization', 'Bearer ' + server.accessToken) + .expect(400, done) + }) + + it('Should fail with a video which does not exist', function (done) { + request(server.url) + .delete(path + '4da6fde3-88f7-4d16-b119-108df5630b06') + .set('Authorization', 'Bearer ' + server.accessToken) + .expect(404, done) + }) + + it('Should fail with a video of another user') + + it('Should fail with a video of another pod') + + it('Should succeed with the correct parameters') + }) + + after(function (done) { + process.kill(-server.app.pid) + + // Keep the logs if the test failed + if (this.ok) { + serversUtils.flushTests(done) + } else { + done() + } + }) +}) diff --git a/server/tests/utils/login.js b/server/tests/utils/login.js index 465564e14..c984c0baf 100644 --- a/server/tests/utils/login.js +++ b/server/tests/utils/login.js @@ -4,7 +4,8 @@ const request = require('supertest') const loginUtils = { login, - loginAndGetAccessToken + loginAndGetAccessToken, + getUserAccessToken } // ---------------------- Export functions -------------------- @@ -43,6 +44,14 @@ function loginAndGetAccessToken (server, callback) { }) } +function getUserAccessToken (server, user, callback) { + login(server.url, server.client, user, 200, function (err, res) { + if (err) return callback(err) + + return callback(null, res.body.access_token) + }) +} + // --------------------------------------------------------------------------- module.exports = loginUtils diff --git a/server/tests/utils/servers.js b/server/tests/utils/servers.js index e7c756499..1946ef49a 100644 --- a/server/tests/utils/servers.js +++ b/server/tests/utils/servers.js @@ -34,7 +34,7 @@ function flushAndRunMultipleServers (totalServers, serversRun) { runServer(j, function (app, url) { anotherServerDone(j, app, url) }) - }, 1000 * j) + }, 1000 * (j - 1)) } }) } From 8fd66b75bfbd8fd4945f1944411461b05eb74795 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 30 Dec 2016 12:39:49 +0100 Subject: [PATCH 30/47] Server: fix video remoe validation --- server/middlewares/validators/videos.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/middlewares/validators/videos.js b/server/middlewares/validators/videos.js index 1b6dbccf0..295ed05fa 100644 --- a/server/middlewares/validators/videos.js +++ b/server/middlewares/validators/videos.js @@ -78,7 +78,7 @@ function videosRemove (req, res, next) { return res.status(403).send('Cannot remove video of another pod') } - if (res.locals.video.authorId !== res.locals.oauth.token.User.id) { + if (res.locals.video.Author.userId !== res.locals.oauth.token.User.id) { return res.status(403).send('Cannot remove video of another user') } From a6fd2b30bf717eec14972a2175354781f5f43e77 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 30 Dec 2016 12:53:41 +0100 Subject: [PATCH 31/47] Server: move remote routes in their own directory --- server/controllers/api/index.js | 8 +++----- server/controllers/api/remote/index.js | 16 ++++++++++++++++ .../api/{remote.js => remote/videos.js} | 8 ++++---- server/helpers/utils.js | 5 +++++ 4 files changed, 28 insertions(+), 9 deletions(-) create mode 100644 server/controllers/api/remote/index.js rename server/controllers/api/{remote.js => remote/videos.js} (97%) diff --git a/server/controllers/api/index.js b/server/controllers/api/index.js index 4cb65ed55..f13ff922c 100644 --- a/server/controllers/api/index.js +++ b/server/controllers/api/index.js @@ -2,6 +2,8 @@ const express = require('express') +const utils = require('../../helpers/utils') + const router = express.Router() const clientsController = require('./clients') @@ -18,7 +20,7 @@ router.use('/requests', requestsController) router.use('/users', usersController) router.use('/videos', videosController) router.use('/ping', pong) -router.use('/*', badRequest) +router.use('/*', utils.badRequest) // --------------------------------------------------------------------------- @@ -29,7 +31,3 @@ module.exports = router function pong (req, res, next) { return res.send('pong').status(200).end() } - -function badRequest (req, res, next) { - res.type('json').status(400).end() -} diff --git a/server/controllers/api/remote/index.js b/server/controllers/api/remote/index.js new file mode 100644 index 000000000..2947632d5 --- /dev/null +++ b/server/controllers/api/remote/index.js @@ -0,0 +1,16 @@ +'use strict' + +const express = require('express') + +const utils = require('../../../helpers/utils') + +const router = express.Router() + +const videosRemoteController = require('./videos') + +router.use('/videos', videosRemoteController) +router.use('/*', utils.badRequest) + +// --------------------------------------------------------------------------- + +module.exports = router diff --git a/server/controllers/api/remote.js b/server/controllers/api/remote/videos.js similarity index 97% rename from server/controllers/api/remote.js rename to server/controllers/api/remote/videos.js index be5e6dc98..87c49bff9 100644 --- a/server/controllers/api/remote.js +++ b/server/controllers/api/remote/videos.js @@ -4,15 +4,15 @@ const eachSeries = require('async/eachSeries') const express = require('express') const waterfall = require('async/waterfall') -const db = require('../../initializers/database') -const middlewares = require('../../middlewares') +const db = require('../../../initializers/database') +const middlewares = require('../../../middlewares') const secureMiddleware = middlewares.secure const validators = middlewares.validators.remote -const logger = require('../../helpers/logger') +const logger = require('../../../helpers/logger') const router = express.Router() -router.post('/videos', +router.post('/', validators.signature, secureMiddleware.checkSignature, validators.remoteVideos, diff --git a/server/helpers/utils.js b/server/helpers/utils.js index 9f27671b6..7e0c9823c 100644 --- a/server/helpers/utils.js +++ b/server/helpers/utils.js @@ -5,11 +5,16 @@ const crypto = require('crypto') const logger = require('./logger') const utils = { + badRequest, cleanForExit, generateRandomString, isTestInstance } +function badRequest (req, res, next) { + res.type('json').status(400).end() +} + function generateRandomString (size, callback) { crypto.pseudoRandomBytes(size, function (err, raw) { if (err) return callback(err) From 55fa55a9be566cca2ba95322f2ae23b434aed62a Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 4 Jan 2017 20:59:23 +0100 Subject: [PATCH 32/47] Server: add video abuse support --- .../friend-list/friend-list.component.ts | 2 +- .../admin/friends/shared/friend.service.ts | 16 +- config/default.yaml | 4 +- server.js | 3 +- server/controllers/api/pods.js | 17 +- server/controllers/api/remote/videos.js | 70 +++++-- server/controllers/api/users.js | 18 +- server/controllers/api/videos.js | 65 ++++-- server/helpers/custom-validators/index.js | 2 + .../helpers/custom-validators/remote/index.js | 11 + .../custom-validators/remote/videos.js | 74 +++++++ server/helpers/custom-validators/videos.js | 68 ++----- server/helpers/utils.js | 16 +- server/initializers/constants.js | 4 + server/lib/friends.js | 39 ++-- server/middlewares/sort.js | 7 + server/middlewares/validators/remote/index.js | 13 ++ .../{remote.js => remote/signature.js} | 17 +- .../middlewares/validators/remote/videos.js | 20 ++ server/middlewares/validators/sort.js | 23 ++- server/middlewares/validators/videos.js | 15 +- .../{requestToPod.js => request-to-pod.js} | 0 server/models/video-abuse.js | 113 +++++++++++ server/models/{videoTag.js => video-tag.js} | 0 server/models/video.js | 8 + server/tests/api/check-params/index.js | 1 + server/tests/api/check-params/remotes.js | 4 + server/tests/api/check-params/video-abuses.js | 180 +++++++++++++++++ server/tests/api/friends-advanced.js | 10 +- server/tests/api/friends-basic.js | 12 +- server/tests/api/video-abuse.js | 191 ++++++++++++++++++ server/tests/utils/video-abuses.js | 73 +++++++ 32 files changed, 921 insertions(+), 175 deletions(-) create mode 100644 server/helpers/custom-validators/remote/index.js create mode 100644 server/helpers/custom-validators/remote/videos.js create mode 100644 server/middlewares/validators/remote/index.js rename server/middlewares/validators/{remote.js => remote/signature.js} (53%) create mode 100644 server/middlewares/validators/remote/videos.js rename server/models/{requestToPod.js => request-to-pod.js} (100%) create mode 100644 server/models/video-abuse.js rename server/models/{videoTag.js => video-tag.js} (100%) create mode 100644 server/tests/api/check-params/video-abuses.js create mode 100644 server/tests/api/video-abuse.js create mode 100644 server/tests/utils/video-abuses.js diff --git a/client/src/app/admin/friends/friend-list/friend-list.component.ts b/client/src/app/admin/friends/friend-list/friend-list.component.ts index 88c4800ee..bec10162c 100644 --- a/client/src/app/admin/friends/friend-list/friend-list.component.ts +++ b/client/src/app/admin/friends/friend-list/friend-list.component.ts @@ -30,7 +30,7 @@ export class FriendListComponent implements OnInit { private getFriends() { this.friendService.getFriends().subscribe( - friends => this.friends = friends, + res => this.friends = res.friends, err => alert(err.text) ); diff --git a/client/src/app/admin/friends/shared/friend.service.ts b/client/src/app/admin/friends/shared/friend.service.ts index 8a1ba6b02..85ac04ba0 100644 --- a/client/src/app/admin/friends/shared/friend.service.ts +++ b/client/src/app/admin/friends/shared/friend.service.ts @@ -2,7 +2,7 @@ import { Injectable } from '@angular/core'; import { Observable } from 'rxjs/Observable'; import { Friend } from './friend.model'; -import { AuthHttp, RestExtractor } from '../../../shared'; +import { AuthHttp, RestExtractor, ResultList } from '../../../shared'; @Injectable() export class FriendService { @@ -13,11 +13,10 @@ export class FriendService { private restExtractor: RestExtractor ) {} - getFriends(): Observable { + getFriends() { return this.authHttp.get(FriendService.BASE_FRIEND_URL) - // Not implemented as a data list by the server yet - // .map(this.restExtractor.extractDataList) - .map((res) => res.json()) + .map(this.restExtractor.extractDataList) + .map(this.extractFriends) .catch((res) => this.restExtractor.handleError(res)); } @@ -36,4 +35,11 @@ export class FriendService { .map(res => res.status) .catch((res) => this.restExtractor.handleError(res)); } + + private extractFriends(result: ResultList) { + const friends: Friend[] = result.data; + const totalFriends = result.total; + + return { friends, totalFriends }; + } } diff --git a/config/default.yaml b/config/default.yaml index 2dd5e05f9..0939ae4ba 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -10,8 +10,8 @@ database: hostname: 'localhost' port: 5432 suffix: '_dev' - username: peertube - password: peertube + username: 'peertube' + password: 'peertube' # From the project root directory storage: diff --git a/server.js b/server.js index f4ca53907..7503072af 100644 --- a/server.js +++ b/server.js @@ -57,7 +57,8 @@ app.use(expressValidator({ customValidators.misc, customValidators.pods, customValidators.users, - customValidators.videos + customValidators.videos, + customValidators.remote.videos ) })) diff --git a/server/controllers/api/pods.js b/server/controllers/api/pods.js index d9279f1d9..38702face 100644 --- a/server/controllers/api/pods.js +++ b/server/controllers/api/pods.js @@ -5,6 +5,7 @@ const waterfall = require('async/waterfall') const db = require('../../initializers/database') const logger = require('../../helpers/logger') +const utils = require('../../helpers/utils') const friends = require('../../lib/friends') const middlewares = require('../../middlewares') const admin = middlewares.admin @@ -36,7 +37,7 @@ router.get('/quitfriends', ) // Post because this is a secured request router.post('/remove', - signatureValidator, + signatureValidator.signature, checkSignature, removePods ) @@ -86,7 +87,7 @@ function listPods (req, res, next) { db.Pod.list(function (err, podsList) { if (err) return next(err) - res.json(getFormatedPods(podsList)) + res.json(utils.getFormatedObjects(podsList, podsList.length)) }) } @@ -130,15 +131,3 @@ function quitFriends (req, res, next) { res.type('json').status(204).end() }) } - -// --------------------------------------------------------------------------- - -function getFormatedPods (pods) { - const formatedPods = [] - - pods.forEach(function (pod) { - formatedPods.push(pod.toFormatedJSON()) - }) - - return formatedPods -} diff --git a/server/controllers/api/remote/videos.js b/server/controllers/api/remote/videos.js index 87c49bff9..d02da4463 100644 --- a/server/controllers/api/remote/videos.js +++ b/server/controllers/api/remote/videos.js @@ -7,15 +7,16 @@ const waterfall = require('async/waterfall') const db = require('../../../initializers/database') const middlewares = require('../../../middlewares') const secureMiddleware = middlewares.secure -const validators = middlewares.validators.remote +const videosValidators = middlewares.validators.remote.videos +const signatureValidators = middlewares.validators.remote.signature const logger = require('../../../helpers/logger') const router = express.Router() router.post('/', - validators.signature, + signatureValidators.signature, secureMiddleware.checkSignature, - validators.remoteVideos, + videosValidators.remoteVideos, remoteVideos ) @@ -32,19 +33,23 @@ function remoteVideos (req, res, next) { // We need to process in the same order to keep consistency // TODO: optimization eachSeries(requests, function (request, callbackEach) { - const videoData = request.data + const data = request.data switch (request.type) { case 'add': - addRemoteVideo(videoData, fromPod, callbackEach) + addRemoteVideo(data, fromPod, callbackEach) break case 'update': - updateRemoteVideo(videoData, fromPod, callbackEach) + updateRemoteVideo(data, fromPod, callbackEach) break case 'remove': - removeRemoteVideo(videoData, fromPod, callbackEach) + removeRemoteVideo(data, fromPod, callbackEach) + break + + case 'report-abuse': + reportAbuseRemoteVideo(data, fromPod, callbackEach) break default: @@ -164,13 +169,8 @@ function updateRemoteVideo (videoAttributesToUpdate, fromPod, finalCallback) { }, function findVideo (t, callback) { - db.Video.loadByHostAndRemoteId(fromPod.host, videoAttributesToUpdate.remoteId, function (err, videoInstance) { - if (err || !videoInstance) { - logger.error('Cannot load video from host and remote id.', { error: err.message }) - return callback(err) - } - - return callback(null, t, videoInstance) + fetchVideo(fromPod.host, videoAttributesToUpdate.remoteId, function (err, videoInstance) { + return callback(err, t, videoInstance) }) }, @@ -225,13 +225,45 @@ function updateRemoteVideo (videoAttributesToUpdate, fromPod, finalCallback) { function removeRemoteVideo (videoToRemoveData, fromPod, callback) { // We need the instance because we have to remove some other stuffs (thumbnail etc) - db.Video.loadByHostAndRemoteId(fromPod.host, videoToRemoveData.remoteId, function (err, video) { - if (err || !video) { - logger.error('Cannot load video from host and remote id.', { error: err.message }) - return callback(err) - } + fetchVideo(fromPod.host, videoToRemoveData.remoteId, function (err, video) { + if (err) return callback(err) logger.debug('Removing remote video %s.', video.remoteId) video.destroy().asCallback(callback) }) } + +function reportAbuseRemoteVideo (reportData, fromPod, callback) { + db.Video.load(reportData.videoRemoteId, function (err, video) { + if (err || !video) { + if (!err) err = new Error('video not found') + + logger.error('Cannot load video from host and remote id.', { error: err }) + return callback(err) + } + + logger.debug('Reporting remote abuse for video %s.', video.id) + + const videoAbuseData = { + reporterUsername: reportData.reporterUsername, + reason: reportData.reportReason, + reporterPodId: fromPod.id, + videoId: video.id + } + + db.VideoAbuse.create(videoAbuseData).asCallback(callback) + }) +} + +function fetchVideo (podHost, remoteId, callback) { + db.Video.loadByHostAndRemoteId(podHost, remoteId, function (err, video) { + if (err || !video) { + if (!err) err = new Error('video not found') + + logger.error('Cannot load video from host and remote id.', { error: err }) + return callback(err) + } + + return callback(null, video) + }) +} diff --git a/server/controllers/api/users.js b/server/controllers/api/users.js index 53bf56790..6cd0e84f7 100644 --- a/server/controllers/api/users.js +++ b/server/controllers/api/users.js @@ -6,6 +6,7 @@ const waterfall = require('async/waterfall') const constants = require('../../initializers/constants') const db = require('../../initializers/database') const logger = require('../../helpers/logger') +const utils = require('../../helpers/utils') const middlewares = require('../../middlewares') const admin = middlewares.admin const oAuth = middlewares.oauth @@ -82,7 +83,7 @@ function listUsers (req, res, next) { db.User.listForApi(req.query.start, req.query.count, req.query.sort, function (err, usersList, usersTotal) { if (err) return next(err) - res.json(getFormatedUsers(usersList, usersTotal)) + res.json(utils.getFormatedObjects(usersList, usersTotal)) }) } @@ -121,18 +122,3 @@ function updateUser (req, res, next) { function success (req, res, next) { res.end() } - -// --------------------------------------------------------------------------- - -function getFormatedUsers (users, usersTotal) { - const formatedUsers = [] - - users.forEach(function (user) { - formatedUsers.push(user.toFormatedJSON()) - }) - - return { - total: usersTotal, - data: formatedUsers - } -} diff --git a/server/controllers/api/videos.js b/server/controllers/api/videos.js index 35d6979e5..6829804ec 100644 --- a/server/controllers/api/videos.js +++ b/server/controllers/api/videos.js @@ -11,6 +11,7 @@ const db = require('../../initializers/database') const logger = require('../../helpers/logger') const friends = require('../../lib/friends') const middlewares = require('../../middlewares') +const admin = middlewares.admin const oAuth = middlewares.oauth const pagination = middlewares.pagination const validators = middlewares.validators @@ -43,6 +44,21 @@ const storage = multer.diskStorage({ const reqFiles = multer({ storage: storage }).fields([{ name: 'videofile', maxCount: 1 }]) +router.get('/abuse', + oAuth.authenticate, + admin.ensureIsAdmin, + validatorsPagination.pagination, + validatorsSort.videoAbusesSort, + sort.setVideoAbusesSort, + pagination.setPagination, + listVideoAbuses +) +router.post('/:id/abuse', + oAuth.authenticate, + validatorsVideos.videoAbuseReport, + reportVideoAbuse +) + router.get('/', validatorsPagination.pagination, validatorsSort.videosSort, @@ -283,7 +299,7 @@ function listVideos (req, res, next) { db.Video.listForApi(req.query.start, req.query.count, req.query.sort, function (err, videosList, videosTotal) { if (err) return next(err) - res.json(getFormatedVideos(videosList, videosTotal)) + res.json(utils.getFormatedObjects(videosList, videosTotal)) }) } @@ -306,22 +322,45 @@ function searchVideos (req, res, next) { function (err, videosList, videosTotal) { if (err) return next(err) - res.json(getFormatedVideos(videosList, videosTotal)) + res.json(utils.getFormatedObjects(videosList, videosTotal)) } ) } -// --------------------------------------------------------------------------- +function listVideoAbuses (req, res, next) { + db.VideoAbuse.listForApi(req.query.start, req.query.count, req.query.sort, function (err, abusesList, abusesTotal) { + if (err) return next(err) -function getFormatedVideos (videos, videosTotal) { - const formatedVideos = [] - - videos.forEach(function (video) { - formatedVideos.push(video.toFormatedJSON()) + res.json(utils.getFormatedObjects(abusesList, abusesTotal)) }) - - return { - total: videosTotal, - data: formatedVideos - } } + +function reportVideoAbuse (req, res, next) { + const videoInstance = res.locals.video + const reporterUsername = res.locals.oauth.token.User.username + + const abuse = { + reporterUsername, + reason: req.body.reason, + videoId: videoInstance.id, + reporterPodId: null // This is our pod that reported this abuse + } + + db.VideoAbuse.create(abuse).asCallback(function (err) { + if (err) return next(err) + + // We send the information to the destination pod + if (videoInstance.isOwned() === false) { + const reportData = { + reporterUsername, + reportReason: abuse.reason, + videoRemoteId: videoInstance.remoteId + } + + friends.reportAbuseVideoToFriend(reportData, videoInstance) + } + + return res.type('json').status(204).end() + }) +} + diff --git a/server/helpers/custom-validators/index.js b/server/helpers/custom-validators/index.js index 96b5b20b9..9383e0304 100644 --- a/server/helpers/custom-validators/index.js +++ b/server/helpers/custom-validators/index.js @@ -2,12 +2,14 @@ const miscValidators = require('./misc') const podsValidators = require('./pods') +const remoteValidators = require('./remote') const usersValidators = require('./users') const videosValidators = require('./videos') const validators = { misc: miscValidators, pods: podsValidators, + remote: remoteValidators, users: usersValidators, videos: videosValidators } diff --git a/server/helpers/custom-validators/remote/index.js b/server/helpers/custom-validators/remote/index.js new file mode 100644 index 000000000..1939a95f4 --- /dev/null +++ b/server/helpers/custom-validators/remote/index.js @@ -0,0 +1,11 @@ +'use strict' + +const remoteVideosValidators = require('./videos') + +const validators = { + videos: remoteVideosValidators +} + +// --------------------------------------------------------------------------- + +module.exports = validators diff --git a/server/helpers/custom-validators/remote/videos.js b/server/helpers/custom-validators/remote/videos.js new file mode 100644 index 000000000..c3ca00e1c --- /dev/null +++ b/server/helpers/custom-validators/remote/videos.js @@ -0,0 +1,74 @@ +'use strict' + +const videosValidators = require('../videos') +const miscValidators = require('../misc') + +const remoteVideosValidators = { + isEachRemoteRequestVideosValid +} + +function isEachRemoteRequestVideosValid (requests) { + return miscValidators.isArray(requests) && + requests.every(function (request) { + const video = request.data + return ( + isRequestTypeAddValid(request.type) && + videosValidators.isVideoAuthorValid(video.author) && + videosValidators.isVideoDateValid(video.createdAt) && + videosValidators.isVideoDateValid(video.updatedAt) && + videosValidators.isVideoDescriptionValid(video.description) && + videosValidators.isVideoDurationValid(video.duration) && + videosValidators.isVideoInfoHashValid(video.infoHash) && + videosValidators.isVideoNameValid(video.name) && + videosValidators.isVideoTagsValid(video.tags) && + videosValidators.isVideoThumbnailDataValid(video.thumbnailData) && + videosValidators.isVideoRemoteIdValid(video.remoteId) && + videosValidators.isVideoExtnameValid(video.extname) + ) || + ( + isRequestTypeUpdateValid(request.type) && + videosValidators.isVideoDateValid(video.createdAt) && + videosValidators.isVideoDateValid(video.updatedAt) && + videosValidators.isVideoDescriptionValid(video.description) && + videosValidators.isVideoDurationValid(video.duration) && + videosValidators.isVideoInfoHashValid(video.infoHash) && + videosValidators.isVideoNameValid(video.name) && + videosValidators.isVideoTagsValid(video.tags) && + videosValidators.isVideoRemoteIdValid(video.remoteId) && + videosValidators.isVideoExtnameValid(video.extname) + ) || + ( + isRequestTypeRemoveValid(request.type) && + videosValidators.isVideoNameValid(video.name) && + videosValidators.isVideoRemoteIdValid(video.remoteId) + ) || + ( + isRequestTypeReportAbuseValid(request.type) && + videosValidators.isVideoRemoteIdValid(request.data.videoRemoteId) && + videosValidators.isVideoAbuseReasonValid(request.data.reportReason) && + videosValidators.isVideoAbuseReporterUsernameValid(request.data.reporterUsername) + ) + }) +} + +// --------------------------------------------------------------------------- + +module.exports = remoteVideosValidators + +// --------------------------------------------------------------------------- + +function isRequestTypeAddValid (value) { + return value === 'add' +} + +function isRequestTypeUpdateValid (value) { + return value === 'update' +} + +function isRequestTypeRemoveValid (value) { + return value === 'remove' +} + +function isRequestTypeReportAbuseValid (value) { + return value === 'report-abuse' +} diff --git a/server/helpers/custom-validators/videos.js b/server/helpers/custom-validators/videos.js index 8448386d9..7f727854d 100644 --- a/server/helpers/custom-validators/videos.js +++ b/server/helpers/custom-validators/videos.js @@ -6,9 +6,9 @@ const constants = require('../../initializers/constants') const usersValidators = require('./users') const miscValidators = require('./misc') const VIDEOS_CONSTRAINTS_FIELDS = constants.CONSTRAINTS_FIELDS.VIDEOS +const VIDEO_ABUSES_CONSTRAINTS_FIELDS = constants.CONSTRAINTS_FIELDS.VIDEO_ABUSES const videosValidators = { - isEachRemoteVideosValid, isVideoAuthorValid, isVideoDateValid, isVideoDescriptionValid, @@ -17,45 +17,11 @@ const videosValidators = { isVideoNameValid, isVideoTagsValid, isVideoThumbnailValid, - isVideoThumbnailDataValid -} - -function isEachRemoteVideosValid (requests) { - return miscValidators.isArray(requests) && - requests.every(function (request) { - const video = request.data - return ( - isRequestTypeAddValid(request.type) && - isVideoAuthorValid(video.author) && - isVideoDateValid(video.createdAt) && - isVideoDateValid(video.updatedAt) && - isVideoDescriptionValid(video.description) && - isVideoDurationValid(video.duration) && - isVideoInfoHashValid(video.infoHash) && - isVideoNameValid(video.name) && - isVideoTagsValid(video.tags) && - isVideoThumbnailDataValid(video.thumbnailData) && - isVideoRemoteIdValid(video.remoteId) && - isVideoExtnameValid(video.extname) - ) || - ( - isRequestTypeUpdateValid(request.type) && - isVideoDateValid(video.createdAt) && - isVideoDateValid(video.updatedAt) && - isVideoDescriptionValid(video.description) && - isVideoDurationValid(video.duration) && - isVideoInfoHashValid(video.infoHash) && - isVideoNameValid(video.name) && - isVideoTagsValid(video.tags) && - isVideoRemoteIdValid(video.remoteId) && - isVideoExtnameValid(video.extname) - ) || - ( - isRequestTypeRemoveValid(request.type) && - isVideoNameValid(video.name) && - isVideoRemoteIdValid(video.remoteId) - ) - }) + isVideoThumbnailDataValid, + isVideoExtnameValid, + isVideoRemoteIdValid, + isVideoAbuseReasonValid, + isVideoAbuseReporterUsernameValid } function isVideoAuthorValid (value) { @@ -107,20 +73,14 @@ function isVideoRemoteIdValid (value) { return validator.isUUID(value, 4) } +function isVideoAbuseReasonValid (value) { + return validator.isLength(value, VIDEO_ABUSES_CONSTRAINTS_FIELDS.REASON) +} + +function isVideoAbuseReporterUsernameValid (value) { + return usersValidators.isUserUsernameValid(value) +} + // --------------------------------------------------------------------------- module.exports = videosValidators - -// --------------------------------------------------------------------------- - -function isRequestTypeAddValid (value) { - return value === 'add' -} - -function isRequestTypeUpdateValid (value) { - return value === 'update' -} - -function isRequestTypeRemoveValid (value) { - return value === 'remove' -} diff --git a/server/helpers/utils.js b/server/helpers/utils.js index 7e0c9823c..9f4b14582 100644 --- a/server/helpers/utils.js +++ b/server/helpers/utils.js @@ -8,7 +8,8 @@ const utils = { badRequest, cleanForExit, generateRandomString, - isTestInstance + isTestInstance, + getFormatedObjects } function badRequest (req, res, next) { @@ -32,6 +33,19 @@ function isTestInstance () { return (process.env.NODE_ENV === 'test') } +function getFormatedObjects (objects, objectsTotal) { + const formatedObjects = [] + + objects.forEach(function (object) { + formatedObjects.push(object.toFormatedJSON()) + }) + + return { + total: objectsTotal, + data: formatedObjects + } +} + // --------------------------------------------------------------------------- module.exports = utils diff --git a/server/initializers/constants.js b/server/initializers/constants.js index 474a37277..6ba8a9da0 100644 --- a/server/initializers/constants.js +++ b/server/initializers/constants.js @@ -19,6 +19,7 @@ const SEARCHABLE_COLUMNS = { // Sortable columns per schema const SORTABLE_COLUMNS = { USERS: [ 'username', '-username', 'createdAt', '-createdAt' ], + VIDEO_ABUSES: [ 'createdAt', '-createdAt' ], VIDEOS: [ 'name', '-name', 'duration', '-duration', 'createdAt', '-createdAt' ] } @@ -65,6 +66,9 @@ const CONSTRAINTS_FIELDS = { USERNAME: { min: 3, max: 20 }, // Length PASSWORD: { min: 6, max: 255 } // Length }, + VIDEO_ABUSES: { + REASON: { min: 2, max: 300 } // Length + }, VIDEOS: { NAME: { min: 3, max: 50 }, // Length DESCRIPTION: { min: 3, max: 250 }, // Length diff --git a/server/lib/friends.js b/server/lib/friends.js index 589b79660..4afb91b8b 100644 --- a/server/lib/friends.js +++ b/server/lib/friends.js @@ -15,6 +15,7 @@ const requests = require('../helpers/requests') const friends = { addVideoToFriends, updateVideoToFriends, + reportAbuseVideoToFriend, hasFriends, getMyCertificate, makeFriends, @@ -23,12 +24,20 @@ const friends = { sendOwnedVideosToPod } -function addVideoToFriends (video) { - createRequest('add', constants.REQUEST_ENDPOINTS.VIDEOS, video) +function addVideoToFriends (videoData) { + createRequest('add', constants.REQUEST_ENDPOINTS.VIDEOS, videoData) } -function updateVideoToFriends (video) { - createRequest('update', constants.REQUEST_ENDPOINTS.VIDEOS, video) +function updateVideoToFriends (videoData) { + createRequest('update', constants.REQUEST_ENDPOINTS.VIDEOS, videoData) +} + +function removeVideoToFriends (videoParams) { + createRequest('remove', constants.REQUEST_ENDPOINTS.VIDEOS, videoParams) +} + +function reportAbuseVideoToFriend (reportData, video) { + createRequest('report-abuse', constants.REQUEST_ENDPOINTS.VIDEOS, reportData, [ video.Author.podId ]) } function hasFriends (callback) { @@ -120,10 +129,6 @@ function quitFriends (callback) { }) } -function removeVideoToFriends (videoParams) { - createRequest('remove', constants.REQUEST_ENDPOINTS.VIDEOS, videoParams) -} - function sendOwnedVideosToPod (podId) { db.Video.listOwnedAndPopulateAuthorAndTags(function (err, videosList) { if (err) { @@ -152,10 +157,10 @@ module.exports = friends // --------------------------------------------------------------------------- function computeForeignPodsList (host, podsScore, callback) { - getForeignPodsList(host, function (err, foreignPodsList) { + getForeignPodsList(host, function (err, res) { if (err) return callback(err) - if (!foreignPodsList) foreignPodsList = [] + const foreignPodsList = res.data // Let's give 1 point to the pod we ask the friends list foreignPodsList.push({ host }) @@ -252,11 +257,11 @@ function makeRequestsToWinningPods (cert, podsList, callback) { }) } -// Wrapper that populate "to" argument with all our friends if it is not specified -function createRequest (type, endpoint, data, to) { - if (to) return _createRequest(type, endpoint, data, to) +// Wrapper that populate "toIds" argument with all our friends if it is not specified +function createRequest (type, endpoint, data, toIds) { + if (toIds) return _createRequest(type, endpoint, data, toIds) - // If the "to" pods is not specified, we send the request to all our friends + // If the "toIds" pods is not specified, we send the request to all our friends db.Pod.listAllIds(function (err, podIds) { if (err) { logger.error('Cannot get pod ids', { error: err }) @@ -267,13 +272,13 @@ function createRequest (type, endpoint, data, to) { }) } -function _createRequest (type, endpoint, data, to) { +function _createRequest (type, endpoint, data, toIds) { const pods = [] // If there are no destination pods abort - if (to.length === 0) return + if (toIds.length === 0) return - to.forEach(function (toPod) { + toIds.forEach(function (toPod) { pods.push(db.Pod.build({ id: toPod })) }) diff --git a/server/middlewares/sort.js b/server/middlewares/sort.js index 477e10571..39e167265 100644 --- a/server/middlewares/sort.js +++ b/server/middlewares/sort.js @@ -2,6 +2,7 @@ const sortMiddleware = { setUsersSort, + setVideoAbusesSort, setVideosSort } @@ -11,6 +12,12 @@ function setUsersSort (req, res, next) { return next() } +function setVideoAbusesSort (req, res, next) { + if (!req.query.sort) req.query.sort = '-createdAt' + + return next() +} + function setVideosSort (req, res, next) { if (!req.query.sort) req.query.sort = '-createdAt' diff --git a/server/middlewares/validators/remote/index.js b/server/middlewares/validators/remote/index.js new file mode 100644 index 000000000..022a2fe50 --- /dev/null +++ b/server/middlewares/validators/remote/index.js @@ -0,0 +1,13 @@ +'use strict' + +const remoteSignatureValidators = require('./signature') +const remoteVideosValidators = require('./videos') + +const validators = { + signature: remoteSignatureValidators, + videos: remoteVideosValidators +} + +// --------------------------------------------------------------------------- + +module.exports = validators diff --git a/server/middlewares/validators/remote.js b/server/middlewares/validators/remote/signature.js similarity index 53% rename from server/middlewares/validators/remote.js rename to server/middlewares/validators/remote/signature.js index 858d193cc..5880a2c2c 100644 --- a/server/middlewares/validators/remote.js +++ b/server/middlewares/validators/remote/signature.js @@ -1,21 +1,12 @@ 'use strict' -const checkErrors = require('./utils').checkErrors -const logger = require('../../helpers/logger') +const checkErrors = require('../utils').checkErrors +const logger = require('../../../helpers/logger') -const validatorsRemote = { - remoteVideos, +const validatorsRemoteSignature = { signature } -function remoteVideos (req, res, next) { - req.checkBody('data').isEachRemoteVideosValid() - - logger.debug('Checking remoteVideos parameters', { parameters: req.body }) - - checkErrors(req, res, next) -} - function signature (req, res, next) { req.checkBody('signature.host', 'Should have a signature host').isURL() req.checkBody('signature.signature', 'Should have a signature').notEmpty() @@ -27,4 +18,4 @@ function signature (req, res, next) { // --------------------------------------------------------------------------- -module.exports = validatorsRemote +module.exports = validatorsRemoteSignature diff --git a/server/middlewares/validators/remote/videos.js b/server/middlewares/validators/remote/videos.js new file mode 100644 index 000000000..cf9925b6c --- /dev/null +++ b/server/middlewares/validators/remote/videos.js @@ -0,0 +1,20 @@ +'use strict' + +const checkErrors = require('../utils').checkErrors +const logger = require('../../../helpers/logger') + +const validatorsRemoteVideos = { + remoteVideos +} + +function remoteVideos (req, res, next) { + req.checkBody('data').isEachRemoteRequestVideosValid() + + logger.debug('Checking remoteVideos parameters', { parameters: req.body }) + + checkErrors(req, res, next) +} + +// --------------------------------------------------------------------------- + +module.exports = validatorsRemoteVideos diff --git a/server/middlewares/validators/sort.js b/server/middlewares/validators/sort.js index 431d3fffd..b7eec0316 100644 --- a/server/middlewares/validators/sort.js +++ b/server/middlewares/validators/sort.js @@ -6,29 +6,38 @@ const logger = require('../../helpers/logger') const validatorsSort = { usersSort, + videoAbusesSort, videosSort } function usersSort (req, res, next) { const sortableColumns = constants.SORTABLE_COLUMNS.USERS - req.checkQuery('sort', 'Should have correct sortable column').optional().isIn(sortableColumns) + checkSort(req, res, next, sortableColumns) +} - logger.debug('Checking sort parameters', { parameters: req.query }) +function videoAbusesSort (req, res, next) { + const sortableColumns = constants.SORTABLE_COLUMNS.VIDEO_ABUSES - checkErrors(req, res, next) + checkSort(req, res, next, sortableColumns) } function videosSort (req, res, next) { const sortableColumns = constants.SORTABLE_COLUMNS.VIDEOS + checkSort(req, res, next, sortableColumns) +} + +// --------------------------------------------------------------------------- + +module.exports = validatorsSort + +// --------------------------------------------------------------------------- + +function checkSort (req, res, next, sortableColumns) { req.checkQuery('sort', 'Should have correct sortable column').optional().isIn(sortableColumns) logger.debug('Checking sort parameters', { parameters: req.query }) checkErrors(req, res, next) } - -// --------------------------------------------------------------------------- - -module.exports = validatorsSort diff --git a/server/middlewares/validators/videos.js b/server/middlewares/validators/videos.js index 295ed05fa..ff18a99c2 100644 --- a/server/middlewares/validators/videos.js +++ b/server/middlewares/validators/videos.js @@ -11,7 +11,9 @@ const validatorsVideos = { videosUpdate, videosGet, videosRemove, - videosSearch + videosSearch, + + videoAbuseReport } function videosAdd (req, res, next) { @@ -97,6 +99,17 @@ function videosSearch (req, res, next) { checkErrors(req, res, next) } +function videoAbuseReport (req, res, next) { + req.checkParams('id', 'Should have a valid id').notEmpty().isUUID(4) + req.checkBody('reason', 'Should have a valid reason').isVideoAbuseReasonValid() + + logger.debug('Checking videoAbuseReport parameters', { parameters: req.body }) + + checkErrors(req, res, function () { + checkVideoExists(req.params.id, res, next) + }) +} + // --------------------------------------------------------------------------- module.exports = validatorsVideos diff --git a/server/models/requestToPod.js b/server/models/request-to-pod.js similarity index 100% rename from server/models/requestToPod.js rename to server/models/request-to-pod.js diff --git a/server/models/video-abuse.js b/server/models/video-abuse.js new file mode 100644 index 000000000..766a7568d --- /dev/null +++ b/server/models/video-abuse.js @@ -0,0 +1,113 @@ +'use strict' + +const constants = require('../initializers/constants') +const modelUtils = require('./utils') +const customVideosValidators = require('../helpers/custom-validators').videos + +module.exports = function (sequelize, DataTypes) { + const VideoAbuse = sequelize.define('VideoAbuse', + { + reporterUsername: { + type: DataTypes.STRING, + allowNull: false, + validate: { + reporterUsernameValid: function (value) { + const res = customVideosValidators.isVideoAbuseReporterUsernameValid(value) + if (res === false) throw new Error('Video abuse reporter username is not valid.') + } + } + }, + reason: { + type: DataTypes.STRING, + allowNull: false, + validate: { + reasonValid: function (value) { + const res = customVideosValidators.isVideoAbuseReasonValid(value) + if (res === false) throw new Error('Video abuse reason is not valid.') + } + } + } + }, + { + indexes: [ + { + fields: [ 'videoId' ] + }, + { + fields: [ 'reporterPodId' ] + } + ], + classMethods: { + associate, + + listForApi + }, + instanceMethods: { + toFormatedJSON + } + } + ) + + return VideoAbuse +} + +// --------------------------------------------------------------------------- + +function associate (models) { + this.belongsTo(models.Pod, { + foreignKey: { + name: 'reporterPodId', + allowNull: true + }, + onDelete: 'cascade' + }) + + this.belongsTo(models.Video, { + foreignKey: { + name: 'videoId', + allowNull: false + }, + onDelete: 'cascade' + }) +} + +function listForApi (start, count, sort, callback) { + const query = { + offset: start, + limit: count, + order: [ modelUtils.getSort(sort) ], + include: [ + { + model: this.sequelize.models.Pod, + required: false + } + ] + } + + return this.findAndCountAll(query).asCallback(function (err, result) { + if (err) return callback(err) + + return callback(null, result.rows, result.count) + }) +} + +function toFormatedJSON () { + let reporterPodHost + + if (this.Pod) { + reporterPodHost = this.Pod.host + } else { + // It means it's our video + reporterPodHost = constants.CONFIG.WEBSERVER.HOST + } + + const json = { + id: this.id, + reporterPodHost, + reason: this.reason, + reporterUsername: this.reporterUsername, + videoId: this.videoId + } + + return json +} diff --git a/server/models/videoTag.js b/server/models/video-tag.js similarity index 100% rename from server/models/videoTag.js rename to server/models/video-tag.js diff --git a/server/models/video.js b/server/models/video.js index 3fe8368c7..4c197a835 100644 --- a/server/models/video.js +++ b/server/models/video.js @@ -248,6 +248,14 @@ function associate (models) { through: models.VideoTag, onDelete: 'cascade' }) + + this.hasMany(models.VideoAbuse, { + foreignKey: { + name: 'videoId', + allowNull: false + }, + onDelete: 'cascade' + }) } function generateMagnetUri () { diff --git a/server/tests/api/check-params/index.js b/server/tests/api/check-params/index.js index 3d6f09267..d0824f08a 100644 --- a/server/tests/api/check-params/index.js +++ b/server/tests/api/check-params/index.js @@ -6,3 +6,4 @@ require('./remotes') require('./users') require('./requests') require('./videos') +require('./video-abuses') diff --git a/server/tests/api/check-params/remotes.js b/server/tests/api/check-params/remotes.js index 30ba3b697..c1ab9fb2b 100644 --- a/server/tests/api/check-params/remotes.js +++ b/server/tests/api/check-params/remotes.js @@ -47,6 +47,10 @@ describe('Test remote videos API validators', function () { it('Should check when removing a video') }) + describe('When reporting abuse on a video', function () { + it('Should check when reporting a video abuse') + }) + after(function (done) { process.kill(-server.app.pid) diff --git a/server/tests/api/check-params/video-abuses.js b/server/tests/api/check-params/video-abuses.js new file mode 100644 index 000000000..8cb4ccdc1 --- /dev/null +++ b/server/tests/api/check-params/video-abuses.js @@ -0,0 +1,180 @@ +'use strict' + +const request = require('supertest') +const series = require('async/series') + +const loginUtils = require('../../utils/login') +const requestsUtils = require('../../utils/requests') +const serversUtils = require('../../utils/servers') +const usersUtils = require('../../utils/users') +const videosUtils = require('../../utils/videos') + +describe('Test video abuses API validators', function () { + let server = null + let userAccessToken = null + + // --------------------------------------------------------------- + + before(function (done) { + this.timeout(20000) + + series([ + function (next) { + serversUtils.flushTests(next) + }, + function (next) { + serversUtils.runServer(1, function (server1) { + server = server1 + + next() + }) + }, + function (next) { + loginUtils.loginAndGetAccessToken(server, function (err, token) { + if (err) throw err + server.accessToken = token + + next() + }) + }, + function (next) { + const username = 'user1' + const password = 'my super password' + + usersUtils.createUser(server.url, server.accessToken, username, password, next) + }, + function (next) { + const user = { + username: 'user1', + password: 'my super password' + } + + loginUtils.getUserAccessToken(server, user, function (err, accessToken) { + if (err) throw err + + userAccessToken = accessToken + + next() + }) + }, + // Upload some videos on each pods + function (next) { + const name = 'my super name for pod' + const description = 'my super description for pod' + const tags = [ 'tag' ] + const file = 'video_short2.webm' + videosUtils.uploadVideo(server.url, server.accessToken, name, description, tags, file, next) + }, + function (next) { + videosUtils.getVideosList(server.url, function (err, res) { + if (err) throw err + + const videos = res.body.data + server.video = videos[0] + + next() + }) + } + ], done) + }) + + describe('When listing video abuses', function () { + const path = '/api/v1/videos/abuse' + + it('Should fail with a bad start pagination', function (done) { + request(server.url) + .get(path) + .query({ start: 'hello' }) + .set('Authorization', 'Bearer ' + server.accessToken) + .set('Accept', 'application/json') + .expect(400, done) + }) + + it('Should fail with a bad count pagination', function (done) { + request(server.url) + .get(path) + .query({ count: 'hello' }) + .set('Accept', 'application/json') + .set('Authorization', 'Bearer ' + server.accessToken) + .expect(400, done) + }) + + it('Should fail with an incorrect sort', function (done) { + request(server.url) + .get(path) + .query({ sort: 'hello' }) + .set('Accept', 'application/json') + .set('Authorization', 'Bearer ' + server.accessToken) + .expect(400, done) + }) + + it('Should fail with a non authenticated user', function (done) { + request(server.url) + .get(path) + .query({ sort: 'hello' }) + .set('Accept', 'application/json') + .expect(401, done) + }) + + it('Should fail with a non admin user', function (done) { + request(server.url) + .get(path) + .query({ sort: 'hello' }) + .set('Accept', 'application/json') + .set('Authorization', 'Bearer ' + userAccessToken) + .expect(403, done) + }) + }) + + describe('When reporting a video abuse', function () { + const basePath = '/api/v1/videos/' + + it('Should fail with nothing', function (done) { + const path = basePath + server.video + '/abuse' + const data = {} + requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done) + }) + + it('Should fail with a wrong video', function (done) { + const wrongPath = '/api/v1/videos/blabla/abuse' + const data = {} + requestsUtils.makePostBodyRequest(server.url, wrongPath, server.accessToken, data, done) + }) + + it('Should fail with a non authenticated user', function (done) { + const data = {} + const path = basePath + server.video + '/abuse' + requestsUtils.makePostBodyRequest(server.url, path, 'hello', data, done, 401) + }) + + it('Should fail with a reason too short', function (done) { + const data = { + reason: 'h' + } + const path = basePath + server.video + '/abuse' + requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done) + }) + + it('Should fail with a reason too big', function (done) { + const data = { + reason: '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef' + + '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef' + + '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef' + + '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef' + } + const path = basePath + server.video + '/abuse' + requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done) + }) + }) + + after(function (done) { + process.kill(-server.app.pid) + + // Keep the logs if the test failed + if (this.ok) { + serversUtils.flushTests(done) + } else { + done() + } + }) +}) diff --git a/server/tests/api/friends-advanced.js b/server/tests/api/friends-advanced.js index 0a2d58d82..708138bc9 100644 --- a/server/tests/api/friends-advanced.js +++ b/server/tests/api/friends-advanced.js @@ -86,7 +86,7 @@ describe('Test advanced friends', function () { getFriendsList(5, function (err, res) { if (err) throw err - expect(res.body.length).to.equal(0) + expect(res.body.data.length).to.equal(0) done() }) @@ -111,7 +111,7 @@ describe('Test advanced friends', function () { getFriendsList(i, function (err, res) { if (err) throw err - expect(res.body.length).to.equal(0) + expect(res.body.data.length).to.equal(0) callback() }) @@ -140,7 +140,7 @@ describe('Test advanced friends', function () { getFriendsList(i, function (err, res) { if (err) throw err - expect(res.body.length).to.equal(3) + expect(res.body.data.length).to.equal(3) callback() }) @@ -182,7 +182,7 @@ describe('Test advanced friends', function () { if (err) throw err // Pod 4 didn't know pod 1 and 2 removed it - expect(res.body.length).to.equal(3) + expect(res.body.data.length).to.equal(3) next() }) }, @@ -200,7 +200,7 @@ describe('Test advanced friends', function () { if (err) throw err // Pod 4 should not be our friend - const result = res.body + const result = res.body.data expect(result.length).to.equal(3) for (const pod of result) { expect(pod.host).not.equal(servers[3].host) diff --git a/server/tests/api/friends-basic.js b/server/tests/api/friends-basic.js index 3a904dbd7..6f37ff291 100644 --- a/server/tests/api/friends-basic.js +++ b/server/tests/api/friends-basic.js @@ -28,7 +28,7 @@ describe('Test basic friends', function () { podsUtils.getFriendsList(serverToTest.url, function (err, res) { if (err) throw err - const result = res.body + const result = res.body.data expect(result).to.be.an('array') expect(result.length).to.equal(2) @@ -65,7 +65,7 @@ describe('Test basic friends', function () { podsUtils.getFriendsList(server.url, function (err, res) { if (err) throw err - const result = res.body + const result = res.body.data expect(result).to.be.an('array') expect(result.length).to.equal(0) callback() @@ -90,7 +90,7 @@ describe('Test basic friends', function () { podsUtils.getFriendsList(servers[1].url, function (err, res) { if (err) throw err - const result = res.body + const result = res.body.data expect(result).to.be.an('array') expect(result.length).to.equal(1) @@ -107,7 +107,7 @@ describe('Test basic friends', function () { podsUtils.getFriendsList(servers[2].url, function (err, res) { if (err) throw err - const result = res.body + const result = res.body.data expect(result).to.be.an('array') expect(result.length).to.equal(1) @@ -154,7 +154,7 @@ describe('Test basic friends', function () { podsUtils.getFriendsList(servers[1].url, function (err, res) { if (err) throw err - const result = res.body + const result = res.body.data expect(result).to.be.an('array') expect(result.length).to.equal(0) @@ -167,7 +167,7 @@ describe('Test basic friends', function () { podsUtils.getFriendsList(url, function (err, res) { if (err) throw err - const result = res.body + const result = res.body.data expect(result).to.be.an('array') expect(result.length).to.equal(1) expect(result[0].host).not.to.be.equal(servers[1].host) diff --git a/server/tests/api/video-abuse.js b/server/tests/api/video-abuse.js new file mode 100644 index 000000000..58db17c42 --- /dev/null +++ b/server/tests/api/video-abuse.js @@ -0,0 +1,191 @@ +'use strict' + +const chai = require('chai') +const each = require('async/each') +const expect = chai.expect +const series = require('async/series') + +const loginUtils = require('../utils/login') +const podsUtils = require('../utils/pods') +const serversUtils = require('../utils/servers') +const videosUtils = require('../utils/videos') +const videoAbusesUtils = require('../utils/video-abuses') + +describe('Test video abuses', function () { + let servers = [] + + before(function (done) { + this.timeout(30000) + + series([ + // Run servers + function (next) { + serversUtils.flushAndRunMultipleServers(2, function (serversRun) { + servers = serversRun + next() + }) + }, + // Get the access tokens + function (next) { + each(servers, function (server, callbackEach) { + loginUtils.loginAndGetAccessToken(server, function (err, accessToken) { + if (err) return callbackEach(err) + + server.accessToken = accessToken + callbackEach() + }) + }, next) + }, + // Pod 1 make friends too + function (next) { + const server = servers[0] + podsUtils.makeFriends(server.url, server.accessToken, next) + }, + // Upload some videos on each pods + function (next) { + const name = 'my super name for pod 1' + const description = 'my super description for pod 1' + const tags = [ 'tag' ] + const file = 'video_short2.webm' + videosUtils.uploadVideo(servers[0].url, servers[0].accessToken, name, description, tags, file, next) + }, + function (next) { + const name = 'my super name for pod 2' + const description = 'my super description for pod 2' + const tags = [ 'tag' ] + const file = 'video_short2.webm' + videosUtils.uploadVideo(servers[1].url, servers[1].accessToken, name, description, tags, file, next) + }, + // Wait videos propagation + function (next) { + setTimeout(next, 11000) + }, + function (next) { + videosUtils.getVideosList(servers[0].url, function (err, res) { + if (err) throw err + + const videos = res.body.data + + expect(videos.length).to.equal(2) + + servers[0].video = videos.find(function (video) { return video.name === 'my super name for pod 1' }) + servers[1].video = videos.find(function (video) { return video.name === 'my super name for pod 2' }) + + next() + }) + } + ], done) + }) + + it('Should not have video abuses', function (done) { + videoAbusesUtils.getVideoAbusesList(servers[0].url, servers[0].accessToken, function (err, res) { + if (err) throw err + + expect(res.body.total).to.equal(0) + expect(res.body.data).to.be.an('array') + expect(res.body.data.length).to.equal(0) + + done() + }) + }) + + it('Should report abuse on a local video', function (done) { + this.timeout(15000) + + const reason = 'my super bad reason' + videoAbusesUtils.reportVideoAbuse(servers[0].url, servers[0].accessToken, servers[0].video.id, reason, function (err) { + if (err) throw err + + // We wait requests propagation, even if the pod 1 is not supposed to make a request to pod 2 + setTimeout(done, 11000) + }) + }) + + it('Should have 1 video abuses on pod 1 and 0 on pod 2', function (done) { + videoAbusesUtils.getVideoAbusesList(servers[0].url, servers[0].accessToken, function (err, res) { + if (err) throw err + + expect(res.body.total).to.equal(1) + expect(res.body.data).to.be.an('array') + expect(res.body.data.length).to.equal(1) + + const abuse = res.body.data[0] + expect(abuse.reason).to.equal('my super bad reason') + expect(abuse.reporterUsername).to.equal('root') + expect(abuse.reporterPodHost).to.equal('localhost:9001') + expect(abuse.videoId).to.equal(servers[0].video.id) + + videoAbusesUtils.getVideoAbusesList(servers[1].url, servers[1].accessToken, function (err, res) { + if (err) throw err + + expect(res.body.total).to.equal(0) + expect(res.body.data).to.be.an('array') + expect(res.body.data.length).to.equal(0) + + done() + }) + }) + }) + + it('Should report abuse on a remote video', function (done) { + this.timeout(15000) + + const reason = 'my super bad reason 2' + videoAbusesUtils.reportVideoAbuse(servers[0].url, servers[0].accessToken, servers[1].video.id, reason, function (err) { + if (err) throw err + + // We wait requests propagation + setTimeout(done, 11000) + }) + }) + + it('Should have 2 video abuse on pod 1 and 1 on pod 2', function (done) { + videoAbusesUtils.getVideoAbusesList(servers[0].url, servers[0].accessToken, function (err, res) { + if (err) throw err + + expect(res.body.total).to.equal(2) + expect(res.body.data).to.be.an('array') + expect(res.body.data.length).to.equal(2) + + let abuse = res.body.data[0] + expect(abuse.reason).to.equal('my super bad reason') + expect(abuse.reporterUsername).to.equal('root') + expect(abuse.reporterPodHost).to.equal('localhost:9001') + expect(abuse.videoId).to.equal(servers[0].video.id) + + abuse = res.body.data[1] + expect(abuse.reason).to.equal('my super bad reason 2') + expect(abuse.reporterUsername).to.equal('root') + expect(abuse.reporterPodHost).to.equal('localhost:9001') + expect(abuse.videoId).to.equal(servers[1].video.id) + + videoAbusesUtils.getVideoAbusesList(servers[1].url, servers[1].accessToken, function (err, res) { + if (err) throw err + + expect(res.body.total).to.equal(1) + expect(res.body.data).to.be.an('array') + expect(res.body.data.length).to.equal(1) + + let abuse = res.body.data[0] + expect(abuse.reason).to.equal('my super bad reason 2') + expect(abuse.reporterUsername).to.equal('root') + expect(abuse.reporterPodHost).to.equal('localhost:9001') + + done() + }) + }) + }) + + after(function (done) { + servers.forEach(function (server) { + process.kill(-server.app.pid) + }) + + // Keep the logs if the test failed + if (this.ok) { + serversUtils.flushTests(done) + } else { + done() + } + }) +}) diff --git a/server/tests/utils/video-abuses.js b/server/tests/utils/video-abuses.js new file mode 100644 index 000000000..596c824b3 --- /dev/null +++ b/server/tests/utils/video-abuses.js @@ -0,0 +1,73 @@ +'use strict' + +const request = require('supertest') + +const videosUtils = { + getVideoAbusesList, + getVideoAbusesListPagination, + getVideoAbusesListSort, + reportVideoAbuse +} + +// ---------------------- Export functions -------------------- + +function reportVideoAbuse (url, token, videoId, reason, specialStatus, end) { + if (!end) { + end = specialStatus + specialStatus = 204 + } + + const path = '/api/v1/videos/' + videoId + '/abuse' + + request(url) + .post(path) + .set('Accept', 'application/json') + .set('Authorization', 'Bearer ' + token) + .send({ reason }) + .expect(specialStatus) + .end(end) +} + +function getVideoAbusesList (url, token, end) { + const path = '/api/v1/videos/abuse' + + request(url) + .get(path) + .query({ sort: 'createdAt' }) + .set('Accept', 'application/json') + .set('Authorization', 'Bearer ' + token) + .expect(200) + .expect('Content-Type', /json/) + .end(end) +} + +function getVideoAbusesListPagination (url, token, start, count, end) { + const path = '/api/v1/videos/abuse' + + request(url) + .get(path) + .query({ start: start }) + .query({ count: count }) + .set('Accept', 'application/json') + .set('Authorization', 'Bearer ' + token) + .expect(200) + .expect('Content-Type', /json/) + .end(end) +} + +function getVideoAbusesListSort (url, token, sort, end) { + const path = '/api/v1/videos/abuse' + + request(url) + .get(path) + .query({ sort: sort }) + .set('Accept', 'application/json') + .set('Authorization', 'Bearer ' + token) + .expect(200) + .expect('Content-Type', /json/) + .end(end) +} + +// --------------------------------------------------------------------------- + +module.exports = videosUtils From b981a525c37d226b3fa59287a6ce338f54583d0c Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 4 Jan 2017 21:15:57 +0100 Subject: [PATCH 33/47] Server: we don't need the video name when removing a remote video --- server/helpers/custom-validators/remote/videos.js | 1 - server/models/video.js | 1 - 2 files changed, 2 deletions(-) diff --git a/server/helpers/custom-validators/remote/videos.js b/server/helpers/custom-validators/remote/videos.js index c3ca00e1c..7c27b9dbb 100644 --- a/server/helpers/custom-validators/remote/videos.js +++ b/server/helpers/custom-validators/remote/videos.js @@ -39,7 +39,6 @@ function isEachRemoteRequestVideosValid (requests) { ) || ( isRequestTypeRemoveValid(request.type) && - videosValidators.isVideoNameValid(video.name) && videosValidators.isVideoRemoteIdValid(video.remoteId) ) || ( diff --git a/server/models/video.js b/server/models/video.js index 4c197a835..b3060705d 100644 --- a/server/models/video.js +++ b/server/models/video.js @@ -218,7 +218,6 @@ function afterDestroy (video, options, next) { function (callback) { const params = { - name: video.name, remoteId: video.id } From bdfbd4f162d66c3a6bd7c312a99e0b692e830792 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 4 Jan 2017 22:23:07 +0100 Subject: [PATCH 34/47] Server: use crypto instead of ursa for pod signature --- package.json | 1 - server/helpers/peertube-crypto.js | 54 +++++++++++++++---- server/helpers/requests.js | 54 ++++++++++--------- server/initializers/constants.js | 13 ++++- server/middlewares/secure.js | 9 +++- .../validators/remote/signature.js | 2 +- server/models/request.js | 2 +- 7 files changed, 94 insertions(+), 41 deletions(-) diff --git a/package.json b/package.json index 5eadcc363..554ad16df 100644 --- a/package.json +++ b/package.json @@ -69,7 +69,6 @@ "safe-buffer": "^5.0.1", "scripty": "^1.5.0", "sequelize": "^3.27.0", - "ursa": "^0.9.1", "winston": "^2.1.1", "ws": "^1.1.1" }, diff --git a/server/helpers/peertube-crypto.js b/server/helpers/peertube-crypto.js index 610cb16cd..0f1e02ad6 100644 --- a/server/helpers/peertube-crypto.js +++ b/server/helpers/peertube-crypto.js @@ -1,9 +1,9 @@ 'use strict' +const crypto = require('crypto') const bcrypt = require('bcrypt') const fs = require('fs') const openssl = require('openssl-wrapper') -const ursa = require('ursa') const constants = require('../initializers/constants') const logger = require('./logger') @@ -16,12 +16,51 @@ const peertubeCrypto = { sign } -function checkSignature (publicKey, rawData, hexSignature) { - const crt = ursa.createPublicKey(publicKey) - const isValid = crt.hashAndVerify('sha256', new Buffer(rawData).toString('hex'), hexSignature, 'hex') +function checkSignature (publicKey, data, hexSignature) { + const verify = crypto.createVerify(constants.SIGNATURE_ALGORITHM) + + let dataString + if (typeof data === 'string') { + dataString = data + } else { + try { + dataString = JSON.stringify(data) + } catch (err) { + logger.error('Cannot check signature.', { error: err }) + return false + } + } + + verify.update(dataString, 'utf8') + + const isValid = verify.verify(publicKey, hexSignature, constants.SIGNATURE_ENCODING) return isValid } +function sign (data) { + const sign = crypto.createSign(constants.SIGNATURE_ALGORITHM) + + let dataString + if (typeof data === 'string') { + dataString = data + } else { + try { + dataString = JSON.stringify(data) + } catch (err) { + logger.error('Cannot sign data.', { error: err }) + return '' + } + } + + sign.update(dataString, 'utf8') + + // TODO: make async + const myKey = fs.readFileSync(constants.CONFIG.STORAGE.CERT_DIR + 'peertube.key.pem') + const signature = sign.sign(myKey, constants.SIGNATURE_ENCODING) + + return signature +} + function comparePassword (plainPassword, hashPassword, callback) { bcrypt.compare(plainPassword, hashPassword, function (err, isPasswordMatch) { if (err) return callback(err) @@ -52,13 +91,6 @@ function cryptPassword (password, callback) { }) } -function sign (data) { - const myKey = ursa.createPrivateKey(fs.readFileSync(constants.CONFIG.STORAGE.CERT_DIR + 'peertube.key.pem')) - const signature = myKey.hashAndSign('sha256', data, 'utf8', 'hex') - - return signature -} - // --------------------------------------------------------------------------- module.exports = peertubeCrypto diff --git a/server/helpers/requests.js b/server/helpers/requests.js index b0cda09fe..095b95e1c 100644 --- a/server/helpers/requests.js +++ b/server/helpers/requests.js @@ -28,31 +28,37 @@ function makeSecureRequest (params, callback) { url: constants.REMOTE_SCHEME.HTTP + '://' + params.toPod.host + params.path } - // Add data with POST requst ? - if (params.method === 'POST') { - requestParams.json = {} - - // Add signature if it is specified in the params - if (params.sign === true) { - const host = constants.CONFIG.WEBSERVER.HOST - - requestParams.json.signature = { - host, - signature: peertubeCrypto.sign(host) - } - } - - // If there are data informations - if (params.data) { - requestParams.json.data = params.data - request.post(requestParams, callback) - } else { - // No data - request.post(requestParams, callback) - } - } else { - request.get(requestParams, callback) + if (params.method !== 'POST') { + return callback(new Error('Cannot make a secure request with a non POST method.')) } + + requestParams.json = {} + + // Add signature if it is specified in the params + if (params.sign === true) { + const host = constants.CONFIG.WEBSERVER.HOST + + let dataToSign + if (params.data) { + dataToSign = dataToSign = params.data + } else { + // We do not have data to sign so we just take our host + // It is not ideal but the connection should be in HTTPS + dataToSign = host + } + + requestParams.json.signature = { + host, // Which host we pretend to be + signature: peertubeCrypto.sign(dataToSign) + } + } + + // If there are data informations + if (params.data) { + requestParams.json.data = params.data + } + + request.post(requestParams, callback) } // --------------------------------------------------------------------------- diff --git a/server/initializers/constants.js b/server/initializers/constants.js index 6ba8a9da0..a6adb75bf 100644 --- a/server/initializers/constants.js +++ b/server/initializers/constants.js @@ -118,16 +118,21 @@ const REQUEST_ENDPOINTS = { VIDEOS: 'videos' } -// --------------------------------------------------------------------------- - const REMOTE_SCHEME = { HTTP: 'https', WS: 'wss' } +// --------------------------------------------------------------------------- + +const SIGNATURE_ALGORITHM = 'RSA-SHA256' +const SIGNATURE_ENCODING = 'hex' + // Password encryption const BCRYPT_SALT_SIZE = 10 +// --------------------------------------------------------------------------- + // Express static paths (router) const STATIC_PATHS = { PREVIEWS: '/static/previews/', @@ -143,6 +148,8 @@ let STATIC_MAX_AGE = '30d' const THUMBNAILS_SIZE = '200x110' const PREVIEWS_SIZE = '640x480' +// --------------------------------------------------------------------------- + const USER_ROLES = { ADMIN: 'admin', USER: 'user' @@ -180,6 +187,8 @@ module.exports = { REQUESTS_LIMIT, RETRY_REQUESTS, SEARCHABLE_COLUMNS, + SIGNATURE_ALGORITHM, + SIGNATURE_ENCODING, SORTABLE_COLUMNS, STATIC_MAX_AGE, STATIC_PATHS, diff --git a/server/middlewares/secure.js b/server/middlewares/secure.js index 2aae715c4..b6e6d818b 100644 --- a/server/middlewares/secure.js +++ b/server/middlewares/secure.js @@ -23,7 +23,14 @@ function checkSignature (req, res, next) { logger.debug('Checking signature from %s.', host) - const signatureOk = peertubeCrypto.checkSignature(pod.publicKey, host, req.body.signature.signature) + let signatureShouldBe + if (req.body.data) { + signatureShouldBe = req.body.data + } else { + signatureShouldBe = host + } + + const signatureOk = peertubeCrypto.checkSignature(pod.publicKey, signatureShouldBe, req.body.signature.signature) if (signatureOk === true) { res.locals.secure = { diff --git a/server/middlewares/validators/remote/signature.js b/server/middlewares/validators/remote/signature.js index 5880a2c2c..002232c05 100644 --- a/server/middlewares/validators/remote/signature.js +++ b/server/middlewares/validators/remote/signature.js @@ -11,7 +11,7 @@ function signature (req, res, next) { req.checkBody('signature.host', 'Should have a signature host').isURL() req.checkBody('signature.signature', 'Should have a signature').notEmpty() - logger.debug('Checking signature parameters', { parameters: { signatureHost: req.body.signature.host } }) + logger.debug('Checking signature parameters', { parameters: { signature: req.body.signature } }) checkErrors(req, res, next) } diff --git a/server/models/request.js b/server/models/request.js index e18f8fe3d..bae227c05 100644 --- a/server/models/request.js +++ b/server/models/request.js @@ -122,7 +122,7 @@ function makeRequest (toPod, requestEndpoint, requestsToMake, callback) { 'Error sending secure request to %s pod.', toPod.host, { - error: err || new Error('Status code not 20x : ' + res.statusCode) + error: err ? err.message : 'Status code not 20x : ' + res.statusCode } ) From bb0b243c92577872a5f4d98f707e078082af4d2a Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 6 Jan 2017 23:24:20 +0100 Subject: [PATCH 35/47] Server: improve real world script --- server/tests/real-world/real-world.js | 29 ++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/server/tests/real-world/real-world.js b/server/tests/real-world/real-world.js index 2ae3dc15b..751d3923f 100644 --- a/server/tests/real-world/real-world.js +++ b/server/tests/real-world/real-world.js @@ -2,6 +2,7 @@ const each = require('async/each') const isEqual = require('lodash/isEqual') +const differenceWith = require('lodash/differenceWith') const program = require('commander') const series = require('async/series') @@ -20,28 +21,35 @@ program .option('-a, --action [interval]', 'Interval in ms for an action') .option('-i, --integrity [interval]', 'Interval in ms for an integrity check') .option('-f, --flush', 'Flush datas on exit') + .option('-d, --difference', 'Display difference if integrity is not okay') .parse(process.argv) -const createWeight = parseInt(program.create) || 5 -const removeWeight = parseInt(program.remove) || 4 +const createWeight = program.create !== undefined ? parseInt(program.create) : 5 +const removeWeight = program.remove !== undefined ? parseInt(program.remove) : 4 const flushAtExit = program.flush || false -const actionInterval = parseInt(program.action) || 500 -let integrityInterval = parseInt(program.integrity) || 60000 +const actionInterval = program.action !== undefined ? parseInt(program.action) : 500 +let integrityInterval = program.integrity !== undefined ? parseInt(program.integrity) : 60000 +const displayDiffOnFail = program.integrity || false const numberOfPods = 6 + // Wait requests between pods -const requestsMaxPerInterval = constants.REQUESTS_INTERVAL / actionInterval +const baseRequestInterval = integrityInterval < constants.REQUESTS_INTERVAL ? integrityInterval : constants.REQUESTS_INTERVAL +const requestsMaxPerInterval = baseRequestInterval / actionInterval const intervalsToMakeAllRequests = Math.ceil(requestsMaxPerInterval / constants.REQUESTS_LIMIT) const waitForBeforeIntegrityCheck = (intervalsToMakeAllRequests * constants.REQUESTS_INTERVAL) + 1000 -integrityInterval += waitForBeforeIntegrityCheck - console.log('Create weight: %d, remove weight: %d.', createWeight, removeWeight) if (flushAtExit) { console.log('Program will flush data on exit.') } else { console.log('Program will not flush data on exit.') } +if (displayDiffOnFail) { + console.log('Program will display diff on failure.') +} else { + console.log('Program will not display diff on failure') +} console.log('Interval in ms for each action: %d.', actionInterval) console.log('Interval in ms for each integrity check: %d.', integrityInterval) console.log('Will wait %d ms before an integrity check.', waitForBeforeIntegrityCheck) @@ -73,6 +81,8 @@ runServers(numberOfPods, function (err, servers) { }, actionInterval) setInterval(function () { + if (checking === true) return + console.log('Checking integrity...') checking = true @@ -196,6 +206,7 @@ function checkIntegrity (servers, callback) { delete serverVideo.id delete serverVideo.isLocal delete serverVideo.thumbnailPath + delete serverVideo.updatedAt } videos.push(serverVideos) @@ -206,6 +217,10 @@ function checkIntegrity (servers, callback) { if (!isEqual(video, videos[0])) { console.error('Integrity not ok!') + if (displayDiffOnFail) { + console.log(differenceWith(videos[0], video, isEqual)) + } + process.exit(-1) } } From ed04d94f6d7132055f97a2f757b85c03c5f2a0b6 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 6 Jan 2017 23:24:47 +0100 Subject: [PATCH 36/47] Server: try to have a better video integrity --- server/controllers/api/remote/videos.js | 56 ++++++++++-- server/controllers/api/videos.js | 114 ++++++++++++++++-------- server/helpers/utils.js | 16 +++- server/lib/friends.js | 66 +++++++++----- server/models/pod.js | 9 +- server/models/request.js | 4 +- 6 files changed, 194 insertions(+), 71 deletions(-) diff --git a/server/controllers/api/remote/videos.js b/server/controllers/api/remote/videos.js index d02da4463..6d768eae8 100644 --- a/server/controllers/api/remote/videos.js +++ b/server/controllers/api/remote/videos.js @@ -10,6 +10,7 @@ const secureMiddleware = middlewares.secure const videosValidators = middlewares.validators.remote.videos const signatureValidators = middlewares.validators.remote.signature const logger = require('../../../helpers/logger') +const utils = require('../../../helpers/utils') const router = express.Router() @@ -37,11 +38,11 @@ function remoteVideos (req, res, next) { switch (request.type) { case 'add': - addRemoteVideo(data, fromPod, callbackEach) + addRemoteVideoRetryWrapper(data, fromPod, callbackEach) break case 'update': - updateRemoteVideo(data, fromPod, callbackEach) + updateRemoteVideoRetryWrapper(data, fromPod, callbackEach) break case 'remove': @@ -63,13 +64,30 @@ function remoteVideos (req, res, next) { return res.type('json').status(204).end() } +// Handle retries on fail +function addRemoteVideoRetryWrapper (videoToCreateData, fromPod, finalCallback) { + utils.transactionRetryer( + function (callback) { + return addRemoteVideo(videoToCreateData, fromPod, callback) + }, + function (err) { + if (err) { + logger.error('Cannot insert the remote video with many retries.', { error: err }) + return finalCallback(err) + } + + return finalCallback() + } + ) +} + function addRemoteVideo (videoToCreateData, fromPod, finalCallback) { - logger.debug('Adding remote video "%s".', videoToCreateData.name) + logger.debug('Adding remote video "%s".', videoToCreateData.remoteId) waterfall([ function startTransaction (callback) { - db.sequelize.transaction().asCallback(function (err, t) { + db.sequelize.transaction({ isolationLevel: 'SERIALIZABLE' }).asCallback(function (err, t) { return callback(err, t) }) }, @@ -103,6 +121,7 @@ function addRemoteVideo (videoToCreateData, fromPod, finalCallback) { authorId: author.id, duration: videoToCreateData.duration, createdAt: videoToCreateData.createdAt, + // FIXME: updatedAt does not seems to be considered by Sequelize updatedAt: videoToCreateData.updatedAt } @@ -142,7 +161,8 @@ function addRemoteVideo (videoToCreateData, fromPod, finalCallback) { ], function (err, t) { if (err) { - logger.error('Cannot insert the remote video.') + // This is just a debug because we will retry the insert + logger.debug('Cannot insert the remote video.', { error: err }) // Abort transaction? if (t) t.rollback() @@ -157,8 +177,25 @@ function addRemoteVideo (videoToCreateData, fromPod, finalCallback) { }) } +// Handle retries on fail +function updateRemoteVideoRetryWrapper (videoAttributesToUpdate, fromPod, finalCallback) { + utils.transactionRetryer( + function (callback) { + return updateRemoteVideo(videoAttributesToUpdate, fromPod, callback) + }, + function (err) { + if (err) { + logger.error('Cannot update the remote video with many retries.', { error: err }) + return finalCallback(err) + } + + return finalCallback() + } + ) +} + function updateRemoteVideo (videoAttributesToUpdate, fromPod, finalCallback) { - logger.debug('Updating remote video "%s".', videoAttributesToUpdate.name) + logger.debug('Updating remote video "%s".', videoAttributesToUpdate.remoteId) waterfall([ @@ -208,7 +245,8 @@ function updateRemoteVideo (videoAttributesToUpdate, fromPod, finalCallback) { ], function (err, t) { if (err) { - logger.error('Cannot update the remote video.') + // This is just a debug because we will retry the insert + logger.debug('Cannot update the remote video.', { error: err }) // Abort transaction? if (t) t.rollback() @@ -238,7 +276,7 @@ function reportAbuseRemoteVideo (reportData, fromPod, callback) { if (err || !video) { if (!err) err = new Error('video not found') - logger.error('Cannot load video from host and remote id.', { error: err }) + logger.error('Cannot load video from id.', { error: err, id: reportData.videoRemoteId }) return callback(err) } @@ -260,7 +298,7 @@ function fetchVideo (podHost, remoteId, callback) { if (err || !video) { if (!err) err = new Error('video not found') - logger.error('Cannot load video from host and remote id.', { error: err }) + logger.error('Cannot load video from host and remote id.', { error: err, podHost, remoteId }) return callback(err) } diff --git a/server/controllers/api/videos.js b/server/controllers/api/videos.js index 6829804ec..4d45c11c0 100644 --- a/server/controllers/api/videos.js +++ b/server/controllers/api/videos.js @@ -70,13 +70,13 @@ router.put('/:id', oAuth.authenticate, reqFiles, validatorsVideos.videosUpdate, - updateVideo + updateVideoRetryWrapper ) router.post('/', oAuth.authenticate, reqFiles, validatorsVideos.videosAdd, - addVideo + addVideoRetryWrapper ) router.get('/:id', validatorsVideos.videosGet, @@ -103,19 +103,37 @@ module.exports = router // --------------------------------------------------------------------------- -function addVideo (req, res, next) { - const videoFile = req.files.videofile[0] +// Wrapper to video add that retry the function if there is a database error +// We need this because we run the transaction in SERIALIZABLE isolation that can fail +function addVideoRetryWrapper (req, res, next) { + utils.transactionRetryer( + function (callback) { + return addVideo(req, res, req.files.videofile[0], callback) + }, + function (err) { + if (err) { + logger.error('Cannot insert the video with many retries.', { error: err }) + return next(err) + } + + // TODO : include Location of the new video -> 201 + return res.type('json').status(204).end() + } + ) +} + +function addVideo (req, res, videoFile, callback) { const videoInfos = req.body waterfall([ - function startTransaction (callback) { - db.sequelize.transaction().asCallback(function (err, t) { - return callback(err, t) + function startTransaction (callbackWaterfall) { + db.sequelize.transaction({ isolationLevel: 'SERIALIZABLE' }).asCallback(function (err, t) { + return callbackWaterfall(err, t) }) }, - function findOrCreateAuthor (t, callback) { + function findOrCreateAuthor (t, callbackWaterfall) { const user = res.locals.oauth.token.User const name = user.username @@ -124,19 +142,19 @@ function addVideo (req, res, next) { const userId = user.id db.Author.findOrCreateAuthor(name, podId, userId, t, function (err, authorInstance) { - return callback(err, t, authorInstance) + return callbackWaterfall(err, t, authorInstance) }) }, - function findOrCreateTags (t, author, callback) { + function findOrCreateTags (t, author, callbackWaterfall) { const tags = videoInfos.tags db.Tag.findOrCreateTags(tags, t, function (err, tagInstances) { - return callback(err, t, author, tagInstances) + return callbackWaterfall(err, t, author, tagInstances) }) }, - function createVideoObject (t, author, tagInstances, callback) { + function createVideoObject (t, author, tagInstances, callbackWaterfall) { const videoData = { name: videoInfos.name, remoteId: null, @@ -148,74 +166,97 @@ function addVideo (req, res, next) { const video = db.Video.build(videoData) - return callback(null, t, author, tagInstances, video) + return callbackWaterfall(null, t, author, tagInstances, video) }, // Set the videoname the same as the id - function renameVideoFile (t, author, tagInstances, video, callback) { + function renameVideoFile (t, author, tagInstances, video, callbackWaterfall) { const videoDir = constants.CONFIG.STORAGE.VIDEOS_DIR const source = path.join(videoDir, videoFile.filename) const destination = path.join(videoDir, video.getVideoFilename()) fs.rename(source, destination, function (err) { - return callback(err, t, author, tagInstances, video) + if (err) return callbackWaterfall(err) + + // This is important in case if there is another attempt + videoFile.filename = video.getVideoFilename() + return callbackWaterfall(null, t, author, tagInstances, video) }) }, - function insertVideoIntoDB (t, author, tagInstances, video, callback) { + function insertVideoIntoDB (t, author, tagInstances, video, callbackWaterfall) { const options = { transaction: t } // Add tags association video.save(options).asCallback(function (err, videoCreated) { - if (err) return callback(err) + if (err) return callbackWaterfall(err) // Do not forget to add Author informations to the created video videoCreated.Author = author - return callback(err, t, tagInstances, videoCreated) + return callbackWaterfall(err, t, tagInstances, videoCreated) }) }, - function associateTagsToVideo (t, tagInstances, video, callback) { + function associateTagsToVideo (t, tagInstances, video, callbackWaterfall) { const options = { transaction: t } video.setTags(tagInstances, options).asCallback(function (err) { video.Tags = tagInstances - return callback(err, t, video) + return callbackWaterfall(err, t, video) }) }, - function sendToFriends (t, video, callback) { + function sendToFriends (t, video, callbackWaterfall) { video.toAddRemoteJSON(function (err, remoteVideo) { - if (err) return callback(err) + if (err) return callbackWaterfall(err) // Now we'll add the video's meta data to our friends - friends.addVideoToFriends(remoteVideo) - - return callback(null, t) + friends.addVideoToFriends(remoteVideo, t, function (err) { + return callbackWaterfall(err, t) + }) }) } ], function andFinally (err, t) { if (err) { - logger.error('Cannot insert the video.') + // This is just a debug because we will retry the insert + logger.debug('Cannot insert the video.', { error: err }) // Abort transaction? if (t) t.rollback() - return next(err) + return callback(err) } // Commit transaction t.commit() - // TODO : include Location of the new video -> 201 - return res.type('json').status(204).end() + logger.info('Video with name %s created.', videoInfos.name) + + return callback(null) }) } -function updateVideo (req, res, next) { +function updateVideoRetryWrapper (req, res, next) { + utils.transactionRetryer( + function (callback) { + return updateVideo(req, res, callback) + }, + function (err) { + if (err) { + logger.error('Cannot update the video with many retries.', { error: err }) + return next(err) + } + + // TODO : include Location of the new video -> 201 + return res.type('json').status(204).end() + } + ) +} + +function updateVideo (req, res, finalCallback) { const videoInstance = res.locals.video const videoInfosToUpdate = req.body @@ -267,26 +308,25 @@ function updateVideo (req, res, next) { const json = videoInstance.toUpdateRemoteJSON() // Now we'll update the video's meta data to our friends - friends.updateVideoToFriends(json) - - return callback(null, t) + friends.updateVideoToFriends(json, t, function (err) { + return callback(err, t) + }) } ], function andFinally (err, t) { if (err) { - logger.error('Cannot insert the video.') + logger.debug('Cannot update the video.', { error: err }) // Abort transaction? if (t) t.rollback() - return next(err) + return finalCallback(err) } // Commit transaction t.commit() - // TODO : include Location of the new video -> 201 - return res.type('json').status(204).end() + return finalCallback(null) }) } diff --git a/server/helpers/utils.js b/server/helpers/utils.js index 9f4b14582..a902850cd 100644 --- a/server/helpers/utils.js +++ b/server/helpers/utils.js @@ -1,6 +1,7 @@ 'use strict' const crypto = require('crypto') +const retry = require('async/retry') const logger = require('./logger') @@ -9,7 +10,8 @@ const utils = { cleanForExit, generateRandomString, isTestInstance, - getFormatedObjects + getFormatedObjects, + transactionRetryer } function badRequest (req, res, next) { @@ -46,6 +48,18 @@ function getFormatedObjects (objects, objectsTotal) { } } +function transactionRetryer (func, callback) { + retry({ + times: 5, + + errorFilter: function (err) { + const willRetry = (err.name === 'SequelizeDatabaseError') + logger.debug('Maybe retrying the transaction function.', { willRetry }) + return willRetry + } + }, func, callback) +} + // --------------------------------------------------------------------------- module.exports = utils diff --git a/server/lib/friends.js b/server/lib/friends.js index 4afb91b8b..3d3d0fdee 100644 --- a/server/lib/friends.js +++ b/server/lib/friends.js @@ -24,16 +24,33 @@ const friends = { sendOwnedVideosToPod } -function addVideoToFriends (videoData) { - createRequest('add', constants.REQUEST_ENDPOINTS.VIDEOS, videoData) +function addVideoToFriends (videoData, transaction, callback) { + const options = { + type: 'add', + endpoint: constants.REQUEST_ENDPOINTS.VIDEOS, + data: videoData, + transaction + } + createRequest(options, callback) } -function updateVideoToFriends (videoData) { - createRequest('update', constants.REQUEST_ENDPOINTS.VIDEOS, videoData) +function updateVideoToFriends (videoData, transaction, callback) { + const options = { + type: 'update', + endpoint: constants.REQUEST_ENDPOINTS.VIDEOS, + data: videoData, + transaction + } + createRequest(options, callback) } function removeVideoToFriends (videoParams) { - createRequest('remove', constants.REQUEST_ENDPOINTS.VIDEOS, videoParams) + const options = { + type: 'remove', + endpoint: constants.REQUEST_ENDPOINTS.VIDEOS, + data: videoParams + } + createRequest(options) } function reportAbuseVideoToFriend (reportData, video) { @@ -258,25 +275,35 @@ function makeRequestsToWinningPods (cert, podsList, callback) { } // Wrapper that populate "toIds" argument with all our friends if it is not specified -function createRequest (type, endpoint, data, toIds) { - if (toIds) return _createRequest(type, endpoint, data, toIds) +// { type, endpoint, data, toIds, transaction } +function createRequest (options, callback) { + if (!callback) callback = function () {} + if (options.toIds) return _createRequest(options, callback) // If the "toIds" pods is not specified, we send the request to all our friends - db.Pod.listAllIds(function (err, podIds) { + db.Pod.listAllIds(options.transaction, function (err, podIds) { if (err) { logger.error('Cannot get pod ids', { error: err }) return } - return _createRequest(type, endpoint, data, podIds) + const newOptions = Object.assign(options, { toIds: podIds }) + return _createRequest(newOptions, callback) }) } -function _createRequest (type, endpoint, data, toIds) { +// { type, endpoint, data, toIds, transaction } +function _createRequest (options, callback) { + const type = options.type + const endpoint = options.endpoint + const data = options.data + const toIds = options.toIds + const transaction = options.transaction + const pods = [] // If there are no destination pods abort - if (toIds.length === 0) return + if (toIds.length === 0) return callback(null) toIds.forEach(function (toPod) { pods.push(db.Pod.build({ id: toPod })) @@ -290,17 +317,14 @@ function _createRequest (type, endpoint, data, toIds) { } } - // We run in transaction to keep coherency between Request and RequestToPod tables - db.sequelize.transaction(function (t) { - const dbRequestOptions = { - transaction: t - } + const dbRequestOptions = { + transaction + } - return db.Request.create(createQuery, dbRequestOptions).then(function (request) { - return request.setPods(pods, dbRequestOptions) - }) - }).asCallback(function (err) { - if (err) logger.error('Error in createRequest transaction.', { error: err }) + return db.Request.create(createQuery, dbRequestOptions).asCallback(function (err, request) { + if (err) return callback(err) + + return request.setPods(pods, dbRequestOptions).asCallback(callback) }) } diff --git a/server/models/pod.js b/server/models/pod.js index 83ecd732e..8e7dd1fd8 100644 --- a/server/models/pod.js +++ b/server/models/pod.js @@ -115,11 +115,18 @@ function list (callback) { return this.findAll().asCallback(callback) } -function listAllIds (callback) { +function listAllIds (transaction, callback) { + if (!callback) { + callback = transaction + transaction = null + } + const query = { attributes: [ 'id' ] } + if (transaction) query.transaction = transaction + return this.findAll(query).asCallback(function (err, pods) { if (err) return callback(err) diff --git a/server/models/request.js b/server/models/request.js index bae227c05..1d6038044 100644 --- a/server/models/request.js +++ b/server/models/request.js @@ -291,8 +291,8 @@ function listWithLimitAndRandom (limit, callback) { order: [ [ 'id', 'ASC' ] ], - offset: start, - limit: limit, + // offset: start, + // limit: limit, include: [ this.sequelize.models.Pod ] } From bd14d16a29e2f90805d04b48378188517741a071 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 10 Jan 2017 22:24:42 +0100 Subject: [PATCH 37/47] Server: improve requests scheduler --- server/initializers/constants.js | 9 ++- server/lib/friends.js | 16 ++++- server/models/pod.js | 37 ++++++++++ server/models/request.js | 116 +++++++++++++++++++------------ 4 files changed, 128 insertions(+), 50 deletions(-) diff --git a/server/initializers/constants.js b/server/initializers/constants.js index a6adb75bf..97e3c5296 100644 --- a/server/initializers/constants.js +++ b/server/initializers/constants.js @@ -108,8 +108,10 @@ let REQUESTS_INTERVAL = 600000 // Number of requests in parallel we can make const REQUESTS_IN_PARALLEL = 10 -// How many requests we put in request -const REQUESTS_LIMIT = 10 +// To how many pods we send requests +const REQUESTS_LIMIT_PODS = 10 +// How many requests we send to a pod per interval +const REQUESTS_LIMIT_PER_POD = 5 // Number of requests to retry for replay requests module const RETRY_REQUESTS = 5 @@ -184,7 +186,8 @@ module.exports = { REQUEST_ENDPOINTS, REQUESTS_IN_PARALLEL, REQUESTS_INTERVAL, - REQUESTS_LIMIT, + REQUESTS_LIMIT_PODS, + REQUESTS_LIMIT_PER_POD, RETRY_REQUESTS, SEARCHABLE_COLUMNS, SIGNATURE_ALGORITHM, diff --git a/server/lib/friends.js b/server/lib/friends.js index 3d3d0fdee..f0575ff2f 100644 --- a/server/lib/friends.js +++ b/server/lib/friends.js @@ -54,7 +54,13 @@ function removeVideoToFriends (videoParams) { } function reportAbuseVideoToFriend (reportData, video) { - createRequest('report-abuse', constants.REQUEST_ENDPOINTS.VIDEOS, reportData, [ video.Author.podId ]) + const options = { + type: 'report-abuse', + endpoint: constants.REQUEST_ENDPOINTS.VIDEOS, + data: reportData, + toIds: [ video.Author.podId ] + } + createRequest(options) } function hasFriends (callback) { @@ -161,7 +167,13 @@ function sendOwnedVideosToPod (podId) { return } - createRequest('add', constants.REQUEST_ENDPOINTS.VIDEOS, remoteVideo, [ podId ]) + const options = { + type: 'add', + endpoint: constants.REQUEST_ENDPOINTS.VIDEOS, + data: remoteVideo, + toIds: [ podId ] + } + createRequest(options) }) }) }) diff --git a/server/models/pod.js b/server/models/pod.js index 8e7dd1fd8..b3c6db8e8 100644 --- a/server/models/pod.js +++ b/server/models/pod.js @@ -50,6 +50,7 @@ module.exports = function (sequelize, DataTypes) { incrementScores, list, listAllIds, + listRandomPodIdsWithRequest, listBadPods, load, loadByHost, @@ -134,6 +135,42 @@ function listAllIds (transaction, callback) { }) } +function listRandomPodIdsWithRequest (limit, callback) { + const self = this + + self.count().asCallback(function (err, count) { + if (err) return callback(err) + + // Optimization... + if (count === 0) return callback(null, []) + + let start = Math.floor(Math.random() * count) - limit + if (start < 0) start = 0 + + const query = { + attributes: [ 'id' ], + order: [ + [ 'id', 'ASC' ] + ], + offset: start, + limit: limit, + where: { + id: { + $in: [ + this.sequelize.literal('SELECT "podId" FROM "RequestToPods"') + ] + } + } + } + + return this.findAll(query).asCallback(function (err, pods) { + if (err) return callback(err) + + return callback(null, map(pods, 'id')) + }) + }) +} + function listBadPods (callback) { const query = { where: { diff --git a/server/models/request.js b/server/models/request.js index 1d6038044..26953e5f5 100644 --- a/server/models/request.js +++ b/server/models/request.js @@ -138,9 +138,9 @@ function makeRequests () { const self = this const RequestToPod = this.sequelize.models.RequestToPod - // We limit the size of the requests (REQUESTS_LIMIT) + // We limit the size of the requests // We don't want to stuck with the same failing requests so we get a random list - listWithLimitAndRandom.call(self, constants.REQUESTS_LIMIT, function (err, requests) { + listWithLimitAndRandom.call(self, constants.REQUESTS_LIMIT_PODS, constants.REQUESTS_LIMIT_PER_POD, function (err, requests) { if (err) { logger.error('Cannot get the list of requests.', { err: err }) return // Abort @@ -156,13 +156,15 @@ function makeRequests () { // We want to group requests by destinations pod and endpoint const requestsToMakeGrouped = {} + Object.keys(requests).forEach(function (toPodId) { + requests[toPodId].forEach(function (data) { + const request = data.request + const pod = data.pod + const hashKey = toPodId + request.endpoint - requests.forEach(function (request) { - request.Pods.forEach(function (toPod) { - const hashKey = toPod.id + request.endpoint if (!requestsToMakeGrouped[hashKey]) { requestsToMakeGrouped[hashKey] = { - toPodId: toPod.id, + toPod: pod, endpoint: request.endpoint, ids: [], // request ids, to delete them from the DB in the future datas: [] // requests data, @@ -179,36 +181,29 @@ function makeRequests () { eachLimit(Object.keys(requestsToMakeGrouped), constants.REQUESTS_IN_PARALLEL, function (hashKey, callbackEach) { const requestToMake = requestsToMakeGrouped[hashKey] + const toPod = requestToMake.toPod - // FIXME: SQL request inside a loop :/ - self.sequelize.models.Pod.load(requestToMake.toPodId, function (err, toPod) { - if (err) { - logger.error('Error finding pod by id.', { err: err }) - return callbackEach() + // Maybe the pod is not our friend anymore so simply remove it + if (!toPod) { + const requestIdsToDelete = requestToMake.ids + + logger.info('Removing %d requests of unexisting pod %s.', requestIdsToDelete.length, requestToMake.toPod.id) + RequestToPod.removePodOf.call(self, requestIdsToDelete, requestToMake.toPod.id) + return callbackEach() + } + + makeRequest(toPod, requestToMake.endpoint, requestToMake.datas, function (success) { + if (success === true) { + logger.debug('Removing requests for pod %s.', requestToMake.toPod.id, { requestsIds: requestToMake.ids }) + + goodPods.push(requestToMake.toPod.id) + + // Remove the pod id of these request ids + RequestToPod.removePodOf(requestToMake.ids, requestToMake.toPod.id, callbackEach) + } else { + badPods.push(requestToMake.toPod.id) + callbackEach() } - - // Maybe the pod is not our friend anymore so simply remove it - if (!toPod) { - const requestIdsToDelete = requestToMake.ids - - logger.info('Removing %d requests of unexisting pod %s.', requestIdsToDelete.length, requestToMake.toPodId) - RequestToPod.removePodOf.call(self, requestIdsToDelete, requestToMake.toPodId) - return callbackEach() - } - - makeRequest(toPod, requestToMake.endpoint, requestToMake.datas, function (success) { - if (success === true) { - logger.debug('Removing requests for pod %s.', requestToMake.toPodId, { requestsIds: requestToMake.ids }) - - goodPods.push(requestToMake.toPodId) - - // Remove the pod id of these request ids - RequestToPod.removePodOf(requestToMake.ids, requestToMake.toPodId, callbackEach) - } else { - badPods.push(requestToMake.toPodId) - callbackEach() - } - }) }) }, function () { // All the requests were made, we update the pods score @@ -275,31 +270,62 @@ function updatePodsScore (goodPods, badPods) { } } -function listWithLimitAndRandom (limit, callback) { +function listWithLimitAndRandom (limitPods, limitRequestsPerPod, callback) { const self = this + const Pod = this.sequelize.models.Pod - self.count().asCallback(function (err, count) { + Pod.listRandomPodIdsWithRequest(limitPods, function (err, podIds) { if (err) return callback(err) - // Optimization... - if (count === 0) return callback(null, []) - - let start = Math.floor(Math.random() * count) - limit - if (start < 0) start = 0 + // We don't have friends that have requests + if (podIds.length === 0) return callback(null, []) + // The the first x requests of these pods + // It is very important to sort by id ASC to keep the requests order! const query = { order: [ [ 'id', 'ASC' ] ], - // offset: start, - // limit: limit, - include: [ this.sequelize.models.Pod ] + include: [ + { + model: self.sequelize.models.Pod, + where: { + id: { + $in: podIds + } + } + } + ] } - self.findAll(query).asCallback(callback) + self.findAll(query).asCallback(function (err, requests) { + if (err) return callback(err) + + const requestsGrouped = groupAndTruncateRequests(requests, limitRequestsPerPod) + return callback(err, requestsGrouped) + }) }) } +function groupAndTruncateRequests (requests, limitRequestsPerPod) { + const requestsGrouped = {} + + requests.forEach(function (request) { + request.Pods.forEach(function (pod) { + if (!requestsGrouped[pod.id]) requestsGrouped[pod.id] = [] + + if (requestsGrouped[pod.id].length < limitRequestsPerPod) { + requestsGrouped[pod.id].push({ + request, + pod + }) + } + }) + }) + + return requestsGrouped +} + function removeAll (callback) { // Delete all requests this.truncate({ cascade: true }).asCallback(callback) From bf4ff8fe0be63422c05d42e12f25b454bb95d1a5 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 10 Jan 2017 22:33:00 +0100 Subject: [PATCH 38/47] Server: retry video abuse requests too --- server/controllers/api/videos.js | 68 ++++++++++++++++++++++++++------ 1 file changed, 56 insertions(+), 12 deletions(-) diff --git a/server/controllers/api/videos.js b/server/controllers/api/videos.js index 4d45c11c0..6573b1210 100644 --- a/server/controllers/api/videos.js +++ b/server/controllers/api/videos.js @@ -56,7 +56,7 @@ router.get('/abuse', router.post('/:id/abuse', oAuth.authenticate, validatorsVideos.videoAbuseReport, - reportVideoAbuse + reportVideoAbuseRetryWrapper ) router.get('/', @@ -375,7 +375,23 @@ function listVideoAbuses (req, res, next) { }) } -function reportVideoAbuse (req, res, next) { +function reportVideoAbuseRetryWrapper (req, res, next) { + utils.transactionRetryer( + function (callback) { + return reportVideoAbuse(req, res, callback) + }, + function (err) { + if (err) { + logger.error('Cannot report abuse to the video with many retries.', { error: err }) + return next(err) + } + + return res.type('json').status(204).end() + } + ) +} + +function reportVideoAbuse (req, res, finalCallback) { const videoInstance = res.locals.video const reporterUsername = res.locals.oauth.token.User.username @@ -386,21 +402,49 @@ function reportVideoAbuse (req, res, next) { reporterPodId: null // This is our pod that reported this abuse } - db.VideoAbuse.create(abuse).asCallback(function (err) { - if (err) return next(err) + waterfall([ - // We send the information to the destination pod - if (videoInstance.isOwned() === false) { - const reportData = { - reporterUsername, - reportReason: abuse.reason, - videoRemoteId: videoInstance.remoteId + function startTransaction (callback) { + db.sequelize.transaction().asCallback(function (err, t) { + return callback(err, t) + }) + }, + + function createAbuse (t, callback) { + db.VideoAbuse.create(abuse).asCallback(function (err, abuse) { + return callback(err, t, abuse) + }) + }, + + function sendToFriendsIfNeeded (t, abuse, callback) { + // We send the information to the destination pod + if (videoInstance.isOwned() === false) { + const reportData = { + reporterUsername, + reportReason: abuse.reason, + videoRemoteId: videoInstance.remoteId + } + + friends.reportAbuseVideoToFriend(reportData, videoInstance) } - friends.reportAbuseVideoToFriend(reportData, videoInstance) + return callback(null, t) } - return res.type('json').status(204).end() + ], function andFinally (err, t) { + if (err) { + logger.debug('Cannot update the video.', { error: err }) + + // Abort transaction? + if (t) t.rollback() + + return finalCallback(err) + } + + // Commit transaction + t.commit() + + return finalCallback(null) }) } From dea32aacde362a5fbd62a88cd32487768b788468 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 11 Jan 2017 16:22:50 +0100 Subject: [PATCH 39/47] Server: always check commit result --- server/controllers/api/remote/videos.js | 16 +++++++++++----- server/controllers/api/videos.js | 23 +++++++++++++++-------- server/initializers/migrator.js | 2 +- 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/server/controllers/api/remote/videos.js b/server/controllers/api/remote/videos.js index 6d768eae8..17bdce019 100644 --- a/server/controllers/api/remote/videos.js +++ b/server/controllers/api/remote/videos.js @@ -171,9 +171,12 @@ function addRemoteVideo (videoToCreateData, fromPod, finalCallback) { } // Commit transaction - t.commit() + t.commit().asCallback(function (err) { + if (err) return finalCallback(err) - return finalCallback() + logger.info('Remote video %s inserted.', videoToCreateData.videoToCreateData.name) + return finalCallback(null) + }) }) } @@ -254,10 +257,13 @@ function updateRemoteVideo (videoAttributesToUpdate, fromPod, finalCallback) { return finalCallback(err) } - // Commit transaction - t.commit() + // Commit transaction + t.commit().asCallback(function (err) { + if (err) return finalCallback(err) - return finalCallback() + logger.info('Remote video %s updated', videoAttributesToUpdate.name) + return finalCallback(null) + }) }) } diff --git a/server/controllers/api/videos.js b/server/controllers/api/videos.js index 6573b1210..df068f961 100644 --- a/server/controllers/api/videos.js +++ b/server/controllers/api/videos.js @@ -231,11 +231,12 @@ function addVideo (req, res, videoFile, callback) { } // Commit transaction - t.commit() + t.commit().asCallback(function (err) { + if (err) return callback(err) - logger.info('Video with name %s created.', videoInfos.name) - - return callback(null) + logger.info('Video with name %s created.', videoInfos.name) + return callback(null) + }) }) } @@ -324,9 +325,12 @@ function updateVideo (req, res, finalCallback) { } // Commit transaction - t.commit() + t.commit().asCallback(function (err) { + if (err) return finalCallback(err) - return finalCallback(null) + logger.info('Video with name %s updated.', videoInfosToUpdate.name) + return finalCallback(null) + }) }) } @@ -442,9 +446,12 @@ function reportVideoAbuse (req, res, finalCallback) { } // Commit transaction - t.commit() + t.commit().asCallback(function (err) { + if (err) return finalCallback(err) - return finalCallback(null) + logger.info('Abuse report for video %s created.', videoInstance.name) + return finalCallback(null) + }) }) } diff --git a/server/initializers/migrator.js b/server/initializers/migrator.js index eaecb4936..e5288b615 100644 --- a/server/initializers/migrator.js +++ b/server/initializers/migrator.js @@ -91,7 +91,7 @@ function executeMigration (actualVersion, entity, callback) { return callback(err) } - t.commit() + t.commit().asCallback(callback) }) }) }) From d8cc063e9775688a1631eda9203411a2dba0333c Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 11 Jan 2017 18:06:51 +0100 Subject: [PATCH 40/47] Server: do not break remote videos processing on error --- server/controllers/api/remote/videos.js | 33 ++++++++++++++++++------- server/models/request.js | 4 +-- server/tests/real-world/real-world.js | 2 +- 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/server/controllers/api/remote/videos.js b/server/controllers/api/remote/videos.js index 17bdce019..b9494f602 100644 --- a/server/controllers/api/remote/videos.js +++ b/server/controllers/api/remote/videos.js @@ -73,10 +73,10 @@ function addRemoteVideoRetryWrapper (videoToCreateData, fromPod, finalCallback) function (err) { if (err) { logger.error('Cannot insert the remote video with many retries.', { error: err }) - return finalCallback(err) } - return finalCallback() + // Do not return the error, continue the process + return finalCallback(null) } ) } @@ -174,7 +174,7 @@ function addRemoteVideo (videoToCreateData, fromPod, finalCallback) { t.commit().asCallback(function (err) { if (err) return finalCallback(err) - logger.info('Remote video %s inserted.', videoToCreateData.videoToCreateData.name) + logger.info('Remote video %s inserted.', videoToCreateData.name) return finalCallback(null) }) }) @@ -189,10 +189,10 @@ function updateRemoteVideoRetryWrapper (videoAttributesToUpdate, fromPod, finalC function (err) { if (err) { logger.error('Cannot update the remote video with many retries.', { error: err }) - return finalCallback(err) } - return finalCallback() + // Do not return the error, continue the process + return finalCallback(null) } ) } @@ -270,10 +270,18 @@ function updateRemoteVideo (videoAttributesToUpdate, fromPod, finalCallback) { function removeRemoteVideo (videoToRemoveData, fromPod, callback) { // We need the instance because we have to remove some other stuffs (thumbnail etc) fetchVideo(fromPod.host, videoToRemoveData.remoteId, function (err, video) { - if (err) return callback(err) + // Do not return the error, continue the process + if (err) return callback(null) logger.debug('Removing remote video %s.', video.remoteId) - video.destroy().asCallback(callback) + video.destroy().asCallback(function (err) { + // Do not return the error, continue the process + if (err) { + logger.error('Cannot remove remote video with id %s.', videoToRemoveData.remoteId, { error: err }) + } + + return callback(null) + }) }) } @@ -283,7 +291,8 @@ function reportAbuseRemoteVideo (reportData, fromPod, callback) { if (!err) err = new Error('video not found') logger.error('Cannot load video from id.', { error: err, id: reportData.videoRemoteId }) - return callback(err) + // Do not return the error, continue the process + return callback(null) } logger.debug('Reporting remote abuse for video %s.', video.id) @@ -295,7 +304,13 @@ function reportAbuseRemoteVideo (reportData, fromPod, callback) { videoId: video.id } - db.VideoAbuse.create(videoAbuseData).asCallback(callback) + db.VideoAbuse.create(videoAbuseData).asCallback(function (err) { + if (err) { + logger.error('Cannot create remote abuse video.', { error: err }) + } + + return callback(null) + }) }) } diff --git a/server/models/request.js b/server/models/request.js index 26953e5f5..e048c288b 100644 --- a/server/models/request.js +++ b/server/models/request.js @@ -152,8 +152,6 @@ function makeRequests () { return } - logger.info('Making requests to friends.') - // We want to group requests by destinations pod and endpoint const requestsToMakeGrouped = {} Object.keys(requests).forEach(function (toPodId) { @@ -176,6 +174,8 @@ function makeRequests () { }) }) + logger.info('Making requests to friends.', { requests: requestsToMakeGrouped }) + const goodPods = [] const badPods = [] diff --git a/server/tests/real-world/real-world.js b/server/tests/real-world/real-world.js index 751d3923f..9a63860ad 100644 --- a/server/tests/real-world/real-world.js +++ b/server/tests/real-world/real-world.js @@ -36,7 +36,7 @@ const numberOfPods = 6 // Wait requests between pods const baseRequestInterval = integrityInterval < constants.REQUESTS_INTERVAL ? integrityInterval : constants.REQUESTS_INTERVAL const requestsMaxPerInterval = baseRequestInterval / actionInterval -const intervalsToMakeAllRequests = Math.ceil(requestsMaxPerInterval / constants.REQUESTS_LIMIT) +const intervalsToMakeAllRequests = Math.ceil(requestsMaxPerInterval / (constants.REQUESTS_LIMIT_PER_POD * numberOfPods)) const waitForBeforeIntegrityCheck = (intervalsToMakeAllRequests * constants.REQUESTS_INTERVAL) + 1000 console.log('Create weight: %d, remove weight: %d.', createWeight, removeWeight) From 45abb8b97b8313f8f58a4a73b527882ad7b4af9c Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 11 Jan 2017 18:41:09 +0100 Subject: [PATCH 41/47] Server: rights check for update a video --- server/middlewares/validators/videos.js | 8 ++++++++ server/tests/api/check-params/videos.js | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/server/middlewares/validators/videos.js b/server/middlewares/validators/videos.js index ff18a99c2..3d7c04b60 100644 --- a/server/middlewares/validators/videos.js +++ b/server/middlewares/validators/videos.js @@ -53,6 +53,14 @@ function videosUpdate (req, res, next) { logger.debug('Checking videosUpdate parameters', { parameters: req.body }) checkErrors(req, res, function () { + if (res.locals.video.isOwned() === false) { + return res.status(403).send('Cannot update video of another pod') + } + + if (res.locals.video.Author.userId !== res.locals.oauth.token.User.id) { + return res.status(403).send('Cannot update video of another user') + } + checkVideoExists(req.params.id, res, next) }) } diff --git a/server/tests/api/check-params/videos.js b/server/tests/api/check-params/videos.js index d18305291..fac903715 100644 --- a/server/tests/api/check-params/videos.js +++ b/server/tests/api/check-params/videos.js @@ -378,6 +378,10 @@ describe('Test videos API validator', function () { } requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) }) + + it('Should fail with a video of another user') + + it('Should fail with a video of another pod') }) describe('When getting a video', function () { From f2cdb86675c3783ee903640b5b6f794fa09cdff2 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 11 Jan 2017 18:41:28 +0100 Subject: [PATCH 42/47] Server: add update case to real world script --- server/tests/real-world/real-world.js | 28 +++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/server/tests/real-world/real-world.js b/server/tests/real-world/real-world.js index 9a63860ad..896ba6cce 100644 --- a/server/tests/real-world/real-world.js +++ b/server/tests/real-world/real-world.js @@ -17,6 +17,7 @@ const videosUtils = require('../utils/videos') program .option('-c, --create [weight]', 'Weight for creating videos') .option('-r, --remove [weight]', 'Weight for removing videos') + .option('-u, --update [weight]', 'Weight for updating videos') .option('-p, --pods [n]', 'Number of pods to run (3 or 6)', /^3|6$/, 3) .option('-a, --action [interval]', 'Interval in ms for an action') .option('-i, --integrity [interval]', 'Interval in ms for an integrity check') @@ -26,6 +27,7 @@ program const createWeight = program.create !== undefined ? parseInt(program.create) : 5 const removeWeight = program.remove !== undefined ? parseInt(program.remove) : 4 +const updateWeight = program.update !== undefined ? parseInt(program.update) : 4 const flushAtExit = program.flush || false const actionInterval = program.action !== undefined ? parseInt(program.action) : 500 let integrityInterval = program.integrity !== undefined ? parseInt(program.integrity) : 60000 @@ -39,7 +41,7 @@ const requestsMaxPerInterval = baseRequestInterval / actionInterval const intervalsToMakeAllRequests = Math.ceil(requestsMaxPerInterval / (constants.REQUESTS_LIMIT_PER_POD * numberOfPods)) const waitForBeforeIntegrityCheck = (intervalsToMakeAllRequests * constants.REQUESTS_INTERVAL) + 1000 -console.log('Create weight: %d, remove weight: %d.', createWeight, removeWeight) +console.log('Create weight: %d, update weight: %d, remove weight: %d.', createWeight, updateWeight, removeWeight) if (flushAtExit) { console.log('Program will flush data on exit.') } else { @@ -71,10 +73,12 @@ runServers(numberOfPods, function (err, servers) { setInterval(function () { if (checking === true) return - const rand = getRandomInt(0, createWeight + removeWeight) + const rand = getRandomInt(0, createWeight + updateWeight + removeWeight) if (rand < createWeight) { upload(servers, getRandomNumServer(servers)) + } else if (rand < createWeight + updateWeight) { + update(servers, getRandomNumServer(servers)) } else { remove(servers, getRandomNumServer(servers)) } @@ -180,6 +184,26 @@ function upload (servers, numServer, callback) { videosUtils.uploadVideo(servers[numServer].url, servers[numServer].accessToken, name, description, tags, file, callback) } +function update (servers, numServer, callback) { + if (!callback) callback = function () {} + + videosUtils.getVideosList(servers[numServer].url, function (err, res) { + if (err) throw err + + const videos = res.body.data.filter(function (video) { return video.isLocal }) + if (videos.length === 0) return callback() + + const toUpdate = videos[getRandomInt(0, videos.length)].id + const name = Date.now() + ' name' + const description = Date.now() + ' description' + const tags = [ Date.now().toString().substring(0, 5) + 't1', Date.now().toString().substring(0, 5) + 't2' ] + + console.log('Updating video of server ' + numServer) + + videosUtils.updateVideo(servers[numServer].url, servers[numServer].accessToken, toUpdate, name, description, tags, callback) + }) +} + function remove (servers, numServer, callback) { if (!callback) callback = function () {} From edc5e86006bf5e4a2819c380bb65734fe9caa87e Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 11 Jan 2017 18:41:40 +0100 Subject: [PATCH 43/47] Server: transaction serializable for videos --- server/controllers/api/remote/videos.js | 2 +- server/controllers/api/videos.js | 2 +- server/models/request.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/controllers/api/remote/videos.js b/server/controllers/api/remote/videos.js index b9494f602..c45a86dbb 100644 --- a/server/controllers/api/remote/videos.js +++ b/server/controllers/api/remote/videos.js @@ -203,7 +203,7 @@ function updateRemoteVideo (videoAttributesToUpdate, fromPod, finalCallback) { waterfall([ function startTransaction (callback) { - db.sequelize.transaction().asCallback(function (err, t) { + db.sequelize.transaction({ isolationLevel: 'SERIALIZABLE' }).asCallback(function (err, t) { return callback(err, t) }) }, diff --git a/server/controllers/api/videos.js b/server/controllers/api/videos.js index df068f961..55d671f5b 100644 --- a/server/controllers/api/videos.js +++ b/server/controllers/api/videos.js @@ -264,7 +264,7 @@ function updateVideo (req, res, finalCallback) { waterfall([ function startTransaction (callback) { - db.sequelize.transaction().asCallback(function (err, t) { + db.sequelize.transaction({ isolationLevel: 'SERIALIZABLE' }).asCallback(function (err, t) { return callback(err, t) }) }, diff --git a/server/models/request.js b/server/models/request.js index e048c288b..cd52ea767 100644 --- a/server/models/request.js +++ b/server/models/request.js @@ -174,7 +174,7 @@ function makeRequests () { }) }) - logger.info('Making requests to friends.', { requests: requestsToMakeGrouped }) + logger.info('Making requests to friends.') const goodPods = [] const badPods = [] From 63d00f5ded0aad25eeb50111da65b6daa46bcb24 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 11 Jan 2017 19:15:23 +0100 Subject: [PATCH 44/47] Server: fix update right checks --- server/middlewares/validators/videos.js | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/server/middlewares/validators/videos.js b/server/middlewares/validators/videos.js index 3d7c04b60..4fe6dcd8b 100644 --- a/server/middlewares/validators/videos.js +++ b/server/middlewares/validators/videos.js @@ -53,15 +53,18 @@ function videosUpdate (req, res, next) { logger.debug('Checking videosUpdate parameters', { parameters: req.body }) checkErrors(req, res, function () { - if (res.locals.video.isOwned() === false) { - return res.status(403).send('Cannot update video of another pod') - } + checkVideoExists(req.params.id, res, function () { + // We need to make additional checks + if (res.locals.video.isOwned() === false) { + return res.status(403).send('Cannot update video of another pod') + } - if (res.locals.video.Author.userId !== res.locals.oauth.token.User.id) { - return res.status(403).send('Cannot update video of another user') - } + if (res.locals.video.Author.userId !== res.locals.oauth.token.User.id) { + return res.status(403).send('Cannot update video of another user') + } - checkVideoExists(req.params.id, res, next) + next() + }) }) } From 7f4e7c36373217b8e92cf227c71999a0ce9a15d9 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 12 Jan 2017 09:47:21 +0100 Subject: [PATCH 45/47] Server: fix update remote video infohash --- server/controllers/api/videos.js | 14 +++++++++++-- server/models/video.js | 3 ++- server/tests/api/multiple-pods.js | 29 +++++++++++++++++++++------ server/tests/api/single-pod.js | 19 ++++++++++++++++-- server/tests/real-world/real-world.js | 2 +- 5 files changed, 55 insertions(+), 12 deletions(-) diff --git a/server/controllers/api/videos.js b/server/controllers/api/videos.js index 55d671f5b..2c4af520e 100644 --- a/server/controllers/api/videos.js +++ b/server/controllers/api/videos.js @@ -259,6 +259,7 @@ function updateVideoRetryWrapper (req, res, next) { function updateVideo (req, res, finalCallback) { const videoInstance = res.locals.video + const videoFieldsSave = videoInstance.toJSON() const videoInfosToUpdate = req.body waterfall([ @@ -280,12 +281,13 @@ function updateVideo (req, res, finalCallback) { }, function updateVideoIntoDB (t, tagInstances, callback) { - const options = { transaction: t } + const options = { + transaction: t + } if (videoInfosToUpdate.name) videoInstance.set('name', videoInfosToUpdate.name) if (videoInfosToUpdate.description) videoInstance.set('description', videoInfosToUpdate.description) - // Add tags association videoInstance.save(options).asCallback(function (err) { return callback(err, t, tagInstances) }) @@ -321,6 +323,14 @@ function updateVideo (req, res, finalCallback) { // Abort transaction? if (t) t.rollback() + // Force fields we want to update + // If the transaction is retried, sequelize will think the object has not changed + // So it will skip the SQL request, even if the last one was ROLLBACKed! + Object.keys(videoFieldsSave).forEach(function (key) { + const value = videoFieldsSave[key] + videoInstance.set(key, value) + }) + return finalCallback(err) } diff --git a/server/models/video.js b/server/models/video.js index b3060705d..ceed976b0 100644 --- a/server/models/video.js +++ b/server/models/video.js @@ -141,7 +141,8 @@ module.exports = function (sequelize, DataTypes) { } function beforeValidate (video, options, next) { - if (video.isOwned()) { + // Put a fake infoHash if it does not exists yet + if (video.isOwned() && !video.infoHash) { // 40 hexa length video.infoHash = '0123456789abcdef0123456789abcdef01234567' } diff --git a/server/tests/api/multiple-pods.js b/server/tests/api/multiple-pods.js index 4442a7ff7..169a9f2e0 100644 --- a/server/tests/api/multiple-pods.js +++ b/server/tests/api/multiple-pods.js @@ -4,7 +4,8 @@ const chai = require('chai') const each = require('async/each') const expect = chai.expect const series = require('async/series') -const webtorrent = new (require('webtorrent'))() +const WebTorrent = require('webtorrent') +const webtorrent = new WebTorrent() const loginUtils = require('../utils/login') const miscsUtils = require('../utils/miscs') @@ -311,7 +312,7 @@ describe('Test multiple pods', function () { expect(torrent.files.length).to.equal(1) expect(torrent.files[0].path).to.exist.and.to.not.equal('') - done() + webtorrent.remove(video.magnetUri, done) }) }) }) @@ -330,7 +331,7 @@ describe('Test multiple pods', function () { expect(torrent.files.length).to.equal(1) expect(torrent.files[0].path).to.exist.and.to.not.equal('') - done() + webtorrent.remove(video.magnetUri, done) }) }) }) @@ -349,7 +350,7 @@ describe('Test multiple pods', function () { expect(torrent.files.length).to.equal(1) expect(torrent.files[0].path).to.exist.and.to.not.equal('') - done() + webtorrent.remove(video.magnetUri, done) }) }) }) @@ -368,7 +369,7 @@ describe('Test multiple pods', function () { expect(torrent.files.length).to.equal(1) expect(torrent.files[0].path).to.exist.and.to.not.equal('') - done() + webtorrent.remove(video.magnetUri, done) }) }) }) @@ -390,7 +391,12 @@ describe('Test multiple pods', function () { }) it('Should have the video 3 updated on each pod', function (done) { + this.timeout(200000) + each(servers, function (server, callback) { + // Avoid "duplicate torrent" errors + const webtorrent = new WebTorrent() + videosUtils.getVideosList(server.url, function (err, res) { if (err) throw err @@ -404,7 +410,18 @@ describe('Test multiple pods', function () { expect(videoUpdated.tags).to.deep.equal([ 'tagup1', 'tagup2' ]) expect(miscsUtils.dateIsValid(videoUpdated.updatedAt, 20000)).to.be.true - callback() + videosUtils.testVideoImage(server.url, 'video_short3.webm', videoUpdated.thumbnailPath, function (err, test) { + if (err) throw err + expect(test).to.equal(true) + + webtorrent.add(videoUpdated.magnetUri, function (torrent) { + expect(torrent.files).to.exist + expect(torrent.files.length).to.equal(1) + expect(torrent.files[0].path).to.exist.and.to.not.equal('') + + webtorrent.remove(videoUpdated.magnetUri, callback) + }) + }) }) }, done) }) diff --git a/server/tests/api/single-pod.js b/server/tests/api/single-pod.js index 29512dfc6..04b93fac7 100644 --- a/server/tests/api/single-pod.js +++ b/server/tests/api/single-pod.js @@ -96,7 +96,7 @@ describe('Test a single pod', function () { expect(torrent.files.length).to.equal(1) expect(torrent.files[0].path).to.exist.and.to.not.equal('') - done() + webtorrent.remove(video.magnetUri, done) }) }) }) @@ -515,6 +515,8 @@ describe('Test a single pod', function () { }) it('Should have the video updated', function (done) { + this.timeout(60000) + videosUtils.getVideo(server.url, videoId, function (err, res) { if (err) throw err @@ -529,7 +531,20 @@ describe('Test a single pod', function () { expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true expect(miscsUtils.dateIsValid(video.updatedAt)).to.be.true - done() + videosUtils.testVideoImage(server.url, 'video_short3.webm', video.thumbnailPath, function (err, test) { + if (err) throw err + expect(test).to.equal(true) + + videoId = video.id + + webtorrent.add(video.magnetUri, function (torrent) { + expect(torrent.files).to.exist + expect(torrent.files.length).to.equal(1) + expect(torrent.files[0].path).to.exist.and.to.not.equal('') + + done() + }) + }) }) }) diff --git a/server/tests/real-world/real-world.js b/server/tests/real-world/real-world.js index 896ba6cce..941e43a2e 100644 --- a/server/tests/real-world/real-world.js +++ b/server/tests/real-world/real-world.js @@ -38,7 +38,7 @@ const numberOfPods = 6 // Wait requests between pods const baseRequestInterval = integrityInterval < constants.REQUESTS_INTERVAL ? integrityInterval : constants.REQUESTS_INTERVAL const requestsMaxPerInterval = baseRequestInterval / actionInterval -const intervalsToMakeAllRequests = Math.ceil(requestsMaxPerInterval / (constants.REQUESTS_LIMIT_PER_POD * numberOfPods)) +const intervalsToMakeAllRequests = Math.ceil(requestsMaxPerInterval / constants.REQUESTS_LIMIT_PER_POD) const waitForBeforeIntegrityCheck = (intervalsToMakeAllRequests * constants.REQUESTS_INTERVAL) + 1000 console.log('Create weight: %d, update weight: %d, remove weight: %d.', createWeight, updateWeight, removeWeight) From 790e65fcf7a0a9f065ecc68c5982efb80cd2e1ca Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 12 Jan 2017 10:06:03 +0100 Subject: [PATCH 46/47] Try to fix travis build --- server/tests/api/multiple-pods.js | 10 +++++----- server/tests/api/single-pod.js | 2 +- server/tests/real-world/real-world.js | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/server/tests/api/multiple-pods.js b/server/tests/api/multiple-pods.js index 169a9f2e0..df12ba0e9 100644 --- a/server/tests/api/multiple-pods.js +++ b/server/tests/api/multiple-pods.js @@ -312,7 +312,7 @@ describe('Test multiple pods', function () { expect(torrent.files.length).to.equal(1) expect(torrent.files[0].path).to.exist.and.to.not.equal('') - webtorrent.remove(video.magnetUri, done) + done() }) }) }) @@ -331,7 +331,7 @@ describe('Test multiple pods', function () { expect(torrent.files.length).to.equal(1) expect(torrent.files[0].path).to.exist.and.to.not.equal('') - webtorrent.remove(video.magnetUri, done) + done() }) }) }) @@ -350,7 +350,7 @@ describe('Test multiple pods', function () { expect(torrent.files.length).to.equal(1) expect(torrent.files[0].path).to.exist.and.to.not.equal('') - webtorrent.remove(video.magnetUri, done) + done() }) }) }) @@ -369,7 +369,7 @@ describe('Test multiple pods', function () { expect(torrent.files.length).to.equal(1) expect(torrent.files[0].path).to.exist.and.to.not.equal('') - webtorrent.remove(video.magnetUri, done) + done() }) }) }) @@ -419,7 +419,7 @@ describe('Test multiple pods', function () { expect(torrent.files.length).to.equal(1) expect(torrent.files[0].path).to.exist.and.to.not.equal('') - webtorrent.remove(videoUpdated.magnetUri, callback) + callback() }) }) }) diff --git a/server/tests/api/single-pod.js b/server/tests/api/single-pod.js index 04b93fac7..2ac83bbf4 100644 --- a/server/tests/api/single-pod.js +++ b/server/tests/api/single-pod.js @@ -96,7 +96,7 @@ describe('Test a single pod', function () { expect(torrent.files.length).to.equal(1) expect(torrent.files[0].path).to.exist.and.to.not.equal('') - webtorrent.remove(video.magnetUri, done) + done() }) }) }) diff --git a/server/tests/real-world/real-world.js b/server/tests/real-world/real-world.js index 941e43a2e..12ab06d6d 100644 --- a/server/tests/real-world/real-world.js +++ b/server/tests/real-world/real-world.js @@ -30,16 +30,16 @@ const removeWeight = program.remove !== undefined ? parseInt(program.remove) : 4 const updateWeight = program.update !== undefined ? parseInt(program.update) : 4 const flushAtExit = program.flush || false const actionInterval = program.action !== undefined ? parseInt(program.action) : 500 -let integrityInterval = program.integrity !== undefined ? parseInt(program.integrity) : 60000 +const integrityInterval = program.integrity !== undefined ? parseInt(program.integrity) : 60000 const displayDiffOnFail = program.integrity || false const numberOfPods = 6 // Wait requests between pods -const baseRequestInterval = integrityInterval < constants.REQUESTS_INTERVAL ? integrityInterval : constants.REQUESTS_INTERVAL +const baseRequestInterval = integrityInterval < constants.REQUESTS_INTERVAL ? constants.REQUESTS_INTERVAL : integrityInterval const requestsMaxPerInterval = baseRequestInterval / actionInterval const intervalsToMakeAllRequests = Math.ceil(requestsMaxPerInterval / constants.REQUESTS_LIMIT_PER_POD) -const waitForBeforeIntegrityCheck = (intervalsToMakeAllRequests * constants.REQUESTS_INTERVAL) + 1000 +const waitForBeforeIntegrityCheck = (intervalsToMakeAllRequests * constants.REQUESTS_INTERVAL) - integrityInterval + 1000 console.log('Create weight: %d, update weight: %d, remove weight: %d.', createWeight, updateWeight, removeWeight) if (flushAtExit) { From 91cc839af88730ba55f84997c56b85ea100070a7 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 12 Jan 2017 13:08:47 +0100 Subject: [PATCH 47/47] Server: fix single pod tests --- server/models/video.js | 2 +- server/tests/api/single-pod.js | 52 ++++++++++++++++---------------- server/tests/utils/videos.js | 54 ++++++++++++++++++++++------------ 3 files changed, 61 insertions(+), 47 deletions(-) diff --git a/server/models/video.js b/server/models/video.js index ceed976b0..17eff6428 100644 --- a/server/models/video.js +++ b/server/models/video.js @@ -335,7 +335,7 @@ function toFormatedJSON () { author: this.Author.name, duration: this.duration, tags: map(this.Tags, 'name'), - thumbnailPath: constants.STATIC_PATHS.THUMBNAILS + '/' + this.getThumbnailName(), + thumbnailPath: pathUtils.join(constants.STATIC_PATHS.THUMBNAILS, this.getThumbnailName()), createdAt: this.createdAt, updatedAt: this.updatedAt } diff --git a/server/tests/api/single-pod.js b/server/tests/api/single-pod.js index 2ac83bbf4..2db60448f 100644 --- a/server/tests/api/single-pod.js +++ b/server/tests/api/single-pod.js @@ -338,69 +338,69 @@ describe('Test a single pod', function () { }) it('Should list only the two first videos', function (done) { - videosUtils.getVideosListPagination(server.url, 0, 2, function (err, res) { + videosUtils.getVideosListPagination(server.url, 0, 2, 'name', function (err, res) { if (err) throw err const videos = res.body.data expect(res.body.total).to.equal(6) expect(videos.length).to.equal(2) - expect(videos[0].name === videosListBase[0].name) - expect(videos[1].name === videosListBase[1].name) + expect(videos[0].name).to.equal(videosListBase[0].name) + expect(videos[1].name).to.equal(videosListBase[1].name) done() }) }) it('Should list only the next three videos', function (done) { - videosUtils.getVideosListPagination(server.url, 2, 3, function (err, res) { + videosUtils.getVideosListPagination(server.url, 2, 3, 'name', function (err, res) { if (err) throw err const videos = res.body.data expect(res.body.total).to.equal(6) expect(videos.length).to.equal(3) - expect(videos[0].name === videosListBase[2].name) - expect(videos[1].name === videosListBase[3].name) - expect(videos[2].name === videosListBase[4].name) + expect(videos[0].name).to.equal(videosListBase[2].name) + expect(videos[1].name).to.equal(videosListBase[3].name) + expect(videos[2].name).to.equal(videosListBase[4].name) done() }) }) it('Should list the last video', function (done) { - videosUtils.getVideosListPagination(server.url, 5, 6, function (err, res) { + videosUtils.getVideosListPagination(server.url, 5, 6, 'name', function (err, res) { if (err) throw err const videos = res.body.data expect(res.body.total).to.equal(6) expect(videos.length).to.equal(1) - expect(videos[0].name === videosListBase[5].name) + expect(videos[0].name).to.equal(videosListBase[5].name) done() }) }) it('Should search the first video', function (done) { - videosUtils.searchVideoWithPagination(server.url, 'webm', 'name', 0, 1, function (err, res) { + videosUtils.searchVideoWithPagination(server.url, 'webm', 'name', 0, 1, 'name', function (err, res) { if (err) throw err const videos = res.body.data expect(res.body.total).to.equal(4) expect(videos.length).to.equal(1) - expect(videos[0].name === 'video_short.webm name') + expect(videos[0].name).to.equal('video_short1.webm name') done() }) }) it('Should search the last two videos', function (done) { - videosUtils.searchVideoWithPagination(server.url, 'webm', 'name', 2, 2, function (err, res) { + videosUtils.searchVideoWithPagination(server.url, 'webm', 'name', 2, 2, 'name', function (err, res) { if (err) throw err const videos = res.body.data expect(res.body.total).to.equal(4) expect(videos.length).to.equal(2) - expect(videos[0].name === 'video_short2.webm name') - expect(videos[1].name === 'video_short3.webm name') + expect(videos[0].name).to.equal('video_short3.webm name') + expect(videos[1].name).to.equal('video_short.webm name') done() }) @@ -476,12 +476,12 @@ describe('Test a single pod', function () { const videos = res.body.data expect(res.body.total).to.equal(6) expect(videos.length).to.equal(6) - expect(videos[5].name === 'video_short.mp4 name') - expect(videos[4].name === 'video_short.ogv name') - expect(videos[3].name === 'video_short.webm name') - expect(videos[2].name === 'video_short1.webm name') - expect(videos[1].name === 'video_short2.webm name') - expect(videos[0].name === 'video_short3.webm name') + expect(videos[0].name).to.equal('video_short.webm name') + expect(videos[1].name).to.equal('video_short.ogv name') + expect(videos[2].name).to.equal('video_short.mp4 name') + expect(videos[3].name).to.equal('video_short3.webm name') + expect(videos[4].name).to.equal('video_short2.webm name') + expect(videos[5].name).to.equal('video_short1.webm name') done() }) @@ -495,12 +495,12 @@ describe('Test a single pod', function () { expect(res.body.total).to.equal(4) expect(videos.length).to.equal(4) - expect(videos[0].name === 'video_short.webm name') - expect(videos[1].name === 'video_short1.webm name') - expect(videos[2].name === 'video_short2.webm name') - expect(videos[3].name === 'video_short3.webm name') + expect(videos[0].name).to.equal('video_short1.webm name') + expect(videos[1].name).to.equal('video_short2.webm name') + expect(videos[2].name).to.equal('video_short3.webm name') + expect(videos[3].name).to.equal('video_short.webm name') - videoId = videos[3].id + videoId = videos[2].id done() }) @@ -535,8 +535,6 @@ describe('Test a single pod', function () { if (err) throw err expect(test).to.equal(true) - videoId = video.id - webtorrent.add(video.magnetUri, function (torrent) { expect(torrent.files).to.exist expect(torrent.files.length).to.equal(1) diff --git a/server/tests/utils/videos.js b/server/tests/utils/videos.js index beafd3cf5..f94368437 100644 --- a/server/tests/utils/videos.js +++ b/server/tests/utils/videos.js @@ -58,17 +58,25 @@ function getVideosList (url, end) { .end(end) } -function getVideosListPagination (url, start, count, end) { +function getVideosListPagination (url, start, count, sort, end) { + if (!end) { + end = sort + sort = null + } + const path = '/api/v1/videos' - request(url) - .get(path) - .query({ start: start }) - .query({ count: count }) - .set('Accept', 'application/json') - .expect(200) - .expect('Content-Type', /json/) - .end(end) + const req = request(url) + .get(path) + .query({ start: start }) + .query({ count: count }) + + if (sort) req.query({ sort }) + + req.set('Accept', 'application/json') + .expect(200) + .expect('Content-Type', /json/) + .end(end) } function getVideosListSort (url, sort, end) { @@ -116,18 +124,26 @@ function searchVideo (url, search, field, end) { .end(end) } -function searchVideoWithPagination (url, search, field, start, count, end) { +function searchVideoWithPagination (url, search, field, start, count, sort, end) { + if (!end) { + end = sort + sort = null + } + const path = '/api/v1/videos' - request(url) - .get(path + '/search/' + search) - .query({ start: start }) - .query({ count: count }) - .query({ field: field }) - .set('Accept', 'application/json') - .expect(200) - .expect('Content-Type', /json/) - .end(end) + const req = request(url) + .get(path + '/search/' + search) + .query({ start: start }) + .query({ count: count }) + .query({ field: field }) + + if (sort) req.query({ sort }) + + req.set('Accept', 'application/json') + .expect(200) + .expect('Content-Type', /json/) + .end(end) } function searchVideoWithSort (url, search, sort, end) {