Merge branch 'postgresql'

.travis.yml

@@ -1,8 +1,8 @@
 language: node_js
 
 node_js:
-  - "4.6"
+  - "4"
-  - "6.9"
+  - "6"
 
 env:
   - CXX=g++-4.8
@@ -13,11 +13,12 @@ addons:
       - ubuntu-toolchain-r-test
     packages:
       - g++-4.8
+  postgresql: "9.4"
 
 sudo: false
 
 services:
-  - mongodb
+  - postgresql
 
 before_install: if [[ `npm -v` != 3* ]]; then npm i -g npm@3; fi
 
@@ -29,6 +30,13 @@ before_script:
   - cp ffmpeg-*-64bit-static/{ffmpeg,ffprobe,ffserver} $HOME/bin
   - export PATH=$HOME/bin:$PATH
   - export NODE_TEST_IMAGE=true
+  - psql -c 'create database peertube_test1;' -U postgres
+  - psql -c 'create database peertube_test2;' -U postgres
+  - psql -c 'create database peertube_test3;' -U postgres
+  - psql -c 'create database peertube_test4;' -U postgres
+  - psql -c 'create database peertube_test5;' -U postgres
+  - psql -c 'create database peertube_test6;' -U postgres
+  - psql -c "create user peertube with password 'peertube';" -U postgres
 
 after_failure:
   - cat test1/logs/all-logs.log

ARCHITECTURE.md

@@ -54,7 +54,16 @@
   * A pod is a websocket tracker which is responsible for all the video uploaded in it
   * A pod has an administrator that can add/remove users, make friends and quit friends
   * A pod has different user accounts that can upload videos
-  * All pods have an index of all videos of the network (name, origin pod url, small description, uploader username, magnet Uri, thumbnail name, created date and the thumbnail file). For example, a test with 1000000 videos with alphanum characters and the following lengths: name = 50, author = 50, url = 25, description = 250, magnerUri = 200, thumbnail name = 50 has a mongodb size of ~ 4GB. To this, we add 1 000 000 thumbnails of 5-15 KB so 15GB maximum
+  * All pods have an index of all videos of the network (name, origin pod url, small description, uploader username, magnet Uri, thumbnail name, created date and the thumbnail file). For example, a test with 1000000 videos (3 tags each) with alphanum characters and the following lengths: name = 50, author = 50, podHost = 25, description = 250, videoExtension = 4, remoteId = 50, infoHash = 50 and tag = 10 has a PostgreSQL size of ~ 2GB with all the useful indexes. To this, we add 1 000 000 thumbnails of 5-15 KB so 15GB maximum
+
+          table_name | row_estimate |   index    |   toast    |   table
+        pod          |       983416 | 140 MB     | 83 MB      | 57 MB
+        author       |        1e+06 | 229 MB     | 140 MB     | 89 MB
+        tag          |  2.96758e+06 | 309 MB     | 182 MB     | 127 MB
+        video        |        1e+06 | 723 MB     | 263 MB     | 460 MB
+        video_tag    |        3e+06 | 316 MB     | 212 MB     | 104 MB
+
+
   * After having uploaded a video, the server seeds it (WebSeed protocol), adds the meta data in its database and makes a secure request to all of its friends
   * If a user wants to watch a video, he asks its pod the magnetUri and the frontend adds the torrent (with WebTorrent), creates the HTML5 video tag and streams the file into it
   * A user watching a video seeds it too (BitTorrent) so another user who is watching the same video can get the data from the origin server and the user 1 (etc)

README.md

@@ -121,7 +121,7 @@ Thanks to [WebTorrent](https://github.com/feross/webtorrent), we can make P2P (t
   * **NodeJS >= 4.x**
   * **npm >= 3.x**
   * OpenSSL (cli)
-  * MongoDB
+  * PostgreSQL
   * ffmpeg
 
 #### Debian
@@ -131,7 +131,7 @@ Thanks to [WebTorrent](https://github.com/feross/webtorrent), we can make P2P (t
   * Run:
 
         # apt-get update
-        # apt-get install ffmpeg mongodb openssl
+        # apt-get install ffmpeg postgresql-9.4 openssl
         # npm install -g npm@3
 
 #### Other distribution... (PR welcome)
@@ -238,7 +238,7 @@ Here are some simple schemes:
 
 <img src="https://lutim.cpy.re/MyeS4q1g" alt="Join a network" />
 
-<img src="https://lutim.cpy.re/PqpTTzdP" alt="Many networks"
+<img src="https://lutim.cpy.re/PqpTTzdP" alt="Many networks" />
 
 </p>
 

client/package.json

@@ -18,14 +18,14 @@
   },
   "license": "GPLv3",
   "dependencies": {
-    "@angular/common": "~2.3.0",
+    "@angular/common": "~2.4.1",
-    "@angular/compiler": "~2.3.0",
+    "@angular/compiler": "~2.4.1",
-    "@angular/core": "~2.3.0",
+    "@angular/core": "~2.4.1",
-    "@angular/forms": "~2.3.0",
+    "@angular/forms": "~2.4.1",
-    "@angular/http": "~2.3.0",
+    "@angular/http": "~2.4.1",
-    "@angular/platform-browser": "~2.3.0",
+    "@angular/platform-browser": "~2.4.1",
-    "@angular/platform-browser-dynamic": "~2.3.0",
+    "@angular/platform-browser-dynamic": "~2.4.1",
-    "@angular/router": "~3.3.0",
+    "@angular/router": "~3.4.1",
     "@angularclass/hmr": "^1.2.0",
     "@angularclass/hmr-loader": "^3.0.2",
     "@types/core-js": "^0.9.28",
@@ -51,7 +51,7 @@
     "ie-shim": "^0.1.0",
     "intl": "^1.2.4",
     "json-loader": "^0.5.4",
-    "ng2-bootstrap": "1.1.16",
+    "ng2-bootstrap": "1.1.16-10",
     "ng2-file-upload": "^1.1.0",
     "ng2-meta": "^2.0.0",
     "node-sass": "^3.10.0",

client/src/app/admin/friends/friend-list/friend-list.component.html

@@ -15,7 +15,7 @@
       <td>{{ friend.id }}</td>
       <td>{{ friend.host }}</td>
       <td>{{ friend.score }}</td>
-      <td>{{ friend.createdDate | date: 'medium' }}</td>
+      <td>{{ friend.createdAt | date: 'medium' }}</td>
     </tr>
   </tbody>
 </table>

client/src/app/admin/friends/friend-list/friend-list.component.ts

@@ -30,7 +30,7 @@ export class FriendListComponent implements OnInit {
 
   private getFriends() {
     this.friendService.getFriends().subscribe(
-      friends => this.friends = friends,
+      res => this.friends = res.friends,
 
       err => alert(err.text)
     );

client/src/app/admin/friends/shared/friend.model.ts

@@ -2,5 +2,5 @@ export interface Friend {
   id: string;
   host: string;
   score: number;
-  createdDate: Date;
+  createdAt: Date;
 }

client/src/app/admin/friends/shared/friend.service.ts

@@ -2,7 +2,7 @@ import { Injectable } from '@angular/core';
 import { Observable } from 'rxjs/Observable';
 
 import { Friend } from './friend.model';
-import { AuthHttp, RestExtractor } from '../../../shared';
+import { AuthHttp, RestExtractor, ResultList } from '../../../shared';
 
 @Injectable()
 export class FriendService {
@@ -13,11 +13,10 @@ export class FriendService {
     private restExtractor: RestExtractor
   ) {}
 
-  getFriends(): Observable<Friend[]> {
+  getFriends() {
     return this.authHttp.get(FriendService.BASE_FRIEND_URL)
-                        // Not implemented as a data list by the server yet
+                        .map(this.restExtractor.extractDataList)
-                        // .map(this.restExtractor.extractDataList)
+                        .map(this.extractFriends)
-                        .map((res) => res.json())
                         .catch((res) => this.restExtractor.handleError(res));
   }
 
@@ -36,4 +35,11 @@ export class FriendService {
                         .map(res => res.status)
                         .catch((res) => this.restExtractor.handleError(res));
   }
+
+  private extractFriends(result: ResultList) {
+    const friends: Friend[] = result.data;
+    const totalFriends = result.total;
+
+    return { friends, totalFriends };
+  }
 }

client/src/app/admin/requests/request-stats/request-stats.component.html

@@ -18,6 +18,6 @@
 
   <div>
     <span class="label-description">Remaining requests:</span>
-    {{ stats.requests.length }}
+    {{ stats.totalRequests }}
   </div>
 </div>

client/src/app/admin/requests/request-stats/request-stats.component.ts

@@ -19,7 +19,7 @@ export class RequestStatsComponent implements OnInit, OnDestroy {
   }
 
   ngOnDestroy() {
-    if (this.stats.secondsInterval !== null) {
+    if (this.stats !== null && this.stats.secondsInterval !== null) {
       clearInterval(this.interval);
     }
   }

client/src/app/admin/requests/shared/request-stats.model.ts

@@ -7,18 +7,18 @@ export class RequestStats {
   maxRequestsInParallel: number;
   milliSecondsInterval: number;
   remainingMilliSeconds: number;
-  requests: Request[];
+  totalRequests: number;
 
   constructor(hash: {
     maxRequestsInParallel: number,
     milliSecondsInterval: number,
     remainingMilliSeconds: number,
-    requests: Request[];
+    totalRequests: number;
   }) {
     this.maxRequestsInParallel = hash.maxRequestsInParallel;
     this.milliSecondsInterval = hash.milliSecondsInterval;
     this.remainingMilliSeconds = hash.remainingMilliSeconds;
-    this.requests = hash.requests;
+    this.totalRequests = hash.totalRequests;
   }
 
   get remainingSeconds() {

client/src/app/admin/users/user-list/user-list.component.html

@@ -14,7 +14,7 @@
     <tr *ngFor="let user of users">
       <td>{{ user.id }}</td>
       <td>{{ user.username }}</td>
-      <td>{{ user.createdDate | date: 'medium' }}</td>
+      <td>{{ user.createdAt | date: 'medium' }}</td>
       <td class="text-right">
         <span class="glyphicon glyphicon-remove" *ngIf="!user.isAdmin()" (click)="removeUser(user)"></span>
       </td>

client/src/app/shared/auth/auth-user.model.ts

@@ -7,9 +7,6 @@ export class AuthUser extends User {
     USERNAME: 'username'
   };
 
-  id: string;
-  role: string;
-  username: string;
   tokens: Tokens;
 
   static load() {
@@ -17,7 +14,7 @@ export class AuthUser extends User {
     if (usernameLocalStorage) {
       return new AuthUser(
         {
-          id: localStorage.getItem(this.KEYS.ID),
+          id: parseInt(localStorage.getItem(this.KEYS.ID)),
           username: localStorage.getItem(this.KEYS.USERNAME),
           role: localStorage.getItem(this.KEYS.ROLE)
         },
@@ -35,7 +32,7 @@ export class AuthUser extends User {
     Tokens.flush();
   }
 
-  constructor(userHash: { id: string, username: string, role: string }, hashTokens: any) {
+  constructor(userHash: { id: number, username: string, role: string }, hashTokens: any) {
     super(userHash);
     this.tokens = new Tokens(hashTokens);
   }
@@ -58,7 +55,7 @@ export class AuthUser extends User {
   }
 
   save() {
-    localStorage.setItem(AuthUser.KEYS.ID, this.id);
+    localStorage.setItem(AuthUser.KEYS.ID, this.id.toString());
     localStorage.setItem(AuthUser.KEYS.USERNAME, this.username);
     localStorage.setItem(AuthUser.KEYS.ROLE, this.role);
     this.tokens.save();

client/src/app/shared/search/search-field.type.ts

@@ -1 +1 @@
-export type SearchField = "name" | "author" | "podUrl" | "magnetUri" | "tags";
+export type SearchField = "name" | "author" | "host" | "magnetUri" | "tags";

client/src/app/shared/search/search.component.ts

@@ -14,8 +14,8 @@ export class SearchComponent implements OnInit {
   fieldChoices = {
     name: 'Name',
     author: 'Author',
-    podUrl: 'Pod Url',
+    host: 'Pod Host',
-    magnetUri: 'Magnet Uri',
+    magnetUri: 'Magnet URI',
     tags: 'Tags'
   };
   searchCriterias: Search = {

client/src/app/shared/shared.module.ts

@@ -5,10 +5,10 @@ import { FormsModule, ReactiveFormsModule } from '@angular/forms';
 import { RouterModule } from '@angular/router';
 
 import { BytesPipe } from 'angular-pipes/src/math/bytes.pipe';
-import { DropdownModule } from 'ng2-bootstrap/components/dropdown';
+import { DropdownModule } from 'ng2-bootstrap/dropdown';
-import { ProgressbarModule } from 'ng2-bootstrap/components/progressbar';
+import { ProgressbarModule } from 'ng2-bootstrap/progressbar';
-import { PaginationModule } from 'ng2-bootstrap/components/pagination';
+import { PaginationModule } from 'ng2-bootstrap/pagination';
-import { ModalModule } from 'ng2-bootstrap/components/modal';
+import { ModalModule } from 'ng2-bootstrap/modal';
 import { FileUploadModule } from 'ng2-file-upload/ng2-file-upload';
 
 import { AUTH_HTTP_PROVIDERS } from './auth';
@@ -23,11 +23,12 @@ import { SearchComponent, SearchService } from './search';
     HttpModule,
     RouterModule,
 
-    DropdownModule,
+    DropdownModule.forRoot(),
-    FileUploadModule,
+    ModalModule.forRoot(),
-    ModalModule,
+    PaginationModule.forRoot(),
-    PaginationModule,
+    ProgressbarModule.forRoot(),
-    ProgressbarModule
+
+    FileUploadModule
   ],
 
   declarations: [

client/src/app/shared/users/user.model.ts

@@ -1,16 +1,16 @@
 export class User {
-  id: string;
+  id: number;
   username: string;
   role: string;
-  createdDate: Date;
+  createdAt: Date;
 
-  constructor(hash: { id: string, username: string, role: string, createdDate?: Date }) {
+  constructor(hash: { id: number, username: string, role: string, createdAt?: Date }) {
     this.id = hash.id;
     this.username = hash.username;
     this.role = hash.role;
 
-    if (hash.createdDate) {
+    if (hash.createdAt) {
-      this.createdDate = hash.createdDate;
+      this.createdAt = hash.createdAt;
     }
   }
 

client/src/app/videos/shared/sort-field.type.ts

@@ -1,3 +1,3 @@
 export type SortField = "name" | "-name"
                       | "duration" | "-duration"
-                      | "createdDate" | "-createdDate";
+                      | "createdAt" | "-createdAt";

client/src/app/videos/shared/video.model.ts

@@ -1,7 +1,7 @@
 export class Video {
   author: string;
   by: string;
-  createdDate: Date;
+  createdAt: Date;
   description: string;
   duration: string;
   id: string;
@@ -27,7 +27,7 @@ export class Video {
 
   constructor(hash: {
     author: string,
-    createdDate: string,
+    createdAt: string,
     description: string,
     duration: number;
     id: string,
@@ -39,7 +39,7 @@ export class Video {
     thumbnailPath: string
   }) {
     this.author  = hash.author;
-    this.createdDate = new Date(hash.createdDate);
+    this.createdAt = new Date(hash.createdAt);
     this.description = hash.description;
     this.duration = Video.createDurationString(hash.duration);
     this.id = hash.id;

client/src/app/videos/video-list/video-list.component.ts

@@ -145,7 +145,7 @@ export class VideoListComponent implements OnInit, OnDestroy {
       };
     }
 
-    this.sort = <SortField>routeParams['sort'] || '-createdDate';
+    this.sort = <SortField>routeParams['sort'] || '-createdAt';
 
     if (routeParams['page'] !== undefined) {
       this.pagination.currentPage = parseInt(routeParams['page']);

client/src/app/videos/video-list/video-miniature.component.html

@@ -23,6 +23,6 @@
     </span>
 
     <a [routerLink]="['/videos/list', { field: 'author', search: video.author, sort: currentSort }]" class="video-miniature-author">{{ video.by }}</a>
-    <span class="video-miniature-created-date">{{ video.createdDate | date:'short' }}</span>
+    <span class="video-miniature-created-at">{{ video.createdAt | date:'short' }}</span>
   </div>
 </div>

client/src/app/videos/video-list/video-miniature.component.scss

@@ -79,7 +79,7 @@
       }
     }
 
-    .video-miniature-author, .video-miniature-created-date {
+    .video-miniature-author, .video-miniature-created-at {
       display: block;
       margin-left: 1px;
       font-size: 12px;

client/src/app/videos/video-list/video-sort.component.ts

@@ -17,8 +17,8 @@ export class VideoSortComponent {
     '-name': 'Name - Desc',
     'duration': 'Duration - Asc',
     '-duration': 'Duration - Desc',
-    'createdDate': 'Created Date - Asc',
+    'createdAt': 'Created Date - Asc',
-    '-createdDate': 'Created Date - Desc'
+    '-createdAt': 'Created Date - Desc'
   };
 
   get choiceKeys() {

client/src/app/videos/video-watch/video-magnet.component.ts

@@ -1,6 +1,6 @@
 import { Component, Input, ViewChild } from '@angular/core';
 
-import { ModalDirective } from 'ng2-bootstrap/components/modal';
+import { ModalDirective } from 'ng2-bootstrap/modal';
 
 import { Video } from '../shared';
 

client/src/app/videos/video-watch/video-share.component.ts

@@ -1,6 +1,6 @@
 import { Component, Input, ViewChild } from '@angular/core';
 
-import { ModalDirective } from 'ng2-bootstrap/components/modal';
+import { ModalDirective } from 'ng2-bootstrap/modal';
 
 import { Video } from '../shared';
 

client/src/app/videos/video-watch/video-watch.component.html

@@ -47,7 +47,7 @@
                 {{ video.by }}
               </a>
             </span>
-          <span id="video-date">on {{ video.createdDate | date:'short' }}</span>
+          <span id="video-date">on {{ video.createdAt | date:'short' }}</span>
         </div>
       </div>
     </div>

client/src/vendor.ts

@@ -29,7 +29,7 @@ import 'angular-pipes/src/math/bytes.pipe';
 import 'ng2-file-upload';
 import 'video.js';
 import 'ng2-meta';
-import 'ng2-bootstrap/components/pagination';
+import 'ng2-bootstrap/pagination';
-import 'ng2-bootstrap/components/dropdown';
+import 'ng2-bootstrap/dropdown';
-import 'ng2-bootstrap/components/progressbar';
+import 'ng2-bootstrap/progressbar';
-import 'ng2-bootstrap/components/modal';
+import 'ng2-bootstrap/modal';

config/default.yaml

@@ -8,8 +8,10 @@ webserver:
 
 database:
   hostname: 'localhost'
-  port: 27017
+  port: 5432
-  suffix: '-dev'
+  suffix: '_dev'
+  username: 'peertube'
+  password: 'peertube'
 
 # From the project root directory
 storage:

config/production.yaml.example

@@ -1,3 +1,6 @@
+listen:
+  port: 9000
+
 # Correspond to your reverse proxy "listen" configuration
 webserver:
   https: false
@@ -5,4 +8,17 @@ webserver:
   port: 80
 
 database:
-  suffix: '-prod'
+  hostname: 'localhost'
+  port: 5432
+  suffix: '_prod'
+  username: peertube
+  password: peertube
+
+# From the project root directory
+storage:
+  certs: 'certs/'
+  videos: 'videos/'
+  logs: 'logs/'
+  previews: 'previews/'
+  thumbnails: 'thumbnails/'
+  torrents: 'torrents/'

config/test-1.yaml

@@ -6,7 +6,7 @@ webserver:
   port: 9001
 
 database:
-  suffix: '-test1'
+  suffix: '_test1'
 
 # From the project root directory
 storage:

config/test-2.yaml

@@ -6,7 +6,7 @@ webserver:
   port: 9002
 
 database:
-  suffix: '-test2'
+  suffix: '_test2'
 
 # From the project root directory
 storage:

config/test-3.yaml

@@ -6,7 +6,7 @@ webserver:
   port: 9003
 
 database:
-  suffix: '-test3'
+  suffix: '_test3'
 
 # From the project root directory
 storage:

config/test-4.yaml

@@ -6,7 +6,7 @@ webserver:
   port: 9004
 
 database:
-  suffix: '-test4'
+  suffix: '_test4'
 
 # From the project root directory
 storage:

config/test-5.yaml

@@ -6,7 +6,7 @@ webserver:
   port: 9005
 
 database:
-  suffix: '-test5'
+  suffix: '_test5'
 
 # From the project root directory
 storage:

config/test-6.yaml

@@ -6,7 +6,7 @@ webserver:
   port: 9006
 
 database:
-  suffix: '-test6'
+  suffix: '_test6'
 
 # From the project root directory
 storage:

config/test.yaml

@@ -6,4 +6,4 @@ webserver:
 
 database:
   hostname: 'localhost'
-  port: 27017
+  port: 5432

package.json

@@ -56,17 +56,19 @@
     "lodash": "^4.11.1",
     "magnet-uri": "^5.1.4",
     "mkdirp": "^0.5.1",
-    "mongoose": "^4.0.5",
     "morgan": "^1.5.3",
     "multer": "^1.1.0",
     "openssl-wrapper": "^0.3.4",
     "parse-torrent": "^5.8.0",
     "password-generator": "^2.0.2",
+    "pg": "^6.1.0",
+    "pg-hstore": "^2.3.2",
     "request": "^2.57.0",
     "request-replay": "^1.0.2",
     "rimraf": "^2.5.4",
+    "safe-buffer": "^5.0.1",
     "scripty": "^1.5.0",
-    "ursa": "^0.9.1",
+    "sequelize": "^3.27.0",
     "winston": "^2.1.1",
     "ws": "^1.1.1"
   },

scripts/clean/server/test.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env sh
 
 for i in $(seq 1 6); do
-  printf "use peertube-test%s;\ndb.dropDatabase();" "$i" | mongo
+  dropdb "peertube_test$i"
   rm -rf "./test$i"
+  createdb "peertube_test$i"
 done

scripts/danger/clean/cleaner.js

@@ -1,24 +1,25 @@
+const eachSeries = require('async/eachSeries')
 const rimraf = require('rimraf')
-const mongoose = require('mongoose')
-mongoose.Promise = global.Promise
 
 const constants = require('../../../server/initializers/constants')
+const db = require('../../../server/initializers/database')
 
-const mongodbUrl = 'mongodb://' + constants.CONFIG.DATABASE.HOSTNAME + ':' + constants.CONFIG.DATABASE.PORT + '/' + constants.CONFIG.DATABASE.DBNAME
+db.init(true, function () {
-mongoose.connect(mongodbUrl, function () {
+  db.sequelize.drop().asCallback(function (err) {
-  console.info('Deleting MongoDB %s database.', constants.CONFIG.DATABASE.DBNAME)
-  mongoose.connection.dropDatabase(function () {
-    mongoose.connection.close()
-  })
-})
-
-const STORAGE = constants.CONFIG.STORAGE
-Object.keys(STORAGE).forEach(function (storage) {
-  const storageDir = STORAGE[storage]
-
-  rimraf(storageDir, function (err) {
     if (err) throw err
 
-    console.info('Deleting %s.', storageDir)
+    console.info('Tables of %s deleted.', db.sequelize.config.database)
+
+    const STORAGE = constants.CONFIG.STORAGE
+    eachSeries(Object.keys(STORAGE), function (storage, callbackEach) {
+      const storageDir = STORAGE[storage]
+
+      rimraf(storageDir, function (err) {
+        console.info('%s deleted.', storageDir)
+        return callbackEach(err)
+      })
+    }, function () {
+      process.exit(0)
+    })
   })
 })

scripts/danger/clean/dev.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 
-read -p "This will remove all directories and Mongo database. Are you sure? " -n 1 -r
+read -p "This will remove all directories and SQL tables. Are you sure? (y/*) " -n 1 -r
+echo
 
 if [[ "$REPLY" =~ ^[Yy]$ ]]; then
   NODE_ENV=test node "./scripts/danger/clean/cleaner"

scripts/danger/clean/prod.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 
-read -p "This will remove all directories and Mongo database. Are you sure? " -n 1 -r
+read -p "This will remove all directories and SQL tables. Are you sure? (y/*) " -n 1 -r
+echo
 
 if [[ "$REPLY" =~ ^[Yy]$ ]]; then
   NODE_ENV=production node "./scripts/danger/clean/cleaner"

scripts/mongo-to-postgre.js

@@ -0,0 +1,244 @@
+#!/usr/bin/env node
+
+'use strict'
+
+// TODO: document this script
+
+const program = require('commander')
+const eachSeries = require('async/eachSeries')
+const series = require('async/series')
+const waterfall = require('async/waterfall')
+const fs = require('fs')
+const path = require('path')
+const MongoClient = require('mongodb').MongoClient
+
+const constants = require('../server/initializers/constants')
+
+program
+  .option('-mh, --mongo-host [host]', 'MongoDB host', 'localhost')
+  .option('-mp, --mongo-port [weight]', 'MongoDB port', '27017')
+  .option('-md, --mongo-database [dbname]', 'MongoDB database')
+  .parse(process.argv)
+
+if (!program.mongoDatabase) {
+  console.error('The mongodb database is mandatory.')
+  process.exit(-1)
+}
+
+const mongoUrl = 'mongodb://' + program.mongoHost + ':' + program.mongoPort + '/' + program.mongoDatabase
+const dbSequelize = require('../server/initializers/database')
+
+console.log('Connecting to ' + mongoUrl)
+MongoClient.connect(mongoUrl, function (err, dbMongo) {
+  if (err) throw err
+
+  console.log('Connected to ' + mongoUrl)
+
+  const videoMongo = dbMongo.collection('videos')
+  const userMongo = dbMongo.collection('users')
+  const podMongo = dbMongo.collection('pods')
+
+  podMongo.count(function (err, podsLength) {
+    if (err) throw err
+
+    if (podsLength > 0) {
+      console.error('You need to quit friends first.')
+      process.exit(-1)
+    }
+
+    console.log('Connecting to ' + dbSequelize.sequelize.config.database)
+    dbSequelize.init(true, function (err) {
+      if (err) throw err
+
+      console.log('Connected to SQL database %s.', dbSequelize.sequelize.config.database)
+
+      series([
+        function (next) {
+          dbSequelize.sequelize.sync({ force: true }).asCallback(next)
+        },
+
+        function (next) {
+          migrateVideos(videoMongo, dbSequelize, next)
+        },
+
+        function (next) {
+          migrateUsers(userMongo, dbSequelize, next)
+        }
+      ], function (err) {
+        if (err) console.error(err)
+
+        process.exit(0)
+      })
+    })
+  })
+})
+
+// ---------------------------------------------------------------------------
+
+function migrateUsers (userMongo, dbSequelize, callback) {
+  userMongo.find().toArray(function (err, mongoUsers) {
+    if (err) return callback(err)
+
+    eachSeries(mongoUsers, function (mongoUser, callbackEach) {
+      console.log('Migrating user %s', mongoUser.username)
+
+      const userData = {
+        username: mongoUser.username,
+        password: mongoUser.password,
+        role: mongoUser.role
+      }
+      const options = {
+        hooks: false
+      }
+
+      dbSequelize.User.create(userData, options).asCallback(callbackEach)
+    }, callback)
+  })
+}
+
+function migrateVideos (videoMongo, dbSequelize, finalCallback) {
+  videoMongo.find().toArray(function (err, mongoVideos) {
+    if (err) return finalCallback(err)
+
+    eachSeries(mongoVideos, function (mongoVideo, callbackEach) {
+      console.log('Migrating video %s.', mongoVideo.name)
+
+      waterfall([
+
+        function startTransaction (callback) {
+          dbSequelize.sequelize.transaction().asCallback(function (err, t) {
+            return callback(err, t)
+          })
+        },
+
+        function findOrCreatePod (t, callback) {
+          if (mongoVideo.remoteId === null) return callback(null, t, null)
+
+          const query = {
+            where: {
+              host: mongoVideo.podHost
+            },
+            defaults: {
+              host: mongoVideo.podHost
+            },
+            transaction: t
+          }
+
+          dbSequelize.Pod.findOrCreate(query).asCallback(function (err, result) {
+            // [ instance, wasCreated ]
+            return callback(err, t, result[0])
+          })
+        },
+
+        function findOrCreateAuthor (t, pod, callback) {
+          const podId = pod ? pod.id : null
+          const username = mongoVideo.author
+
+          const query = {
+            where: {
+              podId,
+              name: username
+            },
+            defaults: {
+              podId,
+              name: username
+            },
+            transaction: t
+          }
+
+          dbSequelize.Author.findOrCreate(query).asCallback(function (err, result) {
+            // [ instance, wasCreated ]
+            return callback(err, t, result[0])
+          })
+        },
+
+        function findOrCreateTags (t, author, callback) {
+          const tags = mongoVideo.tags
+          const tagInstances = []
+
+          eachSeries(tags, function (tag, callbackEach) {
+            const query = {
+              where: {
+                name: tag
+              },
+              defaults: {
+                name: tag
+              },
+              transaction: t
+            }
+
+            dbSequelize.Tag.findOrCreate(query).asCallback(function (err, res) {
+              if (err) return callbackEach(err)
+
+              // res = [ tag, isCreated ]
+              const tag = res[0]
+              tagInstances.push(tag)
+              return callbackEach()
+            })
+          }, function (err) {
+            return callback(err, t, author, tagInstances)
+          })
+        },
+
+        function createVideoObject (t, author, tagInstances, callback) {
+          const videoData = {
+            name: mongoVideo.name,
+            remoteId: mongoVideo.remoteId,
+            extname: mongoVideo.extname,
+            infoHash: mongoVideo.magnet.infoHash,
+            description: mongoVideo.description,
+            authorId: author.id,
+            duration: mongoVideo.duration,
+            createdAt: mongoVideo.createdDate
+          }
+
+          const video = dbSequelize.Video.build(videoData)
+
+          return callback(null, t, tagInstances, video)
+        },
+
+        function moveVideoFile (t, tagInstances, video, callback) {
+          const basePath = constants.CONFIG.STORAGE.VIDEOS_DIR
+          const src = path.join(basePath, mongoVideo._id.toString()) + video.extname
+          const dst = path.join(basePath, video.id) + video.extname
+          fs.rename(src, dst, function (err) {
+            if (err) return callback(err)
+
+            return callback(null, t, tagInstances, video)
+          })
+        },
+
+        function insertVideoIntoDB (t, tagInstances, video, callback) {
+          const options = {
+            transaction: t
+          }
+
+          video.save(options).asCallback(function (err, videoCreated) {
+            return callback(err, t, tagInstances, videoCreated)
+          })
+        },
+
+        function associateTagsToVideo (t, tagInstances, video, callback) {
+          const options = { transaction: t }
+
+          video.setTags(tagInstances, options).asCallback(function (err) {
+            return callback(err, t)
+          })
+        }
+
+      ], function (err, t) {
+        if (err) {
+          // Abort transaction?
+          if (t) t.rollback()
+
+          return callbackEach(err)
+        }
+
+        // Commit transaction
+        t.commit()
+
+        return callbackEach()
+      })
+    }, finalCallback)
+  })
+}

scripts/play.sh

@@ -5,7 +5,9 @@ if [ ! -f server.js ]; then
   exit -1
 fi
 
-for i in 1 2 3; do
+max=${1:-3}
+
+for i in $(seq 1 $max); do
   NODE_ENV=test NODE_APP_INSTANCE=$i node server.js &
   sleep 1
 done

scripts/update-host.js

@@ -5,31 +5,24 @@
 // TODO: document this script
 
 const fs = require('fs')
-const mongoose = require('mongoose')
 const parseTorrent = require('parse-torrent')
 
 const constants = require('../server/initializers/constants')
-const database = require('../server/initializers/database')
+const db = require('../server/initializers/database')
-
-database.connect()
 
 const friends = require('../server/lib/friends')
-const Video = mongoose.model('Video')
 
-friends.hasFriends(function (err, hasFriends) {
+db.init(true, function () {
-  if (err) throw err
+  friends.hasFriends(function (err, hasFriends) {
-
-  if (hasFriends === true) {
-    console.log('Cannot update host because you have friends!')
-    process.exit(-1)
-  }
-
-  console.log('Updating videos host in database.')
-  Video.update({ }, { podHost: constants.CONFIG.WEBSERVER.HOST }, { multi: true }, function (err) {
     if (err) throw err
 
+    if (hasFriends === true) {
+      console.log('Cannot update host because you have friends!')
+      process.exit(-1)
+    }
+
     console.log('Updating torrent files.')
-    Video.find().lean().exec(function (err, videos) {
+    db.Video.list(function (err, videos) {
       if (err) throw err
 
       videos.forEach(function (video) {

server.js

@@ -17,10 +17,10 @@ const app = express()
 
 // ----------- Database -----------
 const constants = require('./server/initializers/constants')
-const database = require('./server/initializers/database')
 const logger = require('./server/helpers/logger')
-
+// Initialize database and models
-database.connect()
+const db = require('./server/initializers/database')
+db.init()
 
 // ----------- Checker -----------
 const checker = require('./server/initializers/checker')
@@ -39,9 +39,7 @@ if (errorMessage !== null) {
 const customValidators = require('./server/helpers/custom-validators')
 const installer = require('./server/initializers/installer')
 const migrator = require('./server/initializers/migrator')
-const mongoose = require('mongoose')
 const routes = require('./server/controllers')
-const Request = mongoose.model('Request')
 
 // ----------- Command line -----------
 
@@ -59,7 +57,8 @@ app.use(expressValidator({
     customValidators.misc,
     customValidators.pods,
     customValidators.users,
-    customValidators.videos
+    customValidators.videos,
+    customValidators.remote.videos
   )
 }))
 
@@ -130,7 +129,7 @@ installer.installApplication(function (err) {
     // ----------- Make the server listening -----------
     server.listen(port, function () {
       // Activate the pool requests
-      Request.activate()
+      db.Request.activate()
 
       logger.info('Server listening on port %d', port)
       logger.info('Webserver: %s', constants.CONFIG.WEBSERVER.URL)

server/controllers/api/clients.js

@@ -1,13 +1,11 @@
 'use strict'
 
 const express = require('express')
-const mongoose = require('mongoose')
 
 const constants = require('../../initializers/constants')
+const db = require('../../initializers/database')
 const logger = require('../../helpers/logger')
 
-const Client = mongoose.model('OAuthClient')
-
 const router = express.Router()
 
 router.get('/local', getLocalClient)
@@ -27,12 +25,12 @@ function getLocalClient (req, res, next) {
     return res.type('json').status(403).end()
   }
 
-  Client.loadFirstClient(function (err, client) {
+  db.OAuthClient.loadFirstClient(function (err, client) {
     if (err) return next(err)
     if (!client) return next(new Error('No client available.'))
 
     res.json({
-      client_id: client._id,
+      client_id: client.clientId,
       client_secret: client.clientSecret
     })
   })

server/controllers/api/index.js

@@ -2,6 +2,8 @@
 
 const express = require('express')
 
+const utils = require('../../helpers/utils')
+
 const router = express.Router()
 
 const clientsController = require('./clients')
@@ -18,7 +20,7 @@ router.use('/requests', requestsController)
 router.use('/users', usersController)
 router.use('/videos', videosController)
 router.use('/ping', pong)
-router.use('/*', badRequest)
+router.use('/*', utils.badRequest)
 
 // ---------------------------------------------------------------------------
 
@@ -29,7 +31,3 @@ module.exports = router
 function pong (req, res, next) {
   return res.send('pong').status(200).end()
 }
-
-function badRequest (req, res, next) {
-  res.type('json').status(400).end()
-}

server/controllers/api/pods.js

@@ -1,10 +1,11 @@
 'use strict'
 
 const express = require('express')
-const mongoose = require('mongoose')
 const waterfall = require('async/waterfall')
 
+const db = require('../../initializers/database')
 const logger = require('../../helpers/logger')
+const utils = require('../../helpers/utils')
 const friends = require('../../lib/friends')
 const middlewares = require('../../middlewares')
 const admin = middlewares.admin
@@ -15,7 +16,6 @@ const validators = middlewares.validators.pods
 const signatureValidator = middlewares.validators.remote.signature
 
 const router = express.Router()
-const Pod = mongoose.model('Pod')
 
 router.get('/', listPods)
 router.post('/',
@@ -37,7 +37,7 @@ router.get('/quitfriends',
 )
 // Post because this is a secured request
 router.post('/remove',
-  signatureValidator,
+  signatureValidator.signature,
   checkSignature,
   removePods
 )
@@ -53,15 +53,15 @@ function addPods (req, res, next) {
 
   waterfall([
     function addPod (callback) {
-      const pod = new Pod(informations)
+      const pod = db.Pod.build(informations)
-      pod.save(function (err, podCreated) {
+      pod.save().asCallback(function (err, podCreated) {
         // Be sure about the number of parameters for the callback
         return callback(err, podCreated)
       })
     },
 
     function sendMyVideos (podCreated, callback) {
-      friends.sendOwnedVideosToPod(podCreated._id)
+      friends.sendOwnedVideosToPod(podCreated.id)
 
       callback(null)
     },
@@ -84,10 +84,10 @@ function addPods (req, res, next) {
 }
 
 function listPods (req, res, next) {
-  Pod.list(function (err, podsList) {
+  db.Pod.list(function (err, podsList) {
     if (err) return next(err)
 
-    res.json(getFormatedPods(podsList))
+    res.json(utils.getFormatedObjects(podsList, podsList.length))
   })
 }
 
@@ -111,11 +111,11 @@ function removePods (req, res, next) {
 
   waterfall([
     function loadPod (callback) {
-      Pod.loadByHost(host, callback)
+      db.Pod.loadByHost(host, callback)
     },
 
-    function removePod (pod, callback) {
+    function deletePod (pod, callback) {
-      pod.remove(callback)
+      pod.destroy().asCallback(callback)
     }
   ], function (err) {
     if (err) return next(err)
@@ -131,15 +131,3 @@ function quitFriends (req, res, next) {
     res.type('json').status(204).end()
   })
 }
-
-// ---------------------------------------------------------------------------
-
-function getFormatedPods (pods) {
-  const formatedPods = []
-
-  pods.forEach(function (pod) {
-    formatedPods.push(pod.toFormatedJSON())
-  })
-
-  return formatedPods
-}

server/controllers/api/remote.js

@@ -1,86 +0,0 @@
-'use strict'
-
-const each = require('async/each')
-const eachSeries = require('async/eachSeries')
-const express = require('express')
-const mongoose = require('mongoose')
-
-const middlewares = require('../../middlewares')
-const secureMiddleware = middlewares.secure
-const validators = middlewares.validators.remote
-const logger = require('../../helpers/logger')
-
-const router = express.Router()
-const Video = mongoose.model('Video')
-
-router.post('/videos',
-  validators.signature,
-  secureMiddleware.checkSignature,
-  validators.remoteVideos,
-  remoteVideos
-)
-
-// ---------------------------------------------------------------------------
-
-module.exports = router
-
-// ---------------------------------------------------------------------------
-
-function remoteVideos (req, res, next) {
-  const requests = req.body.data
-  const fromHost = req.body.signature.host
-
-  // We need to process in the same order to keep consistency
-  // TODO: optimization
-  eachSeries(requests, function (request, callbackEach) {
-    const videoData = request.data
-
-    if (request.type === 'add') {
-      addRemoteVideo(videoData, fromHost, callbackEach)
-    } else if (request.type === 'remove') {
-      removeRemoteVideo(videoData, fromHost, callbackEach)
-    } else {
-      logger.error('Unkown remote request type %s.', request.type)
-    }
-  }, function (err) {
-    if (err) logger.error('Error managing remote videos.', { error: err })
-  })
-
-  // We don't need to keep the other pod waiting
-  return res.type('json').status(204).end()
-}
-
-function addRemoteVideo (videoToCreateData, fromHost, callback) {
-  logger.debug('Adding remote video "%s".', videoToCreateData.name)
-
-  const video = new Video(videoToCreateData)
-  video.podHost = fromHost
-  Video.generateThumbnailFromBase64(video, videoToCreateData.thumbnailBase64, function (err) {
-    if (err) {
-      logger.error('Cannot generate thumbnail from base 64 data.', { error: err })
-      return callback(err)
-    }
-
-    video.save(callback)
-  })
-}
-
-function removeRemoteVideo (videoToRemoveData, fromHost, callback) {
-  // We need the list because we have to remove some other stuffs (thumbnail etc)
-  Video.listByHostAndRemoteId(fromHost, videoToRemoveData.remoteId, function (err, videosList) {
-    if (err) {
-      logger.error('Cannot list videos from host and magnets.', { error: err })
-      return callback(err)
-    }
-
-    if (videosList.length === 0) {
-      logger.error('No remote video was found for this pod.', { magnetUri: videoToRemoveData.magnetUri, podHost: fromHost })
-    }
-
-    each(videosList, function (video, callbackEach) {
-      logger.debug('Removing remote video %s.', video.magnetUri)
-
-      video.remove(callbackEach)
-    }, callback)
-  })
-}

server/controllers/api/remote/index.js

@@ -0,0 +1,16 @@
+'use strict'
+
+const express = require('express')
+
+const utils = require('../../../helpers/utils')
+
+const router = express.Router()
+
+const videosRemoteController = require('./videos')
+
+router.use('/videos', videosRemoteController)
+router.use('/*', utils.badRequest)
+
+// ---------------------------------------------------------------------------
+
+module.exports = router

server/controllers/api/remote/videos.js

@@ -0,0 +1,328 @@
+'use strict'
+
+const eachSeries = require('async/eachSeries')
+const express = require('express')
+const waterfall = require('async/waterfall')
+
+const db = require('../../../initializers/database')
+const middlewares = require('../../../middlewares')
+const secureMiddleware = middlewares.secure
+const videosValidators = middlewares.validators.remote.videos
+const signatureValidators = middlewares.validators.remote.signature
+const logger = require('../../../helpers/logger')
+const utils = require('../../../helpers/utils')
+
+const router = express.Router()
+
+router.post('/',
+  signatureValidators.signature,
+  secureMiddleware.checkSignature,
+  videosValidators.remoteVideos,
+  remoteVideos
+)
+
+// ---------------------------------------------------------------------------
+
+module.exports = router
+
+// ---------------------------------------------------------------------------
+
+function remoteVideos (req, res, next) {
+  const requests = req.body.data
+  const fromPod = res.locals.secure.pod
+
+  // We need to process in the same order to keep consistency
+  // TODO: optimization
+  eachSeries(requests, function (request, callbackEach) {
+    const data = request.data
+
+    switch (request.type) {
+      case 'add':
+        addRemoteVideoRetryWrapper(data, fromPod, callbackEach)
+        break
+
+      case 'update':
+        updateRemoteVideoRetryWrapper(data, fromPod, callbackEach)
+        break
+
+      case 'remove':
+        removeRemoteVideo(data, fromPod, callbackEach)
+        break
+
+      case 'report-abuse':
+        reportAbuseRemoteVideo(data, fromPod, callbackEach)
+        break
+
+      default:
+        logger.error('Unkown remote request type %s.', request.type)
+    }
+  }, function (err) {
+    if (err) logger.error('Error managing remote videos.', { error: err })
+  })
+
+  // We don't need to keep the other pod waiting
+  return res.type('json').status(204).end()
+}
+
+// Handle retries on fail
+function addRemoteVideoRetryWrapper (videoToCreateData, fromPod, finalCallback) {
+  utils.transactionRetryer(
+    function (callback) {
+      return addRemoteVideo(videoToCreateData, fromPod, callback)
+    },
+    function (err) {
+      if (err) {
+        logger.error('Cannot insert the remote video with many retries.', { error: err })
+      }
+
+      // Do not return the error, continue the process
+      return finalCallback(null)
+    }
+  )
+}
+
+function addRemoteVideo (videoToCreateData, fromPod, finalCallback) {
+  logger.debug('Adding remote video "%s".', videoToCreateData.remoteId)
+
+  waterfall([
+
+    function startTransaction (callback) {
+      db.sequelize.transaction({ isolationLevel: 'SERIALIZABLE' }).asCallback(function (err, t) {
+        return callback(err, t)
+      })
+    },
+
+    function findOrCreateAuthor (t, callback) {
+      const name = videoToCreateData.author
+      const podId = fromPod.id
+      // This author is from another pod so we do not associate a user
+      const userId = null
+
+      db.Author.findOrCreateAuthor(name, podId, userId, t, function (err, authorInstance) {
+        return callback(err, t, authorInstance)
+      })
+    },
+
+    function findOrCreateTags (t, author, callback) {
+      const tags = videoToCreateData.tags
+
+      db.Tag.findOrCreateTags(tags, t, function (err, tagInstances) {
+        return callback(err, t, author, tagInstances)
+      })
+    },
+
+    function createVideoObject (t, author, tagInstances, callback) {
+      const videoData = {
+        name: videoToCreateData.name,
+        remoteId: videoToCreateData.remoteId,
+        extname: videoToCreateData.extname,
+        infoHash: videoToCreateData.infoHash,
+        description: videoToCreateData.description,
+        authorId: author.id,
+        duration: videoToCreateData.duration,
+        createdAt: videoToCreateData.createdAt,
+        // FIXME: updatedAt does not seems to be considered by Sequelize
+        updatedAt: videoToCreateData.updatedAt
+      }
+
+      const video = db.Video.build(videoData)
+
+      return callback(null, t, tagInstances, video)
+    },
+
+    function generateThumbnail (t, tagInstances, video, callback) {
+      db.Video.generateThumbnailFromData(video, videoToCreateData.thumbnailData, function (err) {
+        if (err) {
+          logger.error('Cannot generate thumbnail from data.', { error: err })
+          return callback(err)
+        }
+
+        return callback(err, t, tagInstances, video)
+      })
+    },
+
+    function insertVideoIntoDB (t, tagInstances, video, callback) {
+      const options = {
+        transaction: t
+      }
+
+      video.save(options).asCallback(function (err, videoCreated) {
+        return callback(err, t, tagInstances, videoCreated)
+      })
+    },
+
+    function associateTagsToVideo (t, tagInstances, video, callback) {
+      const options = { transaction: t }
+
+      video.setTags(tagInstances, options).asCallback(function (err) {
+        return callback(err, t)
+      })
+    }
+
+  ], function (err, t) {
+    if (err) {
+      // This is just a debug because we will retry the insert
+      logger.debug('Cannot insert the remote video.', { error: err })
+
+      // Abort transaction?
+      if (t) t.rollback()
+
+      return finalCallback(err)
+    }
+
+    // Commit transaction
+    t.commit().asCallback(function (err) {
+      if (err) return finalCallback(err)
+
+      logger.info('Remote video %s inserted.', videoToCreateData.name)
+      return finalCallback(null)
+    })
+  })
+}
+
+// Handle retries on fail
+function updateRemoteVideoRetryWrapper (videoAttributesToUpdate, fromPod, finalCallback) {
+  utils.transactionRetryer(
+    function (callback) {
+      return updateRemoteVideo(videoAttributesToUpdate, fromPod, callback)
+    },
+    function (err) {
+      if (err) {
+        logger.error('Cannot update the remote video with many retries.', { error: err })
+      }
+
+      // Do not return the error, continue the process
+      return finalCallback(null)
+    }
+  )
+}
+
+function updateRemoteVideo (videoAttributesToUpdate, fromPod, finalCallback) {
+  logger.debug('Updating remote video "%s".', videoAttributesToUpdate.remoteId)
+
+  waterfall([
+
+    function startTransaction (callback) {
+      db.sequelize.transaction({ isolationLevel: 'SERIALIZABLE' }).asCallback(function (err, t) {
+        return callback(err, t)
+      })
+    },
+
+    function findVideo (t, callback) {
+      fetchVideo(fromPod.host, videoAttributesToUpdate.remoteId, function (err, videoInstance) {
+        return callback(err, t, videoInstance)
+      })
+    },
+
+    function findOrCreateTags (t, videoInstance, callback) {
+      const tags = videoAttributesToUpdate.tags
+
+      db.Tag.findOrCreateTags(tags, t, function (err, tagInstances) {
+        return callback(err, t, videoInstance, tagInstances)
+      })
+    },
+
+    function updateVideoIntoDB (t, videoInstance, tagInstances, callback) {
+      const options = { transaction: t }
+
+      videoInstance.set('name', videoAttributesToUpdate.name)
+      videoInstance.set('description', videoAttributesToUpdate.description)
+      videoInstance.set('infoHash', videoAttributesToUpdate.infoHash)
+      videoInstance.set('duration', videoAttributesToUpdate.duration)
+      videoInstance.set('createdAt', videoAttributesToUpdate.createdAt)
+      videoInstance.set('updatedAt', videoAttributesToUpdate.updatedAt)
+      videoInstance.set('extname', videoAttributesToUpdate.extname)
+
+      videoInstance.save(options).asCallback(function (err) {
+        return callback(err, t, videoInstance, tagInstances)
+      })
+    },
+
+    function associateTagsToVideo (t, videoInstance, tagInstances, callback) {
+      const options = { transaction: t }
+
+      videoInstance.setTags(tagInstances, options).asCallback(function (err) {
+        return callback(err, t)
+      })
+    }
+
+  ], function (err, t) {
+    if (err) {
+      // This is just a debug because we will retry the insert
+      logger.debug('Cannot update the remote video.', { error: err })
+
+      // Abort transaction?
+      if (t) t.rollback()
+
+      return finalCallback(err)
+    }
+
+   // Commit transaction
+    t.commit().asCallback(function (err) {
+      if (err) return finalCallback(err)
+
+      logger.info('Remote video %s updated', videoAttributesToUpdate.name)
+      return finalCallback(null)
+    })
+  })
+}
+
+function removeRemoteVideo (videoToRemoveData, fromPod, callback) {
+  // We need the instance because we have to remove some other stuffs (thumbnail etc)
+  fetchVideo(fromPod.host, videoToRemoveData.remoteId, function (err, video) {
+    // Do not return the error, continue the process
+    if (err) return callback(null)
+
+    logger.debug('Removing remote video %s.', video.remoteId)
+    video.destroy().asCallback(function (err) {
+      // Do not return the error, continue the process
+      if (err) {
+        logger.error('Cannot remove remote video with id %s.', videoToRemoveData.remoteId, { error: err })
+      }
+
+      return callback(null)
+    })
+  })
+}
+
+function reportAbuseRemoteVideo (reportData, fromPod, callback) {
+  db.Video.load(reportData.videoRemoteId, function (err, video) {
+    if (err || !video) {
+      if (!err) err = new Error('video not found')
+
+      logger.error('Cannot load video from id.', { error: err, id: reportData.videoRemoteId })
+      // Do not return the error, continue the process
+      return callback(null)
+    }
+
+    logger.debug('Reporting remote abuse for video %s.', video.id)
+
+    const videoAbuseData = {
+      reporterUsername: reportData.reporterUsername,
+      reason: reportData.reportReason,
+      reporterPodId: fromPod.id,
+      videoId: video.id
+    }
+
+    db.VideoAbuse.create(videoAbuseData).asCallback(function (err) {
+      if (err) {
+        logger.error('Cannot create remote abuse video.', { error: err })
+      }
+
+      return callback(null)
+    })
+  })
+}
+
+function fetchVideo (podHost, remoteId, callback) {
+  db.Video.loadByHostAndRemoteId(podHost, remoteId, function (err, video) {
+    if (err || !video) {
+      if (!err) err = new Error('video not found')
+
+      logger.error('Cannot load video from host and remote id.', { error: err, podHost, remoteId })
+      return callback(err)
+    }
+
+    return callback(null, video)
+  })
+}

server/controllers/api/requests.js

@@ -1,15 +1,13 @@
 'use strict'
 
 const express = require('express')
-const mongoose = require('mongoose')
 
 const constants = require('../../initializers/constants')
+const db = require('../../initializers/database')
 const middlewares = require('../../middlewares')
 const admin = middlewares.admin
 const oAuth = middlewares.oauth
 
-const Request = mongoose.model('Request')
-
 const router = express.Router()
 
 router.get('/stats',
@@ -25,13 +23,13 @@ module.exports = router
 // ---------------------------------------------------------------------------
 
 function getStatsRequests (req, res, next) {
-  Request.list(function (err, requests) {
+  db.Request.countTotalRequests(function (err, totalRequests) {
     if (err) return next(err)
 
     return res.json({
-      requests: requests,
+      totalRequests: totalRequests,
       maxRequestsInParallel: constants.REQUESTS_IN_PARALLEL,
-      remainingMilliSeconds: Request.remainingMilliSeconds(),
+      remainingMilliSeconds: db.Request.remainingMilliSeconds(),
       milliSecondsInterval: constants.REQUESTS_INTERVAL
     })
   })

server/controllers/api/users.js

@@ -1,13 +1,12 @@
 'use strict'
 
-const each = require('async/each')
 const express = require('express')
-const mongoose = require('mongoose')
 const waterfall = require('async/waterfall')
 
 const constants = require('../../initializers/constants')
-const friends = require('../../lib/friends')
+const db = require('../../initializers/database')
 const logger = require('../../helpers/logger')
+const utils = require('../../helpers/utils')
 const middlewares = require('../../middlewares')
 const admin = middlewares.admin
 const oAuth = middlewares.oauth
@@ -17,9 +16,6 @@ const validatorsPagination = middlewares.validators.pagination
 const validatorsSort = middlewares.validators.sort
 const validatorsUsers = middlewares.validators.users
 
-const User = mongoose.model('User')
-const Video = mongoose.model('Video')
-
 const router = express.Router()
 
 router.get('/me', oAuth.authenticate, getUserInformation)
@@ -62,13 +58,13 @@ module.exports = router
 // ---------------------------------------------------------------------------
 
 function createUser (req, res, next) {
-  const user = new User({
+  const user = db.User.build({
     username: req.body.username,
     password: req.body.password,
     role: constants.USER_ROLES.USER
   })
 
-  user.save(function (err, createdUser) {
+  user.save().asCallback(function (err, createdUser) {
     if (err) return next(err)
 
     return res.type('json').status(204).end()
@@ -76,7 +72,7 @@ function createUser (req, res, next) {
 }
 
 function getUserInformation (req, res, next) {
-  User.loadByUsername(res.locals.oauth.token.user.username, function (err, user) {
+  db.User.loadByUsername(res.locals.oauth.token.user.username, function (err, user) {
     if (err) return next(err)
 
     return res.json(user.toFormatedJSON())
@@ -84,48 +80,21 @@ function getUserInformation (req, res, next) {
 }
 
 function listUsers (req, res, next) {
-  User.listForApi(req.query.start, req.query.count, req.query.sort, function (err, usersList, usersTotal) {
+  db.User.listForApi(req.query.start, req.query.count, req.query.sort, function (err, usersList, usersTotal) {
     if (err) return next(err)
 
-    res.json(getFormatedUsers(usersList, usersTotal))
+    res.json(utils.getFormatedObjects(usersList, usersTotal))
   })
 }
 
 function removeUser (req, res, next) {
   waterfall([
-    function getUser (callback) {
+    function loadUser (callback) {
-      User.loadById(req.params.id, callback)
+      db.User.loadById(req.params.id, callback)
     },
 
-    function getVideos (user, callback) {
+    function deleteUser (user, callback) {
-      Video.listOwnedByAuthor(user.username, function (err, videos) {
+      user.destroy().asCallback(callback)
-        return callback(err, user, videos)
-      })
-    },
-
-    function removeVideosFromDB (user, videos, callback) {
-      each(videos, function (video, callbackEach) {
-        video.remove(callbackEach)
-      }, function (err) {
-        return callback(err, user, videos)
-      })
-    },
-
-    function sendInformationToFriends (user, videos, callback) {
-      videos.forEach(function (video) {
-        const params = {
-          name: video.name,
-          magnetUri: video.magnetUri
-        }
-
-        friends.removeVideoToFriends(params)
-      })
-
-      return callback(null, user)
-    },
-
-    function removeUserFromDB (user, callback) {
-      user.remove(callback)
     }
   ], function andFinally (err) {
     if (err) {
@@ -138,11 +107,11 @@ function removeUser (req, res, next) {
 }
 
 function updateUser (req, res, next) {
-  User.loadByUsername(res.locals.oauth.token.user.username, function (err, user) {
+  db.User.loadByUsername(res.locals.oauth.token.user.username, function (err, user) {
     if (err) return next(err)
 
     user.password = req.body.password
-    user.save(function (err) {
+    user.save().asCallback(function (err) {
       if (err) return next(err)
 
       return res.sendStatus(204)
@@ -153,18 +122,3 @@ function updateUser (req, res, next) {
 function success (req, res, next) {
   res.end()
 }
-
-// ---------------------------------------------------------------------------
-
-function getFormatedUsers (users, usersTotal) {
-  const formatedUsers = []
-
-  users.forEach(function (user) {
-    formatedUsers.push(user.toFormatedJSON())
-  })
-
-  return {
-    total: usersTotal,
-    data: formatedUsers
-  }
-}

server/controllers/api/videos.js

@@ -2,15 +2,16 @@
 
 const express = require('express')
 const fs = require('fs')
-const mongoose = require('mongoose')
 const multer = require('multer')
 const path = require('path')
 const waterfall = require('async/waterfall')
 
 const constants = require('../../initializers/constants')
+const db = require('../../initializers/database')
 const logger = require('../../helpers/logger')
 const friends = require('../../lib/friends')
 const middlewares = require('../../middlewares')
+const admin = middlewares.admin
 const oAuth = middlewares.oauth
 const pagination = middlewares.pagination
 const validators = middlewares.validators
@@ -22,7 +23,6 @@ const sort = middlewares.sort
 const utils = require('../../helpers/utils')
 
 const router = express.Router()
-const Video = mongoose.model('Video')
 
 // multer configuration
 const storage = multer.diskStorage({
@@ -44,6 +44,21 @@ const storage = multer.diskStorage({
 
 const reqFiles = multer({ storage: storage }).fields([{ name: 'videofile', maxCount: 1 }])
 
+router.get('/abuse',
+  oAuth.authenticate,
+  admin.ensureIsAdmin,
+  validatorsPagination.pagination,
+  validatorsSort.videoAbusesSort,
+  sort.setVideoAbusesSort,
+  pagination.setPagination,
+  listVideoAbuses
+)
+router.post('/:id/abuse',
+  oAuth.authenticate,
+  validatorsVideos.videoAbuseReport,
+  reportVideoAbuseRetryWrapper
+)
+
 router.get('/',
   validatorsPagination.pagination,
   validatorsSort.videosSort,
@@ -51,11 +66,17 @@ router.get('/',
   pagination.setPagination,
   listVideos
 )
+router.put('/:id',
+  oAuth.authenticate,
+  reqFiles,
+  validatorsVideos.videosUpdate,
+  updateVideoRetryWrapper
+)
 router.post('/',
   oAuth.authenticate,
   reqFiles,
   validatorsVideos.videosAdd,
-  addVideo
+  addVideoRetryWrapper
 )
 router.get('/:id',
   validatorsVideos.videosGet,
@@ -82,117 +103,264 @@ module.exports = router
 
 // ---------------------------------------------------------------------------
 
-function addVideo (req, res, next) {
+// Wrapper to video add that retry the function if there is a database error
-  const videoFile = req.files.videofile[0]
+// We need this because we run the transaction in SERIALIZABLE isolation that can fail
+function addVideoRetryWrapper (req, res, next) {
+  utils.transactionRetryer(
+    function (callback) {
+      return addVideo(req, res, req.files.videofile[0], callback)
+    },
+    function (err) {
+      if (err) {
+        logger.error('Cannot insert the video with many retries.', { error: err })
+        return next(err)
+      }
+
+      // TODO : include Location of the new video -> 201
+      return res.type('json').status(204).end()
+    }
+  )
+}
+
+function addVideo (req, res, videoFile, callback) {
   const videoInfos = req.body
 
   waterfall([
-    function createVideoObject (callback) {
-      const id = mongoose.Types.ObjectId()
 
+    function startTransaction (callbackWaterfall) {
+      db.sequelize.transaction({ isolationLevel: 'SERIALIZABLE' }).asCallback(function (err, t) {
+        return callbackWaterfall(err, t)
+      })
+    },
+
+    function findOrCreateAuthor (t, callbackWaterfall) {
+      const user = res.locals.oauth.token.User
+
+      const name = user.username
+      // null because it is OUR pod
+      const podId = null
+      const userId = user.id
+
+      db.Author.findOrCreateAuthor(name, podId, userId, t, function (err, authorInstance) {
+        return callbackWaterfall(err, t, authorInstance)
+      })
+    },
+
+    function findOrCreateTags (t, author, callbackWaterfall) {
+      const tags = videoInfos.tags
+
+      db.Tag.findOrCreateTags(tags, t, function (err, tagInstances) {
+        return callbackWaterfall(err, t, author, tagInstances)
+      })
+    },
+
+    function createVideoObject (t, author, tagInstances, callbackWaterfall) {
       const videoData = {
-        _id: id,
         name: videoInfos.name,
         remoteId: null,
         extname: path.extname(videoFile.filename),
         description: videoInfos.description,
-        author: res.locals.oauth.token.user.username,
         duration: videoFile.duration,
-        tags: videoInfos.tags
+        authorId: author.id
       }
 
-      const video = new Video(videoData)
+      const video = db.Video.build(videoData)
 
-      return callback(null, video)
+      return callbackWaterfall(null, t, author, tagInstances, video)
     },
 
-     // Set the videoname the same as the MongoDB id
+     // Set the videoname the same as the id
-    function renameVideoFile (video, callback) {
+    function renameVideoFile (t, author, tagInstances, video, callbackWaterfall) {
       const videoDir = constants.CONFIG.STORAGE.VIDEOS_DIR
       const source = path.join(videoDir, videoFile.filename)
       const destination = path.join(videoDir, video.getVideoFilename())
 
       fs.rename(source, destination, function (err) {
-        return callback(err, video)
+        if (err) return callbackWaterfall(err)
+
+        // This is important in case if there is another attempt
+        videoFile.filename = video.getVideoFilename()
+        return callbackWaterfall(null, t, author, tagInstances, video)
       })
     },
 
-    function insertIntoDB (video, callback) {
+    function insertVideoIntoDB (t, author, tagInstances, video, callbackWaterfall) {
-      video.save(function (err, video) {
+      const options = { transaction: t }
-        // Assert there are only one argument sent to the next function (video)
+
-        return callback(err, video)
+      // Add tags association
+      video.save(options).asCallback(function (err, videoCreated) {
+        if (err) return callbackWaterfall(err)
+
+        // Do not forget to add Author informations to the created video
+        videoCreated.Author = author
+
+        return callbackWaterfall(err, t, tagInstances, videoCreated)
       })
     },
 
-    function sendToFriends (video, callback) {
+    function associateTagsToVideo (t, tagInstances, video, callbackWaterfall) {
-      video.toRemoteJSON(function (err, remoteVideo) {
+      const options = { transaction: t }
-        if (err) return callback(err)
+
+      video.setTags(tagInstances, options).asCallback(function (err) {
+        video.Tags = tagInstances
+
+        return callbackWaterfall(err, t, video)
+      })
+    },
+
+    function sendToFriends (t, video, callbackWaterfall) {
+      video.toAddRemoteJSON(function (err, remoteVideo) {
+        if (err) return callbackWaterfall(err)
 
         // Now we'll add the video's meta data to our friends
-        friends.addVideoToFriends(remoteVideo)
+        friends.addVideoToFriends(remoteVideo, t, function (err) {
-
+          return callbackWaterfall(err, t)
-        return callback(null)
+        })
       })
     }
 
-  ], function andFinally (err) {
+  ], function andFinally (err, t) {
     if (err) {
-      logger.error('Cannot insert the video.')
+      // This is just a debug because we will retry the insert
-      return next(err)
+      logger.debug('Cannot insert the video.', { error: err })
+
+      // Abort transaction?
+      if (t) t.rollback()
+
+      return callback(err)
     }
 
-    // TODO : include Location of the new video -> 201
+    // Commit transaction
-    return res.type('json').status(204).end()
+    t.commit().asCallback(function (err) {
+      if (err) return callback(err)
+
+      logger.info('Video with name %s created.', videoInfos.name)
+      return callback(null)
+    })
+  })
+}
+
+function updateVideoRetryWrapper (req, res, next) {
+  utils.transactionRetryer(
+    function (callback) {
+      return updateVideo(req, res, callback)
+    },
+    function (err) {
+      if (err) {
+        logger.error('Cannot update the video with many retries.', { error: err })
+        return next(err)
+      }
+
+      // TODO : include Location of the new video -> 201
+      return res.type('json').status(204).end()
+    }
+  )
+}
+
+function updateVideo (req, res, finalCallback) {
+  const videoInstance = res.locals.video
+  const videoFieldsSave = videoInstance.toJSON()
+  const videoInfosToUpdate = req.body
+
+  waterfall([
+
+    function startTransaction (callback) {
+      db.sequelize.transaction({ isolationLevel: 'SERIALIZABLE' }).asCallback(function (err, t) {
+        return callback(err, t)
+      })
+    },
+
+    function findOrCreateTags (t, callback) {
+      if (videoInfosToUpdate.tags) {
+        db.Tag.findOrCreateTags(videoInfosToUpdate.tags, t, function (err, tagInstances) {
+          return callback(err, t, tagInstances)
+        })
+      } else {
+        return callback(null, t, null)
+      }
+    },
+
+    function updateVideoIntoDB (t, tagInstances, callback) {
+      const options = {
+        transaction: t
+      }
+
+      if (videoInfosToUpdate.name) videoInstance.set('name', videoInfosToUpdate.name)
+      if (videoInfosToUpdate.description) videoInstance.set('description', videoInfosToUpdate.description)
+
+      videoInstance.save(options).asCallback(function (err) {
+        return callback(err, t, tagInstances)
+      })
+    },
+
+    function associateTagsToVideo (t, tagInstances, callback) {
+      if (tagInstances) {
+        const options = { transaction: t }
+
+        videoInstance.setTags(tagInstances, options).asCallback(function (err) {
+          videoInstance.Tags = tagInstances
+
+          return callback(err, t)
+        })
+      } else {
+        return callback(null, t)
+      }
+    },
+
+    function sendToFriends (t, callback) {
+      const json = videoInstance.toUpdateRemoteJSON()
+
+      // Now we'll update the video's meta data to our friends
+      friends.updateVideoToFriends(json, t, function (err) {
+        return callback(err, t)
+      })
+    }
+
+  ], function andFinally (err, t) {
+    if (err) {
+      logger.debug('Cannot update the video.', { error: err })
+
+      // Abort transaction?
+      if (t) t.rollback()
+
+      // Force fields we want to update
+      // If the transaction is retried, sequelize will think the object has not changed
+      // So it will skip the SQL request, even if the last one was ROLLBACKed!
+      Object.keys(videoFieldsSave).forEach(function (key) {
+        const value = videoFieldsSave[key]
+        videoInstance.set(key, value)
+      })
+
+      return finalCallback(err)
+    }
+
+    // Commit transaction
+    t.commit().asCallback(function (err) {
+      if (err) return finalCallback(err)
+
+      logger.info('Video with name %s updated.', videoInfosToUpdate.name)
+      return finalCallback(null)
+    })
   })
 }
 
 function getVideo (req, res, next) {
-  Video.load(req.params.id, function (err, video) {
+  const videoInstance = res.locals.video
-    if (err) return next(err)
+  res.json(videoInstance.toFormatedJSON())
-
-    if (!video) {
-      return res.type('json').status(204).end()
-    }
-
-    res.json(video.toFormatedJSON())
-  })
 }
 
 function listVideos (req, res, next) {
-  Video.listForApi(req.query.start, req.query.count, req.query.sort, function (err, videosList, videosTotal) {
+  db.Video.listForApi(req.query.start, req.query.count, req.query.sort, function (err, videosList, videosTotal) {
     if (err) return next(err)
 
-    res.json(getFormatedVideos(videosList, videosTotal))
+    res.json(utils.getFormatedObjects(videosList, videosTotal))
   })
 }
 
 function removeVideo (req, res, next) {
-  const videoId = req.params.id
+  const videoInstance = res.locals.video
 
-  waterfall([
+  videoInstance.destroy().asCallback(function (err) {
-    function getVideo (callback) {
-      Video.load(videoId, callback)
-    },
-
-    function removeFromDB (video, callback) {
-      video.remove(function (err) {
-        if (err) return callback(err)
-
-        return callback(null, video)
-      })
-    },
-
-    function sendInformationToFriends (video, callback) {
-      const params = {
-        name: video.name,
-        remoteId: video._id
-      }
-
-      friends.removeVideoToFriends(params)
-
-      return callback(null)
-    }
-  ], function andFinally (err) {
     if (err) {
       logger.error('Errors when removed the video.', { error: err })
       return next(err)
@@ -203,25 +371,97 @@ function removeVideo (req, res, next) {
 }
 
 function searchVideos (req, res, next) {
-  Video.search(req.params.value, req.query.field, req.query.start, req.query.count, req.query.sort,
+  db.Video.searchAndPopulateAuthorAndPodAndTags(
-  function (err, videosList, videosTotal) {
+    req.params.value, req.query.field, req.query.start, req.query.count, req.query.sort,
+    function (err, videosList, videosTotal) {
+      if (err) return next(err)
+
+      res.json(utils.getFormatedObjects(videosList, videosTotal))
+    }
+  )
+}
+
+function listVideoAbuses (req, res, next) {
+  db.VideoAbuse.listForApi(req.query.start, req.query.count, req.query.sort, function (err, abusesList, abusesTotal) {
     if (err) return next(err)
 
-    res.json(getFormatedVideos(videosList, videosTotal))
+    res.json(utils.getFormatedObjects(abusesList, abusesTotal))
   })
 }
 
-// ---------------------------------------------------------------------------
+function reportVideoAbuseRetryWrapper (req, res, next) {
+  utils.transactionRetryer(
+    function (callback) {
+      return reportVideoAbuse(req, res, callback)
+    },
+    function (err) {
+      if (err) {
+        logger.error('Cannot report abuse to the video with many retries.', { error: err })
+        return next(err)
+      }
 
-function getFormatedVideos (videos, videosTotal) {
+      return res.type('json').status(204).end()
-  const formatedVideos = []
+    }
+  )
+}
 
-  videos.forEach(function (video) {
+function reportVideoAbuse (req, res, finalCallback) {
-    formatedVideos.push(video.toFormatedJSON())
+  const videoInstance = res.locals.video
-  })
+  const reporterUsername = res.locals.oauth.token.User.username
 
-  return {
+  const abuse = {
-    total: videosTotal,
+    reporterUsername,
-    data: formatedVideos
+    reason: req.body.reason,
+    videoId: videoInstance.id,
+    reporterPodId: null // This is our pod that reported this abuse
   }
+
+  waterfall([
+
+    function startTransaction (callback) {
+      db.sequelize.transaction().asCallback(function (err, t) {
+        return callback(err, t)
+      })
+    },
+
+    function createAbuse (t, callback) {
+      db.VideoAbuse.create(abuse).asCallback(function (err, abuse) {
+        return callback(err, t, abuse)
+      })
+    },
+
+    function sendToFriendsIfNeeded (t, abuse, callback) {
+      // We send the information to the destination pod
+      if (videoInstance.isOwned() === false) {
+        const reportData = {
+          reporterUsername,
+          reportReason: abuse.reason,
+          videoRemoteId: videoInstance.remoteId
+        }
+
+        friends.reportAbuseVideoToFriend(reportData, videoInstance)
+      }
+
+      return callback(null, t)
+    }
+
+  ], function andFinally (err, t) {
+    if (err) {
+      logger.debug('Cannot update the video.', { error: err })
+
+      // Abort transaction?
+      if (t) t.rollback()
+
+      return finalCallback(err)
+    }
+
+    // Commit transaction
+    t.commit().asCallback(function (err) {
+      if (err) return finalCallback(err)
+
+      logger.info('Abuse report for video %s created.', videoInstance.name)
+      return finalCallback(null)
+    })
+  })
 }
+

server/controllers/client.js

@@ -3,13 +3,12 @@
 const parallel = require('async/parallel')
 const express = require('express')
 const fs = require('fs')
-const mongoose = require('mongoose')
 const path = require('path')
 const validator = require('express-validator').validator
 
 const constants = require('../initializers/constants')
+const db = require('../initializers/database')
 
-const Video = mongoose.model('Video')
 const router = express.Router()
 
 const opengraphComment = '<!-- opengraph tags -->'
@@ -45,14 +44,14 @@ function addOpenGraphTags (htmlStringPage, video) {
   if (video.isOwned()) {
     basePreviewUrlHttp = constants.CONFIG.WEBSERVER.URL
   } else {
-    basePreviewUrlHttp = constants.REMOTE_SCHEME.HTTP + '://' + video.podHost
+    basePreviewUrlHttp = constants.REMOTE_SCHEME.HTTP + '://' + video.Author.Pod.host
   }
 
   // We fetch the remote preview (bigger than the thumbnail)
   // This should not overhead the remote server since social websites put in a cache the OpenGraph tags
   // We can't use the thumbnail because these social websites want bigger images (> 200x200 for Facebook for example)
   const previewUrl = basePreviewUrlHttp + constants.STATIC_PATHS.PREVIEWS + video.getPreviewName()
-  const videoUrl = constants.CONFIG.WEBSERVER.URL + '/videos/watch/' + video._id
+  const videoUrl = constants.CONFIG.WEBSERVER.URL + '/videos/watch/' + video.id
 
   const metaTags = {
     'og:type': 'video',
@@ -86,7 +85,7 @@ function generateWatchHtmlPage (req, res, next) {
   const videoId = req.params.id
 
   // Let Angular application handle errors
-  if (!validator.isMongoId(videoId)) return res.sendFile(indexPath)
+  if (!validator.isUUID(videoId, 4)) return res.sendFile(indexPath)
 
   parallel({
     file: function (callback) {
@@ -94,7 +93,7 @@ function generateWatchHtmlPage (req, res, next) {
     },
 
     video: function (callback) {
-      Video.load(videoId, callback)
+      db.Video.loadAndPopulateAuthorAndPodAndTags(videoId, callback)
     }
   }, function (err, results) {
     if (err) return next(err)

server/helpers/custom-validators/index.js

@@ -2,12 +2,14 @@
 
 const miscValidators = require('./misc')
 const podsValidators = require('./pods')
+const remoteValidators = require('./remote')
 const usersValidators = require('./users')
 const videosValidators = require('./videos')
 
 const validators = {
   misc: miscValidators,
   pods: podsValidators,
+  remote: remoteValidators,
   users: usersValidators,
   videos: videosValidators
 }

server/helpers/custom-validators/pods.js

@@ -5,14 +5,19 @@ const validator = require('express-validator').validator
 const miscValidators = require('./misc')
 
 const podsValidators = {
-  isEachUniqueHostValid
+  isEachUniqueHostValid,
+  isHostValid
+}
+
+function isHostValid (host) {
+  return validator.isURL(host) && host.split('://').length === 1
 }
 
 function isEachUniqueHostValid (hosts) {
   return miscValidators.isArray(hosts) &&
     hosts.length !== 0 &&
     hosts.every(function (host) {
-      return validator.isURL(host) && host.split('://').length === 1 && hosts.indexOf(host) === hosts.lastIndexOf(host)
+      return isHostValid(host) && hosts.indexOf(host) === hosts.lastIndexOf(host)
     })
 }
 

server/helpers/custom-validators/remote/index.js

@@ -0,0 +1,11 @@
+'use strict'
+
+const remoteVideosValidators = require('./videos')
+
+const validators = {
+  videos: remoteVideosValidators
+}
+
+// ---------------------------------------------------------------------------
+
+module.exports = validators

server/helpers/custom-validators/remote/videos.js

@@ -0,0 +1,73 @@
+'use strict'
+
+const videosValidators = require('../videos')
+const miscValidators = require('../misc')
+
+const remoteVideosValidators = {
+  isEachRemoteRequestVideosValid
+}
+
+function isEachRemoteRequestVideosValid (requests) {
+  return miscValidators.isArray(requests) &&
+    requests.every(function (request) {
+      const video = request.data
+      return (
+        isRequestTypeAddValid(request.type) &&
+        videosValidators.isVideoAuthorValid(video.author) &&
+        videosValidators.isVideoDateValid(video.createdAt) &&
+        videosValidators.isVideoDateValid(video.updatedAt) &&
+        videosValidators.isVideoDescriptionValid(video.description) &&
+        videosValidators.isVideoDurationValid(video.duration) &&
+        videosValidators.isVideoInfoHashValid(video.infoHash) &&
+        videosValidators.isVideoNameValid(video.name) &&
+        videosValidators.isVideoTagsValid(video.tags) &&
+        videosValidators.isVideoThumbnailDataValid(video.thumbnailData) &&
+        videosValidators.isVideoRemoteIdValid(video.remoteId) &&
+        videosValidators.isVideoExtnameValid(video.extname)
+      ) ||
+      (
+        isRequestTypeUpdateValid(request.type) &&
+        videosValidators.isVideoDateValid(video.createdAt) &&
+        videosValidators.isVideoDateValid(video.updatedAt) &&
+        videosValidators.isVideoDescriptionValid(video.description) &&
+        videosValidators.isVideoDurationValid(video.duration) &&
+        videosValidators.isVideoInfoHashValid(video.infoHash) &&
+        videosValidators.isVideoNameValid(video.name) &&
+        videosValidators.isVideoTagsValid(video.tags) &&
+        videosValidators.isVideoRemoteIdValid(video.remoteId) &&
+        videosValidators.isVideoExtnameValid(video.extname)
+      ) ||
+      (
+        isRequestTypeRemoveValid(request.type) &&
+        videosValidators.isVideoRemoteIdValid(video.remoteId)
+      ) ||
+      (
+        isRequestTypeReportAbuseValid(request.type) &&
+        videosValidators.isVideoRemoteIdValid(request.data.videoRemoteId) &&
+        videosValidators.isVideoAbuseReasonValid(request.data.reportReason) &&
+        videosValidators.isVideoAbuseReporterUsernameValid(request.data.reporterUsername)
+      )
+    })
+}
+
+// ---------------------------------------------------------------------------
+
+module.exports = remoteVideosValidators
+
+// ---------------------------------------------------------------------------
+
+function isRequestTypeAddValid (value) {
+  return value === 'add'
+}
+
+function isRequestTypeUpdateValid (value) {
+  return value === 'update'
+}
+
+function isRequestTypeRemoveValid (value) {
+  return value === 'remove'
+}
+
+function isRequestTypeReportAbuseValid (value) {
+  return value === 'report-abuse'
+}

server/helpers/custom-validators/videos.js

@@ -6,43 +6,22 @@ const constants = require('../../initializers/constants')
 const usersValidators = require('./users')
 const miscValidators = require('./misc')
 const VIDEOS_CONSTRAINTS_FIELDS = constants.CONSTRAINTS_FIELDS.VIDEOS
+const VIDEO_ABUSES_CONSTRAINTS_FIELDS = constants.CONSTRAINTS_FIELDS.VIDEO_ABUSES
 
 const videosValidators = {
-  isEachRemoteVideosValid,
   isVideoAuthorValid,
   isVideoDateValid,
   isVideoDescriptionValid,
   isVideoDurationValid,
-  isVideoMagnetValid,
+  isVideoInfoHashValid,
   isVideoNameValid,
-  isVideoPodHostValid,
   isVideoTagsValid,
   isVideoThumbnailValid,
-  isVideoThumbnail64Valid
+  isVideoThumbnailDataValid,
-}
+  isVideoExtnameValid,
-
+  isVideoRemoteIdValid,
-function isEachRemoteVideosValid (requests) {
+  isVideoAbuseReasonValid,
-  return miscValidators.isArray(requests) &&
+  isVideoAbuseReporterUsernameValid
-    requests.every(function (request) {
-      const video = request.data
-      return (
-        isRequestTypeAddValid(request.type) &&
-        isVideoAuthorValid(video.author) &&
-        isVideoDateValid(video.createdDate) &&
-        isVideoDescriptionValid(video.description) &&
-        isVideoDurationValid(video.duration) &&
-        isVideoMagnetValid(video.magnet) &&
-        isVideoNameValid(video.name) &&
-        isVideoTagsValid(video.tags) &&
-        isVideoThumbnail64Valid(video.thumbnailBase64) &&
-        isVideoRemoteIdValid(video.remoteId)
-      ) ||
-      (
-        isRequestTypeRemoveValid(request.type) &&
-        isVideoNameValid(video.name) &&
-        isVideoRemoteIdValid(video.remoteId)
-      )
-    })
 }
 
 function isVideoAuthorValid (value) {
@@ -61,19 +40,18 @@ function isVideoDurationValid (value) {
   return validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.DURATION)
 }
 
-function isVideoMagnetValid (value) {
+function isVideoExtnameValid (value) {
-  return validator.isLength(value.infoHash, VIDEOS_CONSTRAINTS_FIELDS.MAGNET.INFO_HASH)
+  return VIDEOS_CONSTRAINTS_FIELDS.EXTNAME.indexOf(value) !== -1
+}
+
+function isVideoInfoHashValid (value) {
+  return validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.INFO_HASH)
 }
 
 function isVideoNameValid (value) {
   return validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.NAME)
 }
 
-function isVideoPodHostValid (value) {
-  // TODO: set options (TLD...)
-  return validator.isURL(value)
-}
-
 function isVideoTagsValid (tags) {
   return miscValidators.isArray(tags) &&
          validator.isInt(tags.length, VIDEOS_CONSTRAINTS_FIELDS.TAGS) &&
@@ -87,25 +65,22 @@ function isVideoThumbnailValid (value) {
   return validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.THUMBNAIL)
 }
 
-function isVideoThumbnail64Valid (value) {
+function isVideoThumbnailDataValid (value) {
-  return validator.isBase64(value) &&
+  return validator.isByteLength(value, VIDEOS_CONSTRAINTS_FIELDS.THUMBNAIL_DATA)
-         validator.isByteLength(value, VIDEOS_CONSTRAINTS_FIELDS.THUMBNAIL64)
 }
 
 function isVideoRemoteIdValid (value) {
-  return validator.isMongoId(value)
+  return validator.isUUID(value, 4)
+}
+
+function isVideoAbuseReasonValid (value) {
+  return validator.isLength(value, VIDEO_ABUSES_CONSTRAINTS_FIELDS.REASON)
+}
+
+function isVideoAbuseReporterUsernameValid (value) {
+  return usersValidators.isUserUsernameValid(value)
 }
 
 // ---------------------------------------------------------------------------
 
 module.exports = videosValidators
-
-// ---------------------------------------------------------------------------
-
-function isRequestTypeAddValid (value) {
-  return value === 'add'
-}
-
-function isRequestTypeRemoveValid (value) {
-  return value === 'remove'
-}

server/helpers/logger.js

@@ -22,7 +22,8 @@ const logger = new winston.Logger({
       json: true,
       maxsize: 5242880,
       maxFiles: 5,
-      colorize: false
+      colorize: false,
+      prettyPrint: true
     }),
     new winston.transports.Console({
       level: 'debug',
@@ -30,7 +31,8 @@ const logger = new winston.Logger({
       handleExceptions: true,
       humanReadableUnhandledException: true,
       json: false,
-      colorize: true
+      colorize: true,
+      prettyPrint: true
     })
   ],
   exitOnError: true

server/helpers/peertube-crypto.js

@@ -1,16 +1,13 @@
 'use strict'
 
-const bcrypt = require('bcrypt')
 const crypto = require('crypto')
+const bcrypt = require('bcrypt')
 const fs = require('fs')
 const openssl = require('openssl-wrapper')
-const ursa = require('ursa')
 
 const constants = require('../initializers/constants')
 const logger = require('./logger')
 
-const algorithm = 'aes-256-ctr'
-
 const peertubeCrypto = {
   checkSignature,
   comparePassword,
@@ -19,12 +16,51 @@ const peertubeCrypto = {
   sign
 }
 
-function checkSignature (publicKey, rawData, hexSignature) {
+function checkSignature (publicKey, data, hexSignature) {
-  const crt = ursa.createPublicKey(publicKey)
+  const verify = crypto.createVerify(constants.SIGNATURE_ALGORITHM)
-  const isValid = crt.hashAndVerify('sha256', new Buffer(rawData).toString('hex'), hexSignature, 'hex')
+
+  let dataString
+  if (typeof data === 'string') {
+    dataString = data
+  } else {
+    try {
+      dataString = JSON.stringify(data)
+    } catch (err) {
+      logger.error('Cannot check signature.', { error: err })
+      return false
+    }
+  }
+
+  verify.update(dataString, 'utf8')
+
+  const isValid = verify.verify(publicKey, hexSignature, constants.SIGNATURE_ENCODING)
   return isValid
 }
 
+function sign (data) {
+  const sign = crypto.createSign(constants.SIGNATURE_ALGORITHM)
+
+  let dataString
+  if (typeof data === 'string') {
+    dataString = data
+  } else {
+    try {
+      dataString = JSON.stringify(data)
+    } catch (err) {
+      logger.error('Cannot sign data.', { error: err })
+      return ''
+    }
+  }
+
+  sign.update(dataString, 'utf8')
+
+  // TODO: make async
+  const myKey = fs.readFileSync(constants.CONFIG.STORAGE.CERT_DIR + 'peertube.key.pem')
+  const signature = sign.sign(myKey, constants.SIGNATURE_ENCODING)
+
+  return signature
+}
+
 function comparePassword (plainPassword, hashPassword, callback) {
   bcrypt.compare(plainPassword, hashPassword, function (err, isPasswordMatch) {
     if (err) return callback(err)
@@ -55,13 +91,6 @@ function cryptPassword (password, callback) {
   })
 }
 
-function sign (data) {
-  const myKey = ursa.createPrivateKey(fs.readFileSync(constants.CONFIG.STORAGE.CERT_DIR + 'peertube.key.pem'))
-  const signature = myKey.hashAndSign('sha256', data, 'utf8', 'hex')
-
-  return signature
-}
-
 // ---------------------------------------------------------------------------
 
 module.exports = peertubeCrypto
@@ -113,11 +142,3 @@ function createCerts (callback) {
     })
   })
 }
-
-function generatePassword (callback) {
-  crypto.randomBytes(32, function (err, buf) {
-    if (err) return callback(err)
-
-    callback(null, buf.toString('utf8'))
-  })
-}

server/helpers/requests.js

@@ -28,31 +28,37 @@ function makeSecureRequest (params, callback) {
     url: constants.REMOTE_SCHEME.HTTP + '://' + params.toPod.host + params.path
   }
 
-  // Add data with POST requst ?
+  if (params.method !== 'POST') {
-  if (params.method === 'POST') {
+    return callback(new Error('Cannot make a secure request with a non POST method.'))
-    requestParams.json = {}
-
-    // Add signature if it is specified in the params
-    if (params.sign === true) {
-      const host = constants.CONFIG.WEBSERVER.HOST
-
-      requestParams.json.signature = {
-        host,
-        signature: peertubeCrypto.sign(host)
-      }
-    }
-
-    // If there are data informations
-    if (params.data) {
-      requestParams.json.data = params.data
-      request.post(requestParams, callback)
-    } else {
-      // No data
-      request.post(requestParams, callback)
-    }
-  } else {
-    request.get(requestParams, callback)
   }
+
+  requestParams.json = {}
+
+  // Add signature if it is specified in the params
+  if (params.sign === true) {
+    const host = constants.CONFIG.WEBSERVER.HOST
+
+    let dataToSign
+    if (params.data) {
+      dataToSign = dataToSign = params.data
+    } else {
+      // We do not have data to sign so we just take our host
+      // It is not ideal but the connection should be in HTTPS
+      dataToSign = host
+    }
+
+    requestParams.json.signature = {
+      host, // Which host we pretend to be
+      signature: peertubeCrypto.sign(dataToSign)
+    }
+  }
+
+  // If there are data informations
+  if (params.data) {
+    requestParams.json.data = params.data
+  }
+
+  request.post(requestParams, callback)
 }
 
 // ---------------------------------------------------------------------------

server/helpers/utils.js

@@ -1,13 +1,21 @@
 'use strict'
 
 const crypto = require('crypto')
+const retry = require('async/retry')
 
 const logger = require('./logger')
 
 const utils = {
+  badRequest,
   cleanForExit,
   generateRandomString,
-  isTestInstance
+  isTestInstance,
+  getFormatedObjects,
+  transactionRetryer
+}
+
+function badRequest (req, res, next) {
+  res.type('json').status(400).end()
 }
 
 function generateRandomString (size, callback) {
@@ -27,6 +35,31 @@ function isTestInstance () {
   return (process.env.NODE_ENV === 'test')
 }
 
+function getFormatedObjects (objects, objectsTotal) {
+  const formatedObjects = []
+
+  objects.forEach(function (object) {
+    formatedObjects.push(object.toFormatedJSON())
+  })
+
+  return {
+    total: objectsTotal,
+    data: formatedObjects
+  }
+}
+
+function transactionRetryer (func, callback) {
+  retry({
+    times: 5,
+
+    errorFilter: function (err) {
+      const willRetry = (err.name === 'SequelizeDatabaseError')
+      logger.debug('Maybe retrying the transaction function.', { willRetry })
+      return willRetry
+    }
+  }, func, callback)
+}
+
 // ---------------------------------------------------------------------------
 
 module.exports = utils

server/initializers/checker.js

@@ -1,10 +1,8 @@
 'use strict'
 
 const config = require('config')
-const mongoose = require('mongoose')
 
-const Client = mongoose.model('OAuthClient')
+const db = require('./database')
-const User = mongoose.model('User')
 
 const checker = {
   checkConfig,
@@ -29,7 +27,7 @@ function checkConfig () {
 function checkMissedConfig () {
   const required = [ 'listen.port',
     'webserver.https', 'webserver.hostname', 'webserver.port',
-    'database.hostname', 'database.port', 'database.suffix',
+    'database.hostname', 'database.port', 'database.suffix', 'database.username', 'database.password',
     'storage.certs', 'storage.videos', 'storage.logs', 'storage.thumbnails', 'storage.previews'
   ]
   const miss = []
@@ -44,15 +42,15 @@ function checkMissedConfig () {
 }
 
 function clientsExist (callback) {
-  Client.list(function (err, clients) {
+  db.OAuthClient.countTotal(function (err, totalClients) {
     if (err) return callback(err)
 
-    return callback(null, clients.length !== 0)
+    return callback(null, totalClients !== 0)
   })
 }
 
 function usersExist (callback) {
-  User.countTotal(function (err, totalUsers) {
+  db.User.countTotal(function (err, totalUsers) {
     if (err) return callback(err)
 
     return callback(null, totalUsers !== 0)

server/initializers/constants.js

@@ -1,7 +1,6 @@
 'use strict'
 
 const config = require('config')
-const maxBy = require('lodash/maxBy')
 const path = require('path')
 
 // ---------------------------------------------------------------------------
@@ -14,13 +13,14 @@ const PAGINATION_COUNT_DEFAULT = 15
 
 // Sortable columns per schema
 const SEARCHABLE_COLUMNS = {
-  VIDEOS: [ 'name', 'magnetUri', 'podHost', 'author', 'tags' ]
+  VIDEOS: [ 'name', 'magnetUri', 'host', 'author', 'tags' ]
 }
 
 // Sortable columns per schema
 const SORTABLE_COLUMNS = {
-  USERS: [ 'username', '-username', 'createdDate', '-createdDate' ],
+  USERS: [ 'username', '-username', 'createdAt', '-createdAt' ],
-  VIDEOS: [ 'name', '-name', 'duration', '-duration', 'createdDate', '-createdDate' ]
+  VIDEO_ABUSES: [ 'createdAt', '-createdAt' ],
+  VIDEOS: [ 'name', '-name', 'duration', '-duration', 'createdAt', '-createdAt' ]
 }
 
 const OAUTH_LIFETIME = {
@@ -37,7 +37,9 @@ const CONFIG = {
   DATABASE: {
     DBNAME: 'peertube' + config.get('database.suffix'),
     HOSTNAME: config.get('database.hostname'),
-    PORT: config.get('database.port')
+    PORT: config.get('database.port'),
+    USERNAME: config.get('database.username'),
+    PASSWORD: config.get('database.password')
   },
   STORAGE: {
     CERT_DIR: path.join(__dirname, '..', '..', config.get('storage.certs')),
@@ -64,17 +66,19 @@ const CONSTRAINTS_FIELDS = {
     USERNAME: { min: 3, max: 20 }, // Length
     PASSWORD: { min: 6, max: 255 } // Length
   },
+  VIDEO_ABUSES: {
+    REASON: { min: 2, max: 300 } // Length
+  },
   VIDEOS: {
     NAME: { min: 3, max: 50 }, // Length
     DESCRIPTION: { min: 3, max: 250 }, // Length
-    MAGNET: {
+    EXTNAME: [ '.mp4', '.ogv', '.webm' ],
-      INFO_HASH: { min: 10, max: 50 } // Length
+    INFO_HASH: { min: 40, max: 40 }, // Length, infohash is 20 bytes length but we represent it in hexa so 20 * 2
-    },
     DURATION: { min: 1, max: 7200 }, // Number
     TAGS: { min: 1, max: 3 }, // Number of total tags
     TAG: { min: 2, max: 10 }, // Length
     THUMBNAIL: { min: 2, max: 30 },
-    THUMBNAIL64: { min: 0, max: 20000 } // Bytes
+    THUMBNAIL_DATA: { min: 0, max: 20000 } // Bytes
   }
 }
 
@@ -88,41 +92,7 @@ const FRIEND_SCORE = {
 
 // ---------------------------------------------------------------------------
 
-const MONGO_MIGRATION_SCRIPTS = [
+const LAST_MIGRATION_VERSION = 0
-  {
-    script: '0005-create-application',
-    version: 5
-  },
-  {
-    script: '0010-users-password',
-    version: 10
-  },
-  {
-    script: '0015-admin-role',
-    version: 15
-  },
-  {
-    script: '0020-requests-endpoint',
-    version: 20
-  },
-  {
-    script: '0025-video-filenames',
-    version: 25
-  },
-  {
-    script: '0030-video-magnet',
-    version: 30
-  },
-  {
-    script: '0035-url-to-host',
-    version: 35
-  },
-  {
-    script: '0040-video-remote-id',
-    version: 40
-  }
-]
-const LAST_MONGO_SCHEMA_VERSION = (maxBy(MONGO_MIGRATION_SCRIPTS, 'version'))['version']
 
 // ---------------------------------------------------------------------------
 
@@ -138,8 +108,10 @@ let REQUESTS_INTERVAL = 600000
 // Number of requests in parallel we can make
 const REQUESTS_IN_PARALLEL = 10
 
-// How many requests we put in request
+// To how many pods we send requests
-const REQUESTS_LIMIT = 10
+const REQUESTS_LIMIT_PODS = 10
+// How many requests we send to a pod per interval
+const REQUESTS_LIMIT_PER_POD = 5
 
 // Number of requests to retry for replay requests module
 const RETRY_REQUESTS = 5
@@ -148,16 +120,21 @@ const REQUEST_ENDPOINTS = {
   VIDEOS: 'videos'
 }
 
-// ---------------------------------------------------------------------------
-
 const REMOTE_SCHEME = {
   HTTP: 'https',
   WS: 'wss'
 }
 
+// ---------------------------------------------------------------------------
+
+const SIGNATURE_ALGORITHM = 'RSA-SHA256'
+const SIGNATURE_ENCODING = 'hex'
+
 // Password encryption
 const BCRYPT_SALT_SIZE = 10
 
+// ---------------------------------------------------------------------------
+
 // Express static paths (router)
 const STATIC_PATHS = {
   PREVIEWS: '/static/previews/',
@@ -173,6 +150,8 @@ let STATIC_MAX_AGE = '30d'
 const THUMBNAILS_SIZE = '200x110'
 const PREVIEWS_SIZE = '640x480'
 
+// ---------------------------------------------------------------------------
+
 const USER_ROLES = {
   ADMIN: 'admin',
   USER: 'user'
@@ -198,8 +177,7 @@ module.exports = {
   CONFIG,
   CONSTRAINTS_FIELDS,
   FRIEND_SCORE,
-  LAST_MONGO_SCHEMA_VERSION,
+  LAST_MIGRATION_VERSION,
-  MONGO_MIGRATION_SCRIPTS,
   OAUTH_LIFETIME,
   PAGINATION_COUNT_DEFAULT,
   PODS_SCORE,
@@ -208,9 +186,12 @@ module.exports = {
   REQUEST_ENDPOINTS,
   REQUESTS_IN_PARALLEL,
   REQUESTS_INTERVAL,
-  REQUESTS_LIMIT,
+  REQUESTS_LIMIT_PODS,
+  REQUESTS_LIMIT_PER_POD,
   RETRY_REQUESTS,
   SEARCHABLE_COLUMNS,
+  SIGNATURE_ALGORITHM,
+  SIGNATURE_ENCODING,
   SORTABLE_COLUMNS,
   STATIC_MAX_AGE,
   STATIC_PATHS,

server/initializers/database.js

@@ -1,37 +1,77 @@
 'use strict'
 
-const mongoose = require('mongoose')
+const fs = require('fs')
+const path = require('path')
+const Sequelize = require('sequelize')
 
 const constants = require('../initializers/constants')
 const logger = require('../helpers/logger')
+const utils = require('../helpers/utils')
 
-// Bootstrap models
+const database = {}
-require('../models/application')
-require('../models/oauth-token')
-require('../models/user')
-require('../models/oauth-client')
-require('../models/video')
-// Request model needs Video model
-require('../models/pods')
-// Request model needs Pod model
-require('../models/request')
 
-const database = {
+const dbname = constants.CONFIG.DATABASE.DBNAME
-  connect: connect
+const username = constants.CONFIG.DATABASE.USERNAME
-}
+const password = constants.CONFIG.DATABASE.PASSWORD
 
-function connect () {
+const sequelize = new Sequelize(dbname, username, password, {
-  mongoose.Promise = global.Promise
+  dialect: 'postgres',
-  mongoose.connect('mongodb://' + constants.CONFIG.DATABASE.HOSTNAME + ':' + constants.CONFIG.DATABASE.PORT + '/' + constants.CONFIG.DATABASE.DBNAME)
+  host: constants.CONFIG.DATABASE.HOSTNAME,
-  mongoose.connection.on('error', function () {
+  port: constants.CONFIG.DATABASE.PORT,
-    throw new Error('Mongodb connection error.')
+  benchmark: utils.isTestInstance(),
-  })
 
-  mongoose.connection.on('open', function () {
+  logging: function (message, benchmark) {
-    logger.info('Connected to mongodb.')
+    let newMessage = message
-  })
+    if (benchmark !== undefined) {
-}
+      newMessage += ' | ' + benchmark + 'ms'
+    }
+
+    logger.debug(newMessage)
+  }
+})
+
+database.sequelize = sequelize
+database.Sequelize = Sequelize
+database.init = init
 
 // ---------------------------------------------------------------------------
 
 module.exports = database
+
+// ---------------------------------------------------------------------------
+
+function init (silent, callback) {
+  if (!callback) {
+    callback = silent
+    silent = false
+  }
+
+  if (!callback) callback = function () {}
+
+  const modelDirectory = path.join(__dirname, '..', 'models')
+  fs.readdir(modelDirectory, function (err, files) {
+    if (err) throw err
+
+    files.filter(function (file) {
+      // For all models but not utils.js
+      if (file === 'utils.js') return false
+
+      return true
+    })
+    .forEach(function (file) {
+      const model = sequelize.import(path.join(modelDirectory, file))
+
+      database[model.name] = model
+    })
+
+    Object.keys(database).forEach(function (modelName) {
+      if ('associate' in database[modelName]) {
+        database[modelName].associate(database)
+      }
+    })
+
+    if (!silent) logger.info('Database is ready.')
+
+    return callback(null)
+  })
+}

server/initializers/installer.js

@@ -3,26 +3,27 @@
 const config = require('config')
 const each = require('async/each')
 const mkdirp = require('mkdirp')
-const mongoose = require('mongoose')
 const passwordGenerator = require('password-generator')
 const path = require('path')
 const series = require('async/series')
 
 const checker = require('./checker')
 const constants = require('./constants')
+const db = require('./database')
 const logger = require('../helpers/logger')
 const peertubeCrypto = require('../helpers/peertube-crypto')
 
-const Application = mongoose.model('Application')
-const Client = mongoose.model('OAuthClient')
-const User = mongoose.model('User')
-
 const installer = {
   installApplication
 }
 
 function installApplication (callback) {
   series([
+    function createDatabase (callbackAsync) {
+      db.sequelize.sync().asCallback(callbackAsync)
+      // db.sequelize.sync({ force: true }).asCallback(callbackAsync)
+    },
+
     function createDirectories (callbackAsync) {
       createDirectoriesIfNotExist(callbackAsync)
     },
@@ -65,16 +66,18 @@ function createOAuthClientIfNotExist (callback) {
 
     logger.info('Creating a default OAuth Client.')
 
-    const secret = passwordGenerator(32, false)
+    const id = passwordGenerator(32, false, /[a-z0-9]/)
-    const client = new Client({
+    const secret = passwordGenerator(32, false, /[a-zA-Z0-9]/)
+    const client = db.OAuthClient.build({
+      clientId: id,
       clientSecret: secret,
       grants: [ 'password', 'refresh_token' ]
     })
 
-    client.save(function (err, createdClient) {
+    client.save().asCallback(function (err, createdClient) {
       if (err) return callback(err)
 
-      logger.info('Client id: ' + createdClient._id)
+      logger.info('Client id: ' + createdClient.clientId)
       logger.info('Client secret: ' + createdClient.clientSecret)
 
       return callback(null)
@@ -93,6 +96,7 @@ function createOAuthAdminIfNotExist (callback) {
 
     const username = 'root'
     const role = constants.USER_ROLES.ADMIN
+    const createOptions = {}
     let password = ''
 
     // Do not generate a random password for tests
@@ -102,25 +106,27 @@ function createOAuthAdminIfNotExist (callback) {
       if (process.env.NODE_APP_INSTANCE) {
         password += process.env.NODE_APP_INSTANCE
       }
+
+      // Our password is weak so do not validate it
+      createOptions.validate = false
     } else {
       password = passwordGenerator(8, true)
     }
 
-    const user = new User({
+    const userData = {
       username,
       password,
       role
-    })
+    }
 
-    user.save(function (err, createdUser) {
+    db.User.create(userData, createOptions).asCallback(function (err, createdUser) {
       if (err) return callback(err)
 
       logger.info('Username: ' + username)
       logger.info('User password: ' + password)
 
-      logger.info('Creating Application collection.')
+      logger.info('Creating Application table.')
-      const application = new Application({ mongoSchemaVersion: constants.LAST_MONGO_SCHEMA_VERSION })
+      db.Application.create({ migrationVersion: constants.LAST_MIGRATION_VERSION }).asCallback(callback)
-      application.save(callback)
     })
   })
 }

server/initializers/migrations/0005-create-application.js

@@ -1,17 +0,0 @@
-/*
-  Create the application collection in MongoDB.
-  Used to store the actual MongoDB scheme version.
-*/
-
-const mongoose = require('mongoose')
-
-const Application = mongoose.model('Application')
-
-exports.up = function (callback) {
-  const application = new Application()
-  application.save(callback)
-}
-
-exports.down = function (callback) {
-  throw new Error('Not implemented.')
-}

server/initializers/migrations/0005-example.js

@@ -0,0 +1,14 @@
+// /*
+//   This is just an example.
+// */
+
+// const db = require('../database')
+
+// // options contains the transaction
+// exports.up = function (options, callback) {
+//   db.Application.create({ migrationVersion: 42 }, { transaction: options.transaction }).asCallback(callback)
+// }
+
+// exports.down = function (options, callback) {
+//   throw new Error('Not implemented.')
+// }

server/initializers/migrations/0010-users-password.js

@@ -1,22 +0,0 @@
-/*
-  Convert plain user password to encrypted user password.
-*/
-
-const eachSeries = require('async/eachSeries')
-const mongoose = require('mongoose')
-
-const User = mongoose.model('User')
-
-exports.up = function (callback) {
-  User.list(function (err, users) {
-    if (err) return callback(err)
-
-    eachSeries(users, function (user, callbackEach) {
-      user.save(callbackEach)
-    }, callback)
-  })
-}
-
-exports.down = function (callback) {
-  throw new Error('Not implemented.')
-}

server/initializers/migrations/0015-admin-role.js

@@ -1,16 +0,0 @@
-/*
-  Set the admin role to the root user.
-*/
-
-const constants = require('../constants')
-const mongoose = require('mongoose')
-
-const User = mongoose.model('User')
-
-exports.up = function (callback) {
-  User.update({ username: 'root' }, { role: constants.USER_ROLES.ADMIN }, callback)
-}
-
-exports.down = function (callback) {
-  throw new Error('Not implemented.')
-}

server/initializers/migrations/0020-requests-endpoint.js

@@ -1,15 +0,0 @@
-/*
-  Set the endpoint videos for requests.
-*/
-
-const mongoose = require('mongoose')
-
-const Request = mongoose.model('Request')
-
-exports.up = function (callback) {
-  Request.update({ }, { endpoint: 'videos' }, callback)
-}
-
-exports.down = function (callback) {
-  throw new Error('Not implemented.')
-}

server/initializers/migrations/0025-video-filenames.js

@@ -1,57 +0,0 @@
-/*
-  Rename thumbnails and video filenames to _id.extension
-*/
-
-const each = require('async/each')
-const fs = require('fs')
-const path = require('path')
-const mongoose = require('mongoose')
-
-const constants = require('../constants')
-const logger = require('../../helpers/logger')
-
-const Video = mongoose.model('Video')
-
-exports.up = function (callback) {
-  // Use of lean because the new Video scheme does not have filename field
-  Video.find({ filename: { $ne: null } }).lean().exec(function (err, videos) {
-    if (err) throw err
-
-    each(videos, function (video, callbackEach) {
-      const torrentName = video.filename + '.torrent'
-      const thumbnailName = video.thumbnail
-      const thumbnailExtension = path.extname(thumbnailName)
-      const videoName = video.filename
-      const videoExtension = path.extname(videoName)
-
-      const newTorrentName = video._id + '.torrent'
-      const newThumbnailName = video._id + thumbnailExtension
-      const newVideoName = video._id + videoExtension
-
-      const torrentsDir = constants.CONFIG.STORAGE.TORRENTS_DIR
-      const thumbnailsDir = constants.CONFIG.STORAGE.THUMBNAILS_DIR
-      const videosDir = constants.CONFIG.STORAGE.VIDEOS_DIR
-
-      logger.info('Renaming %s to %s.', torrentsDir + torrentName, torrentsDir + newTorrentName)
-      fs.renameSync(torrentsDir + torrentName, torrentsDir + newTorrentName)
-
-      logger.info('Renaming %s to %s.', thumbnailsDir + thumbnailName, thumbnailsDir + newThumbnailName)
-      fs.renameSync(thumbnailsDir + thumbnailName, thumbnailsDir + newThumbnailName)
-
-      logger.info('Renaming %s to %s.', videosDir + videoName, videosDir + newVideoName)
-      fs.renameSync(videosDir + videoName, videosDir + newVideoName)
-
-      Video.load(video._id, function (err, videoObj) {
-        if (err) return callbackEach(err)
-
-        videoObj.extname = videoExtension
-        videoObj.remoteId = null
-        videoObj.save(callbackEach)
-      })
-    }, callback)
-  })
-}
-
-exports.down = function (callback) {
-  throw new Error('Not implemented.')
-}

server/initializers/migrations/0030-video-magnet.js

@@ -1,32 +0,0 @@
-/*
-  Change video magnet structures
-*/
-
-const each = require('async/each')
-const magnet = require('magnet-uri')
-const mongoose = require('mongoose')
-
-const Video = mongoose.model('Video')
-
-exports.up = function (callback) {
-  // Use of lean because the new Video scheme does not have magnetUri field
-  Video.find({ }).lean().exec(function (err, videos) {
-    if (err) throw err
-
-    each(videos, function (video, callbackEach) {
-      const parsed = magnet.decode(video.magnetUri)
-      const infoHash = parsed.infoHash
-
-      Video.load(video._id, function (err, videoObj) {
-        if (err) return callbackEach(err)
-
-        videoObj.magnet.infoHash = infoHash
-        videoObj.save(callbackEach)
-      })
-    }, callback)
-  })
-}
-
-exports.down = function (callback) {
-  throw new Error('Not implemented.')
-}

server/initializers/migrations/0035-url-to-host.js

@@ -1,30 +0,0 @@
-/*
-  Change video magnet structures
-*/
-
-const each = require('async/each')
-const mongoose = require('mongoose')
-
-const Video = mongoose.model('Video')
-
-exports.up = function (callback) {
-  // Use of lean because the new Video scheme does not have podUrl field
-  Video.find({ }).lean().exec(function (err, videos) {
-    if (err) throw err
-
-    each(videos, function (video, callbackEach) {
-      Video.load(video._id, function (err, videoObj) {
-        if (err) return callbackEach(err)
-
-        const host = video.podUrl.split('://')[1]
-
-        videoObj.podHost = host
-        videoObj.save(callbackEach)
-      })
-    }, callback)
-  })
-}
-
-exports.down = function (callback) {
-  throw new Error('Not implemented.')
-}

server/initializers/migrations/0040-video-remote-id.js

@@ -1,59 +0,0 @@
-/*
-  Use remote id as identifier
-*/
-
-const map = require('lodash/map')
-const mongoose = require('mongoose')
-const readline = require('readline')
-
-const rl = readline.createInterface({
-  input: process.stdin,
-  output: process.stdout
-})
-
-const logger = require('../../helpers/logger')
-const friends = require('../../lib/friends')
-
-const Pod = mongoose.model('Pod')
-const Video = mongoose.model('Video')
-
-exports.up = function (callback) {
-  Pod.find({}).lean().exec(function (err, pods) {
-    if (err) return callback(err)
-
-    // We need to quit friends first
-    if (pods.length === 0) {
-      return setVideosRemoteId(callback)
-    }
-
-    const timeout = setTimeout(function () {
-      throw new Error('You need to enter a value!')
-    }, 10000)
-
-    rl.question('I am sorry but I need to quit friends for upgrading. Do you want to continue? (yes/*)', function (answer) {
-      if (answer !== 'yes') throw new Error('I cannot continue.')
-
-      clearTimeout(timeout)
-      rl.close()
-
-      const urls = map(pods, 'url')
-      logger.info('Saying goodbye to: ' + urls.join(', '))
-
-      setVideosRemoteId(function () {
-        friends.quitFriends(callback)
-      })
-    })
-  })
-}
-
-exports.down = function (callback) {
-  throw new Error('Not implemented.')
-}
-
-function setVideosRemoteId (callback) {
-  Video.update({ filename: { $ne: null } }, { remoteId: null }, function (err) {
-    if (err) throw err
-
-    Video.update({ filename: null }, { remoteId: mongoose.Types.ObjectId() }, callback)
-  })
-}

server/initializers/migrator.js

@@ -1,48 +1,36 @@
 'use strict'
 
 const eachSeries = require('async/eachSeries')
-const mongoose = require('mongoose')
+const fs = require('fs')
 const path = require('path')
 
 const constants = require('./constants')
+const db = require('./database')
 const logger = require('../helpers/logger')
 
-const Application = mongoose.model('Application')
-
 const migrator = {
   migrate: migrate
 }
 
 function migrate (callback) {
-  Application.loadMongoSchemaVersion(function (err, actualVersion) {
+  db.Application.loadMigrationVersion(function (err, actualVersion) {
     if (err) return callback(err)
 
-    // If there are a new mongo schemas
+    // If there are a new migration scripts
-    if (!actualVersion || actualVersion < constants.LAST_MONGO_SCHEMA_VERSION) {
+    if (actualVersion < constants.LAST_MIGRATION_VERSION) {
       logger.info('Begin migrations.')
 
-      eachSeries(constants.MONGO_MIGRATION_SCRIPTS, function (entity, callbackEach) {
+      getMigrationScripts(function (err, migrationScripts) {
-        const versionScript = entity.version
-
-        // Do not execute old migration scripts
-        if (versionScript <= actualVersion) return callbackEach(null)
-
-        // Load the migration module and run it
-        const migrationScriptName = entity.script
-        logger.info('Executing %s migration script.', migrationScriptName)
-
-        const migrationScript = require(path.join(__dirname, 'migrations', migrationScriptName))
-        migrationScript.up(function (err) {
-          if (err) return callbackEach(err)
-
-          // Update the new mongo version schema
-          Application.updateMongoSchemaVersion(versionScript, callbackEach)
-        })
-      }, function (err) {
         if (err) return callback(err)
 
-        logger.info('Migrations finished. New mongo version schema: %s', constants.LAST_MONGO_SCHEMA_VERSION)
+        eachSeries(migrationScripts, function (entity, callbackEach) {
-        return callback(null)
+          executeMigration(actualVersion, entity, callbackEach)
+        }, function (err) {
+          if (err) return callback(err)
+
+          logger.info('Migrations finished. New migration version schema: %s', constants.LAST_MIGRATION_VERSION)
+          return callback(null)
+        })
       })
     } else {
       return callback(null)
@@ -54,3 +42,57 @@ function migrate (callback) {
 
 module.exports = migrator
 
+// ---------------------------------------------------------------------------
+
+function getMigrationScripts (callback) {
+  fs.readdir(path.join(__dirname, 'migrations'), function (err, files) {
+    if (err) return callback(err)
+
+    const filesToMigrate = []
+
+    files.forEach(function (file) {
+      // Filename is something like 'version-blabla.js'
+      const version = file.split('-')[0]
+      filesToMigrate.push({
+        version,
+        script: file
+      })
+    })
+
+    return callback(err, filesToMigrate)
+  })
+}
+
+function executeMigration (actualVersion, entity, callback) {
+  const versionScript = entity.version
+
+  // Do not execute old migration scripts
+  if (versionScript <= actualVersion) return callback(null)
+
+  // Load the migration module and run it
+  const migrationScriptName = entity.script
+  logger.info('Executing %s migration script.', migrationScriptName)
+
+  const migrationScript = require(path.join(__dirname, 'migrations', migrationScriptName))
+
+  db.sequelize.transaction().asCallback(function (err, t) {
+    if (err) return callback(err)
+
+    migrationScript.up({ transaction: t }, function (err) {
+      if (err) {
+        t.rollback()
+        return callback(err)
+      }
+
+      // Update the new migration version
+      db.Application.updateMigrationVersion(versionScript, t, function (err) {
+        if (err) {
+          t.rollback()
+          return callback(err)
+        }
+
+        t.commit().asCallback(callback)
+      })
+    })
+  })
+}

server/lib/friends.js

@@ -4,20 +4,18 @@ const each = require('async/each')
 const eachLimit = require('async/eachLimit')
 const eachSeries = require('async/eachSeries')
 const fs = require('fs')
-const mongoose = require('mongoose')
 const request = require('request')
 const waterfall = require('async/waterfall')
 
 const constants = require('../initializers/constants')
+const db = require('../initializers/database')
 const logger = require('../helpers/logger')
 const requests = require('../helpers/requests')
 
-const Pod = mongoose.model('Pod')
-const Request = mongoose.model('Request')
-const Video = mongoose.model('Video')
-
 const friends = {
   addVideoToFriends,
+  updateVideoToFriends,
+  reportAbuseVideoToFriend,
   hasFriends,
   getMyCertificate,
   makeFriends,
@@ -26,12 +24,47 @@ const friends = {
   sendOwnedVideosToPod
 }
 
-function addVideoToFriends (video) {
+function addVideoToFriends (videoData, transaction, callback) {
-  createRequest('add', constants.REQUEST_ENDPOINTS.VIDEOS, video)
+  const options = {
+    type: 'add',
+    endpoint: constants.REQUEST_ENDPOINTS.VIDEOS,
+    data: videoData,
+    transaction
+  }
+  createRequest(options, callback)
+}
+
+function updateVideoToFriends (videoData, transaction, callback) {
+  const options = {
+    type: 'update',
+    endpoint: constants.REQUEST_ENDPOINTS.VIDEOS,
+    data: videoData,
+    transaction
+  }
+  createRequest(options, callback)
+}
+
+function removeVideoToFriends (videoParams) {
+  const options = {
+    type: 'remove',
+    endpoint: constants.REQUEST_ENDPOINTS.VIDEOS,
+    data: videoParams
+  }
+  createRequest(options)
+}
+
+function reportAbuseVideoToFriend (reportData, video) {
+  const options = {
+    type: 'report-abuse',
+    endpoint: constants.REQUEST_ENDPOINTS.VIDEOS,
+    data: reportData,
+    toIds: [ video.Author.podId ]
+  }
+  createRequest(options)
 }
 
 function hasFriends (callback) {
-  Pod.countAll(function (err, count) {
+  db.Pod.countAll(function (err, count) {
     if (err) return callback(err)
 
     const hasFriends = (count !== 0)
@@ -69,13 +102,15 @@ function makeFriends (hosts, callback) {
 
 function quitFriends (callback) {
   // Stop pool requests
-  Request.deactivate()
+  db.Request.deactivate()
-  // Flush pool requests
-  Request.flush()
 
   waterfall([
+    function flushRequests (callbackAsync) {
+      db.Request.flush(callbackAsync)
+    },
+
     function getPodsList (callbackAsync) {
-      return Pod.list(callbackAsync)
+      return db.Pod.list(callbackAsync)
     },
 
     function announceIQuitMyFriends (pods, callbackAsync) {
@@ -103,12 +138,12 @@ function quitFriends (callback) {
 
     function removePodsFromDB (pods, callbackAsync) {
       each(pods, function (pod, callbackEach) {
-        pod.remove(callbackEach)
+        pod.destroy().asCallback(callbackEach)
       }, callbackAsync)
     }
   ], function (err) {
     // Don't forget to re activate the scheduler, even if there was an error
-    Request.activate()
+    db.Request.activate()
 
     if (err) return callback(err)
 
@@ -117,26 +152,28 @@ function quitFriends (callback) {
   })
 }
 
-function removeVideoToFriends (videoParams) {
-  createRequest('remove', constants.REQUEST_ENDPOINTS.VIDEOS, videoParams)
-}
-
 function sendOwnedVideosToPod (podId) {
-  Video.listOwned(function (err, videosList) {
+  db.Video.listOwnedAndPopulateAuthorAndTags(function (err, videosList) {
     if (err) {
       logger.error('Cannot get the list of videos we own.')
       return
     }
 
     videosList.forEach(function (video) {
-      video.toRemoteJSON(function (err, remoteVideo) {
+      video.toAddRemoteJSON(function (err, remoteVideo) {
         if (err) {
           logger.error('Cannot convert video to remote.', { error: err })
           // Don't break the process
           return
         }
 
-        createRequest('add', constants.REQUEST_ENDPOINTS.VIDEOS, remoteVideo, [ podId ])
+        const options = {
+          type: 'add',
+          endpoint: constants.REQUEST_ENDPOINTS.VIDEOS,
+          data: remoteVideo,
+          toIds: [ podId ]
+        }
+        createRequest(options)
       })
     })
   })
@@ -149,10 +186,10 @@ module.exports = friends
 // ---------------------------------------------------------------------------
 
 function computeForeignPodsList (host, podsScore, callback) {
-  getForeignPodsList(host, function (err, foreignPodsList) {
+  getForeignPodsList(host, function (err, res) {
     if (err) return callback(err)
 
-    if (!foreignPodsList) foreignPodsList = []
+    const foreignPodsList = res.data
 
     // Let's give 1 point to the pod we ask the friends list
     foreignPodsList.push({ host })
@@ -200,9 +237,9 @@ function getForeignPodsList (host, callback) {
 
 function makeRequestsToWinningPods (cert, podsList, callback) {
   // Stop pool requests
-  Request.deactivate()
+  db.Request.deactivate()
   // Flush pool requests
-  Request.forceSend()
+  db.Request.forceSend()
 
   eachLimit(podsList, constants.REQUESTS_IN_PARALLEL, function (pod, callbackEach) {
     const params = {
@@ -222,15 +259,15 @@ function makeRequestsToWinningPods (cert, podsList, callback) {
       }
 
       if (res.statusCode === 200) {
-        const podObj = new Pod({ host: pod.host, publicKey: body.cert })
+        const podObj = db.Pod.build({ host: pod.host, publicKey: body.cert })
-        podObj.save(function (err, podCreated) {
+        podObj.save().asCallback(function (err, podCreated) {
           if (err) {
             logger.error('Cannot add friend %s pod.', pod.host, { error: err })
             return callbackEach()
           }
 
           // Add our videos to the request scheduler
-          sendOwnedVideosToPod(podCreated._id)
+          sendOwnedVideosToPod(podCreated.id)
 
           return callbackEach()
         })
@@ -242,28 +279,64 @@ function makeRequestsToWinningPods (cert, podsList, callback) {
   }, function endRequests () {
     // Final callback, we've ended all the requests
     // Now we made new friends, we can re activate the pool of requests
-    Request.activate()
+    db.Request.activate()
 
     logger.debug('makeRequestsToWinningPods finished.')
     return callback()
   })
 }
 
-function createRequest (type, endpoint, data, to) {
+// Wrapper that populate "toIds" argument with all our friends if it is not specified
-  const req = new Request({
+// { type, endpoint, data, toIds, transaction }
+function createRequest (options, callback) {
+  if (!callback) callback = function () {}
+  if (options.toIds) return _createRequest(options, callback)
+
+  // If the "toIds" pods is not specified, we send the request to all our friends
+  db.Pod.listAllIds(options.transaction, function (err, podIds) {
+    if (err) {
+      logger.error('Cannot get pod ids', { error: err })
+      return
+    }
+
+    const newOptions = Object.assign(options, { toIds: podIds })
+    return _createRequest(newOptions, callback)
+  })
+}
+
+// { type, endpoint, data, toIds, transaction }
+function _createRequest (options, callback) {
+  const type = options.type
+  const endpoint = options.endpoint
+  const data = options.data
+  const toIds = options.toIds
+  const transaction = options.transaction
+
+  const pods = []
+
+  // If there are no destination pods abort
+  if (toIds.length === 0) return callback(null)
+
+  toIds.forEach(function (toPod) {
+    pods.push(db.Pod.build({ id: toPod }))
+  })
+
+  const createQuery = {
     endpoint,
     request: {
       type: type,
       data: data
     }
-  })
-
-  if (to) {
-    req.to = to
   }
 
-  req.save(function (err) {
+  const dbRequestOptions = {
-    if (err) logger.error('Cannot save the request.', { error: err })
+    transaction
+  }
+
+  return db.Request.create(createQuery, dbRequestOptions).asCallback(function (err, request) {
+    if (err) return callback(err)
+
+    return request.setPods(pods, dbRequestOptions).asCallback(callback)
   })
 }
 

server/lib/oauth-model.js

@@ -1,11 +1,6 @@
-const mongoose = require('mongoose')
+const db = require('../initializers/database')
-
 const logger = require('../helpers/logger')
 
-const OAuthClient = mongoose.model('OAuthClient')
-const OAuthToken = mongoose.model('OAuthToken')
-const User = mongoose.model('User')
-
 // See https://github.com/oauthjs/node-oauth2-server/wiki/Model-specification for the model specifications
 const OAuthModel = {
   getAccessToken,
@@ -21,27 +16,25 @@ const OAuthModel = {
 function getAccessToken (bearerToken) {
   logger.debug('Getting access token (bearerToken: ' + bearerToken + ').')
 
-  return OAuthToken.getByTokenAndPopulateUser(bearerToken)
+  return db.OAuthToken.getByTokenAndPopulateUser(bearerToken)
 }
 
 function getClient (clientId, clientSecret) {
   logger.debug('Getting Client (clientId: ' + clientId + ', clientSecret: ' + clientSecret + ').')
 
-  // TODO req validator
+  return db.OAuthClient.getByIdAndSecret(clientId, clientSecret)
-  const mongoId = new mongoose.mongo.ObjectID(clientId)
-  return OAuthClient.getByIdAndSecret(mongoId, clientSecret)
 }
 
 function getRefreshToken (refreshToken) {
   logger.debug('Getting RefreshToken (refreshToken: ' + refreshToken + ').')
 
-  return OAuthToken.getByRefreshTokenAndPopulateClient(refreshToken)
+  return db.OAuthToken.getByRefreshTokenAndPopulateClient(refreshToken)
 }
 
 function getUser (username, password) {
   logger.debug('Getting User (username: ' + username + ', password: ' + password + ').')
 
-  return User.getByUsername(username).then(function (user) {
+  return db.User.getByUsername(username).then(function (user) {
     if (!user) return null
 
     // We need to return a promise
@@ -60,8 +53,8 @@ function getUser (username, password) {
 }
 
 function revokeToken (token) {
-  return OAuthToken.getByRefreshTokenAndPopulateUser(token.refreshToken).then(function (tokenDB) {
+  return db.OAuthToken.getByRefreshTokenAndPopulateUser(token.refreshToken).then(function (tokenDB) {
-    if (tokenDB) tokenDB.remove()
+    if (tokenDB) tokenDB.destroy()
 
     /*
       * Thanks to https://github.com/manjeshpv/node-oauth2-server-implementation/blob/master/components/oauth/mongo-models.js
@@ -80,18 +73,19 @@ function revokeToken (token) {
 function saveToken (token, client, user) {
   logger.debug('Saving token ' + token.accessToken + ' for client ' + client.id + ' and user ' + user.id + '.')
 
-  const tokenObj = new OAuthToken({
+  const tokenToCreate = {
     accessToken: token.accessToken,
     accessTokenExpiresAt: token.accessTokenExpiresAt,
-    client: client.id,
     refreshToken: token.refreshToken,
     refreshTokenExpiresAt: token.refreshTokenExpiresAt,
-    user: user.id
+    oAuthClientId: client.id,
-  })
+    userId: user.id
+  }
 
-  return tokenObj.save().then(function (tokenCreated) {
+  return db.OAuthToken.create(tokenToCreate).then(function (tokenCreated) {
     tokenCreated.client = client
     tokenCreated.user = user
+
     return tokenCreated
   }).catch(function (err) {
     throw err

server/middlewares/pods.js

@@ -44,7 +44,6 @@ module.exports = podsMiddleware
 function getHostWithPort (host) {
   const splitted = host.split(':')
 
-  console.log(splitted)
   // The port was not specified
   if (splitted.length === 1) {
     if (constants.REMOTE_SCHEME.HTTP === 'https') return host + ':443'

server/middlewares/secure.js

@@ -1,18 +1,16 @@
 'use strict'
 
+const db = require('../initializers/database')
 const logger = require('../helpers/logger')
-const mongoose = require('mongoose')
 const peertubeCrypto = require('../helpers/peertube-crypto')
 
-const Pod = mongoose.model('Pod')
-
 const secureMiddleware = {
   checkSignature
 }
 
 function checkSignature (req, res, next) {
   const host = req.body.signature.host
-  Pod.loadByHost(host, function (err, pod) {
+  db.Pod.loadByHost(host, function (err, pod) {
     if (err) {
       logger.error('Cannot get signed host in body.', { error: err })
       return res.sendStatus(500)
@@ -25,9 +23,20 @@ function checkSignature (req, res, next) {
 
     logger.debug('Checking signature from %s.', host)
 
-    const signatureOk = peertubeCrypto.checkSignature(pod.publicKey, host, req.body.signature.signature)
+    let signatureShouldBe
+    if (req.body.data) {
+      signatureShouldBe = req.body.data
+    } else {
+      signatureShouldBe = host
+    }
+
+    const signatureOk = peertubeCrypto.checkSignature(pod.publicKey, signatureShouldBe, req.body.signature.signature)
 
     if (signatureOk === true) {
+      res.locals.secure = {
+        pod
+      }
+
       return next()
     }
 

server/middlewares/sort.js

@@ -2,17 +2,24 @@
 
 const sortMiddleware = {
   setUsersSort,
+  setVideoAbusesSort,
   setVideosSort
 }
 
 function setUsersSort (req, res, next) {
-  if (!req.query.sort) req.query.sort = '-createdDate'
+  if (!req.query.sort) req.query.sort = '-createdAt'
+
+  return next()
+}
+
+function setVideoAbusesSort (req, res, next) {
+  if (!req.query.sort) req.query.sort = '-createdAt'
 
   return next()
 }
 
 function setVideosSort (req, res, next) {
-  if (!req.query.sort) req.query.sort = '-createdDate'
+  if (!req.query.sort) req.query.sort = '-createdAt'
 
   return next()
 }

server/middlewares/validators/remote.js

@@ -1,30 +0,0 @@
-'use strict'
-
-const checkErrors = require('./utils').checkErrors
-const logger = require('../../helpers/logger')
-
-const validatorsRemote = {
-  remoteVideos,
-  signature
-}
-
-function remoteVideos (req, res, next) {
-  req.checkBody('data').isEachRemoteVideosValid()
-
-  logger.debug('Checking remoteVideos parameters', { parameters: req.body })
-
-  checkErrors(req, res, next)
-}
-
-function signature (req, res, next) {
-  req.checkBody('signature.host', 'Should have a signature host').isURL()
-  req.checkBody('signature.signature', 'Should have a signature').notEmpty()
-
-  logger.debug('Checking signature parameters', { parameters: { signatureHost: req.body.signature.host } })
-
-  checkErrors(req, res, next)
-}
-
-// ---------------------------------------------------------------------------
-
-module.exports = validatorsRemote

server/middlewares/validators/remote/index.js

@@ -0,0 +1,13 @@
+'use strict'
+
+const remoteSignatureValidators = require('./signature')
+const remoteVideosValidators = require('./videos')
+
+const validators = {
+  signature: remoteSignatureValidators,
+  videos: remoteVideosValidators
+}
+
+// ---------------------------------------------------------------------------
+
+module.exports = validators

server/middlewares/validators/remote/signature.js

@@ -0,0 +1,21 @@
+'use strict'
+
+const checkErrors = require('../utils').checkErrors
+const logger = require('../../../helpers/logger')
+
+const validatorsRemoteSignature = {
+  signature
+}
+
+function signature (req, res, next) {
+  req.checkBody('signature.host', 'Should have a signature host').isURL()
+  req.checkBody('signature.signature', 'Should have a signature').notEmpty()
+
+  logger.debug('Checking signature parameters', { parameters: { signature: req.body.signature } })
+
+  checkErrors(req, res, next)
+}
+
+// ---------------------------------------------------------------------------
+
+module.exports = validatorsRemoteSignature

server/middlewares/validators/remote/videos.js

@@ -0,0 +1,20 @@
+'use strict'
+
+const checkErrors = require('../utils').checkErrors
+const logger = require('../../../helpers/logger')
+
+const validatorsRemoteVideos = {
+  remoteVideos
+}
+
+function remoteVideos (req, res, next) {
+  req.checkBody('data').isEachRemoteRequestVideosValid()
+
+  logger.debug('Checking remoteVideos parameters', { parameters: req.body })
+
+  checkErrors(req, res, next)
+}
+
+// ---------------------------------------------------------------------------
+
+module.exports = validatorsRemoteVideos

server/middlewares/validators/sort.js

@@ -6,29 +6,38 @@ const logger = require('../../helpers/logger')
 
 const validatorsSort = {
   usersSort,
+  videoAbusesSort,
   videosSort
 }
 
 function usersSort (req, res, next) {
   const sortableColumns = constants.SORTABLE_COLUMNS.USERS
 
-  req.checkQuery('sort', 'Should have correct sortable column').optional().isIn(sortableColumns)
+  checkSort(req, res, next, sortableColumns)
+}
 
-  logger.debug('Checking sort parameters', { parameters: req.query })
+function videoAbusesSort (req, res, next) {
+  const sortableColumns = constants.SORTABLE_COLUMNS.VIDEO_ABUSES
 
-  checkErrors(req, res, next)
+  checkSort(req, res, next, sortableColumns)
 }
 
 function videosSort (req, res, next) {
   const sortableColumns = constants.SORTABLE_COLUMNS.VIDEOS
 
+  checkSort(req, res, next, sortableColumns)
+}
+
+// ---------------------------------------------------------------------------
+
+module.exports = validatorsSort
+
+// ---------------------------------------------------------------------------
+
+function checkSort (req, res, next, sortableColumns) {
   req.checkQuery('sort', 'Should have correct sortable column').optional().isIn(sortableColumns)
 
   logger.debug('Checking sort parameters', { parameters: req.query })
 
   checkErrors(req, res, next)
 }
-
-// ---------------------------------------------------------------------------
-
-module.exports = validatorsSort

server/middlewares/validators/users.js

@@ -1,12 +1,9 @@
 'use strict'
 
-const mongoose = require('mongoose')
-
 const checkErrors = require('./utils').checkErrors
+const db = require('../../initializers/database')
 const logger = require('../../helpers/logger')
 
-const User = mongoose.model('User')
-
 const validatorsUsers = {
   usersAdd,
   usersRemove,
@@ -20,7 +17,7 @@ function usersAdd (req, res, next) {
   logger.debug('Checking usersAdd parameters', { parameters: req.body })
 
   checkErrors(req, res, function () {
-    User.loadByUsername(req.body.username, function (err, user) {
+    db.User.loadByUsername(req.body.username, function (err, user) {
       if (err) {
         logger.error('Error in usersAdd request validator.', { error: err })
         return res.sendStatus(500)
@@ -34,12 +31,12 @@ function usersAdd (req, res, next) {
 }
 
 function usersRemove (req, res, next) {
-  req.checkParams('id', 'Should have a valid id').notEmpty().isMongoId()
+  req.checkParams('id', 'Should have a valid id').notEmpty().isInt()
 
   logger.debug('Checking usersRemove parameters', { parameters: req.params })
 
   checkErrors(req, res, function () {
-    User.loadById(req.params.id, function (err, user) {
+    db.User.loadById(req.params.id, function (err, user) {
       if (err) {
         logger.error('Error in usersRemove request validator.', { error: err })
         return res.sendStatus(500)
@@ -55,7 +52,7 @@ function usersRemove (req, res, next) {
 }
 
 function usersUpdate (req, res, next) {
-  req.checkParams('id', 'Should have a valid id').notEmpty().isMongoId()
+  req.checkParams('id', 'Should have a valid id').notEmpty().isInt()
   // Add old password verification
   req.checkBody('password', 'Should have a valid password').isUserPasswordValid()
 

server/middlewares/validators/videos.js

@@ -1,19 +1,19 @@
 'use strict'
 
-const mongoose = require('mongoose')
-
 const checkErrors = require('./utils').checkErrors
 const constants = require('../../initializers/constants')
 const customVideosValidators = require('../../helpers/custom-validators').videos
+const db = require('../../initializers/database')
 const logger = require('../../helpers/logger')
 
-const Video = mongoose.model('Video')
-
 const validatorsVideos = {
   videosAdd,
+  videosUpdate,
   videosGet,
   videosRemove,
-  videosSearch
+  videosSearch,
+
+  videoAbuseReport
 }
 
 function videosAdd (req, res, next) {
@@ -29,7 +29,7 @@ function videosAdd (req, res, next) {
   checkErrors(req, res, function () {
     const videoFile = req.files.videofile[0]
 
-    Video.getDurationFromFile(videoFile.path, function (err, duration) {
+    db.Video.getDurationFromFile(videoFile.path, function (err, duration) {
       if (err) {
         return res.status(400).send('Cannot retrieve metadata of the file.')
       }
@@ -44,40 +44,56 @@ function videosAdd (req, res, next) {
   })
 }
 
-function videosGet (req, res, next) {
+function videosUpdate (req, res, next) {
-  req.checkParams('id', 'Should have a valid id').notEmpty().isMongoId()
+  req.checkParams('id', 'Should have a valid id').notEmpty().isUUID(4)
+  req.checkBody('name', 'Should have a valid name').optional().isVideoNameValid()
+  req.checkBody('description', 'Should have a valid description').optional().isVideoDescriptionValid()
+  req.checkBody('tags', 'Should have correct tags').optional().isVideoTagsValid()
 
-  logger.debug('Checking videosGet parameters', { parameters: req.params })
+  logger.debug('Checking videosUpdate parameters', { parameters: req.body })
 
   checkErrors(req, res, function () {
-    Video.load(req.params.id, function (err, video) {
+    checkVideoExists(req.params.id, res, function () {
-      if (err) {
+      // We need to make additional checks
-        logger.error('Error in videosGet request validator.', { error: err })
+      if (res.locals.video.isOwned() === false) {
-        return res.sendStatus(500)
+        return res.status(403).send('Cannot update video of another pod')
       }
 
-      if (!video) return res.status(404).send('Video not found')
+      if (res.locals.video.Author.userId !== res.locals.oauth.token.User.id) {
+        return res.status(403).send('Cannot update video of another user')
+      }
 
       next()
     })
   })
 }
 
+function videosGet (req, res, next) {
+  req.checkParams('id', 'Should have a valid id').notEmpty().isUUID(4)
+
+  logger.debug('Checking videosGet parameters', { parameters: req.params })
+
+  checkErrors(req, res, function () {
+    checkVideoExists(req.params.id, res, next)
+  })
+}
+
 function videosRemove (req, res, next) {
-  req.checkParams('id', 'Should have a valid id').notEmpty().isMongoId()
+  req.checkParams('id', 'Should have a valid id').notEmpty().isUUID(4)
 
   logger.debug('Checking videosRemove parameters', { parameters: req.params })
 
   checkErrors(req, res, function () {
-    Video.load(req.params.id, function (err, video) {
+    checkVideoExists(req.params.id, res, function () {
-      if (err) {
+      // We need to make additional checks
-        logger.error('Error in videosRemove request validator.', { error: err })
+
-        return res.sendStatus(500)
+      if (res.locals.video.isOwned() === false) {
+        return res.status(403).send('Cannot remove video of another pod')
       }
 
-      if (!video) return res.status(404).send('Video not found')
+      if (res.locals.video.Author.userId !== res.locals.oauth.token.User.id) {
-      else if (video.isOwned() === false) return res.status(403).send('Cannot remove video of another pod')
+        return res.status(403).send('Cannot remove video of another user')
-      else if (video.author !== res.locals.oauth.token.user.username) return res.status(403).send('Cannot remove video of another user')
+      }
 
       next()
     })
@@ -94,6 +110,33 @@ function videosSearch (req, res, next) {
   checkErrors(req, res, next)
 }
 
+function videoAbuseReport (req, res, next) {
+  req.checkParams('id', 'Should have a valid id').notEmpty().isUUID(4)
+  req.checkBody('reason', 'Should have a valid reason').isVideoAbuseReasonValid()
+
+  logger.debug('Checking videoAbuseReport parameters', { parameters: req.body })
+
+  checkErrors(req, res, function () {
+    checkVideoExists(req.params.id, res, next)
+  })
+}
+
 // ---------------------------------------------------------------------------
 
 module.exports = validatorsVideos
+
+// ---------------------------------------------------------------------------
+
+function checkVideoExists (id, res, callback) {
+  db.Video.loadAndPopulateAuthorAndPodAndTags(id, function (err, video) {
+    if (err) {
+      logger.error('Error in video request validator.', { error: err })
+      return res.sendStatus(500)
+    }
+
+    if (!video) return res.status(404).send('Video not found')
+
+    res.locals.video = video
+    callback()
+  })
+}

server/models/application.js

@@ -1,31 +1,52 @@
-const mongoose = require('mongoose')
+'use strict'
 
-// ---------------------------------------------------------------------------
+module.exports = function (sequelize, DataTypes) {
+  const Application = sequelize.define('Application',
+    {
+      migrationVersion: {
+        type: DataTypes.INTEGER,
+        defaultValue: 0,
+        allowNull: false,
+        validate: {
+          isInt: true
+        }
+      }
+    },
+    {
+      classMethods: {
+        loadMigrationVersion,
+        updateMigrationVersion
+      }
+    }
+  )
 
-const ApplicationSchema = mongoose.Schema({
+  return Application
-  mongoSchemaVersion: {
-    type: Number,
-    default: 0
-  }
-})
-
-ApplicationSchema.statics = {
-  loadMongoSchemaVersion,
-  updateMongoSchemaVersion
 }
 
-mongoose.model('Application', ApplicationSchema)
-
 // ---------------------------------------------------------------------------
 
-function loadMongoSchemaVersion (callback) {
+function loadMigrationVersion (callback) {
-  return this.findOne({}, { mongoSchemaVersion: 1 }, function (err, data) {
+  const query = {
-    const version = data ? data.mongoSchemaVersion : 0
+    attributes: [ 'migrationVersion' ]
+  }
+
+  return this.findOne(query).asCallback(function (err, data) {
+    const version = data ? data.migrationVersion : 0
 
     return callback(err, version)
   })
 }
 
-function updateMongoSchemaVersion (newVersion, callback) {
+function updateMigrationVersion (newVersion, transaction, callback) {
-  return this.update({}, { mongoSchemaVersion: newVersion }, callback)
+  const options = {
+    where: {}
+  }
+
+  if (!callback) {
+    transaction = callback
+  } else {
+    options.transaction = transaction
+  }
+
+  return this.update({ migrationVersion: newVersion }, options).asCallback(callback)
 }

server/models/author.js

@@ -0,0 +1,85 @@
+'use strict'
+
+const customUsersValidators = require('../helpers/custom-validators').users
+
+module.exports = function (sequelize, DataTypes) {
+  const Author = sequelize.define('Author',
+    {
+      name: {
+        type: DataTypes.STRING,
+        allowNull: false,
+        validate: {
+          usernameValid: function (value) {
+            const res = customUsersValidators.isUserUsernameValid(value)
+            if (res === false) throw new Error('Username is not valid.')
+          }
+        }
+      }
+    },
+    {
+      indexes: [
+        {
+          fields: [ 'name' ]
+        },
+        {
+          fields: [ 'podId' ]
+        },
+        {
+          fields: [ 'userId' ]
+        }
+      ],
+      classMethods: {
+        associate,
+
+        findOrCreateAuthor
+      }
+    }
+  )
+
+  return Author
+}
+
+// ---------------------------------------------------------------------------
+
+function associate (models) {
+  this.belongsTo(models.Pod, {
+    foreignKey: {
+      name: 'podId',
+      allowNull: true
+    },
+    onDelete: 'cascade'
+  })
+
+  this.belongsTo(models.User, {
+    foreignKey: {
+      name: 'userId',
+      allowNull: true
+    },
+    onDelete: 'cascade'
+  })
+}
+
+function findOrCreateAuthor (name, podId, userId, transaction, callback) {
+  if (!callback) {
+    callback = transaction
+    transaction = null
+  }
+
+  const author = {
+    name,
+    podId,
+    userId
+  }
+
+  const query = {
+    where: author,
+    defaults: author
+  }
+
+  if (transaction) query.transaction = transaction
+
+  this.findOrCreate(query).asCallback(function (err, result) {
+    // [ instance, wasCreated ]
+    return callback(err, result[0])
+  })
+}

server/models/oauth-client.js

@@ -1,33 +1,62 @@
-const mongoose = require('mongoose')
+'use strict'
 
-// ---------------------------------------------------------------------------
+module.exports = function (sequelize, DataTypes) {
+  const OAuthClient = sequelize.define('OAuthClient',
+    {
+      clientId: {
+        type: DataTypes.STRING,
+        allowNull: false
+      },
+      clientSecret: {
+        type: DataTypes.STRING,
+        allowNull: false
+      },
+      grants: {
+        type: DataTypes.ARRAY(DataTypes.STRING)
+      },
+      redirectUris: {
+        type: DataTypes.ARRAY(DataTypes.STRING)
+      }
+    },
+    {
+      indexes: [
+        {
+          fields: [ 'clientId' ],
+          unique: true
+        },
+        {
+          fields: [ 'clientId', 'clientSecret' ],
+          unique: true
+        }
+      ],
+      classMethods: {
+        countTotal,
+        getByIdAndSecret,
+        loadFirstClient
+      }
+    }
+  )
 
-const OAuthClientSchema = mongoose.Schema({
+  return OAuthClient
-  clientSecret: String,
-  grants: Array,
-  redirectUris: Array
-})
-
-OAuthClientSchema.path('clientSecret').required(true)
-
-OAuthClientSchema.statics = {
-  getByIdAndSecret,
-  list,
-  loadFirstClient
 }
 
-mongoose.model('OAuthClient', OAuthClientSchema)
-
 // ---------------------------------------------------------------------------
 
-function list (callback) {
+function countTotal (callback) {
-  return this.find(callback)
+  return this.count().asCallback(callback)
 }
 
 function loadFirstClient (callback) {
-  return this.findOne({}, callback)
+  return this.findOne().asCallback(callback)
 }
 
-function getByIdAndSecret (id, clientSecret) {
+function getByIdAndSecret (clientId, clientSecret) {
-  return this.findOne({ _id: id, clientSecret: clientSecret }).exec()
+  const query = {
+    where: {
+      clientId: clientId,
+      clientSecret: clientSecret
+    }
+  }
+
+  return this.findOne(query)
 }

server/models/oauth-token.js

@@ -1,42 +1,96 @@
-const mongoose = require('mongoose')
+'use strict'
 
 const logger = require('../helpers/logger')
 
 // ---------------------------------------------------------------------------
 
-const OAuthTokenSchema = mongoose.Schema({
+module.exports = function (sequelize, DataTypes) {
-  accessToken: String,
+  const OAuthToken = sequelize.define('OAuthToken',
-  accessTokenExpiresAt: Date,
+    {
-  client: { type: mongoose.Schema.Types.ObjectId, ref: 'OAuthClient' },
+      accessToken: {
-  refreshToken: String,
+        type: DataTypes.STRING,
-  refreshTokenExpiresAt: Date,
+        allowNull: false
-  user: { type: mongoose.Schema.Types.ObjectId, ref: 'User' }
+      },
-})
+      accessTokenExpiresAt: {
+        type: DataTypes.DATE,
+        allowNull: false
+      },
+      refreshToken: {
+        type: DataTypes.STRING,
+        allowNull: false
+      },
+      refreshTokenExpiresAt: {
+        type: DataTypes.DATE,
+        allowNull: false
+      }
+    },
+    {
+      indexes: [
+        {
+          fields: [ 'refreshToken' ],
+          unique: true
+        },
+        {
+          fields: [ 'accessToken' ],
+          unique: true
+        },
+        {
+          fields: [ 'userId' ]
+        },
+        {
+          fields: [ 'oAuthClientId' ]
+        }
+      ],
+      classMethods: {
+        associate,
 
-OAuthTokenSchema.path('accessToken').required(true)
+        getByRefreshTokenAndPopulateClient,
-OAuthTokenSchema.path('client').required(true)
+        getByTokenAndPopulateUser,
-OAuthTokenSchema.path('user').required(true)
+        getByRefreshTokenAndPopulateUser,
+        removeByUserId
+      }
+    }
+  )
 
-OAuthTokenSchema.statics = {
+  return OAuthToken
-  getByRefreshTokenAndPopulateClient,
-  getByTokenAndPopulateUser,
-  getByRefreshTokenAndPopulateUser,
-  removeByUserId
 }
 
-mongoose.model('OAuthToken', OAuthTokenSchema)
-
 // ---------------------------------------------------------------------------
 
+function associate (models) {
+  this.belongsTo(models.User, {
+    foreignKey: {
+      name: 'userId',
+      allowNull: false
+    },
+    onDelete: 'cascade'
+  })
+
+  this.belongsTo(models.OAuthClient, {
+    foreignKey: {
+      name: 'oAuthClientId',
+      allowNull: false
+    },
+    onDelete: 'cascade'
+  })
+}
+
 function getByRefreshTokenAndPopulateClient (refreshToken) {
-  return this.findOne({ refreshToken: refreshToken }).populate('client').exec().then(function (token) {
+  const query = {
+    where: {
+      refreshToken: refreshToken
+    },
+    include: [ this.associations.OAuthClient ]
+  }
+
+  return this.findOne(query).then(function (token) {
     if (!token) return token
 
     const tokenInfos = {
       refreshToken: token.refreshToken,
       refreshTokenExpiresAt: token.refreshTokenExpiresAt,
       client: {
-        id: token.client._id.toString()
+        id: token.client.id
       },
       user: {
         id: token.user
@@ -50,13 +104,41 @@ function getByRefreshTokenAndPopulateClient (refreshToken) {
 }
 
 function getByTokenAndPopulateUser (bearerToken) {
-  return this.findOne({ accessToken: bearerToken }).populate('user').exec()
+  const query = {
+    where: {
+      accessToken: bearerToken
+    },
+    include: [ this.sequelize.models.User ]
+  }
+
+  return this.findOne(query).then(function (token) {
+    if (token) token.user = token.User
+
+    return token
+  })
 }
 
 function getByRefreshTokenAndPopulateUser (refreshToken) {
-  return this.findOne({ refreshToken: refreshToken }).populate('user').exec()
+  const query = {
+    where: {
+      refreshToken: refreshToken
+    },
+    include: [ this.sequelize.models.User ]
+  }
+
+  return this.findOne(query).then(function (token) {
+    token.user = token.User
+
+    return token
+  })
 }
 
 function removeByUserId (userId, callback) {
-  return this.remove({ user: userId }, callback)
+  const query = {
+    where: {
+      userId: userId
+    }
+  }
+
+  return this.destroy(query).asCallback(callback)
 }

server/models/pod.js

@@ -0,0 +1,200 @@
+'use strict'
+
+const map = require('lodash/map')
+
+const constants = require('../initializers/constants')
+const customPodsValidators = require('../helpers/custom-validators').pods
+
+// ---------------------------------------------------------------------------
+
+module.exports = function (sequelize, DataTypes) {
+  const Pod = sequelize.define('Pod',
+    {
+      host: {
+        type: DataTypes.STRING,
+        allowNull: false,
+        validate: {
+          isHost: function (value) {
+            const res = customPodsValidators.isHostValid(value)
+            if (res === false) throw new Error('Host not valid.')
+          }
+        }
+      },
+      publicKey: {
+        type: DataTypes.STRING(5000),
+        allowNull: false
+      },
+      score: {
+        type: DataTypes.INTEGER,
+        defaultValue: constants.FRIEND_SCORE.BASE,
+        allowNull: false,
+        validate: {
+          isInt: true,
+          max: constants.FRIEND_SCORE.MAX
+        }
+      }
+    },
+    {
+      indexes: [
+        {
+          fields: [ 'host' ]
+        },
+        {
+          fields: [ 'score' ]
+        }
+      ],
+      classMethods: {
+        associate,
+
+        countAll,
+        incrementScores,
+        list,
+        listAllIds,
+        listRandomPodIdsWithRequest,
+        listBadPods,
+        load,
+        loadByHost,
+        removeAll
+      },
+      instanceMethods: {
+        toFormatedJSON
+      }
+    }
+  )
+
+  return Pod
+}
+
+// ------------------------------ METHODS ------------------------------
+
+function toFormatedJSON () {
+  const json = {
+    id: this.id,
+    host: this.host,
+    score: this.score,
+    createdAt: this.createdAt
+  }
+
+  return json
+}
+
+// ------------------------------ Statics ------------------------------
+
+function associate (models) {
+  this.belongsToMany(models.Request, {
+    foreignKey: 'podId',
+    through: models.RequestToPod,
+    onDelete: 'cascade'
+  })
+}
+
+function countAll (callback) {
+  return this.count().asCallback(callback)
+}
+
+function incrementScores (ids, value, callback) {
+  if (!callback) callback = function () {}
+
+  const update = {
+    score: this.sequelize.literal('score +' + value)
+  }
+
+  const options = {
+    where: {
+      id: {
+        $in: ids
+      }
+    },
+    // In this case score is a literal and not an integer so we do not validate it
+    validate: false
+  }
+
+  return this.update(update, options).asCallback(callback)
+}
+
+function list (callback) {
+  return this.findAll().asCallback(callback)
+}
+
+function listAllIds (transaction, callback) {
+  if (!callback) {
+    callback = transaction
+    transaction = null
+  }
+
+  const query = {
+    attributes: [ 'id' ]
+  }
+
+  if (transaction) query.transaction = transaction
+
+  return this.findAll(query).asCallback(function (err, pods) {
+    if (err) return callback(err)
+
+    return callback(null, map(pods, 'id'))
+  })
+}
+
+function listRandomPodIdsWithRequest (limit, callback) {
+  const self = this
+
+  self.count().asCallback(function (err, count) {
+    if (err) return callback(err)
+
+    // Optimization...
+    if (count === 0) return callback(null, [])
+
+    let start = Math.floor(Math.random() * count) - limit
+    if (start < 0) start = 0
+
+    const query = {
+      attributes: [ 'id' ],
+      order: [
+        [ 'id', 'ASC' ]
+      ],
+      offset: start,
+      limit: limit,
+      where: {
+        id: {
+          $in: [
+            this.sequelize.literal('SELECT "podId" FROM "RequestToPods"')
+          ]
+        }
+      }
+    }
+
+    return this.findAll(query).asCallback(function (err, pods) {
+      if (err) return callback(err)
+
+      return callback(null, map(pods, 'id'))
+    })
+  })
+}
+
+function listBadPods (callback) {
+  const query = {
+    where: {
+      score: { $lte: 0 }
+    }
+  }
+
+  return this.findAll(query).asCallback(callback)
+}
+
+function load (id, callback) {
+  return this.findById(id).asCallback(callback)
+}
+
+function loadByHost (host, callback) {
+  const query = {
+    where: {
+      host: host
+    }
+  }
+
+  return this.findOne(query).asCallback(callback)
+}
+
+function removeAll (callback) {
+  return this.destroy().asCallback(callback)
+}

server/models/pods.js

@@ -1,119 +0,0 @@
-'use strict'
-
-const each = require('async/each')
-const mongoose = require('mongoose')
-const map = require('lodash/map')
-const validator = require('express-validator').validator
-
-const constants = require('../initializers/constants')
-
-const Video = mongoose.model('Video')
-
-// ---------------------------------------------------------------------------
-
-const PodSchema = mongoose.Schema({
-  host: String,
-  publicKey: String,
-  score: { type: Number, max: constants.FRIEND_SCORE.MAX },
-  createdDate: {
-    type: Date,
-    default: Date.now
-  }
-})
-
-PodSchema.path('host').validate(validator.isURL)
-PodSchema.path('publicKey').required(true)
-PodSchema.path('score').validate(function (value) { return !isNaN(value) })
-
-PodSchema.methods = {
-  toFormatedJSON
-}
-
-PodSchema.statics = {
-  countAll,
-  incrementScores,
-  list,
-  listAllIds,
-  listBadPods,
-  load,
-  loadByHost,
-  removeAll
-}
-
-PodSchema.pre('save', function (next) {
-  const self = this
-
-  Pod.loadByHost(this.host, function (err, pod) {
-    if (err) return next(err)
-
-    if (pod) return next(new Error('Pod already exists.'))
-
-    self.score = constants.FRIEND_SCORE.BASE
-    return next()
-  })
-})
-
-PodSchema.pre('remove', function (next) {
-  // Remove the videos owned by this pod too
-  Video.listByHost(this.host, function (err, videos) {
-    if (err) return next(err)
-
-    each(videos, function (video, callbackEach) {
-      video.remove(callbackEach)
-    }, next)
-  })
-})
-
-const Pod = mongoose.model('Pod', PodSchema)
-
-// ------------------------------ METHODS ------------------------------
-
-function toFormatedJSON () {
-  const json = {
-    id: this._id,
-    host: this.host,
-    score: this.score,
-    createdDate: this.createdDate
-  }
-
-  return json
-}
-
-// ------------------------------ Statics ------------------------------
-
-function countAll (callback) {
-  return this.count(callback)
-}
-
-function incrementScores (ids, value, callback) {
-  if (!callback) callback = function () {}
-  return this.update({ _id: { $in: ids } }, { $inc: { score: value } }, { multi: true }, callback)
-}
-
-function list (callback) {
-  return this.find(callback)
-}
-
-function listAllIds (callback) {
-  return this.find({}, { _id: 1 }, function (err, pods) {
-    if (err) return callback(err)
-
-    return callback(null, map(pods, '_id'))
-  })
-}
-
-function listBadPods (callback) {
-  return this.find({ score: 0 }, callback)
-}
-
-function load (id, callback) {
-  return this.findById(id, callback)
-}
-
-function loadByHost (host, callback) {
-  return this.findOne({ host }, callback)
-}
-
-function removeAll (callback) {
-  return this.remove({}, callback)
-}

server/models/request-to-pod.js

@@ -0,0 +1,42 @@
+'use strict'
+
+// ---------------------------------------------------------------------------
+
+module.exports = function (sequelize, DataTypes) {
+  const RequestToPod = sequelize.define('RequestToPod', {}, {
+    indexes: [
+      {
+        fields: [ 'requestId' ]
+      },
+      {
+        fields: [ 'podId' ]
+      },
+      {
+        fields: [ 'requestId', 'podId' ],
+        unique: true
+      }
+    ],
+    classMethods: {
+      removePodOf
+    }
+  })
+
+  return RequestToPod
+}
+
+// ---------------------------------------------------------------------------
+
+function removePodOf (requestsIds, podId, callback) {
+  if (!callback) callback = function () {}
+
+  const query = {
+    where: {
+      requestId: {
+        $in: requestsIds
+      },
+      podId: podId
+    }
+  }
+
+  this.destroy(query).asCallback(callback)
+}

server/models/request.js

@@ -2,66 +2,60 @@
 
 const each = require('async/each')
 const eachLimit = require('async/eachLimit')
-const values = require('lodash/values')
-const mongoose = require('mongoose')
 const waterfall = require('async/waterfall')
+const values = require('lodash/values')
 
 const constants = require('../initializers/constants')
 const logger = require('../helpers/logger')
 const requests = require('../helpers/requests')
 
-const Pod = mongoose.model('Pod')
-
 let timer = null
 let lastRequestTimestamp = 0
 
 // ---------------------------------------------------------------------------
 
-const RequestSchema = mongoose.Schema({
+module.exports = function (sequelize, DataTypes) {
-  request: mongoose.Schema.Types.Mixed,
+  const Request = sequelize.define('Request',
-  endpoint: {
-    type: String,
-    enum: [ values(constants.REQUEST_ENDPOINTS) ]
-  },
-  to: [
     {
-      type: mongoose.Schema.Types.ObjectId,
+      request: {
-      ref: 'Pod'
+        type: DataTypes.JSON,
-    }
+        allowNull: false
-  ]
+      },
-})
+      endpoint: {
+        type: DataTypes.ENUM(values(constants.REQUEST_ENDPOINTS)),
+        allowNull: false
+      }
+    },
+    {
+      classMethods: {
+        associate,
 
-RequestSchema.statics = {
+        activate,
-  activate,
+        countTotalRequests,
-  deactivate,
+        deactivate,
-  flush,
+        flush,
-  forceSend,
+        forceSend,
-  list,
+        remainingMilliSeconds
-  remainingMilliSeconds
+      }
+    }
+  )
+
+  return Request
 }
 
-RequestSchema.pre('save', function (next) {
-  const self = this
-
-  if (self.to.length === 0) {
-    Pod.listAllIds(function (err, podIds) {
-      if (err) return next(err)
-
-      // No friends
-      if (podIds.length === 0) return
-
-      self.to = podIds
-      return next()
-    })
-  } else {
-    return next()
-  }
-})
-
-mongoose.model('Request', RequestSchema)
-
 // ------------------------------ STATICS ------------------------------
 
+function associate (models) {
+  this.belongsToMany(models.Pod, {
+    foreignKey: {
+      name: 'requestId',
+      allowNull: false
+    },
+    through: models.RequestToPod,
+    onDelete: 'CASCADE'
+  })
+}
+
 function activate () {
   logger.info('Requests scheduler activated.')
   lastRequestTimestamp = Date.now()
@@ -73,15 +67,25 @@ function activate () {
   }, constants.REQUESTS_INTERVAL)
 }
 
+function countTotalRequests (callback) {
+  const query = {
+    include: [ this.sequelize.models.Pod ]
+  }
+
+  return this.count(query).asCallback(callback)
+}
+
 function deactivate () {
   logger.info('Requests scheduler deactivated.')
   clearInterval(timer)
   timer = null
 }
 
-function flush () {
+function flush (callback) {
   removeAll.call(this, function (err) {
     if (err) logger.error('Cannot flush the requests.', { error: err })
+
+    return callback(err)
   })
 }
 
@@ -90,10 +94,6 @@ function forceSend () {
   makeRequests.call(this)
 }
 
-function list (callback) {
-  this.find({ }, callback)
-}
-
 function remainingMilliSeconds () {
   if (timer === null) return -1
 
@@ -122,7 +122,7 @@ function makeRequest (toPod, requestEndpoint, requestsToMake, callback) {
         'Error sending secure request to %s pod.',
         toPod.host,
         {
-          error: err || new Error('Status code not 20x : ' + res.statusCode)
+          error: err ? err.message : 'Status code not 20x : ' + res.statusCode
         }
       )
 
@@ -136,10 +136,11 @@ function makeRequest (toPod, requestEndpoint, requestsToMake, callback) {
 // Make all the requests of the scheduler
 function makeRequests () {
   const self = this
+  const RequestToPod = this.sequelize.models.RequestToPod
 
-  // We limit the size of the requests (REQUESTS_LIMIT)
+  // We limit the size of the requests
   // We don't want to stuck with the same failing requests so we get a random list
-  listWithLimitAndRandom.call(self, constants.REQUESTS_LIMIT, function (err, requests) {
+  listWithLimitAndRandom.call(self, constants.REQUESTS_LIMIT_PODS, constants.REQUESTS_LIMIT_PER_POD, function (err, requests) {
     if (err) {
       logger.error('Cannot get the list of requests.', { err: err })
       return // Abort
@@ -151,78 +152,77 @@ function makeRequests () {
       return
     }
 
-    logger.info('Making requests to friends.')
-
     // We want to group requests by destinations pod and endpoint
     const requestsToMakeGrouped = {}
+    Object.keys(requests).forEach(function (toPodId) {
+      requests[toPodId].forEach(function (data) {
+        const request = data.request
+        const pod = data.pod
+        const hashKey = toPodId + request.endpoint
 
-    requests.forEach(function (poolRequest) {
-      poolRequest.to.forEach(function (toPodId) {
-        const hashKey = toPodId + poolRequest.endpoint
         if (!requestsToMakeGrouped[hashKey]) {
           requestsToMakeGrouped[hashKey] = {
-            toPodId,
+            toPod: pod,
-            endpoint: poolRequest.endpoint,
+            endpoint: request.endpoint,
-            ids: [], // pool request ids, to delete them from the DB in the future
+            ids: [], // request ids, to delete them from the DB in the future
             datas: [] // requests data,
           }
         }
 
-        requestsToMakeGrouped[hashKey].ids.push(poolRequest._id)
+        requestsToMakeGrouped[hashKey].ids.push(request.id)
-        requestsToMakeGrouped[hashKey].datas.push(poolRequest.request)
+        requestsToMakeGrouped[hashKey].datas.push(request.request)
       })
     })
 
+    logger.info('Making requests to friends.')
+
     const goodPods = []
     const badPods = []
 
     eachLimit(Object.keys(requestsToMakeGrouped), constants.REQUESTS_IN_PARALLEL, function (hashKey, callbackEach) {
       const requestToMake = requestsToMakeGrouped[hashKey]
+      const toPod = requestToMake.toPod
 
-      // FIXME: mongodb request inside a loop :/
+      // Maybe the pod is not our friend anymore so simply remove it
-      Pod.load(requestToMake.toPodId, function (err, toPod) {
+      if (!toPod) {
-        if (err) {
+        const requestIdsToDelete = requestToMake.ids
-          logger.error('Error finding pod by id.', { err: err })
+
-          return callbackEach()
+        logger.info('Removing %d requests of unexisting pod %s.', requestIdsToDelete.length, requestToMake.toPod.id)
+        RequestToPod.removePodOf.call(self, requestIdsToDelete, requestToMake.toPod.id)
+        return callbackEach()
+      }
+
+      makeRequest(toPod, requestToMake.endpoint, requestToMake.datas, function (success) {
+        if (success === true) {
+          logger.debug('Removing requests for pod %s.', requestToMake.toPod.id, { requestsIds: requestToMake.ids })
+
+          goodPods.push(requestToMake.toPod.id)
+
+          // Remove the pod id of these request ids
+          RequestToPod.removePodOf(requestToMake.ids, requestToMake.toPod.id, callbackEach)
+        } else {
+          badPods.push(requestToMake.toPod.id)
+          callbackEach()
         }
-
-        // Maybe the pod is not our friend anymore so simply remove it
-        if (!toPod) {
-          const requestIdsToDelete = requestToMake.ids
-
-          logger.info('Removing %d requests of unexisting pod %s.', requestIdsToDelete.length, requestToMake.toPodId)
-          removePodOf.call(self, requestIdsToDelete, requestToMake.toPodId)
-          return callbackEach()
-        }
-
-        makeRequest(toPod, requestToMake.endpoint, requestToMake.datas, function (success) {
-          if (success === true) {
-            logger.debug('Removing requests for %s pod.', requestToMake.toPodId, { requestsIds: requestToMake.ids })
-
-            goodPods.push(requestToMake.toPodId)
-
-            // Remove the pod id of these request ids
-            removePodOf.call(self, requestToMake.ids, requestToMake.toPodId, callbackEach)
-          } else {
-            badPods.push(requestToMake.toPodId)
-            callbackEach()
-          }
-        })
       })
     }, function () {
       // All the requests were made, we update the pods score
-      updatePodsScore(goodPods, badPods)
+      updatePodsScore.call(self, goodPods, badPods)
       // Flush requests with no pod
-      removeWithEmptyTo.call(self)
+      removeWithEmptyTo.call(self, function (err) {
+        if (err) logger.error('Error when removing requests with no pods.', { error: err })
+      })
     })
   })
 }
 
 // Remove pods with a score of 0 (too many requests where they were unreachable)
 function removeBadPods () {
+  const self = this
+
   waterfall([
     function findBadPods (callback) {
-      Pod.listBadPods(function (err, pods) {
+      self.sequelize.models.Pod.listBadPods(function (err, pods) {
         if (err) {
           logger.error('Cannot find bad pods.', { error: err })
           return callback(err)
@@ -233,10 +233,8 @@ function removeBadPods () {
     },
 
     function removeTheseBadPods (pods, callback) {
-      if (pods.length === 0) return callback(null, 0)
-
       each(pods, function (pod, callbackEach) {
-        pod.remove(callbackEach)
+        pod.destroy().asCallback(callbackEach)
       }, function (err) {
         return callback(err, pods.length)
       })
@@ -253,43 +251,98 @@ function removeBadPods () {
 }
 
 function updatePodsScore (goodPods, badPods) {
+  const self = this
+  const Pod = this.sequelize.models.Pod
+
   logger.info('Updating %d good pods and %d bad pods scores.', goodPods.length, badPods.length)
 
-  Pod.incrementScores(goodPods, constants.PODS_SCORE.BONUS, function (err) {
+  if (goodPods.length !== 0) {
-    if (err) logger.error('Cannot increment scores of good pods.')
+    Pod.incrementScores(goodPods, constants.PODS_SCORE.BONUS, function (err) {
-  })
+      if (err) logger.error('Cannot increment scores of good pods.', { error: err })
+    })
+  }
 
-  Pod.incrementScores(badPods, constants.PODS_SCORE.MALUS, function (err) {
+  if (badPods.length !== 0) {
-    if (err) logger.error('Cannot decrement scores of bad pods.')
+    Pod.incrementScores(badPods, constants.PODS_SCORE.MALUS, function (err) {
-    removeBadPods()
+      if (err) logger.error('Cannot decrement scores of bad pods.', { error: err })
+      removeBadPods.call(self)
+    })
+  }
+}
+
+function listWithLimitAndRandom (limitPods, limitRequestsPerPod, callback) {
+  const self = this
+  const Pod = this.sequelize.models.Pod
+
+  Pod.listRandomPodIdsWithRequest(limitPods, function (err, podIds) {
+    if (err) return callback(err)
+
+    // We don't have friends that have requests
+    if (podIds.length === 0) return callback(null, [])
+
+    // The the first x requests of these pods
+    // It is very important to sort by id ASC to keep the requests order!
+    const query = {
+      order: [
+        [ 'id', 'ASC' ]
+      ],
+      include: [
+        {
+          model: self.sequelize.models.Pod,
+          where: {
+            id: {
+              $in: podIds
+            }
+          }
+        }
+      ]
+    }
+
+    self.findAll(query).asCallback(function (err, requests) {
+      if (err) return callback(err)
+
+      const requestsGrouped = groupAndTruncateRequests(requests, limitRequestsPerPod)
+      return callback(err, requestsGrouped)
+    })
   })
 }
 
-function listWithLimitAndRandom (limit, callback) {
+function groupAndTruncateRequests (requests, limitRequestsPerPod) {
-  const self = this
+  const requestsGrouped = {}
 
-  self.count(function (err, count) {
+  requests.forEach(function (request) {
-    if (err) return callback(err)
+    request.Pods.forEach(function (pod) {
+      if (!requestsGrouped[pod.id]) requestsGrouped[pod.id] = []
 
-    let start = Math.floor(Math.random() * count) - limit
+      if (requestsGrouped[pod.id].length < limitRequestsPerPod) {
-    if (start < 0) start = 0
+        requestsGrouped[pod.id].push({
-
+          request,
-    self.find().sort({ _id: 1 }).skip(start).limit(limit).exec(callback)
+          pod
+        })
+      }
+    })
   })
+
+  return requestsGrouped
 }
 
 function removeAll (callback) {
-  this.remove({ }, callback)
+  // Delete all requests
-}
+  this.truncate({ cascade: true }).asCallback(callback)
-
-function removePodOf (requestsIds, podId, callback) {
-  if (!callback) callback = function () {}
-
-  this.update({ _id: { $in: requestsIds } }, { $pull: { to: podId } }, { multi: true }, callback)
 }
 
 function removeWithEmptyTo (callback) {
   if (!callback) callback = function () {}
 
-  this.remove({ to: { $size: 0 } }, callback)
+  const query = {
+    where: {
+      id: {
+        $notIn: [
+          this.sequelize.literal('SELECT "requestId" FROM "RequestToPods"')
+        ]
+      }
+    }
+  }
+
+  this.destroy(query).asCallback(callback)
 }

server/models/tag.js

@@ -0,0 +1,76 @@
+'use strict'
+
+const each = require('async/each')
+
+// ---------------------------------------------------------------------------
+
+module.exports = function (sequelize, DataTypes) {
+  const Tag = sequelize.define('Tag',
+    {
+      name: {
+        type: DataTypes.STRING,
+        allowNull: false
+      }
+    },
+    {
+      timestamps: false,
+      indexes: [
+        {
+          fields: [ 'name' ],
+          unique: true
+        }
+      ],
+      classMethods: {
+        associate,
+
+        findOrCreateTags
+      }
+    }
+  )
+
+  return Tag
+}
+
+// ---------------------------------------------------------------------------
+
+function associate (models) {
+  this.belongsToMany(models.Video, {
+    foreignKey: 'tagId',
+    through: models.VideoTag,
+    onDelete: 'cascade'
+  })
+}
+
+function findOrCreateTags (tags, transaction, callback) {
+  if (!callback) {
+    callback = transaction
+    transaction = null
+  }
+
+  const self = this
+  const tagInstances = []
+
+  each(tags, function (tag, callbackEach) {
+    const query = {
+      where: {
+        name: tag
+      },
+      defaults: {
+        name: tag
+      }
+    }
+
+    if (transaction) query.transaction = transaction
+
+    self.findOrCreate(query).asCallback(function (err, res) {
+      if (err) return callbackEach(err)
+
+      // res = [ tag, isCreated ]
+      const tag = res[0]
+      tagInstances.push(tag)
+      return callbackEach()
+    })
+  }, function (err) {
+    return callback(err, tagInstances)
+  })
+}

server/models/user.js

@@ -1,60 +1,81 @@
-const mongoose = require('mongoose')
+'use strict'
+
+const values = require('lodash/values')
 
-const customUsersValidators = require('../helpers/custom-validators').users
 const modelUtils = require('./utils')
+const constants = require('../initializers/constants')
 const peertubeCrypto = require('../helpers/peertube-crypto')
-
+const customUsersValidators = require('../helpers/custom-validators').users
-const OAuthToken = mongoose.model('OAuthToken')
 
 // ---------------------------------------------------------------------------
 
-const UserSchema = mongoose.Schema({
+module.exports = function (sequelize, DataTypes) {
-  createdDate: {
+  const User = sequelize.define('User',
-    type: Date,
+    {
-    default: Date.now
+      password: {
-  },
+        type: DataTypes.STRING,
-  password: String,
+        allowNull: false,
-  username: String,
+        validate: {
-  role: String
+          passwordValid: function (value) {
-})
+            const res = customUsersValidators.isUserPasswordValid(value)
+            if (res === false) throw new Error('Password not valid.')
+          }
+        }
+      },
+      username: {
+        type: DataTypes.STRING,
+        allowNull: false,
+        validate: {
+          usernameValid: function (value) {
+            const res = customUsersValidators.isUserUsernameValid(value)
+            if (res === false) throw new Error('Username not valid.')
+          }
+        }
+      },
+      role: {
+        type: DataTypes.ENUM(values(constants.USER_ROLES)),
+        allowNull: false
+      }
+    },
+    {
+      indexes: [
+        {
+          fields: [ 'username' ]
+        }
+      ],
+      classMethods: {
+        associate,
 
-UserSchema.path('password').required(customUsersValidators.isUserPasswordValid)
+        countTotal,
-UserSchema.path('username').required(customUsersValidators.isUserUsernameValid)
+        getByUsername,
-UserSchema.path('role').validate(customUsersValidators.isUserRoleValid)
+        list,
+        listForApi,
+        loadById,
+        loadByUsername
+      },
+      instanceMethods: {
+        isPasswordMatch,
+        toFormatedJSON
+      },
+      hooks: {
+        beforeCreate: beforeCreateOrUpdate,
+        beforeUpdate: beforeCreateOrUpdate
+      }
+    }
+  )
 
-UserSchema.methods = {
+  return User
-  isPasswordMatch,
-  toFormatedJSON
 }
 
-UserSchema.statics = {
+function beforeCreateOrUpdate (user, options, next) {
-  countTotal,
+  peertubeCrypto.cryptPassword(user.password, function (err, hash) {
-  getByUsername,
-  list,
-  listForApi,
-  loadById,
-  loadByUsername
-}
-
-UserSchema.pre('save', function (next) {
-  const user = this
-
-  peertubeCrypto.cryptPassword(this.password, function (err, hash) {
     if (err) return next(err)
 
     user.password = hash
 
     return next()
   })
-})
+}
-
-UserSchema.pre('remove', function (next) {
-  const user = this
-
-  OAuthToken.removeByUserId(user._id, next)
-})
-
-mongoose.model('User', UserSchema)
 
 // ------------------------------ METHODS ------------------------------
 
@@ -64,35 +85,68 @@ function isPasswordMatch (password, callback) {
 
 function toFormatedJSON () {
   return {
-    id: this._id,
+    id: this.id,
     username: this.username,
     role: this.role,
-    createdDate: this.createdDate
+    createdAt: this.createdAt
   }
 }
 // ------------------------------ STATICS ------------------------------
 
+function associate (models) {
+  this.hasOne(models.Author, {
+    foreignKey: 'userId',
+    onDelete: 'cascade'
+  })
+
+  this.hasMany(models.OAuthToken, {
+    foreignKey: 'userId',
+    onDelete: 'cascade'
+  })
+}
+
 function countTotal (callback) {
-  return this.count(callback)
+  return this.count().asCallback(callback)
 }
 
 function getByUsername (username) {
-  return this.findOne({ username: username })
+  const query = {
+    where: {
+      username: username
+    }
+  }
+
+  return this.findOne(query)
 }
 
 function list (callback) {
-  return this.find(callback)
+  return this.find().asCallback(callback)
 }
 
 function listForApi (start, count, sort, callback) {
-  const query = {}
+  const query = {
-  return modelUtils.listForApiWithCount.call(this, query, start, count, sort, callback)
+    offset: start,
+    limit: count,
+    order: [ modelUtils.getSort(sort) ]
+  }
+
+  return this.findAndCountAll(query).asCallback(function (err, result) {
+    if (err) return callback(err)
+
+    return callback(null, result.rows, result.count)
+  })
 }
 
 function loadById (id, callback) {
-  return this.findById(id, callback)
+  return this.findById(id).asCallback(callback)
 }
 
 function loadByUsername (username, callback) {
-  return this.findOne({ username: username }, callback)
+  const query = {
+    where: {
+      username: username
+    }
+  }
+
+  return this.findOne(query).asCallback(callback)
 }