Add migration (for db, folders...) mechanism

server.js

@@ -32,6 +32,7 @@ if (miss.length !== 0) {
 // ----------- PeerTube modules -----------
 const customValidators = require('./server/helpers/custom-validators')
 const installer = require('./server/initializers/installer')
+const migrator = require('./server/initializers/migrator')
 const mongoose = require('mongoose')
 const routes = require('./server/controllers')
 const utils = require('./server/helpers/utils')
@@ -127,6 +128,10 @@ app.use(function (err, req, res, next) {
 installer.installApplication(function (err) {
   if (err) throw err
 
+  // Run the migration scripts if needed
+  migrator.migrate(function (err) {
+    if (err) throw err
+
     // Create/activate the webtorrent module
     webtorrent.create(function () {
       function cleanForExit () {
@@ -155,6 +160,7 @@ installer.installApplication(function (err) {
         })
       })
     })
+  })
 })
 
 module.exports = app

server/initializers/constants.js

@@ -54,6 +54,18 @@ const FRIEND_SCORE = {
   MAX: 1000
 }
 
+const MONGO_MIGRATION_SCRIPTS = [
+  {
+    script: '0005-create-application',
+    version: 5
+  },
+  {
+    script: '0010-users-password',
+    version: 10
+  }
+]
+const LAST_MONGO_SCHEMA_VERSION = 10
+
 // Time to wait between requests to the friends (10 min)
 let REQUESTS_INTERVAL = 600000
 
@@ -121,11 +133,13 @@ module.exports = {
   CONFIG: CONFIG,
   CONSTRAINTS_FIELDS: CONSTRAINTS_FIELDS,
   FRIEND_SCORE: FRIEND_SCORE,
-  REQUESTS_INTERVAL: REQUESTS_INTERVAL,
+  LAST_MONGO_SCHEMA_VERSION: LAST_MONGO_SCHEMA_VERSION,
+  MONGO_MIGRATION_SCRIPTS: MONGO_MIGRATION_SCRIPTS,
   OAUTH_LIFETIME: OAUTH_LIFETIME,
   PAGINATION_COUNT_DEFAULT: PAGINATION_COUNT_DEFAULT,
   PODS_SCORE: PODS_SCORE,
   REQUESTS_IN_PARALLEL: REQUESTS_IN_PARALLEL,
+  REQUESTS_INTERVAL: REQUESTS_INTERVAL,
   REQUESTS_LIMIT: REQUESTS_LIMIT,
   RETRY_REQUESTS: RETRY_REQUESTS,
   SEARCHABLE_COLUMNS: SEARCHABLE_COLUMNS,

server/initializers/database.js

@@ -6,6 +6,7 @@ const constants = require('../initializers/constants')
 const logger = require('../helpers/logger')
 
 // Bootstrap models
+require('../models/application')
 require('../models/user')
 require('../models/oauth-client')
 require('../models/oauth-token')

server/initializers/migrations/0005-create-application.js

@@ -0,0 +1,17 @@
+/*
+  Create the application collection in MongoDB.
+  Used to store the actual MongoDB scheme version.
+*/
+
+const mongoose = require('mongoose')
+
+const Application = mongoose.model('Application')
+
+exports.up = function (callback) {
+  const application = new Application()
+  application.save(callback)
+}
+
+exports.down = function (callback) {
+  throw new Error('Not implemented.')
+}

server/initializers/migrations/0010-users-password.js

@@ -0,0 +1,23 @@
+/*
+  Convert plain user password to encrypted user password.
+*/
+
+const mongoose = require('mongoose')
+
+const User = mongoose.model('User')
+
+exports.up = function (callback) {
+  User.list(function (err, users) {
+    if (err) return callback(err)
+
+    users.forEach(function (user) {
+      user.save()
+    })
+
+    return callback(null)
+  })
+}
+
+exports.down = function (callback) {
+  throw new Error('Not implemented.')
+}

server/initializers/migrator.js

@@ -0,0 +1,56 @@
+'use strict'
+
+const eachSeries = require('async/eachSeries')
+const mongoose = require('mongoose')
+const path = require('path')
+
+const constants = require('./constants')
+const logger = require('../helpers/logger')
+
+const Application = mongoose.model('Application')
+
+const migrator = {
+  migrate: migrate
+}
+
+function migrate (callback) {
+  Application.loadMongoSchemaVersion(function (err, actualVersion) {
+    if (err) return callback(err)
+
+    // If there are a new mongo schemas
+    if (!actualVersion || actualVersion < constants.LAST_MONGO_SCHEMA_VERSION) {
+      logger.info('Begin migrations.')
+
+      eachSeries(constants.MONGO_MIGRATION_SCRIPTS, function (entity, callbackEach) {
+        const versionScript = entity.version
+
+        // Do not execute old migration scripts
+        if (versionScript <= actualVersion) return callbackEach(null)
+
+        // Load the migration module and run it
+        const migrationScriptName = entity.script
+        logger.info('Executing %s migration script.', migrationScriptName)
+
+        const migrationScript = require(path.join(__dirname, 'migrations', migrationScriptName))
+        migrationScript.up(function (err) {
+          if (err) return callbackEach(err)
+
+          // Update the new mongo version schema
+          Application.updateMongoSchemaVersion(versionScript, callbackEach)
+        })
+      }, function (err) {
+        if (err) return callback(err)
+
+        logger.info('Migrations finished. New mongo version schema: %s', constants.LAST_MONGO_SCHEMA_VERSION)
+        return callback(null)
+      })
+    } else {
+      return callback(null)
+    }
+  })
+}
+
+// ---------------------------------------------------------------------------
+
+module.exports = migrator
+

server/models/application.js

@@ -0,0 +1,31 @@
+const mongoose = require('mongoose')
+
+// ---------------------------------------------------------------------------
+
+const ApplicationSchema = mongoose.Schema({
+  mongoSchemaVersion: {
+    type: Number,
+    default: 0
+  }
+})
+
+ApplicationSchema.statics = {
+  loadMongoSchemaVersion: loadMongoSchemaVersion,
+  updateMongoSchemaVersion: updateMongoSchemaVersion
+}
+
+mongoose.model('Application', ApplicationSchema)
+
+// ---------------------------------------------------------------------------
+
+function loadMongoSchemaVersion (callback) {
+  return this.findOne({}, { mongoSchemaVersion: 1 }, function (err, data) {
+    const version = data ? data.mongoSchemaVersion : 0
+
+    return callback(err, version)
+  })
+}
+
+function updateMongoSchemaVersion (newVersion, callback) {
+  return this.update({}, { mongoSchemaVersion: newVersion }, callback)
+}

server/models/user.js

@@ -28,6 +28,7 @@ UserSchema.methods = {
 UserSchema.statics = {
   countTotal: countTotal,
   getByUsername: getByUsername,
+  list: list,
   listForApi: listForApi,
   loadById: loadById,
   loadByUsername: loadByUsername
@@ -71,6 +72,10 @@ function getByUsername (username) {
   return this.findOne({ username: username })
 }
 
+function list (callback) {
+  return this.find(callback)
+}
+
 function listForApi (start, count, sort, callback) {
   const query = {}
   return modelUtils.listForApiWithCount.call(this, query, start, count, sort, callback)