diff --git a/.gitignore b/.gitignore index 18a2e808d..7ca89dca8 100644 --- a/.gitignore +++ b/.gitignore @@ -14,7 +14,11 @@ uploads thumbnails config/production.yaml ffmpeg +<<<<<<< HEAD +torrents +======= .tags *.sublime-project *.sublime-workspace torrents/ +>>>>>>> master diff --git a/client/src/app/shared/auth/auth.service.ts b/client/src/app/shared/auth/auth.service.ts index c57486d9a..a30c79c86 100644 --- a/client/src/app/shared/auth/auth.service.ts +++ b/client/src/app/shared/auth/auth.service.ts @@ -123,6 +123,7 @@ export class AuthService { logout() { // TODO: make an HTTP request to revoke the tokens this.user = null; + AuthUser.flush(); this.setStatus(AuthStatus.LoggedOut); diff --git a/client/src/app/videos/video-watch/video-watch.component.ts b/client/src/app/videos/video-watch/video-watch.component.ts index ada20fc96..239e24c99 100644 --- a/client/src/app/videos/video-watch/video-watch.component.ts +++ b/client/src/app/videos/video-watch/video-watch.component.ts @@ -68,6 +68,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy { ngOnDestroy() { console.log('Removing video from webtorrent.'); clearInterval(this.torrentInfosInterval); + clearTimeout(this.errorTimer); this.webTorrentService.remove(this.video.magnetUri); this.sub.unsubscribe(); diff --git a/config/default.yaml b/config/default.yaml index d32d3e17b..b44be31b0 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -17,6 +17,7 @@ storage: uploads: 'uploads/' logs: 'logs/' thumbnails: 'thumbnails/' + torrents: 'torrents/' electron: debug: false diff --git a/config/test-1.yaml b/config/test-1.yaml index 0a8dd3937..a59566cc4 100644 --- a/config/test-1.yaml +++ b/config/test-1.yaml @@ -14,3 +14,4 @@ storage: uploads: 'test1/uploads/' logs: 'test1/logs/' thumbnails: 'test1/thumbnails/' + torrents: 'test1/torrents/' diff --git a/config/test-2.yaml b/config/test-2.yaml index 40f410559..1b937898f 100644 --- a/config/test-2.yaml +++ b/config/test-2.yaml @@ -14,3 +14,4 @@ storage: uploads: 'test2/uploads/' logs: 'test2/logs/' thumbnails: 'test2/thumbnails/' + torrents: 'test2/torrents/' diff --git a/config/test-3.yaml b/config/test-3.yaml index 87b335228..e522c13e7 100644 --- a/config/test-3.yaml +++ b/config/test-3.yaml @@ -14,3 +14,4 @@ storage: uploads: 'test3/uploads/' logs: 'test3/logs/' thumbnails: 'test3/thumbnails/' + torrents: 'test3/torrents/' diff --git a/config/test-4.yaml b/config/test-4.yaml index 22abc0a58..e30cd7978 100644 --- a/config/test-4.yaml +++ b/config/test-4.yaml @@ -14,3 +14,4 @@ storage: uploads: 'test4/uploads/' logs: 'test4/logs/' thumbnails: 'test4/thumbnails/' + torrents: 'test4/torrents/' diff --git a/config/test-5.yaml b/config/test-5.yaml index af5619994..3a54599f5 100644 --- a/config/test-5.yaml +++ b/config/test-5.yaml @@ -14,3 +14,4 @@ storage: uploads: 'test5/uploads/' logs: 'test5/logs/' thumbnails: 'test5/thumbnails/' + torrents: 'test5/torrents/' diff --git a/config/test-6.yaml b/config/test-6.yaml index 5b8bf306b..31608add2 100644 --- a/config/test-6.yaml +++ b/config/test-6.yaml @@ -14,3 +14,4 @@ storage: uploads: 'test6/uploads/' logs: 'test6/logs/' thumbnails: 'test6/thumbnails/' + torrents: 'test6/torrents/' diff --git a/package.json b/package.json index 9a844baaf..59c7a4332 100644 --- a/package.json +++ b/package.json @@ -40,27 +40,27 @@ "body-parser": "^1.12.4", "concurrently": "^2.0.0", "config": "^1.14.0", + "create-torrent": "^3.24.5", "debug": "^2.2.0", "dezalgo": "^1.0.3", - "electron-spawn": "https://github.com/Chocobozzz/electron-spawn", "express": "^4.12.4", "express-oauth-server": "https://github.com/Chocobozzz/express-oauth-server", "express-validator": "^2.11.0", "fluent-ffmpeg": "^2.1.0", "js-yaml": "^3.5.4", "lodash": "^4.11.1", + "magnet-uri": "^5.1.4", "mkdirp": "^0.5.1", "mongoose": "^4.0.5", "morgan": "^1.5.3", "multer": "^1.1.0", - "node-ipc": "8.9.0", "openssl-wrapper": "^0.3.4", + "parse-torrent": "^5.8.0", "password-generator": "^2.0.2", "request": "^2.57.0", "request-replay": "^1.0.2", "scripty": "^1.5.0", "ursa": "^0.9.1", - "webtorrent": "^0.96.0", "winston": "^2.1.1", "ws": "^1.1.1" }, diff --git a/server.js b/server.js index ca275ccec..5feb21476 100644 --- a/server.js +++ b/server.js @@ -35,10 +35,7 @@ const installer = require('./server/initializers/installer') const migrator = require('./server/initializers/migrator') const mongoose = require('mongoose') const routes = require('./server/controllers') -const utils = require('./server/helpers/utils') -const webtorrent = require('./server/lib/webtorrent') const Request = mongoose.model('Request') -const Video = mongoose.model('Video') // Get configurations const port = config.get('listen.port') @@ -76,9 +73,16 @@ app.use('/client/*', function (req, res, next) { res.sendStatus(404) }) +const torrentsPhysicalPath = path.join(__dirname, config.get('storage.torrents')) +app.use(constants.STATIC_PATHS.TORRENTS, express.static(torrentsPhysicalPath, { maxAge: 0 })) + +// Uploads path for webseeding +const uploadsPhysicalPath = path.join(__dirname, config.get('storage.uploads')) +app.use(constants.STATIC_PATHS.WEBSEED, express.static(uploadsPhysicalPath, { maxAge: 0 })) + // Thumbnails path for express const thumbnailsPhysicalPath = path.join(__dirname, config.get('storage.thumbnails')) -app.use(constants.THUMBNAILS_STATIC_PATH, express.static(thumbnailsPhysicalPath, { maxAge: 0 })) +app.use(constants.STATIC_PATHS.THUMBNAILS, express.static(thumbnailsPhysicalPath, { maxAge: 0 })) // Client application app.use('/*', function (req, res, next) { @@ -129,33 +133,14 @@ installer.installApplication(function (err) { migrator.migrate(function (err) { if (err) throw err - // Create/activate the webtorrent module - webtorrent.create(function () { - function cleanForExit () { - utils.cleanForExit(webtorrent.app) - } + // ----------- Make the server listening ----------- + server.listen(port, function () { + // Activate the pool requests + Request.activate() - function exitGracefullyOnSignal () { - process.exit(-1) - } - - process.on('exit', cleanForExit) - process.on('SIGINT', exitGracefullyOnSignal) - process.on('SIGTERM', exitGracefullyOnSignal) - - // ----------- Make the server listening ----------- - server.listen(port, function () { - // Activate the pool requests - Request.activate() - - Video.seedAllExisting(function (err) { - if (err) throw err - - logger.info('Seeded all the videos') - logger.info('Server listening on port %d', port) - app.emit('ready') - }) - }) + logger.info('Seeded all the videos') + logger.info('Server listening on port %d', port) + app.emit('ready') }) }) }) diff --git a/server/initializers/constants.js b/server/initializers/constants.js index b1d033377..be2e3e943 100644 --- a/server/initializers/constants.js +++ b/server/initializers/constants.js @@ -122,12 +122,16 @@ const RETRY_REQUESTS = 5 // Password encryption const BCRYPT_SALT_SIZE = 10 +// Express static paths (router) +const STATIC_PATHS = { + THUMBNAILS: '/static/thumbnails', + TORRENTS: '/static/torrents/', + WEBSEED: '/static/webseed/' +} + // Videos thumbnail size const THUMBNAILS_SIZE = '200x110' -// Path for access to thumbnails with express router -const THUMBNAILS_STATIC_PATH = '/static/thumbnails' - const USER_ROLES = { ADMIN: 'admin', USER: 'user' @@ -166,8 +170,8 @@ module.exports = { SEARCHABLE_COLUMNS, SEEDS_IN_PARALLEL, SORTABLE_COLUMNS, + STATIC_PATHS, THUMBNAILS_SIZE, - THUMBNAILS_STATIC_PATH, USER_ROLES } diff --git a/server/lib/webtorrent-process.js b/server/lib/webtorrent-process.js deleted file mode 100644 index be7ac5bb4..000000000 --- a/server/lib/webtorrent-process.js +++ /dev/null @@ -1,92 +0,0 @@ -'use strict' - -const WebTorrent = require('webtorrent') -const ipc = require('node-ipc') - -function webtorrent (args) { - if (args.length !== 3) { - throw new Error('Wrong arguments number: ' + args.length + '/3') - } - - const host = args[1] - const port = args[2] - const nodeKey = 'webtorrentnode' + port - const processKey = 'webtorrentprocess' + port - - ipc.config.silent = true - ipc.config.id = processKey - - if (host === 'client' && port === '1') global.WEBTORRENT_ANNOUNCE = [] - else global.WEBTORRENT_ANNOUNCE = 'ws://' + host + ':' + port + '/tracker/socket' - const wt = new WebTorrent({ dht: false }) - - function seed (data) { - const args = data.args - const path = args.path - const _id = data._id - - wt.seed(path, { announceList: '' }, function (torrent) { - const toSend = { - magnetUri: torrent.magnetURI - } - - ipc.of[nodeKey].emit(nodeKey + '.seedDone.' + _id, toSend) - }) - } - - function add (data) { - const args = data.args - const magnetUri = args.magnetUri - const _id = data._id - - wt.add(magnetUri, function (torrent) { - const toSend = { - files: [] - } - - torrent.files.forEach(function (file) { - toSend.files.push({ path: file.path }) - }) - - ipc.of[nodeKey].emit(nodeKey + '.addDone.' + _id, toSend) - }) - } - - function remove (data) { - const args = data.args - const magnetUri = args.magnetUri - const _id = data._id - - try { - wt.remove(magnetUri, callback) - } catch (err) { - console.log('Cannot remove the torrent from WebTorrent.') - return callback(null) - } - - function callback () { - const toSend = {} - ipc.of[nodeKey].emit(nodeKey + '.removeDone.' + _id, toSend) - } - } - - console.log('Configuration: ' + host + ':' + port) - console.log('Connecting to IPC...') - - ipc.connectTo(nodeKey, function () { - ipc.of[nodeKey].on(processKey + '.seed', seed) - ipc.of[nodeKey].on(processKey + '.add', add) - ipc.of[nodeKey].on(processKey + '.remove', remove) - - ipc.of[nodeKey].emit(processKey + '.ready') - console.log('Ready.') - }) - - process.on('uncaughtException', function (e) { - ipc.of[nodeKey].emit(processKey + '.exception', { exception: e.toString() }) - }) -} - -// --------------------------------------------------------------------------- - -module.exports = webtorrent diff --git a/server/lib/webtorrent.js b/server/lib/webtorrent.js deleted file mode 100644 index dde046828..000000000 --- a/server/lib/webtorrent.js +++ /dev/null @@ -1,160 +0,0 @@ -'use strict' - -const ipc = require('node-ipc') -const pathUtils = require('path') -const spawn = require('electron-spawn') - -const constants = require('../initializers/constants') -const logger = require('../helpers/logger') - -let host = constants.CONFIG.WEBSERVER.HOST -let port = constants.CONFIG.WEBSERVER.PORT -let nodeKey = 'webtorrentnode' + port -let processKey = 'webtorrentprocess' + port -ipc.config.silent = true -ipc.config.id = nodeKey - -const webtorrent = { - add: add, - app: null, // Pid of the app - create: create, - remove: remove, - seed: seed, - silent: false // Useful for beautiful tests -} - -function create (options, callback) { - if (typeof options === 'function') { - callback = options - options = {} - } - - // Override options - if (options.host) host = options.host - if (options.port) { - port = options.port - nodeKey = 'webtorrentnode' + port - processKey = 'webtorrentprocess' + port - ipc.config.id = nodeKey - } - - ipc.serve(function () { - if (!webtorrent.silent) logger.info('IPC server ready.') - - // Run a timeout of 30s after which we exit the process - const timeoutWebtorrentProcess = setTimeout(function () { - throw new Error('Timeout : cannot run the webtorrent process. Please ensure you have electron npm package installed with xvfb-run.') - }, 30000) - - ipc.server.on(processKey + '.ready', function () { - if (!webtorrent.silent) logger.info('Webtorrent process ready.') - clearTimeout(timeoutWebtorrentProcess) - callback() - }) - - ipc.server.on(processKey + '.exception', function (data) { - throw new Error('Received exception error from webtorrent process : ' + data.exception) - }) - - const webtorrentProcess = spawn(pathUtils.join(__dirname, 'webtorrent-process.js'), host, port, { detached: true }) - - if (constants.CONFIG.ELECTRON.DEBUG === true) { - webtorrentProcess.stderr.on('data', function (data) { - logger.debug('Webtorrent process stderr: ', data.toString()) - }) - - webtorrentProcess.stdout.on('data', function (data) { - logger.debug('Webtorrent process:', data.toString()) - }) - } - - webtorrent.app = webtorrentProcess - }) - - ipc.server.start() -} - -function seed (path, callback) { - const extension = pathUtils.extname(path) - const basename = pathUtils.basename(path, extension) - const data = { - _id: basename, - args: { - path: path - } - } - - if (!webtorrent.silent) logger.debug('Node wants to seed %s.', data._id) - - // Finish signal - const eventKey = nodeKey + '.seedDone.' + data._id - ipc.server.on(eventKey, function listener (received) { - if (!webtorrent.silent) logger.debug('Process seeded torrent %s.', received.magnetUri) - - // This is a fake object, we just use the magnetUri in this project - const torrent = { - magnetURI: received.magnetUri - } - - ipc.server.off(eventKey, '*') - callback(torrent) - }) - - ipc.server.broadcast(processKey + '.seed', data) -} - -function add (magnetUri, callback) { - const data = { - _id: magnetUri, - args: { - magnetUri: magnetUri - } - } - - if (!webtorrent.silent) logger.debug('Node wants to add ' + data._id) - - // Finish signal - const eventKey = nodeKey + '.addDone.' + data._id - ipc.server.on(eventKey, function (received) { - if (!webtorrent.silent) logger.debug('Process added torrent.') - - // This is a fake object, we just use the magnetUri in this project - const torrent = { - files: received.files - } - - ipc.server.off(eventKey, '*') - callback(torrent) - }) - - ipc.server.broadcast(processKey + '.add', data) -} - -function remove (magnetUri, callback) { - const data = { - _id: magnetUri, - args: { - magnetUri: magnetUri - } - } - - if (!webtorrent.silent) logger.debug('Node wants to stop seeding %s.', data._id) - - // Finish signal - const eventKey = nodeKey + '.removeDone.' + data._id - ipc.server.on(eventKey, function (received) { - if (!webtorrent.silent) logger.debug('Process removed torrent %s.', data._id) - - let err = null - if (received.err) err = received.err - - ipc.server.off(eventKey, '*') - callback(err) - }) - - ipc.server.broadcast(processKey + '.remove', data) -} - -// --------------------------------------------------------------------------- - -module.exports = webtorrent diff --git a/server/models/video.js b/server/models/video.js index b9999c8f6..7d073cffa 100644 --- a/server/models/video.js +++ b/server/models/video.js @@ -1,10 +1,13 @@ 'use strict' -const eachLimit = require('async/eachLimit') +const config = require('config') +const createTorrent = require('create-torrent') const ffmpeg = require('fluent-ffmpeg') const fs = require('fs') const parallel = require('async/parallel') +const parseTorrent = require('parse-torrent') const pathUtils = require('path') +const magnet = require('magnet-uri') const mongoose = require('mongoose') const constants = require('../initializers/constants') @@ -12,7 +15,14 @@ const customVideosValidators = require('../helpers/custom-validators').videos const logger = require('../helpers/logger') const modelUtils = require('./utils') const utils = require('../helpers/utils') -const webtorrent = require('../lib/webtorrent') + +const http = config.get('webserver.https') === true ? 'https' : 'http' +const host = config.get('webserver.host') +const port = config.get('webserver.port') +const uploadsDir = pathUtils.join(__dirname, '..', '..', config.get('storage.uploads')) +const thumbnailsDir = pathUtils.join(__dirname, '..', '..', config.get('storage.thumbnails')) +const torrentsDir = pathUtils.join(__dirname, '..', '..', config.get('storage.torrents')) +const webseedBaseUrl = http + '://' + host + ':' + port + constants.STATIC_PATHS.WEBSEED // --------------------------------------------------------------------------- @@ -61,8 +71,7 @@ VideoSchema.statics = { listOwnedByAuthor, listRemotes, load, - search, - seedAllExisting + search } VideoSchema.pre('remove', function (next) { @@ -98,8 +107,21 @@ VideoSchema.pre('save', function (next) { this.podUrl = constants.CONFIG.WEBSERVER.URL tasks.push( + // TODO: refractoring function (callback) { - seed(videoPath, callback) + createTorrent(videoPath, { announceList: [ [ 'ws://' + host + ':' + port + '/tracker/socket' ] ], urlList: [ webseedBaseUrl + video.filename ] }, function (err, torrent) { + if (err) return callback(err) + + fs.writeFile(torrentsDir + video.filename + '.torrent', torrent, function (err) { + if (err) return callback(err) + + const parsedTorrent = parseTorrent(torrent) + parsedTorrent.xs = video.podUrl + constants.STATIC_PATHS.TORRENTS + video.filename + '.torrent' + video.magnetUri = magnet.encode(parsedTorrent) + + callback(null) + }) + }) }, function (callback) { createThumbnail(videoPath, callback) @@ -109,7 +131,6 @@ VideoSchema.pre('save', function (next) { parallel(tasks, function (err, results) { if (err) return next(err) - video.magnetUri = results[0].magnetURI video.thumbnail = results[1] return next() @@ -144,7 +165,7 @@ function toFormatedJSON () { author: this.author, duration: this.duration, tags: this.tags, - thumbnailPath: constants.THUMBNAILS_STATIC_PATH + '/' + this.thumbnail, + thumbnailPath: constants.STATIC_PATHS.THUMBNAILS + '/' + this.thumbnail, createdDate: this.createdDate } @@ -230,17 +251,6 @@ function search (value, field, start, count, sort, callback) { modelUtils.listForApiWithCount.call(this, query, start, count, sort, callback) } -function seedAllExisting (callback) { - listOwned.call(this, function (err, videos) { - if (err) return callback(err) - - eachLimit(videos, constants.SEEDS_IN_PARALLEL, function (video, callbackEach) { - const videoPath = pathUtils.join(constants.CONFIG.STORAGE.UPLOAD_DIR, video.filename) - seed(videoPath, callbackEach) - }, callback) - }) -} - // --------------------------------------------------------------------------- function removeThumbnail (video, callback) { @@ -253,12 +263,7 @@ function removeFile (video, callback) { // Maybe the torrent is not seeded, but we catch the error to don't stop the removing process function removeTorrent (video, callback) { - try { - webtorrent.remove(video.magnetUri, callback) - } catch (err) { - logger.warn('Cannot remove the torrent from WebTorrent', { err: err }) - return callback(null) - } + fs.unlink(torrentsDir + video.filename + '.torrent') } function createThumbnail (videoPath, callback) { @@ -276,16 +281,6 @@ function createThumbnail (videoPath, callback) { }) } -function seed (path, callback) { - logger.info('Seeding %s...', path) - - webtorrent.seed(path, function (torrent) { - logger.info('%s seeded (%s).', path, torrent.magnetURI) - - return callback(null, torrent) - }) -} - function generateThumbnailFromBase64 (data, callback) { // Creating the thumbnail for this remote video utils.generateRandomString(16, function (err, randomString) {