From 62689b942b71cd1dd0d050c6ed05f884a0b325c2 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 27 Aug 2018 16:23:34 +0200 Subject: [PATCH] Correctly migrate to fs-extra --- package.json | 2 +- scripts/prune-storage.ts | 6 ++-- server/controllers/api/config.ts | 6 ++-- server/controllers/api/videos/import.ts | 6 ++-- server/controllers/api/videos/index.ts | 4 +-- server/helpers/captions-utils.ts | 7 ++-- server/helpers/core-utils.ts | 22 +------------ server/helpers/ffmpeg-utils.ts | 6 ++-- server/helpers/image-utils.ts | 4 +-- server/helpers/utils.ts | 5 +-- server/helpers/webtorrent.ts | 7 ++-- server/initializers/installer.ts | 10 +++--- .../migrations/0075-video-resolutions.ts | 9 +++-- server/initializers/migrator.ts | 4 +-- .../cache/abstract-video-static-file-cache.ts | 5 ++- server/lib/client-html.ts | 7 ++-- server/lib/job-queue/handlers/video-import.ts | 6 ++-- .../schedulers/youtube-dl-update-scheduler.ts | 6 ++-- server/models/avatar/avatar.ts | 4 +-- server/models/video/video-caption.ts | 4 +-- server/models/video/video.ts | 33 ++++++++----------- server/tests/feeds/feeds.ts | 1 - server/tests/utils/feeds/feeds.ts | 1 - server/tests/utils/miscs/miscs.ts | 4 +-- server/tests/utils/users/accounts.ts | 5 ++- server/tests/utils/videos/videos.ts | 8 ++--- server/tools/import-videos.ts | 8 ++--- server/tools/upload.ts | 5 +-- yarn.lock | 22 ++++++++++++- 29 files changed, 100 insertions(+), 117 deletions(-) diff --git a/package.json b/package.json index 635ceaf76..c452a9f52 100644 --- a/package.json +++ b/package.json @@ -125,7 +125,6 @@ "redis": "^2.8.0", "reflect-metadata": "^0.1.10", "request": "^2.81.0", - "rimraf": "^2.5.4", "safe-buffer": "^5.0.1", "scripty": "^1.5.0", "sequelize": "4.38.0", @@ -154,6 +153,7 @@ "@types/config": "^0.0.34", "@types/express": "^4.0.35", "@types/express-rate-limit": "^2.9.3", + "@types/fs-extra": "^5.0.4", "@types/libxmljs": "^0.18.0", "@types/lodash": "^4.14.64", "@types/magnet-uri": "^5.1.1", diff --git a/scripts/prune-storage.ts b/scripts/prune-storage.ts index bc59da6af..572283868 100755 --- a/scripts/prune-storage.ts +++ b/scripts/prune-storage.ts @@ -1,9 +1,9 @@ import * as prompt from 'prompt' import { join } from 'path' -import { readdirPromise, unlinkPromise } from '../server/helpers/core-utils' import { CONFIG } from '../server/initializers/constants' import { VideoModel } from '../server/models/video/video' import { initDatabaseModels } from '../server/initializers' +import { remove, readdir } from 'fs-extra' run() .then(() => process.exit(0)) @@ -39,7 +39,7 @@ async function run () { console.log('Processing delete...\n') for (const path of toDelete) { - await unlinkPromise(path) + await remove(path) } console.log('Done!') @@ -49,7 +49,7 @@ async function run () { } async function pruneDirectory (directory: string) { - const files = await readdirPromise(directory) + const files = await readdir(directory) const toDelete: string[] = [] for (const file of files) { diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts index e0539c414..b25f739bb 100644 --- a/server/controllers/api/config.ts +++ b/server/controllers/api/config.ts @@ -3,13 +3,13 @@ import { omit } from 'lodash' import { ServerConfig, UserRight } from '../../../shared' import { About } from '../../../shared/models/server/about.model' import { CustomConfig } from '../../../shared/models/server/custom-config.model' -import { unlinkPromise, writeFilePromise } from '../../helpers/core-utils' import { isSignupAllowed, isSignupAllowedForCurrentIP } from '../../helpers/signup' import { CONFIG, CONSTRAINTS_FIELDS, reloadConfig } from '../../initializers' import { asyncMiddleware, authenticate, ensureUserHasRight } from '../../middlewares' import { customConfigUpdateValidator } from '../../middlewares/validators/config' import { ClientHtml } from '../../lib/client-html' import { auditLoggerFactory, CustomConfigAuditView } from '../../helpers/audit-logger' +import { remove, writeJSON } from 'fs-extra' const packageJSON = require('../../../../package.json') const configRouter = express.Router() @@ -130,7 +130,7 @@ async function getCustomConfig (req: express.Request, res: express.Response, nex } async function deleteCustomConfig (req: express.Request, res: express.Response, next: express.NextFunction) { - await unlinkPromise(CONFIG.CUSTOM_FILE) + await remove(CONFIG.CUSTOM_FILE) auditLogger.delete( res.locals.oauth.token.User.Account.Actor.getIdentifier(), @@ -163,7 +163,7 @@ async function updateCustomConfig (req: express.Request, res: express.Response, toUpdateJSON.instance['short_description'] = toUpdate.instance.shortDescription toUpdateJSON.instance['default_nsfw_policy'] = toUpdate.instance.defaultNSFWPolicy - await writeFilePromise(CONFIG.CUSTOM_FILE, JSON.stringify(toUpdateJSON, undefined, 2)) + await writeJSON(CONFIG.CUSTOM_FILE, toUpdateJSON, { spaces: 2 }) reloadConfig() ClientHtml.invalidCache() diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts index b2f73fa48..44f15ef74 100644 --- a/server/controllers/api/videos/import.ts +++ b/server/controllers/api/videos/import.ts @@ -27,8 +27,8 @@ import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model' import { VideoChannelModel } from '../../../models/video/video-channel' import * as Bluebird from 'bluebird' import * as parseTorrent from 'parse-torrent' -import { readFileBufferPromise, renamePromise } from '../../../helpers/core-utils' import { getSecureTorrentName } from '../../../helpers/utils' +import { readFile, rename } from 'fs-extra' const auditLogger = auditLoggerFactory('video-imports') const videoImportsRouter = express.Router() @@ -78,10 +78,10 @@ async function addTorrentImport (req: express.Request, res: express.Response, to // Rename the torrent to a secured name const newTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, getSecureTorrentName(torrentName)) - await renamePromise(torrentfile.path, newTorrentPath) + await rename(torrentfile.path, newTorrentPath) torrentfile.path = newTorrentPath - const buf = await readFileBufferPromise(torrentfile.path) + const buf = await readFile(torrentfile.path) const parsedTorrent = parseTorrent(buf) videoName = isArray(parsedTorrent.name) ? parsedTorrent.name[ 0 ] : parsedTorrent.name as string diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts index e973aa43f..a86cf4f99 100644 --- a/server/controllers/api/videos/index.ts +++ b/server/controllers/api/videos/index.ts @@ -1,7 +1,6 @@ import * as express from 'express' import { extname, join } from 'path' import { VideoCreate, VideoPrivacy, VideoState, VideoUpdate } from '../../../../shared' -import { renamePromise } from '../../../helpers/core-utils' import { getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils' import { processImage } from '../../../helpers/image-utils' import { logger } from '../../../helpers/logger' @@ -56,6 +55,7 @@ import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-u import { videoCaptionsRouter } from './captions' import { videoImportsRouter } from './import' import { resetSequelizeInstance } from '../../../helpers/database-utils' +import { rename } from 'fs-extra' const auditLogger = auditLoggerFactory('videos') const videosRouter = express.Router() @@ -194,7 +194,7 @@ async function addVideo (req: express.Request, res: express.Response) { // Move physical file const videoDir = CONFIG.STORAGE.VIDEOS_DIR const destination = join(videoDir, video.getVideoFilename(videoFile)) - await renamePromise(videoPhysicalFile.path, destination) + await rename(videoPhysicalFile.path, destination) // This is important in case if there is another attempt in the retry process videoPhysicalFile.filename = video.getVideoFilename(videoFile) videoPhysicalFile.path = destination diff --git a/server/helpers/captions-utils.ts b/server/helpers/captions-utils.ts index 20c9fe5aa..660dce65c 100644 --- a/server/helpers/captions-utils.ts +++ b/server/helpers/captions-utils.ts @@ -1,9 +1,8 @@ -import { renamePromise, unlinkPromise } from './core-utils' import { join } from 'path' import { CONFIG } from '../initializers' import { VideoCaptionModel } from '../models/video/video-caption' import * as srt2vtt from 'srt-to-vtt' -import { createReadStream, createWriteStream } from 'fs-extra' +import { createReadStream, createWriteStream, remove, rename } from 'fs-extra' async function moveAndProcessCaptionFile (physicalFile: { filename: string, path: string }, videoCaption: VideoCaptionModel) { const videoCaptionsDir = CONFIG.STORAGE.CAPTIONS_DIR @@ -12,9 +11,9 @@ async function moveAndProcessCaptionFile (physicalFile: { filename: string, path // Convert this srt file to vtt if (physicalFile.path.endsWith('.srt')) { await convertSrtToVtt(physicalFile.path, destination) - await unlinkPromise(physicalFile.path) + await remove(physicalFile.path) } else { // Just move the vtt file - await renamePromise(physicalFile.path, destination) + await rename(physicalFile.path, destination) } // This is important in case if there is another attempt in the retry process diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts index 9830d41a8..f5ef187fe 100644 --- a/server/helpers/core-utils.ts +++ b/server/helpers/core-utils.ts @@ -6,7 +6,6 @@ import * as bcrypt from 'bcrypt' import * as createTorrent from 'create-torrent' import { createHash, pseudoRandomBytes } from 'crypto' -import { copyFile, readdir, readFile, rename, stat, Stats, unlink, writeFile, mkdirp } from 'fs-extra' import { isAbsolute, join } from 'path' import * as pem from 'pem' import * as rimraf from 'rimraf' @@ -168,14 +167,6 @@ function promisify2WithVoid (func: (arg1: T, arg2: U, cb: (err: any) => vo } } -const copyFilePromise = promisify2WithVoid(copyFile) -const readFileBufferPromise = promisify1(readFile) -const unlinkPromise = promisify1WithVoid(unlink) -const renamePromise = promisify2WithVoid(rename) -const writeFilePromise = promisify2WithVoid(writeFile) -const readdirPromise = promisify1(readdir) -const mkdirpPromise = promisify1(mkdirp) -// we cannot modify the Promise types, so we should make the promisify instance check mkdirp const pseudoRandomBytesPromise = promisify1(pseudoRandomBytes) const createPrivateKey = promisify1(pem.createPrivateKey) const getPublicKey = promisify1(pem.getPublicKey) @@ -183,8 +174,6 @@ const bcryptComparePromise = promisify2(bcrypt.compare) const bcryptGenSaltPromise = promisify1(bcrypt.genSalt) const bcryptHashPromise = promisify2(bcrypt.hash) const createTorrentPromise = promisify2(createTorrent) -const rimrafPromise = promisify1WithVoid(rimraf) -const statPromise = promisify1(stat) // --------------------------------------------------------------------------- @@ -202,20 +191,11 @@ export { promisify0, promisify1, - copyFilePromise, - readdirPromise, - readFileBufferPromise, - unlinkPromise, - renamePromise, - writeFilePromise, - mkdirpPromise, pseudoRandomBytesPromise, createPrivateKey, getPublicKey, bcryptComparePromise, bcryptGenSaltPromise, bcryptHashPromise, - createTorrentPromise, - rimrafPromise, - statPromise + createTorrentPromise } diff --git a/server/helpers/ffmpeg-utils.ts b/server/helpers/ffmpeg-utils.ts index 8936005e0..7c45f3632 100644 --- a/server/helpers/ffmpeg-utils.ts +++ b/server/helpers/ffmpeg-utils.ts @@ -1,11 +1,11 @@ import * as ffmpeg from 'fluent-ffmpeg' import { join } from 'path' import { VideoResolution } from '../../shared/models/videos' -import { CONFIG, VIDEO_TRANSCODING_FPS, FFMPEG_NICE } from '../initializers' -import { unlinkPromise } from './core-utils' +import { CONFIG, FFMPEG_NICE, VIDEO_TRANSCODING_FPS } from '../initializers' import { processImage } from './image-utils' import { logger } from './logger' import { checkFFmpegEncoders } from '../initializers/checker' +import { remove } from 'fs-extra' function computeResolutionsToTranscode (videoFileHeight: number) { const resolutionsEnabled: number[] = [] @@ -90,7 +90,7 @@ async function generateImageFromVideoFile (fromPath: string, folder: string, ima logger.error('Cannot generate image from video %s.', fromPath, { err }) try { - await unlinkPromise(pendingImagePath) + await remove(pendingImagePath) } catch (err) { logger.debug('Cannot remove pending image path after generation error.', { err }) } diff --git a/server/helpers/image-utils.ts b/server/helpers/image-utils.ts index 0065f4210..3eaa674ed 100644 --- a/server/helpers/image-utils.ts +++ b/server/helpers/image-utils.ts @@ -1,6 +1,6 @@ import 'multer' import * as sharp from 'sharp' -import { unlinkPromise } from './core-utils' +import { remove } from 'fs-extra' async function processImage ( physicalFile: { path: string }, @@ -11,7 +11,7 @@ async function processImage ( .resize(newSize.width, newSize.height) .toFile(destination) - await unlinkPromise(physicalFile.path) + await remove(physicalFile.path) } // --------------------------------------------------------------------------- diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts index 703e57887..a1ed8e72d 100644 --- a/server/helpers/utils.ts +++ b/server/helpers/utils.ts @@ -2,13 +2,14 @@ import { ResultList } from '../../shared' import { CONFIG } from '../initializers' import { ActorModel } from '../models/activitypub/actor' import { ApplicationModel } from '../models/application/application' -import { pseudoRandomBytesPromise, sha256, unlinkPromise } from './core-utils' +import { pseudoRandomBytesPromise, sha256 } from './core-utils' import { logger } from './logger' import { join } from 'path' import { Instance as ParseTorrent } from 'parse-torrent' +import { remove } from 'fs-extra' function deleteFileAsync (path: string) { - unlinkPromise(path) + remove(path) .catch(err => logger.error('Cannot delete the file %s asynchronously.', path, { err })) } diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts index 1c0d00d70..1c0cc7058 100644 --- a/server/helpers/webtorrent.ts +++ b/server/helpers/webtorrent.ts @@ -1,10 +1,9 @@ import { logger } from './logger' import { generateVideoTmpPath } from './utils' import * as WebTorrent from 'webtorrent' -import { createWriteStream } from 'fs-extra' +import { createWriteStream, remove } from 'fs-extra' import { CONFIG } from '../initializers' import { join } from 'path' -import { unlinkPromise } from './core-utils' function downloadWebTorrentVideo (target: { magnetUri: string, torrentName: string }) { const id = target.magnetUri || target.torrentName @@ -29,11 +28,11 @@ function downloadWebTorrentVideo (target: { magnetUri: string, torrentName: stri if (err) return rej(err) if (target.torrentName) { - unlinkPromise(torrentId) + remove(torrentId) .catch(err => logger.error('Cannot remove torrent %s in webtorrent download.', torrentId, { err })) } - unlinkPromise(join(CONFIG.STORAGE.VIDEOS_DIR, file.name)) + remove(join(CONFIG.STORAGE.VIDEOS_DIR, file.name)) .catch(err => logger.error('Cannot remove torrent file %s in webtorrent download.', file.name, { err })) res(path) diff --git a/server/initializers/installer.ts b/server/initializers/installer.ts index 1926c40dd..e319164e4 100644 --- a/server/initializers/installer.ts +++ b/server/initializers/installer.ts @@ -1,6 +1,5 @@ import * as passwordGenerator from 'password-generator' import { UserRole } from '../../shared' -import { mkdirpPromise, rimrafPromise } from '../helpers/core-utils' import { logger } from '../helpers/logger' import { createApplicationActor, createUserAccountAndChannel } from '../lib/user' import { UserModel } from '../models/account/user' @@ -9,6 +8,7 @@ import { OAuthClientModel } from '../models/oauth/oauth-client' import { applicationExist, clientsExist, usersExist } from './checker' import { CACHE, CONFIG, LAST_MIGRATION_VERSION } from './constants' import { sequelizeTypescript } from './database' +import { remove, ensureDir } from 'fs-extra' async function installApplication () { try { @@ -41,7 +41,7 @@ function removeCacheDirectories () { // Cache directories for (const key of Object.keys(cacheDirectories)) { const dir = cacheDirectories[key] - tasks.push(rimrafPromise(dir)) + tasks.push(remove(dir)) } return Promise.all(tasks) @@ -52,16 +52,16 @@ function createDirectoriesIfNotExist () { const cacheDirectories = Object.keys(CACHE) .map(k => CACHE[k].DIRECTORY) - const tasks: Promise[] = [] + const tasks: Promise[] = [] for (const key of Object.keys(storage)) { const dir = storage[key] - tasks.push(mkdirpPromise(dir)) + tasks.push(ensureDir(dir)) } // Cache directories for (const key of Object.keys(cacheDirectories)) { const dir = cacheDirectories[key] - tasks.push(mkdirpPromise(dir)) + tasks.push(ensureDir(dir)) } return Promise.all(tasks) diff --git a/server/initializers/migrations/0075-video-resolutions.ts b/server/initializers/migrations/0075-video-resolutions.ts index 54ea852b1..26a188e5e 100644 --- a/server/initializers/migrations/0075-video-resolutions.ts +++ b/server/initializers/migrations/0075-video-resolutions.ts @@ -1,9 +1,8 @@ import * as Sequelize from 'sequelize' import { join } from 'path' - -import { readdirPromise, renamePromise } from '../../helpers/core-utils' import { CONFIG } from '../../initializers/constants' import { getVideoFileResolution } from '../../helpers/ffmpeg-utils' +import { readdir, rename } from 'fs-extra' function up (utils: { transaction: Sequelize.Transaction, @@ -14,7 +13,7 @@ function up (utils: { const torrentDir = CONFIG.STORAGE.TORRENTS_DIR const videoFileDir = CONFIG.STORAGE.VIDEOS_DIR - return readdirPromise(videoFileDir) + return readdir(videoFileDir) .then(videoFiles => { const tasks: Promise[] = [] for (const videoFile of videoFiles) { @@ -31,11 +30,11 @@ function up (utils: { .then(height => { const oldTorrentName = uuid + '.torrent' const newTorrentName = uuid + '-' + height + '.torrent' - return renamePromise(join(torrentDir, oldTorrentName), join(torrentDir, newTorrentName)).then(() => height) + return rename(join(torrentDir, oldTorrentName), join(torrentDir, newTorrentName)).then(() => height) }) .then(height => { const newVideoFileName = uuid + '-' + height + '.' + ext - return renamePromise(join(videoFileDir, videoFile), join(videoFileDir, newVideoFileName)).then(() => height) + return rename(join(videoFileDir, videoFile), join(videoFileDir, newVideoFileName)).then(() => height) }) .then(height => { const query = 'UPDATE "VideoFiles" SET "resolution" = ' + height + diff --git a/server/initializers/migrator.ts b/server/initializers/migrator.ts index 539e2bc8f..adc2f9fb3 100644 --- a/server/initializers/migrator.ts +++ b/server/initializers/migrator.ts @@ -1,8 +1,8 @@ import * as path from 'path' -import { readdirPromise } from '../helpers/core-utils' import { logger } from '../helpers/logger' import { LAST_MIGRATION_VERSION } from './constants' import { sequelizeTypescript } from './database' +import { readdir } from 'fs-extra' async function migrate () { const tables = await sequelizeTypescript.getQueryInterface().showAllTables() @@ -52,7 +52,7 @@ export { // --------------------------------------------------------------------------- async function getMigrationScripts () { - const files = await readdirPromise(path.join(__dirname, 'migrations')) + const files = await readdir(path.join(__dirname, 'migrations')) const filesToMigrate: { version: string, script: string diff --git a/server/lib/cache/abstract-video-static-file-cache.ts b/server/lib/cache/abstract-video-static-file-cache.ts index 3e20c5d2a..7512f2b9d 100644 --- a/server/lib/cache/abstract-video-static-file-cache.ts +++ b/server/lib/cache/abstract-video-static-file-cache.ts @@ -1,6 +1,5 @@ import * as AsyncLRU from 'async-lru' -import { createWriteStream } from 'fs-extra' -import { unlinkPromise } from '../../helpers/core-utils' +import { createWriteStream, remove } from 'fs-extra' import { logger } from '../../helpers/logger' import { VideoModel } from '../../models/video/video' import { fetchRemoteVideoStaticFile } from '../activitypub' @@ -26,7 +25,7 @@ export abstract class AbstractVideoStaticFileCache { }) this.lru.on('evict', (obj: { key: string, value: string }) => { - unlinkPromise(obj.value) + remove(obj.value) .then(() => logger.debug('%s evicted from %s', obj.value, this.constructor.name)) }) } diff --git a/server/lib/client-html.ts b/server/lib/client-html.ts index 72984e778..a69e09c32 100644 --- a/server/lib/client-html.ts +++ b/server/lib/client-html.ts @@ -1,12 +1,13 @@ import * as express from 'express' import * as Bluebird from 'bluebird' import { buildFileLocale, getDefaultLocale, is18nLocale, POSSIBLE_LOCALES } from '../../shared/models/i18n/i18n' -import { CONFIG, EMBED_SIZE, CUSTOM_HTML_TAG_COMMENTS, STATIC_PATHS } from '../initializers' +import { CONFIG, CUSTOM_HTML_TAG_COMMENTS, EMBED_SIZE, STATIC_PATHS } from '../initializers' import { join } from 'path' -import { escapeHTML, readFileBufferPromise } from '../helpers/core-utils' +import { escapeHTML } from '../helpers/core-utils' import { VideoModel } from '../models/video/video' import * as validator from 'validator' import { VideoPrivacy } from '../../shared/models/videos' +import { readFile } from 'fs-extra' export class ClientHtml { @@ -20,7 +21,7 @@ export class ClientHtml { const path = ClientHtml.getIndexPath(req, res, paramLang) if (ClientHtml.htmlCache[path]) return ClientHtml.htmlCache[path] - const buffer = await readFileBufferPromise(path) + const buffer = await readFile(path) let html = buffer.toString() diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index d6984ef92..ebcb2090c 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts @@ -6,7 +6,6 @@ import { VideoImportState } from '../../../../shared/models/videos' import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils' import { extname, join } from 'path' import { VideoFileModel } from '../../../models/video/video-file' -import { renamePromise, statPromise, unlinkPromise } from '../../../helpers/core-utils' import { CONFIG, sequelizeTypescript } from '../../../initializers' import { doRequestAndSaveToFile } from '../../../helpers/requests' import { VideoState } from '../../../../shared' @@ -15,6 +14,7 @@ import { federateVideoIfNeeded } from '../../activitypub' import { VideoModel } from '../../../models/video/video' import { downloadWebTorrentVideo } from '../../../helpers/webtorrent' import { getSecureTorrentName } from '../../../helpers/utils' +import { rename, stat } from 'fs-extra' type VideoImportYoutubeDLPayload = { type: 'youtube-dl' @@ -114,7 +114,7 @@ async function processFile (downloader: () => Promise, videoImport: Vide tempVideoPath = await downloader() // Get information about this video - const stats = await statPromise(tempVideoPath) + const stats = await stat(tempVideoPath) const isAble = await videoImport.User.isAbleToUploadVideo({ size: stats.size }) if (isAble === false) { throw new Error('The user video quota is exceeded with this video to import.') @@ -138,7 +138,7 @@ async function processFile (downloader: () => Promise, videoImport: Vide // Move file videoDestFile = join(CONFIG.STORAGE.VIDEOS_DIR, videoImport.Video.getVideoFilename(videoFile)) - await renamePromise(tempVideoPath, videoDestFile) + await rename(tempVideoPath, videoDestFile) tempVideoPath = null // This path is not used anymore // Process thumbnail diff --git a/server/lib/schedulers/youtube-dl-update-scheduler.ts b/server/lib/schedulers/youtube-dl-update-scheduler.ts index da47378e8..faadb4334 100644 --- a/server/lib/schedulers/youtube-dl-update-scheduler.ts +++ b/server/lib/schedulers/youtube-dl-update-scheduler.ts @@ -5,9 +5,9 @@ import { AbstractScheduler } from './abstract-scheduler' import { SCHEDULER_INTERVALS_MS } from '../../initializers' import { logger } from '../../helpers/logger' import * as request from 'request' -import { createWriteStream, writeFile } from 'fs-extra' +import { createWriteStream, ensureDir, writeFile } from 'fs-extra' import { join } from 'path' -import { mkdirpPromise, root } from '../../helpers/core-utils' +import { root } from '../../helpers/core-utils' export class YoutubeDlUpdateScheduler extends AbstractScheduler { @@ -27,7 +27,7 @@ export class YoutubeDlUpdateScheduler extends AbstractScheduler { const detailsPath = join(binDirectory, 'details') const url = 'https://yt-dl.org/downloads/latest/youtube-dl' - await mkdirpPromise(binDirectory) + await ensureDir(binDirectory) return new Promise(res => { request.get(url, { followRedirect: false }, (err, result) => { diff --git a/server/models/avatar/avatar.ts b/server/models/avatar/avatar.ts index 51e26c87b..5d73e24fa 100644 --- a/server/models/avatar/avatar.ts +++ b/server/models/avatar/avatar.ts @@ -1,9 +1,9 @@ import { join } from 'path' import { AfterDestroy, AllowNull, Column, CreatedAt, Model, Table, UpdatedAt } from 'sequelize-typescript' import { Avatar } from '../../../shared/models/avatars/avatar.model' -import { unlinkPromise } from '../../helpers/core-utils' import { CONFIG, STATIC_PATHS } from '../../initializers' import { logger } from '../../helpers/logger' +import { remove } from 'fs-extra' @Table({ tableName: 'avatar' @@ -40,6 +40,6 @@ export class AvatarModel extends Model { removeAvatar () { const avatarPath = join(CONFIG.STORAGE.AVATARS_DIR, this.filename) - return unlinkPromise(avatarPath) + return remove(avatarPath) } } diff --git a/server/models/video/video-caption.ts b/server/models/video/video-caption.ts index 3593646a2..10ef46c14 100644 --- a/server/models/video/video-caption.ts +++ b/server/models/video/video-caption.ts @@ -19,7 +19,7 @@ import { VideoCaption } from '../../../shared/models/videos/caption/video-captio import { CONFIG, STATIC_PATHS, VIDEO_LANGUAGES } from '../../initializers' import { join } from 'path' import { logger } from '../../helpers/logger' -import { unlinkPromise } from '../../helpers/core-utils' +import { remove } from 'fs-extra' export enum ScopeNames { WITH_VIDEO_UUID_AND_REMOTE = 'WITH_VIDEO_UUID_AND_REMOTE' @@ -172,6 +172,6 @@ export class VideoCaptionModel extends Model { } removeCaptionFile () { - return unlinkPromise(CONFIG.STORAGE.CAPTIONS_DIR + this.getCaptionName()) + return remove(CONFIG.STORAGE.CAPTIONS_DIR + this.getCaptionName()) } } diff --git a/server/models/video/video.ts b/server/models/video/video.ts index a956da16e..6271db1b3 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -30,15 +30,7 @@ import { VideoPrivacy, VideoResolution, VideoState } from '../../../shared' import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos' import { VideoFilter } from '../../../shared/models/videos/video-query.type' -import { - copyFilePromise, - createTorrentPromise, - peertubeTruncate, - renamePromise, - statPromise, - unlinkPromise, - writeFilePromise -} from '../../helpers/core-utils' +import { createTorrentPromise, peertubeTruncate } from '../../helpers/core-utils' import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' import { isBooleanValid } from '../../helpers/custom-validators/misc' import { @@ -95,6 +87,7 @@ import { VideoTagModel } from './video-tag' import { ScheduleVideoUpdateModel } from './schedule-video-update' import { VideoCaptionModel } from './video-caption' import { VideoBlacklistModel } from './video-blacklist' +import { copy, remove, rename, stat, writeFile } from 'fs-extra' // FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation const indexes: Sequelize.DefineIndexesOptions[] = [ @@ -1187,7 +1180,7 @@ export class VideoModel extends Model { const filePath = join(CONFIG.STORAGE.TORRENTS_DIR, this.getTorrentFileName(videoFile)) logger.info('Creating torrent %s.', filePath) - await writeFilePromise(filePath, torrent) + await writeFile(filePath, torrent) const parsedTorrent = parseTorrent(torrent) videoFile.infoHash = parsedTorrent.infoHash @@ -1497,14 +1490,14 @@ export class VideoModel extends Model { await transcode(transcodeOptions) try { - await unlinkPromise(videoInputPath) + await remove(videoInputPath) // Important to do this before getVideoFilename() to take in account the new file extension inputVideoFile.set('extname', newExtname) const videoOutputPath = this.getVideoFilePath(inputVideoFile) - await renamePromise(videoTranscodedPath, videoOutputPath) - const stats = await statPromise(videoOutputPath) + await rename(videoTranscodedPath, videoOutputPath) + const stats = await stat(videoOutputPath) const fps = await getVideoFileFPS(videoOutputPath) inputVideoFile.set('size', stats.size) @@ -1545,7 +1538,7 @@ export class VideoModel extends Model { await transcode(transcodeOptions) - const stats = await statPromise(videoOutputPath) + const stats = await stat(videoOutputPath) const fps = await getVideoFileFPS(videoOutputPath) newVideoFile.set('size', stats.size) @@ -1560,7 +1553,7 @@ export class VideoModel extends Model { async importVideoFile (inputFilePath: string) { const { videoFileResolution } = await getVideoFileResolution(inputFilePath) - const { size } = await statPromise(inputFilePath) + const { size } = await stat(inputFilePath) const fps = await getVideoFileFPS(inputFilePath) let updatedVideoFile = new VideoFileModel({ @@ -1589,7 +1582,7 @@ export class VideoModel extends Model { } const outputPath = this.getVideoFilePath(updatedVideoFile) - await copyFilePromise(inputFilePath, outputPath) + await copy(inputFilePath, outputPath) await this.createTorrentAndSetInfoHash(updatedVideoFile) @@ -1610,25 +1603,25 @@ export class VideoModel extends Model { removeThumbnail () { const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, this.getThumbnailName()) - return unlinkPromise(thumbnailPath) + return remove(thumbnailPath) .catch(err => logger.warn('Cannot delete thumbnail %s.', thumbnailPath, { err })) } removePreview () { const previewPath = join(CONFIG.STORAGE.PREVIEWS_DIR + this.getPreviewName()) - return unlinkPromise(previewPath) + return remove(previewPath) .catch(err => logger.warn('Cannot delete preview %s.', previewPath, { err })) } removeFile (videoFile: VideoFileModel) { const filePath = join(CONFIG.STORAGE.VIDEOS_DIR, this.getVideoFilename(videoFile)) - return unlinkPromise(filePath) + return remove(filePath) .catch(err => logger.warn('Cannot delete file %s.', filePath, { err })) } removeTorrent (videoFile: VideoFileModel) { const torrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, this.getTorrentFileName(videoFile)) - return unlinkPromise(torrentPath) + return remove(torrentPath) .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err })) } diff --git a/server/tests/feeds/feeds.ts b/server/tests/feeds/feeds.ts index 90450a0bb..72c9bf9a0 100644 --- a/server/tests/feeds/feeds.ts +++ b/server/tests/feeds/feeds.ts @@ -13,7 +13,6 @@ import { setAccessTokensToServers, uploadVideo } from '../utils' -import { join } from 'path' import * as libxmljs from 'libxmljs' import { addVideoCommentThread } from '../utils/videos/video-comments' import { waitJobs } from '../utils/server/jobs' diff --git a/server/tests/utils/feeds/feeds.ts b/server/tests/utils/feeds/feeds.ts index ffd23a1ad..fb480b704 100644 --- a/server/tests/utils/feeds/feeds.ts +++ b/server/tests/utils/feeds/feeds.ts @@ -1,5 +1,4 @@ import * as request from 'supertest' -import { readFileBufferPromise } from '../../../helpers/core-utils' type FeedType = 'videos' | 'video-comments' diff --git a/server/tests/utils/miscs/miscs.ts b/server/tests/utils/miscs/miscs.ts index b0667491b..b2f80e9b1 100644 --- a/server/tests/utils/miscs/miscs.ts +++ b/server/tests/utils/miscs/miscs.ts @@ -4,7 +4,7 @@ import * as chai from 'chai' import { isAbsolute, join } from 'path' import * as request from 'supertest' import * as WebTorrent from 'webtorrent' -import { readFileBufferPromise } from '../../../helpers/core-utils' +import { readFile } from 'fs-extra' const expect = chai.expect let webtorrent = new WebTorrent() @@ -43,7 +43,7 @@ async function testImage (url: string, imageName: string, imagePath: string, ext const body = res.body - const data = await readFileBufferPromise(join(__dirname, '..', '..', 'fixtures', imageName + extension)) + const data = await readFile(join(__dirname, '..', '..', 'fixtures', imageName + extension)) const minLength = body.length - ((20 * body.length) / 100) const maxLength = body.length + ((20 * body.length) / 100) diff --git a/server/tests/utils/users/accounts.ts b/server/tests/utils/users/accounts.ts index 024a315c7..f82b8d906 100644 --- a/server/tests/utils/users/accounts.ts +++ b/server/tests/utils/users/accounts.ts @@ -1,10 +1,9 @@ /* tslint:disable:no-unused-expression */ import { expect } from 'chai' -import { existsSync } from 'fs-extra' +import { existsSync, readdir } from 'fs-extra' import { join } from 'path' import { Account } from '../../../../shared/models/actors' -import { readdirPromise } from '../../../helpers/core-utils' import { root } from '../index' import { makeGetRequest } from '../requests/requests' @@ -47,7 +46,7 @@ async function checkActorFilesWereRemoved (actorUUID: string, serverNumber: numb const directoryExists = existsSync(directoryPath) expect(directoryExists).to.be.true - const files = await readdirPromise(directoryPath) + const files = await readdir(directoryPath) for (const file of files) { expect(file).to.not.contain(actorUUID) } diff --git a/server/tests/utils/videos/videos.ts b/server/tests/utils/videos/videos.ts index 973bbbe87..7eee25402 100644 --- a/server/tests/utils/videos/videos.ts +++ b/server/tests/utils/videos/videos.ts @@ -1,13 +1,14 @@ /* tslint:disable:no-unused-expression */ import { expect } from 'chai' -import { existsSync, readFile } from 'fs-extra' +import { existsSync, readdir, readFile } from 'fs-extra' import * as parseTorrent from 'parse-torrent' import { extname, join } from 'path' import * as request from 'supertest' import { buildAbsoluteFixturePath, - getMyUserInformation, immutableAssign, + getMyUserInformation, + immutableAssign, makeGetRequest, makePutBodyRequest, makeUploadRequest, @@ -16,7 +17,6 @@ import { testImage } from '../' import { VideoDetails, VideoPrivacy } from '../../../../shared/models/videos' -import { readdirPromise } from '../../../helpers/core-utils' import { VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES, VIDEO_PRIVACIES } from '../../../initializers' import { dateIsValid, webtorrentAdd } from '../index' @@ -276,7 +276,7 @@ async function checkVideoFilesWereRemoved (videoUUID: string, serverNumber: numb const directoryExists = existsSync(directoryPath) expect(directoryExists).to.be.true - const files = await readdirPromise(directoryPath) + const files = await readdir(directoryPath) for (const file of files) { expect(file).to.not.contain(videoUUID) } diff --git a/server/tools/import-videos.ts b/server/tools/import-videos.ts index e49fbb2f5..812600f8b 100644 --- a/server/tools/import-videos.ts +++ b/server/tools/import-videos.ts @@ -5,12 +5,12 @@ import * as program from 'commander' import { join } from 'path' import * as youtubeDL from 'youtube-dl' import { VideoPrivacy } from '../../shared/models/videos' -import { unlinkPromise } from '../helpers/core-utils' import { doRequestAndSaveToFile } from '../helpers/requests' import { CONSTRAINTS_FIELDS } from '../initializers' import { getClient, getVideoCategories, login, searchVideo, uploadVideo } from '../tests/utils' import { truncate } from 'lodash' import * as prompt from 'prompt' +import { remove } from 'fs-extra' program .option('-u, --url ', 'Server url') @@ -204,10 +204,8 @@ async function uploadVideoOnPeerTube (videoInfo: any, videoPath: string, languag } } - await unlinkPromise(videoPath) - if (thumbnailfile) { - await unlinkPromise(thumbnailfile) - } + await remove(videoPath) + if (thumbnailfile) await remove(thumbnailfile) console.log('Uploaded video "%s"!\n', videoAttributes.name) } diff --git a/server/tools/upload.ts b/server/tools/upload.ts index b5630bb9c..796a0f723 100644 --- a/server/tools/upload.ts +++ b/server/tools/upload.ts @@ -1,13 +1,10 @@ import * as program from 'commander' import { access, constants } from 'fs-extra' import { isAbsolute } from 'path' -import { promisify } from 'util' import { getClient, login } from '../tests/utils' import { uploadVideo } from '../tests/utils/index' import { VideoPrivacy } from '../../shared/models/videos' -const accessPromise = promisify(access) - program .option('-u, --url ', 'Server url') .option('-U, --username ', 'Username') @@ -68,7 +65,7 @@ async function run () { const res2 = await login(program[ 'url' ], client, user) const accessToken = res2.body.access_token - await accessPromise(program[ 'file' ], constants.F_OK) + await access(program[ 'file' ], constants.F_OK) console.log('Uploading %s video...', program[ 'videoName' ]) diff --git a/yarn.lock b/yarn.lock index c1a3d6e88..b0e1d49f7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -116,6 +116,12 @@ dependencies: "@types/node" "*" +"@types/fs-extra@^5.0.4": + version "5.0.4" + resolved "https://registry.yarnpkg.com/@types/fs-extra/-/fs-extra-5.0.4.tgz#b971134d162cc0497d221adde3dbb67502225599" + dependencies: + "@types/node" "*" + "@types/geojson@^1.0.0": version "1.0.6" resolved "https://registry.yarnpkg.com/@types/geojson/-/geojson-1.0.6.tgz#3e02972728c69248c2af08d60a48cbb8680fffdf" @@ -2702,6 +2708,14 @@ fs-extra@^3.0.1: jsonfile "^3.0.0" universalify "^0.1.0" +fs-extra@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-7.0.0.tgz#8cc3f47ce07ef7b3593a11b9fb245f7e34c041d6" + dependencies: + graceful-fs "^4.1.2" + jsonfile "^4.0.0" + universalify "^0.1.0" + fs-minipass@^1.2.5: version "1.2.5" resolved "https://registry.yarnpkg.com/fs-minipass/-/fs-minipass-1.2.5.tgz#06c277218454ec288df77ada54a03b8702aacb9d" @@ -3976,6 +3990,12 @@ jsonfile@^3.0.0: optionalDependencies: graceful-fs "^4.1.6" +jsonfile@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-4.0.0.tgz#8771aae0799b64076b76640fca058f9c10e33ecb" + optionalDependencies: + graceful-fs "^4.1.6" + jsonify@~0.0.0: version "0.0.0" resolved "https://registry.yarnpkg.com/jsonify/-/jsonify-0.0.0.tgz#2c74b6ee41d93ca51b7b5aaee8f503631d252a73" @@ -6197,7 +6217,7 @@ right-align@^0.1.1: dependencies: align-text "^0.1.1" -rimraf@2, rimraf@2.x.x, rimraf@^2.2.8, rimraf@^2.4.2, rimraf@^2.5.1, rimraf@^2.5.4, rimraf@^2.6.1, rimraf@~2.6.2: +rimraf@2, rimraf@2.x.x, rimraf@^2.2.8, rimraf@^2.4.2, rimraf@^2.5.1, rimraf@^2.6.1, rimraf@~2.6.2: version "2.6.2" resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.2.tgz#2ed8150d24a16ea8651e6d6ef0f47c4158ce7a36" dependencies: