Use got instead of request

package.json

@@ -83,8 +83,7 @@
     "sass-lint": "sass-lint"
   },
   "resolutions": {
-    "oauth2-server": "3.1.0-beta.1",
+    "oauth2-server": "3.1.0-beta.1"
-    "http-signature": "1.3.5"
   },
   "dependencies": {
     "apicache": "1.6.2",
@@ -111,6 +110,7 @@
     "flat": "^5.0.0",
     "fluent-ffmpeg": "^2.1.0",
     "fs-extra": "^9.0.0",
+    "got": "^11.8.2",
     "helmet": "^4.1.0",
     "http-signature": "1.3.5",
     "ip-anonymize": "^0.1.0",
@@ -140,7 +140,6 @@
     "pug": "^3.0.0",
     "redis": "^3.0.2",
     "reflect-metadata": "^0.1.12",
-    "request": "^2.81.0",
     "sanitize-html": "2.x",
     "scripty": "^2.0.0",
     "sequelize": "6.5.0",

server/controllers/api/search.ts

@@ -1,6 +1,6 @@
 import * as express from 'express'
 import { sanitizeUrl } from '@server/helpers/core-utils'
-import { doRequest } from '@server/helpers/requests'
+import { doJSONRequest } from '@server/helpers/requests'
 import { CONFIG } from '@server/initializers/config'
 import { getOrCreateVideoAndAccountAndChannel } from '@server/lib/activitypub/videos'
 import { AccountBlocklistModel } from '@server/models/account/account-blocklist'
@@ -94,9 +94,9 @@ async function searchVideoChannelsIndex (query: VideoChannelsSearchQuery, res: e
   try {
     logger.debug('Doing video channels search index request on %s.', url, { body })
 
-    const searchIndexResult = await doRequest<ResultList<VideoChannel>>({ uri: url, body, json: true })
+    const searchIndexResult = await doJSONRequest<ResultList<VideoChannel>>(url, { json: body })
 
-    return res.json(searchIndexResult.body)
+    return res.json(searchIndexResult)
   } catch (err) {
     logger.warn('Cannot use search index to make video channels search.', { err })
 
@@ -186,9 +186,9 @@ async function searchVideosIndex (query: VideosSearchQuery, res: express.Respons
   try {
     logger.debug('Doing videos search index request on %s.', url, { body })
 
-    const searchIndexResult = await doRequest<ResultList<Video>>({ uri: url, body, json: true })
+    const searchIndexResult = await doJSONRequest<ResultList<Video>>(url, { json: body })
 
-    return res.json(searchIndexResult.body)
+    return res.json(searchIndexResult)
   } catch (err) {
     logger.warn('Cannot use search index to make video search.', { err })
 

server/helpers/activitypub.ts

@@ -3,7 +3,6 @@ import { URL } from 'url'
 import validator from 'validator'
 import { ContextType } from '@shared/models/activitypub/context'
 import { ResultList } from '../../shared/models'
-import { Activity } from '../../shared/models/activitypub'
 import { ACTIVITY_PUB, REMOTE_SCHEME } from '../initializers/constants'
 import { MActor, MVideoWithHost } from '../types/models'
 import { pageToStartAndCount } from './core-utils'
@@ -182,10 +181,10 @@ async function activityPubCollectionPagination (
 
 }
 
-function buildSignedActivity (byActor: MActor, data: Object, contextType?: ContextType) {
+function buildSignedActivity <T> (byActor: MActor, data: T, contextType?: ContextType) {
   const activity = activityPubContextify(data, contextType)
 
-  return signJsonLDObject(byActor, activity) as Promise<Activity>
+  return signJsonLDObject(byActor, activity)
 }
 
 function getAPId (activity: string | { id: string }) {

server/helpers/core-utils.ts

@@ -10,7 +10,9 @@ import { BinaryToTextEncoding, createHash, randomBytes } from 'crypto'
 import { truncate } from 'lodash'
 import { basename, isAbsolute, join, resolve } from 'path'
 import * as pem from 'pem'
+import { pipeline } from 'stream'
 import { URL } from 'url'
+import { promisify } from 'util'
 
 const objectConverter = (oldObject: any, keyConverter: (e: string) => string, valueConverter: (e: any) => any) => {
   if (!oldObject || typeof oldObject !== 'object') {
@@ -254,6 +256,7 @@ const createPrivateKey = promisify1<number, { key: string }>(pem.createPrivateKe
 const getPublicKey = promisify1<string, { publicKey: string }>(pem.getPublicKey)
 const execPromise2 = promisify2<string, any, string>(exec)
 const execPromise = promisify1<string, string>(exec)
+const pipelinePromise = promisify(pipeline)
 
 // ---------------------------------------------------------------------------
 
@@ -284,5 +287,6 @@ export {
   createPrivateKey,
   getPublicKey,
   execPromise2,
-  execPromise
+  execPromise,
+  pipelinePromise
 }

server/helpers/custom-validators/activitypub/activity.ts

@@ -41,7 +41,7 @@ const activityCheckers: { [ P in ActivityType ]: (activity: Activity) => boolean
 }
 
 function isActivityValid (activity: any) {
-  const checker = activityCheckers[activity.tswype]
+  const checker = activityCheckers[activity.type]
   // Unknown activity type
   if (!checker) return false
 

server/helpers/peertube-crypto.ts

@@ -84,7 +84,7 @@ async function isJsonLDRSA2017Verified (fromActor: MActor, signedDocument: any)
   return verify.verify(fromActor.publicKey, signedDocument.signature.signatureValue, 'base64')
 }
 
-async function signJsonLDObject (byActor: MActor, data: any) {
+async function signJsonLDObject <T> (byActor: MActor, data: T) {
   const signature = {
     type: 'RsaSignature2017',
     creator: byActor.url,

server/helpers/requests.ts

@@ -1,58 +1,124 @@
-import * as Bluebird from 'bluebird'
 import { createWriteStream, remove } from 'fs-extra'
-import * as request from 'request'
+import got, { CancelableRequest, Options as GotOptions } from 'got'
-import { ACTIVITY_PUB, PEERTUBE_VERSION, WEBSERVER } from '../initializers/constants'
-import { processImage } from './image-utils'
 import { join } from 'path'
-import { logger } from './logger'
 import { CONFIG } from '../initializers/config'
+import { ACTIVITY_PUB, PEERTUBE_VERSION, WEBSERVER } from '../initializers/constants'
+import { pipelinePromise } from './core-utils'
+import { processImage } from './image-utils'
+import { logger } from './logger'
 
-function doRequest <T> (
+const httpSignature = require('http-signature')
-  requestOptions: request.CoreOptions & request.UriOptions & { activityPub?: boolean },
-  bodyKBLimit = 1000 // 1MB
-): Bluebird<{ response: request.RequestResponse, body: T }> {
-  if (!(requestOptions.headers)) requestOptions.headers = {}
-  requestOptions.headers['User-Agent'] = getUserAgent()
 
-  if (requestOptions.activityPub === true) {
+type PeerTubeRequestOptions = {
-    requestOptions.headers['accept'] = ACTIVITY_PUB.ACCEPT_HEADER
+  activityPub?: boolean
+  bodyKBLimit?: number // 1MB
+  httpSignature?: {
+    algorithm: string
+    authorizationHeaderName: string
+    keyId: string
+    key: string
+    headers: string[]
   }
+  jsonResponse?: boolean
+} & Pick<GotOptions, 'headers' | 'json' | 'method' | 'searchParams'>
 
-  return new Bluebird<{ response: request.RequestResponse, body: T }>((res, rej) => {
+const peertubeGot = got.extend({
-    request(requestOptions, (err, response, body) => err ? rej(err) : res({ response, body }))
+  headers: {
-      .on('data', onRequestDataLengthCheck(bodyKBLimit))
+    'user-agent': getUserAgent()
+  },
+
+  handlers: [
+    (options, next) => {
+      const promiseOrStream = next(options) as CancelableRequest<any>
+      const bodyKBLimit = options.context?.bodyKBLimit
+      if (!bodyKBLimit) throw new Error('No KB limit for this request')
+
+      /* eslint-disable @typescript-eslint/no-floating-promises */
+      promiseOrStream.on('downloadProgress', progress => {
+        if (progress.transferred * 1000 > bodyKBLimit && progress.percent !== 1) {
+          promiseOrStream.cancel(`Exceeded the download limit of ${bodyKBLimit} bytes`)
+        }
       })
+
+      return promiseOrStream
+    }
+  ],
+
+  hooks: {
+    beforeRequest: [
+      options => {
+        const headers = options.headers || {}
+        headers['host'] = options.url.host
+      },
+
+      options => {
+        const httpSignatureOptions = options.context?.httpSignature
+
+        if (httpSignatureOptions) {
+          const method = options.method ?? 'GET'
+          const path = options.path ?? options.url.pathname
+
+          if (!method || !path) {
+            throw new Error(`Cannot sign request without method (${method}) or path (${path}) ${options}`)
           }
 
-function doRequestAndSaveToFile (
+          httpSignature.signRequest({
-  requestOptions: request.CoreOptions & request.UriOptions,
+            getHeader: function (header) {
+              return options.headers[header]
+            },
+
+            setHeader: function (header, value) {
+              options.headers[header] = value
+            },
+
+            method,
+            path
+          }, httpSignatureOptions)
+        }
+      }
+    ]
+  }
+})
+
+function doRequest (url: string, options: PeerTubeRequestOptions = {}) {
+  const gotOptions = buildGotOptions(options)
+
+  return peertubeGot(url, gotOptions)
+    .catch(err => { throw buildRequestError(err) })
+}
+
+function doJSONRequest <T> (url: string, options: PeerTubeRequestOptions = {}) {
+  const gotOptions = buildGotOptions(options)
+
+  return peertubeGot<T>(url, { ...gotOptions, responseType: 'json' })
+    .catch(err => { throw buildRequestError(err) })
+}
+
+async function doRequestAndSaveToFile (
+  url: string,
   destPath: string,
-  bodyKBLimit = 10000 // 10MB
+  options: PeerTubeRequestOptions = {}
 ) {
-  if (!requestOptions.headers) requestOptions.headers = {}
+  const gotOptions = buildGotOptions(options)
-  requestOptions.headers['User-Agent'] = getUserAgent()
 
-  return new Bluebird<void>((res, rej) => {
+  const outFile = createWriteStream(destPath)
-    const file = createWriteStream(destPath)
-    file.on('finish', () => res())
-
-    request(requestOptions)
-      .on('data', onRequestDataLengthCheck(bodyKBLimit))
-      .on('error', err => {
-        file.close()
 
+  try {
+    await pipelinePromise(
+      peertubeGot.stream(url, gotOptions),
+      outFile
+    )
+  } catch (err) {
     remove(destPath)
       .catch(err => logger.error('Cannot remove %s after request failure.', destPath, { err }))
 
-        return rej(err)
+    throw buildRequestError(err)
-      })
+  }
-      .pipe(file)
-  })
 }
 
 async function downloadImage (url: string, destDir: string, destName: string, size: { width: number, height: number }) {
   const tmpPath = join(CONFIG.STORAGE.TMP_DIR, 'pending-' + destName)
-  await doRequestAndSaveToFile({ method: 'GET', uri: url }, tmpPath)
+  await doRequestAndSaveToFile(url, tmpPath)
 
   const destPath = join(destDir, destName)
 
@@ -73,24 +139,43 @@ function getUserAgent () {
 
 export {
   doRequest,
+  doJSONRequest,
   doRequestAndSaveToFile,
   downloadImage
 }
 
 // ---------------------------------------------------------------------------
 
-// Thanks to https://github.com/request/request/issues/2470#issuecomment-268929907 <3
+function buildGotOptions (options: PeerTubeRequestOptions) {
-function onRequestDataLengthCheck (bodyKBLimit: number) {
+  const { activityPub, bodyKBLimit = 1000 } = options
-  let bufferLength = 0
-  const bytesLimit = bodyKBLimit * 1000
 
-  return function (chunk) {
+  const context = { bodyKBLimit, httpSignature: options.httpSignature }
-    bufferLength += chunk.length
-    if (bufferLength > bytesLimit) {
-      this.abort()
 
-      const error = new Error(`Response was too large - aborted after ${bytesLimit} bytes.`)
+  let headers = options.headers || {}
-      this.emit('error', error)
+
+  headers = { ...headers, date: new Date().toUTCString() }
+
+  if (activityPub) {
+    headers = { ...headers, accept: ACTIVITY_PUB.ACCEPT_HEADER }
+  }
+
+  return {
+    method: options.method,
+    json: options.json,
+    searchParams: options.searchParams,
+    headers,
+    context
   }
 }
+
+function buildRequestError (error: any) {
+  const newError = new Error(error.message)
+  newError.name = error.name
+  newError.stack = error.stack
+
+  if (error.response?.body) {
+    error.responseBody = error.response.body
+  }
+
+  return newError
 }

server/helpers/youtube-dl.ts

@@ -1,13 +1,13 @@
 import { createWriteStream } from 'fs'
 import { ensureDir, move, pathExists, remove, writeFile } from 'fs-extra'
+import got from 'got'
 import { join } from 'path'
-import * as request from 'request'
 import { CONFIG } from '@server/initializers/config'
 import { HttpStatusCode } from '../../shared/core-utils/miscs/http-error-codes'
 import { VideoResolution } from '../../shared/models/videos'
 import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES } from '../initializers/constants'
 import { getEnabledResolutions } from '../lib/video-transcoding'
-import { peertubeTruncate, root } from './core-utils'
+import { peertubeTruncate, pipelinePromise, root } from './core-utils'
 import { isVideoFileExtnameValid } from './custom-validators/videos'
 import { logger } from './logger'
 import { generateVideoImportTmpPath } from './utils'
@@ -195,55 +195,32 @@ async function updateYoutubeDLBinary () {
 
   await ensureDir(binDirectory)
 
-  return new Promise<void>(res => {
+  try {
-    request.get(url, { followRedirect: false }, (err, result) => {
+    const result = await got(url, { followRedirect: false })
-      if (err) {
-        logger.error('Cannot update youtube-dl.', { err })
-        return res()
-      }
 
     if (result.statusCode !== HttpStatusCode.FOUND_302) {
       logger.error('youtube-dl update error: did not get redirect for the latest version link. Status %d', result.statusCode)
-        return res()
+      return
     }
 
-      const url = result.headers.location
+    const newUrl = result.headers.location
-      const downloadFile = request.get(url)
+    const newVersion = /yt-dl\.org\/downloads\/(\d{4}\.\d\d\.\d\d(\.\d)?)\/youtube-dl/.exec(newUrl)[1]
-      const newVersion = /yt-dl\.org\/downloads\/(\d{4}\.\d\d\.\d\d(\.\d)?)\/youtube-dl/.exec(url)[1]
 
-      downloadFile.on('response', result => {
+    const downloadFileStream = got.stream(newUrl)
-        if (result.statusCode !== HttpStatusCode.OK_200) {
+    const writeStream = createWriteStream(bin, { mode: 493 })
-          logger.error('Cannot update youtube-dl: new version response is not 200, it\'s %d.', result.statusCode)
-          return res()
-        }
 
-        const writeStream = createWriteStream(bin, { mode: 493 }).on('error', err => {
+    await pipelinePromise(
-          logger.error('youtube-dl update error in write stream', { err })
+      downloadFileStream,
-          return res()
+      writeStream
-        })
+    )
 
-        downloadFile.pipe(writeStream)
-      })
-
-      downloadFile.on('error', err => {
-        logger.error('youtube-dl update error.', { err })
-        return res()
-      })
-
-      downloadFile.on('end', () => {
     const details = JSON.stringify({ version: newVersion, path: bin, exec: 'youtube-dl' })
-        writeFile(detailsPath, details, { encoding: 'utf8' }, err => {
+    await writeFile(detailsPath, details, { encoding: 'utf8' })
-          if (err) {
-            logger.error('youtube-dl update error: cannot write details.', { err })
-            return res()
-          }
 
     logger.info('youtube-dl updated to version %s.', newVersion)
-          return res()
+  } catch (err) {
-        })
+    logger.error('Cannot update youtube-dl.', { err })
-      })
+  }
-    })
-  })
 }
 
 async function safeGetYoutubeDL () {

server/initializers/constants.ts

@@ -29,7 +29,7 @@ const LAST_MIGRATION_VERSION = 610
 // ---------------------------------------------------------------------------
 
 const API_VERSION = 'v1'
-const PEERTUBE_VERSION = require(join(root(), 'package.json')).version
+const PEERTUBE_VERSION: string = require(join(root(), 'package.json')).version
 
 const PAGINATION = {
   GLOBAL: {

server/lib/activitypub/actor.ts

@@ -1,26 +1,28 @@
 import * as Bluebird from 'bluebird'
+import { extname } from 'path'
 import { Op, Transaction } from 'sequelize'
 import { URL } from 'url'
 import { v4 as uuidv4 } from 'uuid'
+import { getServerActor } from '@server/models/application/application'
+import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
 import { ActivityPubActor, ActivityPubActorType, ActivityPubOrderedCollection } from '../../../shared/models/activitypub'
 import { ActivityPubAttributedTo } from '../../../shared/models/activitypub/objects'
 import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
+import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor'
 import { sanitizeAndCheckActorObject } from '../../helpers/custom-validators/activitypub/actor'
 import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
 import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils'
 import { logger } from '../../helpers/logger'
 import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto'
-import { doRequest } from '../../helpers/requests'
+import { doJSONRequest } from '../../helpers/requests'
 import { getUrlFromWebfinger } from '../../helpers/webfinger'
 import { MIMETYPES, WEBSERVER } from '../../initializers/constants'
+import { sequelizeTypescript } from '../../initializers/database'
 import { AccountModel } from '../../models/account/account'
 import { ActorModel } from '../../models/activitypub/actor'
 import { AvatarModel } from '../../models/avatar/avatar'
 import { ServerModel } from '../../models/server/server'
 import { VideoChannelModel } from '../../models/video/video-channel'
-import { JobQueue } from '../job-queue'
-import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor'
-import { sequelizeTypescript } from '../../initializers/database'
 import {
   MAccount,
   MAccountDefault,
@@ -34,9 +36,7 @@ import {
   MActorId,
   MChannel
 } from '../../types/models'
-import { extname } from 'path'
+import { JobQueue } from '../job-queue'
-import { getServerActor } from '@server/models/application/application'
-import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
 
 // Set account keys, this could be long so process after the account creation and do not block the client
 async function generateAndSaveActorKeys <T extends MActor> (actor: T) {
@@ -209,16 +209,10 @@ async function deleteActorAvatarInstance (actor: MActorDefault, t: Transaction)
 }
 
 async function fetchActorTotalItems (url: string) {
-  const options = {
-    uri: url,
-    method: 'GET',
-    json: true,
-    activityPub: true
-  }
-
   try {
-    const { body } = await doRequest<ActivityPubOrderedCollection<unknown>>(options)
+    const { body } = await doJSONRequest<ActivityPubOrderedCollection<unknown>>(url, { activityPub: true })
-    return body.totalItems ? body.totalItems : 0
+
+    return body.totalItems || 0
   } catch (err) {
     logger.warn('Cannot fetch remote actor count %s.', url, { err })
     return 0
@@ -449,26 +443,19 @@ type FetchRemoteActorResult = {
   attributedTo: ActivityPubAttributedTo[]
 }
 async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: number, result: FetchRemoteActorResult }> {
-  const options = {
-    uri: actorUrl,
-    method: 'GET',
-    json: true,
-    activityPub: true
-  }
-
   logger.info('Fetching remote actor %s.', actorUrl)
 
-  const requestResult = await doRequest<ActivityPubActor>(options)
+  const requestResult = await doJSONRequest<ActivityPubActor>(actorUrl, { activityPub: true })
   const actorJSON = requestResult.body
 
   if (sanitizeAndCheckActorObject(actorJSON) === false) {
     logger.debug('Remote actor JSON is not valid.', { actorJSON })
-    return { result: undefined, statusCode: requestResult.response.statusCode }
+    return { result: undefined, statusCode: requestResult.statusCode }
   }
 
   if (checkUrlsSameHost(actorJSON.id, actorUrl) !== true) {
     logger.warn('Actor url %s has not the same host than its AP id %s', actorUrl, actorJSON.id)
-    return { result: undefined, statusCode: requestResult.response.statusCode }
+    return { result: undefined, statusCode: requestResult.statusCode }
   }
 
   const followersCount = await fetchActorTotalItems(actorJSON.followers)
@@ -496,7 +483,7 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe
 
   const name = actorJSON.name || actorJSON.preferredUsername
   return {
-    statusCode: requestResult.response.statusCode,
+    statusCode: requestResult.statusCode,
     result: {
       actor,
       name,

server/lib/activitypub/crawl.ts

@@ -1,27 +1,26 @@
-import { ACTIVITY_PUB, REQUEST_TIMEOUT, WEBSERVER } from '../../initializers/constants'
-import { doRequest } from '../../helpers/requests'
-import { logger } from '../../helpers/logger'
 import * as Bluebird from 'bluebird'
-import { ActivityPubOrderedCollection } from '../../../shared/models/activitypub'
 import { URL } from 'url'
+import { ActivityPubOrderedCollection } from '../../../shared/models/activitypub'
+import { logger } from '../../helpers/logger'
+import { doJSONRequest } from '../../helpers/requests'
+import { ACTIVITY_PUB, REQUEST_TIMEOUT, WEBSERVER } from '../../initializers/constants'
 
 type HandlerFunction<T> = (items: T[]) => (Promise<any> | Bluebird<any>)
 type CleanerFunction = (startedDate: Date) => (Promise<any> | Bluebird<any>)
 
-async function crawlCollectionPage <T> (uri: string, handler: HandlerFunction<T>, cleaner?: CleanerFunction) {
+async function crawlCollectionPage <T> (argUrl: string, handler: HandlerFunction<T>, cleaner?: CleanerFunction) {
-  logger.info('Crawling ActivityPub data on %s.', uri)
+  let url = argUrl
+
+  logger.info('Crawling ActivityPub data on %s.', url)
 
   const options = {
-    method: 'GET',
-    uri,
-    json: true,
     activityPub: true,
     timeout: REQUEST_TIMEOUT
   }
 
   const startDate = new Date()
 
-  const response = await doRequest<ActivityPubOrderedCollection<T>>(options)
+  const response = await doJSONRequest<ActivityPubOrderedCollection<T>>(url, options)
   const firstBody = response.body
 
   const limit = ACTIVITY_PUB.FETCH_PAGE_LIMIT
@@ -35,9 +34,9 @@ async function crawlCollectionPage <T> (uri: string, handler: HandlerFunction<T>
       const remoteHost = new URL(nextLink).host
       if (remoteHost === WEBSERVER.HOST) continue
 
-      options.uri = nextLink
+      url = nextLink
 
-      const res = await doRequest<ActivityPubOrderedCollection<T>>(options)
+      const res = await doJSONRequest<ActivityPubOrderedCollection<T>>(url, options)
       body = res.body
     } else {
       // nextLink is already the object we want
@@ -49,7 +48,7 @@ async function crawlCollectionPage <T> (uri: string, handler: HandlerFunction<T>
 
     if (Array.isArray(body.orderedItems)) {
       const items = body.orderedItems
-      logger.info('Processing %i ActivityPub items for %s.', items.length, options.uri)
+      logger.info('Processing %i ActivityPub items for %s.', items.length, url)
 
       await handler(items)
     }

server/lib/activitypub/playlist.ts

@@ -1,24 +1,24 @@
-import { PlaylistObject } from '../../../shared/models/activitypub/objects/playlist-object'
-import { crawlCollectionPage } from './crawl'
-import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
-import { isArray } from '../../helpers/custom-validators/misc'
-import { getOrCreateActorAndServerAndModel } from './actor'
-import { logger } from '../../helpers/logger'
-import { VideoPlaylistModel } from '../../models/video/video-playlist'
-import { doRequest } from '../../helpers/requests'
-import { checkUrlsSameHost } from '../../helpers/activitypub'
 import * as Bluebird from 'bluebird'
+import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
 import { PlaylistElementObject } from '../../../shared/models/activitypub/objects/playlist-element-object'
-import { getOrCreateVideoAndAccountAndChannel } from './videos'
+import { PlaylistObject } from '../../../shared/models/activitypub/objects/playlist-object'
-import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '../../helpers/custom-validators/activitypub/playlist'
-import { VideoPlaylistElementModel } from '../../models/video/video-playlist-element'
 import { VideoPlaylistPrivacy } from '../../../shared/models/videos/playlist/video-playlist-privacy.model'
+import { checkUrlsSameHost } from '../../helpers/activitypub'
+import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '../../helpers/custom-validators/activitypub/playlist'
+import { isArray } from '../../helpers/custom-validators/misc'
+import { logger } from '../../helpers/logger'
+import { doJSONRequest } from '../../helpers/requests'
+import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
 import { sequelizeTypescript } from '../../initializers/database'
-import { createPlaylistMiniatureFromUrl } from '../thumbnail'
+import { VideoPlaylistModel } from '../../models/video/video-playlist'
-import { FilteredModelAttributes } from '../../types/sequelize'
+import { VideoPlaylistElementModel } from '../../models/video/video-playlist-element'
 import { MAccountDefault, MAccountId, MVideoId } from '../../types/models'
 import { MVideoPlaylist, MVideoPlaylistId, MVideoPlaylistOwner } from '../../types/models/video/video-playlist'
-import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
+import { FilteredModelAttributes } from '../../types/sequelize'
+import { createPlaylistMiniatureFromUrl } from '../thumbnail'
+import { getOrCreateActorAndServerAndModel } from './actor'
+import { crawlCollectionPage } from './crawl'
+import { getOrCreateVideoAndAccountAndChannel } from './videos'
 
 function playlistObjectToDBAttributes (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) {
   const privacy = to.includes(ACTIVITY_PUB.PUBLIC)
@@ -56,11 +56,7 @@ async function createAccountPlaylists (playlistUrls: string[], account: MAccount
       if (exists === true) return
 
       // Fetch url
-      const { body } = await doRequest<PlaylistObject>({
+      const { body } = await doJSONRequest<PlaylistObject>(playlistUrl, { activityPub: true })
-        uri: playlistUrl,
-        json: true,
-        activityPub: true
-      })
 
       if (!isPlaylistObjectValid(body)) {
         throw new Error(`Invalid playlist object when fetch account playlists: ${JSON.stringify(body)}`)
@@ -164,12 +160,7 @@ async function resetVideoPlaylistElements (elementUrls: string[], playlist: MVid
 
   await Bluebird.map(elementUrls, async elementUrl => {
     try {
-      // Fetch url
+      const { body } = await doJSONRequest<PlaylistElementObject>(elementUrl, { activityPub: true })
-      const { body } = await doRequest<PlaylistElementObject>({
-        uri: elementUrl,
-        json: true,
-        activityPub: true
-      })
 
       if (!isPlaylistElementObjectValid(body)) throw new Error(`Invalid body in video get playlist element ${elementUrl}`)
 
@@ -199,21 +190,14 @@ async function resetVideoPlaylistElements (elementUrls: string[], playlist: MVid
 }
 
 async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusCode: number, playlistObject: PlaylistObject }> {
-  const options = {
-    uri: playlistUrl,
-    method: 'GET',
-    json: true,
-    activityPub: true
-  }
-
   logger.info('Fetching remote playlist %s.', playlistUrl)
 
-  const { response, body } = await doRequest<any>(options)
+  const { body, statusCode } = await doJSONRequest<any>(playlistUrl, { activityPub: true })
 
   if (isPlaylistObjectValid(body) === false || checkUrlsSameHost(body.id, playlistUrl) !== true) {
     logger.debug('Remote video playlist JSON is not valid.', { body })
-    return { statusCode: response.statusCode, playlistObject: undefined }
+    return { statusCode, playlistObject: undefined }
   }
 
-  return { statusCode: response.statusCode, playlistObject: body }
+  return { statusCode, playlistObject: body }
 }

server/lib/activitypub/share.ts

@@ -3,7 +3,7 @@ import { Transaction } from 'sequelize'
 import { getServerActor } from '@server/models/application/application'
 import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
 import { logger, loggerTagsFactory } from '../../helpers/logger'
-import { doRequest } from '../../helpers/requests'
+import { doJSONRequest } from '../../helpers/requests'
 import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
 import { VideoShareModel } from '../../models/video/video-share'
 import { MChannelActorLight, MVideo, MVideoAccountLight, MVideoId } from '../../types/models/video'
@@ -40,12 +40,7 @@ async function changeVideoChannelShare (
 async function addVideoShares (shareUrls: string[], video: MVideoId) {
   await Bluebird.map(shareUrls, async shareUrl => {
     try {
-      // Fetch url
+      const { body } = await doJSONRequest<any>(shareUrl, { activityPub: true })
-      const { body } = await doRequest<any>({
-        uri: shareUrl,
-        json: true,
-        activityPub: true
-      })
       if (!body || !body.actor) throw new Error('Body or body actor is invalid')
 
       const actorUrl = getAPId(body.actor)

server/lib/activitypub/video-comments.ts

@@ -1,13 +1,13 @@
-import { sanitizeAndCheckVideoCommentObject } from '../../helpers/custom-validators/activitypub/video-comments'
-import { logger } from '../../helpers/logger'
-import { doRequest } from '../../helpers/requests'
-import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
-import { VideoCommentModel } from '../../models/video/video-comment'
-import { getOrCreateActorAndServerAndModel } from './actor'
-import { getOrCreateVideoAndAccountAndChannel } from './videos'
 import * as Bluebird from 'bluebird'
 import { checkUrlsSameHost } from '../../helpers/activitypub'
+import { sanitizeAndCheckVideoCommentObject } from '../../helpers/custom-validators/activitypub/video-comments'
+import { logger } from '../../helpers/logger'
+import { doJSONRequest } from '../../helpers/requests'
+import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
+import { VideoCommentModel } from '../../models/video/video-comment'
 import { MCommentOwner, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../types/models/video'
+import { getOrCreateActorAndServerAndModel } from './actor'
+import { getOrCreateVideoAndAccountAndChannel } from './videos'
 
 type ResolveThreadParams = {
   url: string
@@ -126,11 +126,7 @@ async function resolveRemoteParentComment (params: ResolveThreadParams) {
     throw new Error('Recursion limit reached when resolving a thread')
   }
 
-  const { body } = await doRequest<any>({
+  const { body } = await doJSONRequest<any>(url, { activityPub: true })
-    uri: url,
-    json: true,
-    activityPub: true
-  })
 
   if (sanitizeAndCheckVideoCommentObject(body) === false) {
     throw new Error(`Remote video comment JSON ${url} is not valid:` + JSON.stringify(body))

server/lib/activitypub/video-rates.ts

@@ -1,26 +1,22 @@
-import { Transaction } from 'sequelize'
-import { sendLike, sendUndoDislike, sendUndoLike } from './send'
-import { VideoRateType } from '../../../shared/models/videos'
 import * as Bluebird from 'bluebird'
-import { getOrCreateActorAndServerAndModel } from './actor'
+import { Transaction } from 'sequelize'
-import { AccountVideoRateModel } from '../../models/account/account-video-rate'
+import { doJSONRequest } from '@server/helpers/requests'
+import { VideoRateType } from '../../../shared/models/videos'
+import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
 import { logger } from '../../helpers/logger'
 import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
-import { doRequest } from '../../helpers/requests'
+import { AccountVideoRateModel } from '../../models/account/account-video-rate'
-import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
-import { getVideoDislikeActivityPubUrlByLocalActor, getVideoLikeActivityPubUrlByLocalActor } from './url'
-import { sendDislike } from './send/send-dislike'
 import { MAccountActor, MActorUrl, MVideo, MVideoAccountLight, MVideoId } from '../../types/models'
+import { getOrCreateActorAndServerAndModel } from './actor'
+import { sendLike, sendUndoDislike, sendUndoLike } from './send'
+import { sendDislike } from './send/send-dislike'
+import { getVideoDislikeActivityPubUrlByLocalActor, getVideoLikeActivityPubUrlByLocalActor } from './url'
 
 async function createRates (ratesUrl: string[], video: MVideo, rate: VideoRateType) {
   await Bluebird.map(ratesUrl, async rateUrl => {
     try {
       // Fetch url
-      const { body } = await doRequest<any>({
+      const { body } = await doJSONRequest<any>(rateUrl, { activityPub: true })
-        uri: rateUrl,
-        json: true,
-        activityPub: true
-      })
       if (!body || !body.actor) throw new Error('Body or body actor is invalid')
 
       const actorUrl = getAPId(body.actor)

server/lib/activitypub/videos.ts

@@ -2,7 +2,6 @@ import * as Bluebird from 'bluebird'
 import { maxBy, minBy } from 'lodash'
 import * as magnetUtil from 'magnet-uri'
 import { basename, join } from 'path'
-import * as request from 'request'
 import { Transaction } from 'sequelize/types'
 import { TrackerModel } from '@server/models/server/tracker'
 import { VideoLiveModel } from '@server/models/video/video-live'
@@ -31,7 +30,7 @@ import { isArray } from '../../helpers/custom-validators/misc'
 import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
 import { deleteNonExistingModels, resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils'
 import { logger } from '../../helpers/logger'
-import { doRequest } from '../../helpers/requests'
+import { doJSONRequest } from '../../helpers/requests'
 import { fetchVideoByUrl, getExtFromMimetype, VideoFetchByUrlType } from '../../helpers/video'
 import {
   ACTIVITY_PUB,
@@ -115,36 +114,26 @@ async function federateVideoIfNeeded (videoArg: MVideoAPWithoutCaption, isNewVid
   }
 }
 
-async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.RequestResponse, videoObject: VideoObject }> {
+async function fetchRemoteVideo (videoUrl: string): Promise<{ statusCode: number, videoObject: VideoObject }> {
-  const options = {
-    uri: videoUrl,
-    method: 'GET',
-    json: true,
-    activityPub: true
-  }
-
   logger.info('Fetching remote video %s.', videoUrl)
 
-  const { response, body } = await doRequest<any>(options)
+  const { statusCode, body } = await doJSONRequest<any>(videoUrl, { activityPub: true })
 
   if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) {
     logger.debug('Remote video JSON is not valid.', { body })
-    return { response, videoObject: undefined }
+    return { statusCode, videoObject: undefined }
   }
 
-  return { response, videoObject: body }
+  return { statusCode, videoObject: body }
 }
 
 async function fetchRemoteVideoDescription (video: MVideoAccountLight) {
   const host = video.VideoChannel.Account.Actor.Server.host
   const path = video.getDescriptionAPIPath()
-  const options = {
+  const url = REMOTE_SCHEME.HTTP + '://' + host + path
-    uri: REMOTE_SCHEME.HTTP + '://' + host + path,
-    json: true
-  }
 
-  const { body } = await doRequest<any>(options)
+  const { body } = await doJSONRequest<any>(url)
-  return body.description ? body.description : ''
+  return body.description || ''
 }
 
 function getOrCreateVideoChannelFromVideoObject (videoObject: VideoObject) {
@@ -534,8 +523,8 @@ async function refreshVideoIfNeeded (options: {
     : await VideoModel.loadByUrlAndPopulateAccount(options.video.url)
 
   try {
-    const { response, videoObject } = await fetchRemoteVideo(video.url)
+    const { statusCode, videoObject } = await fetchRemoteVideo(video.url)
-    if (response.statusCode === HttpStatusCode.NOT_FOUND_404) {
+    if (statusCode === HttpStatusCode.NOT_FOUND_404) {
       logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url)
 
       // Video does not exist anymore

server/lib/files-cache/videos-caption-cache.ts

@@ -41,7 +41,7 @@ class VideosCaptionCache extends AbstractVideoStaticFileCache <string> {
     const remoteUrl = videoCaption.getFileUrl(video)
     const destPath = join(FILES_CACHE.VIDEO_CAPTIONS.DIRECTORY, videoCaption.filename)
 
-    await doRequestAndSaveToFile({ uri: remoteUrl }, destPath)
+    await doRequestAndSaveToFile(remoteUrl, destPath)
 
     return { isOwned: false, path: destPath }
   }

server/lib/files-cache/videos-preview-cache.ts

@@ -39,7 +39,7 @@ class VideosPreviewCache extends AbstractVideoStaticFileCache <string> {
     const destPath = join(FILES_CACHE.PREVIEWS.DIRECTORY, preview.filename)
 
     const remoteUrl = preview.getFileUrl(video)
-    await doRequestAndSaveToFile({ uri: remoteUrl }, destPath)
+    await doRequestAndSaveToFile(remoteUrl, destPath)
 
     logger.debug('Fetched remote preview %s to %s.', remoteUrl, destPath)
 

server/lib/files-cache/videos-torrent-cache.ts

@@ -41,7 +41,7 @@ class VideosTorrentCache extends AbstractVideoStaticFileCache <string> {
     const remoteUrl = file.getRemoteTorrentUrl(video)
     const destPath = join(FILES_CACHE.TORRENTS.DIRECTORY, file.torrentFilename)
 
-    await doRequestAndSaveToFile({ uri: remoteUrl }, destPath)
+    await doRequestAndSaveToFile(remoteUrl, destPath)
 
     const downloadName = `${video.name}-${file.resolution}p.torrent`
 

server/lib/hls.ts

@@ -135,7 +135,7 @@ function downloadPlaylistSegments (playlistUrl: string, destinationDir: string,
         const destPath = join(tmpDirectory, basename(fileUrl))
 
         const bodyKBLimit = 10 * 1000 * 1000 // 10GB
-        await doRequestAndSaveToFile({ uri: fileUrl }, destPath, bodyKBLimit)
+        await doRequestAndSaveToFile(fileUrl, destPath, { bodyKBLimit })
       }
 
       clearTimeout(timer)
@@ -156,7 +156,7 @@ function downloadPlaylistSegments (playlistUrl: string, destinationDir: string,
   }
 
   async function fetchUniqUrls (playlistUrl: string) {
-    const { body } = await doRequest<string>({ uri: playlistUrl })
+    const { body } = await doRequest(playlistUrl)
 
     if (!body) return []
 

server/lib/job-queue/handlers/activitypub-cleaner.ts

@@ -7,7 +7,7 @@ import {
   isLikeActivityValid
 } from '@server/helpers/custom-validators/activitypub/activity'
 import { sanitizeAndCheckVideoCommentObject } from '@server/helpers/custom-validators/activitypub/video-comments'
-import { doRequest } from '@server/helpers/requests'
+import { doJSONRequest } from '@server/helpers/requests'
 import { AP_CLEANER_CONCURRENCY } from '@server/initializers/constants'
 import { VideoModel } from '@server/models/video/video'
 import { VideoCommentModel } from '@server/models/video/video-comment'
@@ -81,15 +81,10 @@ async function updateObjectIfNeeded <T> (
   updater: (url: string, newUrl: string) => Promise<T>,
   deleter: (url: string) => Promise<T>
 ): Promise<{ data: T, status: 'deleted' | 'updated' } | null> {
-  // Fetch url
+  const { statusCode, body } = await doJSONRequest<any>(url, { activityPub: true })
-  const { response, body } = await doRequest<any>({
-    uri: url,
-    json: true,
-    activityPub: true
-  })
 
   // Does not exist anymore, remove entry
-  if (response.statusCode === HttpStatusCode.NOT_FOUND_404) {
+  if (statusCode === HttpStatusCode.NOT_FOUND_404) {
     logger.info('Removing remote AP object %s.', url)
     const data = await deleter(url)
 

server/lib/job-queue/handlers/activitypub-http-broadcast.ts

@@ -16,8 +16,7 @@ async function processActivityPubHttpBroadcast (job: Bull.Job) {
   const httpSignatureOptions = await buildSignedRequestOptions(payload)
 
   const options = {
-    method: 'POST',
+    method: 'POST' as 'POST',
-    uri: '',
     json: body,
     httpSignature: httpSignatureOptions,
     timeout: REQUEST_TIMEOUT,
@@ -28,7 +27,7 @@ async function processActivityPubHttpBroadcast (job: Bull.Job) {
   const goodUrls: string[] = []
 
   await Bluebird.map(payload.uris, uri => {
-    return doRequest(Object.assign({}, options, { uri }))
+    return doRequest(uri, options)
       .then(() => goodUrls.push(uri))
       .catch(() => badUrls.push(uri))
   }, { concurrency: BROADCAST_CONCURRENCY })

server/lib/job-queue/handlers/activitypub-http-unicast.ts

@@ -16,8 +16,7 @@ async function processActivityPubHttpUnicast (job: Bull.Job) {
   const httpSignatureOptions = await buildSignedRequestOptions(payload)
 
   const options = {
-    method: 'POST',
+    method: 'POST' as 'POST',
-    uri,
     json: body,
     httpSignature: httpSignatureOptions,
     timeout: REQUEST_TIMEOUT,
@@ -25,7 +24,7 @@ async function processActivityPubHttpUnicast (job: Bull.Job) {
   }
 
   try {
-    await doRequest(options)
+    await doRequest(uri, options)
     ActorFollowScoreCache.Instance.updateActorFollowsScore([ uri ], [])
   } catch (err) {
     ActorFollowScoreCache.Instance.updateActorFollowsScore([], [ uri ])

server/lib/job-queue/handlers/utils/activitypub-http-utils.ts

@@ -6,21 +6,24 @@ import { getServerActor } from '@server/models/application/application'
 import { buildDigest } from '@server/helpers/peertube-crypto'
 import { ContextType } from '@shared/models/activitypub/context'
 
-type Payload = { body: any, contextType?: ContextType, signatureActorId?: number }
+type Payload <T> = { body: T, contextType?: ContextType, signatureActorId?: number }
 
-async function computeBody (payload: Payload) {
+async function computeBody <T> (
+  payload: Payload<T>
+): Promise<T | T & { type: 'RsaSignature2017', creator: string, created: string }> {
   let body = payload.body
 
   if (payload.signatureActorId) {
     const actorSignature = await ActorModel.load(payload.signatureActorId)
     if (!actorSignature) throw new Error('Unknown signature actor id.')
+
     body = await buildSignedActivity(actorSignature, payload.body, payload.contextType)
   }
 
   return body
 }
 
-async function buildSignedRequestOptions (payload: Payload) {
+async function buildSignedRequestOptions (payload: Payload<any>) {
   let actor: MActor | null
 
   if (payload.signatureActorId) {

server/lib/plugins/plugin-index.ts

@@ -1,22 +1,22 @@
-import { doRequest } from '../../helpers/requests'
+import { sanitizeUrl } from '@server/helpers/core-utils'
-import { CONFIG } from '../../initializers/config'
+import { ResultList } from '../../../shared/models'
+import { PeertubePluginIndexList } from '../../../shared/models/plugins/peertube-plugin-index-list.model'
+import { PeerTubePluginIndex } from '../../../shared/models/plugins/peertube-plugin-index.model'
 import {
   PeertubePluginLatestVersionRequest,
   PeertubePluginLatestVersionResponse
 } from '../../../shared/models/plugins/peertube-plugin-latest-version.model'
-import { PeertubePluginIndexList } from '../../../shared/models/plugins/peertube-plugin-index-list.model'
+import { logger } from '../../helpers/logger'
-import { ResultList } from '../../../shared/models'
+import { doJSONRequest } from '../../helpers/requests'
-import { PeerTubePluginIndex } from '../../../shared/models/plugins/peertube-plugin-index.model'
+import { CONFIG } from '../../initializers/config'
+import { PEERTUBE_VERSION } from '../../initializers/constants'
 import { PluginModel } from '../../models/server/plugin'
 import { PluginManager } from './plugin-manager'
-import { logger } from '../../helpers/logger'
-import { PEERTUBE_VERSION } from '../../initializers/constants'
-import { sanitizeUrl } from '@server/helpers/core-utils'
 
 async function listAvailablePluginsFromIndex (options: PeertubePluginIndexList) {
   const { start = 0, count = 20, search, sort = 'npmName', pluginType } = options
 
-  const qs: PeertubePluginIndexList = {
+  const searchParams: PeertubePluginIndexList & Record<string, string | number> = {
     start,
     count,
     sort,
@@ -28,7 +28,7 @@ async function listAvailablePluginsFromIndex (options: PeertubePluginIndexList)
   const uri = CONFIG.PLUGINS.INDEX.URL + '/api/v1/plugins'
 
   try {
-    const { body } = await doRequest<any>({ uri, qs, json: true })
+    const { body } = await doJSONRequest<any>(uri, { searchParams })
 
     logger.debug('Got result from PeerTube index.', { body })
 
@@ -58,7 +58,11 @@ async function getLatestPluginsVersion (npmNames: string[]): Promise<PeertubePlu
 
   const uri = sanitizeUrl(CONFIG.PLUGINS.INDEX.URL) + '/api/v1/plugins/latest-version'
 
-  const { body } = await doRequest<any>({ uri, body: bodyRequest, json: true, method: 'POST' })
+  const options = {
+    json: bodyRequest,
+    method: 'POST' as 'POST'
+  }
+  const { body } = await doJSONRequest<PeertubePluginLatestVersionResponse>(uri, options)
 
   return body
 }

server/lib/schedulers/auto-follow-index-instances.ts

@@ -1,5 +1,5 @@
 import { chunk } from 'lodash'
-import { doRequest } from '@server/helpers/requests'
+import { doJSONRequest } from '@server/helpers/requests'
 import { JobQueue } from '@server/lib/job-queue'
 import { ActorFollowModel } from '@server/models/activitypub/actor-follow'
 import { getServerActor } from '@server/models/application/application'
@@ -34,12 +34,12 @@ export class AutoFollowIndexInstances extends AbstractScheduler {
     try {
       const serverActor = await getServerActor()
 
-      const qs = { count: 1000 }
+      const searchParams = { count: 1000 }
-      if (this.lastCheck) Object.assign(qs, { since: this.lastCheck.toISOString() })
+      if (this.lastCheck) Object.assign(searchParams, { since: this.lastCheck.toISOString() })
 
       this.lastCheck = new Date()
 
-      const { body } = await doRequest<any>({ uri: indexUrl, qs, json: true })
+      const { body } = await doJSONRequest<any>(indexUrl, { searchParams })
       if (!body.data || Array.isArray(body.data) === false) {
         logger.error('Cannot auto follow instances of index %s. Please check the auto follow URL.', indexUrl, { body })
         return

server/tests/api/activitypub/security.ts

@@ -79,9 +79,9 @@ describe('Test ActivityPub security', function () {
         Digest: buildDigest({ hello: 'coucou' })
       }
 
-      const { response } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers)
+      const { statusCode } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers)
 
-      expect(response.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
+      expect(statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
     })
 
     it('Should fail with an invalid date', async function () {
@@ -89,9 +89,9 @@ describe('Test ActivityPub security', function () {
       const headers = buildGlobalHeaders(body)
       headers['date'] = 'Wed, 21 Oct 2015 07:28:00 GMT'
 
-      const { response } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers)
+      const { statusCode } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers)
 
-      expect(response.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
+      expect(statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
     })
 
     it('Should fail with bad keys', async function () {
@@ -101,9 +101,9 @@ describe('Test ActivityPub security', function () {
       const body = activityPubContextify(getAnnounceWithoutContext(servers[1]))
       const headers = buildGlobalHeaders(body)
 
-      const { response } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers)
+      const { statusCode } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers)
 
-      expect(response.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
+      expect(statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
     })
 
     it('Should reject requests without appropriate signed headers', async function () {
@@ -123,8 +123,8 @@ describe('Test ActivityPub security', function () {
       for (const badHeaders of badHeadersMatrix) {
         signatureOptions.headers = badHeaders
 
-        const { response } = await makePOSTAPRequest(url, body, signatureOptions, headers)
+        const { statusCode } = await makePOSTAPRequest(url, body, signatureOptions, headers)
-        expect(response.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
+        expect(statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
       }
     })
 
@@ -132,9 +132,9 @@ describe('Test ActivityPub security', function () {
       const body = activityPubContextify(getAnnounceWithoutContext(servers[1]))
       const headers = buildGlobalHeaders(body)
 
-      const { response } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers)
+      const { statusCode } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers)
 
-      expect(response.statusCode).to.equal(HttpStatusCode.NO_CONTENT_204)
+      expect(statusCode).to.equal(HttpStatusCode.NO_CONTENT_204)
     })
 
     it('Should refresh the actor keys', async function () {
@@ -150,9 +150,9 @@ describe('Test ActivityPub security', function () {
       const body = activityPubContextify(getAnnounceWithoutContext(servers[1]))
       const headers = buildGlobalHeaders(body)
 
-      const { response } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers)
+      const { statusCode } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers)
 
-      expect(response.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
+      expect(statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
     })
   })
 
@@ -183,9 +183,9 @@ describe('Test ActivityPub security', function () {
 
       const headers = buildGlobalHeaders(signedBody)
 
-      const { response } = await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers)
+      const { statusCode } = await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers)
 
-      expect(response.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
+      expect(statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
     })
 
     it('Should fail with an altered body', async function () {
@@ -204,9 +204,9 @@ describe('Test ActivityPub security', function () {
 
       const headers = buildGlobalHeaders(signedBody)
 
-      const { response } = await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers)
+      const { statusCode } = await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers)
 
-      expect(response.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
+      expect(statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
     })
 
     it('Should succeed with a valid signature', async function () {
@@ -220,9 +220,9 @@ describe('Test ActivityPub security', function () {
 
       const headers = buildGlobalHeaders(signedBody)
 
-      const { response } = await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers)
+      const { statusCode } = await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers)
 
-      expect(response.statusCode).to.equal(HttpStatusCode.NO_CONTENT_204)
+      expect(statusCode).to.equal(HttpStatusCode.NO_CONTENT_204)
     })
 
     it('Should refresh the actor keys', async function () {
@@ -243,9 +243,9 @@ describe('Test ActivityPub security', function () {
 
       const headers = buildGlobalHeaders(signedBody)
 
-      const { response } = await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers)
+      const { statusCode } = await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers)
 
-      expect(response.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
+      expect(statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
     })
   })
 

server/tests/api/server/handle-down.ts

@@ -348,8 +348,8 @@ describe('Test handle downs', function () {
 
     for (let i = 0; i < 3; i++) {
       await getVideo(servers[1].url, videoIdsServer1[i])
-      await wait(1000)
       await waitJobs([ servers[1] ])
+      await wait(1500)
     }
 
     for (const id of videoIdsServer1) {

server/tests/helpers/request.ts

@@ -1,11 +1,11 @@
 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
 
 import 'mocha'
-import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests'
-import { get4KFileUrl, root, wait } from '../../../shared/extra-utils'
-import { join } from 'path'
-import { pathExists, remove } from 'fs-extra'
 import { expect } from 'chai'
+import { pathExists, remove } from 'fs-extra'
+import { join } from 'path'
+import { get4KFileUrl, root, wait } from '../../../shared/extra-utils'
+import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests'
 
 describe('Request helpers', function () {
   const destPath1 = join(root(), 'test-output-1.txt')
@@ -13,7 +13,7 @@ describe('Request helpers', function () {
 
   it('Should throw an error when the bytes limit is exceeded for request', async function () {
     try {
-      await doRequest({ uri: get4KFileUrl() }, 3)
+      await doRequest(get4KFileUrl(), { bodyKBLimit: 3 })
     } catch {
       return
     }
@@ -23,7 +23,7 @@ describe('Request helpers', function () {
 
   it('Should throw an error when the bytes limit is exceeded for request and save file', async function () {
     try {
-      await doRequestAndSaveToFile({ uri: get4KFileUrl() }, destPath1, 3)
+      await doRequestAndSaveToFile(get4KFileUrl(), destPath1, { bodyKBLimit: 3 })
     } catch {
 
       await wait(500)
@@ -35,8 +35,8 @@ describe('Request helpers', function () {
   })
 
   it('Should succeed if the file is below the limit', async function () {
-    await doRequest({ uri: get4KFileUrl() }, 5)
+    await doRequest(get4KFileUrl(), { bodyKBLimit: 5 })
-    await doRequestAndSaveToFile({ uri: get4KFileUrl() }, destPath2, 5)
+    await doRequestAndSaveToFile(get4KFileUrl(), destPath2, { bodyKBLimit: 5 })
 
     expect(await pathExists(destPath2)).to.be.true
   })

shared/extra-utils/requests/activitypub.ts

@@ -5,14 +5,13 @@ import { activityPubContextify } from '../../../server/helpers/activitypub'
 
 function makePOSTAPRequest (url: string, body: any, httpSignature: any, headers: any) {
   const options = {
-    method: 'POST',
+    method: 'POST' as 'POST',
-    uri: url,
     json: body,
     httpSignature,
     headers
   }
 
-  return doRequest(options)
+  return doRequest(url, options)
 }
 
 async function makeFollowRequest (to: { url: string }, by: { url: string, privateKey }) {

shared/models/server/job.model.ts

@@ -59,7 +59,7 @@ export type ActivitypubHttpFetcherPayload = {
 export type ActivitypubHttpUnicastPayload = {
   uri: string
   signatureActorId?: number
-  body: any
+  body: object
   contextType?: ContextType
 }
 

yarn.lock

@@ -3925,6 +3925,23 @@ globby@^11.0.1:
     merge2 "^1.3.0"
     slash "^3.0.0"
 
+got@^11.8.2, got@~11.8.1:
+  version "11.8.2"
+  resolved "https://registry.yarnpkg.com/got/-/got-11.8.2.tgz#7abb3959ea28c31f3576f1576c1effce23f33599"
+  integrity sha512-D0QywKgIe30ODs+fm8wMZiAcZjypcCodPNuMz5H9Mny7RJ+IjJ10BdmGW7OM7fHXP+O7r6ZwapQ/YQmMSvB0UQ==
+  dependencies:
+    "@sindresorhus/is" "^4.0.0"
+    "@szmarczak/http-timer" "^4.0.5"
+    "@types/cacheable-request" "^6.0.1"
+    "@types/responselike" "^1.0.0"
+    cacheable-lookup "^5.0.3"
+    cacheable-request "^7.0.1"
+    decompress-response "^6.0.0"
+    http2-wrapper "^1.0.0-beta.5.2"
+    lowercase-keys "^2.0.0"
+    p-cancelable "^2.0.0"
+    responselike "^2.0.0"
+
 got@^9.6.0:
   version "9.6.0"
   resolved "https://registry.yarnpkg.com/got/-/got-9.6.0.tgz#edf45e7d67f99545705de1f7bbeeeb121765ed85"
@@ -3942,23 +3959,6 @@ got@^9.6.0:
     to-readable-stream "^1.0.0"
     url-parse-lax "^3.0.0"
 
-got@~11.8.1:
-  version "11.8.2"
-  resolved "https://registry.yarnpkg.com/got/-/got-11.8.2.tgz#7abb3959ea28c31f3576f1576c1effce23f33599"
-  integrity sha512-D0QywKgIe30ODs+fm8wMZiAcZjypcCodPNuMz5H9Mny7RJ+IjJ10BdmGW7OM7fHXP+O7r6ZwapQ/YQmMSvB0UQ==
-  dependencies:
-    "@sindresorhus/is" "^4.0.0"
-    "@szmarczak/http-timer" "^4.0.5"
-    "@types/cacheable-request" "^6.0.1"
-    "@types/responselike" "^1.0.0"
-    cacheable-lookup "^5.0.3"
-    cacheable-request "^7.0.1"
-    decompress-response "^6.0.0"
-    http2-wrapper "^1.0.0-beta.5.2"
-    lowercase-keys "^2.0.0"
-    p-cancelable "^2.0.0"
-    responselike "^2.0.0"
-
 graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.0:
   version "4.2.6"
   resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.6.tgz#ff040b2b0853b23c3d31027523706f1885d76bee"
@@ -4167,7 +4167,7 @@ http-proxy-agent@^4.0.1:
     agent-base "6"
     debug "4"
 
-http-signature@1.3.5, http-signature@~1.2.0:
+http-signature@1.3.5:
   version "1.3.5"
   resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.3.5.tgz#9f19496ffbf3227298d7b5f156e0e1a948678683"
   integrity sha512-NwoTQYSJoFt34jSBbwzDHDofoA61NGXzu6wXh95o1Ry62EnmKjXb/nR/RknLeZ3G/uGwrlKNY2z7uPt+Cdl7Tw==
@@ -4176,6 +4176,15 @@ http-signature@1.3.5, http-signature@~1.2.0:
     jsprim "^1.2.2"
     sshpk "^1.14.1"
 
+http-signature@~1.2.0:
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1"
+  integrity sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=
+  dependencies:
+    assert-plus "^1.0.0"
+    jsprim "^1.2.2"
+    sshpk "^1.7.0"
+
 http2-wrapper@^1.0.0-beta.5.2:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/http2-wrapper/-/http2-wrapper-1.0.3.tgz#b8f55e0c1f25d4ebd08b3b0c2c079f9590800b3d"
@@ -7060,7 +7069,7 @@ render-media@^4.1.0:
     stream-to-blob-url "^3.0.2"
     videostream "^3.2.2"
 
-request@^2.81.0, request@^2.88.0:
+request@^2.88.0:
   version "2.88.2"
   resolved "https://registry.yarnpkg.com/request/-/request-2.88.2.tgz#d73c918731cb5a87da047e207234146f664d12b3"
   integrity sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==
@@ -7700,7 +7709,7 @@ srt-to-vtt@^1.1.2:
     through2 "^0.6.3"
     to-utf-8 "^1.2.0"
 
-sshpk@^1.14.1:
+sshpk@^1.14.1, sshpk@^1.7.0:
   version "1.16.1"
   resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.16.1.tgz#fb661c0bef29b39db40769ee39fa70093d6f6877"
   integrity sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg==