interesting
/
PeerTube
mirror of https://github.com/Chocobozzz/PeerTube.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Use got instead of request

This commit is contained in:
Chocobozzz 2021-03-08 14:24:11 +01:00
parent 71926aae07
commit db4b15f21f
No known key found for this signature in database
GPG Key ID: 583A612D890159BE
32 changed files with 346 additions and 328 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
package.json
View File

@ -83,8 +83,7 @@
"sass-lint": "sass-lint"
},
"resolutions": {
"oauth2-server": "3.1.0-beta.1",
"http-signature": "1.3.5"
"oauth2-server": "3.1.0-beta.1"
},
"dependencies": {
"apicache": "1.6.2",
@ -111,6 +110,7 @@
"flat": "^5.0.0",
"fluent-ffmpeg": "^2.1.0",
"fs-extra": "^9.0.0",
"got": "^11.8.2",
"helmet": "^4.1.0",
"http-signature": "1.3.5",
"ip-anonymize": "^0.1.0",
@ -140,7 +140,6 @@
"pug": "^3.0.0",
"redis": "^3.0.2",
"reflect-metadata": "^0.1.12",
"request": "^2.81.0",
"sanitize-html": "2.x",
"scripty": "^2.0.0",
"sequelize": "6.5.0",

10
server/controllers/api/search.ts
View File

@ -1,6 +1,6 @@
import * as express from 'express'
import { sanitizeUrl } from '@server/helpers/core-utils'
import { doRequest } from '@server/helpers/requests'
import { doJSONRequest } from '@server/helpers/requests'
import { CONFIG } from '@server/initializers/config'
import { getOrCreateVideoAndAccountAndChannel } from '@server/lib/activitypub/videos'
import { AccountBlocklistModel } from '@server/models/account/account-blocklist'
@ -94,9 +94,9 @@ async function searchVideoChannelsIndex (query: VideoChannelsSearchQuery, res: e
try {
logger.debug('Doing video channels search index request on %s.', url, { body })
const searchIndexResult = await doRequest<ResultList<VideoChannel>>({ uri: url, body, json: true })
const searchIndexResult = await doJSONRequest<ResultList<VideoChannel>>(url, { json: body })
return res.json(searchIndexResult.body)
return res.json(searchIndexResult)
} catch (err) {
logger.warn('Cannot use search index to make video channels search.', { err })
@ -186,9 +186,9 @@ async function searchVideosIndex (query: VideosSearchQuery, res: express.Respons
try {
logger.debug('Doing videos search index request on %s.', url, { body })
const searchIndexResult = await doRequest<ResultList<Video>>({ uri: url, body, json: true })
const searchIndexResult = await doJSONRequest<ResultList<Video>>(url, { json: body })
return res.json(searchIndexResult.body)
return res.json(searchIndexResult)
} catch (err) {
logger.warn('Cannot use search index to make video search.', { err })

5
server/helpers/activitypub.ts
View File

@ -3,7 +3,6 @@ import { URL } from 'url'
import validator from 'validator'
import { ContextType } from '@shared/models/activitypub/context'
import { ResultList } from '../../shared/models'
import { Activity } from '../../shared/models/activitypub'
import { ACTIVITY_PUB, REMOTE_SCHEME } from '../initializers/constants'
import { MActor, MVideoWithHost } from '../types/models'
import { pageToStartAndCount } from './core-utils'
@ -182,10 +181,10 @@ async function activityPubCollectionPagination (
}
function buildSignedActivity (byActor: MActor, data: Object, contextType?: ContextType) {
function buildSignedActivity <T> (byActor: MActor, data: T, contextType?: ContextType) {
const activity = activityPubContextify(data, contextType)
return signJsonLDObject(byActor, activity) as Promise<Activity>
return signJsonLDObject(byActor, activity)
}
function getAPId (activity: string | { id: string }) {

6
server/helpers/core-utils.ts
View File

@ -10,7 +10,9 @@ import { BinaryToTextEncoding, createHash, randomBytes } from 'crypto'
import { truncate } from 'lodash'
import { basename, isAbsolute, join, resolve } from 'path'
import * as pem from 'pem'
import { pipeline } from 'stream'
import { URL } from 'url'
import { promisify } from 'util'
const objectConverter = (oldObject: any, keyConverter: (e: string) => string, valueConverter: (e: any) => any) => {
if (!oldObject || typeof oldObject !== 'object') {
@ -254,6 +256,7 @@ const createPrivateKey = promisify1<number, { key: string }>(pem.createPrivateKe
const getPublicKey = promisify1<string, { publicKey: string }>(pem.getPublicKey)
const execPromise2 = promisify2<string, any, string>(exec)
const execPromise = promisify1<string, string>(exec)
const pipelinePromise = promisify(pipeline)
// ---------------------------------------------------------------------------
@ -284,5 +287,6 @@ export {
createPrivateKey,
getPublicKey,
execPromise2,
execPromise
execPromise,
pipelinePromise
}

2
server/helpers/custom-validators/activitypub/activity.ts
View File

@ -41,7 +41,7 @@ const activityCheckers: { [ P in ActivityType ]: (activity: Activity) => boolean
}
function isActivityValid (activity: any) {
const checker = activityCheckers[activity.tswype]
const checker = activityCheckers[activity.type]
// Unknown activity type
if (!checker) return false

2
server/helpers/peertube-crypto.ts
View File

@ -84,7 +84,7 @@ async function isJsonLDRSA2017Verified (fromActor: MActor, signedDocument: any)
return verify.verify(fromActor.publicKey, signedDocument.signature.signatureValue, 'base64')
}
async function signJsonLDObject (byActor: MActor, data: any) {
async function signJsonLDObject <T> (byActor: MActor, data: T) {
const signature = {
type: 'RsaSignature2017',
creator: byActor.url,

181
server/helpers/requests.ts
View File

@ -1,58 +1,124 @@
import * as Bluebird from 'bluebird'
import { createWriteStream, remove } from 'fs-extra'
import * as request from 'request'
import { ACTIVITY_PUB, PEERTUBE_VERSION, WEBSERVER } from '../initializers/constants'
import { processImage } from './image-utils'
import got, { CancelableRequest, Options as GotOptions } from 'got'
import { join } from 'path'
import { logger } from './logger'
import { CONFIG } from '../initializers/config'
import { ACTIVITY_PUB, PEERTUBE_VERSION, WEBSERVER } from '../initializers/constants'
import { pipelinePromise } from './core-utils'
import { processImage } from './image-utils'
import { logger } from './logger'
function doRequest <T> (
requestOptions: request.CoreOptions & request.UriOptions & { activityPub?: boolean },
bodyKBLimit = 1000 // 1MB
): Bluebird<{ response: request.RequestResponse, body: T }> {
if (!(requestOptions.headers)) requestOptions.headers = {}
requestOptions.headers['User-Agent'] = getUserAgent()
const httpSignature = require('http-signature')
if (requestOptions.activityPub === true) {
requestOptions.headers['accept'] = ACTIVITY_PUB.ACCEPT_HEADER
type PeerTubeRequestOptions = {
activityPub?: boolean
bodyKBLimit?: number // 1MB
httpSignature?: {
algorithm: string
authorizationHeaderName: string
keyId: string
key: string
headers: string[]
}
jsonResponse?: boolean
} & Pick<GotOptions, 'headers' | 'json' | 'method' | 'searchParams'>
return new Bluebird<{ response: request.RequestResponse, body: T }>((res, rej) => {
request(requestOptions, (err, response, body) => err ? rej(err) : res({ response, body }))
.on('data', onRequestDataLengthCheck(bodyKBLimit))
})
const peertubeGot = got.extend({
headers: {
'user-agent': getUserAgent()
},
handlers: [
(options, next) => {
const promiseOrStream = next(options) as CancelableRequest<any>
const bodyKBLimit = options.context?.bodyKBLimit
if (!bodyKBLimit) throw new Error('No KB limit for this request')
/* eslint-disable @typescript-eslint/no-floating-promises */
promiseOrStream.on('downloadProgress', progress => {
if (progress.transferred * 1000 > bodyKBLimit && progress.percent !== 1) {
promiseOrStream.cancel(`Exceeded the download limit of ${bodyKBLimit} bytes`)
}
})
return promiseOrStream
}
],
hooks: {
beforeRequest: [
options => {
const headers = options.headers || {}
headers['host'] = options.url.host
},
options => {
const httpSignatureOptions = options.context?.httpSignature
if (httpSignatureOptions) {
const method = options.method ?? 'GET'
const path = options.path ?? options.url.pathname
if (!method || !path) {
throw new Error(`Cannot sign request without method (${method}) or path (${path}) ${options}`)
}
httpSignature.signRequest({
getHeader: function (header) {
return options.headers[header]
},
setHeader: function (header, value) {
options.headers[header] = value
},
method,
path
}, httpSignatureOptions)
}
}
]
}
})
function doRequest (url: string, options: PeerTubeRequestOptions = {}) {
const gotOptions = buildGotOptions(options)
return peertubeGot(url, gotOptions)
.catch(err => { throw buildRequestError(err) })
}
function doRequestAndSaveToFile (
requestOptions: request.CoreOptions & request.UriOptions,
function doJSONRequest <T> (url: string, options: PeerTubeRequestOptions = {}) {
const gotOptions = buildGotOptions(options)
return peertubeGot<T>(url, { ...gotOptions, responseType: 'json' })
.catch(err => { throw buildRequestError(err) })
}
async function doRequestAndSaveToFile (
url: string,
destPath: string,
bodyKBLimit = 10000 // 10MB
options: PeerTubeRequestOptions = {}
) {
if (!requestOptions.headers) requestOptions.headers = {}
requestOptions.headers['User-Agent'] = getUserAgent()
const gotOptions = buildGotOptions(options)
return new Bluebird<void>((res, rej) => {
const file = createWriteStream(destPath)
file.on('finish', () => res())
const outFile = createWriteStream(destPath)
request(requestOptions)
.on('data', onRequestDataLengthCheck(bodyKBLimit))
.on('error', err => {
file.close()
try {
await pipelinePromise(
peertubeGot.stream(url, gotOptions),
outFile
)
} catch (err) {
remove(destPath)
.catch(err => logger.error('Cannot remove %s after request failure.', destPath, { err }))
remove(destPath)
.catch(err => logger.error('Cannot remove %s after request failure.', destPath, { err }))
return rej(err)
})
.pipe(file)
})
throw buildRequestError(err)
}
}
async function downloadImage (url: string, destDir: string, destName: string, size: { width: number, height: number }) {
const tmpPath = join(CONFIG.STORAGE.TMP_DIR, 'pending-' + destName)
await doRequestAndSaveToFile({ method: 'GET', uri: url }, tmpPath)
await doRequestAndSaveToFile(url, tmpPath)
const destPath = join(destDir, destName)
@ -73,24 +139,43 @@ function getUserAgent () {
export {
doRequest,
doJSONRequest,
doRequestAndSaveToFile,
downloadImage
}
// ---------------------------------------------------------------------------
// Thanks to https://github.com/request/request/issues/2470#issuecomment-268929907 <3
function onRequestDataLengthCheck (bodyKBLimit: number) {
let bufferLength = 0
const bytesLimit = bodyKBLimit * 1000
function buildGotOptions (options: PeerTubeRequestOptions) {
const { activityPub, bodyKBLimit = 1000 } = options
return function (chunk) {
bufferLength += chunk.length
if (bufferLength > bytesLimit) {
this.abort()
const context = { bodyKBLimit, httpSignature: options.httpSignature }
const error = new Error(`Response was too large - aborted after ${bytesLimit} bytes.`)
this.emit('error', error)
}
let headers = options.headers || {}
headers = { ...headers, date: new Date().toUTCString() }
if (activityPub) {
headers = { ...headers, accept: ACTIVITY_PUB.ACCEPT_HEADER }
}
return {
method: options.method,
json: options.json,
searchParams: options.searchParams,
headers,
context
}
}
function buildRequestError (error: any) {
const newError = new Error(error.message)
newError.name = error.name
newError.stack = error.stack
if (error.response?.body) {
error.responseBody = error.response.body
}
return newError
}

67
server/helpers/youtube-dl.ts
View File

@ -1,13 +1,13 @@
import { createWriteStream } from 'fs'
import { ensureDir, move, pathExists, remove, writeFile } from 'fs-extra'
import got from 'got'
import { join } from 'path'
import * as request from 'request'
import { CONFIG } from '@server/initializers/config'
import { HttpStatusCode } from '../../shared/core-utils/miscs/http-error-codes'
import { VideoResolution } from '../../shared/models/videos'
import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES } from '../initializers/constants'
import { getEnabledResolutions } from '../lib/video-transcoding'
import { peertubeTruncate, root } from './core-utils'
import { peertubeTruncate, pipelinePromise, root } from './core-utils'
import { isVideoFileExtnameValid } from './custom-validators/videos'
import { logger } from './logger'
import { generateVideoImportTmpPath } from './utils'
@ -195,55 +195,32 @@ async function updateYoutubeDLBinary () {
await ensureDir(binDirectory)
return new Promise<void>(res => {
request.get(url, { followRedirect: false }, (err, result) => {
if (err) {
logger.error('Cannot update youtube-dl.', { err })
return res()
}
try {
const result = await got(url, { followRedirect: false })
if (result.statusCode !== HttpStatusCode.FOUND_302) {
logger.error('youtube-dl update error: did not get redirect for the latest version link. Status %d', result.statusCode)
return res()
}
if (result.statusCode !== HttpStatusCode.FOUND_302) {
logger.error('youtube-dl update error: did not get redirect for the latest version link. Status %d', result.statusCode)
return
}
const url = result.headers.location
const downloadFile = request.get(url)
const newVersion = /yt-dl\.org\/downloads\/(\d{4}\.\d\d\.\d\d(\.\d)?)\/youtube-dl/.exec(url)[1]
const newUrl = result.headers.location
const newVersion = /yt-dl\.org\/downloads\/(\d{4}\.\d\d\.\d\d(\.\d)?)\/youtube-dl/.exec(newUrl)[1]
downloadFile.on('response', result => {
if (result.statusCode !== HttpStatusCode.OK_200) {
logger.error('Cannot update youtube-dl: new version response is not 200, it\'s %d.', result.statusCode)
return res()
}
const downloadFileStream = got.stream(newUrl)
const writeStream = createWriteStream(bin, { mode: 493 })
const writeStream = createWriteStream(bin, { mode: 493 }).on('error', err => {
logger.error('youtube-dl update error in write stream', { err })
return res()
})
await pipelinePromise(
downloadFileStream,
writeStream
)
downloadFile.pipe(writeStream)
})
const details = JSON.stringify({ version: newVersion, path: bin, exec: 'youtube-dl' })
await writeFile(detailsPath, details, { encoding: 'utf8' })
downloadFile.on('error', err => {
logger.error('youtube-dl update error.', { err })
return res()
})
downloadFile.on('end', () => {
const details = JSON.stringify({ version: newVersion, path: bin, exec: 'youtube-dl' })
writeFile(detailsPath, details, { encoding: 'utf8' }, err => {
if (err) {
logger.error('youtube-dl update error: cannot write details.', { err })
return res()
}
logger.info('youtube-dl updated to version %s.', newVersion)
return res()
})
})
})
})
logger.info('youtube-dl updated to version %s.', newVersion)
} catch (err) {
logger.error('Cannot update youtube-dl.', { err })
}
}
async function safeGetYoutubeDL () {

2
server/initializers/constants.ts
View File

@ -29,7 +29,7 @@ const LAST_MIGRATION_VERSION = 610
// ---------------------------------------------------------------------------
const API_VERSION = 'v1'
const PEERTUBE_VERSION = require(join(root(), 'package.json')).version
const PEERTUBE_VERSION: string = require(join(root(), 'package.json')).version
const PAGINATION = {
GLOBAL: {

41
server/lib/activitypub/actor.ts
View File

@ -1,26 +1,28 @@
import * as Bluebird from 'bluebird'
import { extname } from 'path'
import { Op, Transaction } from 'sequelize'
import { URL } from 'url'
import { v4 as uuidv4 } from 'uuid'
import { getServerActor } from '@server/models/application/application'
import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
import { ActivityPubActor, ActivityPubActorType, ActivityPubOrderedCollection } from '../../../shared/models/activitypub'
import { ActivityPubAttributedTo } from '../../../shared/models/activitypub/objects'
import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor'
import { sanitizeAndCheckActorObject } from '../../helpers/custom-validators/activitypub/actor'
import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils'
import { logger } from '../../helpers/logger'
import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto'
import { doRequest } from '../../helpers/requests'
import { doJSONRequest } from '../../helpers/requests'
import { getUrlFromWebfinger } from '../../helpers/webfinger'
import { MIMETYPES, WEBSERVER } from '../../initializers/constants'
import { sequelizeTypescript } from '../../initializers/database'
import { AccountModel } from '../../models/account/account'
import { ActorModel } from '../../models/activitypub/actor'
import { AvatarModel } from '../../models/avatar/avatar'
import { ServerModel } from '../../models/server/server'
import { VideoChannelModel } from '../../models/video/video-channel'
import { JobQueue } from '../job-queue'
import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor'
import { sequelizeTypescript } from '../../initializers/database'
import {
MAccount,
MAccountDefault,
@ -34,9 +36,7 @@ import {
MActorId,
MChannel
} from '../../types/models'
import { extname } from 'path'
import { getServerActor } from '@server/models/application/application'
import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
import { JobQueue } from '../job-queue'
// Set account keys, this could be long so process after the account creation and do not block the client
async function generateAndSaveActorKeys <T extends MActor> (actor: T) {
@ -209,16 +209,10 @@ async function deleteActorAvatarInstance (actor: MActorDefault, t: Transaction)
}
async function fetchActorTotalItems (url: string) {
const options = {
uri: url,
method: 'GET',
json: true,
activityPub: true
}
try {
const { body } = await doRequest<ActivityPubOrderedCollection<unknown>>(options)
return body.totalItems ? body.totalItems : 0
const { body } = await doJSONRequest<ActivityPubOrderedCollection<unknown>>(url, { activityPub: true })
return body.totalItems || 0
} catch (err) {
logger.warn('Cannot fetch remote actor count %s.', url, { err })
return 0
@ -449,26 +443,19 @@ type FetchRemoteActorResult = {
attributedTo: ActivityPubAttributedTo[]
}
async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: number, result: FetchRemoteActorResult }> {
const options = {
uri: actorUrl,
method: 'GET',
json: true,
activityPub: true
}
logger.info('Fetching remote actor %s.', actorUrl)
const requestResult = await doRequest<ActivityPubActor>(options)
const requestResult = await doJSONRequest<ActivityPubActor>(actorUrl, { activityPub: true })
const actorJSON = requestResult.body
if (sanitizeAndCheckActorObject(actorJSON) === false) {
logger.debug('Remote actor JSON is not valid.', { actorJSON })
return { result: undefined, statusCode: requestResult.response.statusCode }
return { result: undefined, statusCode: requestResult.statusCode }
}
if (checkUrlsSameHost(actorJSON.id, actorUrl) !== true) {
logger.warn('Actor url %s has not the same host than its AP id %s', actorUrl, actorJSON.id)
return { result: undefined, statusCode: requestResult.response.statusCode }
return { result: undefined, statusCode: requestResult.statusCode }
}
const followersCount = await fetchActorTotalItems(actorJSON.followers)
@ -496,7 +483,7 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe
const name = actorJSON.name || actorJSON.preferredUsername
return {
statusCode: requestResult.response.statusCode,
statusCode: requestResult.statusCode,
result: {
actor,
name,

25
server/lib/activitypub/crawl.ts
View File

@ -1,27 +1,26 @@
import { ACTIVITY_PUB, REQUEST_TIMEOUT, WEBSERVER } from '../../initializers/constants'
import { doRequest } from '../../helpers/requests'
import { logger } from '../../helpers/logger'
import * as Bluebird from 'bluebird'
import { ActivityPubOrderedCollection } from '../../../shared/models/activitypub'
import { URL } from 'url'
import { ActivityPubOrderedCollection } from '../../../shared/models/activitypub'
import { logger } from '../../helpers/logger'
import { doJSONRequest } from '../../helpers/requests'
import { ACTIVITY_PUB, REQUEST_TIMEOUT, WEBSERVER } from '../../initializers/constants'
type HandlerFunction<T> = (items: T[]) => (Promise<any> | Bluebird<any>)
type CleanerFunction = (startedDate: Date) => (Promise<any> | Bluebird<any>)
async function crawlCollectionPage <T> (uri: string, handler: HandlerFunction<T>, cleaner?: CleanerFunction) {
logger.info('Crawling ActivityPub data on %s.', uri)
async function crawlCollectionPage <T> (argUrl: string, handler: HandlerFunction<T>, cleaner?: CleanerFunction) {
let url = argUrl
logger.info('Crawling ActivityPub data on %s.', url)
const options = {
method: 'GET',
uri,
json: true,
activityPub: true,
timeout: REQUEST_TIMEOUT
}
const startDate = new Date()
const response = await doRequest<ActivityPubOrderedCollection<T>>(options)
const response = await doJSONRequest<ActivityPubOrderedCollection<T>>(url, options)
const firstBody = response.body
const limit = ACTIVITY_PUB.FETCH_PAGE_LIMIT
@ -35,9 +34,9 @@ async function crawlCollectionPage <T> (uri: string, handler: HandlerFunction<T>
const remoteHost = new URL(nextLink).host
if (remoteHost === WEBSERVER.HOST) continue
options.uri = nextLink
url = nextLink
const res = await doRequest<ActivityPubOrderedCollection<T>>(options)
const res = await doJSONRequest<ActivityPubOrderedCollection<T>>(url, options)
body = res.body
} else {
// nextLink is already the object we want
@ -49,7 +48,7 @@ async function crawlCollectionPage <T> (uri: string, handler: HandlerFunction<T>
if (Array.isArray(body.orderedItems)) {
const items = body.orderedItems
logger.info('Processing %i ActivityPub items for %s.', items.length, options.uri)
logger.info('Processing %i ActivityPub items for %s.', items.length, url)
await handler(items)
}

56
server/lib/activitypub/playlist.ts
View File

@ -1,24 +1,24 @@
import { PlaylistObject } from '../../../shared/models/activitypub/objects/playlist-object'
import { crawlCollectionPage } from './crawl'
import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
import { isArray } from '../../helpers/custom-validators/misc'
import { getOrCreateActorAndServerAndModel } from './actor'
import { logger } from '../../helpers/logger'
import { VideoPlaylistModel } from '../../models/video/video-playlist'
import { doRequest } from '../../helpers/requests'
import { checkUrlsSameHost } from '../../helpers/activitypub'
import * as Bluebird from 'bluebird'
import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
import { PlaylistElementObject } from '../../../shared/models/activitypub/objects/playlist-element-object'
import { getOrCreateVideoAndAccountAndChannel } from './videos'
import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '../../helpers/custom-validators/activitypub/playlist'
import { VideoPlaylistElementModel } from '../../models/video/video-playlist-element'
import { PlaylistObject } from '../../../shared/models/activitypub/objects/playlist-object'
import { VideoPlaylistPrivacy } from '../../../shared/models/videos/playlist/video-playlist-privacy.model'
import { checkUrlsSameHost } from '../../helpers/activitypub'
import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '../../helpers/custom-validators/activitypub/playlist'
import { isArray } from '../../helpers/custom-validators/misc'
import { logger } from '../../helpers/logger'
import { doJSONRequest } from '../../helpers/requests'
import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
import { sequelizeTypescript } from '../../initializers/database'
import { createPlaylistMiniatureFromUrl } from '../thumbnail'
import { FilteredModelAttributes } from '../../types/sequelize'
import { VideoPlaylistModel } from '../../models/video/video-playlist'
import { VideoPlaylistElementModel } from '../../models/video/video-playlist-element'
import { MAccountDefault, MAccountId, MVideoId } from '../../types/models'
import { MVideoPlaylist, MVideoPlaylistId, MVideoPlaylistOwner } from '../../types/models/video/video-playlist'
import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
import { FilteredModelAttributes } from '../../types/sequelize'
import { createPlaylistMiniatureFromUrl } from '../thumbnail'
import { getOrCreateActorAndServerAndModel } from './actor'
import { crawlCollectionPage } from './crawl'
import { getOrCreateVideoAndAccountAndChannel } from './videos'
function playlistObjectToDBAttributes (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) {
const privacy = to.includes(ACTIVITY_PUB.PUBLIC)
@ -56,11 +56,7 @@ async function createAccountPlaylists (playlistUrls: string[], account: MAccount
if (exists === true) return
// Fetch url
const { body } = await doRequest<PlaylistObject>({
uri: playlistUrl,
json: true,
activityPub: true
})
const { body } = await doJSONRequest<PlaylistObject>(playlistUrl, { activityPub: true })
if (!isPlaylistObjectValid(body)) {
throw new Error(`Invalid playlist object when fetch account playlists: ${JSON.stringify(body)}`)
@ -164,12 +160,7 @@ async function resetVideoPlaylistElements (elementUrls: string[], playlist: MVid
await Bluebird.map(elementUrls, async elementUrl => {
try {
// Fetch url
const { body } = await doRequest<PlaylistElementObject>({
uri: elementUrl,
json: true,
activityPub: true
})
const { body } = await doJSONRequest<PlaylistElementObject>(elementUrl, { activityPub: true })
if (!isPlaylistElementObjectValid(body)) throw new Error(`Invalid body in video get playlist element ${elementUrl}`)
@ -199,21 +190,14 @@ async function resetVideoPlaylistElements (elementUrls: string[], playlist: MVid
}
async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusCode: number, playlistObject: PlaylistObject }> {
const options = {
uri: playlistUrl,
method: 'GET',
json: true,
activityPub: true
}
logger.info('Fetching remote playlist %s.', playlistUrl)
const { response, body } = await doRequest<any>(options)
const { body, statusCode } = await doJSONRequest<any>(playlistUrl, { activityPub: true })
if (isPlaylistObjectValid(body) === false || checkUrlsSameHost(body.id, playlistUrl) !== true) {
logger.debug('Remote video playlist JSON is not valid.', { body })
return { statusCode: response.statusCode, playlistObject: undefined }
return { statusCode, playlistObject: undefined }
}
return { statusCode: response.statusCode, playlistObject: body }
return { statusCode, playlistObject: body }
}

9
server/lib/activitypub/share.ts
View File

@ -3,7 +3,7 @@ import { Transaction } from 'sequelize'
import { getServerActor } from '@server/models/application/application'
import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
import { logger, loggerTagsFactory } from '../../helpers/logger'
import { doRequest } from '../../helpers/requests'
import { doJSONRequest } from '../../helpers/requests'
import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
import { VideoShareModel } from '../../models/video/video-share'
import { MChannelActorLight, MVideo, MVideoAccountLight, MVideoId } from '../../types/models/video'
@ -40,12 +40,7 @@ async function changeVideoChannelShare (
async function addVideoShares (shareUrls: string[], video: MVideoId) {
await Bluebird.map(shareUrls, async shareUrl => {
try {
// Fetch url
const { body } = await doRequest<any>({
uri: shareUrl,
json: true,
activityPub: true
})
const { body } = await doJSONRequest<any>(shareUrl, { activityPub: true })
if (!body || !body.actor) throw new Error('Body or body actor is invalid')
const actorUrl = getAPId(body.actor)

20
server/lib/activitypub/video-comments.ts
View File

@ -1,13 +1,13 @@
import { sanitizeAndCheckVideoCommentObject } from '../../helpers/custom-validators/activitypub/video-comments'
import { logger } from '../../helpers/logger'
import { doRequest } from '../../helpers/requests'
import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
import { VideoCommentModel } from '../../models/video/video-comment'
import { getOrCreateActorAndServerAndModel } from './actor'
import { getOrCreateVideoAndAccountAndChannel } from './videos'
import * as Bluebird from 'bluebird'
import { checkUrlsSameHost } from '../../helpers/activitypub'
import { sanitizeAndCheckVideoCommentObject } from '../../helpers/custom-validators/activitypub/video-comments'
import { logger } from '../../helpers/logger'
import { doJSONRequest } from '../../helpers/requests'
import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
import { VideoCommentModel } from '../../models/video/video-comment'
import { MCommentOwner, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../types/models/video'
import { getOrCreateActorAndServerAndModel } from './actor'
import { getOrCreateVideoAndAccountAndChannel } from './videos'
type ResolveThreadParams = {
url: string
@ -126,11 +126,7 @@ async function resolveRemoteParentComment (params: ResolveThreadParams) {
throw new Error('Recursion limit reached when resolving a thread')
}
const { body } = await doRequest<any>({
uri: url,
json: true,
activityPub: true
})
const { body } = await doJSONRequest<any>(url, { activityPub: true })
if (sanitizeAndCheckVideoCommentObject(body) === false) {
throw new Error(`Remote video comment JSON ${url} is not valid:` + JSON.stringify(body))

24
server/lib/activitypub/video-rates.ts
View File

@ -1,26 +1,22 @@
import { Transaction } from 'sequelize'
import { sendLike, sendUndoDislike, sendUndoLike } from './send'
import { VideoRateType } from '../../../shared/models/videos'
import * as Bluebird from 'bluebird'
import { getOrCreateActorAndServerAndModel } from './actor'
import { AccountVideoRateModel } from '../../models/account/account-video-rate'
import { Transaction } from 'sequelize'
import { doJSONRequest } from '@server/helpers/requests'
import { VideoRateType } from '../../../shared/models/videos'
import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
import { logger } from '../../helpers/logger'
import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
import { doRequest } from '../../helpers/requests'
import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
import { getVideoDislikeActivityPubUrlByLocalActor, getVideoLikeActivityPubUrlByLocalActor } from './url'
import { sendDislike } from './send/send-dislike'
import { AccountVideoRateModel } from '../../models/account/account-video-rate'
import { MAccountActor, MActorUrl, MVideo, MVideoAccountLight, MVideoId } from '../../types/models'
import { getOrCreateActorAndServerAndModel } from './actor'
import { sendLike, sendUndoDislike, sendUndoLike } from './send'
import { sendDislike } from './send/send-dislike'
import { getVideoDislikeActivityPubUrlByLocalActor, getVideoLikeActivityPubUrlByLocalActor } from './url'
async function createRates (ratesUrl: string[], video: MVideo, rate: VideoRateType) {
await Bluebird.map(ratesUrl, async rateUrl => {
try {
// Fetch url
const { body } = await doRequest<any>({
uri: rateUrl,
json: true,
activityPub: true
})
const { body } = await doJSONRequest<any>(rateUrl, { activityPub: true })
if (!body || !body.actor) throw new Error('Body or body actor is invalid')
const actorUrl = getAPId(body.actor)

31
server/lib/activitypub/videos.ts
View File

@ -2,7 +2,6 @@ import * as Bluebird from 'bluebird'
import { maxBy, minBy } from 'lodash'
import * as magnetUtil from 'magnet-uri'
import { basename, join } from 'path'
import * as request from 'request'
import { Transaction } from 'sequelize/types'
import { TrackerModel } from '@server/models/server/tracker'
import { VideoLiveModel } from '@server/models/video/video-live'
@ -31,7 +30,7 @@ import { isArray } from '../../helpers/custom-validators/misc'
import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
import { deleteNonExistingModels, resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils'
import { logger } from '../../helpers/logger'
import { doRequest } from '../../helpers/requests'
import { doJSONRequest } from '../../helpers/requests'
import { fetchVideoByUrl, getExtFromMimetype, VideoFetchByUrlType } from '../../helpers/video'
import {
ACTIVITY_PUB,
@ -115,36 +114,26 @@ async function federateVideoIfNeeded (videoArg: MVideoAPWithoutCaption, isNewVid
}
}
async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.RequestResponse, videoObject: VideoObject }> {
const options = {
uri: videoUrl,
method: 'GET',
json: true,
activityPub: true
}
async function fetchRemoteVideo (videoUrl: string): Promise<{ statusCode: number, videoObject: VideoObject }> {
logger.info('Fetching remote video %s.', videoUrl)
const { response, body } = await doRequest<any>(options)
const { statusCode, body } = await doJSONRequest<any>(videoUrl, { activityPub: true })
if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) {
logger.debug('Remote video JSON is not valid.', { body })
return { response, videoObject: undefined }
return { statusCode, videoObject: undefined }
}
return { response, videoObject: body }
return { statusCode, videoObject: body }
}
async function fetchRemoteVideoDescription (video: MVideoAccountLight) {
const host = video.VideoChannel.Account.Actor.Server.host
const path = video.getDescriptionAPIPath()
const options = {
uri: REMOTE_SCHEME.HTTP + '://' + host + path,
json: true
}
const url = REMOTE_SCHEME.HTTP + '://' + host + path
const { body } = await doRequest<any>(options)
return body.description ? body.description : ''
const { body } = await doJSONRequest<any>(url)
return body.description || ''
}
function getOrCreateVideoChannelFromVideoObject (videoObject: VideoObject) {
@ -534,8 +523,8 @@ async function refreshVideoIfNeeded (options: {
: await VideoModel.loadByUrlAndPopulateAccount(options.video.url)
try {
const { response, videoObject } = await fetchRemoteVideo(video.url)
if (response.statusCode === HttpStatusCode.NOT_FOUND_404) {
const { statusCode, videoObject } = await fetchRemoteVideo(video.url)
if (statusCode === HttpStatusCode.NOT_FOUND_404) {
logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url)
// Video does not exist anymore

2
server/lib/files-cache/videos-caption-cache.ts
View File

@ -41,7 +41,7 @@ class VideosCaptionCache extends AbstractVideoStaticFileCache <string> {
const remoteUrl = videoCaption.getFileUrl(video)
const destPath = join(FILES_CACHE.VIDEO_CAPTIONS.DIRECTORY, videoCaption.filename)
await doRequestAndSaveToFile({ uri: remoteUrl }, destPath)
await doRequestAndSaveToFile(remoteUrl, destPath)
return { isOwned: false, path: destPath }
}

2
server/lib/files-cache/videos-preview-cache.ts
View File

@ -39,7 +39,7 @@ class VideosPreviewCache extends AbstractVideoStaticFileCache <string> {
const destPath = join(FILES_CACHE.PREVIEWS.DIRECTORY, preview.filename)
const remoteUrl = preview.getFileUrl(video)
await doRequestAndSaveToFile({ uri: remoteUrl }, destPath)
await doRequestAndSaveToFile(remoteUrl, destPath)
logger.debug('Fetched remote preview %s to %s.', remoteUrl, destPath)

2
server/lib/files-cache/videos-torrent-cache.ts
View File

@ -41,7 +41,7 @@ class VideosTorrentCache extends AbstractVideoStaticFileCache <string> {
const remoteUrl = file.getRemoteTorrentUrl(video)
const destPath = join(FILES_CACHE.TORRENTS.DIRECTORY, file.torrentFilename)
await doRequestAndSaveToFile({ uri: remoteUrl }, destPath)
await doRequestAndSaveToFile(remoteUrl, destPath)
const downloadName = `${video.name}-${file.resolution}p.torrent`

4
server/lib/hls.ts
View File

@ -135,7 +135,7 @@ function downloadPlaylistSegments (playlistUrl: string, destinationDir: string,
const destPath = join(tmpDirectory, basename(fileUrl))
const bodyKBLimit = 10 * 1000 * 1000 // 10GB
await doRequestAndSaveToFile({ uri: fileUrl }, destPath, bodyKBLimit)
await doRequestAndSaveToFile(fileUrl, destPath, { bodyKBLimit })
}
clearTimeout(timer)
@ -156,7 +156,7 @@ function downloadPlaylistSegments (playlistUrl: string, destinationDir: string,
}
async function fetchUniqUrls (playlistUrl: string) {
const { body } = await doRequest<string>({ uri: playlistUrl })
const { body } = await doRequest(playlistUrl)
if (!body) return []

11
server/lib/job-queue/handlers/activitypub-cleaner.ts
View File

@ -7,7 +7,7 @@ import {
isLikeActivityValid
} from '@server/helpers/custom-validators/activitypub/activity'
import { sanitizeAndCheckVideoCommentObject } from '@server/helpers/custom-validators/activitypub/video-comments'
import { doRequest } from '@server/helpers/requests'
import { doJSONRequest } from '@server/helpers/requests'
import { AP_CLEANER_CONCURRENCY } from '@server/initializers/constants'
import { VideoModel } from '@server/models/video/video'
import { VideoCommentModel } from '@server/models/video/video-comment'
@ -81,15 +81,10 @@ async function updateObjectIfNeeded <T> (
updater: (url: string, newUrl: string) => Promise<T>,
deleter: (url: string) => Promise<T>
): Promise<{ data: T, status: 'deleted' | 'updated' } | null> {
// Fetch url
const { response, body } = await doRequest<any>({
uri: url,
json: true,
activityPub: true
})
const { statusCode, body } = await doJSONRequest<any>(url, { activityPub: true })
// Does not exist anymore, remove entry
if (response.statusCode === HttpStatusCode.NOT_FOUND_404) {
if (statusCode === HttpStatusCode.NOT_FOUND_404) {
logger.info('Removing remote AP object %s.', url)
const data = await deleter(url)

5
server/lib/job-queue/handlers/activitypub-http-broadcast.ts
View File

@ -16,8 +16,7 @@ async function processActivityPubHttpBroadcast (job: Bull.Job) {
const httpSignatureOptions = await buildSignedRequestOptions(payload)
const options = {
method: 'POST',
uri: '',
method: 'POST' as 'POST',
json: body,
httpSignature: httpSignatureOptions,
timeout: REQUEST_TIMEOUT,
@ -28,7 +27,7 @@ async function processActivityPubHttpBroadcast (job: Bull.Job) {
const goodUrls: string[] = []
await Bluebird.map(payload.uris, uri => {
return doRequest(Object.assign({}, options, { uri }))
return doRequest(uri, options)
.then(() => goodUrls.push(uri))
.catch(() => badUrls.push(uri))
}, { concurrency: BROADCAST_CONCURRENCY })

5
server/lib/job-queue/handlers/activitypub-http-unicast.ts
View File

@ -16,8 +16,7 @@ async function processActivityPubHttpUnicast (job: Bull.Job) {
const httpSignatureOptions = await buildSignedRequestOptions(payload)
const options = {
method: 'POST',
uri,
method: 'POST' as 'POST',
json: body,
httpSignature: httpSignatureOptions,
timeout: REQUEST_TIMEOUT,
@ -25,7 +24,7 @@ async function processActivityPubHttpUnicast (job: Bull.Job) {
}
try {
await doRequest(options)
await doRequest(uri, options)
ActorFollowScoreCache.Instance.updateActorFollowsScore([ uri ], [])
} catch (err) {
ActorFollowScoreCache.Instance.updateActorFollowsScore([], [ uri ])

9
server/lib/job-queue/handlers/utils/activitypub-http-utils.ts
View File

@ -6,21 +6,24 @@ import { getServerActor } from '@server/models/application/application'
import { buildDigest } from '@server/helpers/peertube-crypto'
import { ContextType } from '@shared/models/activitypub/context'
type Payload = { body: any, contextType?: ContextType, signatureActorId?: number }
type Payload <T> = { body: T, contextType?: ContextType, signatureActorId?: number }
async function computeBody (payload: Payload) {
async function computeBody <T> (
payload: Payload<T>
): Promise<T | T & { type: 'RsaSignature2017', creator: string, created: string }> {
let body = payload.body
if (payload.signatureActorId) {
const actorSignature = await ActorModel.load(payload.signatureActorId)
if (!actorSignature) throw new Error('Unknown signature actor id.')
body = await buildSignedActivity(actorSignature, payload.body, payload.contextType)
}
return body
}
async function buildSignedRequestOptions (payload: Payload) {
async function buildSignedRequestOptions (payload: Payload<any>) {
let actor: MActor | null
if (payload.signatureActorId) {

26
server/lib/plugins/plugin-index.ts
View File

@ -1,22 +1,22 @@
import { doRequest } from '../../helpers/requests'
import { CONFIG } from '../../initializers/config'
import { sanitizeUrl } from '@server/helpers/core-utils'
import { ResultList } from '../../../shared/models'
import { PeertubePluginIndexList } from '../../../shared/models/plugins/peertube-plugin-index-list.model'
import { PeerTubePluginIndex } from '../../../shared/models/plugins/peertube-plugin-index.model'
import {
PeertubePluginLatestVersionRequest,
PeertubePluginLatestVersionResponse
} from '../../../shared/models/plugins/peertube-plugin-latest-version.model'
import { PeertubePluginIndexList } from '../../../shared/models/plugins/peertube-plugin-index-list.model'
import { ResultList } from '../../../shared/models'
import { PeerTubePluginIndex } from '../../../shared/models/plugins/peertube-plugin-index.model'
import { logger } from '../../helpers/logger'
import { doJSONRequest } from '../../helpers/requests'
import { CONFIG } from '../../initializers/config'
import { PEERTUBE_VERSION } from '../../initializers/constants'
import { PluginModel } from '../../models/server/plugin'
import { PluginManager } from './plugin-manager'
import { logger } from '../../helpers/logger'
import { PEERTUBE_VERSION } from '../../initializers/constants'
import { sanitizeUrl } from '@server/helpers/core-utils'
async function listAvailablePluginsFromIndex (options: PeertubePluginIndexList) {
const { start = 0, count = 20, search, sort = 'npmName', pluginType } = options
const qs: PeertubePluginIndexList = {
const searchParams: PeertubePluginIndexList & Record<string, string | number> = {
start,
count,
sort,
@ -28,7 +28,7 @@ async function listAvailablePluginsFromIndex (options: PeertubePluginIndexList)
const uri = CONFIG.PLUGINS.INDEX.URL + '/api/v1/plugins'
try {
const { body } = await doRequest<any>({ uri, qs, json: true })
const { body } = await doJSONRequest<any>(uri, { searchParams })
logger.debug('Got result from PeerTube index.', { body })
@ -58,7 +58,11 @@ async function getLatestPluginsVersion (npmNames: string[]): Promise<PeertubePlu
const uri = sanitizeUrl(CONFIG.PLUGINS.INDEX.URL) + '/api/v1/plugins/latest-version'
const { body } = await doRequest<any>({ uri, body: bodyRequest, json: true, method: 'POST' })
const options = {
json: bodyRequest,
method: 'POST' as 'POST'
}
const { body } = await doJSONRequest<PeertubePluginLatestVersionResponse>(uri, options)
return body
}

8
server/lib/schedulers/auto-follow-index-instances.ts
View File

@ -1,5 +1,5 @@
import { chunk } from 'lodash'
import { doRequest } from '@server/helpers/requests'
import { doJSONRequest } from '@server/helpers/requests'
import { JobQueue } from '@server/lib/job-queue'
import { ActorFollowModel } from '@server/models/activitypub/actor-follow'
import { getServerActor } from '@server/models/application/application'
@ -34,12 +34,12 @@ export class AutoFollowIndexInstances extends AbstractScheduler {
try {
const serverActor = await getServerActor()
const qs = { count: 1000 }
if (this.lastCheck) Object.assign(qs, { since: this.lastCheck.toISOString() })
const searchParams = { count: 1000 }
if (this.lastCheck) Object.assign(searchParams, { since: this.lastCheck.toISOString() })
this.lastCheck = new Date()
const { body } = await doRequest<any>({ uri: indexUrl, qs, json: true })
const { body } = await doJSONRequest<any>(indexUrl, { searchParams })
if (!body.data || Array.isArray(body.data) === false) {
logger.error('Cannot auto follow instances of index %s. Please check the auto follow URL.', indexUrl, { body })
return

40
server/tests/api/activitypub/security.ts
View File

@ -79,9 +79,9 @@ describe('Test ActivityPub security', function () {
Digest: buildDigest({ hello: 'coucou' })
}
const { response } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers)
const { statusCode } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers)
expect(response.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
expect(statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
})
it('Should fail with an invalid date', async function () {
@ -89,9 +89,9 @@ describe('Test ActivityPub security', function () {
const headers = buildGlobalHeaders(body)
headers['date'] = 'Wed, 21 Oct 2015 07:28:00 GMT'
const { response } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers)
const { statusCode } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers)
expect(response.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
expect(statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
})
it('Should fail with bad keys', async function () {
@ -101,9 +101,9 @@ describe('Test ActivityPub security', function () {
const body = activityPubContextify(getAnnounceWithoutContext(servers[1]))
const headers = buildGlobalHeaders(body)
const { response } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers)
const { statusCode } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers)
expect(response.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
expect(statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
})
it('Should reject requests without appropriate signed headers', async function () {
@ -123,8 +123,8 @@ describe('Test ActivityPub security', function () {
for (const badHeaders of badHeadersMatrix) {
signatureOptions.headers = badHeaders
const { response } = await makePOSTAPRequest(url, body, signatureOptions, headers)
expect(response.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
const { statusCode } = await makePOSTAPRequest(url, body, signatureOptions, headers)
expect(statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
}
})
@ -132,9 +132,9 @@ describe('Test ActivityPub security', function () {
const body = activityPubContextify(getAnnounceWithoutContext(servers[1]))
const headers = buildGlobalHeaders(body)
const { response } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers)
const { statusCode } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers)
expect(response.statusCode).to.equal(HttpStatusCode.NO_CONTENT_204)
expect(statusCode).to.equal(HttpStatusCode.NO_CONTENT_204)
})
it('Should refresh the actor keys', async function () {
@ -150,9 +150,9 @@ describe('Test ActivityPub security', function () {
const body = activityPubContextify(getAnnounceWithoutContext(servers[1]))
const headers = buildGlobalHeaders(body)
const { response } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers)
const { statusCode } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers)
expect(response.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
expect(statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
})
})
@ -183,9 +183,9 @@ describe('Test ActivityPub security', function () {
const headers = buildGlobalHeaders(signedBody)
const { response } = await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers)
const { statusCode } = await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers)
expect(response.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
expect(statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
})
it('Should fail with an altered body', async function () {
@ -204,9 +204,9 @@ describe('Test ActivityPub security', function () {
const headers = buildGlobalHeaders(signedBody)
const { response } = await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers)
const { statusCode } = await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers)
expect(response.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
expect(statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
})
it('Should succeed with a valid signature', async function () {
@ -220,9 +220,9 @@ describe('Test ActivityPub security', function () {
const headers = buildGlobalHeaders(signedBody)
const { response } = await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers)
const { statusCode } = await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers)
expect(response.statusCode).to.equal(HttpStatusCode.NO_CONTENT_204)
expect(statusCode).to.equal(HttpStatusCode.NO_CONTENT_204)
})
it('Should refresh the actor keys', async function () {
@ -243,9 +243,9 @@ describe('Test ActivityPub security', function () {
const headers = buildGlobalHeaders(signedBody)
const { response } = await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers)
const { statusCode } = await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers)
expect(response.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
expect(statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
})
})

2
server/tests/api/server/handle-down.ts
View File

@ -348,8 +348,8 @@ describe('Test handle downs', function () {
for (let i = 0; i < 3; i++) {
await getVideo(servers[1].url, videoIdsServer1[i])
await wait(1000)
await waitJobs([ servers[1] ])
await wait(1500)
}
for (const id of videoIdsServer1) {

16
server/tests/helpers/request.ts
View File

@ -1,11 +1,11 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import 'mocha'
import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests'
import { get4KFileUrl, root, wait } from '../../../shared/extra-utils'
import { join } from 'path'
import { pathExists, remove } from 'fs-extra'
import { expect } from 'chai'
import { pathExists, remove } from 'fs-extra'
import { join } from 'path'
import { get4KFileUrl, root, wait } from '../../../shared/extra-utils'
import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests'
describe('Request helpers', function () {
const destPath1 = join(root(), 'test-output-1.txt')
@ -13,7 +13,7 @@ describe('Request helpers', function () {
it('Should throw an error when the bytes limit is exceeded for request', async function () {
try {
await doRequest({ uri: get4KFileUrl() }, 3)
await doRequest(get4KFileUrl(), { bodyKBLimit: 3 })
} catch {
return
}
@ -23,7 +23,7 @@ describe('Request helpers', function () {
it('Should throw an error when the bytes limit is exceeded for request and save file', async function () {
try {
await doRequestAndSaveToFile({ uri: get4KFileUrl() }, destPath1, 3)
await doRequestAndSaveToFile(get4KFileUrl(), destPath1, { bodyKBLimit: 3 })
} catch {
await wait(500)
@ -35,8 +35,8 @@ describe('Request helpers', function () {
})
it('Should succeed if the file is below the limit', async function () {
await doRequest({ uri: get4KFileUrl() }, 5)
await doRequestAndSaveToFile({ uri: get4KFileUrl() }, destPath2, 5)
await doRequest(get4KFileUrl(), { bodyKBLimit: 5 })
await doRequestAndSaveToFile(get4KFileUrl(), destPath2, { bodyKBLimit: 5 })
expect(await pathExists(destPath2)).to.be.true
})

5
shared/extra-utils/requests/activitypub.ts
View File

@ -5,14 +5,13 @@ import { activityPubContextify } from '../../../server/helpers/activitypub'
function makePOSTAPRequest (url: string, body: any, httpSignature: any, headers: any) {
const options = {
method: 'POST',
uri: url,
method: 'POST' as 'POST',
json: body,
httpSignature,
headers
}
return doRequest(options)
return doRequest(url, options)
}
async function makeFollowRequest (to: { url: string }, by: { url: string, privateKey }) {

2
shared/models/server/job.model.ts
View File

@ -59,7 +59,7 @@ export type ActivitypubHttpFetcherPayload = {
export type ActivitypubHttpUnicastPayload = {
uri: string
signatureActorId?: number
body: any
body: object
contextType?: ContextType
}

49
yarn.lock
View File

@ -3925,6 +3925,23 @@ globby@^11.0.1:
merge2 "^1.3.0"
slash "^3.0.0"
got@^11.8.2, got@~11.8.1:
version "11.8.2"
resolved "https://registry.yarnpkg.com/got/-/got-11.8.2.tgz#7abb3959ea28c31f3576f1576c1effce23f33599"
integrity sha512-D0QywKgIe30ODs+fm8wMZiAcZjypcCodPNuMz5H9Mny7RJ+IjJ10BdmGW7OM7fHXP+O7r6ZwapQ/YQmMSvB0UQ==
dependencies:
"@sindresorhus/is" "^4.0.0"
"@szmarczak/http-timer" "^4.0.5"
"@types/cacheable-request" "^6.0.1"
"@types/responselike" "^1.0.0"
cacheable-lookup "^5.0.3"
cacheable-request "^7.0.1"
decompress-response "^6.0.0"
http2-wrapper "^1.0.0-beta.5.2"
lowercase-keys "^2.0.0"
p-cancelable "^2.0.0"
responselike "^2.0.0"
got@^9.6.0:
version "9.6.0"
resolved "https://registry.yarnpkg.com/got/-/got-9.6.0.tgz#edf45e7d67f99545705de1f7bbeeeb121765ed85"
@ -3942,23 +3959,6 @@ got@^9.6.0:
to-readable-stream "^1.0.0"
url-parse-lax "^3.0.0"
got@~11.8.1:
version "11.8.2"
resolved "https://registry.yarnpkg.com/got/-/got-11.8.2.tgz#7abb3959ea28c31f3576f1576c1effce23f33599"
integrity sha512-D0QywKgIe30ODs+fm8wMZiAcZjypcCodPNuMz5H9Mny7RJ+IjJ10BdmGW7OM7fHXP+O7r6ZwapQ/YQmMSvB0UQ==
dependencies:
"@sindresorhus/is" "^4.0.0"
"@szmarczak/http-timer" "^4.0.5"
"@types/cacheable-request" "^6.0.1"
"@types/responselike" "^1.0.0"
cacheable-lookup "^5.0.3"
cacheable-request "^7.0.1"
decompress-response "^6.0.0"
http2-wrapper "^1.0.0-beta.5.2"
lowercase-keys "^2.0.0"
p-cancelable "^2.0.0"
responselike "^2.0.0"
graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.0:
version "4.2.6"
resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.6.tgz#ff040b2b0853b23c3d31027523706f1885d76bee"
@ -4167,7 +4167,7 @@ http-proxy-agent@^4.0.1:
agent-base "6"
debug "4"
http-signature@1.3.5, http-signature@~1.2.0:
http-signature@1.3.5:
version "1.3.5"
resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.3.5.tgz#9f19496ffbf3227298d7b5f156e0e1a948678683"
integrity sha512-NwoTQYSJoFt34jSBbwzDHDofoA61NGXzu6wXh95o1Ry62EnmKjXb/nR/RknLeZ3G/uGwrlKNY2z7uPt+Cdl7Tw==
@ -4176,6 +4176,15 @@ http-signature@1.3.5, http-signature@~1.2.0:
jsprim "^1.2.2"
sshpk "^1.14.1"
http-signature@~1.2.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1"
integrity sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=
dependencies:
assert-plus "^1.0.0"
jsprim "^1.2.2"
sshpk "^1.7.0"
http2-wrapper@^1.0.0-beta.5.2:
version "1.0.3"
resolved "https://registry.yarnpkg.com/http2-wrapper/-/http2-wrapper-1.0.3.tgz#b8f55e0c1f25d4ebd08b3b0c2c079f9590800b3d"
@ -7060,7 +7069,7 @@ render-media@^4.1.0:
stream-to-blob-url "^3.0.2"
videostream "^3.2.2"
request@^2.81.0, request@^2.88.0:
request@^2.88.0:
version "2.88.2"
resolved "https://registry.yarnpkg.com/request/-/request-2.88.2.tgz#d73c918731cb5a87da047e207234146f664d12b3"
integrity sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==
@ -7700,7 +7709,7 @@ srt-to-vtt@^1.1.2:
through2 "^0.6.3"
to-utf-8 "^1.2.0"
sshpk@^1.14.1:
sshpk@^1.14.1, sshpk@^1.7.0:
version "1.16.1"
resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.16.1.tgz#fb661c0bef29b39db40769ee39fa70093d6f6877"
integrity sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg==