Add use proxy for s3 (#4973)
* Fix object storage to be accessible via proxy * fix lint * Use hpagent * Fix lint * Fix PR Co-authored-by: noellabo <noel.yoshiba@gmail.com>
This commit is contained in:
Chocobozzz
GitHub
parent
644014cc55
commit
ca3d5912e1
|
|
@ -79,6 +79,7 @@
|
|||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "^3.23.0",
|
||||
"@aws-sdk/lib-storage": "^3.72.0",
|
||||
"@aws-sdk/node-http-handler": "^3.82.0",
|
||||
"@babel/parser": "7.17.8",
|
||||
"@peertube/feed": "^5.0.1",
|
||||
"@peertube/http-signature": "^1.4.0",
|
||||
|
|
|
|||
|
|
@ -212,6 +212,7 @@ export {
|
|||
doRequestAndSaveToFile,
|
||||
isBinaryResponse,
|
||||
downloadImage,
|
||||
getAgent,
|
||||
findLatestRedirection,
|
||||
peertubeGot
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,8 +1,22 @@
|
|||
import { S3Client } from '@aws-sdk/client-s3'
|
||||
import { NodeHttpHandler } from '@aws-sdk/node-http-handler'
|
||||
import { logger } from '@server/helpers/logger'
|
||||
import { isProxyEnabled } from '@server/helpers/proxy'
|
||||
import { getAgent } from '@server/helpers/requests'
|
||||
import { CONFIG } from '@server/initializers/config'
|
||||
import { lTags } from './logger'
|
||||
|
||||
function getProxyRequestHandler () {
|
||||
if (!isProxyEnabled()) return null
|
||||
|
||||
const { agent } = getAgent()
|
||||
|
||||
return new NodeHttpHandler({
|
||||
httpAgent: agent.http,
|
||||
httpsAgent: agent.https
|
||||
})
|
||||
}
|
||||
|
||||
let endpointParsed: URL
|
||||
function getEndpointParsed () {
|
||||
if (endpointParsed) return endpointParsed
|
||||
|
|
@ -26,7 +40,8 @@ function getClient () {
|
|||
accessKeyId: OBJECT_STORAGE.CREDENTIALS.ACCESS_KEY_ID,
|
||||
secretAccessKey: OBJECT_STORAGE.CREDENTIALS.SECRET_ACCESS_KEY
|
||||
}
|
||||
: undefined
|
||||
: undefined,
|
||||
requestHandler: getProxyRequestHandler()
|
||||
})
|
||||
|
||||
logger.info('Initialized S3 client %s with region %s.', getEndpoint(), OBJECT_STORAGE.REGION, lTags())
|
||||
|
|
|
|||
|
|
@ -2,12 +2,14 @@
|
|||
|
||||
import 'mocha'
|
||||
import * as chai from 'chai'
|
||||
import { FIXTURE_URLS, MockProxy } from '@server/tests/shared'
|
||||
import { expectNotStartWith, expectStartWith, FIXTURE_URLS, MockProxy } from '@server/tests/shared'
|
||||
import { areObjectStorageTestsDisabled } from '@shared/core-utils'
|
||||
import { HttpStatusCode, VideoPrivacy } from '@shared/models'
|
||||
import {
|
||||
cleanupTests,
|
||||
createMultipleServers,
|
||||
doubleFollow,
|
||||
ObjectStorageCommand,
|
||||
PeerTubeServer,
|
||||
setAccessTokensToServers,
|
||||
setDefaultVideoChannel,
|
||||
|
|
@ -120,6 +122,44 @@ describe('Test proxy', function () {
|
|||
})
|
||||
})
|
||||
|
||||
describe('Object storage', function () {
|
||||
if (areObjectStorageTestsDisabled()) return
|
||||
|
||||
before(async function () {
|
||||
this.timeout(30000)
|
||||
|
||||
await ObjectStorageCommand.prepareDefaultBuckets()
|
||||
})
|
||||
|
||||
it('Should succeed to upload to object storage with the appropriate proxy config', async function () {
|
||||
this.timeout(120000)
|
||||
|
||||
await servers[0].kill()
|
||||
await servers[0].run(ObjectStorageCommand.getDefaultConfig(), { env: goodEnv })
|
||||
|
||||
const { uuid } = await servers[0].videos.quickUpload({ name: 'video' })
|
||||
await waitJobs(servers)
|
||||
|
||||
const video = await servers[0].videos.get({ id: uuid })
|
||||
|
||||
expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
|
||||
})
|
||||
|
||||
it('Should fail to upload to object storage with a wrong proxy config', async function () {
|
||||
this.timeout(120000)
|
||||
|
||||
await servers[0].kill()
|
||||
await servers[0].run(ObjectStorageCommand.getDefaultConfig(), { env: badEnv })
|
||||
|
||||
const { uuid } = await servers[0].videos.quickUpload({ name: 'video' })
|
||||
await waitJobs(servers)
|
||||
|
||||
const video = await servers[0].videos.get({ id: uuid })
|
||||
|
||||
expectNotStartWith(video.files[0].fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
|
||||
})
|
||||
})
|
||||
|
||||
after(async function () {
|
||||
await proxy.terminate()
|
||||
|
||||
|
|
|
|||
|
|
@ -19,6 +19,10 @@ function expectStartWith (str: string, start: string) {
|
|||
expect(str.startsWith(start), `${str} does not start with ${start}`).to.be.true
|
||||
}
|
||||
|
||||
function expectNotStartWith (str: string, start: string) {
|
||||
expect(str.startsWith(start), `${str} does not start with ${start}`).to.be.false
|
||||
}
|
||||
|
||||
async function expectLogDoesNotContain (server: PeerTubeServer, str: string) {
|
||||
const content = await server.servers.getLogContent()
|
||||
|
||||
|
|
@ -92,6 +96,7 @@ export {
|
|||
expectLogDoesNotContain,
|
||||
testFileExistsOrNot,
|
||||
expectStartWith,
|
||||
expectNotStartWith,
|
||||
checkBadStartPagination,
|
||||
checkBadCountPagination,
|
||||
checkBadSortPagination
|
||||
|
|
|
|||
41
yarn.lock
41
yarn.lock
|
|
@ -142,6 +142,14 @@
|
|||
"@aws-sdk/types" "3.55.0"
|
||||
tslib "^2.3.1"
|
||||
|
||||
"@aws-sdk/abort-controller@3.78.0":
|
||||
version "3.78.0"
|
||||
resolved "https://registry.yarnpkg.com/@aws-sdk/abort-controller/-/abort-controller-3.78.0.tgz#f2b0f8d63954afe51136254f389a18dd24a8f6f3"
|
||||
integrity sha512-iz1YLwM2feJUj/y97yO4XmDeTxs+yZ1XJwQgoawKuc8IDBKUutnJNCHL5jL04WUKU7Nrlq+Hr2fCTScFh2z9zg==
|
||||
dependencies:
|
||||
"@aws-sdk/types" "3.78.0"
|
||||
tslib "^2.3.1"
|
||||
|
||||
"@aws-sdk/chunked-blob-reader-native@3.58.0":
|
||||
version "3.58.0"
|
||||
resolved "https://registry.yarnpkg.com/@aws-sdk/chunked-blob-reader-native/-/chunked-blob-reader-native-3.58.0.tgz#1db413c5c80b32e24f1b62b22e15e9ad74d75cda"
|
||||
|
|
@ -678,6 +686,17 @@
|
|||
"@aws-sdk/types" "3.55.0"
|
||||
tslib "^2.3.1"
|
||||
|
||||
"@aws-sdk/node-http-handler@^3.82.0":
|
||||
version "3.82.0"
|
||||
resolved "https://registry.yarnpkg.com/@aws-sdk/node-http-handler/-/node-http-handler-3.82.0.tgz#e28064815c6c6caf22a16bb7fee4e9e7e73ef3bb"
|
||||
integrity sha512-yyq/DA/IMzL4fLJhV7zVfP7aUQWPHfOKTCJjWB3KeV5YPiviJtSKb/KyzNi+gQyO7SmsL/8vQbQrf3/s7N/2OA==
|
||||
dependencies:
|
||||
"@aws-sdk/abort-controller" "3.78.0"
|
||||
"@aws-sdk/protocol-http" "3.78.0"
|
||||
"@aws-sdk/querystring-builder" "3.78.0"
|
||||
"@aws-sdk/types" "3.78.0"
|
||||
tslib "^2.3.1"
|
||||
|
||||
"@aws-sdk/property-provider@3.55.0":
|
||||
version "3.55.0"
|
||||
resolved "https://registry.yarnpkg.com/@aws-sdk/property-provider/-/property-provider-3.55.0.tgz#0eabe5e84d9258c85c2c5e44bcb09379ae9429d2"
|
||||
|
|
@ -694,6 +713,14 @@
|
|||
"@aws-sdk/types" "3.55.0"
|
||||
tslib "^2.3.1"
|
||||
|
||||
"@aws-sdk/protocol-http@3.78.0":
|
||||
version "3.78.0"
|
||||
resolved "https://registry.yarnpkg.com/@aws-sdk/protocol-http/-/protocol-http-3.78.0.tgz#8a30db90e3373fe94e2b0007c3cba47b5c9e08bd"
|
||||
integrity sha512-SQB26MhEK96yDxyXd3UAaxLz1Y/ZvgE4pzv7V3wZiokdEedM0kawHKEn1UQJlqJLEZcQI9QYyysh3rTvHZ3fyg==
|
||||
dependencies:
|
||||
"@aws-sdk/types" "3.78.0"
|
||||
tslib "^2.3.1"
|
||||
|
||||
"@aws-sdk/querystring-builder@3.55.0":
|
||||
version "3.55.0"
|
||||
resolved "https://registry.yarnpkg.com/@aws-sdk/querystring-builder/-/querystring-builder-3.55.0.tgz#7d6d4e2c597eb3d636bd3a368b494dac175ba329"
|
||||
|
|
@ -703,6 +730,15 @@
|
|||
"@aws-sdk/util-uri-escape" "3.55.0"
|
||||
tslib "^2.3.1"
|
||||
|
||||
"@aws-sdk/querystring-builder@3.78.0":
|
||||
version "3.78.0"
|
||||
resolved "https://registry.yarnpkg.com/@aws-sdk/querystring-builder/-/querystring-builder-3.78.0.tgz#29068c4d1fad056e26f848779a31335469cb0038"
|
||||
integrity sha512-aib6RW1WAaTQDqVgRU1Ku9idkhm90gJKbCxVaGId+as6QHNUqMChEfK2v+0afuKiPNOs5uWmqvOXI9+Gt+UGDg==
|
||||
dependencies:
|
||||
"@aws-sdk/types" "3.78.0"
|
||||
"@aws-sdk/util-uri-escape" "3.55.0"
|
||||
tslib "^2.3.1"
|
||||
|
||||
"@aws-sdk/querystring-parser@3.55.0":
|
||||
version "3.55.0"
|
||||
resolved "https://registry.yarnpkg.com/@aws-sdk/querystring-parser/-/querystring-parser-3.55.0.tgz#ea35642c1b8324dd896d45185f99ad9d6c3af6d2"
|
||||
|
|
@ -760,6 +796,11 @@
|
|||
resolved "https://registry.yarnpkg.com/@aws-sdk/types/-/types-3.55.0.tgz#d524d567e2b2722f2d6be83e2417dd6d46ce1490"
|
||||
integrity sha512-wrDZjuy1CVAYxDCbm3bWQIKMGfNs7XXmG0eG4858Ixgqmq2avsIn5TORy8ynBxcXn9aekV/+tGEQ7BBSYzIVNQ==
|
||||
|
||||
"@aws-sdk/types@3.78.0":
|
||||
version "3.78.0"
|
||||
resolved "https://registry.yarnpkg.com/@aws-sdk/types/-/types-3.78.0.tgz#51dc80b2142ee20821fb9f476bdca6e541021443"
|
||||
integrity sha512-I9PTlVNSbwhIgMfmDM5as1tqRIkVZunjVmfogb2WVVPp4CaX0Ll01S0FSMSLL9k6tcQLXqh45pFRjrxCl9WKdQ==
|
||||
|
||||
"@aws-sdk/url-parser@3.55.0":
|
||||
version "3.55.0"
|
||||
resolved "https://registry.yarnpkg.com/@aws-sdk/url-parser/-/url-parser-3.55.0.tgz#03b47a45c591d52c9d00dc40c630b91094991fe7"
|
||||
|
|
|
|||
Loading…
Reference in New Issue