405 lines
11 KiB
TypeScript
405 lines
11 KiB
TypeScript
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
|
|
|
|
import 'mocha'
|
|
import * as chai from 'chai'
|
|
import { merge } from 'lodash'
|
|
import {
|
|
areObjectStorageTestsDisabled,
|
|
checkTmpIsEmpty,
|
|
cleanupTests,
|
|
createMultipleServers,
|
|
createSingleServer,
|
|
doubleFollow,
|
|
expectLogDoesNotContain,
|
|
expectStartWith,
|
|
killallServers,
|
|
makeRawRequest,
|
|
MockObjectStorage,
|
|
ObjectStorageCommand,
|
|
PeerTubeServer,
|
|
setAccessTokensToServers,
|
|
waitJobs,
|
|
webtorrentAdd
|
|
} from '@shared/extra-utils'
|
|
import { HttpStatusCode, VideoDetails } from '@shared/models'
|
|
|
|
const expect = chai.expect
|
|
|
|
async function checkFiles (options: {
|
|
video: VideoDetails
|
|
|
|
baseMockUrl?: string
|
|
|
|
playlistBucket: string
|
|
playlistPrefix?: string
|
|
|
|
webtorrentBucket: string
|
|
webtorrentPrefix?: string
|
|
}) {
|
|
const {
|
|
video,
|
|
playlistBucket,
|
|
webtorrentBucket,
|
|
baseMockUrl,
|
|
playlistPrefix,
|
|
webtorrentPrefix
|
|
} = options
|
|
|
|
let allFiles = video.files
|
|
|
|
for (const file of video.files) {
|
|
const baseUrl = baseMockUrl
|
|
? `${baseMockUrl}/${webtorrentBucket}/`
|
|
: `http://${webtorrentBucket}.${ObjectStorageCommand.getEndpointHost()}/`
|
|
|
|
const prefix = webtorrentPrefix || ''
|
|
const start = baseUrl + prefix
|
|
|
|
expectStartWith(file.fileUrl, start)
|
|
|
|
const res = await makeRawRequest(file.fileDownloadUrl, HttpStatusCode.FOUND_302)
|
|
const location = res.headers['location']
|
|
expectStartWith(location, start)
|
|
|
|
await makeRawRequest(location, HttpStatusCode.OK_200)
|
|
}
|
|
|
|
const hls = video.streamingPlaylists[0]
|
|
|
|
if (hls) {
|
|
allFiles = allFiles.concat(hls.files)
|
|
|
|
const baseUrl = baseMockUrl
|
|
? `${baseMockUrl}/${playlistBucket}/`
|
|
: `http://${playlistBucket}.${ObjectStorageCommand.getEndpointHost()}/`
|
|
|
|
const prefix = playlistPrefix || ''
|
|
const start = baseUrl + prefix
|
|
|
|
expectStartWith(hls.playlistUrl, start)
|
|
expectStartWith(hls.segmentsSha256Url, start)
|
|
|
|
await makeRawRequest(hls.playlistUrl, HttpStatusCode.OK_200)
|
|
|
|
const resSha = await makeRawRequest(hls.segmentsSha256Url, HttpStatusCode.OK_200)
|
|
expect(JSON.stringify(resSha.body)).to.not.throw
|
|
|
|
for (const file of hls.files) {
|
|
expectStartWith(file.fileUrl, start)
|
|
|
|
const res = await makeRawRequest(file.fileDownloadUrl, HttpStatusCode.FOUND_302)
|
|
const location = res.headers['location']
|
|
expectStartWith(location, start)
|
|
|
|
await makeRawRequest(location, HttpStatusCode.OK_200)
|
|
}
|
|
}
|
|
|
|
for (const file of allFiles) {
|
|
const torrent = await webtorrentAdd(file.magnetUri, true)
|
|
|
|
expect(torrent.files).to.be.an('array')
|
|
expect(torrent.files.length).to.equal(1)
|
|
expect(torrent.files[0].path).to.exist.and.to.not.equal('')
|
|
|
|
const res = await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
|
|
expect(res.body).to.have.length.above(100)
|
|
}
|
|
|
|
return allFiles.map(f => f.fileUrl)
|
|
}
|
|
|
|
function runTestSuite (options: {
|
|
playlistBucket: string
|
|
playlistPrefix?: string
|
|
|
|
webtorrentBucket: string
|
|
webtorrentPrefix?: string
|
|
|
|
useMockBaseUrl?: boolean
|
|
|
|
maxUploadPart?: string
|
|
}) {
|
|
const mockObjectStorage = new MockObjectStorage()
|
|
let baseMockUrl: string
|
|
|
|
let servers: PeerTubeServer[]
|
|
|
|
let keptUrls: string[] = []
|
|
|
|
const uuidsToDelete: string[] = []
|
|
let deletedUrls: string[] = []
|
|
|
|
before(async function () {
|
|
this.timeout(120000)
|
|
|
|
const port = await mockObjectStorage.initialize()
|
|
baseMockUrl = options.useMockBaseUrl ? `http://localhost:${port}` : undefined
|
|
|
|
await ObjectStorageCommand.createBucket(options.playlistBucket)
|
|
await ObjectStorageCommand.createBucket(options.webtorrentBucket)
|
|
|
|
const config = {
|
|
object_storage: {
|
|
enabled: true,
|
|
endpoint: 'http://' + ObjectStorageCommand.getEndpointHost(),
|
|
region: ObjectStorageCommand.getRegion(),
|
|
|
|
credentials: ObjectStorageCommand.getCredentialsConfig(),
|
|
|
|
max_upload_part: options.maxUploadPart || '2MB',
|
|
|
|
streaming_playlists: {
|
|
bucket_name: options.playlistBucket,
|
|
prefix: options.playlistPrefix,
|
|
base_url: baseMockUrl
|
|
? `${baseMockUrl}/${options.playlistBucket}`
|
|
: undefined
|
|
},
|
|
|
|
videos: {
|
|
bucket_name: options.webtorrentBucket,
|
|
prefix: options.webtorrentPrefix,
|
|
base_url: baseMockUrl
|
|
? `${baseMockUrl}/${options.webtorrentBucket}`
|
|
: undefined
|
|
}
|
|
}
|
|
}
|
|
|
|
servers = await createMultipleServers(2, config)
|
|
|
|
await setAccessTokensToServers(servers)
|
|
await doubleFollow(servers[0], servers[1])
|
|
|
|
for (const server of servers) {
|
|
const { uuid } = await server.videos.quickUpload({ name: 'video to keep' })
|
|
await waitJobs(servers)
|
|
|
|
const files = await server.videos.listFiles({ id: uuid })
|
|
keptUrls = keptUrls.concat(files.map(f => f.fileUrl))
|
|
}
|
|
})
|
|
|
|
it('Should upload a video and move it to the object storage without transcoding', async function () {
|
|
this.timeout(20000)
|
|
|
|
const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1' })
|
|
uuidsToDelete.push(uuid)
|
|
|
|
await waitJobs(servers)
|
|
|
|
for (const server of servers) {
|
|
const video = await server.videos.get({ id: uuid })
|
|
const files = await checkFiles({ ...options, video, baseMockUrl })
|
|
|
|
deletedUrls = deletedUrls.concat(files)
|
|
}
|
|
})
|
|
|
|
it('Should upload a video and move it to the object storage with transcoding', async function () {
|
|
this.timeout(40000)
|
|
|
|
const { uuid } = await servers[1].videos.quickUpload({ name: 'video 2' })
|
|
uuidsToDelete.push(uuid)
|
|
|
|
await waitJobs(servers)
|
|
|
|
for (const server of servers) {
|
|
const video = await server.videos.get({ id: uuid })
|
|
const files = await checkFiles({ ...options, video, baseMockUrl })
|
|
|
|
deletedUrls = deletedUrls.concat(files)
|
|
}
|
|
})
|
|
|
|
it('Should fetch correctly all the files', async function () {
|
|
for (const url of deletedUrls.concat(keptUrls)) {
|
|
await makeRawRequest(url, HttpStatusCode.OK_200)
|
|
}
|
|
})
|
|
|
|
it('Should correctly delete the files', async function () {
|
|
await servers[0].videos.remove({ id: uuidsToDelete[0] })
|
|
await servers[1].videos.remove({ id: uuidsToDelete[1] })
|
|
|
|
await waitJobs(servers)
|
|
|
|
for (const url of deletedUrls) {
|
|
await makeRawRequest(url, HttpStatusCode.NOT_FOUND_404)
|
|
}
|
|
})
|
|
|
|
it('Should have kept other files', async function () {
|
|
for (const url of keptUrls) {
|
|
await makeRawRequest(url, HttpStatusCode.OK_200)
|
|
}
|
|
})
|
|
|
|
it('Should have an empty tmp directory', async function () {
|
|
for (const server of servers) {
|
|
await checkTmpIsEmpty(server)
|
|
}
|
|
})
|
|
|
|
it('Should not have downloaded files from object storage', async function () {
|
|
for (const server of servers) {
|
|
await expectLogDoesNotContain(server, 'from object storage')
|
|
}
|
|
})
|
|
|
|
after(async function () {
|
|
await mockObjectStorage.terminate()
|
|
|
|
await cleanupTests(servers)
|
|
})
|
|
}
|
|
|
|
describe('Object storage for videos', function () {
|
|
if (areObjectStorageTestsDisabled()) return
|
|
|
|
describe('Test config', function () {
|
|
let server: PeerTubeServer
|
|
|
|
const baseConfig = {
|
|
object_storage: {
|
|
enabled: true,
|
|
endpoint: 'http://' + ObjectStorageCommand.getEndpointHost(),
|
|
region: ObjectStorageCommand.getRegion(),
|
|
|
|
credentials: ObjectStorageCommand.getCredentialsConfig(),
|
|
|
|
streaming_playlists: {
|
|
bucket_name: ObjectStorageCommand.DEFAULT_PLAYLIST_BUCKET
|
|
},
|
|
|
|
videos: {
|
|
bucket_name: ObjectStorageCommand.DEFAULT_WEBTORRENT_BUCKET
|
|
}
|
|
}
|
|
}
|
|
|
|
const badCredentials = {
|
|
access_key_id: 'AKIAIOSFODNN7EXAMPLE',
|
|
secret_access_key: 'aJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'
|
|
}
|
|
|
|
it('Should fail with same bucket names without prefix', function (done) {
|
|
const config = merge({}, baseConfig, {
|
|
object_storage: {
|
|
streaming_playlists: {
|
|
bucket_name: 'aaa'
|
|
},
|
|
|
|
videos: {
|
|
bucket_name: 'aaa'
|
|
}
|
|
}
|
|
})
|
|
|
|
createSingleServer(1, config)
|
|
.then(() => done(new Error('Did not throw')))
|
|
.catch(() => done())
|
|
})
|
|
|
|
it('Should fail with bad credentials', async function () {
|
|
this.timeout(60000)
|
|
|
|
await ObjectStorageCommand.prepareDefaultBuckets()
|
|
|
|
const config = merge({}, baseConfig, {
|
|
object_storage: {
|
|
credentials: badCredentials
|
|
}
|
|
})
|
|
|
|
server = await createSingleServer(1, config)
|
|
await setAccessTokensToServers([ server ])
|
|
|
|
const { uuid } = await server.videos.quickUpload({ name: 'video' })
|
|
|
|
await waitJobs([ server ], true)
|
|
const video = await server.videos.get({ id: uuid })
|
|
|
|
expectStartWith(video.files[0].fileUrl, server.url)
|
|
|
|
await killallServers([ server ])
|
|
})
|
|
|
|
it('Should succeed with credentials from env', async function () {
|
|
this.timeout(60000)
|
|
|
|
await ObjectStorageCommand.prepareDefaultBuckets()
|
|
|
|
const config = merge({}, baseConfig, {
|
|
object_storage: {
|
|
credentials: {
|
|
access_key_id: '',
|
|
secret_access_key: ''
|
|
}
|
|
}
|
|
})
|
|
|
|
const goodCredentials = ObjectStorageCommand.getCredentialsConfig()
|
|
|
|
server = await createSingleServer(1, config, {
|
|
env: {
|
|
AWS_ACCESS_KEY_ID: goodCredentials.access_key_id,
|
|
AWS_SECRET_ACCESS_KEY: goodCredentials.secret_access_key
|
|
}
|
|
})
|
|
|
|
await setAccessTokensToServers([ server ])
|
|
|
|
const { uuid } = await server.videos.quickUpload({ name: 'video' })
|
|
|
|
await waitJobs([ server ], true)
|
|
const video = await server.videos.get({ id: uuid })
|
|
|
|
expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
|
|
})
|
|
|
|
after(async function () {
|
|
await killallServers([ server ])
|
|
})
|
|
})
|
|
|
|
describe('Test simple object storage', function () {
|
|
runTestSuite({
|
|
playlistBucket: 'streaming-playlists',
|
|
webtorrentBucket: 'videos'
|
|
})
|
|
})
|
|
|
|
describe('Test object storage with prefix', function () {
|
|
runTestSuite({
|
|
playlistBucket: 'mybucket',
|
|
webtorrentBucket: 'mybucket',
|
|
|
|
playlistPrefix: 'streaming-playlists_',
|
|
webtorrentPrefix: 'webtorrent_'
|
|
})
|
|
})
|
|
|
|
describe('Test object storage with prefix and base URL', function () {
|
|
runTestSuite({
|
|
playlistBucket: 'mybucket',
|
|
webtorrentBucket: 'mybucket',
|
|
|
|
playlistPrefix: 'streaming-playlists/',
|
|
webtorrentPrefix: 'webtorrent/',
|
|
|
|
useMockBaseUrl: true
|
|
})
|
|
})
|
|
|
|
describe('Test object storage with small upload part', function () {
|
|
runTestSuite({
|
|
playlistBucket: 'streaming-playlists',
|
|
webtorrentBucket: 'videos',
|
|
|
|
maxUploadPart: '5KB'
|
|
})
|
|
})
|
|
})
|