439 lines
12 KiB
TypeScript
439 lines
12 KiB
TypeScript
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
|
|
|
|
import bytes from 'bytes'
|
|
import { expect } from 'chai'
|
|
import { stat } from 'fs-extra'
|
|
import { merge } from 'lodash'
|
|
import {
|
|
checkTmpIsEmpty,
|
|
checkWebTorrentWorks,
|
|
expectLogDoesNotContain,
|
|
expectStartWith,
|
|
generateHighBitrateVideo,
|
|
MockObjectStorageProxy,
|
|
SQLCommand
|
|
} from '@server/tests/shared'
|
|
import { areMockObjectStorageTestsDisabled } from '@shared/core-utils'
|
|
import { sha1 } from '@shared/extra-utils'
|
|
import { HttpStatusCode, VideoDetails } from '@shared/models'
|
|
import {
|
|
cleanupTests,
|
|
createMultipleServers,
|
|
createSingleServer,
|
|
doubleFollow,
|
|
killallServers,
|
|
makeRawRequest,
|
|
ObjectStorageCommand,
|
|
PeerTubeServer,
|
|
setAccessTokensToServers,
|
|
waitJobs
|
|
} from '@shared/server-commands'
|
|
|
|
async function checkFiles (options: {
|
|
server: PeerTubeServer
|
|
originServer: PeerTubeServer
|
|
originSQLCommand: SQLCommand
|
|
|
|
video: VideoDetails
|
|
|
|
baseMockUrl?: string
|
|
|
|
playlistBucket: string
|
|
playlistPrefix?: string
|
|
|
|
webVideoBucket: string
|
|
webVideoPrefix?: string
|
|
}) {
|
|
const {
|
|
server,
|
|
originServer,
|
|
originSQLCommand,
|
|
video,
|
|
playlistBucket,
|
|
webVideoBucket,
|
|
baseMockUrl,
|
|
playlistPrefix,
|
|
webVideoPrefix
|
|
} = options
|
|
|
|
let allFiles = video.files
|
|
|
|
for (const file of video.files) {
|
|
const baseUrl = baseMockUrl
|
|
? `${baseMockUrl}/${webVideoBucket}/`
|
|
: `http://${webVideoBucket}.${ObjectStorageCommand.getMockEndpointHost()}/`
|
|
|
|
const prefix = webVideoPrefix || ''
|
|
const start = baseUrl + prefix
|
|
|
|
expectStartWith(file.fileUrl, start)
|
|
|
|
const res = await makeRawRequest({ url: file.fileDownloadUrl, expectedStatus: HttpStatusCode.FOUND_302 })
|
|
const location = res.headers['location']
|
|
expectStartWith(location, start)
|
|
|
|
await makeRawRequest({ url: location, expectedStatus: HttpStatusCode.OK_200 })
|
|
}
|
|
|
|
const hls = video.streamingPlaylists[0]
|
|
|
|
if (hls) {
|
|
allFiles = allFiles.concat(hls.files)
|
|
|
|
const baseUrl = baseMockUrl
|
|
? `${baseMockUrl}/${playlistBucket}/`
|
|
: `http://${playlistBucket}.${ObjectStorageCommand.getMockEndpointHost()}/`
|
|
|
|
const prefix = playlistPrefix || ''
|
|
const start = baseUrl + prefix
|
|
|
|
expectStartWith(hls.playlistUrl, start)
|
|
expectStartWith(hls.segmentsSha256Url, start)
|
|
|
|
await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
|
|
|
|
const resSha = await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })
|
|
expect(JSON.stringify(resSha.body)).to.not.throw
|
|
|
|
let i = 0
|
|
for (const file of hls.files) {
|
|
expectStartWith(file.fileUrl, start)
|
|
|
|
const res = await makeRawRequest({ url: file.fileDownloadUrl, expectedStatus: HttpStatusCode.FOUND_302 })
|
|
const location = res.headers['location']
|
|
expectStartWith(location, start)
|
|
|
|
await makeRawRequest({ url: location, expectedStatus: HttpStatusCode.OK_200 })
|
|
|
|
if (originServer.internalServerNumber === server.internalServerNumber) {
|
|
const infohash = sha1(`${2 + hls.playlistUrl}+V${i}`)
|
|
const dbInfohashes = await originSQLCommand.getPlaylistInfohash(hls.id)
|
|
|
|
expect(dbInfohashes).to.include(infohash)
|
|
}
|
|
|
|
i++
|
|
}
|
|
}
|
|
|
|
for (const file of allFiles) {
|
|
await checkWebTorrentWorks(file.magnetUri)
|
|
|
|
const res = await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
|
|
expect(res.body).to.have.length.above(100)
|
|
}
|
|
|
|
return allFiles.map(f => f.fileUrl)
|
|
}
|
|
|
|
function runTestSuite (options: {
|
|
fixture?: string
|
|
|
|
maxUploadPart?: string
|
|
|
|
playlistBucket: string
|
|
playlistPrefix?: string
|
|
|
|
webVideoBucket: string
|
|
webVideoPrefix?: string
|
|
|
|
useMockBaseUrl?: boolean
|
|
}) {
|
|
const mockObjectStorageProxy = new MockObjectStorageProxy()
|
|
const { fixture } = options
|
|
let baseMockUrl: string
|
|
|
|
let servers: PeerTubeServer[]
|
|
let sqlCommands: SQLCommand[] = []
|
|
const objectStorage = new ObjectStorageCommand()
|
|
|
|
let keptUrls: string[] = []
|
|
|
|
const uuidsToDelete: string[] = []
|
|
let deletedUrls: string[] = []
|
|
|
|
before(async function () {
|
|
this.timeout(240000)
|
|
|
|
const port = await mockObjectStorageProxy.initialize()
|
|
baseMockUrl = options.useMockBaseUrl
|
|
? `http://127.0.0.1:${port}`
|
|
: undefined
|
|
|
|
await objectStorage.createMockBucket(options.playlistBucket)
|
|
await objectStorage.createMockBucket(options.webVideoBucket)
|
|
|
|
const config = {
|
|
object_storage: {
|
|
enabled: true,
|
|
endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(),
|
|
region: ObjectStorageCommand.getMockRegion(),
|
|
|
|
credentials: ObjectStorageCommand.getMockCredentialsConfig(),
|
|
|
|
max_upload_part: options.maxUploadPart || '5MB',
|
|
|
|
streaming_playlists: {
|
|
bucket_name: options.playlistBucket,
|
|
prefix: options.playlistPrefix,
|
|
base_url: baseMockUrl
|
|
? `${baseMockUrl}/${options.playlistBucket}`
|
|
: undefined
|
|
},
|
|
|
|
web_videos: {
|
|
bucket_name: options.webVideoBucket,
|
|
prefix: options.webVideoPrefix,
|
|
base_url: baseMockUrl
|
|
? `${baseMockUrl}/${options.webVideoBucket}`
|
|
: undefined
|
|
}
|
|
}
|
|
}
|
|
|
|
servers = await createMultipleServers(2, config)
|
|
|
|
await setAccessTokensToServers(servers)
|
|
await doubleFollow(servers[0], servers[1])
|
|
|
|
for (const server of servers) {
|
|
const { uuid } = await server.videos.quickUpload({ name: 'video to keep' })
|
|
await waitJobs(servers)
|
|
|
|
const files = await server.videos.listFiles({ id: uuid })
|
|
keptUrls = keptUrls.concat(files.map(f => f.fileUrl))
|
|
}
|
|
|
|
sqlCommands = servers.map(s => new SQLCommand(s))
|
|
})
|
|
|
|
it('Should upload a video and move it to the object storage without transcoding', async function () {
|
|
this.timeout(40000)
|
|
|
|
const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1', fixture })
|
|
uuidsToDelete.push(uuid)
|
|
|
|
await waitJobs(servers)
|
|
|
|
for (const server of servers) {
|
|
const video = await server.videos.get({ id: uuid })
|
|
const files = await checkFiles({ ...options, server, originServer: servers[0], originSQLCommand: sqlCommands[0], video, baseMockUrl })
|
|
|
|
deletedUrls = deletedUrls.concat(files)
|
|
}
|
|
})
|
|
|
|
it('Should upload a video and move it to the object storage with transcoding', async function () {
|
|
this.timeout(120000)
|
|
|
|
const { uuid } = await servers[1].videos.quickUpload({ name: 'video 2', fixture })
|
|
uuidsToDelete.push(uuid)
|
|
|
|
await waitJobs(servers)
|
|
|
|
for (const server of servers) {
|
|
const video = await server.videos.get({ id: uuid })
|
|
const files = await checkFiles({ ...options, server, originServer: servers[0], originSQLCommand: sqlCommands[0], video, baseMockUrl })
|
|
|
|
deletedUrls = deletedUrls.concat(files)
|
|
}
|
|
})
|
|
|
|
it('Should fetch correctly all the files', async function () {
|
|
for (const url of deletedUrls.concat(keptUrls)) {
|
|
await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 })
|
|
}
|
|
})
|
|
|
|
it('Should correctly delete the files', async function () {
|
|
await servers[0].videos.remove({ id: uuidsToDelete[0] })
|
|
await servers[1].videos.remove({ id: uuidsToDelete[1] })
|
|
|
|
await waitJobs(servers)
|
|
|
|
for (const url of deletedUrls) {
|
|
await makeRawRequest({ url, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
|
|
}
|
|
})
|
|
|
|
it('Should have kept other files', async function () {
|
|
for (const url of keptUrls) {
|
|
await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 })
|
|
}
|
|
})
|
|
|
|
it('Should have an empty tmp directory', async function () {
|
|
for (const server of servers) {
|
|
await checkTmpIsEmpty(server)
|
|
}
|
|
})
|
|
|
|
it('Should not have downloaded files from object storage', async function () {
|
|
for (const server of servers) {
|
|
await expectLogDoesNotContain(server, 'from object storage')
|
|
}
|
|
})
|
|
|
|
after(async function () {
|
|
await mockObjectStorageProxy.terminate()
|
|
await objectStorage.cleanupMock()
|
|
|
|
for (const sqlCommand of sqlCommands) {
|
|
await sqlCommand.cleanup()
|
|
}
|
|
|
|
await cleanupTests(servers)
|
|
})
|
|
}
|
|
|
|
describe('Object storage for videos', function () {
|
|
if (areMockObjectStorageTestsDisabled()) return
|
|
|
|
const objectStorage = new ObjectStorageCommand()
|
|
|
|
describe('Test config', function () {
|
|
let server: PeerTubeServer
|
|
|
|
const baseConfig = objectStorage.getDefaultMockConfig()
|
|
|
|
const badCredentials = {
|
|
access_key_id: 'AKIAIOSFODNN7EXAMPLE',
|
|
secret_access_key: 'aJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'
|
|
}
|
|
|
|
it('Should fail with same bucket names without prefix', function (done) {
|
|
const config = merge({}, baseConfig, {
|
|
object_storage: {
|
|
streaming_playlists: {
|
|
bucket_name: 'aaa'
|
|
},
|
|
|
|
web_videos: {
|
|
bucket_name: 'aaa'
|
|
}
|
|
}
|
|
})
|
|
|
|
createSingleServer(1, config)
|
|
.then(() => done(new Error('Did not throw')))
|
|
.catch(() => done())
|
|
})
|
|
|
|
it('Should fail with bad credentials', async function () {
|
|
this.timeout(60000)
|
|
|
|
await objectStorage.prepareDefaultMockBuckets()
|
|
|
|
const config = merge({}, baseConfig, {
|
|
object_storage: {
|
|
credentials: badCredentials
|
|
}
|
|
})
|
|
|
|
server = await createSingleServer(1, config)
|
|
await setAccessTokensToServers([ server ])
|
|
|
|
const { uuid } = await server.videos.quickUpload({ name: 'video' })
|
|
|
|
await waitJobs([ server ], { skipDelayed: true })
|
|
const video = await server.videos.get({ id: uuid })
|
|
|
|
expectStartWith(video.files[0].fileUrl, server.url)
|
|
|
|
await killallServers([ server ])
|
|
})
|
|
|
|
it('Should succeed with credentials from env', async function () {
|
|
this.timeout(60000)
|
|
|
|
await objectStorage.prepareDefaultMockBuckets()
|
|
|
|
const config = merge({}, baseConfig, {
|
|
object_storage: {
|
|
credentials: {
|
|
access_key_id: '',
|
|
secret_access_key: ''
|
|
}
|
|
}
|
|
})
|
|
|
|
const goodCredentials = ObjectStorageCommand.getMockCredentialsConfig()
|
|
|
|
server = await createSingleServer(1, config, {
|
|
env: {
|
|
AWS_ACCESS_KEY_ID: goodCredentials.access_key_id,
|
|
AWS_SECRET_ACCESS_KEY: goodCredentials.secret_access_key
|
|
}
|
|
})
|
|
|
|
await setAccessTokensToServers([ server ])
|
|
|
|
const { uuid } = await server.videos.quickUpload({ name: 'video' })
|
|
|
|
await waitJobs([ server ], { skipDelayed: true })
|
|
const video = await server.videos.get({ id: uuid })
|
|
|
|
expectStartWith(video.files[0].fileUrl, objectStorage.getMockWebVideosBaseUrl())
|
|
})
|
|
|
|
after(async function () {
|
|
await objectStorage.cleanupMock()
|
|
|
|
await cleanupTests([ server ])
|
|
})
|
|
})
|
|
|
|
describe('Test simple object storage', function () {
|
|
runTestSuite({
|
|
playlistBucket: objectStorage.getMockBucketName('streaming-playlists'),
|
|
webVideoBucket: objectStorage.getMockBucketName('web-videos')
|
|
})
|
|
})
|
|
|
|
describe('Test object storage with prefix', function () {
|
|
runTestSuite({
|
|
playlistBucket: objectStorage.getMockBucketName('mybucket'),
|
|
webVideoBucket: objectStorage.getMockBucketName('mybucket'),
|
|
|
|
playlistPrefix: 'streaming-playlists_',
|
|
webVideoPrefix: 'webvideo_'
|
|
})
|
|
})
|
|
|
|
describe('Test object storage with prefix and base URL', function () {
|
|
runTestSuite({
|
|
playlistBucket: objectStorage.getMockBucketName('mybucket'),
|
|
webVideoBucket: objectStorage.getMockBucketName('mybucket'),
|
|
|
|
playlistPrefix: 'streaming-playlists/',
|
|
webVideoPrefix: 'webvideo/',
|
|
|
|
useMockBaseUrl: true
|
|
})
|
|
})
|
|
|
|
describe('Test object storage with file bigger than upload part', function () {
|
|
let fixture: string
|
|
const maxUploadPart = '5MB'
|
|
|
|
before(async function () {
|
|
this.timeout(120000)
|
|
|
|
fixture = await generateHighBitrateVideo()
|
|
|
|
const { size } = await stat(fixture)
|
|
|
|
if (bytes.parse(maxUploadPart) > size) {
|
|
throw Error(`Fixture file is too small (${size}) to make sense for this test.`)
|
|
}
|
|
})
|
|
|
|
runTestSuite({
|
|
maxUploadPart,
|
|
playlistBucket: objectStorage.getMockBucketName('streaming-playlists'),
|
|
webVideoBucket: objectStorage.getMockBucketName('web-videos'),
|
|
fixture
|
|
})
|
|
})
|
|
})
|