123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424 |
- /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
- import bytes from 'bytes'
- import { expect } from 'chai'
- import { stat } from 'fs-extra'
- import { merge } from 'lodash'
- import {
- checkTmpIsEmpty,
- expectLogDoesNotContain,
- expectStartWith,
- generateHighBitrateVideo,
- MockObjectStorage
- } from '@server/tests/shared'
- import { areMockObjectStorageTestsDisabled } from '@shared/core-utils'
- import { HttpStatusCode, VideoDetails } from '@shared/models'
- import {
- cleanupTests,
- createMultipleServers,
- createSingleServer,
- doubleFollow,
- killallServers,
- makeRawRequest,
- ObjectStorageCommand,
- PeerTubeServer,
- setAccessTokensToServers,
- waitJobs,
- webtorrentAdd
- } from '@shared/server-commands'
- async function checkFiles (options: {
- video: VideoDetails
- baseMockUrl?: string
- playlistBucket: string
- playlistPrefix?: string
- webtorrentBucket: string
- webtorrentPrefix?: string
- }) {
- const {
- video,
- playlistBucket,
- webtorrentBucket,
- baseMockUrl,
- playlistPrefix,
- webtorrentPrefix
- } = options
- let allFiles = video.files
- for (const file of video.files) {
- const baseUrl = baseMockUrl
- ? `${baseMockUrl}/${webtorrentBucket}/`
- : `http://${webtorrentBucket}.${ObjectStorageCommand.getMockEndpointHost()}/`
- const prefix = webtorrentPrefix || ''
- const start = baseUrl + prefix
- expectStartWith(file.fileUrl, start)
- const res = await makeRawRequest({ url: file.fileDownloadUrl, expectedStatus: HttpStatusCode.FOUND_302 })
- const location = res.headers['location']
- expectStartWith(location, start)
- await makeRawRequest({ url: location, expectedStatus: HttpStatusCode.OK_200 })
- }
- const hls = video.streamingPlaylists[0]
- if (hls) {
- allFiles = allFiles.concat(hls.files)
- const baseUrl = baseMockUrl
- ? `${baseMockUrl}/${playlistBucket}/`
- : `http://${playlistBucket}.${ObjectStorageCommand.getMockEndpointHost()}/`
- const prefix = playlistPrefix || ''
- const start = baseUrl + prefix
- expectStartWith(hls.playlistUrl, start)
- expectStartWith(hls.segmentsSha256Url, start)
- await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
- const resSha = await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })
- expect(JSON.stringify(resSha.body)).to.not.throw
- for (const file of hls.files) {
- expectStartWith(file.fileUrl, start)
- const res = await makeRawRequest({ url: file.fileDownloadUrl, expectedStatus: HttpStatusCode.FOUND_302 })
- const location = res.headers['location']
- expectStartWith(location, start)
- await makeRawRequest({ url: location, expectedStatus: HttpStatusCode.OK_200 })
- }
- }
- for (const file of allFiles) {
- const torrent = await webtorrentAdd(file.magnetUri, true)
- expect(torrent.files).to.be.an('array')
- expect(torrent.files.length).to.equal(1)
- expect(torrent.files[0].path).to.exist.and.to.not.equal('')
- const res = await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
- expect(res.body).to.have.length.above(100)
- }
- return allFiles.map(f => f.fileUrl)
- }
- function runTestSuite (options: {
- fixture?: string
- maxUploadPart?: string
- playlistBucket: string
- playlistPrefix?: string
- webtorrentBucket: string
- webtorrentPrefix?: string
- useMockBaseUrl?: boolean
- }) {
- const mockObjectStorage = new MockObjectStorage()
- const { fixture } = options
- let baseMockUrl: string
- let servers: PeerTubeServer[]
- let keptUrls: string[] = []
- const uuidsToDelete: string[] = []
- let deletedUrls: string[] = []
- before(async function () {
- this.timeout(120000)
- const port = await mockObjectStorage.initialize()
- baseMockUrl = options.useMockBaseUrl ? `http://127.0.0.1:${port}` : undefined
- await ObjectStorageCommand.createMockBucket(options.playlistBucket)
- await ObjectStorageCommand.createMockBucket(options.webtorrentBucket)
- const config = {
- object_storage: {
- enabled: true,
- endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(),
- region: ObjectStorageCommand.getMockRegion(),
- credentials: ObjectStorageCommand.getMockCredentialsConfig(),
- max_upload_part: options.maxUploadPart || '5MB',
- streaming_playlists: {
- bucket_name: options.playlistBucket,
- prefix: options.playlistPrefix,
- base_url: baseMockUrl
- ? `${baseMockUrl}/${options.playlistBucket}`
- : undefined
- },
- videos: {
- bucket_name: options.webtorrentBucket,
- prefix: options.webtorrentPrefix,
- base_url: baseMockUrl
- ? `${baseMockUrl}/${options.webtorrentBucket}`
- : undefined
- }
- }
- }
- servers = await createMultipleServers(2, config)
- await setAccessTokensToServers(servers)
- await doubleFollow(servers[0], servers[1])
- for (const server of servers) {
- const { uuid } = await server.videos.quickUpload({ name: 'video to keep' })
- await waitJobs(servers)
- const files = await server.videos.listFiles({ id: uuid })
- keptUrls = keptUrls.concat(files.map(f => f.fileUrl))
- }
- })
- it('Should upload a video and move it to the object storage without transcoding', async function () {
- this.timeout(40000)
- const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1', fixture })
- uuidsToDelete.push(uuid)
- await waitJobs(servers)
- for (const server of servers) {
- const video = await server.videos.get({ id: uuid })
- const files = await checkFiles({ ...options, video, baseMockUrl })
- deletedUrls = deletedUrls.concat(files)
- }
- })
- it('Should upload a video and move it to the object storage with transcoding', async function () {
- this.timeout(120000)
- const { uuid } = await servers[1].videos.quickUpload({ name: 'video 2', fixture })
- uuidsToDelete.push(uuid)
- await waitJobs(servers)
- for (const server of servers) {
- const video = await server.videos.get({ id: uuid })
- const files = await checkFiles({ ...options, video, baseMockUrl })
- deletedUrls = deletedUrls.concat(files)
- }
- })
- it('Should fetch correctly all the files', async function () {
- for (const url of deletedUrls.concat(keptUrls)) {
- await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 })
- }
- })
- it('Should correctly delete the files', async function () {
- await servers[0].videos.remove({ id: uuidsToDelete[0] })
- await servers[1].videos.remove({ id: uuidsToDelete[1] })
- await waitJobs(servers)
- for (const url of deletedUrls) {
- await makeRawRequest({ url, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
- }
- })
- it('Should have kept other files', async function () {
- for (const url of keptUrls) {
- await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 })
- }
- })
- it('Should have an empty tmp directory', async function () {
- for (const server of servers) {
- await checkTmpIsEmpty(server)
- }
- })
- it('Should not have downloaded files from object storage', async function () {
- for (const server of servers) {
- await expectLogDoesNotContain(server, 'from object storage')
- }
- })
- after(async function () {
- await mockObjectStorage.terminate()
- await cleanupTests(servers)
- })
- }
- describe('Object storage for videos', function () {
- if (areMockObjectStorageTestsDisabled()) return
- describe('Test config', function () {
- let server: PeerTubeServer
- const baseConfig = {
- object_storage: {
- enabled: true,
- endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(),
- region: ObjectStorageCommand.getMockRegion(),
- credentials: ObjectStorageCommand.getMockCredentialsConfig(),
- streaming_playlists: {
- bucket_name: ObjectStorageCommand.DEFAULT_PLAYLIST_MOCK_BUCKET
- },
- videos: {
- bucket_name: ObjectStorageCommand.DEFAULT_WEBTORRENT_MOCK_BUCKET
- }
- }
- }
- const badCredentials = {
- access_key_id: 'AKIAIOSFODNN7EXAMPLE',
- secret_access_key: 'aJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'
- }
- it('Should fail with same bucket names without prefix', function (done) {
- const config = merge({}, baseConfig, {
- object_storage: {
- streaming_playlists: {
- bucket_name: 'aaa'
- },
- videos: {
- bucket_name: 'aaa'
- }
- }
- })
- createSingleServer(1, config)
- .then(() => done(new Error('Did not throw')))
- .catch(() => done())
- })
- it('Should fail with bad credentials', async function () {
- this.timeout(60000)
- await ObjectStorageCommand.prepareDefaultMockBuckets()
- const config = merge({}, baseConfig, {
- object_storage: {
- credentials: badCredentials
- }
- })
- server = await createSingleServer(1, config)
- await setAccessTokensToServers([ server ])
- const { uuid } = await server.videos.quickUpload({ name: 'video' })
- await waitJobs([ server ], { skipDelayed: true })
- const video = await server.videos.get({ id: uuid })
- expectStartWith(video.files[0].fileUrl, server.url)
- await killallServers([ server ])
- })
- it('Should succeed with credentials from env', async function () {
- this.timeout(60000)
- await ObjectStorageCommand.prepareDefaultMockBuckets()
- const config = merge({}, baseConfig, {
- object_storage: {
- credentials: {
- access_key_id: '',
- secret_access_key: ''
- }
- }
- })
- const goodCredentials = ObjectStorageCommand.getMockCredentialsConfig()
- server = await createSingleServer(1, config, {
- env: {
- AWS_ACCESS_KEY_ID: goodCredentials.access_key_id,
- AWS_SECRET_ACCESS_KEY: goodCredentials.secret_access_key
- }
- })
- await setAccessTokensToServers([ server ])
- const { uuid } = await server.videos.quickUpload({ name: 'video' })
- await waitJobs([ server ], { skipDelayed: true })
- const video = await server.videos.get({ id: uuid })
- expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
- })
- after(async function () {
- await killallServers([ server ])
- })
- })
- describe('Test simple object storage', function () {
- runTestSuite({
- playlistBucket: 'streaming-playlists',
- webtorrentBucket: 'videos'
- })
- })
- describe('Test object storage with prefix', function () {
- runTestSuite({
- playlistBucket: 'mybucket',
- webtorrentBucket: 'mybucket',
- playlistPrefix: 'streaming-playlists_',
- webtorrentPrefix: 'webtorrent_'
- })
- })
- describe('Test object storage with prefix and base URL', function () {
- runTestSuite({
- playlistBucket: 'mybucket',
- webtorrentBucket: 'mybucket',
- playlistPrefix: 'streaming-playlists/',
- webtorrentPrefix: 'webtorrent/',
- useMockBaseUrl: true
- })
- })
- describe('Test object storage with file bigger than upload part', function () {
- let fixture: string
- const maxUploadPart = '5MB'
- before(async function () {
- this.timeout(120000)
- fixture = await generateHighBitrateVideo()
- const { size } = await stat(fixture)
- if (bytes.parse(maxUploadPart) > size) {
- throw Error(`Fixture file is too small (${size}) to make sense for this test.`)
- }
- })
- runTestSuite({
- maxUploadPart,
- playlistBucket: 'streaming-playlists',
- webtorrentBucket: 'videos',
- fixture
- })
- })
- })
|