|
@@ -1,7 +1,7 @@
|
|
|
import { Job } from 'bullmq'
|
|
|
import { remove } from 'fs-extra/esm'
|
|
|
import { join } from 'path'
|
|
|
-import { MoveObjectStoragePayload, VideoStateType, VideoStorage } from '@peertube/peertube-models'
|
|
|
+import { MoveStoragePayload, VideoStateType, VideoStorage } from '@peertube/peertube-models'
|
|
|
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
|
|
|
import { updateTorrentMetadata } from '@server/helpers/webtorrent.js'
|
|
|
import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants.js'
|
|
@@ -9,68 +9,36 @@ import { storeHLSFileFromFilename, storeWebVideoFile } from '@server/lib/object-
|
|
|
import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths.js'
|
|
|
import { VideoPathManager } from '@server/lib/video-path-manager.js'
|
|
|
import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state.js'
|
|
|
-import { VideoJobInfoModel } from '@server/models/video/video-job-info.js'
|
|
|
-import { VideoModel } from '@server/models/video/video.js'
|
|
|
import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models/index.js'
|
|
|
+import { moveToJob, onMoveToStorageFailure } from './shared/move-video.js'
|
|
|
|
|
|
const lTagsBase = loggerTagsFactory('move-object-storage')
|
|
|
|
|
|
export async function processMoveToObjectStorage (job: Job) {
|
|
|
- const payload = job.data as MoveObjectStoragePayload
|
|
|
- logger.info('Moving video %s in job %s.', payload.videoUUID, job.id)
|
|
|
-
|
|
|
- const fileMutexReleaser = await VideoPathManager.Instance.lockFiles(payload.videoUUID)
|
|
|
-
|
|
|
- const video = await VideoModel.loadWithFiles(payload.videoUUID)
|
|
|
- // No video, maybe deleted?
|
|
|
- if (!video) {
|
|
|
- logger.info('Can\'t process job %d, video does not exist.', job.id, lTagsBase(payload.videoUUID))
|
|
|
- fileMutexReleaser()
|
|
|
- return undefined
|
|
|
- }
|
|
|
-
|
|
|
- const lTags = lTagsBase(video.uuid, video.url)
|
|
|
-
|
|
|
- try {
|
|
|
- if (video.VideoFiles) {
|
|
|
- logger.debug('Moving %d web video files for video %s.', video.VideoFiles.length, video.uuid, lTags)
|
|
|
-
|
|
|
- await moveWebVideoFiles(video)
|
|
|
- }
|
|
|
-
|
|
|
- if (video.VideoStreamingPlaylists) {
|
|
|
- logger.debug('Moving HLS playlist of %s.', video.uuid)
|
|
|
-
|
|
|
- await moveHLSFiles(video)
|
|
|
- }
|
|
|
-
|
|
|
- const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove')
|
|
|
- if (pendingMove === 0) {
|
|
|
- logger.info('Running cleanup after moving files to object storage (video %s in job %s)', video.uuid, job.id, lTags)
|
|
|
-
|
|
|
- await doAfterLastJob({ video, previousVideoState: payload.previousVideoState, isNewVideo: payload.isNewVideo })
|
|
|
- }
|
|
|
- } catch (err) {
|
|
|
- await onMoveToObjectStorageFailure(job, err)
|
|
|
-
|
|
|
- throw err
|
|
|
- } finally {
|
|
|
- fileMutexReleaser()
|
|
|
- }
|
|
|
-
|
|
|
- return payload.videoUUID
|
|
|
+ const payload = job.data as MoveStoragePayload
|
|
|
+ logger.info('Moving video %s to object storage in job %s.', payload.videoUUID, job.id)
|
|
|
+
|
|
|
+ await moveToJob({
|
|
|
+ jobId: job.id,
|
|
|
+ videoUUID: payload.videoUUID,
|
|
|
+ loggerTags: lTagsBase().tags,
|
|
|
+
|
|
|
+ moveWebVideoFiles,
|
|
|
+ moveHLSFiles,
|
|
|
+ doAfterLastMove: video => doAfterLastMove({ video, previousVideoState: payload.previousVideoState, isNewVideo: payload.isNewVideo }),
|
|
|
+ moveToFailedState: moveToFailedMoveToObjectStorageState
|
|
|
+ })
|
|
|
}
|
|
|
|
|
|
export async function onMoveToObjectStorageFailure (job: Job, err: any) {
|
|
|
- const payload = job.data as MoveObjectStoragePayload
|
|
|
-
|
|
|
- const video = await VideoModel.loadWithFiles(payload.videoUUID)
|
|
|
- if (!video) return
|
|
|
-
|
|
|
- logger.error('Cannot move video %s to object storage.', video.url, { err, ...lTagsBase(video.uuid, video.url) })
|
|
|
-
|
|
|
- await moveToFailedMoveToObjectStorageState(video)
|
|
|
- await VideoJobInfoModel.abortAllTasks(video.uuid, 'pendingMove')
|
|
|
+ const payload = job.data as MoveStoragePayload
|
|
|
+
|
|
|
+ await onMoveToStorageFailure({
|
|
|
+ videoUUID: payload.videoUUID,
|
|
|
+ err,
|
|
|
+ lTags: lTagsBase(),
|
|
|
+ moveToFailedState: moveToFailedMoveToObjectStorageState
|
|
|
+ })
|
|
|
}
|
|
|
|
|
|
// ---------------------------------------------------------------------------
|
|
@@ -107,7 +75,25 @@ async function moveHLSFiles (video: MVideoWithAllFiles) {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
-async function doAfterLastJob (options: {
|
|
|
+async function onFileMoved (options: {
|
|
|
+ videoOrPlaylist: MVideo | MStreamingPlaylistVideo
|
|
|
+ file: MVideoFile
|
|
|
+ fileUrl: string
|
|
|
+ oldPath: string
|
|
|
+}) {
|
|
|
+ const { videoOrPlaylist, file, fileUrl, oldPath } = options
|
|
|
+
|
|
|
+ file.fileUrl = fileUrl
|
|
|
+ file.storage = VideoStorage.OBJECT_STORAGE
|
|
|
+
|
|
|
+ await updateTorrentMetadata(videoOrPlaylist, file)
|
|
|
+ await file.save()
|
|
|
+
|
|
|
+ logger.debug('Removing %s because it\'s now on object storage', oldPath, lTagsBase())
|
|
|
+ await remove(oldPath)
|
|
|
+}
|
|
|
+
|
|
|
+async function doAfterLastMove (options: {
|
|
|
video: MVideoWithAllFiles
|
|
|
previousVideoState: VideoStateType
|
|
|
isNewVideo: boolean
|
|
@@ -119,11 +105,8 @@ async function doAfterLastJob (options: {
|
|
|
|
|
|
const playlistWithVideo = playlist.withVideo(video)
|
|
|
|
|
|
- // Master playlist
|
|
|
playlist.playlistUrl = await storeHLSFileFromFilename(playlistWithVideo, playlist.playlistFilename)
|
|
|
- // Sha256 segments file
|
|
|
playlist.segmentsSha256Url = await storeHLSFileFromFilename(playlistWithVideo, playlist.segmentsSha256Filename)
|
|
|
-
|
|
|
playlist.storage = VideoStorage.OBJECT_STORAGE
|
|
|
|
|
|
playlist.assignP2PMediaLoaderInfoHashes(video, playlist.VideoFiles)
|
|
@@ -132,28 +115,6 @@ async function doAfterLastJob (options: {
|
|
|
await playlist.save()
|
|
|
}
|
|
|
|
|
|
- // Remove empty hls video directory
|
|
|
- if (video.VideoStreamingPlaylists) {
|
|
|
- await remove(getHLSDirectory(video))
|
|
|
- }
|
|
|
-
|
|
|
+ await remove(getHLSDirectory(video))
|
|
|
await moveToNextState({ video, previousVideoState, isNewVideo })
|
|
|
}
|
|
|
-
|
|
|
-async function onFileMoved (options: {
|
|
|
- videoOrPlaylist: MVideo | MStreamingPlaylistVideo
|
|
|
- file: MVideoFile
|
|
|
- fileUrl: string
|
|
|
- oldPath: string
|
|
|
-}) {
|
|
|
- const { videoOrPlaylist, file, fileUrl, oldPath } = options
|
|
|
-
|
|
|
- file.fileUrl = fileUrl
|
|
|
- file.storage = VideoStorage.OBJECT_STORAGE
|
|
|
-
|
|
|
- await updateTorrentMetadata(videoOrPlaylist, file)
|
|
|
- await file.save()
|
|
|
-
|
|
|
- logger.debug('Removing %s because it\'s now on object storage', oldPath)
|
|
|
- await remove(oldPath)
|
|
|
-}
|