Add peertube runner cli

.eslintrc.json

@@ -126,7 +126,8 @@
     ]
   },
   "ignorePatterns": [
-    "node_modules/"
+    "node_modules/",
+    "server/tests/fixtures"
   ],
   "parserOptions": {
     "EXPERIMENTAL_useSourceOfProjectReferenceRedirect": true,
@@ -135,7 +136,8 @@
       "./shared/tsconfig.json",
       "./scripts/tsconfig.json",
       "./server/tsconfig.json",
-      "./server/tools/tsconfig.json"
+      "./server/tools/tsconfig.json",
+      "./packages/peertube-runner/tsconfig.json"
     ]
   }
 }

.github/actions/reusable-prepare-peertube-build/action.yml

@@ -29,3 +29,7 @@ runs:
     - name: Install dependencies
       shell: bash
       run: yarn install --frozen-lockfile
+
+    - name: Install peertube runner dependencies
+      shell: bash
+      run: cd packages/peertube-runner && yarn install --frozen-lockfile

package.json

@@ -29,6 +29,7 @@
     "build:embed": "bash ./scripts/build/embed.sh",
     "build:server": "bash ./scripts/build/server.sh",
     "build:client": "bash ./scripts/build/client.sh",
+    "build:peertube-runner": "bash ./scripts/build/peertube-runner.sh",
     "clean:client": "bash ./scripts/clean/client/index.sh",
     "clean:server:test": "bash ./scripts/clean/server/test.sh",
     "i18n:update": "bash ./scripts/i18n/update.sh",
@@ -41,6 +42,7 @@
     "dev:embed": "bash ./scripts/dev/embed.sh",
     "dev:client": "bash ./scripts/dev/client.sh",
     "dev:cli": "bash ./scripts/dev/cli.sh",
+    "dev:peertube-runner": "bash ./scripts/dev/peertube-runner.sh",
     "start": "node dist/server",
     "start:server": "node dist/server --no-client",
     "update-host": "node ./dist/scripts/update-host.js",

packages/peertube-runner/.gitignore

@@ -0,0 +1,2 @@
+node_modules
+dist

packages/peertube-runner/README.md

@@ -0,0 +1 @@
+# PeerTube runner

packages/peertube-runner/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "peertube-runner",
+  "version": "1.0.0",
+  "main": "dist/peertube-runner.js",
+  "license": "AGPL-3.0",
+  "dependencies": {},
+  "devDependencies": {
+    "@commander-js/extra-typings": "^10.0.3",
+    "@iarna/toml": "^2.2.5",
+    "env-paths": "^3.0.0",
+    "esbuild": "^0.17.15",
+    "net-ipc": "^2.0.1",
+    "pino": "^8.11.0",
+    "pino-pretty": "^10.0.0"
+  }
+}

packages/peertube-runner/peertube-runner.ts

@@ -0,0 +1,84 @@
+import { Command, InvalidArgumentError } from '@commander-js/extra-typings'
+import { listRegistered, registerRunner, unregisterRunner } from './register'
+import { RunnerServer } from './server'
+import { ConfigManager, logger } from './shared'
+
+const program = new Command()
+  .option(
+    '--id <id>',
+    'Runner server id, so you can run multiple PeerTube server runners with different configurations on the same machine',
+    'default'
+  )
+  .option('--verbose', 'Run in verbose mode')
+  .hook('preAction', thisCommand => {
+    const options = thisCommand.opts()
+
+    ConfigManager.Instance.init(options.id)
+
+    if (options.verbose === true) {
+      logger.level = 'debug'
+    }
+  })
+
+program.command('server')
+  .description('Run in server mode, to execute remote jobs of registered PeerTube instances')
+  .action(async () => {
+    try {
+      await RunnerServer.Instance.run()
+    } catch (err) {
+      console.error('Cannot run PeerTube runner as server mode', err)
+      process.exit(-1)
+    }
+  })
+
+program.command('register')
+  .description('Register a new PeerTube instance to process runner jobs')
+  .requiredOption('--url <url>', 'PeerTube instance URL', parseUrl)
+  .requiredOption('--registration-token <token>', 'Runner registration token (can be found in PeerTube instance administration')
+  .requiredOption('--runner-name <name>', 'Runner name')
+  .option('--runner-description <description>', 'Runner description')
+  .action(async options => {
+    try {
+      await registerRunner(options)
+    } catch (err) {
+      console.error('Cannot register this PeerTube runner.', err)
+      process.exit(-1)
+    }
+  })
+
+program.command('unregister')
+  .description('Unregister the runner from PeerTube instance')
+  .requiredOption('--url <url>', 'PeerTube instance URL', parseUrl)
+  .action(async options => {
+    try {
+      await unregisterRunner(options)
+    } catch (err) {
+      console.error('Cannot unregister this PeerTube runner.', err)
+      process.exit(-1)
+    }
+  })
+
+program.command('list-registered')
+  .description('List registered PeerTube instances')
+  .action(async () => {
+    try {
+      await listRegistered()
+    } catch (err) {
+      console.error('Cannot list registered PeerTube instances.', err)
+      process.exit(-1)
+    }
+  })
+
+program.parse()
+
+// ---------------------------------------------------------------------------
+// Private
+// ---------------------------------------------------------------------------
+
+function parseUrl (url: string) {
+  if (url.startsWith('http://') !== true && url.startsWith('https://') !== true) {
+    throw new InvalidArgumentError('URL should start with a http:// or https://')
+  }
+
+  return url
+}

packages/peertube-runner/register/index.ts

@@ -0,0 +1 @@
+export * from './register'

packages/peertube-runner/register/register.ts

@@ -0,0 +1,35 @@
+import { IPCClient } from '../shared/ipc'
+
+export async function registerRunner (options: {
+  url: string
+  registrationToken: string
+  runnerName: string
+  runnerDescription?: string
+}) {
+  const client = new IPCClient()
+  await client.run()
+
+  await client.askRegister(options)
+
+  client.stop()
+}
+
+export async function unregisterRunner (options: {
+  url: string
+}) {
+  const client = new IPCClient()
+  await client.run()
+
+  await client.askUnregister(options)
+
+  client.stop()
+}
+
+export async function listRegistered () {
+  const client = new IPCClient()
+  await client.run()
+
+  await client.askListRegistered()
+
+  client.stop()
+}

packages/peertube-runner/server/index.ts

@@ -0,0 +1 @@
+export * from './server'

packages/peertube-runner/server/process/index.ts

@@ -0,0 +1,2 @@
+export * from './shared'
+export * from './process'

packages/peertube-runner/server/process/process.ts

@@ -0,0 +1,30 @@
+import { logger } from 'packages/peertube-runner/shared/logger'
+import {
+  RunnerJobLiveRTMPHLSTranscodingPayload,
+  RunnerJobVODAudioMergeTranscodingPayload,
+  RunnerJobVODHLSTranscodingPayload,
+  RunnerJobVODWebVideoTranscodingPayload
+} from '@shared/models'
+import { processAudioMergeTranscoding, processHLSTranscoding, ProcessOptions, processWebVideoTranscoding } from './shared'
+import { ProcessLiveRTMPHLSTranscoding } from './shared/process-live'
+
+export async function processJob (options: ProcessOptions) {
+  const { server, job } = options
+
+  logger.info(`[${server.url}] Processing job of type ${job.type}: ${job.uuid}`, { payload: job.payload })
+
+  if (job.type === 'vod-audio-merge-transcoding') {
+    await processAudioMergeTranscoding(options as ProcessOptions<RunnerJobVODAudioMergeTranscodingPayload>)
+  } else if (job.type === 'vod-web-video-transcoding') {
+    await processWebVideoTranscoding(options as ProcessOptions<RunnerJobVODWebVideoTranscodingPayload>)
+  } else if (job.type === 'vod-hls-transcoding') {
+    await processHLSTranscoding(options as ProcessOptions<RunnerJobVODHLSTranscodingPayload>)
+  } else if (job.type === 'live-rtmp-hls-transcoding') {
+    await new ProcessLiveRTMPHLSTranscoding(options as ProcessOptions<RunnerJobLiveRTMPHLSTranscodingPayload>).process()
+  } else {
+    logger.error(`Unknown job ${job.type} to process`)
+    return
+  }
+
+  logger.info(`[${server.url}] Finished processing job of type ${job.type}: ${job.uuid}`)
+}

packages/peertube-runner/server/process/shared/common.ts

@@ -0,0 +1,91 @@
+import { throttle } from 'lodash'
+import { ConfigManager, downloadFile, logger } from 'packages/peertube-runner/shared'
+import { join } from 'path'
+import { buildUUID } from '@shared/extra-utils'
+import { FFmpegLive, FFmpegVOD } from '@shared/ffmpeg'
+import { RunnerJob, RunnerJobPayload } from '@shared/models'
+import { PeerTubeServer } from '@shared/server-commands'
+import { getTranscodingLogger } from './transcoding-logger'
+import { getAvailableEncoders, getEncodersToTry } from './transcoding-profiles'
+
+export type JobWithToken <T extends RunnerJobPayload = RunnerJobPayload> = RunnerJob<T> & { jobToken: string }
+
+export type ProcessOptions <T extends RunnerJobPayload = RunnerJobPayload> = {
+  server: PeerTubeServer
+  job: JobWithToken<T>
+  runnerToken: string
+}
+
+export async function downloadInputFile (options: {
+  url: string
+  job: JobWithToken
+  runnerToken: string
+}) {
+  const { url, job, runnerToken } = options
+  const destination = join(ConfigManager.Instance.getTranscodingDirectory(), buildUUID())
+
+  await downloadFile({ url, jobToken: job.jobToken, runnerToken, destination })
+
+  return destination
+}
+
+export async function updateTranscodingProgress (options: {
+  server: PeerTubeServer
+  runnerToken: string
+  job: JobWithToken
+  progress: number
+}) {
+  const { server, job, runnerToken, progress } = options
+
+  return server.runnerJobs.update({ jobToken: job.jobToken, jobUUID: job.uuid, runnerToken, progress })
+}
+
+export function buildFFmpegVOD (options: {
+  server: PeerTubeServer
+  runnerToken: string
+  job: JobWithToken
+}) {
+  const { server, job, runnerToken } = options
+
+  const updateInterval = ConfigManager.Instance.isTestInstance()
+    ? 500
+    : 60000
+
+  const updateJobProgress = throttle((progress: number) => {
+    if (progress < 0 || progress > 100) progress = undefined
+
+    updateTranscodingProgress({ server, job, runnerToken, progress })
+      .catch(err => logger.error({ err }, 'Cannot send job progress'))
+  }, updateInterval, { trailing: false })
+
+  const config = ConfigManager.Instance.getConfig()
+
+  return new FFmpegVOD({
+    niceness: config.ffmpeg.nice,
+    threads: config.ffmpeg.threads,
+    tmpDirectory: ConfigManager.Instance.getTranscodingDirectory(),
+    profile: 'default',
+    availableEncoders: {
+      available: getAvailableEncoders(),
+      encodersToTry: getEncodersToTry()
+    },
+    logger: getTranscodingLogger(),
+    updateJobProgress
+  })
+}
+
+export function buildFFmpegLive () {
+  const config = ConfigManager.Instance.getConfig()
+
+  return new FFmpegLive({
+    niceness: config.ffmpeg.nice,
+    threads: config.ffmpeg.threads,
+    tmpDirectory: ConfigManager.Instance.getTranscodingDirectory(),
+    profile: 'default',
+    availableEncoders: {
+      available: getAvailableEncoders(),
+      encodersToTry: getEncodersToTry()
+    },
+    logger: getTranscodingLogger()
+  })
+}

packages/peertube-runner/server/process/shared/index.ts

@@ -0,0 +1,4 @@
+export * from './common'
+export * from './process-vod'
+export * from './transcoding-logger'
+export * from './transcoding-profiles'

packages/peertube-runner/server/process/shared/process-live.ts

@@ -0,0 +1,295 @@
+import { FSWatcher, watch } from 'chokidar'
+import { FfmpegCommand } from 'fluent-ffmpeg'
+import { ensureDir, remove } from 'fs-extra'
+import { logger } from 'packages/peertube-runner/shared'
+import { basename, join } from 'path'
+import { wait } from '@shared/core-utils'
+import { buildUUID } from '@shared/extra-utils'
+import { ffprobePromise, getVideoStreamBitrate, getVideoStreamDimensionsInfo, hasAudioStream } from '@shared/ffmpeg'
+import {
+  LiveRTMPHLSTranscodingSuccess,
+  LiveRTMPHLSTranscodingUpdatePayload,
+  PeerTubeProblemDocument,
+  RunnerJobLiveRTMPHLSTranscodingPayload,
+  ServerErrorCode
+} from '@shared/models'
+import { ConfigManager } from '../../../shared/config-manager'
+import { buildFFmpegLive, ProcessOptions } from './common'
+
+export class ProcessLiveRTMPHLSTranscoding {
+
+  private readonly outputPath: string
+  private readonly fsWatchers: FSWatcher[] = []
+
+  private readonly playlistsCreated = new Set<string>()
+  private allPlaylistsCreated = false
+
+  private ffmpegCommand: FfmpegCommand
+
+  private ended = false
+  private errored = false
+
+  constructor (private readonly options: ProcessOptions<RunnerJobLiveRTMPHLSTranscodingPayload>) {
+    this.outputPath = join(ConfigManager.Instance.getTranscodingDirectory(), buildUUID())
+  }
+
+  process () {
+    const job = this.options.job
+    const payload = job.payload
+
+    return new Promise<void>(async (res, rej) => {
+      try {
+        await ensureDir(this.outputPath)
+
+        logger.info(`Probing ${payload.input.rtmpUrl}`)
+        const probe = await ffprobePromise(payload.input.rtmpUrl)
+        logger.info({ probe }, `Probed ${payload.input.rtmpUrl}`)
+
+        const hasAudio = await hasAudioStream(payload.input.rtmpUrl, probe)
+        const bitrate = await getVideoStreamBitrate(payload.input.rtmpUrl, probe)
+        const { ratio } = await getVideoStreamDimensionsInfo(payload.input.rtmpUrl, probe)
+
+        const m3u8Watcher = watch(this.outputPath + '/*.m3u8')
+        this.fsWatchers.push(m3u8Watcher)
+
+        const tsWatcher = watch(this.outputPath + '/*.ts')
+        this.fsWatchers.push(tsWatcher)
+
+        m3u8Watcher.on('change', p => {
+          logger.debug(`${p} m3u8 playlist changed`)
+        })
+
+        m3u8Watcher.on('add', p => {
+          this.playlistsCreated.add(p)
+
+          if (this.playlistsCreated.size === this.options.job.payload.output.toTranscode.length + 1) {
+            this.allPlaylistsCreated = true
+            logger.info('All m3u8 playlists are created.')
+          }
+        })
+
+        tsWatcher.on('add', p => {
+          this.sendAddedChunkUpdate(p)
+            .catch(err => this.onUpdateError(err, rej))
+        })
+
+        tsWatcher.on('unlink', p => {
+          this.sendDeletedChunkUpdate(p)
+            .catch(err => this.onUpdateError(err, rej))
+        })
+
+        this.ffmpegCommand = await buildFFmpegLive().getLiveTranscodingCommand({
+          inputUrl: payload.input.rtmpUrl,
+
+          outPath: this.outputPath,
+          masterPlaylistName: 'master.m3u8',
+
+          segmentListSize: payload.output.segmentListSize,
+          segmentDuration: payload.output.segmentDuration,
+
+          toTranscode: payload.output.toTranscode,
+
+          bitrate,
+          ratio,
+
+          hasAudio
+        })
+
+        logger.info(`Running live transcoding for ${payload.input.rtmpUrl}`)
+
+        this.ffmpegCommand.on('error', (err, stdout, stderr) => {
+          this.onFFmpegError({ err, stdout, stderr })
+
+          res()
+        })
+
+        this.ffmpegCommand.on('end', () => {
+          this.onFFmpegEnded()
+            .catch(err => logger.error({ err }, 'Error in FFmpeg end handler'))
+
+          res()
+        })
+
+        this.ffmpegCommand.run()
+      } catch (err) {
+        rej(err)
+      }
+    })
+  }
+
+  // ---------------------------------------------------------------------------
+
+  private onUpdateError (err: Error, reject: (reason?: any) => void) {
+    if (this.errored) return
+    if (this.ended) return
+
+    this.errored = true
+
+    reject(err)
+    this.ffmpegCommand.kill('SIGINT')
+
+    const type = ((err as any).res?.body as PeerTubeProblemDocument)?.code
+    if (type === ServerErrorCode.RUNNER_JOB_NOT_IN_PROCESSING_STATE) {
+      logger.info({ err }, 'Stopping transcoding as the job is not in processing state anymore')
+    } else {
+      logger.error({ err }, 'Cannot send update after added/deleted chunk, stopping live transcoding')
+
+      this.sendError(err)
+        .catch(subErr => logger.error({ err: subErr }, 'Cannot send error'))
+    }
+
+    this.cleanup()
+  }
+
+  // ---------------------------------------------------------------------------
+
+  private onFFmpegError (options: {
+    err: any
+    stdout: string
+    stderr: string
+  }) {
+    const { err, stdout, stderr } = options
+
+    // Don't care that we killed the ffmpeg process
+    if (err?.message?.includes('Exiting normally')) return
+    if (this.errored) return
+    if (this.ended) return
+
+    this.errored = true
+
+    logger.error({ err, stdout, stderr }, 'FFmpeg transcoding error.')
+
+    this.sendError(err)
+      .catch(subErr => logger.error({ err: subErr }, 'Cannot send error'))
+
+    this.cleanup()
+  }
+
+  private async sendError (err: Error) {
+    await this.options.server.runnerJobs.error({
+      jobToken: this.options.job.jobToken,
+      jobUUID: this.options.job.uuid,
+      runnerToken: this.options.runnerToken,
+      message: err.message
+    })
+  }
+
+  // ---------------------------------------------------------------------------
+
+  private async onFFmpegEnded () {
+    if (this.ended) return
+
+    this.ended = true
+    logger.info('FFmpeg ended, sending success to server')
+
+    // Wait last ffmpeg chunks generation
+    await wait(1500)
+
+    this.sendSuccess()
+      .catch(err => logger.error({ err }, 'Cannot send success'))
+
+    this.cleanup()
+  }
+
+  private async sendSuccess () {
+    const successBody: LiveRTMPHLSTranscodingSuccess = {}
+
+    await this.options.server.runnerJobs.success({
+      jobToken: this.options.job.jobToken,
+      jobUUID: this.options.job.uuid,
+      runnerToken: this.options.runnerToken,
+      payload: successBody
+    })
+  }
+
+  // ---------------------------------------------------------------------------
+
+  private sendDeletedChunkUpdate (deletedChunk: string) {
+    if (this.ended) return
+
+    logger.debug(`Sending removed live chunk ${deletedChunk} update`)
+
+    const videoChunkFilename = basename(deletedChunk)
+
+    let payload: LiveRTMPHLSTranscodingUpdatePayload = {
+      type: 'remove-chunk',
+      videoChunkFilename
+    }
+
+    if (this.allPlaylistsCreated) {
+      const playlistName = this.getPlaylistName(videoChunkFilename)
+
+      payload = {
+        ...payload,
+        masterPlaylistFile: join(this.outputPath, 'master.m3u8'),
+        resolutionPlaylistFilename: playlistName,
+        resolutionPlaylistFile: join(this.outputPath, playlistName)
+      }
+    }
+
+    return this.updateWithRetry(payload)
+  }
+
+  private sendAddedChunkUpdate (addedChunk: string) {
+    if (this.ended) return
+
+    logger.debug(`Sending added live chunk ${addedChunk} update`)
+
+    const videoChunkFilename = basename(addedChunk)
+
+    let payload: LiveRTMPHLSTranscodingUpdatePayload = {
+      type: 'add-chunk',
+      videoChunkFilename,
+      videoChunkFile: addedChunk
+    }
+
+    if (this.allPlaylistsCreated) {
+      const playlistName = this.getPlaylistName(videoChunkFilename)
+
+      payload = {
+        ...payload,
+        masterPlaylistFile: join(this.outputPath, 'master.m3u8'),
+        resolutionPlaylistFilename: playlistName,
+        resolutionPlaylistFile: join(this.outputPath, playlistName)
+      }
+    }
+
+    return this.updateWithRetry(payload)
+  }
+
+  private async updateWithRetry (payload: LiveRTMPHLSTranscodingUpdatePayload, currentTry = 1) {
+    if (this.ended || this.errored) return
+
+    try {
+      await this.options.server.runnerJobs.update({
+        jobToken: this.options.job.jobToken,
+        jobUUID: this.options.job.uuid,
+        runnerToken: this.options.runnerToken,
+        payload
+      })
+    } catch (err) {
+      if (currentTry >= 3) throw err
+
+      logger.warn({ err }, 'Will retry update after error')
+      await wait(250)
+
+      return this.updateWithRetry(payload, currentTry + 1)
+    }
+  }
+
+  private getPlaylistName (videoChunkFilename: string) {
+    return `${videoChunkFilename.split('-')[0]}.m3u8`
+  }
+
+  // ---------------------------------------------------------------------------
+
+  private cleanup () {
+    for (const fsWatcher of this.fsWatchers) {
+      fsWatcher.close()
+        .catch(err => logger.error({ err }, 'Cannot close watcher'))
+    }
+
+    remove(this.outputPath)
+      .catch(err => logger.error({ err }, `Cannot remove ${this.outputPath}`))
+  }
+}

packages/peertube-runner/server/process/shared/process-vod.ts

@@ -0,0 +1,131 @@
+import { remove } from 'fs-extra'
+import { join } from 'path'
+import { buildUUID } from '@shared/extra-utils'
+import {
+  RunnerJobVODAudioMergeTranscodingPayload,
+  RunnerJobVODHLSTranscodingPayload,
+  RunnerJobVODWebVideoTranscodingPayload,
+  VODAudioMergeTranscodingSuccess,
+  VODHLSTranscodingSuccess,
+  VODWebVideoTranscodingSuccess
+} from '@shared/models'
+import { ConfigManager } from '../../../shared/config-manager'
+import { buildFFmpegVOD, downloadInputFile, ProcessOptions } from './common'
+
+export async function processWebVideoTranscoding (options: ProcessOptions<RunnerJobVODWebVideoTranscodingPayload>) {
+  const { server, job, runnerToken } = options
+  const payload = job.payload
+
+  const inputPath = await downloadInputFile({ url: payload.input.videoFileUrl, runnerToken, job })
+
+  const ffmpegVod = buildFFmpegVOD({ job, server, runnerToken })
+
+  const outputPath = join(ConfigManager.Instance.getTranscodingDirectory(), `output-${buildUUID()}.mp4`)
+
+  await ffmpegVod.transcode({
+    type: 'video',
+
+    inputPath,
+
+    outputPath,
+
+    inputFileMutexReleaser: () => {},
+
+    resolution: payload.output.resolution,
+    fps: payload.output.fps
+  })
+
+  const successBody: VODWebVideoTranscodingSuccess = {
+    videoFile: outputPath
+  }
+
+  await server.runnerJobs.success({
+    jobToken: job.jobToken,
+    jobUUID: job.uuid,
+    runnerToken,
+    payload: successBody
+  })
+
+  await remove(outputPath)
+}
+
+export async function processHLSTranscoding (options: ProcessOptions<RunnerJobVODHLSTranscodingPayload>) {
+  const { server, job, runnerToken } = options
+  const payload = job.payload
+
+  const inputPath = await downloadInputFile({ url: payload.input.videoFileUrl, runnerToken, job })
+  const uuid = buildUUID()
+
+  const outputPath = join(ConfigManager.Instance.getTranscodingDirectory(), `${uuid}-${payload.output.resolution}.m3u8`)
+  const videoFilename = `${uuid}-${payload.output.resolution}-fragmented.mp4`
+  const videoPath = join(join(ConfigManager.Instance.getTranscodingDirectory(), videoFilename))
+
+  const ffmpegVod = buildFFmpegVOD({ job, server, runnerToken })
+
+  await ffmpegVod.transcode({
+    type: 'hls',
+    copyCodecs: false,
+    inputPath,
+    hlsPlaylist: { videoFilename },
+    outputPath,
+
+    inputFileMutexReleaser: () => {},
+
+    resolution: payload.output.resolution,
+    fps: payload.output.fps
+  })
+
+  const successBody: VODHLSTranscodingSuccess = {
+    resolutionPlaylistFile: outputPath,
+    videoFile: videoPath
+  }
+
+  await server.runnerJobs.success({
+    jobToken: job.jobToken,
+    jobUUID: job.uuid,
+    runnerToken,
+    payload: successBody
+  })
+
+  await remove(outputPath)
+  await remove(videoPath)
+}
+
+export async function processAudioMergeTranscoding (options: ProcessOptions<RunnerJobVODAudioMergeTranscodingPayload>) {
+  const { server, job, runnerToken } = options
+  const payload = job.payload
+
+  const audioPath = await downloadInputFile({ url: payload.input.audioFileUrl, runnerToken, job })
+  const inputPath = await downloadInputFile({ url: payload.input.previewFileUrl, runnerToken, job })
+
+  const outputPath = join(ConfigManager.Instance.getTranscodingDirectory(), `output-${buildUUID()}.mp4`)
+
+  const ffmpegVod = buildFFmpegVOD({ job, server, runnerToken })
+
+  await ffmpegVod.transcode({
+    type: 'merge-audio',
+
+    audioPath,
+    inputPath,
+
+    outputPath,
+
+    inputFileMutexReleaser: () => {},
+
+    resolution: payload.output.resolution,
+    fps: payload.output.fps
+  })
+
+  const successBody: VODAudioMergeTranscodingSuccess = {
+    videoFile: outputPath
+  }
+
+  await server.runnerJobs.success({
+    jobToken: job.jobToken,
+    jobUUID: job.uuid,
+    runnerToken,
+    payload: successBody
+  })
+
+  await remove(outputPath)
+}

packages/peertube-runner/server/process/shared/transcoding-logger.ts

@@ -0,0 +1,10 @@
+import { logger } from 'packages/peertube-runner/shared/logger'
+
+export function getTranscodingLogger () {
+  return {
+    info: logger.info.bind(logger),
+    debug: logger.debug.bind(logger),
+    warn: logger.warn.bind(logger),
+    error: logger.error.bind(logger)
+  }
+}

packages/peertube-runner/server/process/shared/transcoding-profiles.ts

@@ -0,0 +1,134 @@
+import { getAverageBitrate, getMinLimitBitrate } from '@shared/core-utils'
+import { buildStreamSuffix, ffprobePromise, getAudioStream, getMaxAudioBitrate } from '@shared/ffmpeg'
+import { EncoderOptionsBuilder, EncoderOptionsBuilderParams, VideoResolution } from '@shared/models'
+
+const defaultX264VODOptionsBuilder: EncoderOptionsBuilder = (options: EncoderOptionsBuilderParams) => {
+  const { fps, inputRatio, inputBitrate, resolution } = options
+
+  const targetBitrate = getTargetBitrate({ inputBitrate, ratio: inputRatio, fps, resolution })
+
+  return {
+    outputOptions: [
+      ...getCommonOutputOptions(targetBitrate),
+
+      `-r ${fps}`
+    ]
+  }
+}
+
+const defaultX264LiveOptionsBuilder: EncoderOptionsBuilder = (options: EncoderOptionsBuilderParams) => {
+  const { streamNum, fps, inputBitrate, inputRatio, resolution } = options
+
+  const targetBitrate = getTargetBitrate({ inputBitrate, ratio: inputRatio, fps, resolution })
+
+  return {
+    outputOptions: [
+      ...getCommonOutputOptions(targetBitrate, streamNum),
+
+      `${buildStreamSuffix('-r:v', streamNum)} ${fps}`,
+      `${buildStreamSuffix('-b:v', streamNum)} ${targetBitrate}`
+    ]
+  }
+}
+
+const defaultAACOptionsBuilder: EncoderOptionsBuilder = async ({ input, streamNum, canCopyAudio }) => {
+  const probe = await ffprobePromise(input)
+
+  const parsedAudio = await getAudioStream(input, probe)
+
+  // We try to reduce the ceiling bitrate by making rough matches of bitrates
+  // Of course this is far from perfect, but it might save some space in the end
+
+  const audioCodecName = parsedAudio.audioStream['codec_name']
+
+  const bitrate = getMaxAudioBitrate(audioCodecName, parsedAudio.bitrate)
+
+  // Force stereo as it causes some issues with HLS playback in Chrome
+  const base = [ '-channel_layout', 'stereo' ]
+
+  if (bitrate !== -1) {
+    return { outputOptions: base.concat([ buildStreamSuffix('-b:a', streamNum), bitrate + 'k' ]) }
+  }
+
+  return { outputOptions: base }
+}
+
+const defaultLibFDKAACVODOptionsBuilder: EncoderOptionsBuilder = ({ streamNum }) => {
+  return { outputOptions: [ buildStreamSuffix('-q:a', streamNum), '5' ] }
+}
+
+export function getAvailableEncoders () {
+  return {
+    vod: {
+      libx264: {
+        default: defaultX264VODOptionsBuilder
+      },
+      aac: {
+        default: defaultAACOptionsBuilder
+      },
+      libfdk_aac: {
+        default: defaultLibFDKAACVODOptionsBuilder
+      }
+    },
+    live: {
+      libx264: {
+        default: defaultX264LiveOptionsBuilder
+      },
+      aac: {
+        default: defaultAACOptionsBuilder
+      }
+    }
+  }
+}
+
+export function getEncodersToTry () {
+  return {
+    vod: {
+      video: [ 'libx264' ],
+      audio: [ 'libfdk_aac', 'aac' ]
+    },
+
+    live: {
+      video: [ 'libx264' ],
+      audio: [ 'libfdk_aac', 'aac' ]
+    }
+  }
+}
+
+// ---------------------------------------------------------------------------
+
+function getTargetBitrate (options: {
+  inputBitrate: number
+  resolution: VideoResolution
+  ratio: number
+  fps: number
+}) {
+  const { inputBitrate, resolution, ratio, fps } = options
+
+  const capped = capBitrate(inputBitrate, getAverageBitrate({ resolution, fps, ratio }))
+  const limit = getMinLimitBitrate({ resolution, fps, ratio })
+
+  return Math.max(limit, capped)
+}
+
+function capBitrate (inputBitrate: number, targetBitrate: number) {
+  if (!inputBitrate) return targetBitrate
+
+  // Add 30% margin to input bitrate
+  const inputBitrateWithMargin = inputBitrate + (inputBitrate * 0.3)
+
+  return Math.min(targetBitrate, inputBitrateWithMargin)
+}
+
+function getCommonOutputOptions (targetBitrate: number, streamNum?: number) {
+  return [
+    `-preset veryfast`,
+    `${buildStreamSuffix('-maxrate:v', streamNum)} ${targetBitrate}`,
+    `${buildStreamSuffix('-bufsize:v', streamNum)} ${targetBitrate * 2}`,
+
+    // NOTE: b-strategy 1 - heuristic algorithm, 16 is optimal B-frames for it
+    `-b_strategy 1`,
+    // NOTE: Why 16: https://github.com/Chocobozzz/PeerTube/pull/774. b-strategy 2 -> B-frames<16
+    `-bf 16`
+  ]
+}

packages/peertube-runner/server/server.ts

@@ -0,0 +1,269 @@
+import { ensureDir, readdir, remove } from 'fs-extra'
+import { join } from 'path'
+import { io, Socket } from 'socket.io-client'
+import { pick } from '@shared/core-utils'
+import { PeerTubeProblemDocument, ServerErrorCode } from '@shared/models'
+import { PeerTubeServer as PeerTubeServerCommand } from '@shared/server-commands'
+import { ConfigManager } from '../shared'
+import { IPCServer } from '../shared/ipc'
+import { logger } from '../shared/logger'
+import { JobWithToken, processJob } from './process'
+
+type PeerTubeServer = PeerTubeServerCommand & {
+  runnerToken: string
+  runnerName: string
+  runnerDescription?: string
+}
+
+export class RunnerServer {
+  private static instance: RunnerServer
+
+  private servers: PeerTubeServer[] = []
+  private processingJobs: { job: JobWithToken, server: PeerTubeServer }[] = []
+
+  private checkingAvailableJobs = false
+
+  private readonly sockets = new Map<PeerTubeServer, Socket>()
+
+  private constructor () {}
+
+  async run () {
+    logger.info('Running PeerTube runner in server mode')
+
+    await ConfigManager.Instance.load()
+
+    for (const registered of ConfigManager.Instance.getConfig().registeredInstances) {
+      const serverCommand = new PeerTubeServerCommand({ url: registered.url })
+
+      this.loadServer(Object.assign(serverCommand, registered))
+
+      logger.info(`Loading registered instance ${registered.url}`)
+    }
+
+    // Run IPC
+    const ipcServer = new IPCServer()
+    try {
+      await ipcServer.run(this)
+    } catch (err) {
+      console.error('Cannot start local socket for IPC communication', err)
+      process.exit(-1)
+    }
+
+    // Cleanup on exit
+    for (const code of [ 'SIGINT', 'SIGUSR1', 'SIGUSR2', 'uncaughtException' ]) {
+      process.on(code, async () => {
+        await this.onExit()
+      })
+    }
+
+    // Process jobs
+    await ensureDir(ConfigManager.Instance.getTranscodingDirectory())
+    await this.cleanupTMP()
+
+    logger.info(`Using ${ConfigManager.Instance.getTranscodingDirectory()} for transcoding directory`)
+
+    await this.checkAvailableJobs()
+  }
+
+  // ---------------------------------------------------------------------------
+
+  async registerRunner (options: {
+    url: string
+    registrationToken: string
+    runnerName: string
+    runnerDescription?: string
+  }) {
+    const { url, registrationToken, runnerName, runnerDescription } = options
+
+    logger.info(`Registering runner ${runnerName} on ${url}...`)
+
+    const serverCommand = new PeerTubeServerCommand({ url })
+    const { runnerToken } = await serverCommand.runners.register({ name: runnerName, description: runnerDescription, registrationToken })
+
+    const server: PeerTubeServer = Object.assign(serverCommand, {
+      runnerToken,
+      runnerName,
+      runnerDescription
+    })
+
+    this.loadServer(server)
+    await this.saveRegisteredInstancesInConf()
+
+    logger.info(`Registered runner ${runnerName} on ${url}`)
+
+    await this.checkAvailableJobs()
+  }
+
+  private loadServer (server: PeerTubeServer) {
+    this.servers.push(server)
+
+    const url = server.url + '/runners'
+    const socket = io(url, {
+      auth: {
+        runnerToken: server.runnerToken
+      },
+      transports: [ 'websocket' ]
+    })
+
+    socket.on('connect_error', err => logger.warn({ err }, `Cannot connect to ${url} socket`))
+    socket.on('connect', () => logger.info(`Connected to ${url} socket`))
+    socket.on('available-jobs', () => this.checkAvailableJobs())
+
+    this.sockets.set(server, socket)
+  }
+
+  async unregisterRunner (options: {
+    url: string
+  }) {
+    const { url } = options
+
+    const server = this.servers.find(s => s.url === url)
+    if (!server) {
+      logger.error(`Unknown server ${url} to unregister`)
+      return
+    }
+
+    logger.info(`Unregistering runner ${server.runnerName} on ${url}...`)
+
+    try {
+      await server.runners.unregister({ runnerToken: server.runnerToken })
+    } catch (err) {
+      logger.error({ err }, `Cannot unregister runner ${server.runnerName} on ${url}`)
+    }
+
+    this.unloadServer(server)
+    await this.saveRegisteredInstancesInConf()
+
+    logger.info(`Unregistered runner ${server.runnerName} on ${server.url}`)
+  }
+
+  private unloadServer (server: PeerTubeServer) {
+    this.servers = this.servers.filter(s => s !== server)
+
+    const socket = this.sockets.get(server)
+    socket.disconnect()
+
+    this.sockets.delete(server)
+  }
+
+  listRegistered () {
+    return {
+      servers: this.servers.map(s => {
+        return {
+          url: s.url,
+          runnerName: s.runnerName,
+          runnerDescription: s.runnerDescription
+        }
+      })
+    }
+  }
+
+  // ---------------------------------------------------------------------------
+
+  private async checkAvailableJobs () {
+    if (this.checkingAvailableJobs) return
+
+    logger.info('Checking available jobs')
+
+    this.checkingAvailableJobs = true
+
+    for (const server of this.servers) {
+      try {
+        const job = await this.requestJob(server)
+        if (!job) continue
+
+        await this.tryToExecuteJobAsync(server, job)
+      } catch (err) {
+        if ((err.res?.body as PeerTubeProblemDocument)?.code === ServerErrorCode.UNKNOWN_RUNNER_TOKEN) {
+          logger.error({ err }, `Unregistering ${server.url} as the runner token ${server.runnerToken} is invalid`)
+
+          await this.unregisterRunner({ url: server.url })
+          return
+        }
+
+        logger.error({ err }, `Cannot request/accept job on ${server.url} for runner ${server.runnerName}`)
+      }
+    }
+
+    this.checkingAvailableJobs = false
+  }
+
+  private async requestJob (server: PeerTubeServer) {
+    logger.debug(`Requesting jobs on ${server.url} for runner ${server.runnerName}`)
+
+    const { availableJobs } = await server.runnerJobs.request({ runnerToken: server.runnerToken })
+
+    if (availableJobs.length === 0) {
+      logger.debug(`No job available on ${server.url} for runner ${server.runnerName}`)
+      return undefined
+    }
+
+    return availableJobs[0]
+  }
+
+  private async tryToExecuteJobAsync (server: PeerTubeServer, jobToAccept: { uuid: string }) {
+    if (this.processingJobs.length >= ConfigManager.Instance.getConfig().jobs.concurrency) return
+
+    const { job } = await server.runnerJobs.accept({ runnerToken: server.runnerToken, jobUUID: jobToAccept.uuid })
+
+    const processingJob = { job, server }
+    this.processingJobs.push(processingJob)
+
+    processJob({ server, job, runnerToken: server.runnerToken })
+      .catch(err => {
+        logger.error({ err }, 'Cannot process job')
+
+        server.runnerJobs.error({ jobToken: job.jobToken, jobUUID: job.uuid, runnerToken: server.runnerToken, message: err.message })
+          .catch(err2 => logger.error({ err: err2 }, 'Cannot abort job after error'))
+      })
+      .finally(() => {
+        this.processingJobs = this.processingJobs.filter(p => p !== processingJob)
+
+        return this.checkAvailableJobs()
+      })
+  }
+
+  // ---------------------------------------------------------------------------
+
+  private saveRegisteredInstancesInConf () {
+    const data = this.servers.map(s => {
+      return pick(s, [ 'url', 'runnerToken', 'runnerName', 'runnerDescription' ])
+    })
+
+    return ConfigManager.Instance.setRegisteredInstances(data)
+  }
+
+  // ---------------------------------------------------------------------------
+
+  private async cleanupTMP () {
+    const files = await readdir(ConfigManager.Instance.getTranscodingDirectory())
+
+    for (const file of files) {
+      await remove(join(ConfigManager.Instance.getTranscodingDirectory(), file))
+    }
+  }
+
+  private async onExit () {
+    try {
+      for (const { server, job } of this.processingJobs) {
+        await server.runnerJobs.abort({
+          jobToken: job.jobToken,
+          jobUUID: job.uuid,
+          reason: 'Runner stopped',
+          runnerToken: server.runnerToken
+        })
+      }
+
+      await this.cleanupTMP()
+    } catch (err) {
+      console.error(err)
+      process.exit(-1)
+    }
+
+    process.exit()
+  }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+}

packages/peertube-runner/shared/config-manager.ts

@@ -0,0 +1,139 @@
+import envPaths from 'env-paths'
+import { ensureDir, pathExists, readFile, remove, writeFile } from 'fs-extra'
+import { merge } from 'lodash'
+import { logger } from 'packages/peertube-runner/shared/logger'
+import { dirname, join } from 'path'
+import { parse, stringify } from '@iarna/toml'
+
+const paths = envPaths('peertube-runner')
+
+type Config = {
+  jobs: {
+    concurrency: number
+  }
+
+  ffmpeg: {
+    threads: number
+    nice: number
+  }
+
+  registeredInstances: {
+    url: string
+    runnerToken: string
+    runnerName: string
+    runnerDescription?: string
+  }[]
+}
+
+export class ConfigManager {
+  private static instance: ConfigManager
+
+  private config: Config = {
+    jobs: {
+      concurrency: 2
+    },
+    ffmpeg: {
+      threads: 2,
+      nice: 20
+    },
+    registeredInstances: []
+  }
+
+  private id: string
+  private configFilePath: string
+
+  private constructor () {}
+
+  init (id: string) {
+    this.id = id
+    this.configFilePath = join(this.getConfigDir(), 'config.toml')
+  }
+
+  async load () {
+    logger.info(`Using ${this.configFilePath} as configuration file`)
+
+    if (this.isTestInstance()) {
+      logger.info('Removing configuration file as we are using the "test" id')
+      await remove(this.configFilePath)
+    }
+
+    await ensureDir(dirname(this.configFilePath))
+
+    if (!await pathExists(this.configFilePath)) {
+      await this.save()
+    }
+
+    const file = await readFile(this.configFilePath, 'utf-8')
+
+    this.config = merge(this.config, parse(file))
+  }
+
+  save () {
+    return writeFile(this.configFilePath, stringify(this.config))
+  }
+
+  // ---------------------------------------------------------------------------
+
+  async setRegisteredInstances (registeredInstances: {
+    url: string
+    runnerToken: string
+    runnerName: string
+    runnerDescription?: string
+  }[]) {
+    this.config.registeredInstances = registeredInstances
+
+    await this.save()
+  }
+
+  // ---------------------------------------------------------------------------
+
+  getConfig () {
+    return this.deepFreeze(this.config)
+  }
+
+  // ---------------------------------------------------------------------------
+
+  getTranscodingDirectory () {
+    return join(paths.cache, this.id, 'transcoding')
+  }
+
+  getSocketDirectory () {
+    return join(paths.data, this.id)
+  }
+
+  getSocketPath () {
+    return join(this.getSocketDirectory(), 'peertube-runner.sock')
+  }
+
+  getConfigDir () {
+    return join(paths.config, this.id)
+  }
+
+  // ---------------------------------------------------------------------------
+
+  isTestInstance () {
+    return this.id === 'test'
+  }
+
+  // ---------------------------------------------------------------------------
+
+  // Thanks: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object/freeze
+  private deepFreeze <T extends object> (object: T) {
+    const propNames = Reflect.ownKeys(object)
+
+    // Freeze properties before freezing self
+    for (const name of propNames) {
+      const value = object[name]
+
+      if ((value && typeof value === 'object') || typeof value === 'function') {
+        this.deepFreeze(value)
+      }
+    }
+
+    return Object.freeze({ ...object })
+  }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+}

packages/peertube-runner/shared/http.ts

@@ -0,0 +1,66 @@
+import { createWriteStream, remove } from 'fs-extra'
+import { request as requestHTTP } from 'http'
+import { request as requestHTTPS, RequestOptions } from 'https'
+import { logger } from './logger'
+
+export function downloadFile (options: {
+  url: string
+  destination: string
+  runnerToken: string
+  jobToken: string
+}) {
+  const { url, destination, runnerToken, jobToken } = options
+
+  logger.debug(`Downloading file ${url}`)
+
+  return new Promise<void>((res, rej) => {
+    const parsed = new URL(url)
+
+    const body = JSON.stringify({
+      runnerToken,
+      jobToken
+    })
+
+    const getOptions: RequestOptions = {
+      method: 'POST',
+      hostname: parsed.hostname,
+      port: parsed.port,
+      path: parsed.pathname,
+      headers: {
+        'Content-Type': 'application/json',
+        'Content-Length': Buffer.byteLength(body, 'utf-8')
+      }
+    }
+
+    const request = getRequest(url)(getOptions, response => {
+      const code = response.statusCode ?? 0
+
+      if (code >= 400) {
+        return rej(new Error(response.statusMessage))
+      }
+
+      const file = createWriteStream(destination)
+      file.on('finish', () => res())
+
+      response.pipe(file)
+    })
+
+    request.on('error', err => {
+      remove(destination)
+        .catch(err => console.error(err))
+
+      return rej(err)
+    })
+
+    request.write(body)
+    request.end()
+  })
+}
+
+// ---------------------------------------------------------------------------
+
+function getRequest (url: string) {
+  if (url.startsWith('https://')) return requestHTTPS
+
+  return requestHTTP
+}

packages/peertube-runner/shared/index.ts

@@ -0,0 +1,3 @@
+export * from './config-manager'
+export * from './http'
+export * from './logger'

packages/peertube-runner/shared/ipc/index.ts

@@ -0,0 +1,2 @@
+export * from './ipc-client'
+export * from './ipc-server'

packages/peertube-runner/shared/ipc/ipc-client.ts

@@ -0,0 +1,74 @@
+import CliTable3 from 'cli-table3'
+import { ensureDir } from 'fs-extra'
+import { Client as NetIPC } from 'net-ipc'
+import { ConfigManager } from '../config-manager'
+import { IPCReponse, IPCReponseData, IPCRequest } from './shared'
+
+export class IPCClient {
+  private netIPC: NetIPC
+
+  async run () {
+    await ensureDir(ConfigManager.Instance.getSocketDirectory())
+
+    const socketPath = ConfigManager.Instance.getSocketPath()
+    this.netIPC = new NetIPC({ path: socketPath })
+    await this.netIPC.connect()
+  }
+
+  async askRegister (options: {
+    url: string
+    registrationToken: string
+    runnerName: string
+    runnerDescription?: string
+  }) {
+    const req: IPCRequest = {
+      type: 'register',
+      ...options
+    }
+
+    const { success, error } = await this.netIPC.request(req) as IPCReponse
+
+    if (success) console.log('PeerTube instance registered')
+    else console.error('Could not register PeerTube instance on runner server side', error)
+  }
+
+  async askUnregister (options: {
+    url: string
+  }) {
+    const req: IPCRequest = {
+      type: 'unregister',
+      ...options
+    }
+
+    const { success, error } = await this.netIPC.request(req) as IPCReponse
+
+    if (success) console.log('PeerTube instance unregistered')
+    else console.error('Could not unregister PeerTube instance on runner server side', error)
+  }
+
+  async askListRegistered () {
+    const req: IPCRequest = {
+      type: 'list-registered'
+    }
+
+    const { success, error, data } = await this.netIPC.request(req) as IPCReponse<IPCReponseData>
+    if (!success) {
+      console.error('Could not list registered PeerTube instances', error)
+      return
+    }
+
+    const table = new CliTable3({
+      head: [ 'instance', 'runner name', 'runner description' ]
+    })
+
+    for (const server of data.servers) {
+      table.push([ server.url, server.runnerName, server.runnerDescription ])
+    }
+
+    console.log(table.toString())
+  }
+
+  stop () {
+    this.netIPC.destroy()
+  }
+}

packages/peertube-runner/shared/ipc/ipc-server.ts

@@ -0,0 +1,61 @@
+import { ensureDir } from 'fs-extra'
+import { Server as NetIPC } from 'net-ipc'
+import { pick } from '@shared/core-utils'
+import { RunnerServer } from '../../server'
+import { ConfigManager } from '../config-manager'
+import { logger } from '../logger'
+import { IPCReponse, IPCReponseData, IPCRequest } from './shared'
+
+export class IPCServer {
+  private netIPC: NetIPC
+  private runnerServer: RunnerServer
+
+  async run (runnerServer: RunnerServer) {
+    this.runnerServer = runnerServer
+
+    await ensureDir(ConfigManager.Instance.getSocketDirectory())
+
+    const socketPath = ConfigManager.Instance.getSocketPath()
+    this.netIPC = new NetIPC({ path: socketPath })
+    await this.netIPC.start()
+
+    logger.info(`IPC socket created on ${socketPath}`)
+
+    this.netIPC.on('request', async (req: IPCRequest, res) => {
+      try {
+        const data = await this.process(req)
+
+        this.sendReponse(res, { success: true, data })
+      } catch (err) {
+        console.error('Cannot execute RPC call', err)
+        this.sendReponse(res, { success: false, error: err.message })
+      }
+    })
+  }
+
+  private async process (req: IPCRequest) {
+    switch (req.type) {
+      case 'register':
+        await this.runnerServer.registerRunner(pick(req, [ 'url', 'registrationToken', 'runnerName', 'runnerDescription' ]))
+        return undefined
+
+      case 'unregister':
+        await this.runnerServer.unregisterRunner({ url: req.url })
+        return undefined
+
+      case 'list-registered':
+        return Promise.resolve(this.runnerServer.listRegistered())
+
+      default:
+        throw new Error('Unknown RPC call ' + (req as any).type)
+    }
+  }
+
+  private sendReponse <T extends IPCReponseData> (
+    response: (data: any) => Promise<void>,
+    body: IPCReponse<T>
+  ) {
+    response(body)
+      .catch(err => console.error('Cannot send response after IPC request', err))
+  }
+}

packages/peertube-runner/shared/ipc/shared/index.ts

@@ -0,0 +1,2 @@
+export * from './ipc-request.model'
+export * from './ipc-response.model'

packages/peertube-runner/shared/ipc/shared/ipc-request.model.ts

@@ -0,0 +1,15 @@
+export type IPCRequest =
+  IPCRequestRegister |
+  IPCRequestUnregister |
+  IPCRequestListRegistered
+
+export type IPCRequestRegister = {
+  type: 'register'
+  url: string
+  registrationToken: string
+  runnerName: string
+  runnerDescription?: string
+}
+
+export type IPCRequestUnregister = { type: 'unregister', url: string }
+export type IPCRequestListRegistered = { type: 'list-registered' }

packages/peertube-runner/shared/ipc/shared/ipc-response.model.ts

@@ -0,0 +1,15 @@
+export type IPCReponse <T extends IPCReponseData = undefined> = {
+  success: boolean
+  error?: string
+  data?: T
+}
+
+export type IPCReponseData =
+  // list registered
+  {
+    servers: {
+      runnerName: string
+      runnerDescription: string
+      url: string
+    }[]
+  }

packages/peertube-runner/shared/logger.ts

@@ -0,0 +1,12 @@
+import { pino } from 'pino'
+import pretty from 'pino-pretty'
+
+const logger = pino(pretty({
+  colorize: true
+}))
+
+logger.level = 'info'
+
+export {
+  logger
+}

packages/peertube-runner/tsconfig.json

@@ -0,0 +1,9 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "./dist"
+  },
+  "references": [
+    { "path": "../../shared" }
+  ]
+}

packages/peertube-runner/yarn.lock

@@ -0,0 +1,528 @@
+# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+
+"@commander-js/extra-typings@^10.0.3":
+  version "10.0.3"
+  resolved "https://registry.yarnpkg.com/@commander-js/extra-typings/-/extra-typings-10.0.3.tgz#8b6c64897231ed9c00461db82018b5131b653aae"
+  integrity sha512-OIw28QV/GlP8k0B5CJTRsl8IyNvd0R8C8rfo54Yz9P388vCNDgdNrFlKxZTGqps+5j6lSw3Ss9JTQwcur1w1oA==
+
+"@esbuild/android-arm64@0.17.15":
+  version "0.17.15"
+  resolved "https://registry.yarnpkg.com/@esbuild/android-arm64/-/android-arm64-0.17.15.tgz#893ad71f3920ccb919e1757c387756a9bca2ef42"
+  integrity sha512-0kOB6Y7Br3KDVgHeg8PRcvfLkq+AccreK///B4Z6fNZGr/tNHX0z2VywCc7PTeWp+bPvjA5WMvNXltHw5QjAIA==
+
+"@esbuild/android-arm@0.17.15":
+  version "0.17.15"
+  resolved "https://registry.yarnpkg.com/@esbuild/android-arm/-/android-arm-0.17.15.tgz#143e0d4e4c08c786ea410b9a7739779a9a1315d8"
+  integrity sha512-sRSOVlLawAktpMvDyJIkdLI/c/kdRTOqo8t6ImVxg8yT7LQDUYV5Rp2FKeEosLr6ZCja9UjYAzyRSxGteSJPYg==
+
+"@esbuild/android-x64@0.17.15":
+  version "0.17.15"
+  resolved "https://registry.yarnpkg.com/@esbuild/android-x64/-/android-x64-0.17.15.tgz#d2d12a7676b2589864281b2274355200916540bc"
+  integrity sha512-MzDqnNajQZ63YkaUWVl9uuhcWyEyh69HGpMIrf+acR4otMkfLJ4sUCxqwbCyPGicE9dVlrysI3lMcDBjGiBBcQ==
+
+"@esbuild/darwin-arm64@0.17.15":
+  version "0.17.15"
+  resolved "https://registry.yarnpkg.com/@esbuild/darwin-arm64/-/darwin-arm64-0.17.15.tgz#2e88e79f1d327a2a7d9d06397e5232eb0a473d61"
+  integrity sha512-7siLjBc88Z4+6qkMDxPT2juf2e8SJxmsbNVKFY2ifWCDT72v5YJz9arlvBw5oB4W/e61H1+HDB/jnu8nNg0rLA==
+
+"@esbuild/darwin-x64@0.17.15":
+  version "0.17.15"
+  resolved "https://registry.yarnpkg.com/@esbuild/darwin-x64/-/darwin-x64-0.17.15.tgz#9384e64c0be91388c57be6d3a5eaf1c32a99c91d"
+  integrity sha512-NbImBas2rXwYI52BOKTW342Tm3LTeVlaOQ4QPZ7XuWNKiO226DisFk/RyPk3T0CKZkKMuU69yOvlapJEmax7cg==
+
+"@esbuild/freebsd-arm64@0.17.15":
+  version "0.17.15"
+  resolved "https://registry.yarnpkg.com/@esbuild/freebsd-arm64/-/freebsd-arm64-0.17.15.tgz#2ad5a35bc52ebd9ca6b845dbc59ba39647a93c1a"
+  integrity sha512-Xk9xMDjBVG6CfgoqlVczHAdJnCs0/oeFOspFap5NkYAmRCT2qTn1vJWA2f419iMtsHSLm+O8B6SLV/HlY5cYKg==
+
+"@esbuild/freebsd-x64@0.17.15":
+  version "0.17.15"
+  resolved "https://registry.yarnpkg.com/@esbuild/freebsd-x64/-/freebsd-x64-0.17.15.tgz#b513a48446f96c75fda5bef470e64d342d4379cd"
+  integrity sha512-3TWAnnEOdclvb2pnfsTWtdwthPfOz7qAfcwDLcfZyGJwm1SRZIMOeB5FODVhnM93mFSPsHB9b/PmxNNbSnd0RQ==
+
+"@esbuild/linux-arm64@0.17.15":
+  version "0.17.15"
+  resolved "https://registry.yarnpkg.com/@esbuild/linux-arm64/-/linux-arm64-0.17.15.tgz#9697b168175bfd41fa9cc4a72dd0d48f24715f31"
+  integrity sha512-T0MVnYw9KT6b83/SqyznTs/3Jg2ODWrZfNccg11XjDehIved2oQfrX/wVuev9N936BpMRaTR9I1J0tdGgUgpJA==
+
+"@esbuild/linux-arm@0.17.15":
+  version "0.17.15"
+  resolved "https://registry.yarnpkg.com/@esbuild/linux-arm/-/linux-arm-0.17.15.tgz#5b22062c54f48cd92fab9ffd993732a52db70cd3"
+  integrity sha512-MLTgiXWEMAMr8nmS9Gigx43zPRmEfeBfGCwxFQEMgJ5MC53QKajaclW6XDPjwJvhbebv+RzK05TQjvH3/aM4Xw==
+
+"@esbuild/linux-ia32@0.17.15":
+  version "0.17.15"
+  resolved "https://registry.yarnpkg.com/@esbuild/linux-ia32/-/linux-ia32-0.17.15.tgz#eb28a13f9b60b5189fcc9e98e1024f6b657ba54c"
+  integrity sha512-wp02sHs015T23zsQtU4Cj57WiteiuASHlD7rXjKUyAGYzlOKDAjqK6bk5dMi2QEl/KVOcsjwL36kD+WW7vJt8Q==
+
+"@esbuild/linux-loong64@0.17.15":
+  version "0.17.15"
+  resolved "https://registry.yarnpkg.com/@esbuild/linux-loong64/-/linux-loong64-0.17.15.tgz#32454bdfe144cf74b77895a8ad21a15cb81cfbe5"
+  integrity sha512-k7FsUJjGGSxwnBmMh8d7IbObWu+sF/qbwc+xKZkBe/lTAF16RqxRCnNHA7QTd3oS2AfGBAnHlXL67shV5bBThQ==
+
+"@esbuild/linux-mips64el@0.17.15":
+  version "0.17.15"
+  resolved "https://registry.yarnpkg.com/@esbuild/linux-mips64el/-/linux-mips64el-0.17.15.tgz#af12bde0d775a318fad90eb13a0455229a63987c"
+  integrity sha512-ZLWk6czDdog+Q9kE/Jfbilu24vEe/iW/Sj2d8EVsmiixQ1rM2RKH2n36qfxK4e8tVcaXkvuV3mU5zTZviE+NVQ==
+
+"@esbuild/linux-ppc64@0.17.15":
+  version "0.17.15"
+  resolved "https://registry.yarnpkg.com/@esbuild/linux-ppc64/-/linux-ppc64-0.17.15.tgz#34c5ed145b2dfc493d3e652abac8bd3baa3865a5"
+  integrity sha512-mY6dPkIRAiFHRsGfOYZC8Q9rmr8vOBZBme0/j15zFUKM99d4ILY4WpOC7i/LqoY+RE7KaMaSfvY8CqjJtuO4xg==
+
+"@esbuild/linux-riscv64@0.17.15":
+  version "0.17.15"
+  resolved "https://registry.yarnpkg.com/@esbuild/linux-riscv64/-/linux-riscv64-0.17.15.tgz#87bd515e837f2eb004b45f9e6a94dc5b93f22b92"
+  integrity sha512-EcyUtxffdDtWjjwIH8sKzpDRLcVtqANooMNASO59y+xmqqRYBBM7xVLQhqF7nksIbm2yHABptoioS9RAbVMWVA==
+
+"@esbuild/linux-s390x@0.17.15":
+  version "0.17.15"
+  resolved "https://registry.yarnpkg.com/@esbuild/linux-s390x/-/linux-s390x-0.17.15.tgz#20bf7947197f199ddac2ec412029a414ceae3aa3"
+  integrity sha512-BuS6Jx/ezxFuHxgsfvz7T4g4YlVrmCmg7UAwboeyNNg0OzNzKsIZXpr3Sb/ZREDXWgt48RO4UQRDBxJN3B9Rbg==
+
+"@esbuild/linux-x64@0.17.15":
+  version "0.17.15"
+  resolved "https://registry.yarnpkg.com/@esbuild/linux-x64/-/linux-x64-0.17.15.tgz#31b93f9c94c195e852c20cd3d1914a68aa619124"
+  integrity sha512-JsdS0EgEViwuKsw5tiJQo9UdQdUJYuB+Mf6HxtJSPN35vez1hlrNb1KajvKWF5Sa35j17+rW1ECEO9iNrIXbNg==
+
+"@esbuild/netbsd-x64@0.17.15":
+  version "0.17.15"
+  resolved "https://registry.yarnpkg.com/@esbuild/netbsd-x64/-/netbsd-x64-0.17.15.tgz#8da299b3ac6875836ca8cdc1925826498069ac65"
+  integrity sha512-R6fKjtUysYGym6uXf6qyNephVUQAGtf3n2RCsOST/neIwPqRWcnc3ogcielOd6pT+J0RDR1RGcy0ZY7d3uHVLA==
+
+"@esbuild/openbsd-x64@0.17.15":
+  version "0.17.15"
+  resolved "https://registry.yarnpkg.com/@esbuild/openbsd-x64/-/openbsd-x64-0.17.15.tgz#04a1ec3d4e919714dba68dcf09eeb1228ad0d20c"
+  integrity sha512-mVD4PGc26b8PI60QaPUltYKeSX0wxuy0AltC+WCTFwvKCq2+OgLP4+fFd+hZXzO2xW1HPKcytZBdjqL6FQFa7w==
+
+"@esbuild/sunos-x64@0.17.15":
+  version "0.17.15"
+  resolved "https://registry.yarnpkg.com/@esbuild/sunos-x64/-/sunos-x64-0.17.15.tgz#6694ebe4e16e5cd7dab6505ff7c28f9c1c695ce5"
+  integrity sha512-U6tYPovOkw3459t2CBwGcFYfFRjivcJJc1WC8Q3funIwX8x4fP+R6xL/QuTPNGOblbq/EUDxj9GU+dWKX0oWlQ==
+
+"@esbuild/win32-arm64@0.17.15":
+  version "0.17.15"
+  resolved "https://registry.yarnpkg.com/@esbuild/win32-arm64/-/win32-arm64-0.17.15.tgz#1f95b2564193c8d1fee8f8129a0609728171d500"
+  integrity sha512-W+Z5F++wgKAleDABemiyXVnzXgvRFs+GVKThSI+mGgleLWluv0D7Diz4oQpgdpNzh4i2nNDzQtWbjJiqutRp6Q==
+
+"@esbuild/win32-ia32@0.17.15":
+  version "0.17.15"
+  resolved "https://registry.yarnpkg.com/@esbuild/win32-ia32/-/win32-ia32-0.17.15.tgz#c362b88b3df21916ed7bcf75c6d09c6bf3ae354a"
+  integrity sha512-Muz/+uGgheShKGqSVS1KsHtCyEzcdOn/W/Xbh6H91Etm+wiIfwZaBn1W58MeGtfI8WA961YMHFYTthBdQs4t+w==
+
+"@esbuild/win32-x64@0.17.15":
+  version "0.17.15"
+  resolved "https://registry.yarnpkg.com/@esbuild/win32-x64/-/win32-x64-0.17.15.tgz#c2e737f3a201ebff8e2ac2b8e9f246b397ad19b8"
+  integrity sha512-DjDa9ywLUUmjhV2Y9wUTIF+1XsmuFGvZoCmOWkli1XcNAh5t25cc7fgsCx4Zi/Uurep3TTLyDiKATgGEg61pkA==
+
+"@iarna/toml@^2.2.5":
+  version "2.2.5"
+  resolved "https://registry.yarnpkg.com/@iarna/toml/-/toml-2.2.5.tgz#b32366c89b43c6f8cefbdefac778b9c828e3ba8c"
+  integrity sha512-trnsAYxU3xnS1gPHPyU961coFyLkh4gAD/0zQ5mymY4yOZ+CYvsPqUbOFSw0aDM4y0tV7tiFxL/1XfXPNC6IPg==
+
+"@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.2":
+  version "3.0.2"
+  resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.2.tgz#44d752c1a2dc113f15f781b7cc4f53a307e3fa38"
+  integrity sha512-9bfjwDxIDWmmOKusUcqdS4Rw+SETlp9Dy39Xui9BEGEk19dDwH0jhipwFzEff/pFg95NKymc6TOTbRKcWeRqyQ==
+
+"@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.2":
+  version "3.0.2"
+  resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.2.tgz#f954f34355712212a8e06c465bc06c40852c6bb3"
+  integrity sha512-lwriRAHm1Yg4iDf23Oxm9n/t5Zpw1lVnxYU3HnJPTi2lJRkKTrps1KVgvL6m7WvmhYVt/FIsssWay+k45QHeuw==
+
+"@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.2":
+  version "3.0.2"
+  resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.2.tgz#45c63037f045c2b15c44f80f0393fa24f9655367"
+  integrity sha512-FU20Bo66/f7He9Fp9sP2zaJ1Q8L9uLPZQDub/WlUip78JlPeMbVL8546HbZfcW9LNciEXc8d+tThSJjSC+tmsg==
+
+"@msgpackr-extract/msgpackr-extract-linux-arm@3.0.2":
+  version "3.0.2"
+  resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.2.tgz#35707efeafe6d22b3f373caf9e8775e8920d1399"
+  integrity sha512-MOI9Dlfrpi2Cuc7i5dXdxPbFIgbDBGgKR5F2yWEa6FVEtSWncfVNKW5AKjImAQ6CZlBK9tympdsZJ2xThBiWWA==
+
+"@msgpackr-extract/msgpackr-extract-linux-x64@3.0.2":
+  version "3.0.2"
+  resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.2.tgz#091b1218b66c341f532611477ef89e83f25fae4f"
+  integrity sha512-gsWNDCklNy7Ajk0vBBf9jEx04RUxuDQfBse918Ww+Qb9HCPoGzS+XJTLe96iN3BVK7grnLiYghP/M4L8VsaHeA==
+
+"@msgpackr-extract/msgpackr-extract-win32-x64@3.0.2":
+  version "3.0.2"
+  resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.2.tgz#0f164b726869f71da3c594171df5ebc1c4b0a407"
+  integrity sha512-O+6Gs8UeDbyFpbSh2CPEz/UOrrdWPTBYNblZK5CxxLisYt4kGX3Sc+czffFonyjiGSq3jWLwJS/CCJc7tBr4sQ==
+
+abort-controller@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/abort-controller/-/abort-controller-3.0.0.tgz#eaf54d53b62bae4138e809ca225c8439a6efb392"
+  integrity sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==
+  dependencies:
+    event-target-shim "^5.0.0"
+
+atomic-sleep@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/atomic-sleep/-/atomic-sleep-1.0.0.tgz#eb85b77a601fc932cfe432c5acd364a9e2c9075b"
+  integrity sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==
+
+balanced-match@^1.0.0:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee"
+  integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==
+
+base64-js@^1.3.1:
+  version "1.5.1"
+  resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a"
+  integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==
+
+brace-expansion@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-2.0.1.tgz#1edc459e0f0c548486ecf9fc99f2221364b9a0ae"
+  integrity sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==
+  dependencies:
+    balanced-match "^1.0.0"
+
+buffer@^6.0.3:
+  version "6.0.3"
+  resolved "https://registry.yarnpkg.com/buffer/-/buffer-6.0.3.tgz#2ace578459cc8fbe2a70aaa8f52ee63b6a74c6c6"
+  integrity sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==
+  dependencies:
+    base64-js "^1.3.1"
+    ieee754 "^1.2.1"
+
+colorette@^2.0.7:
+  version "2.0.19"
+  resolved "https://registry.yarnpkg.com/colorette/-/colorette-2.0.19.tgz#cdf044f47ad41a0f4b56b3a0d5b4e6e1a2d5a798"
+  integrity sha512-3tlv/dIP7FWvj3BsbHrGLJ6l/oKh1O3TcgBqMn+yyCagOxc23fyzDS6HypQbgxWbkpDnf52p1LuR4eWDQ/K9WQ==
+
+dateformat@^4.6.3:
+  version "4.6.3"
+  resolved "https://registry.yarnpkg.com/dateformat/-/dateformat-4.6.3.tgz#556fa6497e5217fedb78821424f8a1c22fa3f4b5"
+  integrity sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA==
+
+end-of-stream@^1.1.0:
+  version "1.4.4"
+  resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0"
+  integrity sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==
+  dependencies:
+    once "^1.4.0"
+
+env-paths@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/env-paths/-/env-paths-3.0.0.tgz#2f1e89c2f6dbd3408e1b1711dd82d62e317f58da"
+  integrity sha512-dtJUTepzMW3Lm/NPxRf3wP4642UWhjL2sQxc+ym2YMj1m/H2zDNQOlezafzkHwn6sMstjHTwG6iQQsctDW/b1A==
+
+esbuild@^0.17.15:
+  version "0.17.15"
+  resolved "https://registry.yarnpkg.com/esbuild/-/esbuild-0.17.15.tgz#209ebc87cb671ffb79574db93494b10ffaf43cbc"
+  integrity sha512-LBUV2VsUIc/iD9ME75qhT4aJj0r75abCVS0jakhFzOtR7TQsqQA5w0tZ+KTKnwl3kXE0MhskNdHDh/I5aCR1Zw==
+  optionalDependencies:
+    "@esbuild/android-arm" "0.17.15"
+    "@esbuild/android-arm64" "0.17.15"
+    "@esbuild/android-x64" "0.17.15"
+    "@esbuild/darwin-arm64" "0.17.15"
+    "@esbuild/darwin-x64" "0.17.15"
+    "@esbuild/freebsd-arm64" "0.17.15"
+    "@esbuild/freebsd-x64" "0.17.15"
+    "@esbuild/linux-arm" "0.17.15"
+    "@esbuild/linux-arm64" "0.17.15"
+    "@esbuild/linux-ia32" "0.17.15"
+    "@esbuild/linux-loong64" "0.17.15"
+    "@esbuild/linux-mips64el" "0.17.15"
+    "@esbuild/linux-ppc64" "0.17.15"
+    "@esbuild/linux-riscv64" "0.17.15"
+    "@esbuild/linux-s390x" "0.17.15"
+    "@esbuild/linux-x64" "0.17.15"
+    "@esbuild/netbsd-x64" "0.17.15"
+    "@esbuild/openbsd-x64" "0.17.15"
+    "@esbuild/sunos-x64" "0.17.15"
+    "@esbuild/win32-arm64" "0.17.15"
+    "@esbuild/win32-ia32" "0.17.15"
+    "@esbuild/win32-x64" "0.17.15"
+
+event-target-shim@^5.0.0:
+  version "5.0.1"
+  resolved "https://registry.yarnpkg.com/event-target-shim/-/event-target-shim-5.0.1.tgz#5d4d3ebdf9583d63a5333ce2deb7480ab2b05789"
+  integrity sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==
+
+events@^3.3.0:
+  version "3.3.0"
+  resolved "https://registry.yarnpkg.com/events/-/events-3.3.0.tgz#31a95ad0a924e2d2c419a813aeb2c4e878ea7400"
+  integrity sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==
+
+fast-copy@^3.0.0:
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/fast-copy/-/fast-copy-3.0.1.tgz#9e89ef498b8c04c1cd76b33b8e14271658a732aa"
+  integrity sha512-Knr7NOtK3HWRYGtHoJrjkaWepqT8thIVGAwt0p0aUs1zqkAzXZV4vo9fFNwyb5fcqK1GKYFYxldQdIDVKhUAfA==
+
+fast-redact@^3.1.1:
+  version "3.1.2"
+  resolved "https://registry.yarnpkg.com/fast-redact/-/fast-redact-3.1.2.tgz#d58e69e9084ce9fa4c1a6fa98a3e1ecf5d7839aa"
+  integrity sha512-+0em+Iya9fKGfEQGcd62Yv6onjBmmhV1uh86XVfOU8VwAe6kaFdQCWI9s0/Nnugx5Vd9tdbZ7e6gE2tR9dzXdw==
+
+fast-safe-stringify@^2.1.1:
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz#c406a83b6e70d9e35ce3b30a81141df30aeba884"
+  integrity sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==
+
+fast-zlib@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/fast-zlib/-/fast-zlib-2.0.1.tgz#be624f592fc80ad8019ee2025d16a367a4e9b024"
+  integrity sha512-DCoYgNagM2Bt1VIpXpdGnRx4LzqJeYG0oh6Nf/7cWo6elTXkFGMw9CrRCYYUIapYNrozYMoyDRflx9mgT3Awyw==
+
+fs.realpath@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f"
+  integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==
+
+glob@^8.0.0:
+  version "8.1.0"
+  resolved "https://registry.yarnpkg.com/glob/-/glob-8.1.0.tgz#d388f656593ef708ee3e34640fdfb99a9fd1c33e"
+  integrity sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==
+  dependencies:
+    fs.realpath "^1.0.0"
+    inflight "^1.0.4"
+    inherits "2"
+    minimatch "^5.0.1"
+    once "^1.3.0"
+
+help-me@^4.0.1:
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/help-me/-/help-me-4.2.0.tgz#50712bfd799ff1854ae1d312c36eafcea85b0563"
+  integrity sha512-TAOnTB8Tz5Dw8penUuzHVrKNKlCIbwwbHnXraNJxPwf8LRtE2HlM84RYuezMFcwOJmoYOCWVDyJ8TQGxn9PgxA==
+  dependencies:
+    glob "^8.0.0"
+    readable-stream "^3.6.0"
+
+ieee754@^1.2.1:
+  version "1.2.1"
+  resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352"
+  integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==
+
+inflight@^1.0.4:
+  version "1.0.6"
+  resolved "https://registry.yarnpkg.com/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9"
+  integrity sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==
+  dependencies:
+    once "^1.3.0"
+    wrappy "1"
+
+inherits@2, inherits@^2.0.3:
+  version "2.0.4"
+  resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c"
+  integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==
+
+joycon@^3.1.1:
+  version "3.1.1"
+  resolved "https://registry.yarnpkg.com/joycon/-/joycon-3.1.1.tgz#bce8596d6ae808f8b68168f5fc69280996894f03"
+  integrity sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==
+
+minimatch@^5.0.1:
+  version "5.1.6"
+  resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-5.1.6.tgz#1cfcb8cf5522ea69952cd2af95ae09477f122a96"
+  integrity sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==
+  dependencies:
+    brace-expansion "^2.0.1"
+
+minimist@^1.2.6:
+  version "1.2.8"
+  resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c"
+  integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==
+
+msgpackr-extract@^3.0.1:
+  version "3.0.2"
+  resolved "https://registry.yarnpkg.com/msgpackr-extract/-/msgpackr-extract-3.0.2.tgz#e05ec1bb4453ddf020551bcd5daaf0092a2c279d"
+  integrity sha512-SdzXp4kD/Qf8agZ9+iTu6eql0m3kWm1A2y1hkpTeVNENutaB0BwHlSvAIaMxwntmRUAUjon2V4L8Z/njd0Ct8A==
+  dependencies:
+    node-gyp-build-optional-packages "5.0.7"
+  optionalDependencies:
+    "@msgpackr-extract/msgpackr-extract-darwin-arm64" "3.0.2"
+    "@msgpackr-extract/msgpackr-extract-darwin-x64" "3.0.2"
+    "@msgpackr-extract/msgpackr-extract-linux-arm" "3.0.2"
+    "@msgpackr-extract/msgpackr-extract-linux-arm64" "3.0.2"
+    "@msgpackr-extract/msgpackr-extract-linux-x64" "3.0.2"
+    "@msgpackr-extract/msgpackr-extract-win32-x64" "3.0.2"
+
+msgpackr@^1.3.2:
+  version "1.8.5"
+  resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.8.5.tgz#8cadfb935357680648f33699d0e833c9179dbfeb"
+  integrity sha512-mpPs3qqTug6ahbblkThoUY2DQdNXcm4IapwOS3Vm/87vmpzLVelvp9h3It1y9l1VPpiFLV11vfOXnmeEwiIXwg==
+  optionalDependencies:
+    msgpackr-extract "^3.0.1"
+
+net-ipc@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/net-ipc/-/net-ipc-2.0.1.tgz#1da79ca16f1624f2ed1099a124cb065912c595a5"
+  integrity sha512-4HLjZ/Xorj4kxA7WUajF2EAXlS+OR+XliDLkqQA53Wm7eIr/hWLjdXt4zzB6q4Ii8BB+HbuRbM9yLov3+ttRUw==
+  optionalDependencies:
+    fast-zlib "^2.0.1"
+    msgpackr "^1.3.2"
+
+node-gyp-build-optional-packages@5.0.7:
+  version "5.0.7"
+  resolved "https://registry.yarnpkg.com/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.7.tgz#5d2632bbde0ab2f6e22f1bbac2199b07244ae0b3"
+  integrity sha512-YlCCc6Wffkx0kHkmam79GKvDQ6x+QZkMjFGrIMxgFNILFvGSbCp2fCBC55pGTT9gVaz8Na5CLmxt/urtzRv36w==
+
+on-exit-leak-free@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/on-exit-leak-free/-/on-exit-leak-free-2.1.0.tgz#5c703c968f7e7f851885f6459bf8a8a57edc9cc4"
+  integrity sha512-VuCaZZAjReZ3vUwgOB8LxAosIurDiAW0s13rI1YwmaP++jvcxP77AWoQvenZebpCA2m8WC1/EosPYPMjnRAp/w==
+
+once@^1.3.0, once@^1.3.1, once@^1.4.0:
+  version "1.4.0"
+  resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1"
+  integrity sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==
+  dependencies:
+    wrappy "1"
+
+pino-abstract-transport@^1.0.0, pino-abstract-transport@v1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/pino-abstract-transport/-/pino-abstract-transport-1.0.0.tgz#cc0d6955fffcadb91b7b49ef220a6cc111d48bb3"
+  integrity sha512-c7vo5OpW4wIS42hUVcT5REsL8ZljsUfBjqV/e2sFxmFEFZiq1XLUp5EYLtuDH6PEHq9W1egWqRbnLUP5FuZmOA==
+  dependencies:
+    readable-stream "^4.0.0"
+    split2 "^4.0.0"
+
+pino-pretty@^10.0.0:
+  version "10.0.0"
+  resolved "https://registry.yarnpkg.com/pino-pretty/-/pino-pretty-10.0.0.tgz#fd2f307ee897289f63d09b0b804ac2ecc9a18516"
+  integrity sha512-zKFjYXBzLaLTEAN1ayKpHXtL5UeRQC7R3lvhKe7fWs7hIVEjKGG/qIXwQt9HmeUp71ogUd/YcW+LmMwRp4KT6Q==
+  dependencies:
+    colorette "^2.0.7"
+    dateformat "^4.6.3"
+    fast-copy "^3.0.0"
+    fast-safe-stringify "^2.1.1"
+    help-me "^4.0.1"
+    joycon "^3.1.1"
+    minimist "^1.2.6"
+    on-exit-leak-free "^2.1.0"
+    pino-abstract-transport "^1.0.0"
+    pump "^3.0.0"
+    readable-stream "^4.0.0"
+    secure-json-parse "^2.4.0"
+    sonic-boom "^3.0.0"
+    strip-json-comments "^3.1.1"
+
+pino-std-serializers@^6.0.0:
+  version "6.2.0"
+  resolved "https://registry.yarnpkg.com/pino-std-serializers/-/pino-std-serializers-6.2.0.tgz#169048c0df3f61352fce56aeb7fb962f1b66ab43"
+  integrity sha512-IWgSzUL8X1w4BIWTwErRgtV8PyOGOOi60uqv0oKuS/fOA8Nco/OeI6lBuc4dyP8MMfdFwyHqTMcBIA7nDiqEqA==
+
+pino@^8.11.0:
+  version "8.11.0"
+  resolved "https://registry.yarnpkg.com/pino/-/pino-8.11.0.tgz#2a91f454106b13e708a66c74ebc1c2ab7ab38498"
+  integrity sha512-Z2eKSvlrl2rH8p5eveNUnTdd4AjJk8tAsLkHYZQKGHP4WTh2Gi1cOSOs3eWPqaj+niS3gj4UkoreoaWgF3ZWYg==
+  dependencies:
+    atomic-sleep "^1.0.0"
+    fast-redact "^3.1.1"
+    on-exit-leak-free "^2.1.0"
+    pino-abstract-transport v1.0.0
+    pino-std-serializers "^6.0.0"
+    process-warning "^2.0.0"
+    quick-format-unescaped "^4.0.3"
+    real-require "^0.2.0"
+    safe-stable-stringify "^2.3.1"
+    sonic-boom "^3.1.0"
+    thread-stream "^2.0.0"
+
+process-warning@^2.0.0:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/process-warning/-/process-warning-2.2.0.tgz#008ec76b579820a8e5c35d81960525ca64feb626"
+  integrity sha512-/1WZ8+VQjR6avWOgHeEPd7SDQmFQ1B5mC1eRXsCm5TarlNmx/wCsa5GEaxGm05BORRtyG/Ex/3xq3TuRvq57qg==
+
+process@^0.11.10:
+  version "0.11.10"
+  resolved "https://registry.yarnpkg.com/process/-/process-0.11.10.tgz#7332300e840161bda3e69a1d1d91a7d4bc16f182"
+  integrity sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==
+
+pump@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/pump/-/pump-3.0.0.tgz#b4a2116815bde2f4e1ea602354e8c75565107a64"
+  integrity sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==
+  dependencies:
+    end-of-stream "^1.1.0"
+    once "^1.3.1"
+
+quick-format-unescaped@^4.0.3:
+  version "4.0.4"
+  resolved "https://registry.yarnpkg.com/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz#93ef6dd8d3453cbc7970dd614fad4c5954d6b5a7"
+  integrity sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==
+
+readable-stream@^3.6.0:
+  version "3.6.2"
+  resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.2.tgz#56a9b36ea965c00c5a93ef31eb111a0f11056967"
+  integrity sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==
+  dependencies:
+    inherits "^2.0.3"
+    string_decoder "^1.1.1"
+    util-deprecate "^1.0.1"
+
+readable-stream@^4.0.0:
+  version "4.3.0"
+  resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-4.3.0.tgz#0914d0c72db03b316c9733bb3461d64a3cc50cba"
+  integrity sha512-MuEnA0lbSi7JS8XM+WNJlWZkHAAdm7gETHdFK//Q/mChGyj2akEFtdLZh32jSdkWGbRwCW9pn6g3LWDdDeZnBQ==
+  dependencies:
+    abort-controller "^3.0.0"
+    buffer "^6.0.3"
+    events "^3.3.0"
+    process "^0.11.10"
+
+real-require@^0.2.0:
+  version "0.2.0"
+  resolved "https://registry.yarnpkg.com/real-require/-/real-require-0.2.0.tgz#209632dea1810be2ae063a6ac084fee7e33fba78"
+  integrity sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==
+
+safe-buffer@~5.2.0:
+  version "5.2.1"
+  resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6"
+  integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==
+
+safe-stable-stringify@^2.3.1:
+  version "2.4.3"
+  resolved "https://registry.yarnpkg.com/safe-stable-stringify/-/safe-stable-stringify-2.4.3.tgz#138c84b6f6edb3db5f8ef3ef7115b8f55ccbf886"
+  integrity sha512-e2bDA2WJT0wxseVd4lsDP4+3ONX6HpMXQa1ZhFQ7SU+GjvORCmShbCMltrtIDfkYhVHrOcPtj+KhmDBdPdZD1g==
+
+secure-json-parse@^2.4.0:
+  version "2.7.0"
+  resolved "https://registry.yarnpkg.com/secure-json-parse/-/secure-json-parse-2.7.0.tgz#5a5f9cd6ae47df23dba3151edd06855d47e09862"
+  integrity sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==
+
+sonic-boom@^3.0.0, sonic-boom@^3.1.0:
+  version "3.3.0"
+  resolved "https://registry.yarnpkg.com/sonic-boom/-/sonic-boom-3.3.0.tgz#cffab6dafee3b2bcb88d08d589394198bee1838c"
+  integrity sha512-LYxp34KlZ1a2Jb8ZQgFCK3niIHzibdwtwNUWKg0qQRzsDoJ3Gfgkf8KdBTFU3SkejDEIlWwnSnpVdOZIhFMl/g==
+  dependencies:
+    atomic-sleep "^1.0.0"
+
+split2@^4.0.0:
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/split2/-/split2-4.2.0.tgz#c9c5920904d148bab0b9f67145f245a86aadbfa4"
+  integrity sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==
+
+string_decoder@^1.1.1:
+  version "1.3.0"
+  resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.3.0.tgz#42f114594a46cf1a8e30b0a84f56c78c3edac21e"
+  integrity sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==
+  dependencies:
+    safe-buffer "~5.2.0"
+
+strip-json-comments@^3.1.1:
+  version "3.1.1"
+  resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-3.1.1.tgz#31f1281b3832630434831c310c01cccda8cbe006"
+  integrity sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==
+
+thread-stream@^2.0.0:
+  version "2.3.0"
+  resolved "https://registry.yarnpkg.com/thread-stream/-/thread-stream-2.3.0.tgz#4fc07fb39eff32ae7bad803cb7dd9598349fed33"
+  integrity sha512-kaDqm1DET9pp3NXwR8382WHbnpXnRkN9xGN9dQt3B2+dmXiW8X1SOwmFOxAErEQ47ObhZ96J6yhZNXuyCOL7KA==
+  dependencies:
+    real-require "^0.2.0"
+
+util-deprecate@^1.0.1:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"
+  integrity sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==
+
+wrappy@1:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f"
+  integrity sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==

scripts/build/peertube-runner.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -eu
+
+
+cd ./packages/peertube-runner
+rm -rf ./dist
+
+../../node_modules/.bin/tsc -b --verbose
+rm -rf ./dist
+mkdir ./dist
+
+./node_modules/.bin/esbuild ./peertube-runner.ts --bundle --platform=node --external:"./lib-cov/fluent-ffmpeg" --external:pg-hstore --outfile=dist/peertube-runner.js

scripts/ci.sh

@@ -104,14 +104,17 @@ elif [ "$1" = "api-5" ]; then
     npm run build:server
 
     transcodingFiles=$(findTestFiles ./dist/server/tests/api/transcoding)
+    runnersFiles=$(findTestFiles ./dist/server/tests/api/runners)
 
-    MOCHA_PARALLEL=true runTest "$1" $((2*$speedFactor)) $transcodingFiles
+    MOCHA_PARALLEL=true runTest "$1" $((2*$speedFactor)) $transcodingFiles $runnersFiles
 elif [ "$1" = "external-plugins" ]; then
     npm run build:server
+    npm run build:peertube-runner
 
     externalPluginsFiles=$(findTestFiles ./dist/server/tests/external-plugins)
+    peertubeRunnerFiles=$(findTestFiles ./dist/server/tests/peertube-runner)
 
-    runTest "$1" 1 $externalPluginsFiles
+    runTest "$1" 1 $externalPluginsFiles $peertubeRunnerFiles
 elif [ "$1" = "lint" ]; then
     npm run eslint -- --ext .ts "./server/**/*.ts" "shared/**/*.ts" "scripts/**/*.ts"
     npm run swagger-cli -- validate support/doc/api/openapi.yaml

scripts/dev/peertube-runner.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -eu
+
+rm -rf ./packages/peertube-runner/dist
+
+cd ./packages/peertube-runner
+
+../../node_modules/.bin/concurrently -k \
+  "../../node_modules/.bin/tsc -w --noEmit" \
+  "./node_modules/.bin/esbuild ./peertube-runner.ts --bundle --sourcemap --platform=node --external:"./lib-cov/fluent-ffmpeg" --external:pg-hstore --watch --outfile=dist/peertube-runner.js"