WIP plugins: add ability to register plugins

config/default.yaml

@@ -80,6 +80,7 @@ storage:
   torrents: 'storage/torrents/'
   captions: 'storage/captions/'
   cache: 'storage/cache/'
+  plugins: 'storage/plugins/'
 
 log:
   level: 'info' # debug/info/warning/error

config/production.yaml.example

@@ -81,6 +81,7 @@ storage:
   torrents: '/var/www/peertube/storage/torrents/'
   captions: '/var/www/peertube/storage/captions/'
   cache: '/var/www/peertube/storage/cache/'
+  plugins: '/var/www/peertube/storage/plugins/'
 
 log:
   level: 'info' # debug/info/warning/error

server.ts

@@ -94,6 +94,8 @@ import {
   feedsRouter,
   staticRouter,
   servicesRouter,
+  pluginsRouter,
+  themesRouter,
   webfingerRouter,
   trackerRouter,
   createWebsocketTrackerServer, botsRouter
@@ -173,6 +175,10 @@ app.use(apiRoute, apiRouter)
 // Services (oembed...)
 app.use('/services', servicesRouter)
 
+// Plugins & themes
+app.use('/plugins', pluginsRouter)
+app.use('/themes', themesRouter)
+
 app.use('/', activityPubRouter)
 app.use('/', feedsRouter)
 app.use('/', webfingerRouter)

server/controllers/index.ts

@@ -7,3 +7,5 @@ export * from './static'
 export * from './webfinger'
 export * from './tracker'
 export * from './bots'
+export * from './plugins'
+export * from './themes'

server/controllers/plugins.ts

@@ -0,0 +1,48 @@
+import * as express from 'express'
+import { PLUGIN_GLOBAL_CSS_PATH } from '../initializers/constants'
+import { join } from 'path'
+import { RegisteredPlugin } from '../lib/plugins/plugin-manager'
+import { servePluginStaticDirectoryValidator } from '../middlewares/validators/plugins'
+
+const pluginsRouter = express.Router()
+
+pluginsRouter.get('/global.css',
+  express.static(PLUGIN_GLOBAL_CSS_PATH, { fallthrough: false })
+)
+
+pluginsRouter.get('/:pluginName/:pluginVersion/statics/:staticEndpoint',
+  servePluginStaticDirectoryValidator,
+  servePluginStaticDirectory
+)
+
+pluginsRouter.get('/:pluginName/:pluginVersion/client-scripts/:staticEndpoint',
+  servePluginStaticDirectoryValidator,
+  servePluginClientScripts
+)
+
+// ---------------------------------------------------------------------------
+
+export {
+  pluginsRouter
+}
+
+// ---------------------------------------------------------------------------
+
+function servePluginStaticDirectory (req: express.Request, res: express.Response) {
+  const plugin: RegisteredPlugin = res.locals.registeredPlugin
+  const staticEndpoint = req.params.staticEndpoint
+
+  const staticPath = plugin.staticDirs[staticEndpoint]
+  if (!staticPath) {
+    return res.sendStatus(404)
+  }
+
+  return express.static(join(plugin.path, staticPath), { fallthrough: false })
+}
+
+function servePluginClientScripts (req: express.Request, res: express.Response) {
+  const plugin: RegisteredPlugin = res.locals.registeredPlugin
+  const staticEndpoint = req.params.staticEndpoint
+
+  return express.static(join(plugin.path, staticEndpoint), { fallthrough: false })
+}

server/controllers/themes.ts

@@ -0,0 +1,28 @@
+import * as express from 'express'
+import { PLUGIN_GLOBAL_CSS_PATH } from '../initializers/constants'
+import { join } from 'path'
+import { RegisteredPlugin } from '../lib/plugins/plugin-manager'
+import { servePluginStaticDirectoryValidator } from '../middlewares/validators/plugins'
+import { serveThemeCSSValidator } from '../middlewares/validators/themes'
+
+const themesRouter = express.Router()
+
+themesRouter.get('/:themeName/:themeVersion/css/:staticEndpoint',
+  serveThemeCSSValidator,
+  serveThemeCSSDirectory
+)
+
+// ---------------------------------------------------------------------------
+
+export {
+  themesRouter
+}
+
+// ---------------------------------------------------------------------------
+
+function serveThemeCSSDirectory (req: express.Request, res: express.Response) {
+  const plugin: RegisteredPlugin = res.locals.registeredPlugin
+  const staticEndpoint = req.params.staticEndpoint
+
+  return express.static(join(plugin.path, staticEndpoint), { fallthrough: false })
+}

server/helpers/custom-validators/misc.ts

@@ -1,10 +1,18 @@
 import 'multer'
 import * as validator from 'validator'
+import { sep } from 'path'
 
 function exists (value: any) {
   return value !== undefined && value !== null
 }
 
+function isSafePath (p: string) {
+  return exists(p) &&
+    (p + '').split(sep).every(part => {
+      return [ '', '.', '..' ].includes(part) === false
+    })
+}
+
 function isArray (value: any) {
   return Array.isArray(value)
 }
@@ -97,6 +105,7 @@ export {
   isNotEmptyIntArray,
   isArray,
   isIdValid,
+  isSafePath,
   isUUIDValid,
   isIdOrUUIDValid,
   isDateValid,

server/helpers/custom-validators/plugins.ts

@@ -0,0 +1,82 @@
+import { exists, isArray, isSafePath } from './misc'
+import * as validator from 'validator'
+import { PluginType } from '../../../shared/models/plugins/plugin.type'
+import { CONSTRAINTS_FIELDS } from '../../initializers/constants'
+import { PluginPackageJson } from '../../../shared/models/plugins/plugin-package-json.model'
+import { isUrlValid } from './activitypub/misc'
+
+const PLUGINS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.PLUGINS
+
+function isPluginTypeValid (value: any) {
+  return exists(value) && validator.isInt('' + value) && PluginType[value] !== undefined
+}
+
+function isPluginNameValid (value: string) {
+  return exists(value) &&
+    validator.isLength(value, PLUGINS_CONSTRAINTS_FIELDS.NAME) &&
+    validator.matches(value, /^[a-z\-]+$/)
+}
+
+function isPluginDescriptionValid (value: string) {
+  return exists(value) && validator.isLength(value, PLUGINS_CONSTRAINTS_FIELDS.DESCRIPTION)
+}
+
+function isPluginVersionValid (value: string) {
+  if (!exists(value)) return false
+
+  const parts = (value + '').split('.')
+
+  return parts.length === 3 && parts.every(p => validator.isInt(p))
+}
+
+function isPluginEngineValid (engine: any) {
+  return exists(engine) && exists(engine.peertube)
+}
+
+function isStaticDirectoriesValid (staticDirs: any) {
+  if (!exists(staticDirs) || typeof staticDirs !== 'object') return false
+
+  for (const key of Object.keys(staticDirs)) {
+    if (!isSafePath(staticDirs[key])) return false
+  }
+
+  return true
+}
+
+function isClientScriptsValid (clientScripts: any[]) {
+  return isArray(clientScripts) &&
+    clientScripts.every(c => {
+      return isSafePath(c.script) && isArray(c.scopes)
+    })
+}
+
+function isCSSPathsValid (css: any[]) {
+  return isArray(css) && css.every(c => isSafePath(c))
+}
+
+function isPackageJSONValid (packageJSON: PluginPackageJson, pluginType: PluginType) {
+  return isPluginNameValid(packageJSON.name) &&
+    isPluginDescriptionValid(packageJSON.description) &&
+    isPluginEngineValid(packageJSON.engine) &&
+    isUrlValid(packageJSON.homepage) &&
+    exists(packageJSON.author) &&
+    isUrlValid(packageJSON.bugs) &&
+    (pluginType === PluginType.THEME || isSafePath(packageJSON.library)) &&
+    isStaticDirectoriesValid(packageJSON.staticDirs) &&
+    isCSSPathsValid(packageJSON.css) &&
+    isClientScriptsValid(packageJSON.clientScripts)
+}
+
+function isLibraryCodeValid (library: any) {
+  return typeof library.register === 'function'
+    && typeof library.unregister === 'function'
+}
+
+export {
+  isPluginTypeValid,
+  isPackageJSONValid,
+  isPluginVersionValid,
+  isPluginNameValid,
+  isPluginDescriptionValid,
+  isLibraryCodeValid
+}

server/initializers/checker-before-init.ts

@@ -12,7 +12,7 @@ function checkMissedConfig () {
     'smtp.hostname', 'smtp.port', 'smtp.username', 'smtp.password', 'smtp.tls', 'smtp.from_address',
     'email.body.signature', 'email.object.prefix',
     'storage.avatars', 'storage.videos', 'storage.logs', 'storage.previews', 'storage.thumbnails', 'storage.torrents', 'storage.cache',
-    'storage.redundancy', 'storage.tmp', 'storage.streaming_playlists',
+    'storage.redundancy', 'storage.tmp', 'storage.streaming_playlists', 'storage.plugins',
     'log.level',
     'user.video_quota', 'user.video_quota_daily',
     'csp.enabled', 'csp.report_only', 'csp.report_uri',

server/initializers/config.ts

@@ -63,7 +63,8 @@ const CONFIG = {
     PREVIEWS_DIR: buildPath(config.get<string>('storage.previews')),
     CAPTIONS_DIR: buildPath(config.get<string>('storage.captions')),
     TORRENTS_DIR: buildPath(config.get<string>('storage.torrents')),
-    CACHE_DIR: buildPath(config.get<string>('storage.cache'))
+    CACHE_DIR: buildPath(config.get<string>('storage.cache')),
+    PLUGINS_DIR: buildPath(config.get<string>('storage.plugins'))
   },
   WEBSERVER: {
     SCHEME: config.get<boolean>('webserver.https') === true ? 'https' : 'http',

server/initializers/constants.ts

@@ -277,6 +277,10 @@ let CONSTRAINTS_FIELDS = {
   CONTACT_FORM: {
     FROM_NAME: { min: 1, max: 120 }, // Length
     BODY: { min: 3, max: 5000 } // Length
+  },
+  PLUGINS: {
+    NAME: { min: 1, max: 214 }, // Length
+    DESCRIPTION: { min: 1, max: 20000 } // Length
   }
 }
 
@@ -578,6 +582,11 @@ const P2P_MEDIA_LOADER_PEER_VERSION = 2
 
 // ---------------------------------------------------------------------------
 
+const PLUGIN_GLOBAL_CSS_FILE_NAME = 'plugins-global.css'
+const PLUGIN_GLOBAL_CSS_PATH = join(CONFIG.STORAGE.TMP_DIR, PLUGIN_GLOBAL_CSS_FILE_NAME)
+
+// ---------------------------------------------------------------------------
+
 // Special constants for a test instance
 if (isTestInstance() === true) {
   PRIVATE_RSA_KEY_SIZE = 1024
@@ -650,6 +659,8 @@ export {
   REMOTE_SCHEME,
   FOLLOW_STATES,
   SERVER_ACTOR_NAME,
+  PLUGIN_GLOBAL_CSS_FILE_NAME,
+  PLUGIN_GLOBAL_CSS_PATH,
   PRIVATE_RSA_KEY_SIZE,
   ROUTE_CACHE_LIFETIME,
   SORTABLE_COLUMNS,

server/lib/plugins/plugin-manager.ts

@@ -0,0 +1,169 @@
+import { PluginModel } from '../../models/server/plugin'
+import { logger } from '../../helpers/logger'
+import { RegisterHookOptions } from '../../../shared/models/plugins/register.model'
+import { join } from 'path'
+import { CONFIG } from '../../initializers/config'
+import { isLibraryCodeValid, isPackageJSONValid } from '../../helpers/custom-validators/plugins'
+import { PluginPackageJson } from '../../../shared/models/plugins/plugin-package-json.model'
+import { PluginLibrary } from '../../../shared/models/plugins/plugin-library.model'
+import { createReadStream, createWriteStream } from 'fs'
+import { PLUGIN_GLOBAL_CSS_PATH } from '../../initializers/constants'
+import { PluginType } from '../../../shared/models/plugins/plugin.type'
+
+export interface RegisteredPlugin {
+  name: string
+  version: string
+  description: string
+  peertubeEngine: string
+
+  type: PluginType
+
+  path: string
+
+  staticDirs: { [name: string]: string }
+
+  css: string[]
+
+  // Only if this is a plugin
+  unregister?: Function
+}
+
+export interface HookInformationValue {
+  pluginName: string
+  handler: Function
+  priority: number
+}
+
+export class PluginManager {
+
+  private static instance: PluginManager
+
+  private registeredPlugins: { [ name: string ]: RegisteredPlugin } = {}
+  private hooks: { [ name: string ]: HookInformationValue[] } = {}
+
+  private constructor () {
+  }
+
+  async registerPlugins () {
+    const plugins = await PluginModel.listEnabledPluginsAndThemes()
+
+    for (const plugin of plugins) {
+      try {
+        await this.registerPluginOrTheme(plugin)
+      } catch (err) {
+        logger.error('Cannot register plugin %s, skipping.', plugin.name, { err })
+      }
+    }
+
+    this.sortHooksByPriority()
+  }
+
+  getRegisteredPlugin (name: string) {
+    return this.registeredPlugins[ name ]
+  }
+
+  getRegisteredTheme (name: string) {
+    const registered = this.getRegisteredPlugin(name)
+
+    if (!registered || registered.type !== PluginType.THEME) return undefined
+
+    return registered
+  }
+
+  async unregister (name: string) {
+    const plugin = this.getRegisteredPlugin(name)
+
+    if (!plugin) {
+      throw new Error(`Unknown plugin ${name} to unregister`)
+    }
+
+    if (plugin.type === PluginType.THEME) {
+      throw new Error(`Cannot unregister ${name}: this is a theme`)
+    }
+
+    await plugin.unregister()
+  }
+
+  private async registerPluginOrTheme (plugin: PluginModel) {
+    logger.info('Registering plugin or theme %s.', plugin.name)
+
+    const pluginPath = join(CONFIG.STORAGE.PLUGINS_DIR, plugin.name, plugin.version)
+    const packageJSON: PluginPackageJson = require(join(pluginPath, 'package.json'))
+
+    if (!isPackageJSONValid(packageJSON, plugin.type)) {
+      throw new Error('Package.JSON is invalid.')
+    }
+
+    let library: PluginLibrary
+    if (plugin.type === PluginType.PLUGIN) {
+      library = await this.registerPlugin(plugin, pluginPath, packageJSON)
+    }
+
+    this.registeredPlugins[ plugin.name ] = {
+      name: plugin.name,
+      type: plugin.type,
+      version: plugin.version,
+      description: plugin.description,
+      peertubeEngine: plugin.peertubeEngine,
+      path: pluginPath,
+      staticDirs: packageJSON.staticDirs,
+      css: packageJSON.css,
+      unregister: library ? library.unregister : undefined
+    }
+  }
+
+  private async registerPlugin (plugin: PluginModel, pluginPath: string, packageJSON: PluginPackageJson) {
+    const registerHook = (options: RegisterHookOptions) => {
+      if (!this.hooks[options.target]) this.hooks[options.target] = []
+
+      this.hooks[options.target].push({
+        pluginName: plugin.name,
+        handler: options.handler,
+        priority: options.priority || 0
+      })
+    }
+
+    const library: PluginLibrary = require(join(pluginPath, packageJSON.library))
+    if (!isLibraryCodeValid(library)) {
+      throw new Error('Library code is not valid (miss register or unregister function)')
+    }
+
+    library.register({ registerHook })
+
+    logger.info('Add plugin %s CSS to global file.', plugin.name)
+
+    await this.addCSSToGlobalFile(pluginPath, packageJSON.css)
+
+    return library
+  }
+
+  private sortHooksByPriority () {
+    for (const hookName of Object.keys(this.hooks)) {
+      this.hooks[hookName].sort((a, b) => {
+        return b.priority - a.priority
+      })
+    }
+  }
+
+  private async addCSSToGlobalFile (pluginPath: string, cssRelativePaths: string[]) {
+    for (const cssPath of cssRelativePaths) {
+      await this.concatFiles(join(pluginPath, cssPath), PLUGIN_GLOBAL_CSS_PATH)
+    }
+  }
+
+  private concatFiles (input: string, output: string) {
+    return new Promise<void>((res, rej) => {
+      const outputStream = createWriteStream(input)
+      const inputStream = createReadStream(output)
+
+      inputStream.pipe(outputStream)
+
+      inputStream.on('end', () => res())
+      inputStream.on('error', err => rej(err))
+    })
+  }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+}

server/lib/schedulers/remove-old-history-scheduler.ts

@@ -3,7 +3,6 @@ import { AbstractScheduler } from './abstract-scheduler'
 import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
 import { UserVideoHistoryModel } from '../../models/account/user-video-history'
 import { CONFIG } from '../../initializers/config'
-import { isTestInstance } from '../../helpers/core-utils'
 
 export class RemoveOldHistoryScheduler extends AbstractScheduler {
 

server/middlewares/validators/plugins.ts

@@ -0,0 +1,35 @@
+import * as express from 'express'
+import { param } from 'express-validator/check'
+import { logger } from '../../helpers/logger'
+import { areValidationErrors } from './utils'
+import { isPluginNameValid, isPluginVersionValid } from '../../helpers/custom-validators/plugins'
+import { PluginManager } from '../../lib/plugins/plugin-manager'
+import { isSafePath } from '../../helpers/custom-validators/misc'
+
+const servePluginStaticDirectoryValidator = [
+  param('pluginName').custom(isPluginNameValid).withMessage('Should have a valid plugin name'),
+  param('pluginVersion').custom(isPluginVersionValid).withMessage('Should have a valid plugin version'),
+  param('staticEndpoint').custom(isSafePath).withMessage('Should have a valid static endpoint'),
+
+  (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    logger.debug('Checking servePluginStaticDirectory parameters', { parameters: req.params })
+
+    if (areValidationErrors(req, res)) return
+
+    const plugin = PluginManager.Instance.getRegisteredPlugin(req.params.pluginName)
+
+    if (!plugin || plugin.version !== req.params.pluginVersion) {
+      return res.sendStatus(404)
+    }
+
+    res.locals.registeredPlugin = plugin
+
+    return next()
+  }
+]
+
+// ---------------------------------------------------------------------------
+
+export {
+  servePluginStaticDirectoryValidator
+}

server/middlewares/validators/themes.ts

@@ -0,0 +1,39 @@
+import * as express from 'express'
+import { param } from 'express-validator/check'
+import { logger } from '../../helpers/logger'
+import { areValidationErrors } from './utils'
+import { isPluginNameValid, isPluginVersionValid } from '../../helpers/custom-validators/plugins'
+import { PluginManager } from '../../lib/plugins/plugin-manager'
+import { isSafePath } from '../../helpers/custom-validators/misc'
+
+const serveThemeCSSValidator = [
+  param('themeName').custom(isPluginNameValid).withMessage('Should have a valid theme name'),
+  param('themeVersion').custom(isPluginVersionValid).withMessage('Should have a valid theme version'),
+  param('staticEndpoint').custom(isSafePath).withMessage('Should have a valid static endpoint'),
+
+  (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    logger.debug('Checking serveThemeCSS parameters', { parameters: req.params })
+
+    if (areValidationErrors(req, res)) return
+
+    const theme = PluginManager.Instance.getRegisteredTheme(req.params.themeName)
+
+    if (!theme || theme.version !== req.params.themeVersion) {
+      return res.sendStatus(404)
+    }
+
+    if (theme.css.includes(req.params.staticEndpoint) === false) {
+      return res.sendStatus(404)
+    }
+
+    res.locals.registeredPlugin = theme
+
+    return next()
+  }
+]
+
+// ---------------------------------------------------------------------------
+
+export {
+  serveThemeCSSValidator
+}

server/models/server/plugin.ts

@@ -0,0 +1,79 @@
+import { AllowNull, Column, CreatedAt, DataType, Is, Model, Table, UpdatedAt } from 'sequelize-typescript'
+import { throwIfNotValid } from '../utils'
+import {
+  isPluginDescriptionValid,
+  isPluginNameValid,
+  isPluginTypeValid,
+  isPluginVersionValid
+} from '../../helpers/custom-validators/plugins'
+
+@Table({
+  tableName: 'plugin',
+  indexes: [
+    {
+      fields: [ 'name' ],
+      unique: true
+    }
+  ]
+})
+export class PluginModel extends Model<PluginModel> {
+
+  @AllowNull(false)
+  @Is('PluginName', value => throwIfNotValid(value, isPluginNameValid, 'name'))
+  @Column
+  name: string
+
+  @AllowNull(false)
+  @Is('PluginType', value => throwIfNotValid(value, isPluginTypeValid, 'type'))
+  @Column
+  type: number
+
+  @AllowNull(false)
+  @Is('PluginVersion', value => throwIfNotValid(value, isPluginVersionValid, 'version'))
+  @Column
+  version: string
+
+  @AllowNull(false)
+  @Column
+  enabled: boolean
+
+  @AllowNull(false)
+  @Column
+  uninstalled: boolean
+
+  @AllowNull(false)
+  @Is('PluginPeertubeEngine', value => throwIfNotValid(value, isPluginVersionValid, 'peertubeEngine'))
+  @Column
+  peertubeEngine: string
+
+  @AllowNull(true)
+  @Is('PluginDescription', value => throwIfNotValid(value, isPluginDescriptionValid, 'description'))
+  @Column
+  description: string
+
+  @AllowNull(true)
+  @Column(DataType.JSONB)
+  settings: any
+
+  @AllowNull(true)
+  @Column(DataType.JSONB)
+  storage: any
+
+  @CreatedAt
+  createdAt: Date
+
+  @UpdatedAt
+  updatedAt: Date
+
+  static listEnabledPluginsAndThemes () {
+    const query = {
+      where: {
+        enabled: true,
+        uninstalled: false
+      }
+    }
+
+    return PluginModel.findAll(query)
+  }
+
+}

server/typings/express.ts

@@ -20,9 +20,11 @@ import { VideoAbuseModel } from '../models/video/video-abuse'
 import { VideoBlacklistModel } from '../models/video/video-blacklist'
 import { VideoCaptionModel } from '../models/video/video-caption'
 import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
+import { RegisteredPlugin } from '../lib/plugins/plugin-manager'
 
 declare module 'express' {
 
+
   interface Response {
     locals: {
       video?: VideoModel
@@ -77,6 +79,8 @@ declare module 'express' {
       }
 
       authenticated?: boolean
+
+      registeredPlugin?: RegisteredPlugin
     }
   }
 }

shared/models/plugins/plugin-library.model.ts

@@ -0,0 +1,6 @@
+import { RegisterOptions } from './register-options.type'
+
+export interface PluginLibrary {
+  register: (options: RegisterOptions) => void
+  unregister: () => Promise<any>
+}

shared/models/plugins/plugin-package-json.model.ts

@@ -0,0 +1,15 @@
+export type PluginPackageJson = {
+  name: string
+  description: string
+  engine: { peertube: string },
+
+  homepage: string,
+  author: string,
+  bugs: string,
+  library: string,
+
+  staticDirs: { [ name: string ]: string }
+  css: string[]
+
+  clientScripts: { script: string, scopes: string[] }[]
+}

shared/models/plugins/plugin.type.ts

@@ -0,0 +1,4 @@
+export enum PluginType {
+  PLUGIN = 1,
+  THEME = 2
+}

shared/models/plugins/register-options.type.ts

@@ -0,0 +1,5 @@
+import { RegisterHookOptions } from './register.model'
+
+export type RegisterOptions = {
+  registerHook: (options: RegisterHookOptions) => void
+}

shared/models/plugins/register.model.ts

@@ -0,0 +1,5 @@
+export type RegisterHookOptions = {
+  target: string
+  handler: Function
+  priority?: number
+}

support/docker/production/config/production.yaml

@@ -52,6 +52,7 @@ storage:
   torrents: '../data/torrents/'
   captions: '../data/captions/'
   cache: '../data/cache/'
+  plugins: '../data/plugins/'
 
 log:
   level: 'info' # debug/info/warning/error