server.ts 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379
  1. // ----------- Node modules -----------
  2. import { registerOpentelemetryTracing } from './server/lib/opentelemetry/tracing'
  3. registerOpentelemetryTracing()
  4. import express from 'express'
  5. import morgan, { token } from 'morgan'
  6. import cors from 'cors'
  7. import cookieParser from 'cookie-parser'
  8. import { frameguard } from 'helmet'
  9. import { parse } from 'useragent'
  10. import anonymize from 'ip-anonymize'
  11. import { program as cli } from 'commander'
  12. process.title = 'peertube'
  13. // Create our main app
  14. const app = express().disable('x-powered-by')
  15. // ----------- Core checker -----------
  16. import { checkMissedConfig, checkFFmpeg, checkNodeVersion } from './server/initializers/checker-before-init'
  17. // Do not use barrels because we don't want to load all modules here (we need to initialize database first)
  18. import { CONFIG } from './server/initializers/config'
  19. import { API_VERSION, FILES_CACHE, WEBSERVER, loadLanguages } from './server/initializers/constants'
  20. import { logger } from './server/helpers/logger'
  21. const missed = checkMissedConfig()
  22. if (missed.length !== 0) {
  23. logger.error('Your configuration files miss keys: ' + missed)
  24. process.exit(-1)
  25. }
  26. checkFFmpeg(CONFIG)
  27. .catch(err => {
  28. logger.error('Error in ffmpeg check.', { err })
  29. process.exit(-1)
  30. })
  31. try {
  32. checkNodeVersion()
  33. } catch (err) {
  34. logger.error('Error in NodeJS check.', { err })
  35. process.exit(-1)
  36. }
  37. import { checkConfig, checkActivityPubUrls, checkFFmpegVersion } from './server/initializers/checker-after-init'
  38. try {
  39. checkConfig()
  40. } catch (err) {
  41. logger.error('Config error.', { err })
  42. process.exit(-1)
  43. }
  44. // Trust our proxy (IP forwarding...)
  45. app.set('trust proxy', CONFIG.TRUST_PROXY)
  46. app.use((_req, res, next) => {
  47. res.locals.requestStart = Date.now()
  48. return next()
  49. })
  50. // Security middleware
  51. import { baseCSP } from './server/middlewares/csp'
  52. if (CONFIG.CSP.ENABLED) {
  53. app.use(baseCSP)
  54. }
  55. if (CONFIG.SECURITY.FRAMEGUARD.ENABLED) {
  56. app.use(frameguard({
  57. action: 'deny' // we only allow it for /videos/embed, see server/controllers/client.ts
  58. }))
  59. }
  60. // ----------- Database -----------
  61. // Initialize database and models
  62. import { initDatabaseModels, checkDatabaseConnectionOrDie } from './server/initializers/database'
  63. checkDatabaseConnectionOrDie()
  64. import { migrate } from './server/initializers/migrator'
  65. migrate()
  66. .then(() => initDatabaseModels(false))
  67. .then(() => startApplication())
  68. .catch(err => {
  69. logger.error('Cannot start application.', { err })
  70. process.exit(-1)
  71. })
  72. // ----------- Initialize -----------
  73. loadLanguages()
  74. // ----------- PeerTube modules -----------
  75. import { installApplication } from './server/initializers/installer'
  76. import { Emailer } from './server/lib/emailer'
  77. import { JobQueue } from './server/lib/job-queue'
  78. import { VideosPreviewCache, VideosCaptionCache } from './server/lib/files-cache'
  79. import {
  80. activityPubRouter,
  81. apiRouter,
  82. miscRouter,
  83. clientsRouter,
  84. feedsRouter,
  85. staticRouter,
  86. wellKnownRouter,
  87. lazyStaticRouter,
  88. servicesRouter,
  89. pluginsRouter,
  90. webfingerRouter,
  91. trackerRouter,
  92. createWebsocketTrackerServer,
  93. botsRouter,
  94. downloadRouter
  95. } from './server/controllers'
  96. import { advertiseDoNotTrack } from './server/middlewares/dnt'
  97. import { apiFailMiddleware } from './server/middlewares/error'
  98. import { Redis } from './server/lib/redis'
  99. import { ActorFollowScheduler } from './server/lib/schedulers/actor-follow-scheduler'
  100. import { RemoveOldViewsScheduler } from './server/lib/schedulers/remove-old-views-scheduler'
  101. import { RemoveOldJobsScheduler } from './server/lib/schedulers/remove-old-jobs-scheduler'
  102. import { UpdateVideosScheduler } from './server/lib/schedulers/update-videos-scheduler'
  103. import { YoutubeDlUpdateScheduler } from './server/lib/schedulers/youtube-dl-update-scheduler'
  104. import { VideosRedundancyScheduler } from './server/lib/schedulers/videos-redundancy-scheduler'
  105. import { RemoveOldHistoryScheduler } from './server/lib/schedulers/remove-old-history-scheduler'
  106. import { AutoFollowIndexInstances } from './server/lib/schedulers/auto-follow-index-instances'
  107. import { RemoveDanglingResumableUploadsScheduler } from './server/lib/schedulers/remove-dangling-resumable-uploads-scheduler'
  108. import { VideoViewsBufferScheduler } from './server/lib/schedulers/video-views-buffer-scheduler'
  109. import { GeoIPUpdateScheduler } from './server/lib/schedulers/geo-ip-update-scheduler'
  110. import { isHTTPSignatureDigestValid } from './server/helpers/peertube-crypto'
  111. import { PeerTubeSocket } from './server/lib/peertube-socket'
  112. import { updateStreamingPlaylistsInfohashesIfNeeded } from './server/lib/hls'
  113. import { PluginsCheckScheduler } from './server/lib/schedulers/plugins-check-scheduler'
  114. import { PeerTubeVersionCheckScheduler } from './server/lib/schedulers/peertube-version-check-scheduler'
  115. import { Hooks } from './server/lib/plugins/hooks'
  116. import { PluginManager } from './server/lib/plugins/plugin-manager'
  117. import { LiveManager } from './server/lib/live'
  118. import { HttpStatusCode } from './shared/models/http/http-error-codes'
  119. import { VideosTorrentCache } from '@server/lib/files-cache/videos-torrent-cache'
  120. import { ServerConfigManager } from '@server/lib/server-config-manager'
  121. import { VideoViewsManager } from '@server/lib/views/video-views-manager'
  122. import { isTestOrDevInstance } from './server/helpers/core-utils'
  123. import { OpenTelemetryMetrics } from '@server/lib/opentelemetry/metrics'
  124. import { ApplicationModel } from '@server/models/application/application'
  125. import { VideoChannelSyncLatestScheduler } from '@server/lib/schedulers/video-channel-sync-latest-scheduler'
  126. // ----------- Command line -----------
  127. cli
  128. .option('--no-client', 'Start PeerTube without client interface')
  129. .option('--no-plugins', 'Start PeerTube without plugins/themes enabled')
  130. .option('--benchmark-startup', 'Automatically stop server when initialized')
  131. .parse(process.argv)
  132. // ----------- App -----------
  133. // Enable CORS for develop
  134. if (isTestOrDevInstance()) {
  135. app.use(cors({
  136. origin: '*',
  137. exposedHeaders: 'Retry-After',
  138. credentials: true
  139. }))
  140. }
  141. // For the logger
  142. token('remote-addr', (req: express.Request) => {
  143. if (CONFIG.LOG.ANONYMIZE_IP === true || req.get('DNT') === '1') {
  144. return anonymize(req.ip, 16, 16)
  145. }
  146. return req.ip
  147. })
  148. token('user-agent', (req: express.Request) => {
  149. if (req.get('DNT') === '1') {
  150. return parse(req.get('user-agent')).family
  151. }
  152. return req.get('user-agent')
  153. })
  154. app.use(morgan('combined', {
  155. stream: {
  156. write: (str: string) => logger.info(str.trim(), { tags: [ 'http' ] })
  157. },
  158. skip: req => CONFIG.LOG.LOG_PING_REQUESTS === false && req.originalUrl === '/api/v1/ping'
  159. }))
  160. // Add .fail() helper to response
  161. app.use(apiFailMiddleware)
  162. // For body requests
  163. app.use(express.urlencoded({ extended: false }))
  164. app.use(express.json({
  165. type: [ 'application/json', 'application/*+json' ],
  166. limit: '500kb',
  167. verify: (req: express.Request, res: express.Response, buf: Buffer) => {
  168. const valid = isHTTPSignatureDigestValid(buf, req)
  169. if (valid !== true) {
  170. res.fail({
  171. status: HttpStatusCode.FORBIDDEN_403,
  172. message: 'Invalid digest'
  173. })
  174. }
  175. }
  176. }))
  177. // Cookies
  178. app.use(cookieParser())
  179. // W3C DNT Tracking Status
  180. app.use(advertiseDoNotTrack)
  181. // ----------- Open Telemetry -----------
  182. OpenTelemetryMetrics.Instance.init(app)
  183. // ----------- Views, routes and static files -----------
  184. // API
  185. const apiRoute = '/api/' + API_VERSION
  186. app.use(apiRoute, apiRouter)
  187. // Services (oembed...)
  188. app.use('/services', servicesRouter)
  189. // Plugins & themes
  190. app.use('/', pluginsRouter)
  191. app.use('/', activityPubRouter)
  192. app.use('/', feedsRouter)
  193. app.use('/', webfingerRouter)
  194. app.use('/', trackerRouter)
  195. app.use('/', botsRouter)
  196. // Static files
  197. app.use('/', staticRouter)
  198. app.use('/', wellKnownRouter)
  199. app.use('/', miscRouter)
  200. app.use('/', downloadRouter)
  201. app.use('/', lazyStaticRouter)
  202. // Client files, last valid routes!
  203. const cliOptions = cli.opts<{ client: boolean, plugins: boolean }>()
  204. if (cliOptions.client) app.use('/', clientsRouter)
  205. // ----------- Errors -----------
  206. // Catch unmatched routes
  207. app.use((_req, res: express.Response) => {
  208. res.status(HttpStatusCode.NOT_FOUND_404).end()
  209. })
  210. // Catch thrown errors
  211. app.use((err, _req, res: express.Response, _next) => {
  212. // Format error to be logged
  213. let error = 'Unknown error.'
  214. if (err) {
  215. error = err.stack || err.message || err
  216. }
  217. // Handling Sequelize error traces
  218. const sql = err?.parent ? err.parent.sql : undefined
  219. // Help us to debug SequelizeConnectionAcquireTimeoutError errors
  220. const activeRequests = err?.name === 'SequelizeConnectionAcquireTimeoutError' && typeof (process as any)._getActiveRequests !== 'function'
  221. ? (process as any)._getActiveRequests()
  222. : undefined
  223. logger.error('Error in controller.', { err: error, sql, activeRequests })
  224. return res.fail({
  225. status: err.status || HttpStatusCode.INTERNAL_SERVER_ERROR_500,
  226. message: err.message,
  227. type: err.name
  228. })
  229. })
  230. const server = createWebsocketTrackerServer(app)
  231. // ----------- Run -----------
  232. async function startApplication () {
  233. const port = CONFIG.LISTEN.PORT
  234. const hostname = CONFIG.LISTEN.HOSTNAME
  235. await installApplication()
  236. // Check activity pub urls are valid
  237. checkActivityPubUrls()
  238. .catch(err => {
  239. logger.error('Error in ActivityPub URLs checker.', { err })
  240. process.exit(-1)
  241. })
  242. checkFFmpegVersion()
  243. .catch(err => logger.error('Cannot check ffmpeg version', { err }))
  244. Redis.Instance.init()
  245. Emailer.Instance.init()
  246. await Promise.all([
  247. Emailer.Instance.checkConnection(),
  248. JobQueue.Instance.init(),
  249. ServerConfigManager.Instance.init()
  250. ])
  251. // Caches initializations
  252. VideosPreviewCache.Instance.init(CONFIG.CACHE.PREVIEWS.SIZE, FILES_CACHE.PREVIEWS.MAX_AGE)
  253. VideosCaptionCache.Instance.init(CONFIG.CACHE.VIDEO_CAPTIONS.SIZE, FILES_CACHE.VIDEO_CAPTIONS.MAX_AGE)
  254. VideosTorrentCache.Instance.init(CONFIG.CACHE.TORRENTS.SIZE, FILES_CACHE.TORRENTS.MAX_AGE)
  255. // Enable Schedulers
  256. ActorFollowScheduler.Instance.enable()
  257. RemoveOldJobsScheduler.Instance.enable()
  258. UpdateVideosScheduler.Instance.enable()
  259. YoutubeDlUpdateScheduler.Instance.enable()
  260. VideosRedundancyScheduler.Instance.enable()
  261. RemoveOldHistoryScheduler.Instance.enable()
  262. RemoveOldViewsScheduler.Instance.enable()
  263. PluginsCheckScheduler.Instance.enable()
  264. PeerTubeVersionCheckScheduler.Instance.enable()
  265. AutoFollowIndexInstances.Instance.enable()
  266. RemoveDanglingResumableUploadsScheduler.Instance.enable()
  267. VideoChannelSyncLatestScheduler.Instance.enable()
  268. VideoViewsBufferScheduler.Instance.enable()
  269. GeoIPUpdateScheduler.Instance.enable()
  270. OpenTelemetryMetrics.Instance.registerMetrics()
  271. PluginManager.Instance.init(server)
  272. // Before PeerTubeSocket init
  273. PluginManager.Instance.registerWebSocketRouter()
  274. PeerTubeSocket.Instance.init(server)
  275. VideoViewsManager.Instance.init()
  276. updateStreamingPlaylistsInfohashesIfNeeded()
  277. .catch(err => logger.error('Cannot update streaming playlist infohashes.', { err }))
  278. LiveManager.Instance.init()
  279. if (CONFIG.LIVE.ENABLED) await LiveManager.Instance.run()
  280. // Make server listening
  281. server.listen(port, hostname, async () => {
  282. if (cliOptions.plugins) {
  283. try {
  284. await PluginManager.Instance.rebuildNativePluginsIfNeeded()
  285. await PluginManager.Instance.registerPluginsAndThemes()
  286. } catch (err) {
  287. logger.error('Cannot register plugins and themes.', { err })
  288. }
  289. }
  290. ApplicationModel.updateNodeVersions()
  291. .catch(err => logger.error('Cannot update node versions.', { err }))
  292. JobQueue.Instance.start()
  293. .catch(err => {
  294. logger.error('Cannot start job queue.', { err })
  295. process.exit(-1)
  296. })
  297. logger.info('HTTP server listening on %s:%d', hostname, port)
  298. logger.info('Web server: %s', WEBSERVER.URL)
  299. Hooks.runAction('action:application.listening')
  300. if (cliOptions['benchmarkStartup']) process.exit(0)
  301. })
  302. process.on('exit', () => {
  303. JobQueue.Instance.terminate()
  304. .catch(err => logger.error('Cannot terminate job queue.', { err }))
  305. })
  306. process.on('SIGINT', () => process.exit(0))
  307. }