video-redundancy.ts 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544
  1. import {
  2. AllowNull,
  3. BeforeDestroy,
  4. BelongsTo,
  5. Column,
  6. CreatedAt,
  7. DataType,
  8. ForeignKey,
  9. Is,
  10. Model,
  11. Scopes,
  12. Table,
  13. UpdatedAt
  14. } from 'sequelize-typescript'
  15. import { ActorModel } from '../activitypub/actor'
  16. import { getVideoSort, parseAggregateResult, throwIfNotValid } from '../utils'
  17. import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc'
  18. import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../initializers/constants'
  19. import { VideoFileModel } from '../video/video-file'
  20. import { getServerActor } from '../../helpers/utils'
  21. import { VideoModel } from '../video/video'
  22. import { VideoRedundancyStrategy } from '../../../shared/models/redundancy'
  23. import { logger } from '../../helpers/logger'
  24. import { CacheFileObject, VideoPrivacy } from '../../../shared'
  25. import { VideoChannelModel } from '../video/video-channel'
  26. import { ServerModel } from '../server/server'
  27. import { sample } from 'lodash'
  28. import { isTestInstance } from '../../helpers/core-utils'
  29. import * as Bluebird from 'bluebird'
  30. import { col, FindOptions, fn, literal, Op, Transaction } from 'sequelize'
  31. import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist'
  32. import { CONFIG } from '../../initializers/config'
  33. export enum ScopeNames {
  34. WITH_VIDEO = 'WITH_VIDEO'
  35. }
  36. @Scopes(() => ({
  37. [ ScopeNames.WITH_VIDEO ]: {
  38. include: [
  39. {
  40. model: VideoFileModel,
  41. required: false,
  42. include: [
  43. {
  44. model: VideoModel,
  45. required: true
  46. }
  47. ]
  48. },
  49. {
  50. model: VideoStreamingPlaylistModel,
  51. required: false,
  52. include: [
  53. {
  54. model: VideoModel,
  55. required: true
  56. }
  57. ]
  58. }
  59. ]
  60. }
  61. }))
  62. @Table({
  63. tableName: 'videoRedundancy',
  64. indexes: [
  65. {
  66. fields: [ 'videoFileId' ]
  67. },
  68. {
  69. fields: [ 'actorId' ]
  70. },
  71. {
  72. fields: [ 'url' ],
  73. unique: true
  74. }
  75. ]
  76. })
  77. export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
  78. @CreatedAt
  79. createdAt: Date
  80. @UpdatedAt
  81. updatedAt: Date
  82. @AllowNull(false)
  83. @Column
  84. expiresOn: Date
  85. @AllowNull(false)
  86. @Is('VideoRedundancyFileUrl', value => throwIfNotValid(value, isUrlValid, 'fileUrl'))
  87. @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
  88. fileUrl: string
  89. @AllowNull(false)
  90. @Is('VideoRedundancyUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url'))
  91. @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
  92. url: string
  93. @AllowNull(true)
  94. @Column
  95. strategy: string // Only used by us
  96. @ForeignKey(() => VideoFileModel)
  97. @Column
  98. videoFileId: number
  99. @BelongsTo(() => VideoFileModel, {
  100. foreignKey: {
  101. allowNull: true
  102. },
  103. onDelete: 'cascade'
  104. })
  105. VideoFile: VideoFileModel
  106. @ForeignKey(() => VideoStreamingPlaylistModel)
  107. @Column
  108. videoStreamingPlaylistId: number
  109. @BelongsTo(() => VideoStreamingPlaylistModel, {
  110. foreignKey: {
  111. allowNull: true
  112. },
  113. onDelete: 'cascade'
  114. })
  115. VideoStreamingPlaylist: VideoStreamingPlaylistModel
  116. @ForeignKey(() => ActorModel)
  117. @Column
  118. actorId: number
  119. @BelongsTo(() => ActorModel, {
  120. foreignKey: {
  121. allowNull: false
  122. },
  123. onDelete: 'cascade'
  124. })
  125. Actor: ActorModel
  126. @BeforeDestroy
  127. static async removeFile (instance: VideoRedundancyModel) {
  128. if (!instance.isOwned()) return
  129. if (instance.videoFileId) {
  130. const videoFile = await VideoFileModel.loadWithVideo(instance.videoFileId)
  131. const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}`
  132. logger.info('Removing duplicated video file %s.', logIdentifier)
  133. videoFile.Video.removeFile(videoFile, true)
  134. .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
  135. }
  136. if (instance.videoStreamingPlaylistId) {
  137. const videoStreamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(instance.videoStreamingPlaylistId)
  138. const videoUUID = videoStreamingPlaylist.Video.uuid
  139. logger.info('Removing duplicated video streaming playlist %s.', videoUUID)
  140. videoStreamingPlaylist.Video.removeStreamingPlaylist(true)
  141. .catch(err => logger.error('Cannot delete video streaming playlist files of %s.', videoUUID, { err }))
  142. }
  143. return undefined
  144. }
  145. static async loadLocalByFileId (videoFileId: number) {
  146. const actor = await getServerActor()
  147. const query = {
  148. where: {
  149. actorId: actor.id,
  150. videoFileId
  151. }
  152. }
  153. return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
  154. }
  155. static async loadLocalByStreamingPlaylistId (videoStreamingPlaylistId: number) {
  156. const actor = await getServerActor()
  157. const query = {
  158. where: {
  159. actorId: actor.id,
  160. videoStreamingPlaylistId
  161. }
  162. }
  163. return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
  164. }
  165. static loadByUrl (url: string, transaction?: Transaction) {
  166. const query = {
  167. where: {
  168. url
  169. },
  170. transaction
  171. }
  172. return VideoRedundancyModel.findOne(query)
  173. }
  174. static async isLocalByVideoUUIDExists (uuid: string) {
  175. const actor = await getServerActor()
  176. const query = {
  177. raw: true,
  178. attributes: [ 'id' ],
  179. where: {
  180. actorId: actor.id
  181. },
  182. include: [
  183. {
  184. attributes: [ ],
  185. model: VideoFileModel,
  186. required: true,
  187. include: [
  188. {
  189. attributes: [ ],
  190. model: VideoModel,
  191. required: true,
  192. where: {
  193. uuid
  194. }
  195. }
  196. ]
  197. }
  198. ]
  199. }
  200. return VideoRedundancyModel.findOne(query)
  201. .then(r => !!r)
  202. }
  203. static async getVideoSample (p: Bluebird<VideoModel[]>) {
  204. const rows = await p
  205. const ids = rows.map(r => r.id)
  206. const id = sample(ids)
  207. return VideoModel.loadWithFiles(id, undefined, !isTestInstance())
  208. }
  209. static async findMostViewToDuplicate (randomizedFactor: number) {
  210. // On VideoModel!
  211. const query = {
  212. attributes: [ 'id', 'views' ],
  213. limit: randomizedFactor,
  214. order: getVideoSort('-views'),
  215. where: {
  216. privacy: VideoPrivacy.PUBLIC
  217. },
  218. include: [
  219. await VideoRedundancyModel.buildVideoFileForDuplication(),
  220. VideoRedundancyModel.buildServerRedundancyInclude()
  221. ]
  222. }
  223. return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
  224. }
  225. static async findTrendingToDuplicate (randomizedFactor: number) {
  226. // On VideoModel!
  227. const query = {
  228. attributes: [ 'id', 'views' ],
  229. subQuery: false,
  230. group: 'VideoModel.id',
  231. limit: randomizedFactor,
  232. order: getVideoSort('-trending'),
  233. where: {
  234. privacy: VideoPrivacy.PUBLIC
  235. },
  236. include: [
  237. await VideoRedundancyModel.buildVideoFileForDuplication(),
  238. VideoRedundancyModel.buildServerRedundancyInclude(),
  239. VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS)
  240. ]
  241. }
  242. return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
  243. }
  244. static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) {
  245. // On VideoModel!
  246. const query = {
  247. attributes: [ 'id', 'publishedAt' ],
  248. limit: randomizedFactor,
  249. order: getVideoSort('-publishedAt'),
  250. where: {
  251. privacy: VideoPrivacy.PUBLIC,
  252. views: {
  253. [ Op.gte ]: minViews
  254. }
  255. },
  256. include: [
  257. await VideoRedundancyModel.buildVideoFileForDuplication(),
  258. VideoRedundancyModel.buildServerRedundancyInclude()
  259. ]
  260. }
  261. return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
  262. }
  263. static async loadOldestLocalThatAlreadyExpired (strategy: VideoRedundancyStrategy, expiresAfterMs: number) {
  264. const expiredDate = new Date()
  265. expiredDate.setMilliseconds(expiredDate.getMilliseconds() - expiresAfterMs)
  266. const actor = await getServerActor()
  267. const query = {
  268. where: {
  269. actorId: actor.id,
  270. strategy,
  271. createdAt: {
  272. [ Op.lt ]: expiredDate
  273. }
  274. }
  275. }
  276. return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findOne(query)
  277. }
  278. static async getTotalDuplicated (strategy: VideoRedundancyStrategy) {
  279. const actor = await getServerActor()
  280. const query: FindOptions = {
  281. include: [
  282. {
  283. attributes: [],
  284. model: VideoRedundancyModel,
  285. required: true,
  286. where: {
  287. actorId: actor.id,
  288. strategy
  289. }
  290. }
  291. ]
  292. }
  293. return VideoFileModel.aggregate('size', 'SUM', query)
  294. .then(result => parseAggregateResult(result))
  295. }
  296. static async listLocalExpired () {
  297. const actor = await getServerActor()
  298. const query = {
  299. where: {
  300. actorId: actor.id,
  301. expiresOn: {
  302. [ Op.lt ]: new Date()
  303. }
  304. }
  305. }
  306. return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
  307. }
  308. static async listRemoteExpired () {
  309. const actor = await getServerActor()
  310. const query = {
  311. where: {
  312. actorId: {
  313. [Op.ne]: actor.id
  314. },
  315. expiresOn: {
  316. [ Op.lt ]: new Date()
  317. }
  318. }
  319. }
  320. return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
  321. }
  322. static async listLocalOfServer (serverId: number) {
  323. const actor = await getServerActor()
  324. const buildVideoInclude = () => ({
  325. model: VideoModel,
  326. required: true,
  327. include: [
  328. {
  329. attributes: [],
  330. model: VideoChannelModel.unscoped(),
  331. required: true,
  332. include: [
  333. {
  334. attributes: [],
  335. model: ActorModel.unscoped(),
  336. required: true,
  337. where: {
  338. serverId
  339. }
  340. }
  341. ]
  342. }
  343. ]
  344. })
  345. const query = {
  346. where: {
  347. actorId: actor.id
  348. },
  349. include: [
  350. {
  351. model: VideoFileModel,
  352. required: false,
  353. include: [ buildVideoInclude() ]
  354. },
  355. {
  356. model: VideoStreamingPlaylistModel,
  357. required: false,
  358. include: [ buildVideoInclude() ]
  359. }
  360. ]
  361. }
  362. return VideoRedundancyModel.findAll(query)
  363. }
  364. static async getStats (strategy: VideoRedundancyStrategy) {
  365. const actor = await getServerActor()
  366. const query: FindOptions = {
  367. raw: true,
  368. attributes: [
  369. [ fn('COALESCE', fn('SUM', col('VideoFile.size')), '0'), 'totalUsed' ],
  370. [ fn('COUNT', fn('DISTINCT', col('videoId'))), 'totalVideos' ],
  371. [ fn('COUNT', col('videoFileId')), 'totalVideoFiles' ]
  372. ],
  373. where: {
  374. strategy,
  375. actorId: actor.id
  376. },
  377. include: [
  378. {
  379. attributes: [],
  380. model: VideoFileModel,
  381. required: true
  382. }
  383. ]
  384. }
  385. return VideoRedundancyModel.findOne(query)
  386. .then((r: any) => ({
  387. totalUsed: parseAggregateResult(r.totalUsed),
  388. totalVideos: r.totalVideos,
  389. totalVideoFiles: r.totalVideoFiles
  390. }))
  391. }
  392. getVideo () {
  393. if (this.VideoFile) return this.VideoFile.Video
  394. return this.VideoStreamingPlaylist.Video
  395. }
  396. isOwned () {
  397. return !!this.strategy
  398. }
  399. toActivityPubObject (): CacheFileObject {
  400. if (this.VideoStreamingPlaylist) {
  401. return {
  402. id: this.url,
  403. type: 'CacheFile' as 'CacheFile',
  404. object: this.VideoStreamingPlaylist.Video.url,
  405. expires: this.expiresOn.toISOString(),
  406. url: {
  407. type: 'Link',
  408. mimeType: 'application/x-mpegURL',
  409. mediaType: 'application/x-mpegURL',
  410. href: this.fileUrl
  411. }
  412. }
  413. }
  414. return {
  415. id: this.url,
  416. type: 'CacheFile' as 'CacheFile',
  417. object: this.VideoFile.Video.url,
  418. expires: this.expiresOn.toISOString(),
  419. url: {
  420. type: 'Link',
  421. mimeType: MIMETYPES.VIDEO.EXT_MIMETYPE[ this.VideoFile.extname ] as any,
  422. mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[ this.VideoFile.extname ] as any,
  423. href: this.fileUrl,
  424. height: this.VideoFile.resolution,
  425. size: this.VideoFile.size,
  426. fps: this.VideoFile.fps
  427. }
  428. }
  429. }
  430. // Don't include video files we already duplicated
  431. private static async buildVideoFileForDuplication () {
  432. const actor = await getServerActor()
  433. const notIn = literal(
  434. '(' +
  435. `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "videoFileId" IS NOT NULL` +
  436. ')'
  437. )
  438. return {
  439. attributes: [],
  440. model: VideoFileModel.unscoped(),
  441. required: true,
  442. where: {
  443. id: {
  444. [ Op.notIn ]: notIn
  445. }
  446. }
  447. }
  448. }
  449. private static buildServerRedundancyInclude () {
  450. return {
  451. attributes: [],
  452. model: VideoChannelModel.unscoped(),
  453. required: true,
  454. include: [
  455. {
  456. attributes: [],
  457. model: ActorModel.unscoped(),
  458. required: true,
  459. include: [
  460. {
  461. attributes: [],
  462. model: ServerModel.unscoped(),
  463. required: true,
  464. where: {
  465. redundancyAllowed: true
  466. }
  467. }
  468. ]
  469. }
  470. ]
  471. }
  472. }
  473. }