Add ability to limit videos history size

config/default.yaml

@@ -111,6 +111,13 @@ tracker:
   # Reject peers that do a lot of announces (could improve privacy of TCP/UDP peers)
   reject_too_many_announces: false
 
+history:
+  videos:
+    # If you want to limit users videos history
+    # -1 means there is no limitations
+    # Other values could be '6 months' or '30 days' etc (PeerTube will periodically delete old entries from database)
+    max_age: -1
+
 cache:
   previews:
     size: 500 # Max number of previews you want to cache

config/production.yaml.example

@@ -112,6 +112,12 @@ tracker:
   # Reject peers that do a lot of announces (could improve privacy of TCP/UDP peers)
   reject_too_many_announces: false
 
+history:
+  videos:
+    # If you want to limit users videos history
+    # -1 means there is no limitations
+    # Other values could be '6 months' or '30 days' etc (PeerTube will periodically delete old entries from database)
+    max_age: -1
 
 ###############################################################################
 #

server.ts

@@ -105,6 +105,7 @@ import { RemoveOldJobsScheduler } from './server/lib/schedulers/remove-old-jobs-
 import { UpdateVideosScheduler } from './server/lib/schedulers/update-videos-scheduler'
 import { YoutubeDlUpdateScheduler } from './server/lib/schedulers/youtube-dl-update-scheduler'
 import { VideosRedundancyScheduler } from './server/lib/schedulers/videos-redundancy-scheduler'
+import { RemoveOldHistoryScheduler } from './server/lib/schedulers/remove-old-history-scheduler'
 import { isHTTPSignatureDigestValid } from './server/helpers/peertube-crypto'
 import { PeerTubeSocket } from './server/lib/peertube-socket'
 import { updateStreamingPlaylistsInfohashesIfNeeded } from './server/lib/hls'
@@ -240,6 +241,7 @@ async function startApplication () {
   UpdateVideosScheduler.Instance.enable()
   YoutubeDlUpdateScheduler.Instance.enable()
   VideosRedundancyScheduler.Instance.enable()
+  RemoveOldHistoryScheduler.Instance.enable()
 
   // Redis initialization
   Redis.Instance.init()

server/controllers/api/users/my-history.ts

@@ -48,7 +48,7 @@ async function removeUserHistory (req: express.Request, res: express.Response) {
   const beforeDate = req.body.beforeDate || null
 
   await sequelizeTypescript.transaction(t => {
-    return UserVideoHistoryModel.removeHistoryBefore(user, beforeDate, t)
+    return UserVideoHistoryModel.removeUserHistoryBefore(user, beforeDate, t)
   })
 
   // Do not send the delete to other instances, we delete OUR copy of this video abuse

server/helpers/core-utils.ts

@@ -40,7 +40,7 @@ const timeTable = {
   month:        3600000 * 24 * 30
 }
 
-export function parseDuration (duration: number | string): number {
+export function parseDurationToMs (duration: number | string): number {
   if (typeof duration === 'number') return duration
 
   if (typeof duration === 'string') {

server/helpers/custom-validators/videos.ts

@@ -5,7 +5,8 @@ import 'multer'
 import * as validator from 'validator'
 import { UserRight, VideoFilter, VideoPrivacy, VideoRateType } from '../../../shared'
 import {
-  CONSTRAINTS_FIELDS, MIMETYPES,
+  CONSTRAINTS_FIELDS,
+  MIMETYPES,
   VIDEO_CATEGORIES,
   VIDEO_LICENCES,
   VIDEO_PRIVACIES,

server/initializers/config.ts

@@ -2,7 +2,7 @@ import { IConfig } from 'config'
 import { dirname, join } from 'path'
 import { VideosRedundancy } from '../../shared/models'
 // Do not use barrels, remain constants as independent as possible
-import { buildPath, parseBytes, parseDuration, root } from '../helpers/core-utils'
+import { buildPath, parseBytes, parseDurationToMs, root } from '../helpers/core-utils'
 import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type'
 import * as bytes from 'bytes'
 
@@ -80,7 +80,7 @@ const CONFIG = {
   },
   REDUNDANCY: {
     VIDEOS: {
-      CHECK_INTERVAL: parseDuration(config.get<string>('redundancy.videos.check_interval')),
+      CHECK_INTERVAL: parseDurationToMs(config.get<string>('redundancy.videos.check_interval')),
       STRATEGIES: buildVideosRedundancy(config.get<any[]>('redundancy.videos.strategies'))
     }
   },
@@ -94,6 +94,11 @@ const CONFIG = {
     PRIVATE: config.get<boolean>('tracker.private'),
     REJECT_TOO_MANY_ANNOUNCES: config.get<boolean>('tracker.reject_too_many_announces')
   },
+  HISTORY: {
+    VIDEOS: {
+      MAX_AGE: parseDurationToMs(config.get('history.videos.max_age'))
+    }
+  },
   ADMIN: {
     get EMAIL () { return config.get<string>('admin.email') }
   },
@@ -216,7 +221,7 @@ function buildVideosRedundancy (objs: any[]): VideosRedundancy[] {
 
   return objs.map(obj => {
     return Object.assign({}, obj, {
-      minLifetime: parseDuration(obj.min_lifetime),
+      minLifetime: parseDurationToMs(obj.min_lifetime),
       size: bytes.parse(obj.size),
       minViews: obj.min_views
     })

server/initializers/constants.ts

@@ -158,12 +158,12 @@ const JOB_REQUEST_TIMEOUT = 3000 // 3 seconds
 const JOB_COMPLETED_LIFETIME = 60000 * 60 * 24 * 2 // 2 days
 const VIDEO_IMPORT_TIMEOUT = 1000 * 3600 // 1 hour
 
-// 1 hour
-let SCHEDULER_INTERVALS_MS = {
+const SCHEDULER_INTERVALS_MS = {
   actorFollowScores: 60000 * 60, // 1 hour
   removeOldJobs: 60000 * 60, // 1 hour
   updateVideos: 60000, // 1 minute
-  youtubeDLUpdate: 60000 * 60 * 24 // 1 day
+  youtubeDLUpdate: 60000 * 60 * 24, // 1 day
+  removeOldHistory: 60000 * 60 * 24 // 1 day
 }
 
 // ---------------------------------------------------------------------------
@@ -591,6 +591,7 @@ if (isTestInstance() === true) {
 
   SCHEDULER_INTERVALS_MS.actorFollowScores = 1000
   SCHEDULER_INTERVALS_MS.removeOldJobs = 10000
+  SCHEDULER_INTERVALS_MS.removeOldHistory = 5000
   SCHEDULER_INTERVALS_MS.updateVideos = 5000
   REPEAT_JOBS[ 'videos-views' ] = { every: 5000 }
 
@@ -734,7 +735,7 @@ function buildVideosExtname () {
 }
 
 function loadLanguages () {
-  VIDEO_LANGUAGES = buildLanguages()
+  Object.assign(VIDEO_LANGUAGES, buildLanguages())
 }
 
 function buildLanguages () {

server/lib/schedulers/abstract-scheduler.ts

@@ -1,4 +1,5 @@
 import { logger } from '../../helpers/logger'
+import * as Bluebird from 'bluebird'
 
 export abstract class AbstractScheduler {
 
@@ -30,5 +31,5 @@ export abstract class AbstractScheduler {
     }
   }
 
-  protected abstract internalExecute (): Promise<any>
+  protected abstract internalExecute (): Promise<any> | Bluebird<any>
 }

server/lib/schedulers/remove-old-history-scheduler.ts

@@ -0,0 +1,32 @@
+import { logger } from '../../helpers/logger'
+import { AbstractScheduler } from './abstract-scheduler'
+import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
+import { UserVideoHistoryModel } from '../../models/account/user-video-history'
+import { CONFIG } from '../../initializers/config'
+import { isTestInstance } from '../../helpers/core-utils'
+
+export class RemoveOldHistoryScheduler extends AbstractScheduler {
+
+  private static instance: AbstractScheduler
+
+  protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.removeOldHistory
+
+  private constructor () {
+    super()
+  }
+
+  protected internalExecute () {
+    if (CONFIG.HISTORY.VIDEOS.MAX_AGE === -1) return
+
+    logger.info('Removing old videos history.')
+
+    const now = new Date()
+    const beforeDate = new Date(now.getTime() - CONFIG.HISTORY.VIDEOS.MAX_AGE).toISOString()
+
+    return UserVideoHistoryModel.removeOldHistory(beforeDate)
+  }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+}

server/middlewares/cache.ts

@@ -1,6 +1,6 @@
 import * as express from 'express'
 import * as AsyncLock from 'async-lock'
-import { parseDuration } from '../helpers/core-utils'
+import { parseDurationToMs } from '../helpers/core-utils'
 import { Redis } from '../lib/redis'
 import { logger } from '../helpers/logger'
 
@@ -24,7 +24,7 @@ function cacheRoute (lifetimeArg: string | number) {
           res.send = (body) => {
             if (res.statusCode >= 200 && res.statusCode < 400) {
               const contentType = res.get('content-type')
-              const lifetime = parseDuration(lifetimeArg)
+              const lifetime = parseDurationToMs(lifetimeArg)
 
               Redis.Instance.setCachedRoute(req, body, lifetime, contentType, res.statusCode)
                    .then(() => done())

server/models/account/user-video-history.ts

@@ -67,7 +67,7 @@ export class UserVideoHistoryModel extends Model<UserVideoHistoryModel> {
     })
   }
 
-  static removeHistoryBefore (user: UserModel, beforeDate: string, t: Transaction) {
+  static removeUserHistoryBefore (user: UserModel, beforeDate: string, t: Transaction) {
     const query: DestroyOptions = {
       where: {
         userId: user.id
@@ -83,4 +83,16 @@ export class UserVideoHistoryModel extends Model<UserVideoHistoryModel> {
 
     return UserVideoHistoryModel.destroy(query)
   }
+
+  static removeOldHistory (beforeDate: string) {
+    const query: DestroyOptions = {
+      where: {
+        updatedAt: {
+          [Op.lt]: beforeDate
+        }
+      }
+    }
+
+    return UserVideoHistoryModel.destroy(query)
+  }
 }

server/tests/api/videos/videos-history.ts

@@ -7,14 +7,15 @@ import {
   flushTests,
   getVideosListWithToken,
   getVideoWithToken,
-  killallServers,
+  killallServers, reRunServer,
   runServer,
   searchVideoWithToken,
   ServerInfo,
   setAccessTokensToServers,
   updateMyUser,
   uploadVideo,
-  userLogin
+  userLogin,
+  wait
 } from '../../../../shared/utils'
 import { Video, VideoDetails } from '../../../../shared/models/videos'
 import { listMyVideosHistory, removeMyVideosHistory, userWatchVideo } from '../../../../shared/utils/videos/video-history'
@@ -192,6 +193,35 @@ describe('Test videos history', function () {
     expect(videos[1].name).to.equal('video 3')
   })
 
+  it('Should not clean old history', async function () {
+    this.timeout(50000)
+
+    killallServers([ server ])
+
+    await reRunServer(server, { history: { videos: { max_age: '10 days' } } })
+
+    await wait(6000)
+
+    // Should still have history
+
+    const res = await listMyVideosHistory(server.url, server.accessToken)
+
+    expect(res.body.total).to.equal(2)
+  })
+
+  it('Should clean old history', async function () {
+    this.timeout(50000)
+
+    killallServers([ server ])
+
+    await reRunServer(server, { history: { videos: { max_age: '5 seconds' } } })
+
+    await wait(6000)
+
+    const res = await listMyVideosHistory(server.url, server.accessToken)
+    expect(res.body.total).to.equal(0)
+  })
+
   after(async function () {
     killallServers([ server ])
 

shared/utils/videos/videos.ts

@@ -18,9 +18,11 @@ import {
 } from '../'
 import * as validator from 'validator'
 import { VideoDetails, VideoPrivacy } from '../../models/videos'
-import { VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES, VIDEO_PRIVACIES } from '../../../server/initializers/constants'
+import { VIDEO_CATEGORIES, VIDEO_LANGUAGES, loadLanguages, VIDEO_LICENCES, VIDEO_PRIVACIES } from '../../../server/initializers/constants'
 import { dateIsValid, webtorrentAdd } from '../miscs/miscs'
 
+loadLanguages()
+
 type VideoAttributes = {
   name?: string
   category?: number