Basic video redundancy implementation

client/src/app/+admin/admin.module.ts

@@ -14,6 +14,8 @@ import { UserCreateComponent, UserListComponent, UsersComponent, UserService, Us
 import { ModerationCommentModalComponent, VideoAbuseListComponent, VideoBlacklistListComponent } from './moderation'
 import { UserBanModalComponent } from '@app/+admin/users/user-list/user-ban-modal.component'
 import { ModerationComponent } from '@app/+admin/moderation/moderation.component'
+import { RedundancyCheckboxComponent } from '@app/+admin/follows/shared/redundancy-checkbox.component'
+import { RedundancyService } from '@app/+admin/follows/shared/redundancy.service'
 
 @NgModule({
   imports: [
@@ -29,6 +31,7 @@ import { ModerationComponent } from '@app/+admin/moderation/moderation.component
     FollowingAddComponent,
     FollowersListComponent,
     FollowingListComponent,
+    RedundancyCheckboxComponent,
 
     UsersComponent,
     UserCreateComponent,
@@ -54,6 +57,7 @@ import { ModerationComponent } from '@app/+admin/moderation/moderation.component
 
   providers: [
     FollowService,
+    RedundancyService,
     UserService,
     JobService,
     ConfigService

client/src/app/+admin/follows/followers-list/followers-list.component.ts

@@ -2,7 +2,7 @@ import { Component, OnInit } from '@angular/core'
 
 import { NotificationsService } from 'angular2-notifications'
 import { SortMeta } from 'primeng/primeng'
-import { AccountFollow } from '../../../../../../shared/models/actors/follow.model'
+import { ActorFollow } from '../../../../../../shared/models/actors/follow.model'
 import { RestPagination, RestTable } from '../../../shared'
 import { FollowService } from '../shared'
 import { I18n } from '@ngx-translate/i18n-polyfill'
@@ -13,7 +13,7 @@ import { I18n } from '@ngx-translate/i18n-polyfill'
   styleUrls: [ './followers-list.component.scss' ]
 })
 export class FollowersListComponent extends RestTable implements OnInit {
-  followers: AccountFollow[] = []
+  followers: ActorFollow[] = []
   totalRecords = 0
   rowsPerPage = 10
   sort: SortMeta = { field: 'createdAt', order: 1 }

client/src/app/+admin/follows/following-list/following-list.component.html

@@ -8,6 +8,7 @@
       <th i18n>Host</th>
       <th i18n>State</th>
       <th i18n pSortableColumn="createdAt">Created <p-sortIcon field="createdAt"></p-sortIcon></th>
+      <th i18n>Redundancy allowed</th>
       <th></th>
     </tr>
   </ng-template>
@@ -18,6 +19,11 @@
       <td>{{ follow.following.host }}</td>
       <td>{{ follow.state }}</td>
       <td>{{ follow.createdAt }}</td>
+      <td>
+        <my-redundancy-checkbox
+          [host]="follow.following.host" [redundancyAllowed]="follow.following.hostRedundancyAllowed"
+        ></my-redundancy-checkbox>
+      </td>
       <td class="action-cell">
         <my-delete-button (click)="removeFollowing(follow)"></my-delete-button>
       </td>

client/src/app/+admin/follows/following-list/following-list.component.scss

@@ -0,0 +1,13 @@
+@import '_variables';
+@import '_mixins';
+
+my-redundancy-checkbox /deep/ my-peertube-checkbox {
+  .form-group {
+    margin-bottom: 0;
+    align-items: center;
+  }
+
+  label {
+    margin: 0;
+  }
+}

client/src/app/+admin/follows/following-list/following-list.component.ts

@@ -1,7 +1,7 @@
 import { Component, OnInit } from '@angular/core'
 import { NotificationsService } from 'angular2-notifications'
 import { SortMeta } from 'primeng/primeng'
-import { AccountFollow } from '../../../../../../shared/models/actors/follow.model'
+import { ActorFollow } from '../../../../../../shared/models/actors/follow.model'
 import { ConfirmService } from '../../../core/confirm/confirm.service'
 import { RestPagination, RestTable } from '../../../shared'
 import { FollowService } from '../shared'
@@ -9,10 +9,11 @@ import { I18n } from '@ngx-translate/i18n-polyfill'
 
 @Component({
   selector: 'my-followers-list',
-  templateUrl: './following-list.component.html'
+  templateUrl: './following-list.component.html',
+  styleUrls: [ './following-list.component.scss' ]
 })
 export class FollowingListComponent extends RestTable implements OnInit {
-  following: AccountFollow[] = []
+  following: ActorFollow[] = []
   totalRecords = 0
   rowsPerPage = 10
   sort: SortMeta = { field: 'createdAt', order: 1 }
@@ -31,7 +32,7 @@ export class FollowingListComponent extends RestTable implements OnInit {
     this.loadSort()
   }
 
-  async removeFollowing (follow: AccountFollow) {
+  async removeFollowing (follow: ActorFollow) {
     const res = await this.confirmService.confirm(
       this.i18n('Do you really want to unfollow {{host}}?', { host: follow.following.host }),
       this.i18n('Unfollow')

client/src/app/+admin/follows/shared/follow.service.ts

@@ -3,7 +3,7 @@ import { HttpClient, HttpParams } from '@angular/common/http'
 import { Injectable } from '@angular/core'
 import { SortMeta } from 'primeng/primeng'
 import { Observable } from 'rxjs'
-import { AccountFollow, ResultList } from '../../../../../../shared'
+import { ActorFollow, ResultList } from '../../../../../../shared'
 import { environment } from '../../../../environments/environment'
 import { RestExtractor, RestPagination, RestService } from '../../../shared'
 
@@ -18,22 +18,22 @@ export class FollowService {
   ) {
   }
 
-  getFollowing (pagination: RestPagination, sort: SortMeta): Observable<ResultList<AccountFollow>> {
+  getFollowing (pagination: RestPagination, sort: SortMeta): Observable<ResultList<ActorFollow>> {
     let params = new HttpParams()
     params = this.restService.addRestGetParams(params, pagination, sort)
 
-    return this.authHttp.get<ResultList<AccountFollow>>(FollowService.BASE_APPLICATION_URL + '/following', { params })
+    return this.authHttp.get<ResultList<ActorFollow>>(FollowService.BASE_APPLICATION_URL + '/following', { params })
                .pipe(
                  map(res => this.restExtractor.convertResultListDateToHuman(res)),
                  catchError(res => this.restExtractor.handleError(res))
                )
   }
 
-  getFollowers (pagination: RestPagination, sort: SortMeta): Observable<ResultList<AccountFollow>> {
+  getFollowers (pagination: RestPagination, sort: SortMeta): Observable<ResultList<ActorFollow>> {
     let params = new HttpParams()
     params = this.restService.addRestGetParams(params, pagination, sort)
 
-    return this.authHttp.get<ResultList<AccountFollow>>(FollowService.BASE_APPLICATION_URL + '/followers', { params })
+    return this.authHttp.get<ResultList<ActorFollow>>(FollowService.BASE_APPLICATION_URL + '/followers', { params })
                .pipe(
                  map(res => this.restExtractor.convertResultListDateToHuman(res)),
                  catchError(res => this.restExtractor.handleError(res))
@@ -52,7 +52,7 @@ export class FollowService {
                )
   }
 
-  unfollow (follow: AccountFollow) {
+  unfollow (follow: ActorFollow) {
     return this.authHttp.delete(FollowService.BASE_APPLICATION_URL + '/following/' + follow.following.host)
                .pipe(
                  map(this.restExtractor.extractDataBool),

client/src/app/+admin/follows/shared/redundancy-checkbox.component.html

@@ -0,0 +1,3 @@
+<my-peertube-checkbox
+  [inputName]="host + '-redundancy-allowed'" [(ngModel)]="redundancyAllowed" (ngModelChange)="updateRedundancyState()"
+></my-peertube-checkbox>

client/src/app/+admin/follows/shared/redundancy-checkbox.component.scss

@@ -0,0 +1,2 @@
+@import '_variables';
+@import '_mixins';

client/src/app/+admin/follows/shared/redundancy-checkbox.component.ts

@@ -0,0 +1,42 @@
+import { Component, Input } from '@angular/core'
+import { AuthService } from '@app/core'
+import { RestExtractor } from '@app/shared/rest'
+import { RedirectService } from '@app/core/routing/redirect.service'
+import { NotificationsService } from 'angular2-notifications'
+import { I18n } from '@ngx-translate/i18n-polyfill'
+import { RedundancyService } from '@app/+admin/follows/shared/redundancy.service'
+
+@Component({
+  selector: 'my-redundancy-checkbox',
+  templateUrl: './redundancy-checkbox.component.html',
+  styleUrls: [ './redundancy-checkbox.component.scss' ]
+})
+export class RedundancyCheckboxComponent {
+  @Input() redundancyAllowed: boolean
+  @Input() host: string
+
+  constructor (
+    private authService: AuthService,
+    private restExtractor: RestExtractor,
+    private redirectService: RedirectService,
+    private notificationsService: NotificationsService,
+    private redundancyService: RedundancyService,
+    private i18n: I18n
+  ) { }
+
+  updateRedundancyState () {
+    this.redundancyService.updateRedundancy(this.host, this.redundancyAllowed)
+      .subscribe(
+        () => {
+          const stateLabel = this.redundancyAllowed ? this.i18n('enabled') : this.i18n('disabled')
+
+          this.notificationsService.success(
+            this.i18n('Success'),
+            this.i18n('Redundancy for {{host}} is {{stateLabel}}', { host: this.host, stateLabel })
+          )
+        },
+
+          err => this.notificationsService.error(this.i18n('Error'), err.message)
+      )
+  }
+}

client/src/app/+admin/follows/shared/redundancy.service.ts

@@ -0,0 +1,29 @@
+import { catchError, map } from 'rxjs/operators'
+import { HttpClient } from '@angular/common/http'
+import { Injectable } from '@angular/core'
+import { RestExtractor, RestService } from '@app/shared'
+import { environment } from '../../../../environments/environment'
+
+@Injectable()
+export class RedundancyService {
+  static BASE_USER_SUBSCRIPTIONS_URL = environment.apiUrl + '/api/v1/server/redundancy'
+
+  constructor (
+    private authHttp: HttpClient,
+    private restExtractor: RestExtractor,
+    private restService: RestService
+  ) { }
+
+  updateRedundancy (host: string, redundancyAllowed: boolean) {
+    const url = RedundancyService.BASE_USER_SUBSCRIPTIONS_URL + '/' + host
+
+    const body = { redundancyAllowed }
+
+    return this.authHttp.put(url, body)
+               .pipe(
+                 map(this.restExtractor.extractDataBool),
+                 catchError(err => this.restExtractor.handleError(err))
+               )
+  }
+
+}

config/default.yaml

@@ -66,6 +66,15 @@ trending:
   videos:
     interval_days: 7 # Compute trending videos for the last x days
 
+# Cache remote videos on your server, to help other instances to broadcast the video
+# You can define multiple caches using different sizes/strategies
+# Once you have defined your strategies, choose which instances you want to cache in admin -> manage follows -> following
+redundancy:
+  videos:
+#    -
+#      size: '10GB'
+#      strategy: 'most-views' # Cache videos that have the most views
+
 cache:
   previews:
     size: 500 # Max number of previews you want to cache

config/production.yaml.example

@@ -67,6 +67,15 @@ trending:
   videos:
     interval_days: 7 # Compute trending videos for the last x days
 
+# Cache remote videos on your server, to help other instances to broadcast the video
+# You can define multiple caches using different sizes/strategies
+# Once you have defined your strategies, choose which instances you want to cache in admin -> manage follows -> following
+redundancy:
+  videos:
+#    -
+#      size: '10GB'
+#      strategy: 'most-views' # Cache videos that have the most views
+
 ###############################################################################
 #
 # From this point, all the following keys can be overridden by the web interface

config/test.yaml

@@ -21,6 +21,12 @@ smtp:
 log:
   level: 'debug'
 
+redundancy:
+  videos:
+    -
+      size: '100KB'
+      strategy: 'most-views'
+
 cache:
   previews:
     size: 1

package.json

@@ -86,6 +86,7 @@
     "bluebird": "^3.5.0",
     "body-parser": "^1.12.4",
     "bull": "^3.4.2",
+    "bytes": "^3.0.0",
     "commander": "^2.13.0",
     "concurrently": "^4.0.1",
     "config": "^2.0.1",
@@ -145,6 +146,7 @@
     "@types/bluebird": "3.5.21",
     "@types/body-parser": "^1.16.3",
     "@types/bull": "^3.3.12",
+    "@types/bytes": "^3.0.0",
     "@types/chai": "^4.0.4",
     "@types/chai-json-schema": "^1.4.3",
     "@types/chai-xml": "^0.3.1",

server.ts

@@ -94,6 +94,7 @@ import { BadActorFollowScheduler } from './server/lib/schedulers/bad-actor-follo
 import { RemoveOldJobsScheduler } from './server/lib/schedulers/remove-old-jobs-scheduler'
 import { UpdateVideosScheduler } from './server/lib/schedulers/update-videos-scheduler'
 import { YoutubeDlUpdateScheduler } from './server/lib/schedulers/youtube-dl-update-scheduler'
+import { VideosRedundancyScheduler } from './server/lib/schedulers/videos-redundancy-scheduler'
 
 // ----------- Command line -----------
 
@@ -206,6 +207,7 @@ async function startApplication () {
   RemoveOldJobsScheduler.Instance.enable()
   UpdateVideosScheduler.Instance.enable()
   YoutubeDlUpdateScheduler.Instance.enable()
+  VideosRedundancyScheduler.Instance.enable()
 
   // Redis initialization
   Redis.Instance.init()

server/controllers/activitypub/client.ts

@@ -3,9 +3,9 @@ import * as express from 'express'
 import { VideoPrivacy, VideoRateType } from '../../../shared/models/videos'
 import { activityPubCollectionPagination, activityPubContextify } from '../../helpers/activitypub'
 import { CONFIG, ROUTE_CACHE_LIFETIME } from '../../initializers'
-import { buildVideoAnnounce } from '../../lib/activitypub/send'
+import { buildAnnounceWithVideoAudience } from '../../lib/activitypub/send'
 import { audiencify, getAudience } from '../../lib/activitypub/audience'
-import { createActivityData } from '../../lib/activitypub/send/send-create'
+import { buildCreateActivity } from '../../lib/activitypub/send/send-create'
 import { asyncMiddleware, executeIfActivityPub, localAccountValidator, localVideoChannelValidator } from '../../middlewares'
 import { videosGetValidator, videosShareValidator } from '../../middlewares/validators'
 import { videoCommentGetValidator } from '../../middlewares/validators/video-comments'
@@ -26,6 +26,8 @@ import {
   getVideoSharesActivityPubUrl
 } from '../../lib/activitypub'
 import { VideoCaptionModel } from '../../models/video/video-caption'
+import { videoRedundancyGetValidator } from '../../middlewares/validators/redundancy'
+import { getServerActor } from '../../helpers/utils'
 
 const activityPubClientRouter = express.Router()
 
@@ -93,6 +95,11 @@ activityPubClientRouter.get('/video-channels/:name/following',
   executeIfActivityPub(asyncMiddleware(videoChannelFollowingController))
 )
 
+activityPubClientRouter.get('/redundancy/videos/:videoId/:resolution([0-9]+)(-:fps([0-9]+))?',
+  executeIfActivityPub(asyncMiddleware(videoRedundancyGetValidator)),
+  executeIfActivityPub(asyncMiddleware(videoRedundancyController))
+)
+
 // ---------------------------------------------------------------------------
 
 export {
@@ -131,7 +138,7 @@ async function videoController (req: express.Request, res: express.Response, nex
   const videoObject = audiencify(video.toActivityPubObject(), audience)
 
   if (req.path.endsWith('/activity')) {
-    const data = createActivityData(video.url, video.VideoChannel.Account.Actor, videoObject, audience)
+    const data = buildCreateActivity(video.url, video.VideoChannel.Account.Actor, videoObject, audience)
     return activityPubResponse(activityPubContextify(data), res)
   }
 
@@ -140,9 +147,9 @@ async function videoController (req: express.Request, res: express.Response, nex
 
 async function videoAnnounceController (req: express.Request, res: express.Response, next: express.NextFunction) {
   const share = res.locals.videoShare as VideoShareModel
-  const object = await buildVideoAnnounce(share.Actor, share, res.locals.video, undefined)
+  const { activity } = await buildAnnounceWithVideoAudience(share.Actor, share, res.locals.video, undefined)
 
-  return activityPubResponse(activityPubContextify(object), res)
+  return activityPubResponse(activityPubContextify(activity), res)
 }
 
 async function videoAnnouncesController (req: express.Request, res: express.Response, next: express.NextFunction) {
@@ -219,13 +226,28 @@ async function videoCommentController (req: express.Request, res: express.Respon
   const videoCommentObject = audiencify(videoComment.toActivityPubObject(threadParentComments), audience)
 
   if (req.path.endsWith('/activity')) {
-    const data = createActivityData(videoComment.url, videoComment.Account.Actor, videoCommentObject, audience)
+    const data = buildCreateActivity(videoComment.url, videoComment.Account.Actor, videoCommentObject, audience)
     return activityPubResponse(activityPubContextify(data), res)
   }
 
   return activityPubResponse(activityPubContextify(videoCommentObject), res)
 }
 
+async function videoRedundancyController (req: express.Request, res: express.Response) {
+  const videoRedundancy = res.locals.videoRedundancy
+  const serverActor = await getServerActor()
+
+  const audience = getAudience(serverActor)
+  const object = audiencify(videoRedundancy.toActivityPubObject(), audience)
+
+  if (req.path.endsWith('/activity')) {
+    const data = buildCreateActivity(videoRedundancy.url, serverActor, object, audience)
+    return activityPubResponse(activityPubContextify(data), res)
+  }
+
+  return activityPubResponse(activityPubContextify(object), res)
+}
+
 // ---------------------------------------------------------------------------
 
 async function actorFollowing (req: express.Request, actor: ActorModel) {

server/controllers/activitypub/outbox.ts

@@ -3,7 +3,7 @@ import { Activity } from '../../../shared/models/activitypub/activity'
 import { VideoPrivacy } from '../../../shared/models/videos'
 import { activityPubCollectionPagination, activityPubContextify } from '../../helpers/activitypub'
 import { logger } from '../../helpers/logger'
-import { announceActivityData, createActivityData } from '../../lib/activitypub/send'
+import { buildAnnounceActivity, buildCreateActivity } from '../../lib/activitypub/send'
 import { buildAudience } from '../../lib/activitypub/audience'
 import { asyncMiddleware, localAccountValidator, localVideoChannelValidator } from '../../middlewares'
 import { AccountModel } from '../../models/account/account'
@@ -60,12 +60,12 @@ async function buildActivities (actor: ActorModel, start: number, count: number)
     // This is a shared video
     if (video.VideoShares !== undefined && video.VideoShares.length !== 0) {
       const videoShare = video.VideoShares[0]
-      const announceActivity = announceActivityData(videoShare.url, actor, video.url, createActivityAudience)
+      const announceActivity = buildAnnounceActivity(videoShare.url, actor, video.url, createActivityAudience)
 
       activities.push(announceActivity)
     } else {
       const videoObject = video.toActivityPubObject()
-      const createActivity = createActivityData(video.url, byActor, videoObject, createActivityAudience)
+      const createActivity = buildCreateActivity(video.url, byActor, videoObject, createActivityAudience)
 
       activities.push(createActivity)
     }

server/controllers/api/search.ts

@@ -17,8 +17,6 @@ import {
 import { VideoChannelsSearchQuery, VideosSearchQuery } from '../../../shared/models/search'
 import { getOrCreateActorAndServerAndModel, getOrCreateVideoAndAccountAndChannel } from '../../lib/activitypub'
 import { logger } from '../../helpers/logger'
-import { User } from '../../../shared/models/users'
-import { CONFIG } from '../../initializers/constants'
 import { VideoChannelModel } from '../../models/video/video-channel'
 import { loadActorUrlOrGetFromWebfinger } from '../../helpers/webfinger'
 

server/controllers/api/server/follows.ts

@@ -96,6 +96,11 @@ async function removeFollow (req: express.Request, res: express.Response, next:
   await sequelizeTypescript.transaction(async t => {
     if (follow.state === 'accepted') await sendUndoFollow(follow, t)
 
+    // Disable redundancy on unfollowed instances
+    const server = follow.ActorFollowing.Server
+    server.redundancyAllowed = false
+    await server.save({ transaction: t })
+
     await follow.destroy({ transaction: t })
   })
 

server/controllers/api/server/index.ts

@@ -1,10 +1,12 @@
 import * as express from 'express'
 import { serverFollowsRouter } from './follows'
 import { statsRouter } from './stats'
+import { serverRedundancyRouter } from './redundancy'
 
 const serverRouter = express.Router()
 
 serverRouter.use('/', serverFollowsRouter)
+serverRouter.use('/', serverRedundancyRouter)
 serverRouter.use('/', statsRouter)
 
 // ---------------------------------------------------------------------------

server/controllers/api/server/redundancy.ts

@@ -0,0 +1,32 @@
+import * as express from 'express'
+import { UserRight } from '../../../../shared/models/users'
+import { asyncMiddleware, authenticate, ensureUserHasRight } from '../../../middlewares'
+import { updateServerRedundancyValidator } from '../../../middlewares/validators/redundancy'
+import { ServerModel } from '../../../models/server/server'
+
+const serverRedundancyRouter = express.Router()
+
+serverRedundancyRouter.put('/redundancy/:host',
+  authenticate,
+  ensureUserHasRight(UserRight.MANAGE_SERVER_FOLLOW),
+  asyncMiddleware(updateServerRedundancyValidator),
+  asyncMiddleware(updateRedundancy)
+)
+
+// ---------------------------------------------------------------------------
+
+export {
+  serverRedundancyRouter
+}
+
+// ---------------------------------------------------------------------------
+
+async function updateRedundancy (req: express.Request, res: express.Response, next: express.NextFunction) {
+  const server = res.locals.server as ServerModel
+
+  server.redundancyAllowed = req.body.redundancyAllowed
+
+  await server.save()
+
+  return res.sendStatus(204)
+}

server/controllers/api/videos/abuse.ts

@@ -112,7 +112,7 @@ async function reportVideoAbuse (req: express.Request, res: express.Response) {
 
     // We send the video abuse to the origin server
     if (videoInstance.isOwned() === false) {
-      await sendVideoAbuse(reporterAccount.Actor, videoAbuseInstance, videoInstance, t)
+      await sendVideoAbuse(reporterAccount.Actor, videoAbuseInstance, videoInstance)
     }
 
     auditLogger.create(reporterAccount.Actor.getIdentifier(), new VideoAbuseAuditView(videoAbuseInstance.toFormattedJSON()))

server/helpers/activitypub.ts

@@ -14,20 +14,24 @@ function activityPubContextify <T> (data: T) {
       'https://w3id.org/security/v1',
       {
         RsaSignature2017: 'https://w3id.org/security#RsaSignature2017',
+        pt: 'https://joinpeertube.org/ns',
+        schema: 'http://schema.org#',
         Hashtag: 'as:Hashtag',
-        uuid: 'http://schema.org/identifier',
-        category: 'http://schema.org/category',
-        licence: 'http://schema.org/license',
-        subtitleLanguage: 'http://schema.org/subtitleLanguage',
+        uuid: 'schema:identifier',
+        category: 'schema:category',
+        licence: 'schema:license',
+        subtitleLanguage: 'schema:subtitleLanguage',
         sensitive: 'as:sensitive',
-        language: 'http://schema.org/inLanguage',
-        views: 'http://schema.org/Number',
-        stats: 'http://schema.org/Number',
-        size: 'http://schema.org/Number',
-        fps: 'http://schema.org/Number',
-        commentsEnabled: 'http://schema.org/Boolean',
-        waitTranscoding: 'http://schema.org/Boolean',
-        support: 'http://schema.org/Text'
+        language: 'schema:inLanguage',
+        views: 'schema:Number',
+        stats: 'schema:Number',
+        size: 'schema:Number',
+        fps: 'schema:Number',
+        commentsEnabled: 'schema:Boolean',
+        waitTranscoding: 'schema:Boolean',
+        expires: 'schema:expires',
+        support: 'schema:Text',
+        CacheFile: 'pt:CacheFile'
       },
       {
         likes: {

server/helpers/custom-validators/activitypub/activity.ts

@@ -1,7 +1,10 @@
 import * as validator from 'validator'
 import { Activity, ActivityType } from '../../../../shared/models/activitypub'
 import {
-  isActorAcceptActivityValid, isActorDeleteActivityValid, isActorFollowActivityValid, isActorRejectActivityValid,
+  isActorAcceptActivityValid,
+  isActorDeleteActivityValid,
+  isActorFollowActivityValid,
+  isActorRejectActivityValid,
   isActorUpdateActivityValid
 } from './actor'
 import { isAnnounceActivityValid } from './announce'
@@ -11,12 +14,13 @@ import { isUndoActivityValid } from './undo'
 import { isVideoCommentCreateActivityValid, isVideoCommentDeleteActivityValid } from './video-comments'
 import {
   isVideoFlagValid,
-  sanitizeAndCheckVideoTorrentCreateActivity,
   isVideoTorrentDeleteActivityValid,
+  sanitizeAndCheckVideoTorrentCreateActivity,
   sanitizeAndCheckVideoTorrentUpdateActivity
 } from './videos'
 import { isViewActivityValid } from './view'
 import { exists } from '../misc'
+import { isCacheFileCreateActivityValid, isCacheFileUpdateActivityValid } from './cache-file'
 
 function isRootActivityValid (activity: any) {
   return Array.isArray(activity['@context']) && (
@@ -67,11 +71,13 @@ function checkCreateActivity (activity: any) {
     isDislikeActivityValid(activity) ||
     sanitizeAndCheckVideoTorrentCreateActivity(activity) ||
     isVideoFlagValid(activity) ||
-    isVideoCommentCreateActivityValid(activity)
+    isVideoCommentCreateActivityValid(activity) ||
+    isCacheFileCreateActivityValid(activity)
 }
 
 function checkUpdateActivity (activity: any) {
-  return sanitizeAndCheckVideoTorrentUpdateActivity(activity) ||
+  return isCacheFileUpdateActivityValid(activity) ||
+    sanitizeAndCheckVideoTorrentUpdateActivity(activity) ||
     isActorUpdateActivityValid(activity)
 }
 

server/helpers/custom-validators/activitypub/cache-file.ts

@@ -0,0 +1,28 @@
+import { isActivityPubUrlValid, isBaseActivityValid } from './misc'
+import { isRemoteVideoUrlValid } from './videos'
+import { isDateValid, exists } from '../misc'
+import { CacheFileObject } from '../../../../shared/models/activitypub/objects'
+
+function isCacheFileCreateActivityValid (activity: any) {
+  return isBaseActivityValid(activity, 'Create') &&
+    isCacheFileObjectValid(activity.object)
+}
+
+function isCacheFileUpdateActivityValid (activity: any) {
+  return isBaseActivityValid(activity, 'Update') &&
+    isCacheFileObjectValid(activity.object)
+}
+
+function isCacheFileObjectValid (object: CacheFileObject) {
+  return exists(object) &&
+    object.type === 'CacheFile' &&
+    isDateValid(object.expires) &&
+    isActivityPubUrlValid(object.object) &&
+    isRemoteVideoUrlValid(object.url)
+}
+
+export {
+  isCacheFileUpdateActivityValid,
+  isCacheFileCreateActivityValid,
+  isCacheFileObjectValid
+}

server/helpers/custom-validators/activitypub/misc.ts

@@ -3,7 +3,7 @@ import { CONSTRAINTS_FIELDS } from '../../../initializers'
 import { isTestInstance } from '../../core-utils'
 import { exists } from '../misc'
 
-function isActivityPubUrlValid (url: string) {
+function isUrlValid (url: string) {
   const isURLOptions = {
     require_host: true,
     require_tld: true,
@@ -17,13 +17,18 @@ function isActivityPubUrlValid (url: string) {
     isURLOptions.require_tld = false
   }
 
-  return exists(url) && validator.isURL('' + url, isURLOptions) && validator.isLength('' + url, CONSTRAINTS_FIELDS.ACTORS.URL)
+  return exists(url) && validator.isURL('' + url, isURLOptions)
+}
+
+function isActivityPubUrlValid (url: string) {
+  return isUrlValid(url) && validator.isLength('' + url, CONSTRAINTS_FIELDS.ACTORS.URL)
 }
 
 function isBaseActivityValid (activity: any, type: string) {
   return (activity['@context'] === undefined || Array.isArray(activity['@context'])) &&
     activity.type === type &&
     isActivityPubUrlValid(activity.id) &&
+    exists(activity.actor) &&
     (isActivityPubUrlValid(activity.actor) || isActivityPubUrlValid(activity.actor.id)) &&
     (
       activity.to === undefined ||
@@ -49,6 +54,7 @@ function setValidAttributedTo (obj: any) {
 }
 
 export {
+  isUrlValid,
   isActivityPubUrlValid,
   isBaseActivityValid,
   setValidAttributedTo

server/helpers/custom-validators/activitypub/undo.ts

@@ -2,6 +2,7 @@ import { isActorFollowActivityValid } from './actor'
 import { isBaseActivityValid } from './misc'
 import { isDislikeActivityValid, isLikeActivityValid } from './rate'
 import { isAnnounceActivityValid } from './announce'
+import { isCacheFileCreateActivityValid } from './cache-file'
 
 function isUndoActivityValid (activity: any) {
   return isBaseActivityValid(activity, 'Undo') &&
@@ -9,7 +10,8 @@ function isUndoActivityValid (activity: any) {
       isActorFollowActivityValid(activity.object) ||
       isLikeActivityValid(activity.object) ||
       isDislikeActivityValid(activity.object) ||
-      isAnnounceActivityValid(activity.object)
+      isAnnounceActivityValid(activity.object) ||
+      isCacheFileCreateActivityValid(activity.object)
     )
 }
 

server/helpers/custom-validators/activitypub/videos.ts

@@ -75,6 +75,30 @@ function sanitizeAndCheckVideoTorrentObject (video: any) {
     video.attributedTo.length !== 0
 }
 
+function isRemoteVideoUrlValid (url: any) {
+  // FIXME: Old bug, we used the width to represent the resolution. Remove it in a few release (currently beta.11)
+  if (url.width && !url.height) url.height = url.width
+
+  return url.type === 'Link' &&
+    (
+      ACTIVITY_PUB.URL_MIME_TYPES.VIDEO.indexOf(url.mimeType) !== -1 &&
+      isActivityPubUrlValid(url.href) &&
+      validator.isInt(url.height + '', { min: 0 }) &&
+      validator.isInt(url.size + '', { min: 0 }) &&
+      (!url.fps || validator.isInt(url.fps + '', { min: 0 }))
+    ) ||
+    (
+      ACTIVITY_PUB.URL_MIME_TYPES.TORRENT.indexOf(url.mimeType) !== -1 &&
+      isActivityPubUrlValid(url.href) &&
+      validator.isInt(url.height + '', { min: 0 })
+    ) ||
+    (
+      ACTIVITY_PUB.URL_MIME_TYPES.MAGNET.indexOf(url.mimeType) !== -1 &&
+      validator.isLength(url.href, { min: 5 }) &&
+      validator.isInt(url.height + '', { min: 0 })
+    )
+}
+
 // ---------------------------------------------------------------------------
 
 export {
@@ -83,7 +107,8 @@ export {
   isVideoTorrentDeleteActivityValid,
   isRemoteStringIdentifierValid,
   isVideoFlagValid,
-  sanitizeAndCheckVideoTorrentObject
+  sanitizeAndCheckVideoTorrentObject,
+  isRemoteVideoUrlValid
 }
 
 // ---------------------------------------------------------------------------
@@ -147,26 +172,4 @@ function setRemoteVideoTruncatedContent (video: any) {
   return true
 }
 
-function isRemoteVideoUrlValid (url: any) {
-  // FIXME: Old bug, we used the width to represent the resolution. Remove it in a few realease (currently beta.11)
-  if (url.width && !url.height) url.height = url.width
 
-  return url.type === 'Link' &&
-    (
-      ACTIVITY_PUB.URL_MIME_TYPES.VIDEO.indexOf(url.mimeType) !== -1 &&
-      isActivityPubUrlValid(url.href) &&
-      validator.isInt(url.height + '', { min: 0 }) &&
-      validator.isInt(url.size + '', { min: 0 }) &&
-      (!url.fps || validator.isInt(url.fps + '', { min: 0 }))
-    ) ||
-    (
-      ACTIVITY_PUB.URL_MIME_TYPES.TORRENT.indexOf(url.mimeType) !== -1 &&
-      isActivityPubUrlValid(url.href) &&
-      validator.isInt(url.height + '', { min: 0 })
-    ) ||
-    (
-      ACTIVITY_PUB.URL_MIME_TYPES.MAGNET.indexOf(url.mimeType) !== -1 &&
-      validator.isLength(url.href, { min: 5 }) &&
-      validator.isInt(url.height + '', { min: 0 })
-    )
-}

server/helpers/webtorrent.ts

@@ -5,44 +5,49 @@ import { createWriteStream, remove } from 'fs-extra'
 import { CONFIG } from '../initializers'
 import { join } from 'path'
 
-function downloadWebTorrentVideo (target: { magnetUri: string, torrentName: string }) {
+function downloadWebTorrentVideo (target: { magnetUri: string, torrentName?: string }, timeout?: number) {
   const id = target.magnetUri || target.torrentName
+  let timer
 
   const path = generateVideoTmpPath(id)
   logger.info('Importing torrent video %s', id)
 
   return new Promise<string>((res, rej) => {
     const webtorrent = new WebTorrent()
+    let file: WebTorrent.TorrentFile
 
     const torrentId = target.magnetUri || join(CONFIG.STORAGE.TORRENTS_DIR, target.torrentName)
 
     const options = { path: CONFIG.STORAGE.VIDEOS_DIR }
     const torrent = webtorrent.add(torrentId, options, torrent => {
-      if (torrent.files.length !== 1) return rej(new Error('The number of files is not equal to 1 for ' + torrentId))
+      if (torrent.files.length !== 1) {
+        if (timer) clearTimeout(timer)
 
-      const file = torrent.files[ 0 ]
+        return safeWebtorrentDestroy(webtorrent, torrentId, file.name, target.torrentName)
+          .then(() => rej(new Error('The number of files is not equal to 1 for ' + torrentId)))
+      }
+
+      file = torrent.files[ 0 ]
 
       const writeStream = createWriteStream(path)
       writeStream.on('finish', () => {
-        webtorrent.destroy(async err => {
-          if (err) return rej(err)
-
-          if (target.torrentName) {
-            remove(torrentId)
-              .catch(err => logger.error('Cannot remove torrent %s in webtorrent download.', torrentId, { err }))
-          }
+        if (timer) clearTimeout(timer)
 
-          remove(join(CONFIG.STORAGE.VIDEOS_DIR, file.name))
-            .catch(err => logger.error('Cannot remove torrent file %s in webtorrent download.', file.name, { err }))
-
-          res(path)
-        })
+        return safeWebtorrentDestroy(webtorrent, torrentId, file.name, target.torrentName)
+          .then(() => res(path))
       })
 
       file.createReadStream().pipe(writeStream)
     })
 
     torrent.on('error', err => rej(err))
+
+    if (timeout) {
+      timer = setTimeout(async () => {
+        return safeWebtorrentDestroy(webtorrent, torrentId, file ? file.name : undefined, target.torrentName)
+          .then(() => rej(new Error('Webtorrent download timeout.')))
+      }, timeout)
+    }
   })
 }
 
@@ -51,3 +56,29 @@ function downloadWebTorrentVideo (target: { magnetUri: string, torrentName: stri
 export {
   downloadWebTorrentVideo
 }
+
+// ---------------------------------------------------------------------------
+
+function safeWebtorrentDestroy (webtorrent: WebTorrent.Instance, torrentId: string, filename?: string, torrentName?: string) {
+  return new Promise(res => {
+    webtorrent.destroy(err => {
+      // Delete torrent file
+      if (torrentName) {
+        remove(torrentId)
+          .catch(err => logger.error('Cannot remove torrent %s in webtorrent download.', torrentId, { err }))
+      }
+
+      // Delete downloaded file
+      if (filename) {
+        remove(join(CONFIG.STORAGE.VIDEOS_DIR, filename))
+          .catch(err => logger.error('Cannot remove torrent file %s in webtorrent download.', filename, { err }))
+      }
+
+      if (err) {
+        logger.warn('Cannot destroy webtorrent in timeout.', { err })
+      }
+
+      return res()
+    })
+  })
+}

server/initializers/checker.ts

@@ -7,6 +7,9 @@ import { parse } from 'url'
 import { CONFIG } from './constants'
 import { logger } from '../helpers/logger'
 import { getServerActor } from '../helpers/utils'
+import { VideosRedundancy } from '../../shared/models/redundancy'
+import { isArray } from '../helpers/custom-validators/misc'
+import { uniq } from 'lodash'
 
 async function checkActivityPubUrls () {
   const actor = await getServerActor()
@@ -35,6 +38,20 @@ function checkConfig () {
     return 'NSFW policy setting should be "do_not_list" or "blur" or "display" instead of ' + defaultNSFWPolicy
   }
 
+  const redundancyVideos = config.get<VideosRedundancy[]>('redundancy.videos')
+  if (isArray(redundancyVideos)) {
+    for (const r of redundancyVideos) {
+      if ([ 'most-views' ].indexOf(r.strategy) === -1) {
+        return 'Redundancy video entries should have "most-views" strategy instead of ' + r.strategy
+      }
+    }
+
+    const filtered = uniq(redundancyVideos.map(r => r.strategy))
+    if (filtered.length !== redundancyVideos.length) {
+      return 'Redundancy video entries should have uniq strategies'
+    }
+  }
+
   return null
 }
 

server/initializers/constants.ts

@@ -1,6 +1,6 @@
 import { IConfig } from 'config'
 import { dirname, join } from 'path'
-import { JobType, VideoRateType, VideoState } from '../../shared/models'
+import { JobType, VideoRateType, VideoRedundancyStrategy, VideoState, VideosRedundancy } from '../../shared/models'
 import { ActivityPubActorType } from '../../shared/models/activitypub'
 import { FollowState } from '../../shared/models/actors'
 import { VideoAbuseState, VideoImportState, VideoPrivacy } from '../../shared/models/videos'
@@ -9,13 +9,14 @@ import { buildPath, isTestInstance, root, sanitizeHost, sanitizeUrl } from '../h
 import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type'
 import { invert } from 'lodash'
 import { CronRepeatOptions, EveryRepeatOptions } from 'bull'
+import * as bytes from 'bytes'
 
 // Use a variable to reload the configuration if we need
 let config: IConfig = require('config')
 
 // ---------------------------------------------------------------------------
 
-const LAST_MIGRATION_VERSION = 265
+const LAST_MIGRATION_VERSION = 270
 
 // ---------------------------------------------------------------------------
 
@@ -137,7 +138,8 @@ let SCHEDULER_INTERVALS_MS = {
   badActorFollow: 60000 * 60, // 1 hour
   removeOldJobs: 60000 * 60, // 1 hour
   updateVideos: 60000, // 1 minute
-  youtubeDLUpdate: 60000 * 60 * 24 // 1 day
+  youtubeDLUpdate: 60000 * 60 * 24, // 1 day
+  videosRedundancy: 60000 * 2 // 2 hours
 }
 
 // ---------------------------------------------------------------------------
@@ -208,6 +210,9 @@ const CONFIG = {
       INTERVAL_DAYS: config.get<number>('trending.videos.interval_days')
     }
   },
+  REDUNDANCY: {
+    VIDEOS: buildVideosRedundancy(config.get<any[]>('redundancy.videos'))
+  },
   ADMIN: {
     get EMAIL () { return config.get<string>('admin.email') }
   },
@@ -321,6 +326,9 @@ const CONSTRAINTS_FIELDS = {
       }
     }
   },
+  VIDEOS_REDUNDANCY: {
+    URL: { min: 3, max: 2000 } // Length
+  },
   VIDEOS: {
     NAME: { min: 3, max: 120 }, // Length
     LANGUAGE: { min: 1, max: 10 }, // Length
@@ -584,6 +592,13 @@ const CACHE = {
   }
 }
 
+const REDUNDANCY = {
+  VIDEOS: {
+    EXPIRES_AFTER_MS: 48 * 3600 * 1000, // 2 days
+    RANDOMIZED_FACTOR: 5
+  }
+}
+
 const ACCEPT_HEADERS = [ 'html', 'application/json' ].concat(ACTIVITY_PUB.POTENTIAL_ACCEPT_HEADERS)
 
 // ---------------------------------------------------------------------------
@@ -629,8 +644,11 @@ if (isTestInstance() === true) {
   SCHEDULER_INTERVALS_MS.badActorFollow = 10000
   SCHEDULER_INTERVALS_MS.removeOldJobs = 10000
   SCHEDULER_INTERVALS_MS.updateVideos = 5000
+  SCHEDULER_INTERVALS_MS.videosRedundancy = 5000
   REPEAT_JOBS['videos-views'] = { every: 5000 }
 
+  REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR = 1
+
   VIDEO_VIEW_LIFETIME = 1000 // 1 second
 
   JOB_ATTEMPTS['email'] = 1
@@ -653,6 +671,7 @@ export {
   CONFIG,
   CONSTRAINTS_FIELDS,
   EMBED_SIZE,
+  REDUNDANCY,
   JOB_CONCURRENCY,
   JOB_ATTEMPTS,
   LAST_MIGRATION_VERSION,
@@ -722,6 +741,17 @@ function updateWebserverConfig () {
   CONFIG.WEBSERVER.HOST = sanitizeHost(CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT, REMOTE_SCHEME.HTTP)
 }
 
+function buildVideosRedundancy (objs: { strategy: VideoRedundancyStrategy, size: string }[]): VideosRedundancy[] {
+  if (!objs) return []
+
+  return objs.map(obj => {
+    return {
+      strategy: obj.strategy,
+      size: bytes.parse(obj.size)
+    }
+  })
+}
+
 function buildLanguages () {
   const iso639 = require('iso-639-3')
 

server/initializers/database.ts

@@ -27,6 +27,7 @@ import { VideoCaptionModel } from '../models/video/video-caption'
 import { VideoImportModel } from '../models/video/video-import'
 import { VideoViewModel } from '../models/video/video-views'
 import { VideoChangeOwnershipModel } from '../models/video/video-change-ownership'
+import { VideoRedundancyModel } from '../models/redundancy/video-redundancy'
 
 require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string
 
@@ -87,7 +88,8 @@ async function initDatabaseModels (silent: boolean) {
     VideoCommentModel,
     ScheduleVideoUpdateModel,
     VideoImportModel,
-    VideoViewModel
+    VideoViewModel,
+    VideoRedundancyModel
   ])
 
   // Check extensions exist in the database

server/initializers/migrations/0270-server-redundancy.ts

@@ -0,0 +1,24 @@
+import * as Sequelize from 'sequelize'
+
+async function up (utils: {
+  transaction: Sequelize.Transaction
+  queryInterface: Sequelize.QueryInterface
+  sequelize: Sequelize.Sequelize
+}): Promise<any> {
+  {
+    const data = {
+      type: Sequelize.BOOLEAN,
+      allowNull: false,
+      defaultValue: false
+    }
+
+    await utils.queryInterface.addColumn('server', 'redundancyAllowed', data)
+  }
+
+}
+
+function down (options) {
+  throw new Error('Not implemented.')
+}
+
+export { up, down }

server/lib/activitypub/actor.ts

@@ -400,17 +400,15 @@ async function refreshActorIfNeeded (actor: ActorModel): Promise<{ actor: ActorM
       await actor.save({ transaction: t })
 
       if (actor.Account) {
-        await actor.save({ transaction: t })
-
         actor.Account.set('name', result.name)
         actor.Account.set('description', result.summary)
+
         await actor.Account.save({ transaction: t })
       } else if (actor.VideoChannel) {
-        await actor.save({ transaction: t })
-
         actor.VideoChannel.set('name', result.name)
         actor.VideoChannel.set('description', result.summary)
         actor.VideoChannel.set('support', result.support)
+
         await actor.VideoChannel.save({ transaction: t })
       }
 

server/lib/activitypub/cache-file.ts

@@ -0,0 +1,47 @@
+import { CacheFileObject } from '../../../shared/index'
+import { VideoModel } from '../../models/video/video'
+import { ActorModel } from '../../models/activitypub/actor'
+import { sequelizeTypescript } from '../../initializers'
+import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
+
+function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: VideoModel, byActor: ActorModel) {
+  const url = cacheFileObject.url
+
+  const videoFile = video.VideoFiles.find(f => {
+    return f.resolution === url.height && f.fps === url.fps
+  })
+
+  if (!videoFile) throw new Error(`Cannot find video file ${url.height} ${url.fps} of video ${video.url}`)
+
+  return {
+    expiresOn: new Date(cacheFileObject.expires),
+    url: cacheFileObject.id,
+    fileUrl: cacheFileObject.url.href,
+    strategy: null,
+    videoFileId: videoFile.id,
+    actorId: byActor.id
+  }
+}
+
+function createCacheFile (cacheFileObject: CacheFileObject, video: VideoModel, byActor: ActorModel) {
+  return sequelizeTypescript.transaction(async t => {
+    const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor)
+
+    return VideoRedundancyModel.create(attributes, { transaction: t })
+  })
+}
+
+function updateCacheFile (cacheFileObject: CacheFileObject, redundancyModel: VideoRedundancyModel, byActor: ActorModel) {
+  const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, redundancyModel.VideoFile.Video, byActor)
+
+  redundancyModel.set('expires', attributes.expiresOn)
+  redundancyModel.set('fileUrl', attributes.fileUrl)
+
+  return redundancyModel.save()
+}
+
+export {
+  createCacheFile,
+  updateCacheFile,
+  cacheFileActivityObjectToDBAttributes
+}

server/lib/activitypub/process/process-create.ts

@@ -1,4 +1,4 @@
-import { ActivityCreate, VideoAbuseState, VideoTorrentObject } from '../../../../shared'
+import { ActivityCreate, CacheFileObject, VideoAbuseState, VideoTorrentObject } from '../../../../shared'
 import { DislikeObject, VideoAbuseObject, ViewObject } from '../../../../shared/models/activitypub/objects'
 import { VideoCommentObject } from '../../../../shared/models/activitypub/objects/video-comment-object'
 import { retryTransactionWrapper } from '../../../helpers/database-utils'
@@ -12,6 +12,7 @@ import { addVideoComment, resolveThread } from '../video-comments'
 import { getOrCreateVideoAndAccountAndChannel } from '../videos'
 import { forwardActivity, forwardVideoRelatedActivity } from '../send/utils'
 import { Redis } from '../../redis'
+import { createCacheFile } from '../cache-file'
 
 async function processCreateActivity (activity: ActivityCreate) {
   const activityObject = activity.object
@@ -28,6 +29,8 @@ async function processCreateActivity (activity: ActivityCreate) {
     return retryTransactionWrapper(processCreateVideoAbuse, actor, activityObject as VideoAbuseObject)
   } else if (activityType === 'Note') {
     return retryTransactionWrapper(processCreateVideoComment, actor, activity)
+  } else if (activityType === 'CacheFile') {
+    return retryTransactionWrapper(processCacheFile, actor, activity)
   }
 
   logger.warn('Unknown activity object type %s when creating activity.', activityType, { activity: activity.id })
@@ -97,6 +100,20 @@ async function processCreateView (byActor: ActorModel, activity: ActivityCreate)
   }
 }
 
+async function processCacheFile (byActor: ActorModel, activity: ActivityCreate) {
+  const cacheFile = activity.object as CacheFileObject
+
+  const { video } = await getOrCreateVideoAndAccountAndChannel(cacheFile.object)
+
+  await createCacheFile(cacheFile, video, byActor)
+
+  if (video.isOwned()) {
+    // Don't resend the activity to the sender
+    const exceptions = [ byActor ]
+    await forwardActivity(activity, undefined, exceptions)
+  }
+}
+
 async function processCreateVideoAbuse (actor: ActorModel, videoAbuseToCreateData: VideoAbuseObject) {
   logger.debug('Reporting remote abuse for video %s.', videoAbuseToCreateData.object)
 
@@ -113,7 +130,7 @@ async function processCreateVideoAbuse (actor: ActorModel, videoAbuseToCreateDat
       state: VideoAbuseState.PENDING
     }
 
-    await VideoAbuseModel.create(videoAbuseData)
+    await VideoAbuseModel.create(videoAbuseData, { transaction: t })
 
     logger.info('Remote abuse for video uuid %s created', videoAbuseToCreateData.object)
   })

server/lib/activitypub/process/process-undo.ts

@@ -1,4 +1,4 @@
-import { ActivityAnnounce, ActivityFollow, ActivityLike, ActivityUndo } from '../../../../shared/models/activitypub'
+import { ActivityAnnounce, ActivityFollow, ActivityLike, ActivityUndo, CacheFileObject } from '../../../../shared/models/activitypub'
 import { DislikeObject } from '../../../../shared/models/activitypub/objects'
 import { getActorUrl } from '../../../helpers/activitypub'
 import { retryTransactionWrapper } from '../../../helpers/database-utils'
@@ -11,6 +11,7 @@ import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
 import { forwardVideoRelatedActivity } from '../send/utils'
 import { getOrCreateVideoAndAccountAndChannel } from '../videos'
 import { VideoShareModel } from '../../../models/video/video-share'
+import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
 
 async function processUndoActivity (activity: ActivityUndo) {
   const activityToUndo = activity.object
@@ -19,11 +20,21 @@ async function processUndoActivity (activity: ActivityUndo) {
 
   if (activityToUndo.type === 'Like') {
     return retryTransactionWrapper(processUndoLike, actorUrl, activity)
-  } else if (activityToUndo.type === 'Create' && activityToUndo.object.type === 'Dislike') {
-    return retryTransactionWrapper(processUndoDislike, actorUrl, activity)
-  } else if (activityToUndo.type === 'Follow') {
+  }
+
+  if (activityToUndo.type === 'Create') {
+    if (activityToUndo.object.type === 'Dislike') {
+      return retryTransactionWrapper(processUndoDislike, actorUrl, activity)
+    } else if (activityToUndo.object.type === 'CacheFile') {
+      return retryTransactionWrapper(processUndoCacheFile, actorUrl, activity)
+    }
+  }
+
+  if (activityToUndo.type === 'Follow') {
     return retryTransactionWrapper(processUndoFollow, actorUrl, activityToUndo)
-  } else if (activityToUndo.type === 'Announce') {
+  }
+
+  if (activityToUndo.type === 'Announce') {
     return retryTransactionWrapper(processUndoAnnounce, actorUrl, activityToUndo)
   }
 
@@ -88,6 +99,29 @@ async function processUndoDislike (actorUrl: string, activity: ActivityUndo) {
   })
 }
 
+async function processUndoCacheFile (actorUrl: string, activity: ActivityUndo) {
+  const cacheFileObject = activity.object.object as CacheFileObject
+
+  const { video } = await getOrCreateVideoAndAccountAndChannel(cacheFileObject.object)
+
+  return sequelizeTypescript.transaction(async t => {
+    const byActor = await ActorModel.loadByUrl(actorUrl)
+    if (!byActor) throw new Error('Unknown actor ' + actorUrl)
+
+    const cacheFile = await VideoRedundancyModel.loadByUrl(cacheFileObject.id)
+    if (!cacheFile) throw new Error('Unknown video cache ' + cacheFile.url)
+
+    await cacheFile.destroy()
+
+    if (video.isOwned()) {
+      // Don't resend the activity to the sender
+      const exceptions = [ byActor ]
+
+      await forwardVideoRelatedActivity(activity, t, exceptions, video)
+    }
+  })
+}
+
 function processUndoFollow (actorUrl: string, followActivity: ActivityFollow) {
   return sequelizeTypescript.transaction(async t => {
     const follower = await ActorModel.loadByUrl(actorUrl, t)

server/lib/activitypub/process/process-update.ts

@@ -1,4 +1,4 @@
-import { ActivityUpdate, VideoTorrentObject } from '../../../../shared/models/activitypub'
+import { ActivityUpdate, CacheFileObject, VideoTorrentObject } from '../../../../shared/models/activitypub'
 import { ActivityPubActor } from '../../../../shared/models/activitypub/activitypub-actor'
 import { resetSequelizeInstance, retryTransactionWrapper } from '../../../helpers/database-utils'
 import { logger } from '../../../helpers/logger'
@@ -7,8 +7,11 @@ import { AccountModel } from '../../../models/account/account'
 import { ActorModel } from '../../../models/activitypub/actor'
 import { VideoChannelModel } from '../../../models/video/video-channel'
 import { fetchAvatarIfExists, getOrCreateActorAndServerAndModel, updateActorAvatarInstance, updateActorInstance } from '../actor'
-import { getOrCreateVideoAndAccountAndChannel, getOrCreateVideoChannelFromVideoObject, updateVideoFromAP } from '../videos'
+import { getOrCreateVideoAndAccountAndChannel, updateVideoFromAP, getOrCreateVideoChannelFromVideoObject } from '../videos'
 import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-validators/activitypub/videos'
+import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file'
+import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
+import { createCacheFile, updateCacheFile } from '../cache-file'
 
 async function processUpdateActivity (activity: ActivityUpdate) {
   const actor = await getOrCreateActorAndServerAndModel(activity.actor)
@@ -16,10 +19,16 @@ async function processUpdateActivity (activity: ActivityUpdate) {
 
   if (objectType === 'Video') {
     return retryTransactionWrapper(processUpdateVideo, actor, activity)
-  } else if (objectType === 'Person' || objectType === 'Application' || objectType === 'Group') {
+  }
+
+  if (objectType === 'Person' || objectType === 'Application' || objectType === 'Group') {
     return retryTransactionWrapper(processUpdateActor, actor, activity)
   }
 
+  if (objectType === 'CacheFile') {
+    return retryTransactionWrapper(processUpdateCacheFile, actor, activity)
+  }
+
   return undefined
 }
 
@@ -42,7 +51,24 @@ async function processUpdateVideo (actor: ActorModel, activity: ActivityUpdate)
   const { video } = await getOrCreateVideoAndAccountAndChannel(videoObject.id)
   const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject)
 
-  return updateVideoFromAP(video, videoObject, actor, channelActor, activity.to)
+  return updateVideoFromAP(video, videoObject, actor.Account, channelActor.VideoChannel, activity.to)
+}
+
+async function processUpdateCacheFile (byActor: ActorModel, activity: ActivityUpdate) {
+  const cacheFileObject = activity.object as CacheFileObject
+
+  if (!isCacheFileObjectValid(cacheFileObject) === false) {
+    logger.debug('Cahe file object sent by update is not valid.', { cacheFileObject })
+    return undefined
+  }
+
+  const redundancyModel = await VideoRedundancyModel.loadByUrl(cacheFileObject.id)
+  if (!redundancyModel) {
+    const { video } = await getOrCreateVideoAndAccountAndChannel(cacheFileObject.id)
+    return createCacheFile(cacheFileObject, video, byActor)
+  }
+
+  return updateCacheFile(cacheFileObject, redundancyModel, byActor)
 }
 
 async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate) {

server/lib/activitypub/send/send-accept.ts

@@ -3,7 +3,7 @@ import { ActorModel } from '../../../models/activitypub/actor'
 import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
 import { getActorFollowAcceptActivityPubUrl, getActorFollowActivityPubUrl } from '../url'
 import { unicastTo } from './utils'
-import { followActivityData } from './send-follow'
+import { buildFollowActivity } from './send-follow'
 import { logger } from '../../../helpers/logger'
 
 async function sendAccept (actorFollow: ActorFollowModel) {
@@ -18,10 +18,10 @@ async function sendAccept (actorFollow: ActorFollowModel) {
   logger.info('Creating job to accept follower %s.', follower.url)
 
   const followUrl = getActorFollowActivityPubUrl(actorFollow)
-  const followData = followActivityData(followUrl, follower, me)
+  const followData = buildFollowActivity(followUrl, follower, me)
 
   const url = getActorFollowAcceptActivityPubUrl(actorFollow)
-  const data = acceptActivityData(url, me, followData)
+  const data = buildAcceptActivity(url, me, followData)
 
   return unicastTo(data, me, follower.inboxUrl)
 }
@@ -34,7 +34,7 @@ export {
 
 // ---------------------------------------------------------------------------
 
-function acceptActivityData (url: string, byActor: ActorModel, followActivityData: ActivityFollow): ActivityAccept {
+function buildAcceptActivity (url: string, byActor: ActorModel, followActivityData: ActivityFollow): ActivityAccept {
   return {
     type: 'Accept',
     id: url,

server/lib/activitypub/send/send-announce.ts

@@ -4,45 +4,44 @@ import { ActorModel } from '../../../models/activitypub/actor'
 import { VideoModel } from '../../../models/video/video'
 import { VideoShareModel } from '../../../models/video/video-share'
 import { broadcastToFollowers } from './utils'
-import { getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience } from '../audience'
+import { audiencify, getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience } from '../audience'
 import { logger } from '../../../helpers/logger'
 
-async function buildVideoAnnounce (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) {
+async function buildAnnounceWithVideoAudience (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) {
   const announcedObject = video.url
 
-  const accountsToForwardView = await getActorsInvolvedInVideo(video, t)
-  const audience = getObjectFollowersAudience(accountsToForwardView)
-  return announceActivityData(videoShare.url, byActor, announcedObject, audience)
+  const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t)
+  const audience = getObjectFollowersAudience(actorsInvolvedInVideo)
+
+  const activity = buildAnnounceActivity(videoShare.url, byActor, announcedObject, audience)
+
+  return { activity, actorsInvolvedInVideo }
 }
 
 async function sendVideoAnnounce (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) {
-  const data = await buildVideoAnnounce(byActor, videoShare, video, t)
+  const { activity, actorsInvolvedInVideo } = await buildAnnounceWithVideoAudience(byActor, videoShare, video, t)
 
   logger.info('Creating job to send announce %s.', videoShare.url)
 
-  const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t)
   const followersException = [ byActor ]
-
-  return broadcastToFollowers(data, byActor, actorsInvolvedInVideo, t, followersException)
+  return broadcastToFollowers(activity, byActor, actorsInvolvedInVideo, t, followersException)
 }
 
-function announceActivityData (url: string, byActor: ActorModel, object: string, audience?: ActivityAudience): ActivityAnnounce {
+function buildAnnounceActivity (url: string, byActor: ActorModel, object: string, audience?: ActivityAudience): ActivityAnnounce {
   if (!audience) audience = getAudience(byActor)
 
-  return {
-    type: 'Announce',
-    to: audience.to,
-    cc: audience.cc,
+  return audiencify({
+    type: 'Announce' as 'Announce',
     id: url,
     actor: byActor.url,
     object
-  }
+  }, audience)
 }
 
 // ---------------------------------------------------------------------------
 
 export {
   sendVideoAnnounce,
-  announceActivityData,
-  buildVideoAnnounce
+  buildAnnounceActivity,
+  buildAnnounceWithVideoAudience
 }

server/lib/activitypub/send/send-create.ts

@@ -17,6 +17,7 @@ import {
   getVideoCommentAudience
 } from '../audience'
 import { logger } from '../../../helpers/logger'
+import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
 
 async function sendCreateVideo (video: VideoModel, t: Transaction) {
   if (video.privacy === VideoPrivacy.PRIVATE) return undefined
@@ -27,12 +28,12 @@ async function sendCreateVideo (video: VideoModel, t: Transaction) {
   const videoObject = video.toActivityPubObject()
 
   const audience = getAudience(byActor, video.privacy === VideoPrivacy.PUBLIC)
-  const data = createActivityData(video.url, byActor, videoObject, audience)
+  const createActivity = buildCreateActivity(video.url, byActor, videoObject, audience)
 
-  return broadcastToFollowers(data, byActor, [ byActor ], t)
+  return broadcastToFollowers(createActivity, byActor, [ byActor ], t)
 }
 
-async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, video: VideoModel, t: Transaction) {
+async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, video: VideoModel) {
   if (!video.VideoChannel.Account.Actor.serverId) return // Local
 
   const url = getVideoAbuseActivityPubUrl(videoAbuse)
@@ -40,9 +41,23 @@ async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel,
   logger.info('Creating job to send video abuse %s.', url)
 
   const audience = { to: [ video.VideoChannel.Account.Actor.url ], cc: [] }
-  const data = createActivityData(url, byActor, videoAbuse.toActivityPubObject(), audience)
+  const createActivity = buildCreateActivity(url, byActor, videoAbuse.toActivityPubObject(), audience)
 
-  return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
+  return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
+}
+
+async function sendCreateCacheFile (byActor: ActorModel, fileRedundancy: VideoRedundancyModel) {
+  logger.info('Creating job to send file cache of %s.', fileRedundancy.url)
+
+  const redundancyObject = fileRedundancy.toActivityPubObject()
+
+  const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(fileRedundancy.VideoFile.Video.id)
+  const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, undefined)
+
+  const audience = getVideoAudience(video, actorsInvolvedInVideo)
+  const createActivity = buildCreateActivity(fileRedundancy.url, byActor, redundancyObject, audience)
+
+  return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
 }
 
 async function sendCreateVideoComment (comment: VideoCommentModel, t: Transaction) {
@@ -66,73 +81,73 @@ async function sendCreateVideoComment (comment: VideoCommentModel, t: Transactio
     audience = getObjectFollowersAudience(actorsInvolvedInComment.concat(parentsCommentActors))
   }
 
-  const data = createActivityData(comment.url, byActor, commentObject, audience)
+  const createActivity = buildCreateActivity(comment.url, byActor, commentObject, audience)
 
   // This was a reply, send it to the parent actors
   const actorsException = [ byActor ]
-  await broadcastToActors(data, byActor, parentsCommentActors, actorsException)
+  await broadcastToActors(createActivity, byActor, parentsCommentActors, actorsException)
 
   // Broadcast to our followers
-  await broadcastToFollowers(data, byActor, [ byActor ], t)
+  await broadcastToFollowers(createActivity, byActor, [ byActor ], t)
 
   // Send to actors involved in the comment
-  if (isOrigin) return broadcastToFollowers(data, byActor, actorsInvolvedInComment, t, actorsException)
+  if (isOrigin) return broadcastToFollowers(createActivity, byActor, actorsInvolvedInComment, t, actorsException)
 
   // Send to origin
-  return unicastTo(data, byActor, comment.Video.VideoChannel.Account.Actor.sharedInboxUrl)
+  return unicastTo(createActivity, byActor, comment.Video.VideoChannel.Account.Actor.sharedInboxUrl)
 }
 
 async function sendCreateView (byActor: ActorModel, video: VideoModel, t: Transaction) {
   logger.info('Creating job to send view of %s.', video.url)
 
   const url = getVideoViewActivityPubUrl(byActor, video)
-  const viewActivityData = createViewActivityData(byActor, video)
+  const viewActivity = buildViewActivity(byActor, video)
 
   const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t)
 
   // Send to origin
   if (video.isOwned() === false) {
     const audience = getVideoAudience(video, actorsInvolvedInVideo)
-    const data = createActivityData(url, byActor, viewActivityData, audience)
+    const createActivity = buildCreateActivity(url, byActor, viewActivity, audience)
 
-    return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
+    return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
   }
 
   // Send to followers
   const audience = getObjectFollowersAudience(actorsInvolvedInVideo)
-  const data = createActivityData(url, byActor, viewActivityData, audience)
+  const createActivity = buildCreateActivity(url, byActor, viewActivity, audience)
 
   // Use the server actor to send the view
   const serverActor = await getServerActor()
   const actorsException = [ byActor ]
-  return broadcastToFollowers(data, serverActor, actorsInvolvedInVideo, t, actorsException)
+  return broadcastToFollowers(createActivity, serverActor, actorsInvolvedInVideo, t, actorsException)
 }
 
 async function sendCreateDislike (byActor: ActorModel, video: VideoModel, t: Transaction) {
   logger.info('Creating job to dislike %s.', video.url)
 
   const url = getVideoDislikeActivityPubUrl(byActor, video)
-  const dislikeActivityData = createDislikeActivityData(byActor, video)
+  const dislikeActivity = buildDislikeActivity(byActor, video)
 
   const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t)
 
   // Send to origin
   if (video.isOwned() === false) {
     const audience = getVideoAudience(video, actorsInvolvedInVideo)
-    const data = createActivityData(url, byActor, dislikeActivityData, audience)
+    const createActivity = buildCreateActivity(url, byActor, dislikeActivity, audience)
 
-    return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
+    return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
   }
 
   // Send to followers
   const audience = getObjectFollowersAudience(actorsInvolvedInVideo)
-  const data = createActivityData(url, byActor, dislikeActivityData, audience)
+  const createActivity = buildCreateActivity(url, byActor, dislikeActivity, audience)
 
   const actorsException = [ byActor ]
-  return broadcastToFollowers(data, byActor, actorsInvolvedInVideo, t, actorsException)
+  return broadcastToFollowers(createActivity, byActor, actorsInvolvedInVideo, t, actorsException)
 }
 
-function createActivityData (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityCreate {
+function buildCreateActivity (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityCreate {
   if (!audience) audience = getAudience(byActor)
 
   return audiencify(
@@ -146,7 +161,7 @@ function createActivityData (url: string, byActor: ActorModel, object: any, audi
   )
 }
 
-function createDislikeActivityData (byActor: ActorModel, video: VideoModel) {
+function buildDislikeActivity (byActor: ActorModel, video: VideoModel) {
   return {
     type: 'Dislike',
     actor: byActor.url,
@@ -154,7 +169,7 @@ function createDislikeActivityData (byActor: ActorModel, video: VideoModel) {
   }
 }
 
-function createViewActivityData (byActor: ActorModel, video: VideoModel) {
+function buildViewActivity (byActor: ActorModel, video: VideoModel) {
   return {
     type: 'View',
     actor: byActor.url,
@@ -167,9 +182,10 @@ function createViewActivityData (byActor: ActorModel, video: VideoModel) {
 export {
   sendCreateVideo,
   sendVideoAbuse,
-  createActivityData,
+  buildCreateActivity,
   sendCreateView,
   sendCreateDislike,
-  createDislikeActivityData,
-  sendCreateVideoComment
+  buildDislikeActivity,
+  sendCreateVideoComment,
+  sendCreateCacheFile
 }

server/lib/activitypub/send/send-delete.ts

@@ -15,24 +15,23 @@ async function sendDeleteVideo (video: VideoModel, t: Transaction) {
   const url = getDeleteActivityPubUrl(video.url)
   const byActor = video.VideoChannel.Account.Actor
 
-  const data = deleteActivityData(url, video.url, byActor)
+  const activity = buildDeleteActivity(url, video.url, byActor)
 
-  const actorsInvolved = await VideoShareModel.loadActorsByShare(video.id, t)
-  actorsInvolved.push(byActor)
+  const actorsInvolved = await getActorsInvolvedInVideo(video, t)
 
-  return broadcastToFollowers(data, byActor, actorsInvolved, t)
+  return broadcastToFollowers(activity, byActor, actorsInvolved, t)
 }
 
 async function sendDeleteActor (byActor: ActorModel, t: Transaction) {
   logger.info('Creating job to broadcast delete of actor %s.', byActor.url)
 
   const url = getDeleteActivityPubUrl(byActor.url)
-  const data = deleteActivityData(url, byActor.url, byActor)
+  const activity = buildDeleteActivity(url, byActor.url, byActor)
 
   const actorsInvolved = await VideoShareModel.loadActorsByVideoOwner(byActor.id, t)
   actorsInvolved.push(byActor)
 
-  return broadcastToFollowers(data, byActor, actorsInvolved, t)
+  return broadcastToFollowers(activity, byActor, actorsInvolved, t)
 }
 
 async function sendDeleteVideoComment (videoComment: VideoCommentModel, t: Transaction) {
@@ -45,23 +44,23 @@ async function sendDeleteVideoComment (videoComment: VideoCommentModel, t: Trans
   const threadParentComments = await VideoCommentModel.listThreadParentComments(videoComment, t)
 
   const actorsInvolvedInComment = await getActorsInvolvedInVideo(videoComment.Video, t)
-  actorsInvolvedInComment.push(byActor)
+  actorsInvolvedInComment.push(byActor) // Add the actor that commented the video
 
   const audience = getVideoCommentAudience(videoComment, threadParentComments, actorsInvolvedInComment, isVideoOrigin)
-  const data = deleteActivityData(url, videoComment.url, byActor, audience)
+  const activity = buildDeleteActivity(url, videoComment.url, byActor, audience)
 
   // This was a reply, send it to the parent actors
   const actorsException = [ byActor ]
-  await broadcastToActors(data, byActor, threadParentComments.map(c => c.Account.Actor), actorsException)
+  await broadcastToActors(activity, byActor, threadParentComments.map(c => c.Account.Actor), actorsException)
 
   // Broadcast to our followers
-  await broadcastToFollowers(data, byActor, [ byActor ], t)
+  await broadcastToFollowers(activity, byActor, [ byActor ], t)
 
   // Send to actors involved in the comment
-  if (isVideoOrigin) return broadcastToFollowers(data, byActor, actorsInvolvedInComment, t, actorsException)
+  if (isVideoOrigin) return broadcastToFollowers(activity, byActor, actorsInvolvedInComment, t, actorsException)
 
   // Send to origin
-  return unicastTo(data, byActor, videoComment.Video.VideoChannel.Account.Actor.sharedInboxUrl)
+  return unicastTo(activity, byActor, videoComment.Video.VideoChannel.Account.Actor.sharedInboxUrl)
 }
 
 // ---------------------------------------------------------------------------
@@ -74,7 +73,7 @@ export {
 
 // ---------------------------------------------------------------------------
 
-function deleteActivityData (url: string, object: string, byActor: ActorModel, audience?: ActivityAudience): ActivityDelete {
+function buildDeleteActivity (url: string, object: string, byActor: ActorModel, audience?: ActivityAudience): ActivityDelete {
   const activity = {
     type: 'Delete' as 'Delete',
     id: url,

server/lib/activitypub/send/send-follow.ts

@@ -15,12 +15,12 @@ function sendFollow (actorFollow: ActorFollowModel) {
   logger.info('Creating job to send follow request to %s.', following.url)
 
   const url = getActorFollowActivityPubUrl(actorFollow)
-  const data = followActivityData(url, me, following)
+  const data = buildFollowActivity(url, me, following)
 
   return unicastTo(data, me, following.inboxUrl)
 }
 
-function followActivityData (url: string, byActor: ActorModel, targetActor: ActorModel): ActivityFollow {
+function buildFollowActivity (url: string, byActor: ActorModel, targetActor: ActorModel): ActivityFollow {
   return {
     type: 'Follow',
     id: url,
@@ -33,5 +33,5 @@ function followActivityData (url: string, byActor: ActorModel, targetActor: Acto
 
 export {
   sendFollow,
-  followActivityData
+  buildFollowActivity
 }

server/lib/activitypub/send/send-like.ts

@@ -17,20 +17,20 @@ async function sendLike (byActor: ActorModel, video: VideoModel, t: Transaction)
   // Send to origin
   if (video.isOwned() === false) {
     const audience = getVideoAudience(video, accountsInvolvedInVideo)
-    const data = likeActivityData(url, byActor, video, audience)
+    const data = buildLikeActivity(url, byActor, video, audience)
 
     return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
   }
 
   // Send to followers
   const audience = getObjectFollowersAudience(accountsInvolvedInVideo)
-  const data = likeActivityData(url, byActor, video, audience)
+  const activity = buildLikeActivity(url, byActor, video, audience)
 
   const followersException = [ byActor ]
-  return broadcastToFollowers(data, byActor, accountsInvolvedInVideo, t, followersException)
+  return broadcastToFollowers(activity, byActor, accountsInvolvedInVideo, t, followersException)
 }
 
-function likeActivityData (url: string, byActor: ActorModel, video: VideoModel, audience?: ActivityAudience): ActivityLike {
+function buildLikeActivity (url: string, byActor: ActorModel, video: VideoModel, audience?: ActivityAudience): ActivityLike {
   if (!audience) audience = getAudience(byActor)
 
   return audiencify(
@@ -48,5 +48,5 @@ function likeActivityData (url: string, byActor: ActorModel, video: VideoModel,
 
 export {
   sendLike,
-  likeActivityData
+  buildLikeActivity
 }

server/lib/activitypub/send/send-undo.ts

@@ -13,12 +13,13 @@ import { VideoModel } from '../../../models/video/video'
 import { getActorFollowActivityPubUrl, getUndoActivityPubUrl, getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from '../url'
 import { broadcastToFollowers, unicastTo } from './utils'
 import { audiencify, getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience, getVideoAudience } from '../audience'
-import { createActivityData, createDislikeActivityData } from './send-create'
-import { followActivityData } from './send-follow'
-import { likeActivityData } from './send-like'
+import { buildCreateActivity, buildDislikeActivity } from './send-create'
+import { buildFollowActivity } from './send-follow'
+import { buildLikeActivity } from './send-like'
 import { VideoShareModel } from '../../../models/video/video-share'
-import { buildVideoAnnounce } from './send-announce'
+import { buildAnnounceWithVideoAudience } from './send-announce'
 import { logger } from '../../../helpers/logger'
+import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
 
 async function sendUndoFollow (actorFollow: ActorFollowModel, t: Transaction) {
   const me = actorFollow.ActorFollower
@@ -32,10 +33,10 @@ async function sendUndoFollow (actorFollow: ActorFollowModel, t: Transaction) {
   const followUrl = getActorFollowActivityPubUrl(actorFollow)
   const undoUrl = getUndoActivityPubUrl(followUrl)
 
-  const object = followActivityData(followUrl, me, following)
-  const data = undoActivityData(undoUrl, me, object)
+  const followActivity = buildFollowActivity(followUrl, me, following)
+  const undoActivity = undoActivityData(undoUrl, me, followActivity)
 
-  return unicastTo(data, me, following.inboxUrl)
+  return unicastTo(undoActivity, me, following.inboxUrl)
 }
 
 async function sendUndoLike (byActor: ActorModel, video: VideoModel, t: Transaction) {
@@ -45,21 +46,21 @@ async function sendUndoLike (byActor: ActorModel, video: VideoModel, t: Transact
   const undoUrl = getUndoActivityPubUrl(likeUrl)
 
   const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t)
-  const object = likeActivityData(likeUrl, byActor, video)
+  const likeActivity = buildLikeActivity(likeUrl, byActor, video)
 
   // Send to origin
   if (video.isOwned() === false) {
     const audience = getVideoAudience(video, actorsInvolvedInVideo)
-    const data = undoActivityData(undoUrl, byActor, object, audience)
+    const undoActivity = undoActivityData(undoUrl, byActor, likeActivity, audience)
 
-    return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
+    return unicastTo(undoActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
   }
 
   const audience = getObjectFollowersAudience(actorsInvolvedInVideo)
-  const data = undoActivityData(undoUrl, byActor, object, audience)
+  const undoActivity = undoActivityData(undoUrl, byActor, likeActivity, audience)
 
   const followersException = [ byActor ]
-  return broadcastToFollowers(data, byActor, actorsInvolvedInVideo, t, followersException)
+  return broadcastToFollowers(undoActivity, byActor, actorsInvolvedInVideo, t, followersException)
 }
 
 async function sendUndoDislike (byActor: ActorModel, video: VideoModel, t: Transaction) {
@@ -69,20 +70,20 @@ async function sendUndoDislike (byActor: ActorModel, video: VideoModel, t: Trans
   const undoUrl = getUndoActivityPubUrl(dislikeUrl)
 
   const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t)
-  const dislikeActivity = createDislikeActivityData(byActor, video)
-  const object = createActivityData(dislikeUrl, byActor, dislikeActivity)
+  const dislikeActivity = buildDislikeActivity(byActor, video)
+  const createDislikeActivity = buildCreateActivity(dislikeUrl, byActor, dislikeActivity)
 
   if (video.isOwned() === false) {
     const audience = getVideoAudience(video, actorsInvolvedInVideo)
-    const data = undoActivityData(undoUrl, byActor, object, audience)
+    const undoActivity = undoActivityData(undoUrl, byActor, createDislikeActivity, audience)
 
-    return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
+    return unicastTo(undoActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
   }
 
-  const data = undoActivityData(undoUrl, byActor, object)
+  const undoActivity = undoActivityData(undoUrl, byActor, createDislikeActivity)
 
   const followersException = [ byActor ]
-  return broadcastToFollowers(data, byActor, actorsInvolvedInVideo, t, followersException)
+  return broadcastToFollowers(undoActivity, byActor, actorsInvolvedInVideo, t, followersException)
 }
 
 async function sendUndoAnnounce (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) {
@@ -90,12 +91,27 @@ async function sendUndoAnnounce (byActor: ActorModel, videoShare: VideoShareMode
 
   const undoUrl = getUndoActivityPubUrl(videoShare.url)
 
-  const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t)
-  const object = await buildVideoAnnounce(byActor, videoShare, video, t)
-  const data = undoActivityData(undoUrl, byActor, object)
+  const { activity: announceActivity, actorsInvolvedInVideo } = await buildAnnounceWithVideoAudience(byActor, videoShare, video, t)
+  const undoActivity = undoActivityData(undoUrl, byActor, announceActivity)
 
   const followersException = [ byActor ]
-  return broadcastToFollowers(data, byActor, actorsInvolvedInVideo, t, followersException)
+  return broadcastToFollowers(undoActivity, byActor, actorsInvolvedInVideo, t, followersException)
+}
+
+async function sendUndoCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel, t: Transaction) {
+  logger.info('Creating job to undo cache file %s.', redundancyModel.url)
+
+  const undoUrl = getUndoActivityPubUrl(redundancyModel.url)
+
+  const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id)
+  const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t)
+
+  const audience = getVideoAudience(video, actorsInvolvedInVideo)
+  const createActivity = buildCreateActivity(redundancyModel.url, byActor, redundancyModel.toActivityPubObject())
+
+  const undoActivity = undoActivityData(undoUrl, byActor, createActivity, audience)
+
+  return unicastTo(undoActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
 }
 
 // ---------------------------------------------------------------------------
@@ -104,7 +120,8 @@ export {
   sendUndoFollow,
   sendUndoLike,
   sendUndoDislike,
-  sendUndoAnnounce
+  sendUndoAnnounce,
+  sendUndoCacheFile
 }
 
 // ---------------------------------------------------------------------------

server/lib/activitypub/send/send-update.ts

@@ -7,11 +7,11 @@ import { VideoModel } from '../../../models/video/video'
 import { VideoChannelModel } from '../../../models/video/video-channel'
 import { VideoShareModel } from '../../../models/video/video-share'
 import { getUpdateActivityPubUrl } from '../url'
-import { broadcastToFollowers } from './utils'
-import { audiencify, getAudience } from '../audience'
+import { broadcastToFollowers, unicastTo } from './utils'
+import { audiencify, getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience } from '../audience'
 import { logger } from '../../../helpers/logger'
-import { videoFeedsValidator } from '../../../middlewares/validators'
 import { VideoCaptionModel } from '../../../models/video/video-caption'
+import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
 
 async function sendUpdateVideo (video: VideoModel, t: Transaction, overrodeByActor?: ActorModel) {
   logger.info('Creating job to update video %s.', video.url)
@@ -26,12 +26,12 @@ async function sendUpdateVideo (video: VideoModel, t: Transaction, overrodeByAct
   const videoObject = video.toActivityPubObject()
   const audience = getAudience(byActor, video.privacy === VideoPrivacy.PUBLIC)
 
-  const data = updateActivityData(url, byActor, videoObject, audience)
+  const updateActivity = buildUpdateActivity(url, byActor, videoObject, audience)
 
-  const actorsInvolved = await VideoShareModel.loadActorsByShare(video.id, t)
-  actorsInvolved.push(byActor)
+  const actorsInvolved = await getActorsInvolvedInVideo(video, t)
+  if (overrodeByActor) actorsInvolved.push(overrodeByActor)
 
-  return broadcastToFollowers(data, byActor, actorsInvolved, t)
+  return broadcastToFollowers(updateActivity, byActor, actorsInvolved, t)
 }
 
 async function sendUpdateActor (accountOrChannel: AccountModel | VideoChannelModel, t: Transaction) {
@@ -42,7 +42,7 @@ async function sendUpdateActor (accountOrChannel: AccountModel | VideoChannelMod
   const url = getUpdateActivityPubUrl(byActor.url, byActor.updatedAt.toISOString())
   const accountOrChannelObject = accountOrChannel.toActivityPubObject()
   const audience = getAudience(byActor)
-  const data = updateActivityData(url, byActor, accountOrChannelObject, audience)
+  const updateActivity = buildUpdateActivity(url, byActor, accountOrChannelObject, audience)
 
   let actorsInvolved: ActorModel[]
   if (accountOrChannel instanceof AccountModel) {
@@ -55,19 +55,35 @@ async function sendUpdateActor (accountOrChannel: AccountModel | VideoChannelMod
 
   actorsInvolved.push(byActor)
 
-  return broadcastToFollowers(data, byActor, actorsInvolved, t)
+  return broadcastToFollowers(updateActivity, byActor, actorsInvolved, t)
+}
+
+async function sendUpdateCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel) {
+  logger.info('Creating job to update cache file %s.', redundancyModel.url)
+
+  const url = getUpdateActivityPubUrl(redundancyModel.url, redundancyModel.updatedAt.toISOString())
+  const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id)
+
+  const redundancyObject = redundancyModel.toActivityPubObject()
+
+  const accountsInvolvedInVideo = await getActorsInvolvedInVideo(video, undefined)
+  const audience = getObjectFollowersAudience(accountsInvolvedInVideo)
+
+  const updateActivity = buildUpdateActivity(url, byActor, redundancyObject, audience)
+  return unicastTo(updateActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
 }
 
 // ---------------------------------------------------------------------------
 
 export {
   sendUpdateActor,
-  sendUpdateVideo
+  sendUpdateVideo,
+  sendUpdateCacheFile
 }
 
 // ---------------------------------------------------------------------------
 
-function updateActivityData (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityUpdate {
+function buildUpdateActivity (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityUpdate {
   if (!audience) audience = getAudience(byActor)
 
   return audiencify(

server/lib/activitypub/send/utils.ts

@@ -59,11 +59,11 @@ async function forwardActivity (
 async function broadcastToFollowers (
   data: any,
   byActor: ActorModel,
-  toActorFollowers: ActorModel[],
+  toFollowersOf: ActorModel[],
   t: Transaction,
   actorsException: ActorModel[] = []
 ) {
-  const uris = await computeFollowerUris(toActorFollowers, actorsException, t)
+  const uris = await computeFollowerUris(toFollowersOf, actorsException, t)
   return broadcastTo(uris, data, byActor)
 }
 
@@ -115,8 +115,8 @@ export {
 
 // ---------------------------------------------------------------------------
 
-async function computeFollowerUris (toActorFollower: ActorModel[], actorsException: ActorModel[], t: Transaction) {
-  const toActorFollowerIds = toActorFollower.map(a => a.id)
+async function computeFollowerUris (toFollowersOf: ActorModel[], actorsException: ActorModel[], t: Transaction) {
+  const toActorFollowerIds = toFollowersOf.map(a => a.id)
 
   const result = await ActorFollowModel.listAcceptedFollowerSharedInboxUrls(toActorFollowerIds, t)
   const sharedInboxesException = await buildSharedInboxesException(actorsException)

server/lib/activitypub/url.ts

@@ -4,11 +4,18 @@ import { ActorFollowModel } from '../../models/activitypub/actor-follow'
 import { VideoModel } from '../../models/video/video'
 import { VideoAbuseModel } from '../../models/video/video-abuse'
 import { VideoCommentModel } from '../../models/video/video-comment'
+import { VideoFileModel } from '../../models/video/video-file'
 
 function getVideoActivityPubUrl (video: VideoModel) {
   return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid
 }
 
+function getVideoCacheFileActivityPubUrl (videoFile: VideoFileModel) {
+  const suffixFPS = videoFile.fps ? '-' + videoFile.fps : ''
+
+  return `${CONFIG.WEBSERVER.URL}/redundancy/videos/${videoFile.Video.uuid}/${videoFile.resolution}${suffixFPS}`
+}
+
 function getVideoCommentActivityPubUrl (video: VideoModel, videoComment: VideoCommentModel) {
   return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid + '/comments/' + videoComment.id
 }
@@ -101,5 +108,6 @@ export {
   getVideoSharesActivityPubUrl,
   getVideoCommentsActivityPubUrl,
   getVideoLikesActivityPubUrl,
-  getVideoDislikesActivityPubUrl
+  getVideoDislikesActivityPubUrl,
+  getVideoCacheFileActivityPubUrl
 }

server/lib/activitypub/videos.ts

@@ -3,12 +3,12 @@ import * as sequelize from 'sequelize'
 import * as magnetUtil from 'magnet-uri'
 import { join } from 'path'
 import * as request from 'request'
-import { ActivityIconObject, VideoState } from '../../../shared/index'
+import { ActivityIconObject, ActivityVideoUrlObject, VideoState, ActivityUrlObject } from '../../../shared/index'
 import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
 import { VideoPrivacy } from '../../../shared/models/videos'
 import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos'
 import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
-import { resetSequelizeInstance, retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils'
+import { resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils'
 import { logger } from '../../helpers/logger'
 import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests'
 import { ACTIVITY_PUB, CONFIG, REMOTE_SCHEME, sequelizeTypescript, VIDEO_MIMETYPE_EXT } from '../../initializers'
@@ -17,7 +17,7 @@ import { TagModel } from '../../models/video/tag'
 import { VideoModel } from '../../models/video/video'
 import { VideoChannelModel } from '../../models/video/video-channel'
 import { VideoFileModel } from '../../models/video/video-file'
-import { getOrCreateActorAndServerAndModel, updateActorAvatarInstance } from './actor'
+import { getOrCreateActorAndServerAndModel } from './actor'
 import { addVideoComments } from './video-comments'
 import { crawlCollectionPage } from './crawl'
 import { sendCreateVideo, sendUpdateVideo } from './send'
@@ -25,7 +25,6 @@ import { isArray } from '../../helpers/custom-validators/misc'
 import { VideoCaptionModel } from '../../models/video/video-caption'
 import { JobQueue } from '../job-queue'
 import { ActivitypubHttpFetcherPayload } from '../job-queue/handlers/activitypub-http-fetcher'
-import { getUrlFromWebfinger } from '../../helpers/webfinger'
 import { createRates } from './video-rates'
 import { addVideoShares, shareVideoByServerAndChannel } from './share'
 import { AccountModel } from '../../models/account/account'
@@ -137,10 +136,7 @@ async function videoActivityObjectToDBAttributes (
 }
 
 function videoFileActivityUrlToDBAttributes (videoCreated: VideoModel, videoObject: VideoTorrentObject) {
-  const mimeTypes = Object.keys(VIDEO_MIMETYPE_EXT)
-  const fileUrls = videoObject.url.filter(u => {
-    return mimeTypes.indexOf(u.mimeType) !== -1 && u.mimeType.startsWith('video/')
-  })
+  const fileUrls = videoObject.url.filter(u => isActivityVideoUrlObject(u)) as ActivityVideoUrlObject[]
 
   if (fileUrls.length === 0) {
     throw new Error('Cannot find video files for ' + videoCreated.url)
@@ -331,8 +327,8 @@ async function refreshVideoIfNeeded (video: VideoModel): Promise<VideoModel> {
 
     const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject)
     const account = await AccountModel.load(channelActor.VideoChannel.accountId)
-    return updateVideoFromAP(video, videoObject, account.Actor, channelActor)
 
+    return updateVideoFromAP(video, videoObject, account, channelActor.VideoChannel)
   } catch (err) {
     logger.warn('Cannot refresh video.', { err })
     return video
@@ -342,8 +338,8 @@ async function refreshVideoIfNeeded (video: VideoModel): Promise<VideoModel> {
 async function updateVideoFromAP (
   video: VideoModel,
   videoObject: VideoTorrentObject,
-  accountActor: ActorModel,
-  channelActor: ActorModel,
+  account: AccountModel,
+  channel: VideoChannelModel,
   overrideTo?: string[]
 ) {
   logger.debug('Updating remote video "%s".', videoObject.uuid)
@@ -359,12 +355,12 @@ async function updateVideoFromAP (
 
       // Check actor has the right to update the video
       const videoChannel = video.VideoChannel
-      if (videoChannel.Account.Actor.id !== accountActor.id) {
-        throw new Error('Account ' + accountActor.url + ' does not own video channel ' + videoChannel.Actor.url)
+      if (videoChannel.Account.id !== account.id) {
+        throw new Error('Account ' + account.Actor.url + ' does not own video channel ' + videoChannel.Actor.url)
       }
 
       const to = overrideTo ? overrideTo : videoObject.to
-      const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, to)
+      const videoData = await videoActivityObjectToDBAttributes(channel, videoObject, to)
       video.set('name', videoData.name)
       video.set('uuid', videoData.uuid)
       video.set('url', videoData.url)
@@ -444,3 +440,11 @@ export {
   addVideoShares,
   createRates
 }
+
+// ---------------------------------------------------------------------------
+
+function isActivityVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject {
+  const mimeTypes = Object.keys(VIDEO_MIMETYPE_EXT)
+
+  return mimeTypes.indexOf(url.mimeType) !== -1 && url.mimeType.startsWith('video/')
+}

server/lib/redundancy.ts

@@ -0,0 +1,18 @@
+import { VideoRedundancyModel } from '../models/redundancy/video-redundancy'
+import { sendUndoCacheFile } from './activitypub/send'
+import { Transaction } from 'sequelize'
+import { getServerActor } from '../helpers/utils'
+
+async function removeVideoRedundancy (videoRedundancy: VideoRedundancyModel, t?: Transaction) {
+  const serverActor = await getServerActor()
+
+  await sendUndoCacheFile(serverActor, videoRedundancy, t)
+
+  await videoRedundancy.destroy({ transaction: t })
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  removeVideoRedundancy
+}

server/lib/schedulers/videos-redundancy-scheduler.ts

@@ -0,0 +1,161 @@
+import { AbstractScheduler } from './abstract-scheduler'
+import { CONFIG, JOB_TTL, REDUNDANCY, SCHEDULER_INTERVALS_MS } from '../../initializers'
+import { logger } from '../../helpers/logger'
+import { VideoRedundancyStrategy } from '../../../shared/models/redundancy'
+import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
+import { VideoFileModel } from '../../models/video/video-file'
+import { sortBy } from 'lodash'
+import { downloadWebTorrentVideo } from '../../helpers/webtorrent'
+import { join } from 'path'
+import { rename } from 'fs-extra'
+import { getServerActor } from '../../helpers/utils'
+import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send'
+import { VideoModel } from '../../models/video/video'
+import { getVideoCacheFileActivityPubUrl } from '../activitypub/url'
+import { removeVideoRedundancy } from '../redundancy'
+import { isTestInstance } from '../../helpers/core-utils'
+
+export class VideosRedundancyScheduler extends AbstractScheduler {
+
+  private static instance: AbstractScheduler
+  private executing = false
+
+  protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.videosRedundancy
+
+  private constructor () {
+    super()
+  }
+
+  async execute () {
+    if (this.executing) return
+
+    this.executing = true
+
+    for (const obj of CONFIG.REDUNDANCY.VIDEOS) {
+
+      try {
+        const videoToDuplicate = await this.findVideoToDuplicate(obj.strategy)
+        if (!videoToDuplicate) continue
+
+        const videoFiles = videoToDuplicate.VideoFiles
+        videoFiles.forEach(f => f.Video = videoToDuplicate)
+
+        const videosRedundancy = await VideoRedundancyModel.getVideoFiles(obj.strategy)
+        if (this.isTooHeavy(videosRedundancy, videoFiles, obj.size)) {
+          if (!isTestInstance()) logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url)
+          continue
+        }
+
+        logger.info('Will duplicate video %s in redundancy scheduler "%s".', videoToDuplicate.url, obj.strategy)
+
+        await this.createVideoRedundancy(obj.strategy, videoFiles)
+      } catch (err) {
+        logger.error('Cannot run videos redundancy %s.', obj.strategy, { err })
+      }
+    }
+
+    const expired = await VideoRedundancyModel.listAllExpired()
+
+    for (const m of expired) {
+      logger.info('Removing expired video %s from our redundancy system.', this.buildEntryLogId(m))
+
+      try {
+        await m.destroy()
+      } catch (err) {
+        logger.error('Cannot remove %s video from our redundancy system.', this.buildEntryLogId(m))
+      }
+    }
+
+    this.executing = false
+  }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+
+  private findVideoToDuplicate (strategy: VideoRedundancyStrategy) {
+    if (strategy === 'most-views') return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR)
+  }
+
+  private async createVideoRedundancy (strategy: VideoRedundancyStrategy, filesToDuplicate: VideoFileModel[]) {
+    const serverActor = await getServerActor()
+
+    for (const file of filesToDuplicate) {
+      const existing = await VideoRedundancyModel.loadByFileId(file.id)
+      if (existing) {
+        logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', file.Video.url, file.resolution, strategy)
+
+        existing.expiresOn = this.buildNewExpiration()
+        await existing.save()
+
+        await sendUpdateCacheFile(serverActor, existing)
+        continue
+      }
+
+      // We need more attributes and check if the video still exists
+      const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(file.Video.id)
+      if (!video) continue
+
+      logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, strategy)
+
+      const { baseUrlHttp, baseUrlWs } = video.getBaseUrls()
+      const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs)
+
+      const tmpPath = await downloadWebTorrentVideo({ magnetUri }, JOB_TTL['video-import'])
+
+      const destPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(file))
+      await rename(tmpPath, destPath)
+
+      const createdModel = await VideoRedundancyModel.create({
+        expiresOn: new Date(Date.now() + REDUNDANCY.VIDEOS.EXPIRES_AFTER_MS),
+        url: getVideoCacheFileActivityPubUrl(file),
+        fileUrl: video.getVideoFileUrl(file, CONFIG.WEBSERVER.URL),
+        strategy,
+        videoFileId: file.id,
+        actorId: serverActor.id
+      })
+      createdModel.VideoFile = file
+
+      await sendCreateCacheFile(serverActor, createdModel)
+    }
+  }
+
+  // Unused, but could be useful in the future, with a custom strategy
+  private async purgeVideosIfNeeded (videosRedundancy: VideoRedundancyModel[], filesToDuplicate: VideoFileModel[], maxSize: number) {
+    const sortedVideosRedundancy = sortBy(videosRedundancy, 'createdAt')
+
+    while (this.isTooHeavy(sortedVideosRedundancy, filesToDuplicate, maxSize)) {
+      const toDelete = sortedVideosRedundancy.shift()
+
+      const videoFile = toDelete.VideoFile
+      logger.info('Purging video %s (resolution %d) from our redundancy system.', videoFile.Video.url, videoFile.resolution)
+
+      await removeVideoRedundancy(toDelete, undefined)
+    }
+
+    return sortedVideosRedundancy
+  }
+
+  private isTooHeavy (videosRedundancy: VideoRedundancyModel[], filesToDuplicate: VideoFileModel[], maxSizeArg: number) {
+    const maxSize = maxSizeArg - this.getTotalFileSizes(filesToDuplicate)
+
+    const redundancyReducer = (previous: number, current: VideoRedundancyModel) => previous + current.VideoFile.size
+    const totalDuplicated = videosRedundancy.reduce(redundancyReducer, 0)
+
+    return totalDuplicated > maxSize
+  }
+
+  private buildNewExpiration () {
+    return new Date(Date.now() + REDUNDANCY.VIDEOS.EXPIRES_AFTER_MS)
+  }
+
+  private buildEntryLogId (object: VideoRedundancyModel) {
+    return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}`
+  }
+
+  private getTotalFileSizes (files: VideoFileModel[]) {
+    const fileReducer = (previous: number, current: VideoFileModel) => previous + current.size
+
+    return files.reduce(fileReducer, 0)
+  }
+}

server/middlewares/validators/redundancy.ts

@@ -0,0 +1,80 @@
+import * as express from 'express'
+import 'express-validator'
+import { param, body } from 'express-validator/check'
+import { exists, isBooleanValid, isIdOrUUIDValid, toIntOrNull } from '../../helpers/custom-validators/misc'
+import { isVideoExist } from '../../helpers/custom-validators/videos'
+import { logger } from '../../helpers/logger'
+import { areValidationErrors } from './utils'
+import { VideoModel } from '../../models/video/video'
+import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
+import { isHostValid } from '../../helpers/custom-validators/servers'
+import { getServerActor } from '../../helpers/utils'
+import { ActorFollowModel } from '../../models/activitypub/actor-follow'
+import { SERVER_ACTOR_NAME } from '../../initializers'
+import { ServerModel } from '../../models/server/server'
+
+const videoRedundancyGetValidator = [
+  param('videoId').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid video id'),
+  param('resolution')
+    .customSanitizer(toIntOrNull)
+    .custom(exists).withMessage('Should have a valid resolution'),
+  param('fps')
+    .optional()
+    .customSanitizer(toIntOrNull)
+    .custom(exists).withMessage('Should have a valid fps'),
+
+  async (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    logger.debug('Checking videoRedundancyGetValidator parameters', { parameters: req.params })
+
+    if (areValidationErrors(req, res)) return
+    if (!await isVideoExist(req.params.videoId, res)) return
+
+    const video: VideoModel = res.locals.video
+    const videoFile = video.VideoFiles.find(f => {
+      return f.resolution === req.params.resolution && (!req.params.fps || f.fps === req.params.fps)
+    })
+
+    if (!videoFile) return res.status(404).json({ error: 'Video file not found.' })
+    res.locals.videoFile = videoFile
+
+    const videoRedundancy = await VideoRedundancyModel.loadByFileId(videoFile.id)
+    if (!videoRedundancy)return res.status(404).json({ error: 'Video redundancy not found.' })
+    res.locals.videoRedundancy = videoRedundancy
+
+    return next()
+  }
+]
+
+const updateServerRedundancyValidator = [
+  param('host').custom(isHostValid).withMessage('Should have a valid host'),
+  body('redundancyAllowed')
+    .toBoolean()
+    .custom(isBooleanValid).withMessage('Should have a valid redundancyAllowed attribute'),
+
+  async (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    logger.debug('Checking updateServerRedundancy parameters', { parameters: req.params })
+
+    if (areValidationErrors(req, res)) return
+
+    const server = await ServerModel.loadByHost(req.params.host)
+
+    if (!server) {
+      return res
+        .status(404)
+        .json({
+          error: `Server ${req.params.host} not found.`
+        })
+        .end()
+    }
+
+    res.locals.server = server
+    return next()
+  }
+]
+
+// ---------------------------------------------------------------------------
+
+export {
+  videoRedundancyGetValidator,
+  updateServerRedundancyValidator
+}

server/models/activitypub/actor-follow.ts

@@ -19,7 +19,7 @@ import {
   UpdatedAt
 } from 'sequelize-typescript'
 import { FollowState } from '../../../shared/models/actors'
-import { AccountFollow } from '../../../shared/models/actors/follow.model'
+import { ActorFollow } from '../../../shared/models/actors/follow.model'
 import { logger } from '../../helpers/logger'
 import { getServerActor } from '../../helpers/utils'
 import { ACTOR_FOLLOW_SCORE } from '../../initializers'
@@ -529,7 +529,7 @@ export class ActorFollowModel extends Model<ActorFollowModel> {
     return ActorFollowModel.findAll(query)
   }
 
-  toFormattedJSON (): AccountFollow {
+  toFormattedJSON (): ActorFollow {
     const follower = this.ActorFollower.toFormattedJSON()
     const following = this.ActorFollowing.toFormattedJSON()
 

server/models/activitypub/actor.ts

@@ -76,7 +76,13 @@ export const unusedActorAttributesForAPI = [
       },
       {
         model: () => VideoChannelModel.unscoped(),
-        required: false
+        required: false,
+        include: [
+          {
+            model: () => AccountModel,
+            required: true
+          }
+        ]
       },
       {
         model: () => ServerModel,
@@ -337,6 +343,7 @@ export class ActorModel extends Model<ActorModel> {
       uuid: this.uuid,
       name: this.preferredUsername,
       host: this.getHost(),
+      hostRedundancyAllowed: this.getRedundancyAllowed(),
       followingCount: this.followingCount,
       followersCount: this.followersCount,
       avatar,
@@ -440,6 +447,10 @@ export class ActorModel extends Model<ActorModel> {
     return this.Server ? this.Server.host : CONFIG.WEBSERVER.HOST
   }
 
+  getRedundancyAllowed () {
+    return this.Server ? this.Server.redundancyAllowed : false
+  }
+
   getAvatarUrl () {
     if (!this.avatarId) return undefined
 

server/models/redundancy/video-redundancy.ts

@@ -0,0 +1,249 @@
+import {
+  AfterDestroy,
+  AllowNull,
+  BelongsTo,
+  Column,
+  CreatedAt,
+  DataType,
+  ForeignKey,
+  Is,
+  Model,
+  Scopes,
+  Sequelize,
+  Table,
+  UpdatedAt
+} from 'sequelize-typescript'
+import { ActorModel } from '../activitypub/actor'
+import { throwIfNotValid } from '../utils'
+import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc'
+import { CONSTRAINTS_FIELDS, VIDEO_EXT_MIMETYPE } from '../../initializers'
+import { VideoFileModel } from '../video/video-file'
+import { isDateValid } from '../../helpers/custom-validators/misc'
+import { getServerActor } from '../../helpers/utils'
+import { VideoModel } from '../video/video'
+import { VideoRedundancyStrategy } from '../../../shared/models/redundancy'
+import { logger } from '../../helpers/logger'
+import { CacheFileObject } from '../../../shared'
+import { VideoChannelModel } from '../video/video-channel'
+import { ServerModel } from '../server/server'
+import { sample } from 'lodash'
+import { isTestInstance } from '../../helpers/core-utils'
+
+export enum ScopeNames {
+  WITH_VIDEO = 'WITH_VIDEO'
+}
+
+@Scopes({
+  [ ScopeNames.WITH_VIDEO ]: {
+    include: [
+      {
+        model: () => VideoFileModel,
+        required: true,
+        include: [
+          {
+            model: () => VideoModel,
+            required: true
+          }
+        ]
+      }
+    ]
+  }
+})
+
+@Table({
+  tableName: 'videoRedundancy',
+  indexes: [
+    {
+      fields: [ 'videoFileId' ]
+    },
+    {
+      fields: [ 'actorId' ]
+    },
+    {
+      fields: [ 'url' ],
+      unique: true
+    }
+  ]
+})
+export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
+
+  @CreatedAt
+  createdAt: Date
+
+  @UpdatedAt
+  updatedAt: Date
+
+  @AllowNull(false)
+  @Column
+  expiresOn: Date
+
+  @AllowNull(false)
+  @Is('VideoRedundancyFileUrl', value => throwIfNotValid(value, isUrlValid, 'fileUrl'))
+  @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
+  fileUrl: string
+
+  @AllowNull(false)
+  @Is('VideoRedundancyUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url'))
+  @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
+  url: string
+
+  @AllowNull(true)
+  @Column
+  strategy: string // Only used by us
+
+  @ForeignKey(() => VideoFileModel)
+  @Column
+  videoFileId: number
+
+  @BelongsTo(() => VideoFileModel, {
+    foreignKey: {
+      allowNull: false
+    },
+    onDelete: 'cascade'
+  })
+  VideoFile: VideoFileModel
+
+  @ForeignKey(() => ActorModel)
+  @Column
+  actorId: number
+
+  @BelongsTo(() => ActorModel, {
+    foreignKey: {
+      allowNull: false
+    },
+    onDelete: 'cascade'
+  })
+  Actor: ActorModel
+
+  @AfterDestroy
+  static removeFilesAndSendDelete (instance: VideoRedundancyModel) {
+    // Not us
+    if (!instance.strategy) return
+
+    logger.info('Removing video file %s-.', instance.VideoFile.Video.uuid, instance.VideoFile.resolution)
+
+    return instance.VideoFile.Video.removeFile(instance.VideoFile)
+  }
+
+  static loadByFileId (videoFileId: number) {
+    const query = {
+      where: {
+        videoFileId
+      }
+    }
+
+    return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
+  }
+
+  static loadByUrl (url: string) {
+    const query = {
+      where: {
+        url
+      }
+    }
+
+    return VideoRedundancyModel.findOne(query)
+  }
+
+  static async findMostViewToDuplicate (randomizedFactor: number) {
+    // On VideoModel!
+    const query = {
+      logging: !isTestInstance(),
+      limit: randomizedFactor,
+      order: [ [ 'views', 'DESC' ] ],
+      include: [
+        {
+          model: VideoFileModel.unscoped(),
+          required: true,
+          where: {
+            id: {
+              [ Sequelize.Op.notIn ]: await VideoRedundancyModel.buildExcludeIn()
+            }
+          }
+        },
+        {
+          attributes: [],
+          model: VideoChannelModel.unscoped(),
+          required: true,
+          include: [
+            {
+              attributes: [],
+              model: ActorModel.unscoped(),
+              required: true,
+              include: [
+                {
+                  attributes: [],
+                  model: ServerModel.unscoped(),
+                  required: true,
+                  where: {
+                    redundancyAllowed: true
+                  }
+                }
+              ]
+            }
+          ]
+        }
+      ]
+    }
+
+    const rows = await VideoModel.unscoped().findAll(query)
+
+    return sample(rows)
+  }
+
+  static async getVideoFiles (strategy: VideoRedundancyStrategy) {
+    const actor = await getServerActor()
+
+    const queryVideoFiles = {
+      logging: !isTestInstance(),
+      where: {
+        actorId: actor.id,
+        strategy
+      }
+    }
+
+    return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO)
+                               .findAll(queryVideoFiles)
+  }
+
+  static listAllExpired () {
+    const query = {
+      logging: !isTestInstance(),
+      where: {
+        expiresOn: {
+          [Sequelize.Op.lt]: new Date()
+        }
+      }
+    }
+
+    return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO)
+                               .findAll(query)
+  }
+
+  toActivityPubObject (): CacheFileObject {
+    return {
+      id: this.url,
+      type: 'CacheFile' as 'CacheFile',
+      object: this.VideoFile.Video.url,
+      expires: this.expiresOn.toISOString(),
+      url: {
+        type: 'Link',
+        mimeType: VIDEO_EXT_MIMETYPE[ this.VideoFile.extname ] as any,
+        href: this.fileUrl,
+        height: this.VideoFile.resolution,
+        size: this.VideoFile.size,
+        fps: this.VideoFile.fps
+      }
+    }
+  }
+
+  private static async buildExcludeIn () {
+    const actor = await getServerActor()
+
+    return Sequelize.literal(
+      '(' +
+        `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "expiresOn" >= NOW()` +
+      ')'
+    )
+  }
+}

server/models/server/server.ts

@@ -1,4 +1,4 @@
-import { AllowNull, Column, CreatedAt, HasMany, Is, Model, Table, UpdatedAt } from 'sequelize-typescript'
+import { AllowNull, Column, CreatedAt, Default, HasMany, Is, Model, Table, UpdatedAt } from 'sequelize-typescript'
 import { isHostValid } from '../../helpers/custom-validators/servers'
 import { ActorModel } from '../activitypub/actor'
 import { throwIfNotValid } from '../utils'
@@ -19,6 +19,11 @@ export class ServerModel extends Model<ServerModel> {
   @Column
   host: string
 
+  @AllowNull(false)
+  @Default(false)
+  @Column
+  redundancyAllowed: boolean
+
   @CreatedAt
   createdAt: Date
 
@@ -34,4 +39,14 @@ export class ServerModel extends Model<ServerModel> {
     hooks: true
   })
   Actors: ActorModel[]
+
+  static loadByHost (host: string) {
+    const query = {
+      where: {
+        host
+      }
+    }
+
+    return ServerModel.findOne(query)
+  }
 }

server/models/video/video-file.ts

@@ -1,5 +1,18 @@
 import { values } from 'lodash'
-import { AllowNull, BelongsTo, Column, CreatedAt, DataType, Default, ForeignKey, Is, Model, Table, UpdatedAt } from 'sequelize-typescript'
+import {
+  AllowNull,
+  BelongsTo,
+  Column,
+  CreatedAt,
+  DataType,
+  Default,
+  ForeignKey,
+  HasMany,
+  Is,
+  Model,
+  Table,
+  UpdatedAt
+} from 'sequelize-typescript'
 import {
   isVideoFileInfoHashValid,
   isVideoFileResolutionValid,
@@ -10,6 +23,7 @@ import { CONSTRAINTS_FIELDS } from '../../initializers'
 import { throwIfNotValid } from '../utils'
 import { VideoModel } from './video'
 import * as Sequelize from 'sequelize'
+import { VideoRedundancyModel } from '../redundancy/video-redundancy'
 
 @Table({
   tableName: 'videoFile',
@@ -70,6 +84,15 @@ export class VideoFileModel extends Model<VideoFileModel> {
   })
   Video: VideoModel
 
+  @HasMany(() => VideoRedundancyModel, {
+    foreignKey: {
+      allowNull: false
+    },
+    onDelete: 'CASCADE',
+    hooks: true
+  })
+  RedundancyVideos: VideoRedundancyModel[]
+
   static isInfohashExists (infoHash: string) {
     const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1'
     const options = {

server/models/video/video.ts

@@ -27,13 +27,13 @@ import {
   Table,
   UpdatedAt
 } from 'sequelize-typescript'
-import { VideoPrivacy, VideoResolution, VideoState } from '../../../shared'
+import { ActivityUrlObject, VideoPrivacy, VideoResolution, VideoState } from '../../../shared'
 import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
 import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos'
 import { VideoFilter } from '../../../shared/models/videos/video-query.type'
 import { createTorrentPromise, peertubeTruncate } from '../../helpers/core-utils'
 import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
-import { isBooleanValid } from '../../helpers/custom-validators/misc'
+import { isArray, isBooleanValid } from '../../helpers/custom-validators/misc'
 import {
   isVideoCategoryValid,
   isVideoDescriptionValid,
@@ -90,6 +90,7 @@ import { VideoCaptionModel } from './video-caption'
 import { VideoBlacklistModel } from './video-blacklist'
 import { copy, remove, rename, stat, writeFile } from 'fs-extra'
 import { VideoViewModel } from './video-views'
+import { VideoRedundancyModel } from '../redundancy/video-redundancy'
 
 // FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation
 const indexes: Sequelize.DefineIndexesOptions[] = [
@@ -470,7 +471,13 @@ type AvailableForListIDsOptions = {
     include: [
       {
         model: () => VideoFileModel.unscoped(),
-        required: false
+        required: false,
+        include: [
+          {
+            model: () => VideoRedundancyModel.unscoped(),
+            required: false
+          }
+        ]
       }
     ]
   },
@@ -633,6 +640,7 @@ export class VideoModel extends Model<VideoModel> {
       name: 'videoId',
       allowNull: false
     },
+    hooks: true,
     onDelete: 'cascade'
   })
   VideoFiles: VideoFileModel[]
@@ -1325,9 +1333,7 @@ export class VideoModel extends Model<VideoModel> {
         [ CONFIG.WEBSERVER.WS + '://' + CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT + '/tracker/socket' ],
         [ CONFIG.WEBSERVER.URL + '/tracker/announce' ]
       ],
-      urlList: [
-        CONFIG.WEBSERVER.URL + STATIC_PATHS.WEBSEED + this.getVideoFilename(videoFile)
-      ]
+      urlList: [ CONFIG.WEBSERVER.URL + STATIC_PATHS.WEBSEED + this.getVideoFilename(videoFile) ]
     }
 
     const torrent = await createTorrentPromise(this.getVideoFilePath(videoFile), options)
@@ -1535,11 +1541,11 @@ export class VideoModel extends Model<VideoModel> {
       }
     }
 
-    const url = []
+    const url: ActivityUrlObject[] = []
     for (const file of this.VideoFiles) {
       url.push({
         type: 'Link',
-        mimeType: VIDEO_EXT_MIMETYPE[ file.extname ],
+        mimeType: VIDEO_EXT_MIMETYPE[ file.extname ] as any,
         href: this.getVideoFileUrl(file, baseUrlHttp),
         height: file.resolution,
         size: file.size,
@@ -1548,14 +1554,14 @@ export class VideoModel extends Model<VideoModel> {
 
       url.push({
         type: 'Link',
-        mimeType: 'application/x-bittorrent',
+        mimeType: 'application/x-bittorrent' as 'application/x-bittorrent',
         href: this.getTorrentUrl(file, baseUrlHttp),
         height: file.resolution
       })
 
       url.push({
         type: 'Link',
-        mimeType: 'application/x-bittorrent;x-scheme-handler/magnet',
+        mimeType: 'application/x-bittorrent;x-scheme-handler/magnet' as 'application/x-bittorrent;x-scheme-handler/magnet',
         href: this.generateMagnetUri(file, baseUrlHttp, baseUrlWs),
         height: file.resolution
       })
@@ -1796,7 +1802,7 @@ export class VideoModel extends Model<VideoModel> {
       (now - updatedAtTime) > ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL
   }
 
-  private getBaseUrls () {
+  getBaseUrls () {
     let baseUrlHttp
     let baseUrlWs
 
@@ -1811,39 +1817,42 @@ export class VideoModel extends Model<VideoModel> {
     return { baseUrlHttp, baseUrlWs }
   }
 
-  private getThumbnailUrl (baseUrlHttp: string) {
+  generateMagnetUri (videoFile: VideoFileModel, baseUrlHttp: string, baseUrlWs: string) {
+    const xs = this.getTorrentUrl(videoFile, baseUrlHttp)
+    const announce = [ baseUrlWs + '/tracker/socket', baseUrlHttp + '/tracker/announce' ]
+    let urlList = [ this.getVideoFileUrl(videoFile, baseUrlHttp) ]
+
+    const redundancies = videoFile.RedundancyVideos
+    if (isArray(redundancies)) urlList = urlList.concat(redundancies.map(r => r.fileUrl))
+
+    const magnetHash = {
+      xs,
+      announce,
+      urlList,
+      infoHash: videoFile.infoHash,
+      name: this.name
+    }
+
+    return magnetUtil.encode(magnetHash)
+  }
+
+  getThumbnailUrl (baseUrlHttp: string) {
     return baseUrlHttp + STATIC_PATHS.THUMBNAILS + this.getThumbnailName()
   }
 
-  private getTorrentUrl (videoFile: VideoFileModel, baseUrlHttp: string) {
+  getTorrentUrl (videoFile: VideoFileModel, baseUrlHttp: string) {
     return baseUrlHttp + STATIC_PATHS.TORRENTS + this.getTorrentFileName(videoFile)
   }
 
-  private getTorrentDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) {
+  getTorrentDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) {
     return baseUrlHttp + STATIC_DOWNLOAD_PATHS.TORRENTS + this.getTorrentFileName(videoFile)
   }
 
-  private getVideoFileUrl (videoFile: VideoFileModel, baseUrlHttp: string) {
+  getVideoFileUrl (videoFile: VideoFileModel, baseUrlHttp: string) {
     return baseUrlHttp + STATIC_PATHS.WEBSEED + this.getVideoFilename(videoFile)
   }
 
-  private getVideoFileDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) {
+  getVideoFileDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) {
     return baseUrlHttp + STATIC_DOWNLOAD_PATHS.VIDEOS + this.getVideoFilename(videoFile)
   }
-
-  private generateMagnetUri (videoFile: VideoFileModel, baseUrlHttp: string, baseUrlWs: string) {
-    const xs = this.getTorrentUrl(videoFile, baseUrlHttp)
-    const announce = [ baseUrlWs + '/tracker/socket', baseUrlHttp + '/tracker/announce' ]
-    const urlList = [ this.getVideoFileUrl(videoFile, baseUrlHttp) ]
-
-    const magnetHash = {
-      xs,
-      announce,
-      urlList,
-      infoHash: videoFile.infoHash,
-      name: this.name
-    }
-
-    return magnetUtil.encode(magnetHash)
-  }
 }

server/tests/api/check-params/follows.ts

@@ -169,15 +169,6 @@ describe('Test server follows API validators', function () {
           statusCodeExpected: 404
         })
       })
-
-      it('Should succeed with the correct parameters', async function () {
-        await makeDeleteRequest({
-          url: server.url,
-          path: path + '/localhost:9002',
-          token: server.accessToken,
-          statusCodeExpected: 404
-        })
-      })
     })
   })
 

server/tests/api/check-params/index.ts

@@ -1,15 +1,17 @@
 // Order of the tests we want to execute
 import './accounts'
+import './config'
 import './follows'
 import './jobs'
+import './redundancy'
+import './search'
 import './services'
+import './user-subscriptions'
 import './users'
 import './video-abuses'
 import './video-blacklist'
 import './video-captions'
 import './video-channels'
 import './video-comments'
-import './videos'
 import './video-imports'
-import './search'
-import './user-subscriptions'
+import './videos'

server/tests/api/check-params/redundancy.ts

@@ -0,0 +1,103 @@
+/* tslint:disable:no-unused-expression */
+
+import 'mocha'
+
+import {
+  createUser,
+  doubleFollow,
+  flushAndRunMultipleServers,
+  flushTests,
+  killallServers,
+  makePutBodyRequest,
+  ServerInfo,
+  setAccessTokensToServers,
+  userLogin
+} from '../../utils'
+
+describe('Test server redundancy API validators', function () {
+  let servers: ServerInfo[]
+  let userAccessToken = null
+
+  // ---------------------------------------------------------------
+
+  before(async function () {
+    this.timeout(30000)
+
+    await flushTests()
+    servers = await flushAndRunMultipleServers(2)
+
+    await setAccessTokensToServers(servers)
+    await doubleFollow(servers[0], servers[1])
+
+    const user = {
+      username: 'user1',
+      password: 'password'
+    }
+
+    await createUser(servers[0].url, servers[0].accessToken, user.username, user.password)
+    userAccessToken = await userLogin(servers[0], user)
+  })
+
+  describe('When updating redundancy', function () {
+    const path = '/api/v1/server/redundancy'
+
+    it('Should fail with an invalid token', async function () {
+      await makePutBodyRequest({
+        url: servers[0].url,
+        path: path + '/localhost:9002',
+        fields: { redundancyAllowed: true },
+        token: 'fake_token',
+        statusCodeExpected: 401
+      })
+    })
+
+    it('Should fail if the user is not an administrator', async function () {
+      await makePutBodyRequest({
+        url: servers[0].url,
+        path: path + '/localhost:9002',
+        fields: { redundancyAllowed: true },
+        token: userAccessToken,
+        statusCodeExpected: 403
+      })
+    })
+
+    it('Should fail if we do not follow this server', async function () {
+      await makePutBodyRequest({
+        url: servers[0].url,
+        path: path + '/example.com',
+        fields: { redundancyAllowed: true },
+        token: servers[0].accessToken,
+        statusCodeExpected: 404
+      })
+    })
+
+    it('Should fail without de redundancyAllowed param', async function () {
+      await makePutBodyRequest({
+        url: servers[0].url,
+        path: path + '/localhost:9002',
+        fields: { blabla: true },
+        token: servers[0].accessToken,
+        statusCodeExpected: 400
+      })
+    })
+
+    it('Should succeed with the correct parameters', async function () {
+      await makePutBodyRequest({
+        url: servers[0].url,
+        path: path + '/localhost:9002',
+        fields: { redundancyAllowed: true },
+        token: servers[0].accessToken,
+        statusCodeExpected: 204
+      })
+    })
+  })
+
+  after(async function () {
+    killallServers(servers)
+
+    // Keep the logs if the test failed
+    if (this['ok']) {
+      await flushTests()
+    }
+  })
+})

server/tests/api/server/index.ts

@@ -3,6 +3,7 @@ import './email'
 import './follows'
 import './handle-down'
 import './jobs'
+import './redundancy'
 import './reverse-proxy'
 import './stats'
 import './tracker'

server/tests/api/server/redundancy.ts

@@ -0,0 +1,140 @@
+/* tslint:disable:no-unused-expression */
+
+import * as chai from 'chai'
+import 'mocha'
+import { VideoDetails } from '../../../../shared/models/videos'
+import {
+  doubleFollow,
+  flushAndRunMultipleServers,
+  flushTests,
+  getFollowingListPaginationAndSort,
+  getVideo,
+  killallServers,
+  ServerInfo,
+  setAccessTokensToServers,
+  uploadVideo,
+  wait,
+  root, viewVideo
+} from '../../utils'
+import { waitJobs } from '../../utils/server/jobs'
+import * as magnetUtil from 'magnet-uri'
+import { updateRedundancy } from '../../utils/server/redundancy'
+import { ActorFollow } from '../../../../shared/models/actors'
+import { readdir } from 'fs-extra'
+import { join } from 'path'
+
+const expect = chai.expect
+
+function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[]) {
+  const parsed = magnetUtil.decode(file.magnetUri)
+
+  for (const ws of baseWebseeds) {
+    const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`)
+    expect(found, `Webseed ${ws} not found in ${file.magnetUri}`).to.not.be.undefined
+  }
+}
+
+describe('Test videos redundancy', function () {
+  let servers: ServerInfo[] = []
+  let video1Server2UUID: string
+  let video2Server2UUID: string
+
+  before(async function () {
+    this.timeout(120000)
+
+    servers = await flushAndRunMultipleServers(3)
+
+    // Get the access tokens
+    await setAccessTokensToServers(servers)
+
+    {
+      const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' })
+      video1Server2UUID = res.body.video.uuid
+
+      await viewVideo(servers[1].url, video1Server2UUID)
+    }
+
+    {
+      const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' })
+      video2Server2UUID = res.body.video.uuid
+    }
+
+    await waitJobs(servers)
+
+    // Server 1 and server 2 follow each other
+    await doubleFollow(servers[0], servers[1])
+    // Server 1 and server 3 follow each other
+    await doubleFollow(servers[0], servers[2])
+    // Server 2 and server 3 follow each other
+    await doubleFollow(servers[1], servers[2])
+
+    await waitJobs(servers)
+  })
+
+  it('Should have 1 webseed on the first video', async function () {
+    const webseeds = [
+      'http://localhost:9002/static/webseed/' + video1Server2UUID
+    ]
+
+    for (const server of servers) {
+      const res = await getVideo(server.url, video1Server2UUID)
+
+      const video: VideoDetails = res.body
+      video.files.forEach(f => checkMagnetWebseeds(f, webseeds))
+    }
+  })
+
+  it('Should enable redundancy on server 1', async function () {
+    await updateRedundancy(servers[0].url, servers[0].accessToken, servers[1].host, true)
+
+    const res = await getFollowingListPaginationAndSort(servers[0].url, 0, 5, '-createdAt')
+    const follows: ActorFollow[] = res.body.data
+    const server2 = follows.find(f => f.following.host === 'localhost:9002')
+    const server3 = follows.find(f => f.following.host === 'localhost:9003')
+
+    expect(server3).to.not.be.undefined
+    expect(server3.following.hostRedundancyAllowed).to.be.false
+
+    expect(server2).to.not.be.undefined
+    expect(server2.following.hostRedundancyAllowed).to.be.true
+  })
+
+  it('Should have 2 webseed on the first video', async function () {
+    this.timeout(40000)
+
+    await waitJobs(servers)
+    await wait(15000)
+    await waitJobs(servers)
+
+    const webseeds = [
+      'http://localhost:9001/static/webseed/' + video1Server2UUID,
+      'http://localhost:9002/static/webseed/' + video1Server2UUID
+    ]
+
+    for (const server of servers) {
+      const res = await getVideo(server.url, video1Server2UUID)
+
+      const video: VideoDetails = res.body
+
+      for (const file of video.files) {
+        checkMagnetWebseeds(file, webseeds)
+      }
+    }
+
+    const files = await readdir(join(root(), 'test1', 'videos'))
+    expect(files).to.have.lengthOf(4)
+
+    for (const resolution of [ 240, 360, 480, 720 ]) {
+      expect(files.find(f => f === `${video1Server2UUID}-${resolution}.mp4`)).to.not.be.undefined
+    }
+  })
+
+  after(async function () {
+    killallServers(servers)
+
+    // Keep the logs if the test failed
+    if (this['ok']) {
+      await flushTests()
+    }
+  })
+})

server/tests/utils/server/follows.ts

@@ -1,5 +1,4 @@
 import * as request from 'supertest'
-import { wait } from '../miscs/miscs'
 import { ServerInfo } from './servers'
 import { waitJobs } from './jobs'
 

server/tests/utils/server/redundancy.ts

@@ -0,0 +1,17 @@
+import { makePutBodyRequest } from '../requests/requests'
+
+async function updateRedundancy (url: string, accessToken: string, host: string, redundancyAllowed: boolean, expectedStatus = 204) {
+  const path = '/api/v1/server/redundancy/' + host
+
+  return makePutBodyRequest({
+    url,
+    path,
+    token: accessToken,
+    fields: { redundancyAllowed },
+    statusCodeExpected: expectedStatus
+  })
+}
+
+export {
+  updateRedundancy
+}

shared/models/activitypub/activity.ts

@@ -1,6 +1,6 @@
 import { ActivityPubActor } from './activitypub-actor'
 import { ActivityPubSignature } from './activitypub-signature'
-import { VideoTorrentObject } from './objects'
+import { CacheFileObject, VideoTorrentObject } from './objects'
 import { DislikeObject } from './objects/dislike-object'
 import { VideoAbuseObject } from './objects/video-abuse-object'
 import { VideoCommentObject } from './objects/video-comment-object'
@@ -29,12 +29,12 @@ export interface BaseActivity {
 
 export interface ActivityCreate extends BaseActivity {
   type: 'Create'
-  object: VideoTorrentObject | VideoAbuseObject | ViewObject | DislikeObject | VideoCommentObject
+  object: VideoTorrentObject | VideoAbuseObject | ViewObject | DislikeObject | VideoCommentObject | CacheFileObject
 }
 
 export interface ActivityUpdate extends BaseActivity {
   type: 'Update'
-  object: VideoTorrentObject | ActivityPubActor
+  object: VideoTorrentObject | ActivityPubActor | CacheFileObject
 }
 
 export interface ActivityDelete extends BaseActivity {

shared/models/activitypub/objects/cache-file-object.ts

@@ -0,0 +1,9 @@
+import { ActivityVideoUrlObject } from './common-objects'
+
+export interface CacheFileObject {
+  id: string
+  type: 'CacheFile',
+  object: string
+  expires: string
+  url: ActivityVideoUrlObject
+}

shared/models/activitypub/objects/common-objects.ts

@@ -17,16 +17,31 @@ export interface ActivityIconObject {
   height: number
 }
 
-export interface ActivityUrlObject {
+export type ActivityVideoUrlObject = {
   type: 'Link'
-  mimeType: 'video/mp4' | 'video/webm' | 'application/x-bittorrent' | 'application/x-bittorrent;x-scheme-handler/magnet'
+  mimeType: 'video/mp4' | 'video/webm' | 'video/ogg'
   href: string
   height: number
-
-  size?: number
-  fps?: number
+  size: number
+  fps: number
 }
 
+export type ActivityUrlObject =
+  ActivityVideoUrlObject
+  |
+  {
+    type: 'Link'
+    mimeType: 'application/x-bittorrent' | 'application/x-bittorrent;x-scheme-handler/magnet'
+    href: string
+    height: number
+  }
+  |
+  {
+    type: 'Link'
+    mimeType: 'text/html'
+    href: string
+  }
+
 export interface ActivityPubAttributedTo {
   type: 'Group' | 'Person'
   id: string

shared/models/activitypub/objects/index.ts

@@ -1,3 +1,4 @@
+export * from './cache-file-object'
 export * from './common-objects'
 export * from './video-abuse-object'
 export * from './video-torrent-object'

shared/models/activitypub/objects/video-torrent-object.ts

@@ -1,10 +1,10 @@
 import {
   ActivityIconObject,
-  ActivityIdentifierObject, ActivityPubAttributedTo,
+  ActivityIdentifierObject,
+  ActivityPubAttributedTo,
   ActivityTagObject,
   ActivityUrlObject
 } from './common-objects'
-import { ActivityPubOrderedCollection } from '../activitypub-ordered-collection'
 import { VideoState } from '../../videos'
 
 export interface VideoTorrentObject {

shared/models/actors/follow.model.ts

@@ -2,10 +2,10 @@ import { Actor } from './actor.model'
 
 export type FollowState = 'pending' | 'accepted'
 
-export interface AccountFollow {
+export interface ActorFollow {
   id: number
-  follower: Actor
-  following: Actor
+  follower: Actor & { hostRedundancyAllowed: boolean }
+  following: Actor & { hostRedundancyAllowed: boolean }
   score: number
   state: FollowState
   createdAt: Date

shared/models/avatars/index.ts

@@ -0,0 +1 @@
+export * from './avatar.model'

shared/models/index.ts

@@ -1,5 +1,7 @@
-export * from './actors'
 export * from './activitypub'
+export * from './actors'
+export * from './avatars'
+export * from './redundancy'
 export * from './users'
 export * from './videos'
 export * from './feeds'

shared/models/redundancy/index.ts

@@ -0,0 +1 @@
+export * from './videos-redundancy.model'

shared/models/redundancy/videos-redundancy.model.ts

@@ -0,0 +1,6 @@
+export type VideoRedundancyStrategy = 'most-views'
+
+export interface VideosRedundancy {
+  strategy: VideoRedundancyStrategy
+  size: number
+}

shared/models/users/user-right.enum.ts

@@ -3,6 +3,7 @@ export enum UserRight {
 
   MANAGE_USERS,
   MANAGE_SERVER_FOLLOW,
+  MANAGE_SERVER_REDUNDANCY,
   MANAGE_VIDEO_ABUSES,
   MANAGE_JOBS,
   MANAGE_CONFIGURATION,

yarn.lock

@@ -44,6 +44,10 @@
     "@types/bluebird" "*"
     "@types/ioredis" "*"
 
+"@types/bytes@^3.0.0":
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/@types/bytes/-/bytes-3.0.0.tgz#549eeacd0a8fecfaa459334583a4edcee738e6db"
+
 "@types/caseless@*":
   version "0.12.1"
   resolved "https://registry.yarnpkg.com/@types/caseless/-/caseless-0.12.1.tgz#9794c69c8385d0192acc471a540d1f8e0d16218a"
@@ -993,7 +997,7 @@ bytes@1:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/bytes/-/bytes-1.0.0.tgz#3569ede8ba34315fab99c3e92cb04c7220de1fa8"
 
-bytes@3.0.0:
+bytes@3.0.0, bytes@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.0.0.tgz#d32815404d689699f85a4ea4fa8755dd13a96048"