prune-storage.ts 4.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168
  1. import { registerTSPaths } from '../server/helpers/register-ts-paths'
  2. registerTSPaths()
  3. import * as prompt from 'prompt'
  4. import { join } from 'path'
  5. import { CONFIG } from '../server/initializers/config'
  6. import { VideoModel } from '../server/models/video/video'
  7. import { initDatabaseModels } from '../server/initializers/database'
  8. import { readdir, remove } from 'fs-extra'
  9. import { VideoRedundancyModel } from '../server/models/redundancy/video-redundancy'
  10. import * as Bluebird from 'bluebird'
  11. import { getUUIDFromFilename } from '../server/helpers/utils'
  12. import { ThumbnailModel } from '../server/models/video/thumbnail'
  13. import { AvatarModel } from '../server/models/avatar/avatar'
  14. import { uniq, values } from 'lodash'
  15. run()
  16. .then(() => process.exit(0))
  17. .catch(err => {
  18. console.error(err)
  19. process.exit(-1)
  20. })
  21. async function run () {
  22. const dirs = values(CONFIG.STORAGE)
  23. if (uniq(dirs).length !== dirs.length) {
  24. console.error('Cannot prune storage because you put multiple storage keys in the same directory.')
  25. process.exit(0)
  26. }
  27. await initDatabaseModels(true)
  28. let toDelete: string[] = []
  29. toDelete = toDelete.concat(
  30. await pruneDirectory(CONFIG.STORAGE.VIDEOS_DIR, doesVideoExist(true)),
  31. await pruneDirectory(CONFIG.STORAGE.TORRENTS_DIR, doesVideoExist(true)),
  32. await pruneDirectory(CONFIG.STORAGE.REDUNDANCY_DIR, doesRedundancyExist),
  33. await pruneDirectory(CONFIG.STORAGE.PREVIEWS_DIR, doesThumbnailExist(true)),
  34. await pruneDirectory(CONFIG.STORAGE.THUMBNAILS_DIR, doesThumbnailExist(false)),
  35. await pruneDirectory(CONFIG.STORAGE.AVATARS_DIR, doesAvatarExist)
  36. )
  37. const tmpFiles = await readdir(CONFIG.STORAGE.TMP_DIR)
  38. toDelete = toDelete.concat(tmpFiles.map(t => join(CONFIG.STORAGE.TMP_DIR, t)))
  39. if (toDelete.length === 0) {
  40. console.log('No files to delete.')
  41. return
  42. }
  43. console.log('Will delete %d files:\n\n%s\n\n', toDelete.length, toDelete.join('\n'))
  44. const res = await askConfirmation()
  45. if (res === true) {
  46. console.log('Processing delete...\n')
  47. for (const path of toDelete) {
  48. await remove(path)
  49. }
  50. console.log('Done!')
  51. } else {
  52. console.log('Exiting without deleting files.')
  53. }
  54. }
  55. type ExistFun = (file: string) => Promise<boolean>
  56. async function pruneDirectory (directory: string, existFun: ExistFun) {
  57. const files = await readdir(directory)
  58. const toDelete: string[] = []
  59. await Bluebird.map(files, async file => {
  60. if (await existFun(file) !== true) {
  61. toDelete.push(join(directory, file))
  62. }
  63. }, { concurrency: 20 })
  64. return toDelete
  65. }
  66. function doesVideoExist (keepOnlyOwned: boolean) {
  67. return async (file: string) => {
  68. const uuid = getUUIDFromFilename(file)
  69. const video = await VideoModel.loadByUUID(uuid)
  70. return video && (keepOnlyOwned === false || video.isOwned())
  71. }
  72. }
  73. function doesThumbnailExist (keepOnlyOwned: boolean) {
  74. return async (file: string) => {
  75. const thumbnail = await ThumbnailModel.loadByName(file)
  76. if (!thumbnail) return false
  77. if (keepOnlyOwned) {
  78. const video = await VideoModel.load(thumbnail.videoId)
  79. if (video.isOwned() === false) return false
  80. }
  81. return true
  82. }
  83. }
  84. async function doesAvatarExist (file: string) {
  85. const avatar = await AvatarModel.loadByName(file)
  86. return !!avatar
  87. }
  88. async function doesRedundancyExist (file: string) {
  89. const uuid = getUUIDFromFilename(file)
  90. const video = await VideoModel.loadWithFiles(uuid)
  91. if (!video) return false
  92. const isPlaylist = file.includes('.') === false
  93. if (isPlaylist) {
  94. const p = video.getHLSPlaylist()
  95. if (!p) return false
  96. const redundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(p.id)
  97. return !!redundancy
  98. }
  99. const resolution = parseInt(file.split('-')[5], 10)
  100. if (isNaN(resolution)) {
  101. console.error('Cannot prune %s because we cannot guess guess the resolution.', file)
  102. return true
  103. }
  104. const videoFile = video.getWebTorrentFile(resolution)
  105. if (!videoFile) {
  106. console.error('Cannot find webtorrent file of video %s - %d', video.url, resolution)
  107. return true
  108. }
  109. const redundancy = await VideoRedundancyModel.loadLocalByFileId(videoFile.id)
  110. return !!redundancy
  111. }
  112. async function askConfirmation () {
  113. return new Promise((res, rej) => {
  114. prompt.start()
  115. const schema = {
  116. properties: {
  117. confirm: {
  118. type: 'string',
  119. description: 'These following unused files can be deleted, but please check your backups first (bugs happen).' +
  120. ' Notice PeerTube must have been stopped when your ran this script.' +
  121. ' Can we delete these files?',
  122. default: 'n',
  123. required: true
  124. }
  125. }
  126. }
  127. prompt.get(schema, function (err, result) {
  128. if (err) return rej(err)
  129. return res(result.confirm?.match(/y/) !== null)
  130. })
  131. })
  132. }