move_remote_media_to_new_store.py 3.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133
  1. #!/usr/bin/env python
  2. # -*- coding: utf-8 -*-
  3. # Copyright 2017 New Vector Ltd
  4. #
  5. # Licensed under the Apache License, Version 2.0 (the "License");
  6. # you may not use this file except in compliance with the License.
  7. # You may obtain a copy of the License at
  8. #
  9. # http://www.apache.org/licenses/LICENSE-2.0
  10. #
  11. # Unless required by applicable law or agreed to in writing, software
  12. # distributed under the License is distributed on an "AS IS" BASIS,
  13. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. # See the License for the specific language governing permissions and
  15. # limitations under the License.
  16. """
  17. Moves a list of remote media from one media store to another.
  18. The input should be a list of media files to be moved, one per line. Each line
  19. should be formatted::
  20. <origin server>|<file id>
  21. This can be extracted from postgres with::
  22. psql --tuples-only -A -c "select media_origin, filesystem_id from
  23. matrix.remote_media_cache where ..."
  24. To use, pipe the above into::
  25. PYTHON_PATH=. ./scripts/move_remote_media_to_new_store.py <source repo> <dest repo>
  26. """
  27. from __future__ import print_function
  28. import argparse
  29. import logging
  30. import sys
  31. import os
  32. import shutil
  33. from synapse.rest.media.v1.filepath import MediaFilePaths
  34. logger = logging.getLogger()
  35. def main(src_repo, dest_repo):
  36. src_paths = MediaFilePaths(src_repo)
  37. dest_paths = MediaFilePaths(dest_repo)
  38. for line in sys.stdin:
  39. line = line.strip()
  40. parts = line.split('|')
  41. if len(parts) != 2:
  42. print("Unable to parse input line %s" % line, file=sys.stderr)
  43. exit(1)
  44. move_media(parts[0], parts[1], src_paths, dest_paths)
  45. def move_media(origin_server, file_id, src_paths, dest_paths):
  46. """Move the given file, and any thumbnails, to the dest repo
  47. Args:
  48. origin_server (str):
  49. file_id (str):
  50. src_paths (MediaFilePaths):
  51. dest_paths (MediaFilePaths):
  52. """
  53. logger.info("%s/%s", origin_server, file_id)
  54. # check that the original exists
  55. original_file = src_paths.remote_media_filepath(origin_server, file_id)
  56. if not os.path.exists(original_file):
  57. logger.warn(
  58. "Original for %s/%s (%s) does not exist",
  59. origin_server, file_id, original_file,
  60. )
  61. else:
  62. mkdir_and_move(
  63. original_file,
  64. dest_paths.remote_media_filepath(origin_server, file_id),
  65. )
  66. # now look for thumbnails
  67. original_thumb_dir = src_paths.remote_media_thumbnail_dir(
  68. origin_server, file_id,
  69. )
  70. if not os.path.exists(original_thumb_dir):
  71. return
  72. mkdir_and_move(
  73. original_thumb_dir,
  74. dest_paths.remote_media_thumbnail_dir(origin_server, file_id)
  75. )
  76. def mkdir_and_move(original_file, dest_file):
  77. dirname = os.path.dirname(dest_file)
  78. if not os.path.exists(dirname):
  79. logger.debug("mkdir %s", dirname)
  80. os.makedirs(dirname)
  81. logger.debug("mv %s %s", original_file, dest_file)
  82. shutil.move(original_file, dest_file)
  83. if __name__ == "__main__":
  84. parser = argparse.ArgumentParser(
  85. description=__doc__,
  86. formatter_class = argparse.RawDescriptionHelpFormatter,
  87. )
  88. parser.add_argument(
  89. "-v", action='store_true', help='enable debug logging')
  90. parser.add_argument(
  91. "src_repo",
  92. help="Path to source content repo",
  93. )
  94. parser.add_argument(
  95. "dest_repo",
  96. help="Path to source content repo",
  97. )
  98. args = parser.parse_args()
  99. logging_config = {
  100. "level": logging.DEBUG if args.v else logging.INFO,
  101. "format": "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(message)s"
  102. }
  103. logging.basicConfig(**logging_config)
  104. main(args.src_repo, args.dest_repo)