|
@@ -0,0 +1,133 @@
|
|
|
+#!/usr/bin/env python
|
|
|
+# -*- coding: utf-8 -*-
|
|
|
+# Copyright 2017 New Vector Ltd
|
|
|
+#
|
|
|
+# Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
+# you may not use this file except in compliance with the License.
|
|
|
+# You may obtain a copy of the License at
|
|
|
+#
|
|
|
+# http://www.apache.org/licenses/LICENSE-2.0
|
|
|
+#
|
|
|
+# Unless required by applicable law or agreed to in writing, software
|
|
|
+# distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
+# See the License for the specific language governing permissions and
|
|
|
+# limitations under the License.
|
|
|
+
|
|
|
+"""
|
|
|
+Moves a list of remote media from one media store to another.
|
|
|
+
|
|
|
+The input should be a list of media files to be moved, one per line. Each line
|
|
|
+should be formatted::
|
|
|
+
|
|
|
+ <origin server>|<file id>
|
|
|
+
|
|
|
+This can be extracted from postgres with::
|
|
|
+
|
|
|
+ psql --tuples-only -A -c "select media_origin, filesystem_id from
|
|
|
+ matrix.remote_media_cache where ..."
|
|
|
+
|
|
|
+To use, pipe the above into::
|
|
|
+
|
|
|
+ PYTHON_PATH=. ./scripts/move_remote_media_to_new_store.py <source repo> <dest repo>
|
|
|
+"""
|
|
|
+
|
|
|
+from __future__ import print_function
|
|
|
+
|
|
|
+import argparse
|
|
|
+import logging
|
|
|
+
|
|
|
+import sys
|
|
|
+
|
|
|
+import os
|
|
|
+
|
|
|
+import shutil
|
|
|
+
|
|
|
+from synapse.rest.media.v1.filepath import MediaFilePaths
|
|
|
+
|
|
|
+logger = logging.getLogger()
|
|
|
+
|
|
|
+
|
|
|
+def main(src_repo, dest_repo):
|
|
|
+ src_paths = MediaFilePaths(src_repo)
|
|
|
+ dest_paths = MediaFilePaths(dest_repo)
|
|
|
+ for line in sys.stdin:
|
|
|
+ line = line.strip()
|
|
|
+ parts = line.split('|')
|
|
|
+ if len(parts) != 2:
|
|
|
+ print("Unable to parse input line %s" % line, file=sys.stderr)
|
|
|
+ exit(1)
|
|
|
+
|
|
|
+ move_media(parts[0], parts[1], src_paths, dest_paths)
|
|
|
+
|
|
|
+
|
|
|
+def move_media(origin_server, file_id, src_paths, dest_paths):
|
|
|
+ """Move the given file, and any thumbnails, to the dest repo
|
|
|
+
|
|
|
+ Args:
|
|
|
+ origin_server (str):
|
|
|
+ file_id (str):
|
|
|
+ src_paths (MediaFilePaths):
|
|
|
+ dest_paths (MediaFilePaths):
|
|
|
+ """
|
|
|
+ logger.info("%s/%s", origin_server, file_id)
|
|
|
+
|
|
|
+ # check that the original exists
|
|
|
+ original_file = src_paths.remote_media_filepath(origin_server, file_id)
|
|
|
+ if not os.path.exists(original_file):
|
|
|
+ logger.warn(
|
|
|
+ "Original for %s/%s (%s) does not exist",
|
|
|
+ origin_server, file_id, original_file,
|
|
|
+ )
|
|
|
+ else:
|
|
|
+ mkdir_and_move(
|
|
|
+ original_file,
|
|
|
+ dest_paths.remote_media_filepath(origin_server, file_id),
|
|
|
+ )
|
|
|
+
|
|
|
+ # now look for thumbnails
|
|
|
+ original_thumb_dir = src_paths.remote_media_thumbnail_dir(
|
|
|
+ origin_server, file_id,
|
|
|
+ )
|
|
|
+ if not os.path.exists(original_thumb_dir):
|
|
|
+ return
|
|
|
+
|
|
|
+ mkdir_and_move(
|
|
|
+ original_thumb_dir,
|
|
|
+ dest_paths.remote_media_thumbnail_dir(origin_server, file_id)
|
|
|
+ )
|
|
|
+
|
|
|
+
|
|
|
+def mkdir_and_move(original_file, dest_file):
|
|
|
+ dirname = os.path.dirname(dest_file)
|
|
|
+ if not os.path.exists(dirname):
|
|
|
+ logger.debug("mkdir %s", dirname)
|
|
|
+ os.makedirs(dirname)
|
|
|
+ logger.debug("mv %s %s", original_file, dest_file)
|
|
|
+ shutil.move(original_file, dest_file)
|
|
|
+
|
|
|
+
|
|
|
+if __name__ == "__main__":
|
|
|
+ parser = argparse.ArgumentParser(
|
|
|
+ description=__doc__,
|
|
|
+ formatter_class = argparse.RawDescriptionHelpFormatter,
|
|
|
+ )
|
|
|
+ parser.add_argument(
|
|
|
+ "-v", action='store_true', help='enable debug logging')
|
|
|
+ parser.add_argument(
|
|
|
+ "src_repo",
|
|
|
+ help="Path to source content repo",
|
|
|
+ )
|
|
|
+ parser.add_argument(
|
|
|
+ "dest_repo",
|
|
|
+ help="Path to source content repo",
|
|
|
+ )
|
|
|
+ args = parser.parse_args()
|
|
|
+
|
|
|
+ logging_config = {
|
|
|
+ "level": logging.DEBUG if args.v else logging.INFO,
|
|
|
+ "format": "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(message)s"
|
|
|
+ }
|
|
|
+ logging.basicConfig(**logging_config)
|
|
|
+
|
|
|
+ main(args.src_repo, args.dest_repo)
|