#!/usr/bin/env python # Copyright 2015, 2016 OpenMarket Ltd # Copyright 2018 New Vector Ltd # Copyright 2019 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import argparse import curses import logging import os import sys import time import traceback from types import TracebackType from typing import ( Any, Awaitable, Callable, Dict, Generator, Iterable, List, NoReturn, Optional, Set, Tuple, Type, TypeVar, cast, ) import yaml from typing_extensions import TypedDict from twisted.internet import defer, reactor as reactor_ from synapse.config.database import DatabaseConnectionConfig from synapse.config.homeserver import HomeServerConfig from synapse.logging.context import ( LoggingContext, make_deferred_yieldable, run_in_background, ) from synapse.notifier import ReplicationNotifier from synapse.storage.database import DatabasePool, LoggingTransaction, make_conn from synapse.storage.databases.main import FilteringWorkerStore, PushRuleStore from synapse.storage.databases.main.account_data import AccountDataWorkerStore from synapse.storage.databases.main.client_ips import ClientIpBackgroundUpdateStore from synapse.storage.databases.main.deviceinbox import DeviceInboxBackgroundUpdateStore from synapse.storage.databases.main.devices import DeviceBackgroundUpdateStore from synapse.storage.databases.main.e2e_room_keys import EndToEndRoomKeyBackgroundStore from synapse.storage.databases.main.end_to_end_keys import EndToEndKeyBackgroundStore from synapse.storage.databases.main.event_federation import EventFederationWorkerStore from synapse.storage.databases.main.event_push_actions import EventPushActionsStore from synapse.storage.databases.main.events_bg_updates import ( EventsBackgroundUpdatesStore, ) from synapse.storage.databases.main.media_repository import ( MediaRepositoryBackgroundUpdateStore, ) from synapse.storage.databases.main.presence import PresenceBackgroundUpdateStore from synapse.storage.databases.main.profile import ProfileWorkerStore from synapse.storage.databases.main.pusher import ( PusherBackgroundUpdatesStore, PusherWorkerStore, ) from synapse.storage.databases.main.receipts import ReceiptsBackgroundUpdateStore from synapse.storage.databases.main.registration import ( RegistrationBackgroundUpdateStore, find_max_generated_user_id_localpart, ) from synapse.storage.databases.main.relations import RelationsWorkerStore from synapse.storage.databases.main.room import RoomBackgroundUpdateStore from synapse.storage.databases.main.roommember import RoomMemberBackgroundUpdateStore from synapse.storage.databases.main.search import SearchBackgroundUpdateStore from synapse.storage.databases.main.state import MainStateBackgroundUpdateStore from synapse.storage.databases.main.stats import StatsStore from synapse.storage.databases.main.user_directory import ( UserDirectoryBackgroundUpdateStore, ) from synapse.storage.databases.state.bg_updates import StateBackgroundUpdateStore from synapse.storage.engines import create_engine from synapse.storage.prepare_database import prepare_database from synapse.types import ISynapseReactor from synapse.util import SYNAPSE_VERSION, Clock # Cast safety: Twisted does some naughty magic which replaces the # twisted.internet.reactor module with a Reactor instance at runtime. reactor = cast(ISynapseReactor, reactor_) logger = logging.getLogger("synapse_port_db") # SQLite doesn't have a dedicated boolean type (it stores True/False as 1/0). This means # portdb will read sqlite bools as integers, then try to insert them into postgres # boolean columns---which fails. Lacking some Python-parseable metaschema, we must # specify which integer columns should be inserted as booleans into postgres. BOOLEAN_COLUMNS = { "access_tokens": ["used"], "account_validity": ["email_sent"], "device_lists_changes_in_room": ["converted_to_destinations"], "device_lists_outbound_pokes": ["sent"], "devices": ["hidden"], "e2e_fallback_keys_json": ["used"], "e2e_room_keys": ["is_verified"], "event_edges": ["is_state"], "events": ["processed", "outlier", "contains_url"], "local_media_repository": ["safe_from_quarantine"], "presence_list": ["accepted"], "presence_stream": ["currently_active"], "public_room_list_stream": ["visibility"], "pushers": ["enabled"], "redactions": ["have_censored"], "room_stats_state": ["is_federatable"], "rooms": ["is_public", "has_auth_chain_index"], "users": ["shadow_banned", "approved", "locked"], "un_partial_stated_event_stream": ["rejection_status_changed"], "users_who_share_rooms": ["share_private"], "per_user_experimental_features": ["enabled"], } # These tables are never deleted from in normal operation [*], so we can resume porting # over rows from a previous attempt rather than starting from scratch. # # [*]: We do delete from many of these tables when purging a room, and # presumably when purging old events. So we might e.g. # # 1. Run portdb and port half of some table. # 2. Stop portdb. # 3. Purge something, deleting some of the rows we've ported over. # 4. Restart portdb. The rows deleted from sqlite are still present in postgres. # # But this isn't the end of the world: we should be able to repeat the purge # on the postgres DB when porting completes. APPEND_ONLY_TABLES = [ "cache_invalidation_stream_by_instance", "event_auth", "event_edges", "event_json", "event_reference_hashes", "event_search", "event_to_state_groups", "events", "ex_outlier_stream", "local_media_repository", "local_media_repository_thumbnails", "presence_stream", "public_room_list_stream", "push_rules_stream", "received_transactions", "redactions", "rejections", "remote_media_cache", "remote_media_cache_thumbnails", "room_memberships", "room_names", "rooms", "sent_transactions", "state_events", "state_group_edges", "state_groups", "state_groups_state", "stream_ordering_to_exterm", "topics", "transaction_id_to_pdu", "un_partial_stated_event_stream", "users", ] IGNORED_TABLES = { # We don't port these tables, as they're a faff and we can regenerate # them anyway. "user_directory", "user_directory_search", "user_directory_search_content", "user_directory_search_docsize", "user_directory_search_segdir", "user_directory_search_segments", "user_directory_search_stat", "user_directory_search_pos", "users_who_share_private_rooms", "users_in_public_room", # UI auth sessions have foreign keys so additional care needs to be taken, # the sessions are transient anyway, so ignore them. "ui_auth_sessions", "ui_auth_sessions_credentials", "ui_auth_sessions_ips", # Ignore the worker locks table, as a) there shouldn't be any acquired locks # after porting, and b) the circular foreign key constraints make it hard to # port. "worker_read_write_locks_mode", "worker_read_write_locks", } # Error returned by the run function. Used at the top-level part of the script to # handle errors and return codes. end_error: Optional[str] = None # The exec_info for the error, if any. If error is defined but not exec_info the script # will show only the error message without the stacktrace, if exec_info is defined but # not the error then the script will show nothing outside of what's printed in the run # function. If both are defined, the script will print both the error and the stacktrace. end_error_exec_info: Optional[ Tuple[Type[BaseException], BaseException, TracebackType] ] = None R = TypeVar("R") class Store( EventPushActionsStore, ClientIpBackgroundUpdateStore, DeviceInboxBackgroundUpdateStore, DeviceBackgroundUpdateStore, EventsBackgroundUpdatesStore, MediaRepositoryBackgroundUpdateStore, RegistrationBackgroundUpdateStore, RoomBackgroundUpdateStore, RoomMemberBackgroundUpdateStore, SearchBackgroundUpdateStore, StateBackgroundUpdateStore, MainStateBackgroundUpdateStore, UserDirectoryBackgroundUpdateStore, EndToEndKeyBackgroundStore, EndToEndRoomKeyBackgroundStore, StatsStore, AccountDataWorkerStore, FilteringWorkerStore, ProfileWorkerStore, PushRuleStore, PusherWorkerStore, PusherBackgroundUpdatesStore, PresenceBackgroundUpdateStore, ReceiptsBackgroundUpdateStore, RelationsWorkerStore, EventFederationWorkerStore, ): def execute(self, f: Callable[..., R], *args: Any, **kwargs: Any) -> Awaitable[R]: return self.db_pool.runInteraction(f.__name__, f, *args, **kwargs) def execute_sql(self, sql: str, *args: object) -> Awaitable[List[Tuple]]: def r(txn: LoggingTransaction) -> List[Tuple]: txn.execute(sql, args) return txn.fetchall() return self.db_pool.runInteraction("execute_sql", r) def insert_many_txn( self, txn: LoggingTransaction, table: str, headers: List[str], rows: List[Tuple] ) -> None: sql = "INSERT INTO %s (%s) VALUES (%s)" % ( table, ", ".join(k for k in headers), ", ".join("%s" for _ in headers), ) try: txn.executemany(sql, rows) except Exception: logger.exception("Failed to insert: %s", table) raise # Note: the parent method is an `async def`. def set_room_is_public(self, room_id: str, is_public: bool) -> NoReturn: raise Exception( "Attempt to set room_is_public during port_db: database not empty?" ) class MockHomeserver: def __init__(self, config: HomeServerConfig): self.clock = Clock(reactor) self.config = config self.hostname = config.server.server_name self.version_string = SYNAPSE_VERSION def get_clock(self) -> Clock: return self.clock def get_reactor(self) -> ISynapseReactor: return reactor def get_instance_name(self) -> str: return "master" def should_send_federation(self) -> bool: return False def get_replication_notifier(self) -> ReplicationNotifier: return ReplicationNotifier() class Porter: def __init__( self, sqlite_config: Dict[str, Any], progress: "Progress", batch_size: int, hs_config: HomeServerConfig, ): self.sqlite_config = sqlite_config self.progress = progress self.batch_size = batch_size self.hs_config = hs_config async def setup_table(self, table: str) -> Tuple[str, int, int, int, int]: if table in APPEND_ONLY_TABLES: # It's safe to just carry on inserting. row = await self.postgres_store.db_pool.simple_select_one( table="port_from_sqlite3", keyvalues={"table_name": table}, retcols=("forward_rowid", "backward_rowid"), allow_none=True, ) total_to_port = None if row is None: if table == "sent_transactions": ( forward_chunk, already_ported, total_to_port, ) = await self._setup_sent_transactions() backward_chunk = 0 else: await self.postgres_store.db_pool.simple_insert( table="port_from_sqlite3", values={ "table_name": table, "forward_rowid": 1, "backward_rowid": 0, }, ) forward_chunk = 1 backward_chunk = 0 already_ported = 0 else: forward_chunk = row["forward_rowid"] backward_chunk = row["backward_rowid"] if total_to_port is None: already_ported, total_to_port = await self._get_total_count_to_port( table, forward_chunk, backward_chunk ) else: def delete_all(txn: LoggingTransaction) -> None: txn.execute( "DELETE FROM port_from_sqlite3 WHERE table_name = %s", (table,) ) txn.execute("TRUNCATE %s CASCADE" % (table,)) await self.postgres_store.execute(delete_all) await self.postgres_store.db_pool.simple_insert( table="port_from_sqlite3", values={"table_name": table, "forward_rowid": 1, "backward_rowid": 0}, ) forward_chunk = 1 backward_chunk = 0 already_ported, total_to_port = await self._get_total_count_to_port( table, forward_chunk, backward_chunk ) return table, already_ported, total_to_port, forward_chunk, backward_chunk async def get_table_constraints(self) -> Dict[str, Set[str]]: """Returns a map of tables that have foreign key constraints to tables they depend on.""" def _get_constraints(txn: LoggingTransaction) -> Dict[str, Set[str]]: # We can pull the information about foreign key constraints out from # the postgres schema tables. sql = """ SELECT DISTINCT tc.table_name, ccu.table_name AS foreign_table_name FROM information_schema.table_constraints AS tc INNER JOIN information_schema.constraint_column_usage AS ccu USING (table_schema, constraint_name) WHERE tc.constraint_type = 'FOREIGN KEY' AND tc.table_name != ccu.table_name; """ txn.execute(sql) results: Dict[str, Set[str]] = {} for table, foreign_table in txn: results.setdefault(table, set()).add(foreign_table) return results return await self.postgres_store.db_pool.runInteraction( "get_table_constraints", _get_constraints ) async def handle_table( self, table: str, postgres_size: int, table_size: int, forward_chunk: int, backward_chunk: int, ) -> None: logger.info( "Table %s: %i/%i (rows %i-%i) already ported", table, postgres_size, table_size, backward_chunk + 1, forward_chunk - 1, ) if not table_size: return self.progress.add_table(table, postgres_size, table_size) if table == "event_search": await self.handle_search_table( postgres_size, table_size, forward_chunk, backward_chunk ) return if table in IGNORED_TABLES: self.progress.update(table, table_size) # Mark table as done return if table == "user_directory_stream_pos": # We need to make sure there is a single row, `(X, null), as that is # what synapse expects to be there. await self.postgres_store.db_pool.simple_insert( table=table, values={"stream_id": None} ) self.progress.update(table, table_size) # Mark table as done return # We sweep over rowids in two directions: one forwards (rowids 1, 2, 3, ...) # and another backwards (rowids 0, -1, -2, ...). forward_select = ( "SELECT rowid, * FROM %s WHERE rowid >= ? ORDER BY rowid LIMIT ?" % (table,) ) backward_select = ( "SELECT rowid, * FROM %s WHERE rowid <= ? ORDER BY rowid DESC LIMIT ?" % (table,) ) do_forward = [True] do_backward = [True] while True: def r( txn: LoggingTransaction, ) -> Tuple[Optional[List[str]], List[Tuple], List[Tuple]]: forward_rows = [] backward_rows = [] if do_forward[0]: txn.execute(forward_select, (forward_chunk, self.batch_size)) forward_rows = txn.fetchall() if not forward_rows: do_forward[0] = False if do_backward[0]: txn.execute(backward_select, (backward_chunk, self.batch_size)) backward_rows = txn.fetchall() if not backward_rows: do_backward[0] = False if forward_rows or backward_rows: assert txn.description is not None headers: Optional[List[str]] = [ column[0] for column in txn.description ] else: headers = None return headers, forward_rows, backward_rows headers, frows, brows = await self.sqlite_store.db_pool.runInteraction( "select", r ) if frows or brows: assert headers is not None if frows: forward_chunk = max(row[0] for row in frows) + 1 if brows: backward_chunk = min(row[0] for row in brows) - 1 rows = frows + brows rows = self._convert_rows(table, headers, rows) def insert(txn: LoggingTransaction) -> None: assert headers is not None self.postgres_store.insert_many_txn(txn, table, headers[1:], rows) self.postgres_store.db_pool.simple_update_one_txn( txn, table="port_from_sqlite3", keyvalues={"table_name": table}, updatevalues={ "forward_rowid": forward_chunk, "backward_rowid": backward_chunk, }, ) await self.postgres_store.execute(insert) postgres_size += len(rows) self.progress.update(table, postgres_size) else: return async def handle_search_table( self, postgres_size: int, table_size: int, forward_chunk: int, backward_chunk: int, ) -> None: select = ( "SELECT es.rowid, es.*, e.origin_server_ts, e.stream_ordering" " FROM event_search as es" " INNER JOIN events AS e USING (event_id, room_id)" " WHERE es.rowid >= ?" " ORDER BY es.rowid LIMIT ?" ) while True: def r(txn: LoggingTransaction) -> Tuple[List[str], List[Tuple]]: txn.execute(select, (forward_chunk, self.batch_size)) rows = txn.fetchall() assert txn.description is not None headers = [column[0] for column in txn.description] return headers, rows headers, rows = await self.sqlite_store.db_pool.runInteraction("select", r) if rows: forward_chunk = rows[-1][0] + 1 # We have to treat event_search differently since it has a # different structure in the two different databases. def insert(txn: LoggingTransaction) -> None: sql = ( "INSERT INTO event_search (event_id, room_id, key," " sender, vector, origin_server_ts, stream_ordering)" " VALUES (?,?,?,?,to_tsvector('english', ?),?,?)" ) rows_dict = [] for row in rows: d = dict(zip(headers, row)) if "\0" in d["value"]: logger.warning("dropping search row %s", d) else: rows_dict.append(d) txn.executemany( sql, [ ( row["event_id"], row["room_id"], row["key"], row["sender"], row["value"], row["origin_server_ts"], row["stream_ordering"], ) for row in rows_dict ], ) self.postgres_store.db_pool.simple_update_one_txn( txn, table="port_from_sqlite3", keyvalues={"table_name": "event_search"}, updatevalues={ "forward_rowid": forward_chunk, "backward_rowid": backward_chunk, }, ) await self.postgres_store.execute(insert) postgres_size += len(rows) self.progress.update("event_search", postgres_size) else: return def build_db_store( self, db_config: DatabaseConnectionConfig, allow_outdated_version: bool = False, ) -> Store: """Builds and returns a database store using the provided configuration. Args: db_config: The database configuration allow_outdated_version: True to suppress errors about the database server version being too old to run a complete synapse Returns: The built Store object. """ self.progress.set_state("Preparing %s" % db_config.config["name"]) engine = create_engine(db_config.config) hs = MockHomeserver(self.hs_config) with make_conn(db_config, engine, "portdb") as db_conn: engine.check_database( db_conn, allow_outdated_version=allow_outdated_version ) prepare_database(db_conn, engine, config=self.hs_config) # Type safety: ignore that we're using Mock homeservers here. store = Store(DatabasePool(hs, db_config, engine), db_conn, hs) # type: ignore[arg-type] db_conn.commit() return store async def run_background_updates_on_postgres(self) -> None: # Manually apply all background updates on the PostgreSQL database. postgres_ready = ( await self.postgres_store.db_pool.updates.has_completed_background_updates() ) if not postgres_ready: # Only say that we're running background updates when there are background # updates to run. self.progress.set_state("Running background updates on PostgreSQL") while not postgres_ready: await self.postgres_store.db_pool.updates.do_next_background_update(True) postgres_ready = await ( self.postgres_store.db_pool.updates.has_completed_background_updates() ) @staticmethod def _is_sqlite_autovacuum_enabled(txn: LoggingTransaction) -> bool: """ Returns true if auto_vacuum is enabled in SQLite. https://www.sqlite.org/pragma.html#pragma_auto_vacuum Vacuuming changes the rowids on rows in the database. Auto-vacuuming is therefore dangerous when used in conjunction with this script. Note that the auto_vacuum setting can't be changed without performing a VACUUM after trying to change the pragma. """ txn.execute("PRAGMA auto_vacuum") row = txn.fetchone() assert row is not None, "`PRAGMA auto_vacuum` did not give a row." (autovacuum_setting,) = row # 0 means off. 1 means full. 2 means incremental. return autovacuum_setting != 0 async def run(self) -> None: """Ports the SQLite database to a PostgreSQL database. When a fatal error is met, its message is assigned to the global "end_error" variable. When this error comes with a stacktrace, its exec_info is assigned to the global "end_error_exec_info" variable. """ global end_error try: # we allow people to port away from outdated versions of sqlite. self.sqlite_store = self.build_db_store( DatabaseConnectionConfig("master-sqlite", self.sqlite_config), allow_outdated_version=True, ) # For safety, ensure auto_vacuums are disabled. if await self.sqlite_store.db_pool.runInteraction( "is_sqlite_autovacuum_enabled", self._is_sqlite_autovacuum_enabled ): end_error = ( "auto_vacuum is enabled in the SQLite database." " (This is not the default configuration.)\n" " This script relies on rowids being consistent and must not" " be used if the database could be vacuumed between re-runs.\n" " To disable auto_vacuum, you need to stop Synapse and run the following SQL:\n" " PRAGMA auto_vacuum=off;\n" " VACUUM;" ) return # Check if all background updates are done, abort if not. updates_complete = ( await self.sqlite_store.db_pool.updates.has_completed_background_updates() ) if not updates_complete: end_error = ( "Pending background updates exist in the SQLite3 database." " Please start Synapse again and wait until every update has finished" " before running this script.\n" ) return self.postgres_store = self.build_db_store( self.hs_config.database.get_single_database() ) await self.run_background_updates_on_postgres() self.progress.set_state("Creating port tables") def create_port_table(txn: LoggingTransaction) -> None: txn.execute( "CREATE TABLE IF NOT EXISTS port_from_sqlite3 (" " table_name varchar(100) NOT NULL UNIQUE," " forward_rowid bigint NOT NULL," " backward_rowid bigint NOT NULL" ")" ) # The old port script created a table with just a "rowid" column. # We want people to be able to rerun this script from an old port # so that they can pick up any missing events that were not # ported across. def alter_table(txn: LoggingTransaction) -> None: txn.execute( "ALTER TABLE IF EXISTS port_from_sqlite3" " RENAME rowid TO forward_rowid" ) txn.execute( "ALTER TABLE IF EXISTS port_from_sqlite3" " ADD backward_rowid bigint NOT NULL DEFAULT 0" ) try: await self.postgres_store.db_pool.runInteraction( "alter_table", alter_table ) except Exception: # On Error Resume Next pass await self.postgres_store.db_pool.runInteraction( "create_port_table", create_port_table ) # Step 2. Set up sequences # # We do this before porting the tables so that even if we fail half # way through the postgres DB always have sequences that are greater # than their respective tables. If we don't then creating the # `DataStore` object will fail due to the inconsistency. self.progress.set_state("Setting up sequence generators") await self._setup_state_group_id_seq() await self._setup_user_id_seq() await self._setup_events_stream_seqs() await self._setup_sequence( "un_partial_stated_event_stream_sequence", ("un_partial_stated_event_stream",), ) await self._setup_sequence( "device_inbox_sequence", ("device_inbox", "device_federation_outbox") ) await self._setup_sequence( "account_data_sequence", ("room_account_data", "room_tags_revisions", "account_data"), ) await self._setup_sequence("receipts_sequence", ("receipts_linearized",)) await self._setup_sequence("presence_stream_sequence", ("presence_stream",)) await self._setup_auth_chain_sequence() await self._setup_sequence( "application_services_txn_id_seq", ("application_services_txns",), "txn_id", ) # Step 3. Get tables. self.progress.set_state("Fetching tables") sqlite_tables = await self.sqlite_store.db_pool.simple_select_onecol( table="sqlite_master", keyvalues={"type": "table"}, retcol="name" ) postgres_tables = await self.postgres_store.db_pool.simple_select_onecol( table="information_schema.tables", keyvalues={}, retcol="distinct table_name", ) tables = set(sqlite_tables) & set(postgres_tables) logger.info("Found %d tables", len(tables)) # Step 4. Figure out what still needs copying self.progress.set_state("Checking on port progress") setup_res = await make_deferred_yieldable( defer.gatherResults( [ run_in_background(self.setup_table, table) for table in tables if table not in ["schema_version", "applied_schema_deltas"] and not table.startswith("sqlite_") ], consumeErrors=True, ) ) # Map from table name to args passed to `handle_table`, i.e. a tuple # of: `postgres_size`, `table_size`, `forward_chunk`, `backward_chunk`. tables_to_port_info_map = { r[0]: r[1:] for r in setup_res if r[0] not in IGNORED_TABLES } # Step 5. Do the copying. # # This is slightly convoluted as we need to ensure tables are ported # in the correct order due to foreign key constraints. self.progress.set_state("Copying to postgres") constraints = await self.get_table_constraints() tables_ported = set() # type: Set[str] while tables_to_port_info_map: # Pulls out all tables that are still to be ported and which # only depend on tables that are already ported (if any). tables_to_port = [ table for table in tables_to_port_info_map if not constraints.get(table, set()) - tables_ported ] await make_deferred_yieldable( defer.gatherResults( [ run_in_background( self.handle_table, table, *tables_to_port_info_map.pop(table), ) for table in tables_to_port ], consumeErrors=True, ) ) tables_ported.update(tables_to_port) self.progress.done() except Exception as e: global end_error_exec_info end_error = str(e) # Type safety: we're in an exception handler, so the exc_info() tuple # will not be (None, None, None). end_error_exec_info = sys.exc_info() # type: ignore[assignment] logger.exception("") finally: reactor.stop() def _convert_rows( self, table: str, headers: List[str], rows: List[Tuple] ) -> List[Tuple]: bool_col_names = BOOLEAN_COLUMNS.get(table, []) bool_cols = [i for i, h in enumerate(headers) if h in bool_col_names] class BadValueException(Exception): pass def conv(j: int, col: object) -> object: if j in bool_cols: return bool(col) if isinstance(col, bytes): return bytearray(col) elif isinstance(col, str) and "\0" in col: logger.warning( "DROPPING ROW: NUL value in table %s col %s: %r", table, headers[j], col, ) raise BadValueException() return col outrows = [] for row in rows: try: outrows.append( tuple(conv(j, col) for j, col in enumerate(row) if j > 0) ) except BadValueException: pass return outrows async def _setup_sent_transactions(self) -> Tuple[int, int, int]: # Only save things from the last day yesterday = int(time.time() * 1000) - 86400000 # And save the max transaction id from each destination select = ( "SELECT rowid, * FROM sent_transactions WHERE rowid IN (" "SELECT max(rowid) FROM sent_transactions" " GROUP BY destination" ")" ) def r(txn: LoggingTransaction) -> Tuple[List[str], List[Tuple]]: txn.execute(select) rows = txn.fetchall() assert txn.description is not None headers = [column[0] for column in txn.description] ts_ind = headers.index("ts") return headers, [r for r in rows if r[ts_ind] < yesterday] headers, rows = await self.sqlite_store.db_pool.runInteraction("select", r) rows = self._convert_rows("sent_transactions", headers, rows) inserted_rows = len(rows) if inserted_rows: max_inserted_rowid = max(r[0] for r in rows) def insert(txn: LoggingTransaction) -> None: self.postgres_store.insert_many_txn( txn, "sent_transactions", headers[1:], rows ) await self.postgres_store.execute(insert) else: max_inserted_rowid = 0 def get_start_id(txn: LoggingTransaction) -> int: txn.execute( "SELECT rowid FROM sent_transactions WHERE ts >= ?" " ORDER BY rowid ASC LIMIT 1", (yesterday,), ) rows = txn.fetchall() if rows: return rows[0][0] else: return 1 next_chunk = await self.sqlite_store.execute(get_start_id) next_chunk = max(max_inserted_rowid + 1, next_chunk) await self.postgres_store.db_pool.simple_insert( table="port_from_sqlite3", values={ "table_name": "sent_transactions", "forward_rowid": next_chunk, "backward_rowid": 0, }, ) def get_sent_table_size(txn: LoggingTransaction) -> int: txn.execute( "SELECT count(*) FROM sent_transactions" " WHERE ts >= ?", (yesterday,) ) result = txn.fetchone() assert result is not None return int(result[0]) remaining_count = await self.sqlite_store.execute(get_sent_table_size) total_count = remaining_count + inserted_rows return next_chunk, inserted_rows, total_count async def _get_remaining_count_to_port( self, table: str, forward_chunk: int, backward_chunk: int ) -> int: frows = cast( List[Tuple[int]], await self.sqlite_store.execute_sql( "SELECT count(*) FROM %s WHERE rowid >= ?" % (table,), forward_chunk ), ) brows = cast( List[Tuple[int]], await self.sqlite_store.execute_sql( "SELECT count(*) FROM %s WHERE rowid <= ?" % (table,), backward_chunk ), ) return frows[0][0] + brows[0][0] async def _get_already_ported_count(self, table: str) -> int: rows = await self.postgres_store.execute_sql( "SELECT count(*) FROM %s" % (table,) ) return rows[0][0] async def _get_total_count_to_port( self, table: str, forward_chunk: int, backward_chunk: int ) -> Tuple[int, int]: remaining, done = await make_deferred_yieldable( defer.gatherResults( [ run_in_background( self._get_remaining_count_to_port, table, forward_chunk, backward_chunk, ), run_in_background(self._get_already_ported_count, table), ], ) ) remaining = int(remaining) if remaining else 0 done = int(done) if done else 0 return done, remaining + done async def _setup_state_group_id_seq(self) -> None: curr_id: Optional[ int ] = await self.sqlite_store.db_pool.simple_select_one_onecol( table="state_groups", keyvalues={}, retcol="MAX(id)", allow_none=True ) if not curr_id: return def r(txn: LoggingTransaction) -> None: assert curr_id is not None next_id = curr_id + 1 txn.execute("ALTER SEQUENCE state_group_id_seq RESTART WITH %s", (next_id,)) await self.postgres_store.db_pool.runInteraction("setup_state_group_id_seq", r) async def _setup_user_id_seq(self) -> None: curr_id = await self.sqlite_store.db_pool.runInteraction( "setup_user_id_seq", find_max_generated_user_id_localpart ) def r(txn: LoggingTransaction) -> None: next_id = curr_id + 1 txn.execute("ALTER SEQUENCE user_id_seq RESTART WITH %s", (next_id,)) await self.postgres_store.db_pool.runInteraction("setup_user_id_seq", r) async def _setup_events_stream_seqs(self) -> None: """Set the event stream sequences to the correct values.""" # We get called before we've ported the events table, so we need to # fetch the current positions from the SQLite store. curr_forward_id = await self.sqlite_store.db_pool.simple_select_one_onecol( table="events", keyvalues={}, retcol="MAX(stream_ordering)", allow_none=True ) curr_backward_id = await self.sqlite_store.db_pool.simple_select_one_onecol( table="events", keyvalues={}, retcol="MAX(-MIN(stream_ordering), 1)", allow_none=True, ) def _setup_events_stream_seqs_set_pos(txn: LoggingTransaction) -> None: if curr_forward_id: txn.execute( "ALTER SEQUENCE events_stream_seq RESTART WITH %s", (curr_forward_id + 1,), ) if curr_backward_id: txn.execute( "ALTER SEQUENCE events_backfill_stream_seq RESTART WITH %s", (curr_backward_id + 1,), ) await self.postgres_store.db_pool.runInteraction( "_setup_events_stream_seqs", _setup_events_stream_seqs_set_pos, ) async def _setup_sequence( self, sequence_name: str, stream_id_tables: Iterable[str], column_name: str = "stream_id", ) -> None: """Set a sequence to the correct value.""" current_stream_ids = [] for stream_id_table in stream_id_tables: max_stream_id = cast( int, await self.sqlite_store.db_pool.simple_select_one_onecol( table=stream_id_table, keyvalues={}, retcol=f"COALESCE(MAX({column_name}), 1)", allow_none=True, ), ) current_stream_ids.append(max_stream_id) next_id = max(current_stream_ids) + 1 def r(txn: LoggingTransaction) -> None: sql = "ALTER SEQUENCE %s RESTART WITH" % (sequence_name,) txn.execute(sql + " %s", (next_id,)) await self.postgres_store.db_pool.runInteraction( "_setup_%s" % (sequence_name,), r ) async def _setup_auth_chain_sequence(self) -> None: curr_chain_id: Optional[ int ] = await self.sqlite_store.db_pool.simple_select_one_onecol( table="event_auth_chains", keyvalues={}, retcol="MAX(chain_id)", allow_none=True, ) def r(txn: LoggingTransaction) -> None: # Presumably there is at least one row in event_auth_chains. assert curr_chain_id is not None txn.execute( "ALTER SEQUENCE event_auth_chain_id RESTART WITH %s", (curr_chain_id + 1,), ) if curr_chain_id is not None: await self.postgres_store.db_pool.runInteraction( "_setup_event_auth_chain_id", r, ) ############################################## # The following is simply UI stuff ############################################## class TableProgress(TypedDict): start: int num_done: int total: int perc: int class Progress: """Used to report progress of the port""" def __init__(self) -> None: self.tables: Dict[str, TableProgress] = {} self.start_time = int(time.time()) def add_table(self, table: str, cur: int, size: int) -> None: self.tables[table] = { "start": cur, "num_done": cur, "total": size, "perc": int(cur * 100 / size), } def update(self, table: str, num_done: int) -> None: data = self.tables[table] data["num_done"] = num_done data["perc"] = int(num_done * 100 / data["total"]) def done(self) -> None: pass def set_state(self, state: str) -> None: pass class CursesProgress(Progress): """Reports progress to a curses window""" def __init__(self, stdscr: "curses.window"): self.stdscr = stdscr curses.use_default_colors() curses.curs_set(0) curses.init_pair(1, curses.COLOR_RED, -1) curses.init_pair(2, curses.COLOR_GREEN, -1) self.last_update = 0.0 self.finished = False self.total_processed = 0 self.total_remaining = 0 super().__init__() def update(self, table: str, num_done: int) -> None: super().update(table, num_done) self.total_processed = 0 self.total_remaining = 0 for data in self.tables.values(): self.total_processed += data["num_done"] - data["start"] self.total_remaining += data["total"] - data["num_done"] self.render() def render(self, force: bool = False) -> None: now = time.time() if not force and now - self.last_update < 0.2: # reactor.callLater(1, self.render) return self.stdscr.clear() rows, cols = self.stdscr.getmaxyx() duration = int(now) - int(self.start_time) minutes, seconds = divmod(duration, 60) duration_str = "%02dm %02ds" % (minutes, seconds) if self.finished: status = "Time spent: %s (Done!)" % (duration_str,) else: if self.total_processed > 0: left = float(self.total_remaining) / self.total_processed est_remaining = (int(now) - self.start_time) * left est_remaining_str = "%02dm %02ds remaining" % divmod(est_remaining, 60) else: est_remaining_str = "Unknown" status = "Time spent: %s (est. remaining: %s)" % ( duration_str, est_remaining_str, ) self.stdscr.addstr(0, 0, status, curses.A_BOLD) max_len = max(len(t) for t in self.tables.keys()) left_margin = 5 middle_space = 1 items = sorted(self.tables.items(), key=lambda i: (i[1]["perc"], i[0])) for i, (table, data) in enumerate(items): if i + 2 >= rows: break perc = data["perc"] color = curses.color_pair(2) if perc == 100 else curses.color_pair(1) self.stdscr.addstr( i + 2, left_margin + max_len - len(table), table, curses.A_BOLD | color ) size = 20 progress = "[%s%s]" % ( "#" * int(perc * size / 100), " " * (size - int(perc * size / 100)), ) self.stdscr.addstr( i + 2, left_margin + max_len + middle_space, "%s %3d%% (%d/%d)" % (progress, perc, data["num_done"], data["total"]), ) if self.finished: self.stdscr.addstr(rows - 1, 0, "Press any key to exit...") self.stdscr.refresh() self.last_update = time.time() def done(self) -> None: self.finished = True self.render(True) self.stdscr.getch() def set_state(self, state: str) -> None: self.stdscr.clear() self.stdscr.addstr(0, 0, state + "...", curses.A_BOLD) self.stdscr.refresh() class TerminalProgress(Progress): """Just prints progress to the terminal""" def update(self, table: str, num_done: int) -> None: super().update(table, num_done) data = self.tables[table] print( "%s: %d%% (%d/%d)" % (table, data["perc"], data["num_done"], data["total"]) ) def set_state(self, state: str) -> None: print(state + "...") ############################################## ############################################## def main() -> None: parser = argparse.ArgumentParser( description="A script to port an existing synapse SQLite database to" " a new PostgreSQL database." ) parser.add_argument("-v", action="store_true") parser.add_argument( "--sqlite-database", required=True, help="The snapshot of the SQLite database file. This must not be" " currently used by a running synapse server", ) parser.add_argument( "--postgres-config", type=argparse.FileType("r"), required=True, help="The database config file for the PostgreSQL database", ) parser.add_argument( "--curses", action="store_true", help="display a curses based progress UI" ) parser.add_argument( "--batch-size", type=int, default=1000, help="The number of rows to select from the SQLite table each" " iteration [default=1000]", ) args = parser.parse_args() logging.basicConfig( level=logging.DEBUG if args.v else logging.INFO, format="%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(message)s", filename="port-synapse.log" if args.curses else None, ) if not os.path.isfile(args.sqlite_database): sys.stderr.write( "The sqlite database you specified does not exist, please check that you have the" "correct path." ) sys.exit(1) sqlite_config = { "name": "sqlite3", "args": { "database": args.sqlite_database, "cp_min": 1, "cp_max": 1, "check_same_thread": False, }, } hs_config = yaml.safe_load(args.postgres_config) if "database" not in hs_config: sys.stderr.write("The configuration file must have a 'database' section.\n") sys.exit(4) postgres_config = hs_config["database"] if "name" not in postgres_config: sys.stderr.write("Malformed database config: no 'name'\n") sys.exit(2) if postgres_config["name"] != "psycopg2": sys.stderr.write("Database must use the 'psycopg2' connector.\n") sys.exit(3) # Don't run the background tasks that get started by the data stores. hs_config["run_background_tasks_on"] = "some_other_process" config = HomeServerConfig() config.parse_config_dict(hs_config, "", "") def start(stdscr: Optional["curses.window"] = None) -> None: progress: Progress if stdscr: progress = CursesProgress(stdscr) else: progress = TerminalProgress() porter = Porter( sqlite_config=sqlite_config, progress=progress, batch_size=args.batch_size, hs_config=config, ) @defer.inlineCallbacks def run() -> Generator["defer.Deferred[Any]", Any, None]: with LoggingContext("synapse_port_db_run"): yield defer.ensureDeferred(porter.run()) reactor.callWhenRunning(run) reactor.run() if args.curses: curses.wrapper(start) else: start() if end_error: if end_error_exec_info: exc_type, exc_value, exc_traceback = end_error_exec_info traceback.print_exception(exc_type, exc_value, exc_traceback) sys.stderr.write(end_error) sys.exit(5) if __name__ == "__main__": main()