|
@@ -37,6 +37,7 @@ from synapse.metrics.background_process_metrics import run_as_background_process
|
|
|
from synapse.storage.background_updates import BackgroundUpdateStore
|
|
|
from synapse.storage.event_federation import EventFederationStore
|
|
|
from synapse.storage.events_worker import EventsWorkerStore
|
|
|
+from synapse.storage.state import StateGroupWorkerStore
|
|
|
from synapse.types import RoomStreamToken, get_domain_from_id
|
|
|
from synapse.util.async_helpers import ObservableDeferred
|
|
|
from synapse.util.caches.descriptors import cached, cachedInlineCallbacks
|
|
@@ -203,7 +204,8 @@ def _retry_on_integrity_error(func):
|
|
|
|
|
|
# inherits from EventFederationStore so that we can call _update_backward_extremities
|
|
|
# and _handle_mult_prev_events (though arguably those could both be moved in here)
|
|
|
-class EventsStore(EventFederationStore, EventsWorkerStore, BackgroundUpdateStore):
|
|
|
+class EventsStore(StateGroupWorkerStore, EventFederationStore, EventsWorkerStore,
|
|
|
+ BackgroundUpdateStore):
|
|
|
EVENT_ORIGIN_SERVER_TS_NAME = "event_origin_server_ts"
|
|
|
EVENT_FIELDS_SENDER_URL_UPDATE_NAME = "event_fields_sender_url"
|
|
|
|
|
@@ -1995,70 +1997,29 @@ class EventsStore(EventFederationStore, EventsWorkerStore, BackgroundUpdateStore
|
|
|
|
|
|
logger.info("[purge] finding redundant state groups")
|
|
|
|
|
|
- # Get all state groups that are only referenced by events that are
|
|
|
- # to be deleted.
|
|
|
- # This works by first getting state groups that we may want to delete,
|
|
|
- # joining against event_to_state_groups to get events that use that
|
|
|
- # state group, then left joining against events_to_purge again. Any
|
|
|
- # state group where the left join produce *no nulls* are referenced
|
|
|
- # only by events that are going to be purged.
|
|
|
+ # Get all state groups that are referenced by events that are to be
|
|
|
+ # deleted. We then go and check if they are referenced by other events
|
|
|
+ # or state groups, and if not we delete them.
|
|
|
txn.execute("""
|
|
|
- SELECT state_group FROM
|
|
|
- (
|
|
|
- SELECT DISTINCT state_group FROM events_to_purge
|
|
|
- INNER JOIN event_to_state_groups USING (event_id)
|
|
|
- ) AS sp
|
|
|
- INNER JOIN event_to_state_groups USING (state_group)
|
|
|
- LEFT JOIN events_to_purge AS ep USING (event_id)
|
|
|
- GROUP BY state_group
|
|
|
- HAVING SUM(CASE WHEN ep.event_id IS NULL THEN 1 ELSE 0 END) = 0
|
|
|
+ SELECT DISTINCT state_group FROM events_to_purge
|
|
|
+ INNER JOIN event_to_state_groups USING (event_id)
|
|
|
""")
|
|
|
|
|
|
- state_rows = txn.fetchall()
|
|
|
- logger.info("[purge] found %i redundant state groups", len(state_rows))
|
|
|
-
|
|
|
- # make a set of the redundant state groups, so that we can look them up
|
|
|
- # efficiently
|
|
|
- state_groups_to_delete = set([sg for sg, in state_rows])
|
|
|
-
|
|
|
- # Now we get all the state groups that rely on these state groups
|
|
|
- logger.info("[purge] finding state groups which depend on redundant"
|
|
|
- " state groups")
|
|
|
- remaining_state_groups = []
|
|
|
- unreferenced_state_groups = 0
|
|
|
- for i in range(0, len(state_rows), 100):
|
|
|
- chunk = [sg for sg, in state_rows[i:i + 100]]
|
|
|
- # look for state groups whose prev_state_group is one we are about
|
|
|
- # to delete
|
|
|
- rows = self._simple_select_many_txn(
|
|
|
- txn,
|
|
|
- table="state_group_edges",
|
|
|
- column="prev_state_group",
|
|
|
- iterable=chunk,
|
|
|
- retcols=["state_group"],
|
|
|
- keyvalues={},
|
|
|
- )
|
|
|
-
|
|
|
- for row in rows:
|
|
|
- sg = row["state_group"]
|
|
|
-
|
|
|
- if sg in state_groups_to_delete:
|
|
|
- # exclude state groups we are about to delete: no point in
|
|
|
- # updating them
|
|
|
- continue
|
|
|
+ referenced_state_groups = set(sg for sg, in txn)
|
|
|
+ logger.info(
|
|
|
+ "[purge] found %i referenced state groups",
|
|
|
+ len(referenced_state_groups),
|
|
|
+ )
|
|
|
|
|
|
- if not self._is_state_group_referenced(txn, sg):
|
|
|
- # Let's also delete unreferenced state groups while we're
|
|
|
- # here, since otherwise we'd need to de-delta them
|
|
|
- state_groups_to_delete.add(sg)
|
|
|
- unreferenced_state_groups += 1
|
|
|
- continue
|
|
|
+ logger.info("[purge] finding state groups that can be deleted")
|
|
|
|
|
|
- remaining_state_groups.append(sg)
|
|
|
+ state_groups_to_delete, remaining_state_groups = self._find_unreferenced_groups(
|
|
|
+ txn, referenced_state_groups,
|
|
|
+ )
|
|
|
|
|
|
logger.info(
|
|
|
- "[purge] found %i extra unreferenced state groups to delete",
|
|
|
- unreferenced_state_groups,
|
|
|
+ "[purge] found %i state groups to delete",
|
|
|
+ len(state_groups_to_delete),
|
|
|
)
|
|
|
|
|
|
logger.info(
|
|
@@ -2109,11 +2070,11 @@ class EventsStore(EventFederationStore, EventsWorkerStore, BackgroundUpdateStore
|
|
|
logger.info("[purge] removing redundant state groups")
|
|
|
txn.executemany(
|
|
|
"DELETE FROM state_groups_state WHERE state_group = ?",
|
|
|
- state_rows
|
|
|
+ ((sg,) for sg in state_groups_to_delete),
|
|
|
)
|
|
|
txn.executemany(
|
|
|
"DELETE FROM state_groups WHERE id = ?",
|
|
|
- state_rows
|
|
|
+ ((sg,) for sg in state_groups_to_delete),
|
|
|
)
|
|
|
|
|
|
logger.info("[purge] removing events from event_to_state_groups")
|