test_cleanup_extrems.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389
  1. # Copyright 2019 The Matrix.org Foundation C.I.C.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. import os.path
  15. from unittest.mock import Mock, patch
  16. from twisted.test.proto_helpers import MemoryReactor
  17. import synapse.rest.admin
  18. from synapse.api.constants import EventTypes
  19. from synapse.rest.client import login, room
  20. from synapse.server import HomeServer
  21. from synapse.storage import prepare_database
  22. from synapse.storage.types import Cursor
  23. from synapse.types import UserID, create_requester
  24. from synapse.util import Clock
  25. from tests.unittest import HomeserverTestCase
  26. class CleanupExtremBackgroundUpdateStoreTestCase(HomeserverTestCase):
  27. """
  28. Test the background update to clean forward extremities table.
  29. """
  30. def prepare(
  31. self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer
  32. ) -> None:
  33. self.store = homeserver.get_datastores().main
  34. self.room_creator = homeserver.get_room_creation_handler()
  35. # Create a test user and room
  36. self.user = UserID("alice", "test")
  37. self.requester = create_requester(self.user)
  38. info, _ = self.get_success(self.room_creator.create_room(self.requester, {}))
  39. self.room_id = info["room_id"]
  40. def run_background_update(self) -> None:
  41. """Re run the background update to clean up the extremities."""
  42. # Make sure we don't clash with in progress updates.
  43. self.assertTrue(
  44. self.store.db_pool.updates._all_done, "Background updates are still ongoing"
  45. )
  46. schema_path = os.path.join(
  47. prepare_database.schema_path,
  48. "main",
  49. "delta",
  50. "54",
  51. "delete_forward_extremities.sql",
  52. )
  53. def run_delta_file(txn: Cursor) -> None:
  54. prepare_database.executescript(txn, schema_path)
  55. self.get_success(
  56. self.store.db_pool.runInteraction(
  57. "test_delete_forward_extremities", run_delta_file
  58. )
  59. )
  60. # Ugh, have to reset this flag
  61. self.store.db_pool.updates._all_done = False
  62. self.wait_for_background_updates()
  63. def add_extremity(self, room_id: str, event_id: str) -> None:
  64. """
  65. Add the given event as an extremity to the room.
  66. """
  67. self.get_success(
  68. self.hs.get_datastores().main.db_pool.simple_insert(
  69. table="event_forward_extremities",
  70. values={"room_id": room_id, "event_id": event_id},
  71. desc="test_add_extremity",
  72. )
  73. )
  74. self.hs.get_datastores().main.get_latest_event_ids_in_room.invalidate(
  75. (room_id,)
  76. )
  77. def test_soft_failed_extremities_handled_correctly(self) -> None:
  78. """Test that extremities are correctly calculated in the presence of
  79. soft failed events.
  80. Tests a graph like:
  81. A <- SF1 <- SF2 <- B
  82. Where SF* are soft failed.
  83. """
  84. # Create the room graph
  85. event_id_1 = self.create_and_send_event(self.room_id, self.user)
  86. event_id_2 = self.create_and_send_event(
  87. self.room_id, self.user, True, [event_id_1]
  88. )
  89. event_id_3 = self.create_and_send_event(
  90. self.room_id, self.user, True, [event_id_2]
  91. )
  92. event_id_4 = self.create_and_send_event(
  93. self.room_id, self.user, False, [event_id_3]
  94. )
  95. # Check the latest events are as expected
  96. latest_event_ids = self.get_success(
  97. self.store.get_latest_event_ids_in_room(self.room_id)
  98. )
  99. self.assertEqual(latest_event_ids, [event_id_4])
  100. def test_basic_cleanup(self) -> None:
  101. """Test that extremities are correctly calculated in the presence of
  102. soft failed events.
  103. Tests a graph like:
  104. A <- SF1 <- B
  105. Where SF* are soft failed, and with extremities of A and B
  106. """
  107. # Create the room graph
  108. event_id_a = self.create_and_send_event(self.room_id, self.user)
  109. event_id_sf1 = self.create_and_send_event(
  110. self.room_id, self.user, True, [event_id_a]
  111. )
  112. event_id_b = self.create_and_send_event(
  113. self.room_id, self.user, False, [event_id_sf1]
  114. )
  115. # Add the new extremity and check the latest events are as expected
  116. self.add_extremity(self.room_id, event_id_a)
  117. latest_event_ids = self.get_success(
  118. self.store.get_latest_event_ids_in_room(self.room_id)
  119. )
  120. self.assertEqual(set(latest_event_ids), {event_id_a, event_id_b})
  121. # Run the background update and check it did the right thing
  122. self.run_background_update()
  123. latest_event_ids = self.get_success(
  124. self.store.get_latest_event_ids_in_room(self.room_id)
  125. )
  126. self.assertEqual(latest_event_ids, [event_id_b])
  127. def test_chain_of_fail_cleanup(self) -> None:
  128. """Test that extremities are correctly calculated in the presence of
  129. soft failed events.
  130. Tests a graph like:
  131. A <- SF1 <- SF2 <- B
  132. Where SF* are soft failed, and with extremities of A and B
  133. """
  134. # Create the room graph
  135. event_id_a = self.create_and_send_event(self.room_id, self.user)
  136. event_id_sf1 = self.create_and_send_event(
  137. self.room_id, self.user, True, [event_id_a]
  138. )
  139. event_id_sf2 = self.create_and_send_event(
  140. self.room_id, self.user, True, [event_id_sf1]
  141. )
  142. event_id_b = self.create_and_send_event(
  143. self.room_id, self.user, False, [event_id_sf2]
  144. )
  145. # Add the new extremity and check the latest events are as expected
  146. self.add_extremity(self.room_id, event_id_a)
  147. latest_event_ids = self.get_success(
  148. self.store.get_latest_event_ids_in_room(self.room_id)
  149. )
  150. self.assertEqual(set(latest_event_ids), {event_id_a, event_id_b})
  151. # Run the background update and check it did the right thing
  152. self.run_background_update()
  153. latest_event_ids = self.get_success(
  154. self.store.get_latest_event_ids_in_room(self.room_id)
  155. )
  156. self.assertEqual(latest_event_ids, [event_id_b])
  157. def test_forked_graph_cleanup(self) -> None:
  158. r"""Test that extremities are correctly calculated in the presence of
  159. soft failed events.
  160. Tests a graph like, where time flows down the page:
  161. A B
  162. / \ /
  163. / \ /
  164. SF1 SF2
  165. | |
  166. SF3 |
  167. / \ |
  168. | \ |
  169. C SF4
  170. Where SF* are soft failed, and with them A, B and C marked as
  171. extremities. This should resolve to B and C being marked as extremity.
  172. """
  173. # Create the room graph
  174. event_id_a = self.create_and_send_event(self.room_id, self.user)
  175. event_id_b = self.create_and_send_event(self.room_id, self.user)
  176. event_id_sf1 = self.create_and_send_event(
  177. self.room_id, self.user, True, [event_id_a]
  178. )
  179. event_id_sf2 = self.create_and_send_event(
  180. self.room_id, self.user, True, [event_id_a, event_id_b]
  181. )
  182. event_id_sf3 = self.create_and_send_event(
  183. self.room_id, self.user, True, [event_id_sf1]
  184. )
  185. self.create_and_send_event(
  186. self.room_id, self.user, True, [event_id_sf2, event_id_sf3]
  187. ) # SF4
  188. event_id_c = self.create_and_send_event(
  189. self.room_id, self.user, False, [event_id_sf3]
  190. )
  191. # Add the new extremity and check the latest events are as expected
  192. self.add_extremity(self.room_id, event_id_a)
  193. latest_event_ids = self.get_success(
  194. self.store.get_latest_event_ids_in_room(self.room_id)
  195. )
  196. self.assertEqual(set(latest_event_ids), {event_id_a, event_id_b, event_id_c})
  197. # Run the background update and check it did the right thing
  198. self.run_background_update()
  199. latest_event_ids = self.get_success(
  200. self.store.get_latest_event_ids_in_room(self.room_id)
  201. )
  202. self.assertEqual(set(latest_event_ids), {event_id_b, event_id_c})
  203. class CleanupExtremDummyEventsTestCase(HomeserverTestCase):
  204. CONSENT_VERSION = "1"
  205. EXTREMITIES_COUNT = 50
  206. servlets = [
  207. synapse.rest.admin.register_servlets_for_client_rest_resource,
  208. login.register_servlets,
  209. room.register_servlets,
  210. ]
  211. def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer:
  212. config = self.default_config()
  213. config["cleanup_extremities_with_dummy_events"] = True
  214. return self.setup_test_homeserver(config=config)
  215. def prepare(
  216. self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer
  217. ) -> None:
  218. self.store = homeserver.get_datastores().main
  219. self.room_creator = homeserver.get_room_creation_handler()
  220. self.event_creator_handler = homeserver.get_event_creation_handler()
  221. # Create a test user and room
  222. self.user = UserID.from_string(self.register_user("user1", "password"))
  223. self.token1 = self.login("user1", "password")
  224. self.requester = create_requester(self.user)
  225. info, _ = self.get_success(
  226. self.room_creator.create_room(self.requester, {"visibility": "public"})
  227. )
  228. self.room_id = info["room_id"]
  229. self.event_creator = homeserver.get_event_creation_handler()
  230. homeserver.config.consent.user_consent_version = self.CONSENT_VERSION
  231. def test_send_dummy_event(self) -> None:
  232. self._create_extremity_rich_graph()
  233. # Pump the reactor repeatedly so that the background updates have a
  234. # chance to run.
  235. self.pump(20)
  236. latest_event_ids = self.get_success(
  237. self.store.get_latest_event_ids_in_room(self.room_id)
  238. )
  239. self.assertTrue(len(latest_event_ids) < 10, len(latest_event_ids))
  240. @patch("synapse.handlers.message._DUMMY_EVENT_ROOM_EXCLUSION_EXPIRY", new=0)
  241. def test_send_dummy_events_when_insufficient_power(self) -> None:
  242. self._create_extremity_rich_graph()
  243. # Criple power levels
  244. self.helper.send_state(
  245. self.room_id,
  246. EventTypes.PowerLevels,
  247. body={"users": {str(self.user): -1}},
  248. tok=self.token1,
  249. )
  250. # Pump the reactor repeatedly so that the background updates have a
  251. # chance to run.
  252. self.pump(10 * 60)
  253. latest_event_ids = self.get_success(
  254. self.store.get_latest_event_ids_in_room(self.room_id)
  255. )
  256. # Check that the room has not been pruned
  257. self.assertTrue(len(latest_event_ids) > 10)
  258. # New user with regular levels
  259. user2 = self.register_user("user2", "password")
  260. token2 = self.login("user2", "password")
  261. self.helper.join(self.room_id, user2, tok=token2)
  262. self.pump(10 * 60)
  263. latest_event_ids = self.get_success(
  264. self.store.get_latest_event_ids_in_room(self.room_id)
  265. )
  266. self.assertTrue(len(latest_event_ids) < 10, len(latest_event_ids))
  267. @patch("synapse.handlers.message._DUMMY_EVENT_ROOM_EXCLUSION_EXPIRY", new=250)
  268. def test_expiry_logic(self) -> None:
  269. """Simple test to ensure that _expire_rooms_to_exclude_from_dummy_event_insertion()
  270. expires old entries correctly.
  271. """
  272. self.event_creator_handler._rooms_to_exclude_from_dummy_event_insertion[
  273. "1"
  274. ] = 100000
  275. self.event_creator_handler._rooms_to_exclude_from_dummy_event_insertion[
  276. "2"
  277. ] = 200000
  278. self.event_creator_handler._rooms_to_exclude_from_dummy_event_insertion[
  279. "3"
  280. ] = 300000
  281. self.event_creator_handler._expire_rooms_to_exclude_from_dummy_event_insertion()
  282. # All entries within time frame
  283. self.assertEqual(
  284. len(
  285. self.event_creator_handler._rooms_to_exclude_from_dummy_event_insertion
  286. ),
  287. 3,
  288. )
  289. # Oldest room to expire
  290. self.pump(1.01)
  291. self.event_creator_handler._expire_rooms_to_exclude_from_dummy_event_insertion()
  292. self.assertEqual(
  293. len(
  294. self.event_creator_handler._rooms_to_exclude_from_dummy_event_insertion
  295. ),
  296. 2,
  297. )
  298. # All rooms to expire
  299. self.pump(2)
  300. self.assertEqual(
  301. len(
  302. self.event_creator_handler._rooms_to_exclude_from_dummy_event_insertion
  303. ),
  304. 0,
  305. )
  306. def _create_extremity_rich_graph(self) -> None:
  307. """Helper method to create bushy graph on demand"""
  308. event_id_start = self.create_and_send_event(self.room_id, self.user)
  309. for _ in range(self.EXTREMITIES_COUNT):
  310. self.create_and_send_event(
  311. self.room_id, self.user, prev_event_ids=[event_id_start]
  312. )
  313. latest_event_ids = self.get_success(
  314. self.store.get_latest_event_ids_in_room(self.room_id)
  315. )
  316. self.assertEqual(len(latest_event_ids), 50)
  317. def _enable_consent_checking(self) -> None:
  318. """Helper method to enable consent checking"""
  319. self.event_creator._block_events_without_consent_error = "No consent from user"
  320. consent_uri_builder = Mock()
  321. consent_uri_builder.build_user_consent_uri.return_value = "http://example.com"
  322. self.event_creator._consent_uri_builder = consent_uri_builder