test_cleanup_extrems.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389
  1. # Copyright 2019 The Matrix.org Foundation C.I.C.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. import os.path
  15. from unittest.mock import Mock, patch
  16. from twisted.test.proto_helpers import MemoryReactor
  17. import synapse.rest.admin
  18. from synapse.api.constants import EventTypes
  19. from synapse.rest.client import login, room
  20. from synapse.server import HomeServer
  21. from synapse.storage import prepare_database
  22. from synapse.storage.types import Cursor
  23. from synapse.types import UserID, create_requester
  24. from synapse.util import Clock
  25. from tests.unittest import HomeserverTestCase
  26. class CleanupExtremBackgroundUpdateStoreTestCase(HomeserverTestCase):
  27. """
  28. Test the background update to clean forward extremities table.
  29. """
  30. def prepare(
  31. self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer
  32. ) -> None:
  33. self.store = homeserver.get_datastores().main
  34. self.room_creator = homeserver.get_room_creation_handler()
  35. # Create a test user and room
  36. self.user = UserID("alice", "test")
  37. self.requester = create_requester(self.user)
  38. self.room_id, _, _ = self.get_success(
  39. self.room_creator.create_room(self.requester, {})
  40. )
  41. def run_background_update(self) -> None:
  42. """Re run the background update to clean up the extremities."""
  43. # Make sure we don't clash with in progress updates.
  44. self.assertTrue(
  45. self.store.db_pool.updates._all_done, "Background updates are still ongoing"
  46. )
  47. schema_path = os.path.join(
  48. prepare_database.schema_path,
  49. "main",
  50. "delta",
  51. "54",
  52. "delete_forward_extremities.sql",
  53. )
  54. def run_delta_file(txn: Cursor) -> None:
  55. prepare_database.executescript(txn, schema_path)
  56. self.get_success(
  57. self.store.db_pool.runInteraction(
  58. "test_delete_forward_extremities", run_delta_file
  59. )
  60. )
  61. # Ugh, have to reset this flag
  62. self.store.db_pool.updates._all_done = False
  63. self.wait_for_background_updates()
  64. def add_extremity(self, room_id: str, event_id: str) -> None:
  65. """
  66. Add the given event as an extremity to the room.
  67. """
  68. self.get_success(
  69. self.hs.get_datastores().main.db_pool.simple_insert(
  70. table="event_forward_extremities",
  71. values={"room_id": room_id, "event_id": event_id},
  72. desc="test_add_extremity",
  73. )
  74. )
  75. self.hs.get_datastores().main.get_latest_event_ids_in_room.invalidate(
  76. (room_id,)
  77. )
  78. def test_soft_failed_extremities_handled_correctly(self) -> None:
  79. """Test that extremities are correctly calculated in the presence of
  80. soft failed events.
  81. Tests a graph like:
  82. A <- SF1 <- SF2 <- B
  83. Where SF* are soft failed.
  84. """
  85. # Create the room graph
  86. event_id_1 = self.create_and_send_event(self.room_id, self.user)
  87. event_id_2 = self.create_and_send_event(
  88. self.room_id, self.user, True, [event_id_1]
  89. )
  90. event_id_3 = self.create_and_send_event(
  91. self.room_id, self.user, True, [event_id_2]
  92. )
  93. event_id_4 = self.create_and_send_event(
  94. self.room_id, self.user, False, [event_id_3]
  95. )
  96. # Check the latest events are as expected
  97. latest_event_ids = self.get_success(
  98. self.store.get_latest_event_ids_in_room(self.room_id)
  99. )
  100. self.assertEqual(latest_event_ids, [event_id_4])
  101. def test_basic_cleanup(self) -> None:
  102. """Test that extremities are correctly calculated in the presence of
  103. soft failed events.
  104. Tests a graph like:
  105. A <- SF1 <- B
  106. Where SF* are soft failed, and with extremities of A and B
  107. """
  108. # Create the room graph
  109. event_id_a = self.create_and_send_event(self.room_id, self.user)
  110. event_id_sf1 = self.create_and_send_event(
  111. self.room_id, self.user, True, [event_id_a]
  112. )
  113. event_id_b = self.create_and_send_event(
  114. self.room_id, self.user, False, [event_id_sf1]
  115. )
  116. # Add the new extremity and check the latest events are as expected
  117. self.add_extremity(self.room_id, event_id_a)
  118. latest_event_ids = self.get_success(
  119. self.store.get_latest_event_ids_in_room(self.room_id)
  120. )
  121. self.assertEqual(set(latest_event_ids), {event_id_a, event_id_b})
  122. # Run the background update and check it did the right thing
  123. self.run_background_update()
  124. latest_event_ids = self.get_success(
  125. self.store.get_latest_event_ids_in_room(self.room_id)
  126. )
  127. self.assertEqual(latest_event_ids, [event_id_b])
  128. def test_chain_of_fail_cleanup(self) -> None:
  129. """Test that extremities are correctly calculated in the presence of
  130. soft failed events.
  131. Tests a graph like:
  132. A <- SF1 <- SF2 <- B
  133. Where SF* are soft failed, and with extremities of A and B
  134. """
  135. # Create the room graph
  136. event_id_a = self.create_and_send_event(self.room_id, self.user)
  137. event_id_sf1 = self.create_and_send_event(
  138. self.room_id, self.user, True, [event_id_a]
  139. )
  140. event_id_sf2 = self.create_and_send_event(
  141. self.room_id, self.user, True, [event_id_sf1]
  142. )
  143. event_id_b = self.create_and_send_event(
  144. self.room_id, self.user, False, [event_id_sf2]
  145. )
  146. # Add the new extremity and check the latest events are as expected
  147. self.add_extremity(self.room_id, event_id_a)
  148. latest_event_ids = self.get_success(
  149. self.store.get_latest_event_ids_in_room(self.room_id)
  150. )
  151. self.assertEqual(set(latest_event_ids), {event_id_a, event_id_b})
  152. # Run the background update and check it did the right thing
  153. self.run_background_update()
  154. latest_event_ids = self.get_success(
  155. self.store.get_latest_event_ids_in_room(self.room_id)
  156. )
  157. self.assertEqual(latest_event_ids, [event_id_b])
  158. def test_forked_graph_cleanup(self) -> None:
  159. r"""Test that extremities are correctly calculated in the presence of
  160. soft failed events.
  161. Tests a graph like, where time flows down the page:
  162. A B
  163. / \ /
  164. / \ /
  165. SF1 SF2
  166. | |
  167. SF3 |
  168. / \ |
  169. | \ |
  170. C SF4
  171. Where SF* are soft failed, and with them A, B and C marked as
  172. extremities. This should resolve to B and C being marked as extremity.
  173. """
  174. # Create the room graph
  175. event_id_a = self.create_and_send_event(self.room_id, self.user)
  176. event_id_b = self.create_and_send_event(self.room_id, self.user)
  177. event_id_sf1 = self.create_and_send_event(
  178. self.room_id, self.user, True, [event_id_a]
  179. )
  180. event_id_sf2 = self.create_and_send_event(
  181. self.room_id, self.user, True, [event_id_a, event_id_b]
  182. )
  183. event_id_sf3 = self.create_and_send_event(
  184. self.room_id, self.user, True, [event_id_sf1]
  185. )
  186. self.create_and_send_event(
  187. self.room_id, self.user, True, [event_id_sf2, event_id_sf3]
  188. ) # SF4
  189. event_id_c = self.create_and_send_event(
  190. self.room_id, self.user, False, [event_id_sf3]
  191. )
  192. # Add the new extremity and check the latest events are as expected
  193. self.add_extremity(self.room_id, event_id_a)
  194. latest_event_ids = self.get_success(
  195. self.store.get_latest_event_ids_in_room(self.room_id)
  196. )
  197. self.assertEqual(set(latest_event_ids), {event_id_a, event_id_b, event_id_c})
  198. # Run the background update and check it did the right thing
  199. self.run_background_update()
  200. latest_event_ids = self.get_success(
  201. self.store.get_latest_event_ids_in_room(self.room_id)
  202. )
  203. self.assertEqual(set(latest_event_ids), {event_id_b, event_id_c})
  204. class CleanupExtremDummyEventsTestCase(HomeserverTestCase):
  205. CONSENT_VERSION = "1"
  206. EXTREMITIES_COUNT = 50
  207. servlets = [
  208. synapse.rest.admin.register_servlets_for_client_rest_resource,
  209. login.register_servlets,
  210. room.register_servlets,
  211. ]
  212. def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer:
  213. config = self.default_config()
  214. config["cleanup_extremities_with_dummy_events"] = True
  215. return self.setup_test_homeserver(config=config)
  216. def prepare(
  217. self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer
  218. ) -> None:
  219. self.store = homeserver.get_datastores().main
  220. self.room_creator = homeserver.get_room_creation_handler()
  221. self.event_creator_handler = homeserver.get_event_creation_handler()
  222. # Create a test user and room
  223. self.user = UserID.from_string(self.register_user("user1", "password"))
  224. self.token1 = self.login("user1", "password")
  225. self.requester = create_requester(self.user)
  226. self.room_id, _, _ = self.get_success(
  227. self.room_creator.create_room(self.requester, {"visibility": "public"})
  228. )
  229. self.event_creator = homeserver.get_event_creation_handler()
  230. homeserver.config.consent.user_consent_version = self.CONSENT_VERSION
  231. def test_send_dummy_event(self) -> None:
  232. self._create_extremity_rich_graph()
  233. # Pump the reactor repeatedly so that the background updates have a
  234. # chance to run.
  235. self.pump(20)
  236. latest_event_ids = self.get_success(
  237. self.store.get_latest_event_ids_in_room(self.room_id)
  238. )
  239. self.assertTrue(len(latest_event_ids) < 10, len(latest_event_ids))
  240. @patch("synapse.handlers.message._DUMMY_EVENT_ROOM_EXCLUSION_EXPIRY", new=0)
  241. def test_send_dummy_events_when_insufficient_power(self) -> None:
  242. self._create_extremity_rich_graph()
  243. # Criple power levels
  244. self.helper.send_state(
  245. self.room_id,
  246. EventTypes.PowerLevels,
  247. body={"users": {str(self.user): -1}},
  248. tok=self.token1,
  249. )
  250. # Pump the reactor repeatedly so that the background updates have a
  251. # chance to run.
  252. self.pump(10 * 60)
  253. latest_event_ids = self.get_success(
  254. self.store.get_latest_event_ids_in_room(self.room_id)
  255. )
  256. # Check that the room has not been pruned
  257. self.assertTrue(len(latest_event_ids) > 10)
  258. # New user with regular levels
  259. user2 = self.register_user("user2", "password")
  260. token2 = self.login("user2", "password")
  261. self.helper.join(self.room_id, user2, tok=token2)
  262. self.pump(10 * 60)
  263. latest_event_ids = self.get_success(
  264. self.store.get_latest_event_ids_in_room(self.room_id)
  265. )
  266. self.assertTrue(len(latest_event_ids) < 10, len(latest_event_ids))
  267. @patch("synapse.handlers.message._DUMMY_EVENT_ROOM_EXCLUSION_EXPIRY", new=250)
  268. def test_expiry_logic(self) -> None:
  269. """Simple test to ensure that _expire_rooms_to_exclude_from_dummy_event_insertion()
  270. expires old entries correctly.
  271. """
  272. self.event_creator_handler._rooms_to_exclude_from_dummy_event_insertion[
  273. "1"
  274. ] = 100000
  275. self.event_creator_handler._rooms_to_exclude_from_dummy_event_insertion[
  276. "2"
  277. ] = 200000
  278. self.event_creator_handler._rooms_to_exclude_from_dummy_event_insertion[
  279. "3"
  280. ] = 300000
  281. self.event_creator_handler._expire_rooms_to_exclude_from_dummy_event_insertion()
  282. # All entries within time frame
  283. self.assertEqual(
  284. len(
  285. self.event_creator_handler._rooms_to_exclude_from_dummy_event_insertion
  286. ),
  287. 3,
  288. )
  289. # Oldest room to expire
  290. self.pump(1.01)
  291. self.event_creator_handler._expire_rooms_to_exclude_from_dummy_event_insertion()
  292. self.assertEqual(
  293. len(
  294. self.event_creator_handler._rooms_to_exclude_from_dummy_event_insertion
  295. ),
  296. 2,
  297. )
  298. # All rooms to expire
  299. self.pump(2)
  300. self.assertEqual(
  301. len(
  302. self.event_creator_handler._rooms_to_exclude_from_dummy_event_insertion
  303. ),
  304. 0,
  305. )
  306. def _create_extremity_rich_graph(self) -> None:
  307. """Helper method to create bushy graph on demand"""
  308. event_id_start = self.create_and_send_event(self.room_id, self.user)
  309. for _ in range(self.EXTREMITIES_COUNT):
  310. self.create_and_send_event(
  311. self.room_id, self.user, prev_event_ids=[event_id_start]
  312. )
  313. latest_event_ids = self.get_success(
  314. self.store.get_latest_event_ids_in_room(self.room_id)
  315. )
  316. self.assertEqual(len(latest_event_ids), 50)
  317. def _enable_consent_checking(self) -> None:
  318. """Helper method to enable consent checking"""
  319. self.event_creator._block_events_without_consent_error = "No consent from user"
  320. consent_uri_builder = Mock()
  321. consent_uri_builder.build_user_consent_uri.return_value = "http://example.com"
  322. self.event_creator._consent_uri_builder = consent_uri_builder