test_visibility.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340
  1. # -*- coding: utf-8 -*-
  2. # Copyright 2018 New Vector Ltd
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License");
  5. # you may not use this file except in compliance with the License.
  6. # You may obtain a copy of the License at
  7. #
  8. # http://www.apache.org/licenses/LICENSE-2.0
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS,
  12. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13. # See the License for the specific language governing permissions and
  14. # limitations under the License.
  15. import logging
  16. from mock import Mock
  17. from twisted.internet import defer
  18. from twisted.internet.defer import succeed
  19. from synapse.api.room_versions import RoomVersions
  20. from synapse.events import FrozenEvent
  21. from synapse.visibility import filter_events_for_server
  22. import tests.unittest
  23. from tests.utils import create_room, setup_test_homeserver
  24. logger = logging.getLogger(__name__)
  25. TEST_ROOM_ID = "!TEST:ROOM"
  26. class FilterEventsForServerTestCase(tests.unittest.TestCase):
  27. @defer.inlineCallbacks
  28. def setUp(self):
  29. self.hs = yield setup_test_homeserver(self.addCleanup)
  30. self.event_creation_handler = self.hs.get_event_creation_handler()
  31. self.event_builder_factory = self.hs.get_event_builder_factory()
  32. self.store = self.hs.get_datastore()
  33. self.storage = self.hs.get_storage()
  34. yield create_room(self.hs, TEST_ROOM_ID, "@someone:ROOM")
  35. @defer.inlineCallbacks
  36. def test_filtering(self):
  37. #
  38. # The events to be filtered consist of 10 membership events (it doesn't
  39. # really matter if they are joins or leaves, so let's make them joins).
  40. # One of those membership events is going to be for a user on the
  41. # server we are filtering for (so we can check the filtering is doing
  42. # the right thing).
  43. #
  44. # before we do that, we persist some other events to act as state.
  45. self.inject_visibility("@admin:hs", "joined")
  46. for i in range(0, 10):
  47. yield self.inject_room_member("@resident%i:hs" % i)
  48. events_to_filter = []
  49. for i in range(0, 10):
  50. user = "@user%i:%s" % (i, "test_server" if i == 5 else "other_server")
  51. evt = yield self.inject_room_member(user, extra_content={"a": "b"})
  52. events_to_filter.append(evt)
  53. filtered = yield filter_events_for_server(
  54. self.storage, "test_server", events_to_filter
  55. )
  56. # the result should be 5 redacted events, and 5 unredacted events.
  57. for i in range(0, 5):
  58. self.assertEqual(events_to_filter[i].event_id, filtered[i].event_id)
  59. self.assertNotIn("a", filtered[i].content)
  60. for i in range(5, 10):
  61. self.assertEqual(events_to_filter[i].event_id, filtered[i].event_id)
  62. self.assertEqual(filtered[i].content["a"], "b")
  63. @defer.inlineCallbacks
  64. def test_erased_user(self):
  65. # 4 message events, from erased and unerased users, with a membership
  66. # change in the middle of them.
  67. events_to_filter = []
  68. evt = yield self.inject_message("@unerased:local_hs")
  69. events_to_filter.append(evt)
  70. evt = yield self.inject_message("@erased:local_hs")
  71. events_to_filter.append(evt)
  72. evt = yield self.inject_room_member("@joiner:remote_hs")
  73. events_to_filter.append(evt)
  74. evt = yield self.inject_message("@unerased:local_hs")
  75. events_to_filter.append(evt)
  76. evt = yield self.inject_message("@erased:local_hs")
  77. events_to_filter.append(evt)
  78. # the erasey user gets erased
  79. yield self.hs.get_datastore().mark_user_erased("@erased:local_hs")
  80. # ... and the filtering happens.
  81. filtered = yield filter_events_for_server(
  82. self.storage, "test_server", events_to_filter
  83. )
  84. for i in range(0, len(events_to_filter)):
  85. self.assertEqual(
  86. events_to_filter[i].event_id,
  87. filtered[i].event_id,
  88. "Unexpected event at result position %i" % (i,),
  89. )
  90. for i in (0, 3):
  91. self.assertEqual(
  92. events_to_filter[i].content["body"],
  93. filtered[i].content["body"],
  94. "Unexpected event content at result position %i" % (i,),
  95. )
  96. for i in (1, 4):
  97. self.assertNotIn("body", filtered[i].content)
  98. @defer.inlineCallbacks
  99. def inject_visibility(self, user_id, visibility):
  100. content = {"history_visibility": visibility}
  101. builder = self.event_builder_factory.for_room_version(
  102. RoomVersions.V1,
  103. {
  104. "type": "m.room.history_visibility",
  105. "sender": user_id,
  106. "state_key": "",
  107. "room_id": TEST_ROOM_ID,
  108. "content": content,
  109. },
  110. )
  111. event, context = yield self.event_creation_handler.create_new_client_event(
  112. builder
  113. )
  114. yield self.storage.persistence.persist_event(event, context)
  115. return event
  116. @defer.inlineCallbacks
  117. def inject_room_member(self, user_id, membership="join", extra_content={}):
  118. content = {"membership": membership}
  119. content.update(extra_content)
  120. builder = self.event_builder_factory.for_room_version(
  121. RoomVersions.V1,
  122. {
  123. "type": "m.room.member",
  124. "sender": user_id,
  125. "state_key": user_id,
  126. "room_id": TEST_ROOM_ID,
  127. "content": content,
  128. },
  129. )
  130. event, context = yield self.event_creation_handler.create_new_client_event(
  131. builder
  132. )
  133. yield self.storage.persistence.persist_event(event, context)
  134. return event
  135. @defer.inlineCallbacks
  136. def inject_message(self, user_id, content=None):
  137. if content is None:
  138. content = {"body": "testytest", "msgtype": "m.text"}
  139. builder = self.event_builder_factory.for_room_version(
  140. RoomVersions.V1,
  141. {
  142. "type": "m.room.message",
  143. "sender": user_id,
  144. "room_id": TEST_ROOM_ID,
  145. "content": content,
  146. },
  147. )
  148. event, context = yield self.event_creation_handler.create_new_client_event(
  149. builder
  150. )
  151. yield self.storage.persistence.persist_event(event, context)
  152. return event
  153. @defer.inlineCallbacks
  154. def test_large_room(self):
  155. # see what happens when we have a large room with hundreds of thousands
  156. # of membership events
  157. # As above, the events to be filtered consist of 10 membership events,
  158. # where one of them is for a user on the server we are filtering for.
  159. import cProfile
  160. import pstats
  161. import time
  162. # we stub out the store, because building up all that state the normal
  163. # way is very slow.
  164. test_store = _TestStore()
  165. # our initial state is 100000 membership events and one
  166. # history_visibility event.
  167. room_state = []
  168. history_visibility_evt = FrozenEvent(
  169. {
  170. "event_id": "$history_vis",
  171. "type": "m.room.history_visibility",
  172. "sender": "@resident_user_0:test.com",
  173. "state_key": "",
  174. "room_id": TEST_ROOM_ID,
  175. "content": {"history_visibility": "joined"},
  176. }
  177. )
  178. room_state.append(history_visibility_evt)
  179. test_store.add_event(history_visibility_evt)
  180. for i in range(0, 100000):
  181. user = "@resident_user_%i:test.com" % (i,)
  182. evt = FrozenEvent(
  183. {
  184. "event_id": "$res_event_%i" % (i,),
  185. "type": "m.room.member",
  186. "state_key": user,
  187. "sender": user,
  188. "room_id": TEST_ROOM_ID,
  189. "content": {"membership": "join", "extra": "zzz,"},
  190. }
  191. )
  192. room_state.append(evt)
  193. test_store.add_event(evt)
  194. events_to_filter = []
  195. for i in range(0, 10):
  196. user = "@user%i:%s" % (i, "test_server" if i == 5 else "other_server")
  197. evt = FrozenEvent(
  198. {
  199. "event_id": "$evt%i" % (i,),
  200. "type": "m.room.member",
  201. "state_key": user,
  202. "sender": user,
  203. "room_id": TEST_ROOM_ID,
  204. "content": {"membership": "join", "extra": "zzz"},
  205. }
  206. )
  207. events_to_filter.append(evt)
  208. room_state.append(evt)
  209. test_store.add_event(evt)
  210. test_store.set_state_ids_for_event(
  211. evt, {(e.type, e.state_key): e.event_id for e in room_state}
  212. )
  213. pr = cProfile.Profile()
  214. pr.enable()
  215. logger.info("Starting filtering")
  216. start = time.time()
  217. storage = Mock()
  218. storage.main = test_store
  219. storage.state = test_store
  220. filtered = yield filter_events_for_server(
  221. test_store, "test_server", events_to_filter
  222. )
  223. logger.info("Filtering took %f seconds", time.time() - start)
  224. pr.disable()
  225. with open("filter_events_for_server.profile", "w+") as f:
  226. ps = pstats.Stats(pr, stream=f).sort_stats("cumulative")
  227. ps.print_stats()
  228. # the result should be 5 redacted events, and 5 unredacted events.
  229. for i in range(0, 5):
  230. self.assertEqual(events_to_filter[i].event_id, filtered[i].event_id)
  231. self.assertNotIn("extra", filtered[i].content)
  232. for i in range(5, 10):
  233. self.assertEqual(events_to_filter[i].event_id, filtered[i].event_id)
  234. self.assertEqual(filtered[i].content["extra"], "zzz")
  235. test_large_room.skip = "Disabled by default because it's slow"
  236. class _TestStore(object):
  237. """Implements a few methods of the DataStore, so that we can test
  238. filter_events_for_server
  239. """
  240. def __init__(self):
  241. # data for get_events: a map from event_id to event
  242. self.events = {}
  243. # data for get_state_ids_for_events mock: a map from event_id to
  244. # a map from (type_state_key) -> event_id for the state at that
  245. # event
  246. self.state_ids_for_events = {}
  247. def add_event(self, event):
  248. self.events[event.event_id] = event
  249. def set_state_ids_for_event(self, event, state):
  250. self.state_ids_for_events[event.event_id] = state
  251. def get_state_ids_for_events(self, events, types):
  252. res = {}
  253. include_memberships = False
  254. for (type, state_key) in types:
  255. if type == "m.room.history_visibility":
  256. continue
  257. if type != "m.room.member" or state_key is not None:
  258. raise RuntimeError(
  259. "Unimplemented: get_state_ids with type (%s, %s)"
  260. % (type, state_key)
  261. )
  262. include_memberships = True
  263. if include_memberships:
  264. for event_id in events:
  265. res[event_id] = self.state_ids_for_events[event_id]
  266. else:
  267. k = ("m.room.history_visibility", "")
  268. for event_id in events:
  269. hve = self.state_ids_for_events[event_id][k]
  270. res[event_id] = {k: hve}
  271. return succeed(res)
  272. def get_events(self, events):
  273. return succeed({event_id: self.events[event_id] for event_id in events})
  274. def are_users_erased(self, users):
  275. return succeed({u: False for u in users})