test_room_search.py 7.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189
  1. # Copyright 2021 The Matrix.org Foundation C.I.C.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. import synapse.rest.admin
  15. from synapse.api.constants import EventTypes
  16. from synapse.api.errors import StoreError
  17. from synapse.rest.client import login, room
  18. from synapse.storage.engines import PostgresEngine
  19. from tests.unittest import HomeserverTestCase, skip_unless
  20. from tests.utils import USE_POSTGRES_FOR_TESTS
  21. class EventSearchInsertionTest(HomeserverTestCase):
  22. servlets = [
  23. synapse.rest.admin.register_servlets_for_client_rest_resource,
  24. login.register_servlets,
  25. room.register_servlets,
  26. ]
  27. def test_null_byte(self):
  28. """
  29. Postgres/SQLite don't like null bytes going into the search tables. Internally
  30. we replace those with a space.
  31. Ensure this doesn't break anything.
  32. """
  33. # Register a user and create a room, create some messages
  34. self.register_user("alice", "password")
  35. access_token = self.login("alice", "password")
  36. room_id = self.helper.create_room_as("alice", tok=access_token)
  37. # Send messages and ensure they don't cause an internal server
  38. # error
  39. for body in ["hi\u0000bob", "another message", "hi alice"]:
  40. response = self.helper.send(room_id, body, tok=access_token)
  41. self.assertIn("event_id", response)
  42. # Check that search works for the message where the null byte was replaced
  43. store = self.hs.get_datastores().main
  44. result = self.get_success(
  45. store.search_msgs([room_id], "hi bob", ["content.body"])
  46. )
  47. self.assertEqual(result.get("count"), 1)
  48. if isinstance(store.database_engine, PostgresEngine):
  49. self.assertIn("hi", result.get("highlights"))
  50. self.assertIn("bob", result.get("highlights"))
  51. # Check that search works for an unrelated message
  52. result = self.get_success(
  53. store.search_msgs([room_id], "another", ["content.body"])
  54. )
  55. self.assertEqual(result.get("count"), 1)
  56. if isinstance(store.database_engine, PostgresEngine):
  57. self.assertIn("another", result.get("highlights"))
  58. # Check that search works for a search term that overlaps with the message
  59. # containing a null byte and an unrelated message.
  60. result = self.get_success(store.search_msgs([room_id], "hi", ["content.body"]))
  61. self.assertEqual(result.get("count"), 2)
  62. result = self.get_success(
  63. store.search_msgs([room_id], "hi alice", ["content.body"])
  64. )
  65. if isinstance(store.database_engine, PostgresEngine):
  66. self.assertIn("alice", result.get("highlights"))
  67. def test_non_string(self):
  68. """Test that non-string `value`s are not inserted into `event_search`.
  69. This is particularly important when using sqlite, since a sqlite column can hold
  70. both strings and integers. When using Postgres, integers are automatically
  71. converted to strings.
  72. Regression test for #11918.
  73. """
  74. store = self.hs.get_datastores().main
  75. # Register a user and create a room
  76. user_id = self.register_user("alice", "password")
  77. access_token = self.login("alice", "password")
  78. room_id = self.helper.create_room_as("alice", tok=access_token)
  79. room_version = self.get_success(store.get_room_version(room_id))
  80. # Construct a message with a numeric body to be received over federation
  81. # The message can't be sent using the client API, since Synapse's event
  82. # validation will reject it.
  83. prev_event_ids = self.get_success(store.get_prev_events_for_room(room_id))
  84. prev_event = self.get_success(store.get_event(prev_event_ids[0]))
  85. prev_state_map = self.get_success(
  86. self.hs.get_storage_controllers().state.get_state_ids_for_event(
  87. prev_event_ids[0]
  88. )
  89. )
  90. event_dict = {
  91. "type": EventTypes.Message,
  92. "content": {"msgtype": "m.text", "body": 2},
  93. "room_id": room_id,
  94. "sender": user_id,
  95. "depth": prev_event.depth + 1,
  96. "prev_events": prev_event_ids,
  97. "origin_server_ts": self.clock.time_msec(),
  98. }
  99. builder = self.hs.get_event_builder_factory().for_room_version(
  100. room_version, event_dict
  101. )
  102. event = self.get_success(
  103. builder.build(
  104. prev_event_ids=prev_event_ids,
  105. auth_event_ids=self.hs.get_event_auth_handler().compute_auth_events(
  106. builder,
  107. prev_state_map,
  108. for_verification=False,
  109. ),
  110. depth=event_dict["depth"],
  111. )
  112. )
  113. # Receive the event
  114. self.get_success(
  115. self.hs.get_federation_event_handler().on_receive_pdu(
  116. self.hs.hostname, event
  117. )
  118. )
  119. # The event should not have an entry in the `event_search` table
  120. f = self.get_failure(
  121. store.db_pool.simple_select_one_onecol(
  122. "event_search",
  123. {"room_id": room_id, "event_id": event.event_id},
  124. "event_id",
  125. ),
  126. StoreError,
  127. )
  128. self.assertEqual(f.value.code, 404)
  129. @skip_unless(not USE_POSTGRES_FOR_TESTS, "requires sqlite")
  130. def test_sqlite_non_string_deletion_background_update(self):
  131. """Test the background update to delete bad rows from `event_search`."""
  132. store = self.hs.get_datastores().main
  133. # Populate `event_search` with dummy data
  134. self.get_success(
  135. store.db_pool.simple_insert_many(
  136. "event_search",
  137. keys=["event_id", "room_id", "key", "value"],
  138. values=[
  139. ("event1", "room_id", "content.body", "hi"),
  140. ("event2", "room_id", "content.body", "2"),
  141. ("event3", "room_id", "content.body", 3),
  142. ],
  143. desc="populate_event_search",
  144. )
  145. )
  146. # Run the background update
  147. store.db_pool.updates._all_done = False
  148. self.get_success(
  149. store.db_pool.simple_insert(
  150. "background_updates",
  151. {
  152. "update_name": "event_search_sqlite_delete_non_strings",
  153. "progress_json": "{}",
  154. },
  155. )
  156. )
  157. self.wait_for_background_updates()
  158. # The non-string `value`s ought to be gone now.
  159. values = self.get_success(
  160. store.db_pool.simple_select_onecol(
  161. "event_search",
  162. {"room_id": "room_id"},
  163. "value",
  164. ),
  165. )
  166. self.assertCountEqual(values, ["hi", "2"])