utils.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373
  1. # Copyright 2014-2016 OpenMarket Ltd
  2. # Copyright 2018-2019 New Vector Ltd
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License");
  5. # you may not use this file except in compliance with the License.
  6. # You may obtain a copy of the License at
  7. #
  8. # http://www.apache.org/licenses/LICENSE-2.0
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS,
  12. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13. # See the License for the specific language governing permissions and
  14. # limitations under the License.
  15. import atexit
  16. import os
  17. from typing import Any, Callable, Dict, List, Tuple, Type, TypeVar, Union, overload
  18. import attr
  19. from typing_extensions import Literal, ParamSpec
  20. from synapse.api.constants import EventTypes
  21. from synapse.api.room_versions import RoomVersions
  22. from synapse.config.homeserver import HomeServerConfig
  23. from synapse.config.server import DEFAULT_ROOM_VERSION
  24. from synapse.logging.context import current_context, set_current_context
  25. from synapse.server import HomeServer
  26. from synapse.storage.database import LoggingDatabaseConnection
  27. from synapse.storage.engines import create_engine
  28. from synapse.storage.prepare_database import prepare_database
  29. # set this to True to run the tests against postgres instead of sqlite.
  30. #
  31. # When running under postgres, we first create a base database with the name
  32. # POSTGRES_BASE_DB and update it to the current schema. Then, for each test case, we
  33. # create another unique database, using the base database as a template.
  34. USE_POSTGRES_FOR_TESTS = os.environ.get("SYNAPSE_POSTGRES", False)
  35. LEAVE_DB = os.environ.get("SYNAPSE_LEAVE_DB", False)
  36. POSTGRES_USER = os.environ.get("SYNAPSE_POSTGRES_USER", None)
  37. POSTGRES_HOST = os.environ.get("SYNAPSE_POSTGRES_HOST", None)
  38. POSTGRES_PASSWORD = os.environ.get("SYNAPSE_POSTGRES_PASSWORD", None)
  39. POSTGRES_PORT = (
  40. int(os.environ["SYNAPSE_POSTGRES_PORT"])
  41. if "SYNAPSE_POSTGRES_PORT" in os.environ
  42. else None
  43. )
  44. POSTGRES_BASE_DB = "_synapse_unit_tests_base_%s" % (os.getpid(),)
  45. # When debugging a specific test, it's occasionally useful to write the
  46. # DB to disk and query it with the sqlite CLI.
  47. SQLITE_PERSIST_DB = os.environ.get("SYNAPSE_TEST_PERSIST_SQLITE_DB") is not None
  48. # the dbname we will connect to in order to create the base database.
  49. POSTGRES_DBNAME_FOR_INITIAL_CREATE = "postgres"
  50. def setupdb() -> None:
  51. # If we're using PostgreSQL, set up the db once
  52. if USE_POSTGRES_FOR_TESTS:
  53. # create a PostgresEngine
  54. if USE_POSTGRES_FOR_TESTS == "psycopg":
  55. db_engine = create_engine({"name": "psycopg", "args": {}})
  56. else:
  57. db_engine = create_engine({"name": "psycopg2", "args": {}})
  58. # connect to postgres to create the base database.
  59. db_conn = db_engine.module.connect(
  60. user=POSTGRES_USER,
  61. host=POSTGRES_HOST,
  62. port=POSTGRES_PORT,
  63. password=POSTGRES_PASSWORD,
  64. dbname=POSTGRES_DBNAME_FOR_INITIAL_CREATE,
  65. )
  66. db_engine.attempt_to_set_autocommit(db_conn, autocommit=True)
  67. cur = db_conn.cursor()
  68. cur.execute("DROP DATABASE IF EXISTS %s;" % (POSTGRES_BASE_DB,))
  69. cur.execute(
  70. "CREATE DATABASE %s ENCODING 'UTF8' LC_COLLATE='C' LC_CTYPE='C' "
  71. "template=template0;" % (POSTGRES_BASE_DB,)
  72. )
  73. cur.close()
  74. db_conn.close()
  75. # Set up in the db
  76. db_conn = db_engine.module.connect(
  77. dbname=POSTGRES_BASE_DB,
  78. user=POSTGRES_USER,
  79. host=POSTGRES_HOST,
  80. port=POSTGRES_PORT,
  81. password=POSTGRES_PASSWORD,
  82. )
  83. logging_conn = LoggingDatabaseConnection(db_conn, db_engine, "tests")
  84. prepare_database(logging_conn, db_engine, None)
  85. logging_conn.close()
  86. def _cleanup() -> None:
  87. db_conn = db_engine.module.connect(
  88. user=POSTGRES_USER,
  89. host=POSTGRES_HOST,
  90. port=POSTGRES_PORT,
  91. password=POSTGRES_PASSWORD,
  92. dbname=POSTGRES_DBNAME_FOR_INITIAL_CREATE,
  93. )
  94. db_engine.attempt_to_set_autocommit(db_conn, autocommit=True)
  95. cur = db_conn.cursor()
  96. cur.execute("DROP DATABASE IF EXISTS %s;" % (POSTGRES_BASE_DB,))
  97. cur.close()
  98. db_conn.close()
  99. atexit.register(_cleanup)
  100. @overload
  101. def default_config(name: str, parse: Literal[False] = ...) -> Dict[str, object]:
  102. ...
  103. @overload
  104. def default_config(name: str, parse: Literal[True]) -> HomeServerConfig:
  105. ...
  106. def default_config(
  107. name: str, parse: bool = False
  108. ) -> Union[Dict[str, object], HomeServerConfig]:
  109. """
  110. Create a reasonable test config.
  111. """
  112. config_dict = {
  113. "server_name": name,
  114. # Setting this to an empty list turns off federation sending.
  115. "federation_sender_instances": [],
  116. "media_store_path": "media",
  117. # the test signing key is just an arbitrary ed25519 key to keep the config
  118. # parser happy
  119. "signing_key": "ed25519 a_lPym qvioDNmfExFBRPgdTU+wtFYKq4JfwFRv7sYVgWvmgJg",
  120. # Disable trusted key servers, otherwise unit tests might try to actually
  121. # reach out to matrix.org.
  122. "trusted_key_servers": [],
  123. "event_cache_size": 1,
  124. "enable_registration": True,
  125. "enable_registration_captcha": False,
  126. "macaroon_secret_key": "not even a little secret",
  127. "password_providers": [],
  128. "worker_app": None,
  129. "block_non_admin_invites": False,
  130. "federation_domain_whitelist": None,
  131. "filter_timeline_limit": 5000,
  132. "user_directory_search_all_users": False,
  133. "user_consent_server_notice_content": None,
  134. "block_events_without_consent_error": None,
  135. "user_consent_at_registration": False,
  136. "user_consent_policy_name": "Privacy Policy",
  137. "media_storage_providers": [],
  138. "autocreate_auto_join_rooms": True,
  139. "auto_join_rooms": [],
  140. "limit_usage_by_mau": False,
  141. "hs_disabled": False,
  142. "hs_disabled_message": "",
  143. "max_mau_value": 50,
  144. "mau_trial_days": 0,
  145. "mau_stats_only": False,
  146. "mau_limits_reserved_threepids": [],
  147. "admin_contact": None,
  148. "rc_message": {"per_second": 10000, "burst_count": 10000},
  149. "rc_registration": {"per_second": 10000, "burst_count": 10000},
  150. "rc_login": {
  151. "address": {"per_second": 10000, "burst_count": 10000},
  152. "account": {"per_second": 10000, "burst_count": 10000},
  153. "failed_attempts": {"per_second": 10000, "burst_count": 10000},
  154. },
  155. "rc_joins": {
  156. "local": {"per_second": 10000, "burst_count": 10000},
  157. "remote": {"per_second": 10000, "burst_count": 10000},
  158. },
  159. "rc_joins_per_room": {"per_second": 10000, "burst_count": 10000},
  160. "rc_invites": {
  161. "per_room": {"per_second": 10000, "burst_count": 10000},
  162. "per_user": {"per_second": 10000, "burst_count": 10000},
  163. },
  164. "rc_3pid_validation": {"per_second": 10000, "burst_count": 10000},
  165. "saml2_enabled": False,
  166. "public_baseurl": None,
  167. "default_identity_server": None,
  168. "key_refresh_interval": 24 * 60 * 60 * 1000,
  169. "old_signing_keys": {},
  170. "tls_fingerprints": [],
  171. "use_frozen_dicts": False,
  172. # We need a sane default_room_version, otherwise attempts to create
  173. # rooms will fail.
  174. "default_room_version": DEFAULT_ROOM_VERSION,
  175. # disable user directory updates, because they get done in the
  176. # background, which upsets the test runner. Setting this to an
  177. # (obviously) fake worker name disables updating the user directory.
  178. "update_user_directory_from_worker": "does_not_exist_worker_name",
  179. "caches": {"global_factor": 1, "sync_response_cache_duration": 0},
  180. "listeners": [{"port": 0, "type": "http"}],
  181. }
  182. if parse:
  183. config = HomeServerConfig()
  184. config.parse_config_dict(config_dict, "", "")
  185. return config
  186. return config_dict
  187. def mock_getRawHeaders(headers=None): # type: ignore[no-untyped-def]
  188. headers = headers if headers is not None else {}
  189. def getRawHeaders(name, default=None): # type: ignore[no-untyped-def]
  190. # If the requested header is present, the real twisted function returns
  191. # List[str] if name is a str and List[bytes] if name is a bytes.
  192. # This mock doesn't support that behaviour.
  193. # Fortunately, none of the current callers of mock_getRawHeaders() provide a
  194. # headers dict, so we don't encounter this discrepancy in practice.
  195. return headers.get(name, default)
  196. return getRawHeaders
  197. P = ParamSpec("P")
  198. @attr.s(slots=True, auto_attribs=True)
  199. class Timer:
  200. absolute_time: float
  201. callback: Callable[[], None]
  202. expired: bool
  203. # TODO: Make this generic over a ParamSpec?
  204. @attr.s(slots=True, auto_attribs=True)
  205. class Looper:
  206. func: Callable[..., Any]
  207. interval: float # seconds
  208. last: float
  209. args: Tuple[object, ...]
  210. kwargs: Dict[str, object]
  211. class MockClock:
  212. now = 1000.0
  213. def __init__(self) -> None:
  214. # Timers in no particular order
  215. self.timers: List[Timer] = []
  216. self.loopers: List[Looper] = []
  217. def time(self) -> float:
  218. return self.now
  219. def time_msec(self) -> int:
  220. return int(self.time() * 1000)
  221. def call_later(
  222. self,
  223. delay: float,
  224. callback: Callable[P, object],
  225. *args: P.args,
  226. **kwargs: P.kwargs,
  227. ) -> Timer:
  228. ctx = current_context()
  229. def wrapped_callback() -> None:
  230. set_current_context(ctx)
  231. callback(*args, **kwargs)
  232. t = Timer(self.now + delay, wrapped_callback, False)
  233. self.timers.append(t)
  234. return t
  235. def looping_call(
  236. self,
  237. function: Callable[P, object],
  238. interval: float,
  239. *args: P.args,
  240. **kwargs: P.kwargs,
  241. ) -> None:
  242. self.loopers.append(Looper(function, interval / 1000.0, self.now, args, kwargs))
  243. def cancel_call_later(self, timer: Timer, ignore_errs: bool = False) -> None:
  244. if timer.expired:
  245. if not ignore_errs:
  246. raise Exception("Cannot cancel an expired timer")
  247. timer.expired = True
  248. self.timers = [t for t in self.timers if t != timer]
  249. # For unit testing
  250. def advance_time(self, secs: float) -> None:
  251. self.now += secs
  252. timers = self.timers
  253. self.timers = []
  254. for t in timers:
  255. if t.expired:
  256. raise Exception("Timer already expired")
  257. if self.now >= t.absolute_time:
  258. t.expired = True
  259. t.callback()
  260. else:
  261. self.timers.append(t)
  262. for looped in self.loopers:
  263. if looped.last + looped.interval < self.now:
  264. looped.func(*looped.args, **looped.kwargs)
  265. looped.last = self.now
  266. def advance_time_msec(self, ms: float) -> None:
  267. self.advance_time(ms / 1000.0)
  268. async def create_room(hs: HomeServer, room_id: str, creator_id: str) -> None:
  269. """Creates and persist a creation event for the given room"""
  270. persistence_store = hs.get_storage_controllers().persistence
  271. assert persistence_store is not None
  272. store = hs.get_datastores().main
  273. event_builder_factory = hs.get_event_builder_factory()
  274. event_creation_handler = hs.get_event_creation_handler()
  275. await store.store_room(
  276. room_id=room_id,
  277. room_creator_user_id=creator_id,
  278. is_public=False,
  279. room_version=RoomVersions.V1,
  280. )
  281. builder = event_builder_factory.for_room_version(
  282. RoomVersions.V1,
  283. {
  284. "type": EventTypes.Create,
  285. "state_key": "",
  286. "sender": creator_id,
  287. "room_id": room_id,
  288. "content": {},
  289. },
  290. )
  291. event, unpersisted_context = await event_creation_handler.create_new_client_event(
  292. builder
  293. )
  294. context = await unpersisted_context.persist(event)
  295. await persistence_store.persist_event(event, context)
  296. T = TypeVar("T")
  297. def checked_cast(type: Type[T], x: object) -> T:
  298. """A version of typing.cast that is checked at runtime.
  299. We have our own function for this for two reasons:
  300. 1. typing.cast itself is deliberately a no-op at runtime, see
  301. https://docs.python.org/3/library/typing.html#typing.cast
  302. 2. To help workaround a mypy-zope bug https://github.com/Shoobx/mypy-zope/issues/91
  303. where mypy would erroneously consider `isinstance(x, type)` to be false in all
  304. circumstances.
  305. For this to make sense, `T` needs to be something that `isinstance` can check; see
  306. https://docs.python.org/3/library/functions.html?highlight=isinstance#isinstance
  307. https://docs.python.org/3/glossary.html#term-abstract-base-class
  308. https://docs.python.org/3/library/typing.html#typing.runtime_checkable
  309. for more details.
  310. """
  311. assert isinstance(x, type)
  312. return x