utils.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636
  1. # -*- coding: utf-8 -*-
  2. # Copyright 2014-2016 OpenMarket Ltd
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License");
  5. # you may not use this file except in compliance with the License.
  6. # You may obtain a copy of the License at
  7. #
  8. # http://www.apache.org/licenses/LICENSE-2.0
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS,
  12. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13. # See the License for the specific language governing permissions and
  14. # limitations under the License.
  15. import atexit
  16. import hashlib
  17. import os
  18. import time
  19. import uuid
  20. import warnings
  21. from inspect import getcallargs
  22. from mock import Mock, patch
  23. from six.moves.urllib import parse as urlparse
  24. from twisted.internet import defer, reactor
  25. from synapse.api.constants import EventTypes
  26. from synapse.api.errors import CodeMessageException, cs_error
  27. from synapse.config.server import ServerConfig
  28. from synapse.federation.transport import server
  29. from synapse.http.server import HttpServer
  30. from synapse.server import HomeServer
  31. from synapse.storage import DataStore
  32. from synapse.storage.engines import PostgresEngine, create_engine
  33. from synapse.storage.prepare_database import (
  34. _get_or_create_schema_state,
  35. _setup_new_database,
  36. prepare_database,
  37. )
  38. from synapse.util.logcontext import LoggingContext
  39. from synapse.util.ratelimitutils import FederationRateLimiter
  40. # set this to True to run the tests against postgres instead of sqlite.
  41. USE_POSTGRES_FOR_TESTS = os.environ.get("SYNAPSE_POSTGRES", False)
  42. LEAVE_DB = os.environ.get("SYNAPSE_LEAVE_DB", False)
  43. POSTGRES_USER = os.environ.get("SYNAPSE_POSTGRES_USER", "postgres")
  44. POSTGRES_BASE_DB = "_synapse_unit_tests_base_%s" % (os.getpid(),)
  45. def setupdb():
  46. # If we're using PostgreSQL, set up the db once
  47. if USE_POSTGRES_FOR_TESTS:
  48. pgconfig = {
  49. "name": "psycopg2",
  50. "args": {
  51. "database": POSTGRES_BASE_DB,
  52. "user": POSTGRES_USER,
  53. "cp_min": 1,
  54. "cp_max": 5,
  55. },
  56. }
  57. config = Mock()
  58. config.password_providers = []
  59. config.database_config = pgconfig
  60. db_engine = create_engine(pgconfig)
  61. db_conn = db_engine.module.connect(user=POSTGRES_USER)
  62. db_conn.autocommit = True
  63. cur = db_conn.cursor()
  64. cur.execute("DROP DATABASE IF EXISTS %s;" % (POSTGRES_BASE_DB,))
  65. cur.execute("CREATE DATABASE %s;" % (POSTGRES_BASE_DB,))
  66. cur.close()
  67. db_conn.close()
  68. # Set up in the db
  69. db_conn = db_engine.module.connect(
  70. database=POSTGRES_BASE_DB, user=POSTGRES_USER
  71. )
  72. cur = db_conn.cursor()
  73. _get_or_create_schema_state(cur, db_engine)
  74. _setup_new_database(cur, db_engine)
  75. db_conn.commit()
  76. cur.close()
  77. db_conn.close()
  78. def _cleanup():
  79. db_conn = db_engine.module.connect(user=POSTGRES_USER)
  80. db_conn.autocommit = True
  81. cur = db_conn.cursor()
  82. cur.execute("DROP DATABASE IF EXISTS %s;" % (POSTGRES_BASE_DB,))
  83. cur.close()
  84. db_conn.close()
  85. atexit.register(_cleanup)
  86. def default_config(name):
  87. """
  88. Create a reasonable test config.
  89. """
  90. config = Mock()
  91. config.signing_key = [MockKey()]
  92. config.event_cache_size = 1
  93. config.enable_registration = True
  94. config.macaroon_secret_key = "not even a little secret"
  95. config.expire_access_token = False
  96. config.server_name = name
  97. config.trusted_third_party_id_servers = []
  98. config.room_invite_state_types = []
  99. config.password_providers = []
  100. config.worker_replication_url = ""
  101. config.worker_app = None
  102. config.email_enable_notifs = False
  103. config.block_non_admin_invites = False
  104. config.federation_domain_whitelist = None
  105. config.federation_rc_reject_limit = 10
  106. config.federation_rc_sleep_limit = 10
  107. config.federation_rc_sleep_delay = 100
  108. config.federation_rc_concurrent = 10
  109. config.filter_timeline_limit = 5000
  110. config.user_directory_search_all_users = False
  111. config.user_consent_server_notice_content = None
  112. config.block_events_without_consent_error = None
  113. config.user_consent_at_registration = False
  114. config.user_consent_policy_name = "Privacy Policy"
  115. config.media_storage_providers = []
  116. config.autocreate_auto_join_rooms = True
  117. config.auto_join_rooms = []
  118. config.limit_usage_by_mau = False
  119. config.hs_disabled = False
  120. config.hs_disabled_message = ""
  121. config.hs_disabled_limit_type = ""
  122. config.max_mau_value = 50
  123. config.mau_trial_days = 0
  124. config.mau_stats_only = False
  125. config.mau_limits_reserved_threepids = []
  126. config.admin_contact = None
  127. config.rc_messages_per_second = 10000
  128. config.rc_message_burst_count = 10000
  129. config.saml2_enabled = False
  130. config.public_baseurl = None
  131. config.default_identity_server = None
  132. config.use_frozen_dicts = False
  133. # we need a sane default_room_version, otherwise attempts to create rooms will
  134. # fail.
  135. config.default_room_version = "1"
  136. # disable user directory updates, because they get done in the
  137. # background, which upsets the test runner.
  138. config.update_user_directory = False
  139. def is_threepid_reserved(threepid):
  140. return ServerConfig.is_threepid_reserved(config, threepid)
  141. config.is_threepid_reserved.side_effect = is_threepid_reserved
  142. return config
  143. class TestHomeServer(HomeServer):
  144. DATASTORE_CLASS = DataStore
  145. @defer.inlineCallbacks
  146. def setup_test_homeserver(
  147. cleanup_func,
  148. name="test",
  149. datastore=None,
  150. config=None,
  151. reactor=None,
  152. homeserverToUse=TestHomeServer,
  153. **kargs
  154. ):
  155. """
  156. Setup a homeserver suitable for running tests against. Keyword arguments
  157. are passed to the Homeserver constructor.
  158. If no datastore is supplied, one is created and given to the homeserver.
  159. Args:
  160. cleanup_func : The function used to register a cleanup routine for
  161. after the test.
  162. """
  163. if reactor is None:
  164. from twisted.internet import reactor
  165. if config is None:
  166. config = default_config(name)
  167. config.ldap_enabled = False
  168. if "clock" not in kargs:
  169. kargs["clock"] = MockClock()
  170. if USE_POSTGRES_FOR_TESTS:
  171. test_db = "synapse_test_%s" % uuid.uuid4().hex
  172. config.database_config = {
  173. "name": "psycopg2",
  174. "args": {"database": test_db, "cp_min": 1, "cp_max": 5},
  175. }
  176. else:
  177. config.database_config = {
  178. "name": "sqlite3",
  179. "args": {"database": ":memory:", "cp_min": 1, "cp_max": 1},
  180. }
  181. db_engine = create_engine(config.database_config)
  182. # Create the database before we actually try and connect to it, based off
  183. # the template database we generate in setupdb()
  184. if datastore is None and isinstance(db_engine, PostgresEngine):
  185. db_conn = db_engine.module.connect(
  186. database=POSTGRES_BASE_DB, user=POSTGRES_USER
  187. )
  188. db_conn.autocommit = True
  189. cur = db_conn.cursor()
  190. cur.execute("DROP DATABASE IF EXISTS %s;" % (test_db,))
  191. cur.execute(
  192. "CREATE DATABASE %s WITH TEMPLATE %s;" % (test_db, POSTGRES_BASE_DB)
  193. )
  194. cur.close()
  195. db_conn.close()
  196. # we need to configure the connection pool to run the on_new_connection
  197. # function, so that we can test code that uses custom sqlite functions
  198. # (like rank).
  199. config.database_config["args"]["cp_openfun"] = db_engine.on_new_connection
  200. if datastore is None:
  201. hs = homeserverToUse(
  202. name,
  203. config=config,
  204. db_config=config.database_config,
  205. version_string="Synapse/tests",
  206. database_engine=db_engine,
  207. room_list_handler=object(),
  208. tls_server_context_factory=Mock(),
  209. tls_client_options_factory=Mock(),
  210. reactor=reactor,
  211. **kargs
  212. )
  213. # Prepare the DB on SQLite -- PostgreSQL is a copy of an already up to
  214. # date db
  215. if not isinstance(db_engine, PostgresEngine):
  216. db_conn = hs.get_db_conn()
  217. yield prepare_database(db_conn, db_engine, config)
  218. db_conn.commit()
  219. db_conn.close()
  220. else:
  221. # We need to do cleanup on PostgreSQL
  222. def cleanup():
  223. import psycopg2
  224. # Close all the db pools
  225. hs.get_db_pool().close()
  226. dropped = False
  227. # Drop the test database
  228. db_conn = db_engine.module.connect(
  229. database=POSTGRES_BASE_DB, user=POSTGRES_USER
  230. )
  231. db_conn.autocommit = True
  232. cur = db_conn.cursor()
  233. # Try a few times to drop the DB. Some things may hold on to the
  234. # database for a few more seconds due to flakiness, preventing
  235. # us from dropping it when the test is over. If we can't drop
  236. # it, warn and move on.
  237. for x in range(5):
  238. try:
  239. cur.execute("DROP DATABASE IF EXISTS %s;" % (test_db,))
  240. db_conn.commit()
  241. dropped = True
  242. except psycopg2.OperationalError as e:
  243. warnings.warn(
  244. "Couldn't drop old db: " + str(e), category=UserWarning
  245. )
  246. time.sleep(0.5)
  247. cur.close()
  248. db_conn.close()
  249. if not dropped:
  250. warnings.warn("Failed to drop old DB.", category=UserWarning)
  251. if not LEAVE_DB:
  252. # Register the cleanup hook
  253. cleanup_func(cleanup)
  254. hs.setup()
  255. else:
  256. hs = homeserverToUse(
  257. name,
  258. db_pool=None,
  259. datastore=datastore,
  260. config=config,
  261. version_string="Synapse/tests",
  262. database_engine=db_engine,
  263. room_list_handler=object(),
  264. tls_server_context_factory=Mock(),
  265. tls_client_options_factory=Mock(),
  266. reactor=reactor,
  267. **kargs
  268. )
  269. # bcrypt is far too slow to be doing in unit tests
  270. # Need to let the HS build an auth handler and then mess with it
  271. # because AuthHandler's constructor requires the HS, so we can't make one
  272. # beforehand and pass it in to the HS's constructor (chicken / egg)
  273. hs.get_auth_handler().hash = lambda p: hashlib.md5(p.encode('utf8')).hexdigest()
  274. hs.get_auth_handler().validate_hash = (
  275. lambda p, h: hashlib.md5(p.encode('utf8')).hexdigest() == h
  276. )
  277. fed = kargs.get("resource_for_federation", None)
  278. if fed:
  279. server.register_servlets(
  280. hs,
  281. resource=fed,
  282. authenticator=server.Authenticator(hs),
  283. ratelimiter=FederationRateLimiter(
  284. hs.get_clock(),
  285. window_size=hs.config.federation_rc_window_size,
  286. sleep_limit=hs.config.federation_rc_sleep_limit,
  287. sleep_msec=hs.config.federation_rc_sleep_delay,
  288. reject_limit=hs.config.federation_rc_reject_limit,
  289. concurrent_requests=hs.config.federation_rc_concurrent,
  290. ),
  291. )
  292. defer.returnValue(hs)
  293. def get_mock_call_args(pattern_func, mock_func):
  294. """ Return the arguments the mock function was called with interpreted
  295. by the pattern functions argument list.
  296. """
  297. invoked_args, invoked_kargs = mock_func.call_args
  298. return getcallargs(pattern_func, *invoked_args, **invoked_kargs)
  299. def mock_getRawHeaders(headers=None):
  300. headers = headers if headers is not None else {}
  301. def getRawHeaders(name, default=None):
  302. return headers.get(name, default)
  303. return getRawHeaders
  304. # This is a mock /resource/ not an entire server
  305. class MockHttpResource(HttpServer):
  306. def __init__(self, prefix=""):
  307. self.callbacks = [] # 3-tuple of method/pattern/function
  308. self.prefix = prefix
  309. def trigger_get(self, path):
  310. return self.trigger(b"GET", path, None)
  311. @patch('twisted.web.http.Request')
  312. @defer.inlineCallbacks
  313. def trigger(
  314. self, http_method, path, content, mock_request, federation_auth_origin=None
  315. ):
  316. """ Fire an HTTP event.
  317. Args:
  318. http_method : The HTTP method
  319. path : The HTTP path
  320. content : The HTTP body
  321. mock_request : Mocked request to pass to the event so it can get
  322. content.
  323. federation_auth_origin (bytes|None): domain to authenticate as, for federation
  324. Returns:
  325. A tuple of (code, response)
  326. Raises:
  327. KeyError If no event is found which will handle the path.
  328. """
  329. path = self.prefix + path
  330. # annoyingly we return a twisted http request which has chained calls
  331. # to get at the http content, hence mock it here.
  332. mock_content = Mock()
  333. config = {'read.return_value': content}
  334. mock_content.configure_mock(**config)
  335. mock_request.content = mock_content
  336. mock_request.method = http_method.encode('ascii')
  337. mock_request.uri = path.encode('ascii')
  338. mock_request.getClientIP.return_value = "-"
  339. headers = {}
  340. if federation_auth_origin is not None:
  341. headers[b"Authorization"] = [
  342. b"X-Matrix origin=%s,key=,sig=" % (federation_auth_origin,)
  343. ]
  344. mock_request.requestHeaders.getRawHeaders = mock_getRawHeaders(headers)
  345. # return the right path if the event requires it
  346. mock_request.path = path
  347. # add in query params to the right place
  348. try:
  349. mock_request.args = urlparse.parse_qs(path.split('?')[1])
  350. mock_request.path = path.split('?')[0]
  351. path = mock_request.path
  352. except Exception:
  353. pass
  354. if isinstance(path, bytes):
  355. path = path.decode('utf8')
  356. for (method, pattern, func) in self.callbacks:
  357. if http_method != method:
  358. continue
  359. matcher = pattern.match(path)
  360. if matcher:
  361. try:
  362. args = [urlparse.unquote(u) for u in matcher.groups()]
  363. (code, response) = yield func(mock_request, *args)
  364. defer.returnValue((code, response))
  365. except CodeMessageException as e:
  366. defer.returnValue((e.code, cs_error(e.msg, code=e.errcode)))
  367. raise KeyError("No event can handle %s" % path)
  368. def register_paths(self, method, path_patterns, callback):
  369. for path_pattern in path_patterns:
  370. self.callbacks.append((method, path_pattern, callback))
  371. class MockKey(object):
  372. alg = "mock_alg"
  373. version = "mock_version"
  374. signature = b"\x9a\x87$"
  375. @property
  376. def verify_key(self):
  377. return self
  378. def sign(self, message):
  379. return self
  380. def verify(self, message, sig):
  381. assert sig == b"\x9a\x87$"
  382. class MockClock(object):
  383. now = 1000
  384. def __init__(self):
  385. # list of lists of [absolute_time, callback, expired] in no particular
  386. # order
  387. self.timers = []
  388. self.loopers = []
  389. def time(self):
  390. return self.now
  391. def time_msec(self):
  392. return self.time() * 1000
  393. def call_later(self, delay, callback, *args, **kwargs):
  394. current_context = LoggingContext.current_context()
  395. def wrapped_callback():
  396. LoggingContext.thread_local.current_context = current_context
  397. callback(*args, **kwargs)
  398. t = [self.now + delay, wrapped_callback, False]
  399. self.timers.append(t)
  400. return t
  401. def looping_call(self, function, interval):
  402. self.loopers.append([function, interval / 1000., self.now])
  403. def cancel_call_later(self, timer, ignore_errs=False):
  404. if timer[2]:
  405. if not ignore_errs:
  406. raise Exception("Cannot cancel an expired timer")
  407. timer[2] = True
  408. self.timers = [t for t in self.timers if t != timer]
  409. # For unit testing
  410. def advance_time(self, secs):
  411. self.now += secs
  412. timers = self.timers
  413. self.timers = []
  414. for t in timers:
  415. time, callback, expired = t
  416. if expired:
  417. raise Exception("Timer already expired")
  418. if self.now >= time:
  419. t[2] = True
  420. callback()
  421. else:
  422. self.timers.append(t)
  423. for looped in self.loopers:
  424. func, interval, last = looped
  425. if last + interval < self.now:
  426. func()
  427. looped[2] = self.now
  428. def advance_time_msec(self, ms):
  429. self.advance_time(ms / 1000.)
  430. def time_bound_deferred(self, d, *args, **kwargs):
  431. # We don't bother timing things out for now.
  432. return d
  433. def _format_call(args, kwargs):
  434. return ", ".join(
  435. ["%r" % (a) for a in args] + ["%s=%r" % (k, v) for k, v in kwargs.items()]
  436. )
  437. class DeferredMockCallable(object):
  438. """A callable instance that stores a set of pending call expectations and
  439. return values for them. It allows a unit test to assert that the given set
  440. of function calls are eventually made, by awaiting on them to be called.
  441. """
  442. def __init__(self):
  443. self.expectations = []
  444. self.calls = []
  445. def __call__(self, *args, **kwargs):
  446. self.calls.append((args, kwargs))
  447. if not self.expectations:
  448. raise ValueError(
  449. "%r has no pending calls to handle call(%s)"
  450. % (self, _format_call(args, kwargs))
  451. )
  452. for (call, result, d) in self.expectations:
  453. if args == call[1] and kwargs == call[2]:
  454. d.callback(None)
  455. return result
  456. failure = AssertionError(
  457. "Was not expecting call(%s)" % (_format_call(args, kwargs))
  458. )
  459. for _, _, d in self.expectations:
  460. try:
  461. d.errback(failure)
  462. except Exception:
  463. pass
  464. raise failure
  465. def expect_call_and_return(self, call, result):
  466. self.expectations.append((call, result, defer.Deferred()))
  467. @defer.inlineCallbacks
  468. def await_calls(self, timeout=1000):
  469. deferred = defer.DeferredList(
  470. [d for _, _, d in self.expectations], fireOnOneErrback=True
  471. )
  472. timer = reactor.callLater(
  473. timeout / 1000,
  474. deferred.errback,
  475. AssertionError(
  476. "%d pending calls left: %s"
  477. % (
  478. len([e for e in self.expectations if not e[2].called]),
  479. [e for e in self.expectations if not e[2].called],
  480. )
  481. ),
  482. )
  483. yield deferred
  484. timer.cancel()
  485. self.calls = []
  486. def assert_had_no_calls(self):
  487. if self.calls:
  488. calls = self.calls
  489. self.calls = []
  490. raise AssertionError(
  491. "Expected not to received any calls, got:\n"
  492. + "\n".join(["call(%s)" % _format_call(c[0], c[1]) for c in calls])
  493. )
  494. @defer.inlineCallbacks
  495. def create_room(hs, room_id, creator_id):
  496. """Creates and persist a creation event for the given room
  497. Args:
  498. hs
  499. room_id (str)
  500. creator_id (str)
  501. """
  502. store = hs.get_datastore()
  503. event_builder_factory = hs.get_event_builder_factory()
  504. event_creation_handler = hs.get_event_creation_handler()
  505. builder = event_builder_factory.new(
  506. {
  507. "type": EventTypes.Create,
  508. "state_key": "",
  509. "sender": creator_id,
  510. "room_id": room_id,
  511. "content": {},
  512. }
  513. )
  514. event, context = yield event_creation_handler.create_new_client_event(builder)
  515. yield store.persist_event(event, context)