test_appservice.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456
  1. # -*- coding: utf-8 -*-
  2. # Copyright 2015, 2016 OpenMarket Ltd
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License");
  5. # you may not use this file except in compliance with the License.
  6. # You may obtain a copy of the License at
  7. #
  8. # http://www.apache.org/licenses/LICENSE-2.0
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS,
  12. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13. # See the License for the specific language governing permissions and
  14. # limitations under the License.
  15. import json
  16. import os
  17. import tempfile
  18. from mock import Mock
  19. import yaml
  20. from twisted.internet import defer
  21. from synapse.appservice import ApplicationService, ApplicationServiceState
  22. from synapse.config._base import ConfigError
  23. from synapse.storage.appservice import (
  24. ApplicationServiceStore,
  25. ApplicationServiceTransactionStore,
  26. )
  27. from tests import unittest
  28. from tests.utils import setup_test_homeserver
  29. class ApplicationServiceStoreTestCase(unittest.TestCase):
  30. @defer.inlineCallbacks
  31. def setUp(self):
  32. self.as_yaml_files = []
  33. config = Mock(
  34. app_service_config_files=self.as_yaml_files,
  35. event_cache_size=1,
  36. password_providers=[],
  37. )
  38. hs = yield setup_test_homeserver(
  39. self.addCleanup,
  40. config=config,
  41. federation_sender=Mock(),
  42. federation_client=Mock(),
  43. )
  44. self.as_token = "token1"
  45. self.as_url = "some_url"
  46. self.as_id = "as1"
  47. self._add_appservice(
  48. self.as_token, self.as_id, self.as_url, "some_hs_token", "bob"
  49. )
  50. self._add_appservice("token2", "as2", "some_url", "some_hs_token", "bob")
  51. self._add_appservice("token3", "as3", "some_url", "some_hs_token", "bob")
  52. # must be done after inserts
  53. self.store = ApplicationServiceStore(None, hs)
  54. def tearDown(self):
  55. # TODO: suboptimal that we need to create files for tests!
  56. for f in self.as_yaml_files:
  57. try:
  58. os.remove(f)
  59. except Exception:
  60. pass
  61. def _add_appservice(self, as_token, id, url, hs_token, sender):
  62. as_yaml = dict(
  63. url=url,
  64. as_token=as_token,
  65. hs_token=hs_token,
  66. id=id,
  67. sender_localpart=sender,
  68. namespaces={},
  69. )
  70. # use the token as the filename
  71. with open(as_token, 'w') as outfile:
  72. outfile.write(yaml.dump(as_yaml))
  73. self.as_yaml_files.append(as_token)
  74. def test_retrieve_unknown_service_token(self):
  75. service = self.store.get_app_service_by_token("invalid_token")
  76. self.assertEquals(service, None)
  77. def test_retrieval_of_service(self):
  78. stored_service = self.store.get_app_service_by_token(self.as_token)
  79. self.assertEquals(stored_service.token, self.as_token)
  80. self.assertEquals(stored_service.id, self.as_id)
  81. self.assertEquals(stored_service.url, self.as_url)
  82. self.assertEquals(stored_service.namespaces[ApplicationService.NS_ALIASES], [])
  83. self.assertEquals(stored_service.namespaces[ApplicationService.NS_ROOMS], [])
  84. self.assertEquals(stored_service.namespaces[ApplicationService.NS_USERS], [])
  85. def test_retrieval_of_all_services(self):
  86. services = self.store.get_app_services()
  87. self.assertEquals(len(services), 3)
  88. class ApplicationServiceTransactionStoreTestCase(unittest.TestCase):
  89. @defer.inlineCallbacks
  90. def setUp(self):
  91. self.as_yaml_files = []
  92. config = Mock(
  93. app_service_config_files=self.as_yaml_files,
  94. event_cache_size=1,
  95. password_providers=[],
  96. )
  97. hs = yield setup_test_homeserver(
  98. self.addCleanup,
  99. config=config,
  100. federation_sender=Mock(),
  101. federation_client=Mock(),
  102. )
  103. self.db_pool = hs.get_db_pool()
  104. self.as_list = [
  105. {"token": "token1", "url": "https://matrix-as.org", "id": "id_1"},
  106. {"token": "alpha_tok", "url": "https://alpha.com", "id": "id_alpha"},
  107. {"token": "beta_tok", "url": "https://beta.com", "id": "id_beta"},
  108. {"token": "gamma_tok", "url": "https://gamma.com", "id": "id_gamma"},
  109. ]
  110. for s in self.as_list:
  111. yield self._add_service(s["url"], s["token"], s["id"])
  112. self.as_yaml_files = []
  113. self.store = TestTransactionStore(None, hs)
  114. def _add_service(self, url, as_token, id):
  115. as_yaml = dict(
  116. url=url,
  117. as_token=as_token,
  118. hs_token="something",
  119. id=id,
  120. sender_localpart="a_sender",
  121. namespaces={},
  122. )
  123. # use the token as the filename
  124. with open(as_token, 'w') as outfile:
  125. outfile.write(yaml.dump(as_yaml))
  126. self.as_yaml_files.append(as_token)
  127. def _set_state(self, id, state, txn=None):
  128. return self.db_pool.runQuery(
  129. "INSERT INTO application_services_state(as_id, state, last_txn) "
  130. "VALUES(?,?,?)",
  131. (id, state, txn),
  132. )
  133. def _insert_txn(self, as_id, txn_id, events):
  134. return self.db_pool.runQuery(
  135. "INSERT INTO application_services_txns(as_id, txn_id, event_ids) "
  136. "VALUES(?,?,?)",
  137. (as_id, txn_id, json.dumps([e.event_id for e in events])),
  138. )
  139. def _set_last_txn(self, as_id, txn_id):
  140. return self.db_pool.runQuery(
  141. "INSERT INTO application_services_state(as_id, last_txn, state) "
  142. "VALUES(?,?,?)",
  143. (as_id, txn_id, ApplicationServiceState.UP),
  144. )
  145. @defer.inlineCallbacks
  146. def test_get_appservice_state_none(self):
  147. service = Mock(id=999)
  148. state = yield self.store.get_appservice_state(service)
  149. self.assertEquals(None, state)
  150. @defer.inlineCallbacks
  151. def test_get_appservice_state_up(self):
  152. yield self._set_state(self.as_list[0]["id"], ApplicationServiceState.UP)
  153. service = Mock(id=self.as_list[0]["id"])
  154. state = yield self.store.get_appservice_state(service)
  155. self.assertEquals(ApplicationServiceState.UP, state)
  156. @defer.inlineCallbacks
  157. def test_get_appservice_state_down(self):
  158. yield self._set_state(self.as_list[0]["id"], ApplicationServiceState.UP)
  159. yield self._set_state(self.as_list[1]["id"], ApplicationServiceState.DOWN)
  160. yield self._set_state(self.as_list[2]["id"], ApplicationServiceState.DOWN)
  161. service = Mock(id=self.as_list[1]["id"])
  162. state = yield self.store.get_appservice_state(service)
  163. self.assertEquals(ApplicationServiceState.DOWN, state)
  164. @defer.inlineCallbacks
  165. def test_get_appservices_by_state_none(self):
  166. services = yield self.store.get_appservices_by_state(
  167. ApplicationServiceState.DOWN
  168. )
  169. self.assertEquals(0, len(services))
  170. @defer.inlineCallbacks
  171. def test_set_appservices_state_down(self):
  172. service = Mock(id=self.as_list[1]["id"])
  173. yield self.store.set_appservice_state(service, ApplicationServiceState.DOWN)
  174. rows = yield self.db_pool.runQuery(
  175. "SELECT as_id FROM application_services_state WHERE state=?",
  176. (ApplicationServiceState.DOWN,),
  177. )
  178. self.assertEquals(service.id, rows[0][0])
  179. @defer.inlineCallbacks
  180. def test_set_appservices_state_multiple_up(self):
  181. service = Mock(id=self.as_list[1]["id"])
  182. yield self.store.set_appservice_state(service, ApplicationServiceState.UP)
  183. yield self.store.set_appservice_state(service, ApplicationServiceState.DOWN)
  184. yield self.store.set_appservice_state(service, ApplicationServiceState.UP)
  185. rows = yield self.db_pool.runQuery(
  186. "SELECT as_id FROM application_services_state WHERE state=?",
  187. (ApplicationServiceState.UP,),
  188. )
  189. self.assertEquals(service.id, rows[0][0])
  190. @defer.inlineCallbacks
  191. def test_create_appservice_txn_first(self):
  192. service = Mock(id=self.as_list[0]["id"])
  193. events = [Mock(event_id="e1"), Mock(event_id="e2")]
  194. txn = yield self.store.create_appservice_txn(service, events)
  195. self.assertEquals(txn.id, 1)
  196. self.assertEquals(txn.events, events)
  197. self.assertEquals(txn.service, service)
  198. @defer.inlineCallbacks
  199. def test_create_appservice_txn_older_last_txn(self):
  200. service = Mock(id=self.as_list[0]["id"])
  201. events = [Mock(event_id="e1"), Mock(event_id="e2")]
  202. yield self._set_last_txn(service.id, 9643) # AS is falling behind
  203. yield self._insert_txn(service.id, 9644, events)
  204. yield self._insert_txn(service.id, 9645, events)
  205. txn = yield self.store.create_appservice_txn(service, events)
  206. self.assertEquals(txn.id, 9646)
  207. self.assertEquals(txn.events, events)
  208. self.assertEquals(txn.service, service)
  209. @defer.inlineCallbacks
  210. def test_create_appservice_txn_up_to_date_last_txn(self):
  211. service = Mock(id=self.as_list[0]["id"])
  212. events = [Mock(event_id="e1"), Mock(event_id="e2")]
  213. yield self._set_last_txn(service.id, 9643)
  214. txn = yield self.store.create_appservice_txn(service, events)
  215. self.assertEquals(txn.id, 9644)
  216. self.assertEquals(txn.events, events)
  217. self.assertEquals(txn.service, service)
  218. @defer.inlineCallbacks
  219. def test_create_appservice_txn_up_fuzzing(self):
  220. service = Mock(id=self.as_list[0]["id"])
  221. events = [Mock(event_id="e1"), Mock(event_id="e2")]
  222. yield self._set_last_txn(service.id, 9643)
  223. # dump in rows with higher IDs to make sure the queries aren't wrong.
  224. yield self._set_last_txn(self.as_list[1]["id"], 119643)
  225. yield self._set_last_txn(self.as_list[2]["id"], 9)
  226. yield self._set_last_txn(self.as_list[3]["id"], 9643)
  227. yield self._insert_txn(self.as_list[1]["id"], 119644, events)
  228. yield self._insert_txn(self.as_list[1]["id"], 119645, events)
  229. yield self._insert_txn(self.as_list[1]["id"], 119646, events)
  230. yield self._insert_txn(self.as_list[2]["id"], 10, events)
  231. yield self._insert_txn(self.as_list[3]["id"], 9643, events)
  232. txn = yield self.store.create_appservice_txn(service, events)
  233. self.assertEquals(txn.id, 9644)
  234. self.assertEquals(txn.events, events)
  235. self.assertEquals(txn.service, service)
  236. @defer.inlineCallbacks
  237. def test_complete_appservice_txn_first_txn(self):
  238. service = Mock(id=self.as_list[0]["id"])
  239. events = [Mock(event_id="e1"), Mock(event_id="e2")]
  240. txn_id = 1
  241. yield self._insert_txn(service.id, txn_id, events)
  242. yield self.store.complete_appservice_txn(txn_id=txn_id, service=service)
  243. res = yield self.db_pool.runQuery(
  244. "SELECT last_txn FROM application_services_state WHERE as_id=?",
  245. (service.id,),
  246. )
  247. self.assertEquals(1, len(res))
  248. self.assertEquals(txn_id, res[0][0])
  249. res = yield self.db_pool.runQuery(
  250. "SELECT * FROM application_services_txns WHERE txn_id=?", (txn_id,)
  251. )
  252. self.assertEquals(0, len(res))
  253. @defer.inlineCallbacks
  254. def test_complete_appservice_txn_existing_in_state_table(self):
  255. service = Mock(id=self.as_list[0]["id"])
  256. events = [Mock(event_id="e1"), Mock(event_id="e2")]
  257. txn_id = 5
  258. yield self._set_last_txn(service.id, 4)
  259. yield self._insert_txn(service.id, txn_id, events)
  260. yield self.store.complete_appservice_txn(txn_id=txn_id, service=service)
  261. res = yield self.db_pool.runQuery(
  262. "SELECT last_txn, state FROM application_services_state WHERE " "as_id=?",
  263. (service.id,),
  264. )
  265. self.assertEquals(1, len(res))
  266. self.assertEquals(txn_id, res[0][0])
  267. self.assertEquals(ApplicationServiceState.UP, res[0][1])
  268. res = yield self.db_pool.runQuery(
  269. "SELECT * FROM application_services_txns WHERE txn_id=?", (txn_id,)
  270. )
  271. self.assertEquals(0, len(res))
  272. @defer.inlineCallbacks
  273. def test_get_oldest_unsent_txn_none(self):
  274. service = Mock(id=self.as_list[0]["id"])
  275. txn = yield self.store.get_oldest_unsent_txn(service)
  276. self.assertEquals(None, txn)
  277. @defer.inlineCallbacks
  278. def test_get_oldest_unsent_txn(self):
  279. service = Mock(id=self.as_list[0]["id"])
  280. events = [Mock(event_id="e1"), Mock(event_id="e2")]
  281. other_events = [Mock(event_id="e5"), Mock(event_id="e6")]
  282. # we aren't testing store._base stuff here, so mock this out
  283. self.store._get_events = Mock(return_value=events)
  284. yield self._insert_txn(self.as_list[1]["id"], 9, other_events)
  285. yield self._insert_txn(service.id, 10, events)
  286. yield self._insert_txn(service.id, 11, other_events)
  287. yield self._insert_txn(service.id, 12, other_events)
  288. txn = yield self.store.get_oldest_unsent_txn(service)
  289. self.assertEquals(service, txn.service)
  290. self.assertEquals(10, txn.id)
  291. self.assertEquals(events, txn.events)
  292. @defer.inlineCallbacks
  293. def test_get_appservices_by_state_single(self):
  294. yield self._set_state(self.as_list[0]["id"], ApplicationServiceState.DOWN)
  295. yield self._set_state(self.as_list[1]["id"], ApplicationServiceState.UP)
  296. services = yield self.store.get_appservices_by_state(
  297. ApplicationServiceState.DOWN
  298. )
  299. self.assertEquals(1, len(services))
  300. self.assertEquals(self.as_list[0]["id"], services[0].id)
  301. @defer.inlineCallbacks
  302. def test_get_appservices_by_state_multiple(self):
  303. yield self._set_state(self.as_list[0]["id"], ApplicationServiceState.DOWN)
  304. yield self._set_state(self.as_list[1]["id"], ApplicationServiceState.UP)
  305. yield self._set_state(self.as_list[2]["id"], ApplicationServiceState.DOWN)
  306. yield self._set_state(self.as_list[3]["id"], ApplicationServiceState.UP)
  307. services = yield self.store.get_appservices_by_state(
  308. ApplicationServiceState.DOWN
  309. )
  310. self.assertEquals(2, len(services))
  311. self.assertEquals(
  312. set([self.as_list[2]["id"], self.as_list[0]["id"]]),
  313. set([services[0].id, services[1].id]),
  314. )
  315. # required for ApplicationServiceTransactionStoreTestCase tests
  316. class TestTransactionStore(ApplicationServiceTransactionStore, ApplicationServiceStore):
  317. def __init__(self, db_conn, hs):
  318. super(TestTransactionStore, self).__init__(db_conn, hs)
  319. class ApplicationServiceStoreConfigTestCase(unittest.TestCase):
  320. def _write_config(self, suffix, **kwargs):
  321. vals = {
  322. "id": "id" + suffix,
  323. "url": "url" + suffix,
  324. "as_token": "as_token" + suffix,
  325. "hs_token": "hs_token" + suffix,
  326. "sender_localpart": "sender_localpart" + suffix,
  327. "namespaces": {},
  328. }
  329. vals.update(kwargs)
  330. _, path = tempfile.mkstemp(prefix="as_config")
  331. with open(path, "w") as f:
  332. f.write(yaml.dump(vals))
  333. return path
  334. @defer.inlineCallbacks
  335. def test_unique_works(self):
  336. f1 = self._write_config(suffix="1")
  337. f2 = self._write_config(suffix="2")
  338. config = Mock(
  339. app_service_config_files=[f1, f2], event_cache_size=1, password_providers=[]
  340. )
  341. hs = yield setup_test_homeserver(
  342. self.addCleanup,
  343. config=config,
  344. datastore=Mock(),
  345. federation_sender=Mock(),
  346. federation_client=Mock(),
  347. )
  348. ApplicationServiceStore(None, hs)
  349. @defer.inlineCallbacks
  350. def test_duplicate_ids(self):
  351. f1 = self._write_config(id="id", suffix="1")
  352. f2 = self._write_config(id="id", suffix="2")
  353. config = Mock(
  354. app_service_config_files=[f1, f2], event_cache_size=1, password_providers=[]
  355. )
  356. hs = yield setup_test_homeserver(
  357. self.addCleanup,
  358. config=config,
  359. datastore=Mock(),
  360. federation_sender=Mock(),
  361. federation_client=Mock(),
  362. )
  363. with self.assertRaises(ConfigError) as cm:
  364. ApplicationServiceStore(None, hs)
  365. e = cm.exception
  366. self.assertIn(f1, str(e))
  367. self.assertIn(f2, str(e))
  368. self.assertIn("id", str(e))
  369. @defer.inlineCallbacks
  370. def test_duplicate_as_tokens(self):
  371. f1 = self._write_config(as_token="as_token", suffix="1")
  372. f2 = self._write_config(as_token="as_token", suffix="2")
  373. config = Mock(
  374. app_service_config_files=[f1, f2], event_cache_size=1, password_providers=[]
  375. )
  376. hs = yield setup_test_homeserver(
  377. self.addCleanup,
  378. config=config,
  379. datastore=Mock(),
  380. federation_sender=Mock(),
  381. federation_client=Mock(),
  382. )
  383. with self.assertRaises(ConfigError) as cm:
  384. ApplicationServiceStore(None, hs)
  385. e = cm.exception
  386. self.assertIn(f1, str(e))
  387. self.assertIn(f2, str(e))
  388. self.assertIn("as_token", str(e))