1
0

test_client_ips.py 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726
  1. # Copyright 2016 OpenMarket Ltd
  2. # Copyright 2018 New Vector Ltd
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License");
  5. # you may not use this file except in compliance with the License.
  6. # You may obtain a copy of the License at
  7. #
  8. # http://www.apache.org/licenses/LICENSE-2.0
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS,
  12. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13. # See the License for the specific language governing permissions and
  14. # limitations under the License.
  15. from unittest.mock import Mock
  16. from parameterized import parameterized
  17. import synapse.rest.admin
  18. from synapse.http.site import XForwardedForRequest
  19. from synapse.rest.client import login
  20. from synapse.storage.databases.main.client_ips import LAST_SEEN_GRANULARITY
  21. from synapse.types import UserID
  22. from tests import unittest
  23. from tests.server import make_request
  24. from tests.test_utils import make_awaitable
  25. from tests.unittest import override_config
  26. class ClientIpStoreTestCase(unittest.HomeserverTestCase):
  27. def make_homeserver(self, reactor, clock):
  28. hs = self.setup_test_homeserver()
  29. return hs
  30. def prepare(self, hs, reactor, clock):
  31. self.store = self.hs.get_datastores().main
  32. def test_insert_new_client_ip(self):
  33. self.reactor.advance(12345678)
  34. user_id = "@user:id"
  35. device_id = "MY_DEVICE"
  36. # Insert a user IP
  37. self.get_success(
  38. self.store.store_device(
  39. user_id,
  40. device_id,
  41. "display name",
  42. )
  43. )
  44. self.get_success(
  45. self.store.insert_client_ip(
  46. user_id, "access_token", "ip", "user_agent", device_id
  47. )
  48. )
  49. # Trigger the storage loop
  50. self.reactor.advance(10)
  51. result = self.get_success(
  52. self.store.get_last_client_ip_by_device(user_id, device_id)
  53. )
  54. r = result[(user_id, device_id)]
  55. self.assertDictContainsSubset(
  56. {
  57. "user_id": user_id,
  58. "device_id": device_id,
  59. "ip": "ip",
  60. "user_agent": "user_agent",
  61. "last_seen": 12345678000,
  62. },
  63. r,
  64. )
  65. def test_insert_new_client_ip_none_device_id(self):
  66. """
  67. An insert with a device ID of NULL will not create a new entry, but
  68. update an existing entry in the user_ips table.
  69. """
  70. self.reactor.advance(12345678)
  71. user_id = "@user:id"
  72. # Add & trigger the storage loop
  73. self.get_success(
  74. self.store.insert_client_ip(
  75. user_id, "access_token", "ip", "user_agent", None
  76. )
  77. )
  78. self.reactor.advance(200)
  79. self.pump(0)
  80. result = self.get_success(
  81. self.store.db_pool.simple_select_list(
  82. table="user_ips",
  83. keyvalues={"user_id": user_id},
  84. retcols=["access_token", "ip", "user_agent", "device_id", "last_seen"],
  85. desc="get_user_ip_and_agents",
  86. )
  87. )
  88. self.assertEqual(
  89. result,
  90. [
  91. {
  92. "access_token": "access_token",
  93. "ip": "ip",
  94. "user_agent": "user_agent",
  95. "device_id": None,
  96. "last_seen": 12345678000,
  97. }
  98. ],
  99. )
  100. # Add another & trigger the storage loop
  101. self.get_success(
  102. self.store.insert_client_ip(
  103. user_id, "access_token", "ip", "user_agent", None
  104. )
  105. )
  106. self.reactor.advance(10)
  107. self.pump(0)
  108. result = self.get_success(
  109. self.store.db_pool.simple_select_list(
  110. table="user_ips",
  111. keyvalues={"user_id": user_id},
  112. retcols=["access_token", "ip", "user_agent", "device_id", "last_seen"],
  113. desc="get_user_ip_and_agents",
  114. )
  115. )
  116. # Only one result, has been upserted.
  117. self.assertEqual(
  118. result,
  119. [
  120. {
  121. "access_token": "access_token",
  122. "ip": "ip",
  123. "user_agent": "user_agent",
  124. "device_id": None,
  125. "last_seen": 12345878000,
  126. }
  127. ],
  128. )
  129. @parameterized.expand([(False,), (True,)])
  130. def test_get_last_client_ip_by_device(self, after_persisting: bool):
  131. """Test `get_last_client_ip_by_device` for persisted and unpersisted data"""
  132. self.reactor.advance(12345678)
  133. user_id = "@user:id"
  134. device_id = "MY_DEVICE"
  135. # Insert a user IP
  136. self.get_success(
  137. self.store.store_device(
  138. user_id,
  139. device_id,
  140. "display name",
  141. )
  142. )
  143. self.get_success(
  144. self.store.insert_client_ip(
  145. user_id, "access_token", "ip", "user_agent", device_id
  146. )
  147. )
  148. if after_persisting:
  149. # Trigger the storage loop
  150. self.reactor.advance(10)
  151. else:
  152. # Check that the new IP and user agent has not been stored yet
  153. db_result = self.get_success(
  154. self.store.db_pool.simple_select_list(
  155. table="devices",
  156. keyvalues={},
  157. retcols=("user_id", "ip", "user_agent", "device_id", "last_seen"),
  158. ),
  159. )
  160. self.assertEqual(
  161. db_result,
  162. [
  163. {
  164. "user_id": user_id,
  165. "device_id": device_id,
  166. "ip": None,
  167. "user_agent": None,
  168. "last_seen": None,
  169. },
  170. ],
  171. )
  172. result = self.get_success(
  173. self.store.get_last_client_ip_by_device(user_id, device_id)
  174. )
  175. self.assertEqual(
  176. result,
  177. {
  178. (user_id, device_id): {
  179. "user_id": user_id,
  180. "device_id": device_id,
  181. "ip": "ip",
  182. "user_agent": "user_agent",
  183. "last_seen": 12345678000,
  184. },
  185. },
  186. )
  187. def test_get_last_client_ip_by_device_combined_data(self):
  188. """Test that `get_last_client_ip_by_device` combines persisted and unpersisted
  189. data together correctly
  190. """
  191. self.reactor.advance(12345678)
  192. user_id = "@user:id"
  193. device_id_1 = "MY_DEVICE_1"
  194. device_id_2 = "MY_DEVICE_2"
  195. # Insert user IPs
  196. self.get_success(
  197. self.store.store_device(
  198. user_id,
  199. device_id_1,
  200. "display name",
  201. )
  202. )
  203. self.get_success(
  204. self.store.store_device(
  205. user_id,
  206. device_id_2,
  207. "display name",
  208. )
  209. )
  210. self.get_success(
  211. self.store.insert_client_ip(
  212. user_id, "access_token_1", "ip_1", "user_agent_1", device_id_1
  213. )
  214. )
  215. self.get_success(
  216. self.store.insert_client_ip(
  217. user_id, "access_token_2", "ip_2", "user_agent_2", device_id_2
  218. )
  219. )
  220. # Trigger the storage loop and wait for the rate limiting period to be over
  221. self.reactor.advance(10 + LAST_SEEN_GRANULARITY / 1000)
  222. # Update the user agent for the second device, without running the storage loop
  223. self.get_success(
  224. self.store.insert_client_ip(
  225. user_id, "access_token_2", "ip_2", "user_agent_3", device_id_2
  226. )
  227. )
  228. # Check that the new IP and user agent has not been stored yet
  229. db_result = self.get_success(
  230. self.store.db_pool.simple_select_list(
  231. table="devices",
  232. keyvalues={},
  233. retcols=("user_id", "ip", "user_agent", "device_id", "last_seen"),
  234. ),
  235. )
  236. self.assertCountEqual(
  237. db_result,
  238. [
  239. {
  240. "user_id": user_id,
  241. "device_id": device_id_1,
  242. "ip": "ip_1",
  243. "user_agent": "user_agent_1",
  244. "last_seen": 12345678000,
  245. },
  246. {
  247. "user_id": user_id,
  248. "device_id": device_id_2,
  249. "ip": "ip_2",
  250. "user_agent": "user_agent_2",
  251. "last_seen": 12345678000,
  252. },
  253. ],
  254. )
  255. # Check that data from the database and memory are combined together correctly
  256. result = self.get_success(
  257. self.store.get_last_client_ip_by_device(user_id, None)
  258. )
  259. self.assertEqual(
  260. result,
  261. {
  262. (user_id, device_id_1): {
  263. "user_id": user_id,
  264. "device_id": device_id_1,
  265. "ip": "ip_1",
  266. "user_agent": "user_agent_1",
  267. "last_seen": 12345678000,
  268. },
  269. (user_id, device_id_2): {
  270. "user_id": user_id,
  271. "device_id": device_id_2,
  272. "ip": "ip_2",
  273. "user_agent": "user_agent_3",
  274. "last_seen": 12345688000 + LAST_SEEN_GRANULARITY,
  275. },
  276. },
  277. )
  278. @parameterized.expand([(False,), (True,)])
  279. def test_get_user_ip_and_agents(self, after_persisting: bool):
  280. """Test `get_user_ip_and_agents` for persisted and unpersisted data"""
  281. self.reactor.advance(12345678)
  282. user_id = "@user:id"
  283. user = UserID.from_string(user_id)
  284. # Insert a user IP
  285. self.get_success(
  286. self.store.insert_client_ip(
  287. user_id, "access_token", "ip", "user_agent", "MY_DEVICE"
  288. )
  289. )
  290. if after_persisting:
  291. # Trigger the storage loop
  292. self.reactor.advance(10)
  293. else:
  294. # Check that the new IP and user agent has not been stored yet
  295. db_result = self.get_success(
  296. self.store.db_pool.simple_select_list(
  297. table="user_ips",
  298. keyvalues={},
  299. retcols=("access_token", "ip", "user_agent", "last_seen"),
  300. ),
  301. )
  302. self.assertEqual(db_result, [])
  303. self.assertEqual(
  304. self.get_success(self.store.get_user_ip_and_agents(user)),
  305. [
  306. {
  307. "access_token": "access_token",
  308. "ip": "ip",
  309. "user_agent": "user_agent",
  310. "last_seen": 12345678000,
  311. },
  312. ],
  313. )
  314. def test_get_user_ip_and_agents_combined_data(self):
  315. """Test that `get_user_ip_and_agents` combines persisted and unpersisted data
  316. together correctly
  317. """
  318. self.reactor.advance(12345678)
  319. user_id = "@user:id"
  320. user = UserID.from_string(user_id)
  321. # Insert user IPs
  322. self.get_success(
  323. self.store.insert_client_ip(
  324. user_id, "access_token", "ip_1", "user_agent_1", "MY_DEVICE_1"
  325. )
  326. )
  327. self.get_success(
  328. self.store.insert_client_ip(
  329. user_id, "access_token", "ip_2", "user_agent_2", "MY_DEVICE_2"
  330. )
  331. )
  332. # Trigger the storage loop and wait for the rate limiting period to be over
  333. self.reactor.advance(10 + LAST_SEEN_GRANULARITY / 1000)
  334. # Update the user agent for the second device, without running the storage loop
  335. self.get_success(
  336. self.store.insert_client_ip(
  337. user_id, "access_token", "ip_2", "user_agent_3", "MY_DEVICE_2"
  338. )
  339. )
  340. # Check that the new IP and user agent has not been stored yet
  341. db_result = self.get_success(
  342. self.store.db_pool.simple_select_list(
  343. table="user_ips",
  344. keyvalues={},
  345. retcols=("access_token", "ip", "user_agent", "last_seen"),
  346. ),
  347. )
  348. self.assertEqual(
  349. db_result,
  350. [
  351. {
  352. "access_token": "access_token",
  353. "ip": "ip_1",
  354. "user_agent": "user_agent_1",
  355. "last_seen": 12345678000,
  356. },
  357. {
  358. "access_token": "access_token",
  359. "ip": "ip_2",
  360. "user_agent": "user_agent_2",
  361. "last_seen": 12345678000,
  362. },
  363. ],
  364. )
  365. # Check that data from the database and memory are combined together correctly
  366. self.assertCountEqual(
  367. self.get_success(self.store.get_user_ip_and_agents(user)),
  368. [
  369. {
  370. "access_token": "access_token",
  371. "ip": "ip_1",
  372. "user_agent": "user_agent_1",
  373. "last_seen": 12345678000,
  374. },
  375. {
  376. "access_token": "access_token",
  377. "ip": "ip_2",
  378. "user_agent": "user_agent_3",
  379. "last_seen": 12345688000 + LAST_SEEN_GRANULARITY,
  380. },
  381. ],
  382. )
  383. @override_config({"limit_usage_by_mau": False, "max_mau_value": 50})
  384. def test_disabled_monthly_active_user(self):
  385. user_id = "@user:server"
  386. self.get_success(
  387. self.store.insert_client_ip(
  388. user_id, "access_token", "ip", "user_agent", "device_id"
  389. )
  390. )
  391. active = self.get_success(self.store.user_last_seen_monthly_active(user_id))
  392. self.assertFalse(active)
  393. @override_config({"limit_usage_by_mau": True, "max_mau_value": 50})
  394. def test_adding_monthly_active_user_when_full(self):
  395. lots_of_users = 100
  396. user_id = "@user:server"
  397. self.store.get_monthly_active_count = Mock(
  398. return_value=make_awaitable(lots_of_users)
  399. )
  400. self.get_success(
  401. self.store.insert_client_ip(
  402. user_id, "access_token", "ip", "user_agent", "device_id"
  403. )
  404. )
  405. active = self.get_success(self.store.user_last_seen_monthly_active(user_id))
  406. self.assertFalse(active)
  407. @override_config({"limit_usage_by_mau": True, "max_mau_value": 50})
  408. def test_adding_monthly_active_user_when_space(self):
  409. user_id = "@user:server"
  410. active = self.get_success(self.store.user_last_seen_monthly_active(user_id))
  411. self.assertFalse(active)
  412. # Trigger the saving loop
  413. self.reactor.advance(10)
  414. self.get_success(
  415. self.store.insert_client_ip(
  416. user_id, "access_token", "ip", "user_agent", "device_id"
  417. )
  418. )
  419. active = self.get_success(self.store.user_last_seen_monthly_active(user_id))
  420. self.assertTrue(active)
  421. @override_config({"limit_usage_by_mau": True, "max_mau_value": 50})
  422. def test_updating_monthly_active_user_when_space(self):
  423. user_id = "@user:server"
  424. self.get_success(self.store.register_user(user_id=user_id, password_hash=None))
  425. active = self.get_success(self.store.user_last_seen_monthly_active(user_id))
  426. self.assertFalse(active)
  427. # Trigger the saving loop
  428. self.reactor.advance(10)
  429. self.get_success(
  430. self.store.insert_client_ip(
  431. user_id, "access_token", "ip", "user_agent", "device_id"
  432. )
  433. )
  434. active = self.get_success(self.store.user_last_seen_monthly_active(user_id))
  435. self.assertTrue(active)
  436. def test_devices_last_seen_bg_update(self):
  437. # First make sure we have completed all updates.
  438. self.wait_for_background_updates()
  439. user_id = "@user:id"
  440. device_id = "MY_DEVICE"
  441. # Insert a user IP
  442. self.get_success(
  443. self.store.store_device(
  444. user_id,
  445. device_id,
  446. "display name",
  447. )
  448. )
  449. self.get_success(
  450. self.store.insert_client_ip(
  451. user_id, "access_token", "ip", "user_agent", device_id
  452. )
  453. )
  454. # Force persisting to disk
  455. self.reactor.advance(200)
  456. # But clear the associated entry in devices table
  457. self.get_success(
  458. self.store.db_pool.simple_update(
  459. table="devices",
  460. keyvalues={"user_id": user_id, "device_id": device_id},
  461. updatevalues={"last_seen": None, "ip": None, "user_agent": None},
  462. desc="test_devices_last_seen_bg_update",
  463. )
  464. )
  465. # We should now get nulls when querying
  466. result = self.get_success(
  467. self.store.get_last_client_ip_by_device(user_id, device_id)
  468. )
  469. r = result[(user_id, device_id)]
  470. self.assertDictContainsSubset(
  471. {
  472. "user_id": user_id,
  473. "device_id": device_id,
  474. "ip": None,
  475. "user_agent": None,
  476. "last_seen": None,
  477. },
  478. r,
  479. )
  480. # Register the background update to run again.
  481. self.get_success(
  482. self.store.db_pool.simple_insert(
  483. table="background_updates",
  484. values={
  485. "update_name": "devices_last_seen",
  486. "progress_json": "{}",
  487. "depends_on": None,
  488. },
  489. )
  490. )
  491. # ... and tell the DataStore that it hasn't finished all updates yet
  492. self.store.db_pool.updates._all_done = False
  493. # Now let's actually drive the updates to completion
  494. self.wait_for_background_updates()
  495. # We should now get the correct result again
  496. result = self.get_success(
  497. self.store.get_last_client_ip_by_device(user_id, device_id)
  498. )
  499. r = result[(user_id, device_id)]
  500. self.assertDictContainsSubset(
  501. {
  502. "user_id": user_id,
  503. "device_id": device_id,
  504. "ip": "ip",
  505. "user_agent": "user_agent",
  506. "last_seen": 0,
  507. },
  508. r,
  509. )
  510. def test_old_user_ips_pruned(self):
  511. # First make sure we have completed all updates.
  512. self.wait_for_background_updates()
  513. user_id = "@user:id"
  514. device_id = "MY_DEVICE"
  515. # Insert a user IP
  516. self.get_success(
  517. self.store.store_device(
  518. user_id,
  519. device_id,
  520. "display name",
  521. )
  522. )
  523. self.get_success(
  524. self.store.insert_client_ip(
  525. user_id, "access_token", "ip", "user_agent", device_id
  526. )
  527. )
  528. # Force persisting to disk
  529. self.reactor.advance(200)
  530. # We should see that in the DB
  531. result = self.get_success(
  532. self.store.db_pool.simple_select_list(
  533. table="user_ips",
  534. keyvalues={"user_id": user_id},
  535. retcols=["access_token", "ip", "user_agent", "device_id", "last_seen"],
  536. desc="get_user_ip_and_agents",
  537. )
  538. )
  539. self.assertEqual(
  540. result,
  541. [
  542. {
  543. "access_token": "access_token",
  544. "ip": "ip",
  545. "user_agent": "user_agent",
  546. "device_id": device_id,
  547. "last_seen": 0,
  548. }
  549. ],
  550. )
  551. # Now advance by a couple of months
  552. self.reactor.advance(60 * 24 * 60 * 60)
  553. # We should get no results.
  554. result = self.get_success(
  555. self.store.db_pool.simple_select_list(
  556. table="user_ips",
  557. keyvalues={"user_id": user_id},
  558. retcols=["access_token", "ip", "user_agent", "device_id", "last_seen"],
  559. desc="get_user_ip_and_agents",
  560. )
  561. )
  562. self.assertEqual(result, [])
  563. # But we should still get the correct values for the device
  564. result = self.get_success(
  565. self.store.get_last_client_ip_by_device(user_id, device_id)
  566. )
  567. r = result[(user_id, device_id)]
  568. self.assertDictContainsSubset(
  569. {
  570. "user_id": user_id,
  571. "device_id": device_id,
  572. "ip": "ip",
  573. "user_agent": "user_agent",
  574. "last_seen": 0,
  575. },
  576. r,
  577. )
  578. class ClientIpAuthTestCase(unittest.HomeserverTestCase):
  579. servlets = [
  580. synapse.rest.admin.register_servlets,
  581. login.register_servlets,
  582. ]
  583. def make_homeserver(self, reactor, clock):
  584. hs = self.setup_test_homeserver()
  585. return hs
  586. def prepare(self, hs, reactor, clock):
  587. self.store = self.hs.get_datastores().main
  588. self.user_id = self.register_user("bob", "abc123", True)
  589. def test_request_with_xforwarded(self):
  590. """
  591. The IP in X-Forwarded-For is entered into the client IPs table.
  592. """
  593. self._runtest(
  594. {b"X-Forwarded-For": b"127.9.0.1"},
  595. "127.9.0.1",
  596. {"request": XForwardedForRequest},
  597. )
  598. def test_request_from_getPeer(self):
  599. """
  600. The IP returned by getPeer is entered into the client IPs table, if
  601. there's no X-Forwarded-For header.
  602. """
  603. self._runtest({}, "127.0.0.1", {})
  604. def _runtest(self, headers, expected_ip, make_request_args):
  605. device_id = "bleb"
  606. access_token = self.login("bob", "abc123", device_id=device_id)
  607. # Advance to a known time
  608. self.reactor.advance(123456 - self.reactor.seconds())
  609. headers1 = {b"User-Agent": b"Mozzila pizza"}
  610. headers1.update(headers)
  611. make_request(
  612. self.reactor,
  613. self.site,
  614. "GET",
  615. "/_synapse/admin/v2/users/" + self.user_id,
  616. access_token=access_token,
  617. custom_headers=headers1.items(),
  618. **make_request_args,
  619. )
  620. # Advance so the save loop occurs
  621. self.reactor.advance(100)
  622. result = self.get_success(
  623. self.store.get_last_client_ip_by_device(self.user_id, device_id)
  624. )
  625. r = result[(self.user_id, device_id)]
  626. self.assertDictContainsSubset(
  627. {
  628. "user_id": self.user_id,
  629. "device_id": device_id,
  630. "ip": expected_ip,
  631. "user_agent": "Mozzila pizza",
  632. "last_seen": 123456100,
  633. },
  634. r,
  635. )