test_client_ips.py 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727
  1. # Copyright 2016 OpenMarket Ltd
  2. # Copyright 2018 New Vector Ltd
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License");
  5. # you may not use this file except in compliance with the License.
  6. # You may obtain a copy of the License at
  7. #
  8. # http://www.apache.org/licenses/LICENSE-2.0
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS,
  12. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13. # See the License for the specific language governing permissions and
  14. # limitations under the License.
  15. from typing import Any, Dict
  16. from unittest.mock import Mock
  17. from parameterized import parameterized
  18. from twisted.test.proto_helpers import MemoryReactor
  19. import synapse.rest.admin
  20. from synapse.http.site import XForwardedForRequest
  21. from synapse.rest.client import login
  22. from synapse.server import HomeServer
  23. from synapse.storage.databases.main.client_ips import LAST_SEEN_GRANULARITY
  24. from synapse.types import UserID
  25. from synapse.util import Clock
  26. from tests import unittest
  27. from tests.server import make_request
  28. from tests.test_utils import make_awaitable
  29. from tests.unittest import override_config
  30. class ClientIpStoreTestCase(unittest.HomeserverTestCase):
  31. def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None:
  32. self.store = hs.get_datastores().main
  33. def test_insert_new_client_ip(self) -> None:
  34. self.reactor.advance(12345678)
  35. user_id = "@user:id"
  36. device_id = "MY_DEVICE"
  37. # Insert a user IP
  38. self.get_success(
  39. self.store.store_device(
  40. user_id,
  41. device_id,
  42. "display name",
  43. )
  44. )
  45. self.get_success(
  46. self.store.insert_client_ip(
  47. user_id, "access_token", "ip", "user_agent", device_id
  48. )
  49. )
  50. # Trigger the storage loop
  51. self.reactor.advance(10)
  52. result = self.get_success(
  53. self.store.get_last_client_ip_by_device(user_id, device_id)
  54. )
  55. r = result[(user_id, device_id)]
  56. self.assertDictContainsSubset(
  57. {
  58. "user_id": user_id,
  59. "device_id": device_id,
  60. "ip": "ip",
  61. "user_agent": "user_agent",
  62. "last_seen": 12345678000,
  63. },
  64. r,
  65. )
  66. def test_insert_new_client_ip_none_device_id(self) -> None:
  67. """
  68. An insert with a device ID of NULL will not create a new entry, but
  69. update an existing entry in the user_ips table.
  70. """
  71. self.reactor.advance(12345678)
  72. user_id = "@user:id"
  73. # Add & trigger the storage loop
  74. self.get_success(
  75. self.store.insert_client_ip(
  76. user_id, "access_token", "ip", "user_agent", None
  77. )
  78. )
  79. self.reactor.advance(200)
  80. self.pump(0)
  81. result = self.get_success(
  82. self.store.db_pool.simple_select_list(
  83. table="user_ips",
  84. keyvalues={"user_id": user_id},
  85. retcols=["access_token", "ip", "user_agent", "device_id", "last_seen"],
  86. desc="get_user_ip_and_agents",
  87. )
  88. )
  89. self.assertEqual(
  90. result,
  91. [
  92. {
  93. "access_token": "access_token",
  94. "ip": "ip",
  95. "user_agent": "user_agent",
  96. "device_id": None,
  97. "last_seen": 12345678000,
  98. }
  99. ],
  100. )
  101. # Add another & trigger the storage loop
  102. self.get_success(
  103. self.store.insert_client_ip(
  104. user_id, "access_token", "ip", "user_agent", None
  105. )
  106. )
  107. self.reactor.advance(10)
  108. self.pump(0)
  109. result = self.get_success(
  110. self.store.db_pool.simple_select_list(
  111. table="user_ips",
  112. keyvalues={"user_id": user_id},
  113. retcols=["access_token", "ip", "user_agent", "device_id", "last_seen"],
  114. desc="get_user_ip_and_agents",
  115. )
  116. )
  117. # Only one result, has been upserted.
  118. self.assertEqual(
  119. result,
  120. [
  121. {
  122. "access_token": "access_token",
  123. "ip": "ip",
  124. "user_agent": "user_agent",
  125. "device_id": None,
  126. "last_seen": 12345878000,
  127. }
  128. ],
  129. )
  130. @parameterized.expand([(False,), (True,)])
  131. def test_get_last_client_ip_by_device(self, after_persisting: bool) -> None:
  132. """Test `get_last_client_ip_by_device` for persisted and unpersisted data"""
  133. self.reactor.advance(12345678)
  134. user_id = "@user:id"
  135. device_id = "MY_DEVICE"
  136. # Insert a user IP
  137. self.get_success(
  138. self.store.store_device(
  139. user_id,
  140. device_id,
  141. "display name",
  142. )
  143. )
  144. self.get_success(
  145. self.store.insert_client_ip(
  146. user_id, "access_token", "ip", "user_agent", device_id
  147. )
  148. )
  149. if after_persisting:
  150. # Trigger the storage loop
  151. self.reactor.advance(10)
  152. else:
  153. # Check that the new IP and user agent has not been stored yet
  154. db_result = self.get_success(
  155. self.store.db_pool.simple_select_list(
  156. table="devices",
  157. keyvalues={},
  158. retcols=("user_id", "ip", "user_agent", "device_id", "last_seen"),
  159. ),
  160. )
  161. self.assertEqual(
  162. db_result,
  163. [
  164. {
  165. "user_id": user_id,
  166. "device_id": device_id,
  167. "ip": None,
  168. "user_agent": None,
  169. "last_seen": None,
  170. },
  171. ],
  172. )
  173. result = self.get_success(
  174. self.store.get_last_client_ip_by_device(user_id, device_id)
  175. )
  176. self.assertEqual(
  177. result,
  178. {
  179. (user_id, device_id): {
  180. "user_id": user_id,
  181. "device_id": device_id,
  182. "ip": "ip",
  183. "user_agent": "user_agent",
  184. "last_seen": 12345678000,
  185. },
  186. },
  187. )
  188. def test_get_last_client_ip_by_device_combined_data(self) -> None:
  189. """Test that `get_last_client_ip_by_device` combines persisted and unpersisted
  190. data together correctly
  191. """
  192. self.reactor.advance(12345678)
  193. user_id = "@user:id"
  194. device_id_1 = "MY_DEVICE_1"
  195. device_id_2 = "MY_DEVICE_2"
  196. # Insert user IPs
  197. self.get_success(
  198. self.store.store_device(
  199. user_id,
  200. device_id_1,
  201. "display name",
  202. )
  203. )
  204. self.get_success(
  205. self.store.store_device(
  206. user_id,
  207. device_id_2,
  208. "display name",
  209. )
  210. )
  211. self.get_success(
  212. self.store.insert_client_ip(
  213. user_id, "access_token_1", "ip_1", "user_agent_1", device_id_1
  214. )
  215. )
  216. self.get_success(
  217. self.store.insert_client_ip(
  218. user_id, "access_token_2", "ip_2", "user_agent_2", device_id_2
  219. )
  220. )
  221. # Trigger the storage loop and wait for the rate limiting period to be over
  222. self.reactor.advance(10 + LAST_SEEN_GRANULARITY / 1000)
  223. # Update the user agent for the second device, without running the storage loop
  224. self.get_success(
  225. self.store.insert_client_ip(
  226. user_id, "access_token_2", "ip_2", "user_agent_3", device_id_2
  227. )
  228. )
  229. # Check that the new IP and user agent has not been stored yet
  230. db_result = self.get_success(
  231. self.store.db_pool.simple_select_list(
  232. table="devices",
  233. keyvalues={},
  234. retcols=("user_id", "ip", "user_agent", "device_id", "last_seen"),
  235. ),
  236. )
  237. self.assertCountEqual(
  238. db_result,
  239. [
  240. {
  241. "user_id": user_id,
  242. "device_id": device_id_1,
  243. "ip": "ip_1",
  244. "user_agent": "user_agent_1",
  245. "last_seen": 12345678000,
  246. },
  247. {
  248. "user_id": user_id,
  249. "device_id": device_id_2,
  250. "ip": "ip_2",
  251. "user_agent": "user_agent_2",
  252. "last_seen": 12345678000,
  253. },
  254. ],
  255. )
  256. # Check that data from the database and memory are combined together correctly
  257. result = self.get_success(
  258. self.store.get_last_client_ip_by_device(user_id, None)
  259. )
  260. self.assertEqual(
  261. result,
  262. {
  263. (user_id, device_id_1): {
  264. "user_id": user_id,
  265. "device_id": device_id_1,
  266. "ip": "ip_1",
  267. "user_agent": "user_agent_1",
  268. "last_seen": 12345678000,
  269. },
  270. (user_id, device_id_2): {
  271. "user_id": user_id,
  272. "device_id": device_id_2,
  273. "ip": "ip_2",
  274. "user_agent": "user_agent_3",
  275. "last_seen": 12345688000 + LAST_SEEN_GRANULARITY,
  276. },
  277. },
  278. )
  279. @parameterized.expand([(False,), (True,)])
  280. def test_get_user_ip_and_agents(self, after_persisting: bool) -> None:
  281. """Test `get_user_ip_and_agents` for persisted and unpersisted data"""
  282. self.reactor.advance(12345678)
  283. user_id = "@user:id"
  284. user = UserID.from_string(user_id)
  285. # Insert a user IP
  286. self.get_success(
  287. self.store.insert_client_ip(
  288. user_id, "access_token", "ip", "user_agent", "MY_DEVICE"
  289. )
  290. )
  291. if after_persisting:
  292. # Trigger the storage loop
  293. self.reactor.advance(10)
  294. else:
  295. # Check that the new IP and user agent has not been stored yet
  296. db_result = self.get_success(
  297. self.store.db_pool.simple_select_list(
  298. table="user_ips",
  299. keyvalues={},
  300. retcols=("access_token", "ip", "user_agent", "last_seen"),
  301. ),
  302. )
  303. self.assertEqual(db_result, [])
  304. self.assertEqual(
  305. self.get_success(self.store.get_user_ip_and_agents(user)),
  306. [
  307. {
  308. "access_token": "access_token",
  309. "ip": "ip",
  310. "user_agent": "user_agent",
  311. "last_seen": 12345678000,
  312. },
  313. ],
  314. )
  315. def test_get_user_ip_and_agents_combined_data(self) -> None:
  316. """Test that `get_user_ip_and_agents` combines persisted and unpersisted data
  317. together correctly
  318. """
  319. self.reactor.advance(12345678)
  320. user_id = "@user:id"
  321. user = UserID.from_string(user_id)
  322. # Insert user IPs
  323. self.get_success(
  324. self.store.insert_client_ip(
  325. user_id, "access_token", "ip_1", "user_agent_1", "MY_DEVICE_1"
  326. )
  327. )
  328. self.get_success(
  329. self.store.insert_client_ip(
  330. user_id, "access_token", "ip_2", "user_agent_2", "MY_DEVICE_2"
  331. )
  332. )
  333. # Trigger the storage loop and wait for the rate limiting period to be over
  334. self.reactor.advance(10 + LAST_SEEN_GRANULARITY / 1000)
  335. # Update the user agent for the second device, without running the storage loop
  336. self.get_success(
  337. self.store.insert_client_ip(
  338. user_id, "access_token", "ip_2", "user_agent_3", "MY_DEVICE_2"
  339. )
  340. )
  341. # Check that the new IP and user agent has not been stored yet
  342. db_result = self.get_success(
  343. self.store.db_pool.simple_select_list(
  344. table="user_ips",
  345. keyvalues={},
  346. retcols=("access_token", "ip", "user_agent", "last_seen"),
  347. ),
  348. )
  349. self.assertEqual(
  350. db_result,
  351. [
  352. {
  353. "access_token": "access_token",
  354. "ip": "ip_1",
  355. "user_agent": "user_agent_1",
  356. "last_seen": 12345678000,
  357. },
  358. {
  359. "access_token": "access_token",
  360. "ip": "ip_2",
  361. "user_agent": "user_agent_2",
  362. "last_seen": 12345678000,
  363. },
  364. ],
  365. )
  366. # Check that data from the database and memory are combined together correctly
  367. self.assertCountEqual(
  368. self.get_success(self.store.get_user_ip_and_agents(user)),
  369. [
  370. {
  371. "access_token": "access_token",
  372. "ip": "ip_1",
  373. "user_agent": "user_agent_1",
  374. "last_seen": 12345678000,
  375. },
  376. {
  377. "access_token": "access_token",
  378. "ip": "ip_2",
  379. "user_agent": "user_agent_3",
  380. "last_seen": 12345688000 + LAST_SEEN_GRANULARITY,
  381. },
  382. ],
  383. )
  384. @override_config({"limit_usage_by_mau": False, "max_mau_value": 50})
  385. def test_disabled_monthly_active_user(self) -> None:
  386. user_id = "@user:server"
  387. self.get_success(
  388. self.store.insert_client_ip(
  389. user_id, "access_token", "ip", "user_agent", "device_id"
  390. )
  391. )
  392. active = self.get_success(self.store.user_last_seen_monthly_active(user_id))
  393. self.assertFalse(active)
  394. @override_config({"limit_usage_by_mau": True, "max_mau_value": 50})
  395. def test_adding_monthly_active_user_when_full(self) -> None:
  396. lots_of_users = 100
  397. user_id = "@user:server"
  398. self.store.get_monthly_active_count = Mock(
  399. return_value=make_awaitable(lots_of_users)
  400. )
  401. self.get_success(
  402. self.store.insert_client_ip(
  403. user_id, "access_token", "ip", "user_agent", "device_id"
  404. )
  405. )
  406. active = self.get_success(self.store.user_last_seen_monthly_active(user_id))
  407. self.assertFalse(active)
  408. @override_config({"limit_usage_by_mau": True, "max_mau_value": 50})
  409. def test_adding_monthly_active_user_when_space(self) -> None:
  410. user_id = "@user:server"
  411. active = self.get_success(self.store.user_last_seen_monthly_active(user_id))
  412. self.assertFalse(active)
  413. # Trigger the saving loop
  414. self.reactor.advance(10)
  415. self.get_success(
  416. self.store.insert_client_ip(
  417. user_id, "access_token", "ip", "user_agent", "device_id"
  418. )
  419. )
  420. active = self.get_success(self.store.user_last_seen_monthly_active(user_id))
  421. self.assertTrue(active)
  422. @override_config({"limit_usage_by_mau": True, "max_mau_value": 50})
  423. def test_updating_monthly_active_user_when_space(self) -> None:
  424. user_id = "@user:server"
  425. self.get_success(self.store.register_user(user_id=user_id, password_hash=None))
  426. active = self.get_success(self.store.user_last_seen_monthly_active(user_id))
  427. self.assertFalse(active)
  428. # Trigger the saving loop
  429. self.reactor.advance(10)
  430. self.get_success(
  431. self.store.insert_client_ip(
  432. user_id, "access_token", "ip", "user_agent", "device_id"
  433. )
  434. )
  435. active = self.get_success(self.store.user_last_seen_monthly_active(user_id))
  436. self.assertTrue(active)
  437. def test_devices_last_seen_bg_update(self) -> None:
  438. # First make sure we have completed all updates.
  439. self.wait_for_background_updates()
  440. user_id = "@user:id"
  441. device_id = "MY_DEVICE"
  442. # Insert a user IP
  443. self.get_success(
  444. self.store.store_device(
  445. user_id,
  446. device_id,
  447. "display name",
  448. )
  449. )
  450. self.get_success(
  451. self.store.insert_client_ip(
  452. user_id, "access_token", "ip", "user_agent", device_id
  453. )
  454. )
  455. # Force persisting to disk
  456. self.reactor.advance(200)
  457. # But clear the associated entry in devices table
  458. self.get_success(
  459. self.store.db_pool.simple_update(
  460. table="devices",
  461. keyvalues={"user_id": user_id, "device_id": device_id},
  462. updatevalues={"last_seen": None, "ip": None, "user_agent": None},
  463. desc="test_devices_last_seen_bg_update",
  464. )
  465. )
  466. # We should now get nulls when querying
  467. result = self.get_success(
  468. self.store.get_last_client_ip_by_device(user_id, device_id)
  469. )
  470. r = result[(user_id, device_id)]
  471. self.assertDictContainsSubset(
  472. {
  473. "user_id": user_id,
  474. "device_id": device_id,
  475. "ip": None,
  476. "user_agent": None,
  477. "last_seen": None,
  478. },
  479. r,
  480. )
  481. # Register the background update to run again.
  482. self.get_success(
  483. self.store.db_pool.simple_insert(
  484. table="background_updates",
  485. values={
  486. "update_name": "devices_last_seen",
  487. "progress_json": "{}",
  488. "depends_on": None,
  489. },
  490. )
  491. )
  492. # ... and tell the DataStore that it hasn't finished all updates yet
  493. self.store.db_pool.updates._all_done = False
  494. # Now let's actually drive the updates to completion
  495. self.wait_for_background_updates()
  496. # We should now get the correct result again
  497. result = self.get_success(
  498. self.store.get_last_client_ip_by_device(user_id, device_id)
  499. )
  500. r = result[(user_id, device_id)]
  501. self.assertDictContainsSubset(
  502. {
  503. "user_id": user_id,
  504. "device_id": device_id,
  505. "ip": "ip",
  506. "user_agent": "user_agent",
  507. "last_seen": 0,
  508. },
  509. r,
  510. )
  511. def test_old_user_ips_pruned(self) -> None:
  512. # First make sure we have completed all updates.
  513. self.wait_for_background_updates()
  514. user_id = "@user:id"
  515. device_id = "MY_DEVICE"
  516. # Insert a user IP
  517. self.get_success(
  518. self.store.store_device(
  519. user_id,
  520. device_id,
  521. "display name",
  522. )
  523. )
  524. self.get_success(
  525. self.store.insert_client_ip(
  526. user_id, "access_token", "ip", "user_agent", device_id
  527. )
  528. )
  529. # Force persisting to disk
  530. self.reactor.advance(200)
  531. # We should see that in the DB
  532. result = self.get_success(
  533. self.store.db_pool.simple_select_list(
  534. table="user_ips",
  535. keyvalues={"user_id": user_id},
  536. retcols=["access_token", "ip", "user_agent", "device_id", "last_seen"],
  537. desc="get_user_ip_and_agents",
  538. )
  539. )
  540. self.assertEqual(
  541. result,
  542. [
  543. {
  544. "access_token": "access_token",
  545. "ip": "ip",
  546. "user_agent": "user_agent",
  547. "device_id": device_id,
  548. "last_seen": 0,
  549. }
  550. ],
  551. )
  552. # Now advance by a couple of months
  553. self.reactor.advance(60 * 24 * 60 * 60)
  554. # We should get no results.
  555. result = self.get_success(
  556. self.store.db_pool.simple_select_list(
  557. table="user_ips",
  558. keyvalues={"user_id": user_id},
  559. retcols=["access_token", "ip", "user_agent", "device_id", "last_seen"],
  560. desc="get_user_ip_and_agents",
  561. )
  562. )
  563. self.assertEqual(result, [])
  564. # But we should still get the correct values for the device
  565. result2 = self.get_success(
  566. self.store.get_last_client_ip_by_device(user_id, device_id)
  567. )
  568. r = result2[(user_id, device_id)]
  569. self.assertDictContainsSubset(
  570. {
  571. "user_id": user_id,
  572. "device_id": device_id,
  573. "ip": "ip",
  574. "user_agent": "user_agent",
  575. "last_seen": 0,
  576. },
  577. r,
  578. )
  579. class ClientIpAuthTestCase(unittest.HomeserverTestCase):
  580. servlets = [
  581. synapse.rest.admin.register_servlets,
  582. login.register_servlets,
  583. ]
  584. def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None:
  585. self.store = self.hs.get_datastores().main
  586. self.user_id = self.register_user("bob", "abc123", True)
  587. def test_request_with_xforwarded(self) -> None:
  588. """
  589. The IP in X-Forwarded-For is entered into the client IPs table.
  590. """
  591. self._runtest(
  592. {b"X-Forwarded-For": b"127.9.0.1"},
  593. "127.9.0.1",
  594. {"request": XForwardedForRequest},
  595. )
  596. def test_request_from_getPeer(self) -> None:
  597. """
  598. The IP returned by getPeer is entered into the client IPs table, if
  599. there's no X-Forwarded-For header.
  600. """
  601. self._runtest({}, "127.0.0.1", {})
  602. def _runtest(
  603. self,
  604. headers: Dict[bytes, bytes],
  605. expected_ip: str,
  606. make_request_args: Dict[str, Any],
  607. ) -> None:
  608. device_id = "bleb"
  609. access_token = self.login("bob", "abc123", device_id=device_id)
  610. # Advance to a known time
  611. self.reactor.advance(123456 - self.reactor.seconds())
  612. headers1 = {b"User-Agent": b"Mozzila pizza"}
  613. headers1.update(headers)
  614. make_request(
  615. self.reactor,
  616. self.site,
  617. "GET",
  618. "/_synapse/admin/v2/users/" + self.user_id,
  619. access_token=access_token,
  620. custom_headers=headers1.items(),
  621. **make_request_args,
  622. )
  623. # Advance so the save loop occurs
  624. self.reactor.advance(100)
  625. result = self.get_success(
  626. self.store.get_last_client_ip_by_device(self.user_id, device_id)
  627. )
  628. r = result[(self.user_id, device_id)]
  629. self.assertDictContainsSubset(
  630. {
  631. "user_id": self.user_id,
  632. "device_id": device_id,
  633. "ip": expected_ip,
  634. "user_agent": "Mozzila pizza",
  635. "last_seen": 123456100,
  636. },
  637. r,
  638. )