1
0

test_phone_stats_home.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395
  1. import synapse
  2. from synapse.app.phone_stats_home import start_phone_stats_home
  3. from synapse.rest.client import login, room
  4. from tests import unittest
  5. from tests.unittest import HomeserverTestCase
  6. FIVE_MINUTES_IN_SECONDS = 300
  7. ONE_DAY_IN_SECONDS = 86400
  8. class PhoneHomeTestCase(HomeserverTestCase):
  9. servlets = [
  10. synapse.rest.admin.register_servlets_for_client_rest_resource,
  11. room.register_servlets,
  12. login.register_servlets,
  13. ]
  14. # Override the retention time for the user_ips table because otherwise it
  15. # gets pruned too aggressively for our R30 test.
  16. @unittest.override_config({"user_ips_max_age": "365d"})
  17. def test_r30_minimum_usage(self):
  18. """
  19. Tests the minimum amount of interaction necessary for the R30 metric
  20. to consider a user 'retained'.
  21. """
  22. # Register a user, log it in, create a room and send a message
  23. user_id = self.register_user("u1", "secret!")
  24. access_token = self.login("u1", "secret!")
  25. room_id = self.helper.create_room_as(room_creator=user_id, tok=access_token)
  26. self.helper.send(room_id, "message", tok=access_token)
  27. # Check the R30 results do not count that user.
  28. r30_results = self.get_success(self.hs.get_datastore().count_r30_users())
  29. self.assertEqual(r30_results, {"all": 0})
  30. # Advance 30 days (+ 1 second, because strict inequality causes issues if we are
  31. # bang on 30 days later).
  32. self.reactor.advance(30 * ONE_DAY_IN_SECONDS + 1)
  33. # (Make sure the user isn't somehow counted by this point.)
  34. r30_results = self.get_success(self.hs.get_datastore().count_r30_users())
  35. self.assertEqual(r30_results, {"all": 0})
  36. # Send a message (this counts as activity)
  37. self.helper.send(room_id, "message2", tok=access_token)
  38. # We have to wait some time for _update_client_ips_batch to get
  39. # called and update the user_ips table.
  40. self.reactor.advance(2 * 60 * 60)
  41. # *Now* the user is counted.
  42. r30_results = self.get_success(self.hs.get_datastore().count_r30_users())
  43. self.assertEqual(r30_results, {"all": 1, "unknown": 1})
  44. # Advance 29 days. The user has now not posted for 29 days.
  45. self.reactor.advance(29 * ONE_DAY_IN_SECONDS)
  46. # The user is still counted.
  47. r30_results = self.get_success(self.hs.get_datastore().count_r30_users())
  48. self.assertEqual(r30_results, {"all": 1, "unknown": 1})
  49. # Advance another day. The user has now not posted for 30 days.
  50. self.reactor.advance(ONE_DAY_IN_SECONDS)
  51. # The user is now no longer counted in R30.
  52. r30_results = self.get_success(self.hs.get_datastore().count_r30_users())
  53. self.assertEqual(r30_results, {"all": 0})
  54. def test_r30_minimum_usage_using_default_config(self):
  55. """
  56. Tests the minimum amount of interaction necessary for the R30 metric
  57. to consider a user 'retained'.
  58. N.B. This test does not override the `user_ips_max_age` config setting,
  59. which defaults to 28 days.
  60. """
  61. # Register a user, log it in, create a room and send a message
  62. user_id = self.register_user("u1", "secret!")
  63. access_token = self.login("u1", "secret!")
  64. room_id = self.helper.create_room_as(room_creator=user_id, tok=access_token)
  65. self.helper.send(room_id, "message", tok=access_token)
  66. # Check the R30 results do not count that user.
  67. r30_results = self.get_success(self.hs.get_datastore().count_r30_users())
  68. self.assertEqual(r30_results, {"all": 0})
  69. # Advance 30 days (+ 1 second, because strict inequality causes issues if we are
  70. # bang on 30 days later).
  71. self.reactor.advance(30 * ONE_DAY_IN_SECONDS + 1)
  72. # (Make sure the user isn't somehow counted by this point.)
  73. r30_results = self.get_success(self.hs.get_datastore().count_r30_users())
  74. self.assertEqual(r30_results, {"all": 0})
  75. # Send a message (this counts as activity)
  76. self.helper.send(room_id, "message2", tok=access_token)
  77. # We have to wait some time for _update_client_ips_batch to get
  78. # called and update the user_ips table.
  79. self.reactor.advance(2 * 60 * 60)
  80. # *Now* the user is counted.
  81. r30_results = self.get_success(self.hs.get_datastore().count_r30_users())
  82. self.assertEqual(r30_results, {"all": 1, "unknown": 1})
  83. # Advance 27 days. The user has now not posted for 27 days.
  84. self.reactor.advance(27 * ONE_DAY_IN_SECONDS)
  85. # The user is still counted.
  86. r30_results = self.get_success(self.hs.get_datastore().count_r30_users())
  87. self.assertEqual(r30_results, {"all": 1, "unknown": 1})
  88. # Advance another day. The user has now not posted for 28 days.
  89. self.reactor.advance(ONE_DAY_IN_SECONDS)
  90. # The user is now no longer counted in R30.
  91. # (This is because the user_ips table has been pruned, which by default
  92. # only preserves the last 28 days of entries.)
  93. r30_results = self.get_success(self.hs.get_datastore().count_r30_users())
  94. self.assertEqual(r30_results, {"all": 0})
  95. def test_r30_user_must_be_retained_for_at_least_a_month(self):
  96. """
  97. Tests that a newly-registered user must be retained for a whole month
  98. before appearing in the R30 statistic, even if they post every day
  99. during that time!
  100. """
  101. # Register a user and send a message
  102. user_id = self.register_user("u1", "secret!")
  103. access_token = self.login("u1", "secret!")
  104. room_id = self.helper.create_room_as(room_creator=user_id, tok=access_token)
  105. self.helper.send(room_id, "message", tok=access_token)
  106. # Check the user does not contribute to R30 yet.
  107. r30_results = self.get_success(self.hs.get_datastore().count_r30_users())
  108. self.assertEqual(r30_results, {"all": 0})
  109. for _ in range(30):
  110. # This loop posts a message every day for 30 days
  111. self.reactor.advance(ONE_DAY_IN_SECONDS)
  112. self.helper.send(room_id, "I'm still here", tok=access_token)
  113. # Notice that the user *still* does not contribute to R30!
  114. r30_results = self.get_success(self.hs.get_datastore().count_r30_users())
  115. self.assertEqual(r30_results, {"all": 0})
  116. self.reactor.advance(ONE_DAY_IN_SECONDS)
  117. self.helper.send(room_id, "Still here!", tok=access_token)
  118. # *Now* the user appears in R30.
  119. r30_results = self.get_success(self.hs.get_datastore().count_r30_users())
  120. self.assertEqual(r30_results, {"all": 1, "unknown": 1})
  121. class PhoneHomeR30V2TestCase(HomeserverTestCase):
  122. servlets = [
  123. synapse.rest.admin.register_servlets_for_client_rest_resource,
  124. room.register_servlets,
  125. login.register_servlets,
  126. ]
  127. def _advance_to(self, desired_time_secs: float):
  128. now = self.hs.get_clock().time()
  129. assert now < desired_time_secs
  130. self.reactor.advance(desired_time_secs - now)
  131. def make_homeserver(self, reactor, clock):
  132. hs = super(PhoneHomeR30V2TestCase, self).make_homeserver(reactor, clock)
  133. # We don't want our tests to actually report statistics, so check
  134. # that it's not enabled
  135. assert not hs.config.metrics.report_stats
  136. # This starts the needed data collection that we rely on to calculate
  137. # R30v2 metrics.
  138. start_phone_stats_home(hs)
  139. return hs
  140. def test_r30v2_minimum_usage(self):
  141. """
  142. Tests the minimum amount of interaction necessary for the R30v2 metric
  143. to consider a user 'retained'.
  144. """
  145. # Register a user, log it in, create a room and send a message
  146. user_id = self.register_user("u1", "secret!")
  147. access_token = self.login("u1", "secret!")
  148. room_id = self.helper.create_room_as(room_creator=user_id, tok=access_token)
  149. self.helper.send(room_id, "message", tok=access_token)
  150. first_post_at = self.hs.get_clock().time()
  151. # Give time for user_daily_visits table to be updated.
  152. # (user_daily_visits is updated every 5 minutes using a looping call.)
  153. self.reactor.advance(FIVE_MINUTES_IN_SECONDS)
  154. store = self.hs.get_datastore()
  155. # Check the R30 results do not count that user.
  156. r30_results = self.get_success(store.count_r30v2_users())
  157. self.assertEqual(
  158. r30_results, {"all": 0, "android": 0, "electron": 0, "ios": 0, "web": 0}
  159. )
  160. # Advance 31 days.
  161. # (R30v2 includes users with **more** than 30 days between the two visits,
  162. # and user_daily_visits records the timestamp as the start of the day.)
  163. self.reactor.advance(31 * ONE_DAY_IN_SECONDS)
  164. # Also advance 5 minutes to let another user_daily_visits update occur
  165. self.reactor.advance(FIVE_MINUTES_IN_SECONDS)
  166. # (Make sure the user isn't somehow counted by this point.)
  167. r30_results = self.get_success(store.count_r30v2_users())
  168. self.assertEqual(
  169. r30_results, {"all": 0, "android": 0, "electron": 0, "ios": 0, "web": 0}
  170. )
  171. # Send a message (this counts as activity)
  172. self.helper.send(room_id, "message2", tok=access_token)
  173. # We have to wait a few minutes for the user_daily_visits table to
  174. # be updated by a background process.
  175. self.reactor.advance(FIVE_MINUTES_IN_SECONDS)
  176. # *Now* the user is counted.
  177. r30_results = self.get_success(store.count_r30v2_users())
  178. self.assertEqual(
  179. r30_results, {"all": 1, "android": 0, "electron": 0, "ios": 0, "web": 0}
  180. )
  181. # Advance to JUST under 60 days after the user's first post
  182. self._advance_to(first_post_at + 60 * ONE_DAY_IN_SECONDS - 5)
  183. # Check the user is still counted.
  184. r30_results = self.get_success(store.count_r30v2_users())
  185. self.assertEqual(
  186. r30_results, {"all": 1, "android": 0, "electron": 0, "ios": 0, "web": 0}
  187. )
  188. # Advance into the next day. The user's first activity is now more than 60 days old.
  189. self._advance_to(first_post_at + 60 * ONE_DAY_IN_SECONDS + 5)
  190. # Check the user is now no longer counted in R30.
  191. r30_results = self.get_success(store.count_r30v2_users())
  192. self.assertEqual(
  193. r30_results, {"all": 0, "android": 0, "electron": 0, "ios": 0, "web": 0}
  194. )
  195. def test_r30v2_user_must_be_retained_for_at_least_a_month(self):
  196. """
  197. Tests that a newly-registered user must be retained for a whole month
  198. before appearing in the R30v2 statistic, even if they post every day
  199. during that time!
  200. """
  201. # set a custom user-agent to impersonate Element/Android.
  202. headers = (
  203. (
  204. "User-Agent",
  205. "Element/1.1 (Linux; U; Android 9; MatrixAndroidSDK_X 0.0.1)",
  206. ),
  207. )
  208. # Register a user and send a message
  209. user_id = self.register_user("u1", "secret!")
  210. access_token = self.login("u1", "secret!", custom_headers=headers)
  211. room_id = self.helper.create_room_as(
  212. room_creator=user_id, tok=access_token, custom_headers=headers
  213. )
  214. self.helper.send(room_id, "message", tok=access_token, custom_headers=headers)
  215. # Give time for user_daily_visits table to be updated.
  216. # (user_daily_visits is updated every 5 minutes using a looping call.)
  217. self.reactor.advance(FIVE_MINUTES_IN_SECONDS)
  218. store = self.hs.get_datastore()
  219. # Check the user does not contribute to R30 yet.
  220. r30_results = self.get_success(store.count_r30v2_users())
  221. self.assertEqual(
  222. r30_results, {"all": 0, "android": 0, "electron": 0, "ios": 0, "web": 0}
  223. )
  224. for _ in range(30):
  225. # This loop posts a message every day for 30 days
  226. self.reactor.advance(ONE_DAY_IN_SECONDS - FIVE_MINUTES_IN_SECONDS)
  227. self.helper.send(
  228. room_id, "I'm still here", tok=access_token, custom_headers=headers
  229. )
  230. # give time for user_daily_visits to update
  231. self.reactor.advance(FIVE_MINUTES_IN_SECONDS)
  232. # Notice that the user *still* does not contribute to R30!
  233. r30_results = self.get_success(store.count_r30v2_users())
  234. self.assertEqual(
  235. r30_results, {"all": 0, "android": 0, "electron": 0, "ios": 0, "web": 0}
  236. )
  237. # advance yet another day with more activity
  238. self.reactor.advance(ONE_DAY_IN_SECONDS)
  239. self.helper.send(
  240. room_id, "Still here!", tok=access_token, custom_headers=headers
  241. )
  242. # give time for user_daily_visits to update
  243. self.reactor.advance(FIVE_MINUTES_IN_SECONDS)
  244. # *Now* the user appears in R30.
  245. r30_results = self.get_success(store.count_r30v2_users())
  246. self.assertEqual(
  247. r30_results, {"all": 1, "android": 1, "electron": 0, "ios": 0, "web": 0}
  248. )
  249. def test_r30v2_returning_dormant_users_not_counted(self):
  250. """
  251. Tests that dormant users (users inactive for a long time) do not
  252. contribute to R30v2 when they return for just a single day.
  253. This is a key difference between R30 and R30v2.
  254. """
  255. # set a custom user-agent to impersonate Element/iOS.
  256. headers = (
  257. (
  258. "User-Agent",
  259. "Riot/1.4 (iPhone; iOS 13; Scale/4.00)",
  260. ),
  261. )
  262. # Register a user and send a message
  263. user_id = self.register_user("u1", "secret!")
  264. access_token = self.login("u1", "secret!", custom_headers=headers)
  265. room_id = self.helper.create_room_as(
  266. room_creator=user_id, tok=access_token, custom_headers=headers
  267. )
  268. self.helper.send(room_id, "message", tok=access_token, custom_headers=headers)
  269. # the user goes inactive for 2 months
  270. self.reactor.advance(60 * ONE_DAY_IN_SECONDS)
  271. # the user returns for one day, perhaps just to check out a new feature
  272. self.helper.send(room_id, "message", tok=access_token, custom_headers=headers)
  273. # Give time for user_daily_visits table to be updated.
  274. # (user_daily_visits is updated every 5 minutes using a looping call.)
  275. self.reactor.advance(FIVE_MINUTES_IN_SECONDS)
  276. store = self.hs.get_datastore()
  277. # Check that the user does not contribute to R30v2, even though it's been
  278. # more than 30 days since registration.
  279. r30_results = self.get_success(store.count_r30v2_users())
  280. self.assertEqual(
  281. r30_results, {"all": 0, "android": 0, "electron": 0, "ios": 0, "web": 0}
  282. )
  283. # Check that this is a situation where old R30 differs:
  284. # old R30 DOES count this as 'retained'.
  285. r30_results = self.get_success(store.count_r30_users())
  286. self.assertEqual(r30_results, {"all": 1, "ios": 1})
  287. # Now we want to check that the user will still be able to appear in
  288. # R30v2 as long as the user performs some other activity between
  289. # 30 and 60 days later.
  290. self.reactor.advance(32 * ONE_DAY_IN_SECONDS)
  291. self.helper.send(room_id, "message", tok=access_token, custom_headers=headers)
  292. # (give time for tables to update)
  293. self.reactor.advance(FIVE_MINUTES_IN_SECONDS)
  294. # Check the user now satisfies the requirements to appear in R30v2.
  295. r30_results = self.get_success(store.count_r30v2_users())
  296. self.assertEqual(
  297. r30_results, {"all": 1, "ios": 1, "android": 0, "electron": 0, "web": 0}
  298. )
  299. # Advance to 59.5 days after the user's first R30v2-eligible activity.
  300. self.reactor.advance(27.5 * ONE_DAY_IN_SECONDS)
  301. # Check the user still appears in R30v2.
  302. r30_results = self.get_success(store.count_r30v2_users())
  303. self.assertEqual(
  304. r30_results, {"all": 1, "ios": 1, "android": 0, "electron": 0, "web": 0}
  305. )
  306. # Advance to 60.5 days after the user's first R30v2-eligible activity.
  307. self.reactor.advance(ONE_DAY_IN_SECONDS)
  308. # Check the user no longer appears in R30v2.
  309. r30_results = self.get_success(store.count_r30v2_users())
  310. self.assertEqual(
  311. r30_results, {"all": 0, "android": 0, "electron": 0, "ios": 0, "web": 0}
  312. )