test_phone_stats_home.py 9.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248
  1. import synapse
  2. from synapse.app.phone_stats_home import start_phone_stats_home
  3. from synapse.rest.client import login, room
  4. from synapse.server import HomeServer
  5. from synapse.util import Clock
  6. from tests.server import ThreadedMemoryReactorClock
  7. from tests.unittest import HomeserverTestCase
  8. FIVE_MINUTES_IN_SECONDS = 300
  9. ONE_DAY_IN_SECONDS = 86400
  10. class PhoneHomeR30V2TestCase(HomeserverTestCase):
  11. servlets = [
  12. synapse.rest.admin.register_servlets_for_client_rest_resource,
  13. room.register_servlets,
  14. login.register_servlets,
  15. ]
  16. def _advance_to(self, desired_time_secs: float) -> None:
  17. now = self.hs.get_clock().time()
  18. assert now < desired_time_secs
  19. self.reactor.advance(desired_time_secs - now)
  20. def make_homeserver(
  21. self, reactor: ThreadedMemoryReactorClock, clock: Clock
  22. ) -> HomeServer:
  23. hs = super().make_homeserver(reactor, clock)
  24. # We don't want our tests to actually report statistics, so check
  25. # that it's not enabled
  26. assert not hs.config.metrics.report_stats
  27. # This starts the needed data collection that we rely on to calculate
  28. # R30v2 metrics.
  29. start_phone_stats_home(hs)
  30. return hs
  31. def test_r30v2_minimum_usage(self) -> None:
  32. """
  33. Tests the minimum amount of interaction necessary for the R30v2 metric
  34. to consider a user 'retained'.
  35. """
  36. # Register a user, log it in, create a room and send a message
  37. user_id = self.register_user("u1", "secret!")
  38. access_token = self.login("u1", "secret!")
  39. room_id = self.helper.create_room_as(room_creator=user_id, tok=access_token)
  40. self.helper.send(room_id, "message", tok=access_token)
  41. first_post_at = self.hs.get_clock().time()
  42. # Give time for user_daily_visits table to be updated.
  43. # (user_daily_visits is updated every 5 minutes using a looping call.)
  44. self.reactor.advance(FIVE_MINUTES_IN_SECONDS)
  45. store = self.hs.get_datastores().main
  46. # Check the R30 results do not count that user.
  47. r30_results = self.get_success(store.count_r30v2_users())
  48. self.assertEqual(
  49. r30_results, {"all": 0, "android": 0, "electron": 0, "ios": 0, "web": 0}
  50. )
  51. # Advance 31 days.
  52. # (R30v2 includes users with **more** than 30 days between the two visits,
  53. # and user_daily_visits records the timestamp as the start of the day.)
  54. self.reactor.advance(31 * ONE_DAY_IN_SECONDS)
  55. # Also advance 5 minutes to let another user_daily_visits update occur
  56. self.reactor.advance(FIVE_MINUTES_IN_SECONDS)
  57. # (Make sure the user isn't somehow counted by this point.)
  58. r30_results = self.get_success(store.count_r30v2_users())
  59. self.assertEqual(
  60. r30_results, {"all": 0, "android": 0, "electron": 0, "ios": 0, "web": 0}
  61. )
  62. # Send a message (this counts as activity)
  63. self.helper.send(room_id, "message2", tok=access_token)
  64. # We have to wait a few minutes for the user_daily_visits table to
  65. # be updated by a background process.
  66. self.reactor.advance(FIVE_MINUTES_IN_SECONDS)
  67. # *Now* the user is counted.
  68. r30_results = self.get_success(store.count_r30v2_users())
  69. self.assertEqual(
  70. r30_results, {"all": 1, "android": 0, "electron": 0, "ios": 0, "web": 0}
  71. )
  72. # Advance to JUST under 60 days after the user's first post
  73. self._advance_to(first_post_at + 60 * ONE_DAY_IN_SECONDS - 5)
  74. # Check the user is still counted.
  75. r30_results = self.get_success(store.count_r30v2_users())
  76. self.assertEqual(
  77. r30_results, {"all": 1, "android": 0, "electron": 0, "ios": 0, "web": 0}
  78. )
  79. # Advance into the next day. The user's first activity is now more than 60 days old.
  80. self._advance_to(first_post_at + 60 * ONE_DAY_IN_SECONDS + 5)
  81. # Check the user is now no longer counted in R30.
  82. r30_results = self.get_success(store.count_r30v2_users())
  83. self.assertEqual(
  84. r30_results, {"all": 0, "android": 0, "electron": 0, "ios": 0, "web": 0}
  85. )
  86. def test_r30v2_user_must_be_retained_for_at_least_a_month(self) -> None:
  87. """
  88. Tests that a newly-registered user must be retained for a whole month
  89. before appearing in the R30v2 statistic, even if they post every day
  90. during that time!
  91. """
  92. # set a custom user-agent to impersonate Element/Android.
  93. headers = (
  94. (
  95. "User-Agent",
  96. "Element/1.1 (Linux; U; Android 9; MatrixAndroidSDK_X 0.0.1)",
  97. ),
  98. )
  99. # Register a user and send a message
  100. user_id = self.register_user("u1", "secret!")
  101. access_token = self.login("u1", "secret!", custom_headers=headers)
  102. room_id = self.helper.create_room_as(
  103. room_creator=user_id, tok=access_token, custom_headers=headers
  104. )
  105. self.helper.send(room_id, "message", tok=access_token, custom_headers=headers)
  106. # Give time for user_daily_visits table to be updated.
  107. # (user_daily_visits is updated every 5 minutes using a looping call.)
  108. self.reactor.advance(FIVE_MINUTES_IN_SECONDS)
  109. store = self.hs.get_datastores().main
  110. # Check the user does not contribute to R30 yet.
  111. r30_results = self.get_success(store.count_r30v2_users())
  112. self.assertEqual(
  113. r30_results, {"all": 0, "android": 0, "electron": 0, "ios": 0, "web": 0}
  114. )
  115. for _ in range(30):
  116. # This loop posts a message every day for 30 days
  117. self.reactor.advance(ONE_DAY_IN_SECONDS - FIVE_MINUTES_IN_SECONDS)
  118. self.helper.send(
  119. room_id, "I'm still here", tok=access_token, custom_headers=headers
  120. )
  121. # give time for user_daily_visits to update
  122. self.reactor.advance(FIVE_MINUTES_IN_SECONDS)
  123. # Notice that the user *still* does not contribute to R30!
  124. r30_results = self.get_success(store.count_r30v2_users())
  125. self.assertEqual(
  126. r30_results, {"all": 0, "android": 0, "electron": 0, "ios": 0, "web": 0}
  127. )
  128. # advance yet another day with more activity
  129. self.reactor.advance(ONE_DAY_IN_SECONDS)
  130. self.helper.send(
  131. room_id, "Still here!", tok=access_token, custom_headers=headers
  132. )
  133. # give time for user_daily_visits to update
  134. self.reactor.advance(FIVE_MINUTES_IN_SECONDS)
  135. # *Now* the user appears in R30.
  136. r30_results = self.get_success(store.count_r30v2_users())
  137. self.assertEqual(
  138. r30_results, {"all": 1, "android": 1, "electron": 0, "ios": 0, "web": 0}
  139. )
  140. def test_r30v2_returning_dormant_users_not_counted(self) -> None:
  141. """
  142. Tests that dormant users (users inactive for a long time) do not
  143. contribute to R30v2 when they return for just a single day.
  144. This is a key difference between R30 and R30v2.
  145. """
  146. # set a custom user-agent to impersonate Element/iOS.
  147. headers = (
  148. (
  149. "User-Agent",
  150. "Riot/1.4 (iPhone; iOS 13; Scale/4.00)",
  151. ),
  152. )
  153. # Register a user and send a message
  154. user_id = self.register_user("u1", "secret!")
  155. access_token = self.login("u1", "secret!", custom_headers=headers)
  156. room_id = self.helper.create_room_as(
  157. room_creator=user_id, tok=access_token, custom_headers=headers
  158. )
  159. self.helper.send(room_id, "message", tok=access_token, custom_headers=headers)
  160. # the user goes inactive for 2 months
  161. self.reactor.advance(60 * ONE_DAY_IN_SECONDS)
  162. # the user returns for one day, perhaps just to check out a new feature
  163. self.helper.send(room_id, "message", tok=access_token, custom_headers=headers)
  164. # Give time for user_daily_visits table to be updated.
  165. # (user_daily_visits is updated every 5 minutes using a looping call.)
  166. self.reactor.advance(FIVE_MINUTES_IN_SECONDS)
  167. store = self.hs.get_datastores().main
  168. # Check that the user does not contribute to R30v2, even though it's been
  169. # more than 30 days since registration.
  170. r30_results = self.get_success(store.count_r30v2_users())
  171. self.assertEqual(
  172. r30_results, {"all": 0, "android": 0, "electron": 0, "ios": 0, "web": 0}
  173. )
  174. # Now we want to check that the user will still be able to appear in
  175. # R30v2 as long as the user performs some other activity between
  176. # 30 and 60 days later.
  177. self.reactor.advance(32 * ONE_DAY_IN_SECONDS)
  178. self.helper.send(room_id, "message", tok=access_token, custom_headers=headers)
  179. # (give time for tables to update)
  180. self.reactor.advance(FIVE_MINUTES_IN_SECONDS)
  181. # Check the user now satisfies the requirements to appear in R30v2.
  182. r30_results = self.get_success(store.count_r30v2_users())
  183. self.assertEqual(
  184. r30_results, {"all": 1, "ios": 1, "android": 0, "electron": 0, "web": 0}
  185. )
  186. # Advance to 59.5 days after the user's first R30v2-eligible activity.
  187. self.reactor.advance(27.5 * ONE_DAY_IN_SECONDS)
  188. # Check the user still appears in R30v2.
  189. r30_results = self.get_success(store.count_r30v2_users())
  190. self.assertEqual(
  191. r30_results, {"all": 1, "ios": 1, "android": 0, "electron": 0, "web": 0}
  192. )
  193. # Advance to 60.5 days after the user's first R30v2-eligible activity.
  194. self.reactor.advance(ONE_DAY_IN_SECONDS)
  195. # Check the user no longer appears in R30v2.
  196. r30_results = self.get_success(store.count_r30v2_users())
  197. self.assertEqual(
  198. r30_results, {"all": 0, "android": 0, "electron": 0, "ios": 0, "web": 0}
  199. )