test_descriptors.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370
  1. # -*- coding: utf-8 -*-
  2. # Copyright 2016 OpenMarket Ltd
  3. # Copyright 2018 New Vector Ltd
  4. #
  5. # Licensed under the Apache License, Version 2.0 (the "License");
  6. # you may not use this file except in compliance with the License.
  7. # You may obtain a copy of the License at
  8. #
  9. # http://www.apache.org/licenses/LICENSE-2.0
  10. #
  11. # Unless required by applicable law or agreed to in writing, software
  12. # distributed under the License is distributed on an "AS IS" BASIS,
  13. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. # See the License for the specific language governing permissions and
  15. # limitations under the License.
  16. import logging
  17. from functools import partial
  18. import mock
  19. from twisted.internet import defer, reactor
  20. from synapse.api.errors import SynapseError
  21. from synapse.util import logcontext
  22. from synapse.util.caches import descriptors
  23. from tests import unittest
  24. logger = logging.getLogger(__name__)
  25. def run_on_reactor():
  26. d = defer.Deferred()
  27. reactor.callLater(0, d.callback, 0)
  28. return logcontext.make_deferred_yieldable(d)
  29. class CacheTestCase(unittest.TestCase):
  30. def test_invalidate_all(self):
  31. cache = descriptors.Cache("testcache")
  32. callback_record = [False, False]
  33. def record_callback(idx):
  34. callback_record[idx] = True
  35. # add a couple of pending entries
  36. d1 = defer.Deferred()
  37. cache.set("key1", d1, partial(record_callback, 0))
  38. d2 = defer.Deferred()
  39. cache.set("key2", d2, partial(record_callback, 1))
  40. # lookup should return the deferreds
  41. self.assertIs(cache.get("key1"), d1)
  42. self.assertIs(cache.get("key2"), d2)
  43. # let one of the lookups complete
  44. d2.callback("result2")
  45. self.assertEqual(cache.get("key2"), "result2")
  46. # now do the invalidation
  47. cache.invalidate_all()
  48. # lookup should return none
  49. self.assertIsNone(cache.get("key1", None))
  50. self.assertIsNone(cache.get("key2", None))
  51. # both callbacks should have been callbacked
  52. self.assertTrue(callback_record[0], "Invalidation callback for key1 not called")
  53. self.assertTrue(callback_record[1], "Invalidation callback for key2 not called")
  54. # letting the other lookup complete should do nothing
  55. d1.callback("result1")
  56. self.assertIsNone(cache.get("key1", None))
  57. class DescriptorTestCase(unittest.TestCase):
  58. @defer.inlineCallbacks
  59. def test_cache(self):
  60. class Cls(object):
  61. def __init__(self):
  62. self.mock = mock.Mock()
  63. @descriptors.cached()
  64. def fn(self, arg1, arg2):
  65. return self.mock(arg1, arg2)
  66. obj = Cls()
  67. obj.mock.return_value = 'fish'
  68. r = yield obj.fn(1, 2)
  69. self.assertEqual(r, 'fish')
  70. obj.mock.assert_called_once_with(1, 2)
  71. obj.mock.reset_mock()
  72. # a call with different params should call the mock again
  73. obj.mock.return_value = 'chips'
  74. r = yield obj.fn(1, 3)
  75. self.assertEqual(r, 'chips')
  76. obj.mock.assert_called_once_with(1, 3)
  77. obj.mock.reset_mock()
  78. # the two values should now be cached
  79. r = yield obj.fn(1, 2)
  80. self.assertEqual(r, 'fish')
  81. r = yield obj.fn(1, 3)
  82. self.assertEqual(r, 'chips')
  83. obj.mock.assert_not_called()
  84. @defer.inlineCallbacks
  85. def test_cache_num_args(self):
  86. """Only the first num_args arguments should matter to the cache"""
  87. class Cls(object):
  88. def __init__(self):
  89. self.mock = mock.Mock()
  90. @descriptors.cached(num_args=1)
  91. def fn(self, arg1, arg2):
  92. return self.mock(arg1, arg2)
  93. obj = Cls()
  94. obj.mock.return_value = 'fish'
  95. r = yield obj.fn(1, 2)
  96. self.assertEqual(r, 'fish')
  97. obj.mock.assert_called_once_with(1, 2)
  98. obj.mock.reset_mock()
  99. # a call with different params should call the mock again
  100. obj.mock.return_value = 'chips'
  101. r = yield obj.fn(2, 3)
  102. self.assertEqual(r, 'chips')
  103. obj.mock.assert_called_once_with(2, 3)
  104. obj.mock.reset_mock()
  105. # the two values should now be cached; we should be able to vary
  106. # the second argument and still get the cached result.
  107. r = yield obj.fn(1, 4)
  108. self.assertEqual(r, 'fish')
  109. r = yield obj.fn(2, 5)
  110. self.assertEqual(r, 'chips')
  111. obj.mock.assert_not_called()
  112. def test_cache_logcontexts(self):
  113. """Check that logcontexts are set and restored correctly when
  114. using the cache."""
  115. complete_lookup = defer.Deferred()
  116. class Cls(object):
  117. @descriptors.cached()
  118. def fn(self, arg1):
  119. @defer.inlineCallbacks
  120. def inner_fn():
  121. with logcontext.PreserveLoggingContext():
  122. yield complete_lookup
  123. defer.returnValue(1)
  124. return inner_fn()
  125. @defer.inlineCallbacks
  126. def do_lookup():
  127. with logcontext.LoggingContext() as c1:
  128. c1.name = "c1"
  129. r = yield obj.fn(1)
  130. self.assertEqual(logcontext.LoggingContext.current_context(), c1)
  131. defer.returnValue(r)
  132. def check_result(r):
  133. self.assertEqual(r, 1)
  134. obj = Cls()
  135. # set off a deferred which will do a cache lookup
  136. d1 = do_lookup()
  137. self.assertEqual(
  138. logcontext.LoggingContext.current_context(),
  139. logcontext.LoggingContext.sentinel,
  140. )
  141. d1.addCallback(check_result)
  142. # and another
  143. d2 = do_lookup()
  144. self.assertEqual(
  145. logcontext.LoggingContext.current_context(),
  146. logcontext.LoggingContext.sentinel,
  147. )
  148. d2.addCallback(check_result)
  149. # let the lookup complete
  150. complete_lookup.callback(None)
  151. return defer.gatherResults([d1, d2])
  152. def test_cache_logcontexts_with_exception(self):
  153. """Check that the cache sets and restores logcontexts correctly when
  154. the lookup function throws an exception"""
  155. class Cls(object):
  156. @descriptors.cached()
  157. def fn(self, arg1):
  158. @defer.inlineCallbacks
  159. def inner_fn():
  160. # we want this to behave like an asynchronous function
  161. yield run_on_reactor()
  162. raise SynapseError(400, "blah")
  163. return inner_fn()
  164. @defer.inlineCallbacks
  165. def do_lookup():
  166. with logcontext.LoggingContext() as c1:
  167. c1.name = "c1"
  168. try:
  169. d = obj.fn(1)
  170. self.assertEqual(
  171. logcontext.LoggingContext.current_context(),
  172. logcontext.LoggingContext.sentinel,
  173. )
  174. yield d
  175. self.fail("No exception thrown")
  176. except SynapseError:
  177. pass
  178. self.assertEqual(logcontext.LoggingContext.current_context(), c1)
  179. obj = Cls()
  180. # set off a deferred which will do a cache lookup
  181. d1 = do_lookup()
  182. self.assertEqual(
  183. logcontext.LoggingContext.current_context(),
  184. logcontext.LoggingContext.sentinel,
  185. )
  186. return d1
  187. @defer.inlineCallbacks
  188. def test_cache_default_args(self):
  189. class Cls(object):
  190. def __init__(self):
  191. self.mock = mock.Mock()
  192. @descriptors.cached()
  193. def fn(self, arg1, arg2=2, arg3=3):
  194. return self.mock(arg1, arg2, arg3)
  195. obj = Cls()
  196. obj.mock.return_value = 'fish'
  197. r = yield obj.fn(1, 2, 3)
  198. self.assertEqual(r, 'fish')
  199. obj.mock.assert_called_once_with(1, 2, 3)
  200. obj.mock.reset_mock()
  201. # a call with same params shouldn't call the mock again
  202. r = yield obj.fn(1, 2)
  203. self.assertEqual(r, 'fish')
  204. obj.mock.assert_not_called()
  205. obj.mock.reset_mock()
  206. # a call with different params should call the mock again
  207. obj.mock.return_value = 'chips'
  208. r = yield obj.fn(2, 3)
  209. self.assertEqual(r, 'chips')
  210. obj.mock.assert_called_once_with(2, 3, 3)
  211. obj.mock.reset_mock()
  212. # the two values should now be cached
  213. r = yield obj.fn(1, 2)
  214. self.assertEqual(r, 'fish')
  215. r = yield obj.fn(2, 3)
  216. self.assertEqual(r, 'chips')
  217. obj.mock.assert_not_called()
  218. class CachedListDescriptorTestCase(unittest.TestCase):
  219. @defer.inlineCallbacks
  220. def test_cache(self):
  221. class Cls(object):
  222. def __init__(self):
  223. self.mock = mock.Mock()
  224. @descriptors.cached()
  225. def fn(self, arg1, arg2):
  226. pass
  227. @descriptors.cachedList("fn", "args1", inlineCallbacks=True)
  228. def list_fn(self, args1, arg2):
  229. assert logcontext.LoggingContext.current_context().request == "c1"
  230. # we want this to behave like an asynchronous function
  231. yield run_on_reactor()
  232. assert logcontext.LoggingContext.current_context().request == "c1"
  233. defer.returnValue(self.mock(args1, arg2))
  234. with logcontext.LoggingContext() as c1:
  235. c1.request = "c1"
  236. obj = Cls()
  237. obj.mock.return_value = {10: 'fish', 20: 'chips'}
  238. d1 = obj.list_fn([10, 20], 2)
  239. self.assertEqual(
  240. logcontext.LoggingContext.current_context(),
  241. logcontext.LoggingContext.sentinel,
  242. )
  243. r = yield d1
  244. self.assertEqual(logcontext.LoggingContext.current_context(), c1)
  245. obj.mock.assert_called_once_with([10, 20], 2)
  246. self.assertEqual(r, {10: 'fish', 20: 'chips'})
  247. obj.mock.reset_mock()
  248. # a call with different params should call the mock again
  249. obj.mock.return_value = {30: 'peas'}
  250. r = yield obj.list_fn([20, 30], 2)
  251. obj.mock.assert_called_once_with([30], 2)
  252. self.assertEqual(r, {20: 'chips', 30: 'peas'})
  253. obj.mock.reset_mock()
  254. # all the values should now be cached
  255. r = yield obj.fn(10, 2)
  256. self.assertEqual(r, 'fish')
  257. r = yield obj.fn(20, 2)
  258. self.assertEqual(r, 'chips')
  259. r = yield obj.fn(30, 2)
  260. self.assertEqual(r, 'peas')
  261. r = yield obj.list_fn([10, 20, 30], 2)
  262. obj.mock.assert_not_called()
  263. self.assertEqual(r, {10: 'fish', 20: 'chips', 30: 'peas'})
  264. @defer.inlineCallbacks
  265. def test_invalidate(self):
  266. """Make sure that invalidation callbacks are called."""
  267. class Cls(object):
  268. def __init__(self):
  269. self.mock = mock.Mock()
  270. @descriptors.cached()
  271. def fn(self, arg1, arg2):
  272. pass
  273. @descriptors.cachedList("fn", "args1", inlineCallbacks=True)
  274. def list_fn(self, args1, arg2):
  275. # we want this to behave like an asynchronous function
  276. yield run_on_reactor()
  277. defer.returnValue(self.mock(args1, arg2))
  278. obj = Cls()
  279. invalidate0 = mock.Mock()
  280. invalidate1 = mock.Mock()
  281. # cache miss
  282. obj.mock.return_value = {10: 'fish', 20: 'chips'}
  283. r1 = yield obj.list_fn([10, 20], 2, on_invalidate=invalidate0)
  284. obj.mock.assert_called_once_with([10, 20], 2)
  285. self.assertEqual(r1, {10: 'fish', 20: 'chips'})
  286. obj.mock.reset_mock()
  287. # cache hit
  288. r2 = yield obj.list_fn([10, 20], 2, on_invalidate=invalidate1)
  289. obj.mock.assert_not_called()
  290. self.assertEqual(r2, {10: 'fish', 20: 'chips'})
  291. invalidate0.assert_not_called()
  292. invalidate1.assert_not_called()
  293. # now if we invalidate the keys, both invalidations should get called
  294. obj.fn.invalidate((10, 2))
  295. invalidate0.assert_called_once()
  296. invalidate1.assert_called_once()