async_helpers.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456
  1. # -*- coding: utf-8 -*-
  2. # Copyright 2014-2016 OpenMarket Ltd
  3. # Copyright 2018 New Vector Ltd.
  4. #
  5. # Licensed under the Apache License, Version 2.0 (the "License");
  6. # you may not use this file except in compliance with the License.
  7. # You may obtain a copy of the License at
  8. #
  9. # http://www.apache.org/licenses/LICENSE-2.0
  10. #
  11. # Unless required by applicable law or agreed to in writing, software
  12. # distributed under the License is distributed on an "AS IS" BASIS,
  13. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. # See the License for the specific language governing permissions and
  15. # limitations under the License.
  16. import collections
  17. import logging
  18. from contextlib import contextmanager
  19. from six.moves import range
  20. from twisted.internet import defer
  21. from twisted.internet.defer import CancelledError
  22. from twisted.python import failure
  23. from synapse.util import Clock, logcontext, unwrapFirstError
  24. from .logcontext import (
  25. PreserveLoggingContext,
  26. make_deferred_yieldable,
  27. run_in_background,
  28. )
  29. logger = logging.getLogger(__name__)
  30. class ObservableDeferred(object):
  31. """Wraps a deferred object so that we can add observer deferreds. These
  32. observer deferreds do not affect the callback chain of the original
  33. deferred.
  34. If consumeErrors is true errors will be captured from the origin deferred.
  35. Cancelling or otherwise resolving an observer will not affect the original
  36. ObservableDeferred.
  37. NB that it does not attempt to do anything with logcontexts; in general
  38. you should probably make_deferred_yieldable the deferreds
  39. returned by `observe`, and ensure that the original deferred runs its
  40. callbacks in the sentinel logcontext.
  41. """
  42. __slots__ = ["_deferred", "_observers", "_result"]
  43. def __init__(self, deferred, consumeErrors=False):
  44. object.__setattr__(self, "_deferred", deferred)
  45. object.__setattr__(self, "_result", None)
  46. object.__setattr__(self, "_observers", set())
  47. def callback(r):
  48. object.__setattr__(self, "_result", (True, r))
  49. while self._observers:
  50. try:
  51. # TODO: Handle errors here.
  52. self._observers.pop().callback(r)
  53. except Exception:
  54. pass
  55. return r
  56. def errback(f):
  57. object.__setattr__(self, "_result", (False, f))
  58. while self._observers:
  59. try:
  60. # TODO: Handle errors here.
  61. self._observers.pop().errback(f)
  62. except Exception:
  63. pass
  64. if consumeErrors:
  65. return None
  66. else:
  67. return f
  68. deferred.addCallbacks(callback, errback)
  69. def observe(self):
  70. """Observe the underlying deferred.
  71. Can return either a deferred if the underlying deferred is still pending
  72. (or has failed), or the actual value. Callers may need to use maybeDeferred.
  73. """
  74. if not self._result:
  75. d = defer.Deferred()
  76. def remove(r):
  77. self._observers.discard(d)
  78. return r
  79. d.addBoth(remove)
  80. self._observers.add(d)
  81. return d
  82. else:
  83. success, res = self._result
  84. return res if success else defer.fail(res)
  85. def observers(self):
  86. return self._observers
  87. def has_called(self):
  88. return self._result is not None
  89. def has_succeeded(self):
  90. return self._result is not None and self._result[0] is True
  91. def get_result(self):
  92. return self._result[1]
  93. def __getattr__(self, name):
  94. return getattr(self._deferred, name)
  95. def __setattr__(self, name, value):
  96. setattr(self._deferred, name, value)
  97. def __repr__(self):
  98. return "<ObservableDeferred object at %s, result=%r, _deferred=%r>" % (
  99. id(self), self._result, self._deferred,
  100. )
  101. def concurrently_execute(func, args, limit):
  102. """Executes the function with each argument conncurrently while limiting
  103. the number of concurrent executions.
  104. Args:
  105. func (func): Function to execute, should return a deferred.
  106. args (list): List of arguments to pass to func, each invocation of func
  107. gets a signle argument.
  108. limit (int): Maximum number of conccurent executions.
  109. Returns:
  110. deferred: Resolved when all function invocations have finished.
  111. """
  112. it = iter(args)
  113. @defer.inlineCallbacks
  114. def _concurrently_execute_inner():
  115. try:
  116. while True:
  117. yield func(next(it))
  118. except StopIteration:
  119. pass
  120. return logcontext.make_deferred_yieldable(defer.gatherResults([
  121. run_in_background(_concurrently_execute_inner)
  122. for _ in range(limit)
  123. ], consumeErrors=True)).addErrback(unwrapFirstError)
  124. class Linearizer(object):
  125. """Limits concurrent access to resources based on a key. Useful to ensure
  126. only a few things happen at a time on a given resource.
  127. Example:
  128. with (yield limiter.queue("test_key")):
  129. # do some work.
  130. """
  131. def __init__(self, name=None, max_count=1, clock=None):
  132. """
  133. Args:
  134. max_count(int): The maximum number of concurrent accesses
  135. """
  136. if name is None:
  137. self.name = id(self)
  138. else:
  139. self.name = name
  140. if not clock:
  141. from twisted.internet import reactor
  142. clock = Clock(reactor)
  143. self._clock = clock
  144. self.max_count = max_count
  145. # key_to_defer is a map from the key to a 2 element list where
  146. # the first element is the number of things executing, and
  147. # the second element is an OrderedDict, where the keys are deferreds for the
  148. # things blocked from executing.
  149. self.key_to_defer = {}
  150. def queue(self, key):
  151. # we avoid doing defer.inlineCallbacks here, so that cancellation works correctly.
  152. # (https://twistedmatrix.com/trac/ticket/4632 meant that cancellations were not
  153. # propagated inside inlineCallbacks until Twisted 18.7)
  154. entry = self.key_to_defer.setdefault(key, [0, collections.OrderedDict()])
  155. # If the number of things executing is greater than the maximum
  156. # then add a deferred to the list of blocked items
  157. # When one of the things currently executing finishes it will callback
  158. # this item so that it can continue executing.
  159. if entry[0] >= self.max_count:
  160. res = self._await_lock(key)
  161. else:
  162. logger.info(
  163. "Acquired uncontended linearizer lock %r for key %r", self.name, key,
  164. )
  165. entry[0] += 1
  166. res = defer.succeed(None)
  167. # once we successfully get the lock, we need to return a context manager which
  168. # will release the lock.
  169. @contextmanager
  170. def _ctx_manager(_):
  171. try:
  172. yield
  173. finally:
  174. logger.info("Releasing linearizer lock %r for key %r", self.name, key)
  175. # We've finished executing so check if there are any things
  176. # blocked waiting to execute and start one of them
  177. entry[0] -= 1
  178. if entry[1]:
  179. (next_def, _) = entry[1].popitem(last=False)
  180. # we need to run the next thing in the sentinel context.
  181. with PreserveLoggingContext():
  182. next_def.callback(None)
  183. elif entry[0] == 0:
  184. # We were the last thing for this key: remove it from the
  185. # map.
  186. del self.key_to_defer[key]
  187. res.addCallback(_ctx_manager)
  188. return res
  189. def _await_lock(self, key):
  190. """Helper for queue: adds a deferred to the queue
  191. Assumes that we've already checked that we've reached the limit of the number
  192. of lock-holders we allow. Creates a new deferred which is added to the list, and
  193. adds some management around cancellations.
  194. Returns the deferred, which will callback once we have secured the lock.
  195. """
  196. entry = self.key_to_defer[key]
  197. logger.info(
  198. "Waiting to acquire linearizer lock %r for key %r", self.name, key,
  199. )
  200. new_defer = make_deferred_yieldable(defer.Deferred())
  201. entry[1][new_defer] = 1
  202. def cb(_r):
  203. logger.info("Acquired linearizer lock %r for key %r", self.name, key)
  204. entry[0] += 1
  205. # if the code holding the lock completes synchronously, then it
  206. # will recursively run the next claimant on the list. That can
  207. # relatively rapidly lead to stack exhaustion. This is essentially
  208. # the same problem as http://twistedmatrix.com/trac/ticket/9304.
  209. #
  210. # In order to break the cycle, we add a cheeky sleep(0) here to
  211. # ensure that we fall back to the reactor between each iteration.
  212. #
  213. # (This needs to happen while we hold the lock, and the context manager's exit
  214. # code must be synchronous, so this is the only sensible place.)
  215. return self._clock.sleep(0)
  216. def eb(e):
  217. logger.info("defer %r got err %r", new_defer, e)
  218. if isinstance(e, CancelledError):
  219. logger.info(
  220. "Cancelling wait for linearizer lock %r for key %r",
  221. self.name, key,
  222. )
  223. else:
  224. logger.warn(
  225. "Unexpected exception waiting for linearizer lock %r for key %r",
  226. self.name, key,
  227. )
  228. # we just have to take ourselves back out of the queue.
  229. del entry[1][new_defer]
  230. return e
  231. new_defer.addCallbacks(cb, eb)
  232. return new_defer
  233. class ReadWriteLock(object):
  234. """A deferred style read write lock.
  235. Example:
  236. with (yield read_write_lock.read("test_key")):
  237. # do some work
  238. """
  239. # IMPLEMENTATION NOTES
  240. #
  241. # We track the most recent queued reader and writer deferreds (which get
  242. # resolved when they release the lock).
  243. #
  244. # Read: We know its safe to acquire a read lock when the latest writer has
  245. # been resolved. The new reader is appeneded to the list of latest readers.
  246. #
  247. # Write: We know its safe to acquire the write lock when both the latest
  248. # writers and readers have been resolved. The new writer replaces the latest
  249. # writer.
  250. def __init__(self):
  251. # Latest readers queued
  252. self.key_to_current_readers = {}
  253. # Latest writer queued
  254. self.key_to_current_writer = {}
  255. @defer.inlineCallbacks
  256. def read(self, key):
  257. new_defer = defer.Deferred()
  258. curr_readers = self.key_to_current_readers.setdefault(key, set())
  259. curr_writer = self.key_to_current_writer.get(key, None)
  260. curr_readers.add(new_defer)
  261. # We wait for the latest writer to finish writing. We can safely ignore
  262. # any existing readers... as they're readers.
  263. yield make_deferred_yieldable(curr_writer)
  264. @contextmanager
  265. def _ctx_manager():
  266. try:
  267. yield
  268. finally:
  269. new_defer.callback(None)
  270. self.key_to_current_readers.get(key, set()).discard(new_defer)
  271. defer.returnValue(_ctx_manager())
  272. @defer.inlineCallbacks
  273. def write(self, key):
  274. new_defer = defer.Deferred()
  275. curr_readers = self.key_to_current_readers.get(key, set())
  276. curr_writer = self.key_to_current_writer.get(key, None)
  277. # We wait on all latest readers and writer.
  278. to_wait_on = list(curr_readers)
  279. if curr_writer:
  280. to_wait_on.append(curr_writer)
  281. # We can clear the list of current readers since the new writer waits
  282. # for them to finish.
  283. curr_readers.clear()
  284. self.key_to_current_writer[key] = new_defer
  285. yield make_deferred_yieldable(defer.gatherResults(to_wait_on))
  286. @contextmanager
  287. def _ctx_manager():
  288. try:
  289. yield
  290. finally:
  291. new_defer.callback(None)
  292. if self.key_to_current_writer[key] == new_defer:
  293. self.key_to_current_writer.pop(key)
  294. defer.returnValue(_ctx_manager())
  295. def _cancelled_to_timed_out_error(value, timeout):
  296. if isinstance(value, failure.Failure):
  297. value.trap(CancelledError)
  298. raise defer.TimeoutError(timeout, "Deferred")
  299. return value
  300. def timeout_deferred(deferred, timeout, reactor, on_timeout_cancel=None):
  301. """The in built twisted `Deferred.addTimeout` fails to time out deferreds
  302. that have a canceller that throws exceptions. This method creates a new
  303. deferred that wraps and times out the given deferred, correctly handling
  304. the case where the given deferred's canceller throws.
  305. (See https://twistedmatrix.com/trac/ticket/9534)
  306. NOTE: Unlike `Deferred.addTimeout`, this function returns a new deferred
  307. Args:
  308. deferred (Deferred)
  309. timeout (float): Timeout in seconds
  310. reactor (twisted.interfaces.IReactorTime): The twisted reactor to use
  311. on_timeout_cancel (callable): A callable which is called immediately
  312. after the deferred times out, and not if this deferred is
  313. otherwise cancelled before the timeout.
  314. It takes an arbitrary value, which is the value of the deferred at
  315. that exact point in time (probably a CancelledError Failure), and
  316. the timeout.
  317. The default callable (if none is provided) will translate a
  318. CancelledError Failure into a defer.TimeoutError.
  319. Returns:
  320. Deferred
  321. """
  322. new_d = defer.Deferred()
  323. timed_out = [False]
  324. def time_it_out():
  325. timed_out[0] = True
  326. try:
  327. deferred.cancel()
  328. except: # noqa: E722, if we throw any exception it'll break time outs
  329. logger.exception("Canceller failed during timeout")
  330. if not new_d.called:
  331. new_d.errback(defer.TimeoutError(timeout, "Deferred"))
  332. delayed_call = reactor.callLater(timeout, time_it_out)
  333. def convert_cancelled(value):
  334. if timed_out[0]:
  335. to_call = on_timeout_cancel or _cancelled_to_timed_out_error
  336. return to_call(value, timeout)
  337. return value
  338. deferred.addBoth(convert_cancelled)
  339. def cancel_timeout(result):
  340. # stop the pending call to cancel the deferred if it's been fired
  341. if delayed_call.active():
  342. delayed_call.cancel()
  343. return result
  344. deferred.addBoth(cancel_timeout)
  345. def success_cb(val):
  346. if not new_d.called:
  347. new_d.callback(val)
  348. def failure_cb(val):
  349. if not new_d.called:
  350. new_d.errback(val)
  351. deferred.addCallbacks(success_cb, failure_cb)
  352. return new_d