server.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507
  1. # -*- coding: utf-8 -*-
  2. # Copyright 2014-2016 OpenMarket Ltd
  3. # Copyright 2018 New Vector Ltd
  4. #
  5. # Licensed under the Apache License, Version 2.0 (the "License");
  6. # you may not use this file except in compliance with the License.
  7. # You may obtain a copy of the License at
  8. #
  9. # http://www.apache.org/licenses/LICENSE-2.0
  10. #
  11. # Unless required by applicable law or agreed to in writing, software
  12. # distributed under the License is distributed on an "AS IS" BASIS,
  13. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. # See the License for the specific language governing permissions and
  15. # limitations under the License.
  16. import cgi
  17. import collections
  18. import logging
  19. from six import PY3
  20. from six.moves import http_client, urllib
  21. from canonicaljson import encode_canonical_json, encode_pretty_printed_json, json
  22. from twisted.internet import defer
  23. from twisted.python import failure
  24. from twisted.web import resource
  25. from twisted.web.server import NOT_DONE_YET
  26. from twisted.web.static import NoRangeStaticProducer
  27. from twisted.web.util import redirectTo
  28. import synapse.events
  29. import synapse.metrics
  30. from synapse.api.errors import (
  31. CodeMessageException,
  32. Codes,
  33. SynapseError,
  34. UnrecognizedRequestError,
  35. )
  36. from synapse.util.caches import intern_dict
  37. from synapse.util.logcontext import preserve_fn
  38. if PY3:
  39. from io import BytesIO
  40. else:
  41. from cStringIO import StringIO as BytesIO
  42. logger = logging.getLogger(__name__)
  43. HTML_ERROR_TEMPLATE = """<!DOCTYPE html>
  44. <html lang=en>
  45. <head>
  46. <meta charset="utf-8">
  47. <title>Error {code}</title>
  48. </head>
  49. <body>
  50. <p>{msg}</p>
  51. </body>
  52. </html>
  53. """
  54. def wrap_json_request_handler(h):
  55. """Wraps a request handler method with exception handling.
  56. Also does the wrapping with request.processing as per wrap_async_request_handler.
  57. The handler method must have a signature of "handle_foo(self, request)",
  58. where "request" must be a SynapseRequest.
  59. The handler must return a deferred. If the deferred succeeds we assume that
  60. a response has been sent. If the deferred fails with a SynapseError we use
  61. it to send a JSON response with the appropriate HTTP reponse code. If the
  62. deferred fails with any other type of error we send a 500 reponse.
  63. """
  64. @defer.inlineCallbacks
  65. def wrapped_request_handler(self, request):
  66. try:
  67. yield h(self, request)
  68. except SynapseError as e:
  69. code = e.code
  70. logger.info(
  71. "%s SynapseError: %s - %s", request, code, e.msg
  72. )
  73. # Only respond with an error response if we haven't already started
  74. # writing, otherwise lets just kill the connection
  75. if request.startedWriting:
  76. if request.transport:
  77. try:
  78. request.transport.abortConnection()
  79. except Exception:
  80. # abortConnection throws if the connection is already closed
  81. pass
  82. else:
  83. respond_with_json(
  84. request, code, e.error_dict(), send_cors=True,
  85. pretty_print=_request_user_agent_is_curl(request),
  86. )
  87. except Exception:
  88. # failure.Failure() fishes the original Failure out
  89. # of our stack, and thus gives us a sensible stack
  90. # trace.
  91. f = failure.Failure()
  92. logger.error(
  93. "Failed handle request via %r: %r: %s",
  94. h,
  95. request,
  96. f.getTraceback().rstrip(),
  97. )
  98. # Only respond with an error response if we haven't already started
  99. # writing, otherwise lets just kill the connection
  100. if request.startedWriting:
  101. if request.transport:
  102. try:
  103. request.transport.abortConnection()
  104. except Exception:
  105. # abortConnection throws if the connection is already closed
  106. pass
  107. else:
  108. respond_with_json(
  109. request,
  110. 500,
  111. {
  112. "error": "Internal server error",
  113. "errcode": Codes.UNKNOWN,
  114. },
  115. send_cors=True,
  116. pretty_print=_request_user_agent_is_curl(request),
  117. )
  118. return wrap_async_request_handler(wrapped_request_handler)
  119. def wrap_html_request_handler(h):
  120. """Wraps a request handler method with exception handling.
  121. Also does the wrapping with request.processing as per wrap_async_request_handler.
  122. The handler method must have a signature of "handle_foo(self, request)",
  123. where "request" must be a SynapseRequest.
  124. """
  125. def wrapped_request_handler(self, request):
  126. d = defer.maybeDeferred(h, self, request)
  127. d.addErrback(_return_html_error, request)
  128. return d
  129. return wrap_async_request_handler(wrapped_request_handler)
  130. def _return_html_error(f, request):
  131. """Sends an HTML error page corresponding to the given failure
  132. Args:
  133. f (twisted.python.failure.Failure):
  134. request (twisted.web.iweb.IRequest):
  135. """
  136. if f.check(CodeMessageException):
  137. cme = f.value
  138. code = cme.code
  139. msg = cme.msg
  140. if isinstance(cme, SynapseError):
  141. logger.info(
  142. "%s SynapseError: %s - %s", request, code, msg
  143. )
  144. else:
  145. logger.error(
  146. "Failed handle request %r: %s",
  147. request,
  148. f.getTraceback().rstrip(),
  149. )
  150. else:
  151. code = http_client.INTERNAL_SERVER_ERROR
  152. msg = "Internal server error"
  153. logger.error(
  154. "Failed handle request %r: %s",
  155. request,
  156. f.getTraceback().rstrip(),
  157. )
  158. body = HTML_ERROR_TEMPLATE.format(
  159. code=code, msg=cgi.escape(msg),
  160. ).encode("utf-8")
  161. request.setResponseCode(code)
  162. request.setHeader(b"Content-Type", b"text/html; charset=utf-8")
  163. request.setHeader(b"Content-Length", b"%i" % (len(body),))
  164. request.write(body)
  165. finish_request(request)
  166. def wrap_async_request_handler(h):
  167. """Wraps an async request handler so that it calls request.processing.
  168. This helps ensure that work done by the request handler after the request is completed
  169. is correctly recorded against the request metrics/logs.
  170. The handler method must have a signature of "handle_foo(self, request)",
  171. where "request" must be a SynapseRequest.
  172. The handler may return a deferred, in which case the completion of the request isn't
  173. logged until the deferred completes.
  174. """
  175. @defer.inlineCallbacks
  176. def wrapped_async_request_handler(self, request):
  177. with request.processing():
  178. yield h(self, request)
  179. # we need to preserve_fn here, because the synchronous render method won't yield for
  180. # us (obviously)
  181. return preserve_fn(wrapped_async_request_handler)
  182. class HttpServer(object):
  183. """ Interface for registering callbacks on a HTTP server
  184. """
  185. def register_paths(self, method, path_patterns, callback):
  186. """ Register a callback that gets fired if we receive a http request
  187. with the given method for a path that matches the given regex.
  188. If the regex contains groups these gets passed to the calback via
  189. an unpacked tuple.
  190. Args:
  191. method (str): The method to listen to.
  192. path_patterns (list<SRE_Pattern>): The regex used to match requests.
  193. callback (function): The function to fire if we receive a matched
  194. request. The first argument will be the request object and
  195. subsequent arguments will be any matched groups from the regex.
  196. This should return a tuple of (code, response).
  197. """
  198. pass
  199. class JsonResource(HttpServer, resource.Resource):
  200. """ This implements the HttpServer interface and provides JSON support for
  201. Resources.
  202. Register callbacks via register_paths()
  203. Callbacks can return a tuple of status code and a dict in which case the
  204. the dict will automatically be sent to the client as a JSON object.
  205. The JsonResource is primarily intended for returning JSON, but callbacks
  206. may send something other than JSON, they may do so by using the methods
  207. on the request object and instead returning None.
  208. """
  209. isLeaf = True
  210. _PathEntry = collections.namedtuple("_PathEntry", ["pattern", "callback"])
  211. def __init__(self, hs, canonical_json=True):
  212. resource.Resource.__init__(self)
  213. self.canonical_json = canonical_json
  214. self.clock = hs.get_clock()
  215. self.path_regexs = {}
  216. self.hs = hs
  217. def register_paths(self, method, path_patterns, callback):
  218. method = method.encode("utf-8") # method is bytes on py3
  219. for path_pattern in path_patterns:
  220. logger.debug("Registering for %s %s", method, path_pattern.pattern)
  221. self.path_regexs.setdefault(method, []).append(
  222. self._PathEntry(path_pattern, callback)
  223. )
  224. def render(self, request):
  225. """ This gets called by twisted every time someone sends us a request.
  226. """
  227. self._async_render(request)
  228. return NOT_DONE_YET
  229. @wrap_json_request_handler
  230. @defer.inlineCallbacks
  231. def _async_render(self, request):
  232. """ This gets called from render() every time someone sends us a request.
  233. This checks if anyone has registered a callback for that method and
  234. path.
  235. """
  236. callback, group_dict = self._get_handler_for_request(request)
  237. servlet_instance = getattr(callback, "__self__", None)
  238. if servlet_instance is not None:
  239. servlet_classname = servlet_instance.__class__.__name__
  240. else:
  241. servlet_classname = "%r" % callback
  242. request.request_metrics.name = servlet_classname
  243. # Now trigger the callback. If it returns a response, we send it
  244. # here. If it throws an exception, that is handled by the wrapper
  245. # installed by @request_handler.
  246. def _unquote(s):
  247. if PY3:
  248. # On Python 3, unquote is unicode -> unicode
  249. return urllib.parse.unquote(s)
  250. else:
  251. # On Python 2, unquote is bytes -> bytes We need to encode the
  252. # URL again (as it was decoded by _get_handler_for request), as
  253. # ASCII because it's a URL, and then decode it to get the UTF-8
  254. # characters that were quoted.
  255. return urllib.parse.unquote(s.encode('ascii')).decode('utf8')
  256. kwargs = intern_dict({
  257. name: _unquote(value) if value else value
  258. for name, value in group_dict.items()
  259. })
  260. callback_return = yield callback(request, **kwargs)
  261. if callback_return is not None:
  262. code, response = callback_return
  263. self._send_response(request, code, response)
  264. def _get_handler_for_request(self, request):
  265. """Finds a callback method to handle the given request
  266. Args:
  267. request (twisted.web.http.Request):
  268. Returns:
  269. Tuple[Callable, dict[unicode, unicode]]: callback method, and the
  270. dict mapping keys to path components as specified in the
  271. handler's path match regexp.
  272. The callback will normally be a method registered via
  273. register_paths, so will return (possibly via Deferred) either
  274. None, or a tuple of (http code, response body).
  275. """
  276. if request.method == b"OPTIONS":
  277. return _options_handler, {}
  278. # Loop through all the registered callbacks to check if the method
  279. # and path regex match
  280. for path_entry in self.path_regexs.get(request.method, []):
  281. m = path_entry.pattern.match(request.path.decode('ascii'))
  282. if m:
  283. # We found a match!
  284. return path_entry.callback, m.groupdict()
  285. # Huh. No one wanted to handle that? Fiiiiiine. Send 400.
  286. return _unrecognised_request_handler, {}
  287. def _send_response(self, request, code, response_json_object,
  288. response_code_message=None):
  289. # TODO: Only enable CORS for the requests that need it.
  290. respond_with_json(
  291. request, code, response_json_object,
  292. send_cors=True,
  293. response_code_message=response_code_message,
  294. pretty_print=_request_user_agent_is_curl(request),
  295. canonical_json=self.canonical_json,
  296. )
  297. def _options_handler(request):
  298. """Request handler for OPTIONS requests
  299. This is a request handler suitable for return from
  300. _get_handler_for_request. It returns a 200 and an empty body.
  301. Args:
  302. request (twisted.web.http.Request):
  303. Returns:
  304. Tuple[int, dict]: http code, response body.
  305. """
  306. return 200, {}
  307. def _unrecognised_request_handler(request):
  308. """Request handler for unrecognised requests
  309. This is a request handler suitable for return from
  310. _get_handler_for_request. It actually just raises an
  311. UnrecognizedRequestError.
  312. Args:
  313. request (twisted.web.http.Request):
  314. """
  315. raise UnrecognizedRequestError()
  316. class RootRedirect(resource.Resource):
  317. """Redirects the root '/' path to another path."""
  318. def __init__(self, path):
  319. resource.Resource.__init__(self)
  320. self.url = path
  321. def render_GET(self, request):
  322. return redirectTo(self.url.encode('ascii'), request)
  323. def getChild(self, name, request):
  324. if len(name) == 0:
  325. return self # select ourselves as the child to render
  326. return resource.Resource.getChild(self, name, request)
  327. def respond_with_json(request, code, json_object, send_cors=False,
  328. response_code_message=None, pretty_print=False,
  329. canonical_json=True):
  330. # could alternatively use request.notifyFinish() and flip a flag when
  331. # the Deferred fires, but since the flag is RIGHT THERE it seems like
  332. # a waste.
  333. if request._disconnected:
  334. logger.warn(
  335. "Not sending response to request %s, already disconnected.",
  336. request)
  337. return
  338. if pretty_print:
  339. json_bytes = encode_pretty_printed_json(json_object) + b"\n"
  340. else:
  341. if canonical_json or synapse.events.USE_FROZEN_DICTS:
  342. # canonicaljson already encodes to bytes
  343. json_bytes = encode_canonical_json(json_object)
  344. else:
  345. json_bytes = json.dumps(json_object).encode("utf-8")
  346. return respond_with_json_bytes(
  347. request, code, json_bytes,
  348. send_cors=send_cors,
  349. response_code_message=response_code_message,
  350. )
  351. def respond_with_json_bytes(request, code, json_bytes, send_cors=False,
  352. response_code_message=None):
  353. """Sends encoded JSON in response to the given request.
  354. Args:
  355. request (twisted.web.http.Request): The http request to respond to.
  356. code (int): The HTTP response code.
  357. json_bytes (bytes): The json bytes to use as the response body.
  358. send_cors (bool): Whether to send Cross-Origin Resource Sharing headers
  359. http://www.w3.org/TR/cors/
  360. Returns:
  361. twisted.web.server.NOT_DONE_YET"""
  362. request.setResponseCode(code, message=response_code_message)
  363. request.setHeader(b"Content-Type", b"application/json")
  364. request.setHeader(b"Content-Length", b"%d" % (len(json_bytes),))
  365. request.setHeader(b"Cache-Control", b"no-cache, no-store, must-revalidate")
  366. if send_cors:
  367. set_cors_headers(request)
  368. # todo: we can almost certainly avoid this copy and encode the json straight into
  369. # the bytesIO, but it would involve faffing around with string->bytes wrappers.
  370. bytes_io = BytesIO(json_bytes)
  371. producer = NoRangeStaticProducer(request, bytes_io)
  372. producer.start()
  373. return NOT_DONE_YET
  374. def set_cors_headers(request):
  375. """Set the CORs headers so that javascript running in a web browsers can
  376. use this API
  377. Args:
  378. request (twisted.web.http.Request): The http request to add CORs to.
  379. """
  380. request.setHeader(b"Access-Control-Allow-Origin", b"*")
  381. request.setHeader(
  382. b"Access-Control-Allow-Methods", b"GET, POST, PUT, DELETE, OPTIONS"
  383. )
  384. request.setHeader(
  385. b"Access-Control-Allow-Headers",
  386. b"Origin, X-Requested-With, Content-Type, Accept, Authorization"
  387. )
  388. def finish_request(request):
  389. """ Finish writing the response to the request.
  390. Twisted throws a RuntimeException if the connection closed before the
  391. response was written but doesn't provide a convenient or reliable way to
  392. determine if the connection was closed. So we catch and log the RuntimeException
  393. You might think that ``request.notifyFinish`` could be used to tell if the
  394. request was finished. However the deferred it returns won't fire if the
  395. connection was already closed, meaning we'd have to have called the method
  396. right at the start of the request. By the time we want to write the response
  397. it will already be too late.
  398. """
  399. try:
  400. request.finish()
  401. except RuntimeError as e:
  402. logger.info("Connection disconnected before response was written: %r", e)
  403. def _request_user_agent_is_curl(request):
  404. user_agents = request.requestHeaders.getRawHeaders(
  405. b"User-Agent", default=[]
  406. )
  407. for user_agent in user_agents:
  408. if b"curl" in user_agent:
  409. return True
  410. return False