server.py 34 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994
  1. # Copyright 2014-2016 OpenMarket Ltd
  2. # Copyright 2018 New Vector Ltd
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License");
  5. # you may not use this file except in compliance with the License.
  6. # You may obtain a copy of the License at
  7. #
  8. # http://www.apache.org/licenses/LICENSE-2.0
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS,
  12. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13. # See the License for the specific language governing permissions and
  14. # limitations under the License.
  15. import abc
  16. import html
  17. import logging
  18. import types
  19. import urllib
  20. from http import HTTPStatus
  21. from http.client import FOUND
  22. from inspect import isawaitable
  23. from typing import (
  24. TYPE_CHECKING,
  25. Any,
  26. Awaitable,
  27. Callable,
  28. Dict,
  29. Iterable,
  30. Iterator,
  31. List,
  32. NoReturn,
  33. Optional,
  34. Pattern,
  35. Tuple,
  36. Union,
  37. )
  38. import attr
  39. import jinja2
  40. from canonicaljson import encode_canonical_json
  41. from typing_extensions import Protocol
  42. from zope.interface import implementer
  43. from twisted.internet import defer, interfaces
  44. from twisted.internet.defer import CancelledError
  45. from twisted.python import failure
  46. from twisted.web import resource
  47. from twisted.web.server import NOT_DONE_YET, Request
  48. from twisted.web.static import File
  49. from twisted.web.util import redirectTo
  50. from synapse.api.errors import (
  51. CodeMessageException,
  52. Codes,
  53. RedirectException,
  54. SynapseError,
  55. UnrecognizedRequestError,
  56. )
  57. from synapse.config.homeserver import HomeServerConfig
  58. from synapse.http.site import SynapseRequest
  59. from synapse.logging.context import defer_to_thread, preserve_fn, run_in_background
  60. from synapse.logging.opentracing import active_span, start_active_span, trace_servlet
  61. from synapse.util import json_encoder
  62. from synapse.util.caches import intern_dict
  63. from synapse.util.cancellation import is_function_cancellable
  64. from synapse.util.iterutils import chunk_seq
  65. if TYPE_CHECKING:
  66. import opentracing
  67. from synapse.server import HomeServer
  68. logger = logging.getLogger(__name__)
  69. HTML_ERROR_TEMPLATE = """<!DOCTYPE html>
  70. <html lang=en>
  71. <head>
  72. <meta charset="utf-8">
  73. <title>Error {code}</title>
  74. </head>
  75. <body>
  76. <p>{msg}</p>
  77. </body>
  78. </html>
  79. """
  80. # A fictional HTTP status code for requests where the client has disconnected and we
  81. # successfully cancelled the request. Used only for logging purposes. Clients will never
  82. # observe this code unless cancellations leak across requests or we raise a
  83. # `CancelledError` ourselves.
  84. # Analogous to nginx's 499 status code:
  85. # https://github.com/nginx/nginx/blob/release-1.21.6/src/http/ngx_http_request.h#L128-L134
  86. HTTP_STATUS_REQUEST_CANCELLED = 499
  87. def return_json_error(
  88. f: failure.Failure, request: SynapseRequest, config: Optional[HomeServerConfig]
  89. ) -> None:
  90. """Sends a JSON error response to clients."""
  91. if f.check(SynapseError):
  92. # mypy doesn't understand that f.check asserts the type.
  93. exc: SynapseError = f.value # type: ignore
  94. error_code = exc.code
  95. error_dict = exc.error_dict(config)
  96. logger.info("%s SynapseError: %s - %s", request, error_code, exc.msg)
  97. elif f.check(CancelledError):
  98. error_code = HTTP_STATUS_REQUEST_CANCELLED
  99. error_dict = {"error": "Request cancelled", "errcode": Codes.UNKNOWN}
  100. if not request._disconnected:
  101. logger.error(
  102. "Got cancellation before client disconnection from %r: %r",
  103. request.request_metrics.name,
  104. request,
  105. exc_info=(f.type, f.value, f.getTracebackObject()), # type: ignore[arg-type]
  106. )
  107. else:
  108. error_code = 500
  109. error_dict = {"error": "Internal server error", "errcode": Codes.UNKNOWN}
  110. logger.error(
  111. "Failed handle request via %r: %r",
  112. request.request_metrics.name,
  113. request,
  114. exc_info=(f.type, f.value, f.getTracebackObject()), # type: ignore[arg-type]
  115. )
  116. # Only respond with an error response if we haven't already started writing,
  117. # otherwise lets just kill the connection
  118. if request.startedWriting:
  119. if request.transport:
  120. try:
  121. request.transport.abortConnection()
  122. except Exception:
  123. # abortConnection throws if the connection is already closed
  124. pass
  125. else:
  126. respond_with_json(
  127. request,
  128. error_code,
  129. error_dict,
  130. send_cors=True,
  131. )
  132. def return_html_error(
  133. f: failure.Failure,
  134. request: Request,
  135. error_template: Union[str, jinja2.Template],
  136. ) -> None:
  137. """Sends an HTML error page corresponding to the given failure.
  138. Handles RedirectException and other CodeMessageExceptions (such as SynapseError)
  139. Args:
  140. f: the error to report
  141. request: the failing request
  142. error_template: the HTML template. Can be either a string (with `{code}`,
  143. `{msg}` placeholders), or a jinja2 template
  144. """
  145. if f.check(CodeMessageException):
  146. # mypy doesn't understand that f.check asserts the type.
  147. cme: CodeMessageException = f.value # type: ignore
  148. code = cme.code
  149. msg = cme.msg
  150. if isinstance(cme, RedirectException):
  151. logger.info("%s redirect to %s", request, cme.location)
  152. request.setHeader(b"location", cme.location)
  153. request.cookies.extend(cme.cookies)
  154. elif isinstance(cme, SynapseError):
  155. logger.info("%s SynapseError: %s - %s", request, code, msg)
  156. else:
  157. logger.error(
  158. "Failed handle request %r",
  159. request,
  160. exc_info=(f.type, f.value, f.getTracebackObject()), # type: ignore[arg-type]
  161. )
  162. elif f.check(CancelledError):
  163. code = HTTP_STATUS_REQUEST_CANCELLED
  164. msg = "Request cancelled"
  165. if not request._disconnected:
  166. logger.error(
  167. "Got cancellation before client disconnection when handling request %r",
  168. request,
  169. exc_info=(f.type, f.value, f.getTracebackObject()), # type: ignore[arg-type]
  170. )
  171. else:
  172. code = HTTPStatus.INTERNAL_SERVER_ERROR
  173. msg = "Internal server error"
  174. logger.error(
  175. "Failed handle request %r",
  176. request,
  177. exc_info=(f.type, f.value, f.getTracebackObject()), # type: ignore[arg-type]
  178. )
  179. if isinstance(error_template, str):
  180. body = error_template.format(code=code, msg=html.escape(msg))
  181. else:
  182. body = error_template.render(code=code, msg=msg)
  183. respond_with_html(request, code, body)
  184. def wrap_async_request_handler(
  185. h: Callable[["_AsyncResource", SynapseRequest], Awaitable[None]]
  186. ) -> Callable[["_AsyncResource", SynapseRequest], "defer.Deferred[None]"]:
  187. """Wraps an async request handler so that it calls request.processing.
  188. This helps ensure that work done by the request handler after the request is completed
  189. is correctly recorded against the request metrics/logs.
  190. The handler method must have a signature of "handle_foo(self, request)",
  191. where "request" must be a SynapseRequest.
  192. The handler may return a deferred, in which case the completion of the request isn't
  193. logged until the deferred completes.
  194. """
  195. async def wrapped_async_request_handler(
  196. self: "_AsyncResource", request: SynapseRequest
  197. ) -> None:
  198. with request.processing():
  199. await h(self, request)
  200. # we need to preserve_fn here, because the synchronous render method won't yield for
  201. # us (obviously)
  202. return preserve_fn(wrapped_async_request_handler)
  203. # Type of a callback method for processing requests
  204. # it is actually called with a SynapseRequest and a kwargs dict for the params,
  205. # but I can't figure out how to represent that.
  206. ServletCallback = Callable[
  207. ..., Union[None, Awaitable[None], Tuple[int, Any], Awaitable[Tuple[int, Any]]]
  208. ]
  209. class HttpServer(Protocol):
  210. """Interface for registering callbacks on a HTTP server"""
  211. def register_paths(
  212. self,
  213. method: str,
  214. path_patterns: Iterable[Pattern],
  215. callback: ServletCallback,
  216. servlet_classname: str,
  217. ) -> None:
  218. """Register a callback that gets fired if we receive a http request
  219. with the given method for a path that matches the given regex.
  220. If the regex contains groups these gets passed to the callback via
  221. an unpacked tuple.
  222. The callback may be marked with the `@cancellable` decorator, which will
  223. cause request processing to be cancelled when clients disconnect early.
  224. Args:
  225. method: The HTTP method to listen to.
  226. path_patterns: The regex used to match requests.
  227. callback: The function to fire if we receive a matched
  228. request. The first argument will be the request object and
  229. subsequent arguments will be any matched groups from the regex.
  230. This should return either tuple of (code, response), or None.
  231. servlet_classname (str): The name of the handler to be used in prometheus
  232. and opentracing logs.
  233. """
  234. class _AsyncResource(resource.Resource, metaclass=abc.ABCMeta):
  235. """Base class for resources that have async handlers.
  236. Sub classes can either implement `_async_render_<METHOD>` to handle
  237. requests by method, or override `_async_render` to handle all requests.
  238. Args:
  239. extract_context: Whether to attempt to extract the opentracing
  240. context from the request the servlet is handling.
  241. """
  242. def __init__(self, extract_context: bool = False):
  243. super().__init__()
  244. self._extract_context = extract_context
  245. def render(self, request: SynapseRequest) -> int:
  246. """This gets called by twisted every time someone sends us a request."""
  247. request.render_deferred = defer.ensureDeferred(
  248. self._async_render_wrapper(request)
  249. )
  250. return NOT_DONE_YET
  251. @wrap_async_request_handler
  252. async def _async_render_wrapper(self, request: SynapseRequest) -> None:
  253. """This is a wrapper that delegates to `_async_render` and handles
  254. exceptions, return values, metrics, etc.
  255. """
  256. try:
  257. request.request_metrics.name = self.__class__.__name__
  258. with trace_servlet(request, self._extract_context):
  259. callback_return = await self._async_render(request)
  260. if callback_return is not None:
  261. code, response = callback_return
  262. self._send_response(request, code, response)
  263. except Exception:
  264. # failure.Failure() fishes the original Failure out
  265. # of our stack, and thus gives us a sensible stack
  266. # trace.
  267. f = failure.Failure()
  268. self._send_error_response(f, request)
  269. async def _async_render(self, request: SynapseRequest) -> Optional[Tuple[int, Any]]:
  270. """Delegates to `_async_render_<METHOD>` methods, or returns a 400 if
  271. no appropriate method exists. Can be overridden in sub classes for
  272. different routing.
  273. """
  274. # Treat HEAD requests as GET requests.
  275. request_method = request.method.decode("ascii")
  276. if request_method == "HEAD":
  277. request_method = "GET"
  278. method_handler = getattr(self, "_async_render_%s" % (request_method,), None)
  279. if method_handler:
  280. request.is_render_cancellable = is_function_cancellable(method_handler)
  281. raw_callback_return = method_handler(request)
  282. # Is it synchronous? We'll allow this for now.
  283. if isawaitable(raw_callback_return):
  284. callback_return = await raw_callback_return
  285. else:
  286. callback_return = raw_callback_return
  287. return callback_return
  288. return _unrecognised_request_handler(request)
  289. @abc.abstractmethod
  290. def _send_response(
  291. self,
  292. request: SynapseRequest,
  293. code: int,
  294. response_object: Any,
  295. ) -> None:
  296. raise NotImplementedError()
  297. @abc.abstractmethod
  298. def _send_error_response(
  299. self,
  300. f: failure.Failure,
  301. request: SynapseRequest,
  302. ) -> None:
  303. raise NotImplementedError()
  304. class DirectServeJsonResource(_AsyncResource):
  305. """A resource that will call `self._async_on_<METHOD>` on new requests,
  306. formatting responses and errors as JSON.
  307. """
  308. def __init__(self, canonical_json: bool = False, extract_context: bool = False):
  309. super().__init__(extract_context)
  310. self.canonical_json = canonical_json
  311. def _send_response(
  312. self,
  313. request: SynapseRequest,
  314. code: int,
  315. response_object: Any,
  316. ) -> None:
  317. """Implements _AsyncResource._send_response"""
  318. # TODO: Only enable CORS for the requests that need it.
  319. respond_with_json(
  320. request,
  321. code,
  322. response_object,
  323. send_cors=True,
  324. canonical_json=self.canonical_json,
  325. )
  326. def _send_error_response(
  327. self,
  328. f: failure.Failure,
  329. request: SynapseRequest,
  330. ) -> None:
  331. """Implements _AsyncResource._send_error_response"""
  332. return_json_error(f, request, None)
  333. @attr.s(slots=True, frozen=True, auto_attribs=True)
  334. class _PathEntry:
  335. pattern: Pattern
  336. callback: ServletCallback
  337. servlet_classname: str
  338. class JsonResource(DirectServeJsonResource):
  339. """This implements the HttpServer interface and provides JSON support for
  340. Resources.
  341. Register callbacks via register_paths()
  342. Callbacks can return a tuple of status code and a dict in which case the
  343. the dict will automatically be sent to the client as a JSON object.
  344. The JsonResource is primarily intended for returning JSON, but callbacks
  345. may send something other than JSON, they may do so by using the methods
  346. on the request object and instead returning None.
  347. """
  348. isLeaf = True
  349. def __init__(
  350. self,
  351. hs: "HomeServer",
  352. canonical_json: bool = True,
  353. extract_context: bool = False,
  354. ):
  355. super().__init__(canonical_json, extract_context)
  356. self.clock = hs.get_clock()
  357. self.path_regexs: Dict[bytes, List[_PathEntry]] = {}
  358. self.hs = hs
  359. def register_paths(
  360. self,
  361. method: str,
  362. path_patterns: Iterable[Pattern],
  363. callback: ServletCallback,
  364. servlet_classname: str,
  365. ) -> None:
  366. """
  367. Registers a request handler against a regular expression. Later request URLs are
  368. checked against these regular expressions in order to identify an appropriate
  369. handler for that request.
  370. Args:
  371. method: GET, POST etc
  372. path_patterns: A list of regular expressions to which the request
  373. URLs are compared.
  374. callback: The handler for the request. Usually a Servlet
  375. servlet_classname: The name of the handler to be used in prometheus
  376. and opentracing logs.
  377. """
  378. method_bytes = method.encode("utf-8")
  379. for path_pattern in path_patterns:
  380. logger.debug("Registering for %s %s", method, path_pattern.pattern)
  381. self.path_regexs.setdefault(method_bytes, []).append(
  382. _PathEntry(path_pattern, callback, servlet_classname)
  383. )
  384. def _get_handler_for_request(
  385. self, request: SynapseRequest
  386. ) -> Tuple[ServletCallback, str, Dict[str, str]]:
  387. """Finds a callback method to handle the given request.
  388. Returns:
  389. A tuple of the callback to use, the name of the servlet, and the
  390. key word arguments to pass to the callback
  391. """
  392. # At this point the path must be bytes.
  393. request_path_bytes: bytes = request.path # type: ignore
  394. request_path = request_path_bytes.decode("ascii")
  395. # Treat HEAD requests as GET requests.
  396. request_method = request.method
  397. if request_method == b"HEAD":
  398. request_method = b"GET"
  399. # Loop through all the registered callbacks to check if the method
  400. # and path regex match
  401. for path_entry in self.path_regexs.get(request_method, []):
  402. m = path_entry.pattern.match(request_path)
  403. if m:
  404. # We found a match!
  405. return path_entry.callback, path_entry.servlet_classname, m.groupdict()
  406. # Huh. No one wanted to handle that? Fiiiiiine. Send 400.
  407. return _unrecognised_request_handler, "unrecognised_request_handler", {}
  408. async def _async_render(self, request: SynapseRequest) -> Tuple[int, Any]:
  409. callback, servlet_classname, group_dict = self._get_handler_for_request(request)
  410. request.is_render_cancellable = is_function_cancellable(callback)
  411. # Make sure we have an appropriate name for this handler in prometheus
  412. # (rather than the default of JsonResource).
  413. request.request_metrics.name = servlet_classname
  414. # Now trigger the callback. If it returns a response, we send it
  415. # here. If it throws an exception, that is handled by the wrapper
  416. # installed by @request_handler.
  417. kwargs = intern_dict(
  418. {
  419. name: urllib.parse.unquote(value) if value else value
  420. for name, value in group_dict.items()
  421. }
  422. )
  423. raw_callback_return = callback(request, **kwargs)
  424. # Is it synchronous? We'll allow this for now.
  425. if isinstance(raw_callback_return, (defer.Deferred, types.CoroutineType)):
  426. callback_return = await raw_callback_return
  427. else:
  428. callback_return = raw_callback_return
  429. return callback_return
  430. def _send_error_response(
  431. self,
  432. f: failure.Failure,
  433. request: SynapseRequest,
  434. ) -> None:
  435. """Implements _AsyncResource._send_error_response"""
  436. return_json_error(f, request, self.hs.config)
  437. class DirectServeHtmlResource(_AsyncResource):
  438. """A resource that will call `self._async_on_<METHOD>` on new requests,
  439. formatting responses and errors as HTML.
  440. """
  441. # The error template to use for this resource
  442. ERROR_TEMPLATE = HTML_ERROR_TEMPLATE
  443. def _send_response(
  444. self,
  445. request: SynapseRequest,
  446. code: int,
  447. response_object: Any,
  448. ) -> None:
  449. """Implements _AsyncResource._send_response"""
  450. # We expect to get bytes for us to write
  451. assert isinstance(response_object, bytes)
  452. html_bytes = response_object
  453. respond_with_html_bytes(request, 200, html_bytes)
  454. def _send_error_response(
  455. self,
  456. f: failure.Failure,
  457. request: SynapseRequest,
  458. ) -> None:
  459. """Implements _AsyncResource._send_error_response"""
  460. return_html_error(f, request, self.ERROR_TEMPLATE)
  461. class StaticResource(File):
  462. """
  463. A resource that represents a plain non-interpreted file or directory.
  464. Differs from the File resource by adding clickjacking protection.
  465. """
  466. def render_GET(self, request: Request) -> bytes:
  467. set_clickjacking_protection_headers(request)
  468. return super().render_GET(request)
  469. def _unrecognised_request_handler(request: Request) -> NoReturn:
  470. """Request handler for unrecognised requests
  471. This is a request handler suitable for return from
  472. _get_handler_for_request. It actually just raises an
  473. UnrecognizedRequestError.
  474. Args:
  475. request: Unused, but passed in to match the signature of ServletCallback.
  476. """
  477. raise UnrecognizedRequestError()
  478. class RootRedirect(resource.Resource):
  479. """Redirects the root '/' path to another path."""
  480. def __init__(self, path: str):
  481. super().__init__()
  482. self.url = path
  483. def render_GET(self, request: Request) -> bytes:
  484. return redirectTo(self.url.encode("ascii"), request)
  485. def getChild(self, name: str, request: Request) -> resource.Resource:
  486. if len(name) == 0:
  487. return self # select ourselves as the child to render
  488. return super().getChild(name, request)
  489. class OptionsResource(resource.Resource):
  490. """Responds to OPTION requests for itself and all children."""
  491. def render_OPTIONS(self, request: SynapseRequest) -> bytes:
  492. request.setResponseCode(204)
  493. request.setHeader(b"Content-Length", b"0")
  494. set_cors_headers(request)
  495. return b""
  496. def getChildWithDefault(self, path: str, request: Request) -> resource.Resource:
  497. if request.method == b"OPTIONS":
  498. return self # select ourselves as the child to render
  499. return super().getChildWithDefault(path, request)
  500. class RootOptionsRedirectResource(OptionsResource, RootRedirect):
  501. pass
  502. @implementer(interfaces.IPushProducer)
  503. class _ByteProducer:
  504. """
  505. Iteratively write bytes to the request.
  506. """
  507. # The minimum number of bytes for each chunk. Note that the last chunk will
  508. # usually be smaller than this.
  509. min_chunk_size = 1024
  510. def __init__(
  511. self,
  512. request: Request,
  513. iterator: Iterator[bytes],
  514. ):
  515. self._request: Optional[Request] = request
  516. self._iterator = iterator
  517. self._paused = False
  518. try:
  519. self._request.registerProducer(self, True)
  520. except AttributeError as e:
  521. # Calling self._request.registerProducer might raise an AttributeError since
  522. # the underlying Twisted code calls self._request.channel.registerProducer,
  523. # however self._request.channel will be None if the connection was lost.
  524. logger.info("Connection disconnected before response was written: %r", e)
  525. # We drop our references to data we'll not use.
  526. self._request = None
  527. self._iterator = iter(())
  528. else:
  529. # Start producing if `registerProducer` was successful
  530. self.resumeProducing()
  531. def _send_data(self, data: List[bytes]) -> None:
  532. """
  533. Send a list of bytes as a chunk of a response.
  534. """
  535. if not data or not self._request:
  536. return
  537. self._request.write(b"".join(data))
  538. def pauseProducing(self) -> None:
  539. self._paused = True
  540. def resumeProducing(self) -> None:
  541. # We've stopped producing in the meantime (note that this might be
  542. # re-entrant after calling write).
  543. if not self._request:
  544. return
  545. self._paused = False
  546. # Write until there's backpressure telling us to stop.
  547. while not self._paused:
  548. # Get the next chunk and write it to the request.
  549. #
  550. # The output of the JSON encoder is buffered and coalesced until
  551. # min_chunk_size is reached. This is because JSON encoders produce
  552. # very small output per iteration and the Request object converts
  553. # each call to write() to a separate chunk. Without this there would
  554. # be an explosion in bytes written (e.g. b"{" becoming "1\r\n{\r\n").
  555. #
  556. # Note that buffer stores a list of bytes (instead of appending to
  557. # bytes) to hopefully avoid many allocations.
  558. buffer = []
  559. buffered_bytes = 0
  560. while buffered_bytes < self.min_chunk_size:
  561. try:
  562. data = next(self._iterator)
  563. buffer.append(data)
  564. buffered_bytes += len(data)
  565. except StopIteration:
  566. # The entire JSON object has been serialized, write any
  567. # remaining data, finalize the producer and the request, and
  568. # clean-up any references.
  569. self._send_data(buffer)
  570. self._request.unregisterProducer()
  571. self._request.finish()
  572. self.stopProducing()
  573. return
  574. self._send_data(buffer)
  575. def stopProducing(self) -> None:
  576. # Clear a circular reference.
  577. self._request = None
  578. def _encode_json_bytes(json_object: object) -> bytes:
  579. """
  580. Encode an object into JSON. Returns an iterator of bytes.
  581. """
  582. return json_encoder.encode(json_object).encode("utf-8")
  583. def respond_with_json(
  584. request: SynapseRequest,
  585. code: int,
  586. json_object: Any,
  587. send_cors: bool = False,
  588. canonical_json: bool = True,
  589. ) -> Optional[int]:
  590. """Sends encoded JSON in response to the given request.
  591. Args:
  592. request: The http request to respond to.
  593. code: The HTTP response code.
  594. json_object: The object to serialize to JSON.
  595. send_cors: Whether to send Cross-Origin Resource Sharing headers
  596. https://fetch.spec.whatwg.org/#http-cors-protocol
  597. canonical_json: Whether to use the canonicaljson algorithm when encoding
  598. the JSON bytes.
  599. Returns:
  600. twisted.web.server.NOT_DONE_YET if the request is still active.
  601. """
  602. # The response code must always be set, for logging purposes.
  603. request.setResponseCode(code)
  604. # could alternatively use request.notifyFinish() and flip a flag when
  605. # the Deferred fires, but since the flag is RIGHT THERE it seems like
  606. # a waste.
  607. if request._disconnected:
  608. logger.warning(
  609. "Not sending response to request %s, already disconnected.", request
  610. )
  611. return None
  612. if canonical_json:
  613. encoder: Callable[[object], bytes] = encode_canonical_json
  614. else:
  615. encoder = _encode_json_bytes
  616. request.setHeader(b"Content-Type", b"application/json")
  617. request.setHeader(b"Cache-Control", b"no-cache, no-store, must-revalidate")
  618. if send_cors:
  619. set_cors_headers(request)
  620. run_in_background(
  621. _async_write_json_to_request_in_thread, request, encoder, json_object
  622. )
  623. return NOT_DONE_YET
  624. def respond_with_json_bytes(
  625. request: SynapseRequest,
  626. code: int,
  627. json_bytes: bytes,
  628. send_cors: bool = False,
  629. ) -> Optional[int]:
  630. """Sends encoded JSON in response to the given request.
  631. Args:
  632. request: The http request to respond to.
  633. code: The HTTP response code.
  634. json_bytes: The json bytes to use as the response body.
  635. send_cors: Whether to send Cross-Origin Resource Sharing headers
  636. https://fetch.spec.whatwg.org/#http-cors-protocol
  637. Returns:
  638. twisted.web.server.NOT_DONE_YET if the request is still active.
  639. """
  640. # The response code must always be set, for logging purposes.
  641. request.setResponseCode(code)
  642. if request._disconnected:
  643. logger.warning(
  644. "Not sending response to request %s, already disconnected.", request
  645. )
  646. return None
  647. request.setHeader(b"Content-Type", b"application/json")
  648. request.setHeader(b"Content-Length", b"%d" % (len(json_bytes),))
  649. request.setHeader(b"Cache-Control", b"no-cache, no-store, must-revalidate")
  650. if send_cors:
  651. set_cors_headers(request)
  652. _write_bytes_to_request(request, json_bytes)
  653. return NOT_DONE_YET
  654. async def _async_write_json_to_request_in_thread(
  655. request: SynapseRequest,
  656. json_encoder: Callable[[Any], bytes],
  657. json_object: Any,
  658. ) -> None:
  659. """Encodes the given JSON object on a thread and then writes it to the
  660. request.
  661. This is done so that encoding large JSON objects doesn't block the reactor
  662. thread.
  663. Note: We don't use JsonEncoder.iterencode here as that falls back to the
  664. Python implementation (rather than the C backend), which is *much* more
  665. expensive.
  666. """
  667. def encode(opentracing_span: "Optional[opentracing.Span]") -> bytes:
  668. # it might take a while for the threadpool to schedule us, so we write
  669. # opentracing logs once we actually get scheduled, so that we can see how
  670. # much that contributed.
  671. if opentracing_span:
  672. opentracing_span.log_kv({"event": "scheduled"})
  673. res = json_encoder(json_object)
  674. if opentracing_span:
  675. opentracing_span.log_kv({"event": "encoded"})
  676. return res
  677. with start_active_span("encode_json_response"):
  678. span = active_span()
  679. json_str = await defer_to_thread(request.reactor, encode, span)
  680. _write_bytes_to_request(request, json_str)
  681. def _write_bytes_to_request(request: Request, bytes_to_write: bytes) -> None:
  682. """Writes the bytes to the request using an appropriate producer.
  683. Note: This should be used instead of `Request.write` to correctly handle
  684. large response bodies.
  685. """
  686. # The problem with dumping all of the response into the `Request` object at
  687. # once (via `Request.write`) is that doing so starts the timeout for the
  688. # next request to be received: so if it takes longer than 60s to stream back
  689. # the response to the client, the client never gets it.
  690. #
  691. # The correct solution is to use a Producer; then the timeout is only
  692. # started once all of the content is sent over the TCP connection.
  693. # To make sure we don't write all of the bytes at once we split it up into
  694. # chunks.
  695. chunk_size = 4096
  696. bytes_generator = chunk_seq(bytes_to_write, chunk_size)
  697. # We use a `_ByteProducer` here rather than `NoRangeStaticProducer` as the
  698. # unit tests can't cope with being given a pull producer.
  699. _ByteProducer(request, bytes_generator)
  700. def set_cors_headers(request: SynapseRequest) -> None:
  701. """Set the CORS headers so that javascript running in a web browsers can
  702. use this API
  703. Args:
  704. request: The http request to add CORS to.
  705. """
  706. request.setHeader(b"Access-Control-Allow-Origin", b"*")
  707. request.setHeader(
  708. b"Access-Control-Allow-Methods", b"GET, HEAD, POST, PUT, DELETE, OPTIONS"
  709. )
  710. if request.experimental_cors_msc3886:
  711. request.setHeader(
  712. b"Access-Control-Allow-Headers",
  713. b"X-Requested-With, Content-Type, Authorization, Date, If-Match, If-None-Match",
  714. )
  715. request.setHeader(
  716. b"Access-Control-Expose-Headers",
  717. b"ETag, Location, X-Max-Bytes",
  718. )
  719. else:
  720. request.setHeader(
  721. b"Access-Control-Allow-Headers",
  722. b"X-Requested-With, Content-Type, Authorization, Date",
  723. )
  724. def set_corp_headers(request: Request) -> None:
  725. """Set the CORP headers so that javascript running in a web browsers can
  726. embed the resource returned from this request when their client requires
  727. the `Cross-Origin-Embedder-Policy: require-corp` header.
  728. Args:
  729. request: The http request to add the CORP header to.
  730. """
  731. request.setHeader(b"Cross-Origin-Resource-Policy", b"cross-origin")
  732. def respond_with_html(request: Request, code: int, html: str) -> None:
  733. """
  734. Wraps `respond_with_html_bytes` by first encoding HTML from a str to UTF-8 bytes.
  735. """
  736. respond_with_html_bytes(request, code, html.encode("utf-8"))
  737. def respond_with_html_bytes(request: Request, code: int, html_bytes: bytes) -> None:
  738. """
  739. Sends HTML (encoded as UTF-8 bytes) as the response to the given request.
  740. Note that this adds clickjacking protection headers and finishes the request.
  741. Args:
  742. request: The http request to respond to.
  743. code: The HTTP response code.
  744. html_bytes: The HTML bytes to use as the response body.
  745. """
  746. # The response code must always be set, for logging purposes.
  747. request.setResponseCode(code)
  748. # could alternatively use request.notifyFinish() and flip a flag when
  749. # the Deferred fires, but since the flag is RIGHT THERE it seems like
  750. # a waste.
  751. if request._disconnected:
  752. logger.warning(
  753. "Not sending response to request %s, already disconnected.", request
  754. )
  755. return None
  756. request.setHeader(b"Content-Type", b"text/html; charset=utf-8")
  757. request.setHeader(b"Content-Length", b"%d" % (len(html_bytes),))
  758. # Ensure this content cannot be embedded.
  759. set_clickjacking_protection_headers(request)
  760. request.write(html_bytes)
  761. finish_request(request)
  762. def set_clickjacking_protection_headers(request: Request) -> None:
  763. """
  764. Set headers to guard against clickjacking of embedded content.
  765. This sets the X-Frame-Options and Content-Security-Policy headers which instructs
  766. browsers to not allow the HTML of the response to be embedded onto another
  767. page.
  768. Args:
  769. request: The http request to add the headers to.
  770. """
  771. request.setHeader(b"X-Frame-Options", b"DENY")
  772. request.setHeader(b"Content-Security-Policy", b"frame-ancestors 'none';")
  773. def respond_with_redirect(
  774. request: SynapseRequest, url: bytes, statusCode: int = FOUND, cors: bool = False
  775. ) -> None:
  776. """
  777. Write a 302 (or other specified status code) response to the request, if it is still alive.
  778. Args:
  779. request: The http request to respond to.
  780. url: The URL to redirect to.
  781. statusCode: The HTTP status code to use for the redirect (defaults to 302).
  782. cors: Whether to set CORS headers on the response.
  783. """
  784. logger.debug("Redirect to %s", url.decode("utf-8"))
  785. if cors:
  786. set_cors_headers(request)
  787. request.setResponseCode(statusCode)
  788. request.setHeader(b"location", url)
  789. finish_request(request)
  790. def finish_request(request: Request) -> None:
  791. """Finish writing the response to the request.
  792. Twisted throws a RuntimeException if the connection closed before the
  793. response was written but doesn't provide a convenient or reliable way to
  794. determine if the connection was closed. So we catch and log the RuntimeException
  795. You might think that ``request.notifyFinish`` could be used to tell if the
  796. request was finished. However the deferred it returns won't fire if the
  797. connection was already closed, meaning we'd have to have called the method
  798. right at the start of the request. By the time we want to write the response
  799. it will already be too late.
  800. """
  801. try:
  802. request.finish()
  803. except RuntimeError as e:
  804. logger.info("Connection disconnected before response was written: %r", e)