http.py 6.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212
  1. # Copyright 2014-2016 OpenMarket Ltd
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. import json
  15. import urllib
  16. from pprint import pformat
  17. from typing import Optional
  18. from twisted.internet import defer, reactor
  19. from twisted.web.client import Agent, readBody
  20. from twisted.web.http_headers import Headers
  21. class HttpClient:
  22. """Interface for talking json over http"""
  23. def put_json(self, url, data):
  24. """Sends the specifed json data using PUT
  25. Args:
  26. url (str): The URL to PUT data to.
  27. data (dict): A dict containing the data that will be used as
  28. the request body. This will be encoded as JSON.
  29. Returns:
  30. Deferred: Succeeds when we get a 2xx HTTP response. The result
  31. will be the decoded JSON body.
  32. """
  33. pass
  34. def get_json(self, url, args=None):
  35. """Gets some json from the given host homeserver and path
  36. Args:
  37. url (str): The URL to GET data from.
  38. args (dict): A dictionary used to create query strings, defaults to
  39. None.
  40. **Note**: The value of each key is assumed to be an iterable
  41. and *not* a string.
  42. Returns:
  43. Deferred: Succeeds when we get a 2xx HTTP response. The result
  44. will be the decoded JSON body.
  45. """
  46. pass
  47. class TwistedHttpClient(HttpClient):
  48. """Wrapper around the twisted HTTP client api.
  49. Attributes:
  50. agent (twisted.web.client.Agent): The twisted Agent used to send the
  51. requests.
  52. """
  53. def __init__(self):
  54. self.agent = Agent(reactor)
  55. @defer.inlineCallbacks
  56. def put_json(self, url, data):
  57. response = yield self._create_put_request(
  58. url, data, headers_dict={"Content-Type": ["application/json"]}
  59. )
  60. body = yield readBody(response)
  61. defer.returnValue((response.code, body))
  62. @defer.inlineCallbacks
  63. def get_json(self, url, args=None):
  64. if args:
  65. # generates a list of strings of form "k=v".
  66. qs = urllib.urlencode(args, True)
  67. url = "%s?%s" % (url, qs)
  68. response = yield self._create_get_request(url)
  69. body = yield readBody(response)
  70. defer.returnValue(json.loads(body))
  71. def _create_put_request(self, url, json_data, headers_dict: Optional[dict] = None):
  72. """Wrapper of _create_request to issue a PUT request"""
  73. headers_dict = headers_dict or {}
  74. if "Content-Type" not in headers_dict:
  75. raise defer.error(RuntimeError("Must include Content-Type header for PUTs"))
  76. return self._create_request(
  77. "PUT", url, producer=_JsonProducer(json_data), headers_dict=headers_dict
  78. )
  79. def _create_get_request(self, url, headers_dict: Optional[dict] = None):
  80. """Wrapper of _create_request to issue a GET request"""
  81. return self._create_request("GET", url, headers_dict=headers_dict or {})
  82. @defer.inlineCallbacks
  83. def do_request(
  84. self,
  85. method,
  86. url,
  87. data=None,
  88. qparams=None,
  89. jsonreq=True,
  90. headers: Optional[dict] = None,
  91. ):
  92. headers = headers or {}
  93. if qparams:
  94. url = "%s?%s" % (url, urllib.urlencode(qparams, True))
  95. if jsonreq:
  96. prod = _JsonProducer(data)
  97. headers["Content-Type"] = ["application/json"]
  98. else:
  99. prod = _RawProducer(data)
  100. if method in ["POST", "PUT"]:
  101. response = yield self._create_request(
  102. method, url, producer=prod, headers_dict=headers
  103. )
  104. else:
  105. response = yield self._create_request(method, url)
  106. body = yield readBody(response)
  107. defer.returnValue(json.loads(body))
  108. @defer.inlineCallbacks
  109. def _create_request(
  110. self, method, url, producer=None, headers_dict: Optional[dict] = None
  111. ):
  112. """Creates and sends a request to the given url"""
  113. headers_dict = headers_dict or {}
  114. headers_dict["User-Agent"] = ["Synapse Cmd Client"]
  115. retries_left = 5
  116. print("%s to %s with headers %s" % (method, url, headers_dict))
  117. if self.verbose and producer:
  118. if "password" in producer.data:
  119. temp = producer.data["password"]
  120. producer.data["password"] = "[REDACTED]"
  121. print(json.dumps(producer.data, indent=4))
  122. producer.data["password"] = temp
  123. else:
  124. print(json.dumps(producer.data, indent=4))
  125. while True:
  126. try:
  127. response = yield self.agent.request(
  128. method, url.encode("UTF8"), Headers(headers_dict), producer
  129. )
  130. break
  131. except Exception as e:
  132. print("uh oh: %s" % e)
  133. if retries_left:
  134. yield self.sleep(2 ** (5 - retries_left))
  135. retries_left -= 1
  136. else:
  137. raise e
  138. if self.verbose:
  139. print("Status %s %s" % (response.code, response.phrase))
  140. print(pformat(list(response.headers.getAllRawHeaders())))
  141. defer.returnValue(response)
  142. def sleep(self, seconds):
  143. d = defer.Deferred()
  144. reactor.callLater(seconds, d.callback, seconds)
  145. return d
  146. class _RawProducer:
  147. def __init__(self, data):
  148. self.data = data
  149. self.body = data
  150. self.length = len(self.body)
  151. def startProducing(self, consumer):
  152. consumer.write(self.body)
  153. return defer.succeed(None)
  154. def pauseProducing(self):
  155. pass
  156. def stopProducing(self):
  157. pass
  158. class _JsonProducer:
  159. """Used by the twisted http client to create the HTTP body from json"""
  160. def __init__(self, jsn):
  161. self.data = jsn
  162. self.body = json.dumps(jsn).encode("utf8")
  163. self.length = len(self.body)
  164. def startProducing(self, consumer):
  165. consumer.write(self.body)
  166. return defer.succeed(None)
  167. def pauseProducing(self):
  168. pass
  169. def stopProducing(self):
  170. pass