http.py 6.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206
  1. # -*- coding: utf-8 -*-
  2. # Copyright 2014-2016 OpenMarket Ltd
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License");
  5. # you may not use this file except in compliance with the License.
  6. # You may obtain a copy of the License at
  7. #
  8. # http://www.apache.org/licenses/LICENSE-2.0
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS,
  12. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13. # See the License for the specific language governing permissions and
  14. # limitations under the License.
  15. from __future__ import print_function
  16. from twisted.web.client import Agent, readBody
  17. from twisted.web.http_headers import Headers
  18. from twisted.internet import defer, reactor
  19. from pprint import pformat
  20. import json
  21. import urllib
  22. class HttpClient(object):
  23. """ Interface for talking json over http
  24. """
  25. def put_json(self, url, data):
  26. """ Sends the specifed json data using PUT
  27. Args:
  28. url (str): The URL to PUT data to.
  29. data (dict): A dict containing the data that will be used as
  30. the request body. This will be encoded as JSON.
  31. Returns:
  32. Deferred: Succeeds when we get a 2xx HTTP response. The result
  33. will be the decoded JSON body.
  34. """
  35. pass
  36. def get_json(self, url, args=None):
  37. """ Gets some json from the given host homeserver and path
  38. Args:
  39. url (str): The URL to GET data from.
  40. args (dict): A dictionary used to create query strings, defaults to
  41. None.
  42. **Note**: The value of each key is assumed to be an iterable
  43. and *not* a string.
  44. Returns:
  45. Deferred: Succeeds when we get a 2xx HTTP response. The result
  46. will be the decoded JSON body.
  47. """
  48. pass
  49. class TwistedHttpClient(HttpClient):
  50. """ Wrapper around the twisted HTTP client api.
  51. Attributes:
  52. agent (twisted.web.client.Agent): The twisted Agent used to send the
  53. requests.
  54. """
  55. def __init__(self):
  56. self.agent = Agent(reactor)
  57. @defer.inlineCallbacks
  58. def put_json(self, url, data):
  59. response = yield self._create_put_request(
  60. url, data, headers_dict={"Content-Type": ["application/json"]}
  61. )
  62. body = yield readBody(response)
  63. defer.returnValue((response.code, body))
  64. @defer.inlineCallbacks
  65. def get_json(self, url, args=None):
  66. if args:
  67. # generates a list of strings of form "k=v".
  68. qs = urllib.urlencode(args, True)
  69. url = "%s?%s" % (url, qs)
  70. response = yield self._create_get_request(url)
  71. body = yield readBody(response)
  72. defer.returnValue(json.loads(body))
  73. def _create_put_request(self, url, json_data, headers_dict={}):
  74. """ Wrapper of _create_request to issue a PUT request
  75. """
  76. if "Content-Type" not in headers_dict:
  77. raise defer.error(RuntimeError("Must include Content-Type header for PUTs"))
  78. return self._create_request(
  79. "PUT", url, producer=_JsonProducer(json_data), headers_dict=headers_dict
  80. )
  81. def _create_get_request(self, url, headers_dict={}):
  82. """ Wrapper of _create_request to issue a GET request
  83. """
  84. return self._create_request("GET", url, headers_dict=headers_dict)
  85. @defer.inlineCallbacks
  86. def do_request(
  87. self, method, url, data=None, qparams=None, jsonreq=True, headers={}
  88. ):
  89. if qparams:
  90. url = "%s?%s" % (url, urllib.urlencode(qparams, True))
  91. if jsonreq:
  92. prod = _JsonProducer(data)
  93. headers["Content-Type"] = ["application/json"]
  94. else:
  95. prod = _RawProducer(data)
  96. if method in ["POST", "PUT"]:
  97. response = yield self._create_request(
  98. method, url, producer=prod, headers_dict=headers
  99. )
  100. else:
  101. response = yield self._create_request(method, url)
  102. body = yield readBody(response)
  103. defer.returnValue(json.loads(body))
  104. @defer.inlineCallbacks
  105. def _create_request(self, method, url, producer=None, headers_dict={}):
  106. """ Creates and sends a request to the given url
  107. """
  108. headers_dict["User-Agent"] = ["Synapse Cmd Client"]
  109. retries_left = 5
  110. print("%s to %s with headers %s" % (method, url, headers_dict))
  111. if self.verbose and producer:
  112. if "password" in producer.data:
  113. temp = producer.data["password"]
  114. producer.data["password"] = "[REDACTED]"
  115. print(json.dumps(producer.data, indent=4))
  116. producer.data["password"] = temp
  117. else:
  118. print(json.dumps(producer.data, indent=4))
  119. while True:
  120. try:
  121. response = yield self.agent.request(
  122. method, url.encode("UTF8"), Headers(headers_dict), producer
  123. )
  124. break
  125. except Exception as e:
  126. print("uh oh: %s" % e)
  127. if retries_left:
  128. yield self.sleep(2 ** (5 - retries_left))
  129. retries_left -= 1
  130. else:
  131. raise e
  132. if self.verbose:
  133. print("Status %s %s" % (response.code, response.phrase))
  134. print(pformat(list(response.headers.getAllRawHeaders())))
  135. defer.returnValue(response)
  136. def sleep(self, seconds):
  137. d = defer.Deferred()
  138. reactor.callLater(seconds, d.callback, seconds)
  139. return d
  140. class _RawProducer(object):
  141. def __init__(self, data):
  142. self.data = data
  143. self.body = data
  144. self.length = len(self.body)
  145. def startProducing(self, consumer):
  146. consumer.write(self.body)
  147. return defer.succeed(None)
  148. def pauseProducing(self):
  149. pass
  150. def stopProducing(self):
  151. pass
  152. class _JsonProducer(object):
  153. """ Used by the twisted http client to create the HTTP body from json
  154. """
  155. def __init__(self, jsn):
  156. self.data = jsn
  157. self.body = json.dumps(jsn).encode("utf8")
  158. self.length = len(self.body)
  159. def startProducing(self, consumer):
  160. consumer.write(self.body)
  161. return defer.succeed(None)
  162. def pauseProducing(self):
  163. pass
  164. def stopProducing(self):
  165. pass