1
0

http.py 6.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213
  1. # -*- coding: utf-8 -*-
  2. # Copyright 2014-2016 OpenMarket Ltd
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License");
  5. # you may not use this file except in compliance with the License.
  6. # You may obtain a copy of the License at
  7. #
  8. # http://www.apache.org/licenses/LICENSE-2.0
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS,
  12. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13. # See the License for the specific language governing permissions and
  14. # limitations under the License.
  15. from twisted.web.client import Agent, readBody
  16. from twisted.web.http_headers import Headers
  17. from twisted.internet import defer, reactor
  18. from pprint import pformat
  19. import json
  20. import urllib
  21. class HttpClient(object):
  22. """ Interface for talking json over http
  23. """
  24. def put_json(self, url, data):
  25. """ Sends the specifed json data using PUT
  26. Args:
  27. url (str): The URL to PUT data to.
  28. data (dict): A dict containing the data that will be used as
  29. the request body. This will be encoded as JSON.
  30. Returns:
  31. Deferred: Succeeds when we get a 2xx HTTP response. The result
  32. will be the decoded JSON body.
  33. """
  34. pass
  35. def get_json(self, url, args=None):
  36. """ Gets some json from the given host homeserver and path
  37. Args:
  38. url (str): The URL to GET data from.
  39. args (dict): A dictionary used to create query strings, defaults to
  40. None.
  41. **Note**: The value of each key is assumed to be an iterable
  42. and *not* a string.
  43. Returns:
  44. Deferred: Succeeds when we get a 2xx HTTP response. The result
  45. will be the decoded JSON body.
  46. """
  47. pass
  48. class TwistedHttpClient(HttpClient):
  49. """ Wrapper around the twisted HTTP client api.
  50. Attributes:
  51. agent (twisted.web.client.Agent): The twisted Agent used to send the
  52. requests.
  53. """
  54. def __init__(self):
  55. self.agent = Agent(reactor)
  56. @defer.inlineCallbacks
  57. def put_json(self, url, data):
  58. response = yield self._create_put_request(
  59. url,
  60. data,
  61. headers_dict={"Content-Type": ["application/json"]}
  62. )
  63. body = yield readBody(response)
  64. defer.returnValue((response.code, body))
  65. @defer.inlineCallbacks
  66. def get_json(self, url, args=None):
  67. if args:
  68. # generates a list of strings of form "k=v".
  69. qs = urllib.urlencode(args, True)
  70. url = "%s?%s" % (url, qs)
  71. response = yield self._create_get_request(url)
  72. body = yield readBody(response)
  73. defer.returnValue(json.loads(body))
  74. def _create_put_request(self, url, json_data, headers_dict={}):
  75. """ Wrapper of _create_request to issue a PUT request
  76. """
  77. if "Content-Type" not in headers_dict:
  78. raise defer.error(
  79. RuntimeError("Must include Content-Type header for PUTs"))
  80. return self._create_request(
  81. "PUT",
  82. url,
  83. producer=_JsonProducer(json_data),
  84. headers_dict=headers_dict
  85. )
  86. def _create_get_request(self, url, headers_dict={}):
  87. """ Wrapper of _create_request to issue a GET request
  88. """
  89. return self._create_request(
  90. "GET",
  91. url,
  92. headers_dict=headers_dict
  93. )
  94. @defer.inlineCallbacks
  95. def do_request(self, method, url, data=None, qparams=None, jsonreq=True, headers={}):
  96. if qparams:
  97. url = "%s?%s" % (url, urllib.urlencode(qparams, True))
  98. if jsonreq:
  99. prod = _JsonProducer(data)
  100. headers['Content-Type'] = ["application/json"];
  101. else:
  102. prod = _RawProducer(data)
  103. if method in ["POST", "PUT"]:
  104. response = yield self._create_request(method, url,
  105. producer=prod,
  106. headers_dict=headers)
  107. else:
  108. response = yield self._create_request(method, url)
  109. body = yield readBody(response)
  110. defer.returnValue(json.loads(body))
  111. @defer.inlineCallbacks
  112. def _create_request(self, method, url, producer=None, headers_dict={}):
  113. """ Creates and sends a request to the given url
  114. """
  115. headers_dict["User-Agent"] = ["Synapse Cmd Client"]
  116. retries_left = 5
  117. print "%s to %s with headers %s" % (method, url, headers_dict)
  118. if self.verbose and producer:
  119. if "password" in producer.data:
  120. temp = producer.data["password"]
  121. producer.data["password"] = "[REDACTED]"
  122. print json.dumps(producer.data, indent=4)
  123. producer.data["password"] = temp
  124. else:
  125. print json.dumps(producer.data, indent=4)
  126. while True:
  127. try:
  128. response = yield self.agent.request(
  129. method,
  130. url.encode("UTF8"),
  131. Headers(headers_dict),
  132. producer
  133. )
  134. break
  135. except Exception as e:
  136. print "uh oh: %s" % e
  137. if retries_left:
  138. yield self.sleep(2 ** (5 - retries_left))
  139. retries_left -= 1
  140. else:
  141. raise e
  142. if self.verbose:
  143. print "Status %s %s" % (response.code, response.phrase)
  144. print pformat(list(response.headers.getAllRawHeaders()))
  145. defer.returnValue(response)
  146. def sleep(self, seconds):
  147. d = defer.Deferred()
  148. reactor.callLater(seconds, d.callback, seconds)
  149. return d
  150. class _RawProducer(object):
  151. def __init__(self, data):
  152. self.data = data
  153. self.body = data
  154. self.length = len(self.body)
  155. def startProducing(self, consumer):
  156. consumer.write(self.body)
  157. return defer.succeed(None)
  158. def pauseProducing(self):
  159. pass
  160. def stopProducing(self):
  161. pass
  162. class _JsonProducer(object):
  163. """ Used by the twisted http client to create the HTTP body from json
  164. """
  165. def __init__(self, jsn):
  166. self.data = jsn
  167. self.body = json.dumps(jsn).encode("utf8")
  168. self.length = len(self.body)
  169. def startProducing(self, consumer):
  170. consumer.write(self.body)
  171. return defer.succeed(None)
  172. def pauseProducing(self):
  173. pass
  174. def stopProducing(self):
  175. pass