1
0

http.py 6.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217
  1. # -*- coding: utf-8 -*-
  2. # Copyright 2014 OpenMarket Ltd
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License");
  5. # you may not use this file except in compliance with the License.
  6. # You may obtain a copy of the License at
  7. #
  8. # http://www.apache.org/licenses/LICENSE-2.0
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS,
  12. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13. # See the License for the specific language governing permissions and
  14. # limitations under the License.
  15. from twisted.web.client import Agent, readBody
  16. from twisted.web.http_headers import Headers
  17. from twisted.internet import defer, reactor
  18. from pprint import pformat
  19. import json
  20. import urllib
  21. class HttpClient(object):
  22. """ Interface for talking json over http
  23. """
  24. def put_json(self, url, data):
  25. """ Sends the specifed json data using PUT
  26. Args:
  27. url (str): The URL to PUT data to.
  28. data (dict): A dict containing the data that will be used as
  29. the request body. This will be encoded as JSON.
  30. Returns:
  31. Deferred: Succeeds when we get *any* HTTP response.
  32. The result of the deferred is a tuple of `(code, response)`,
  33. where `response` is a dict representing the decoded JSON body.
  34. """
  35. pass
  36. def get_json(self, url, args=None):
  37. """ Get's some json from the given host homeserver and path
  38. Args:
  39. url (str): The URL to GET data from.
  40. args (dict): A dictionary used to create query strings, defaults to
  41. None.
  42. **Note**: The value of each key is assumed to be an iterable
  43. and *not* a string.
  44. Returns:
  45. Deferred: Succeeds when we get *any* HTTP response.
  46. The result of the deferred is a tuple of `(code, response)`,
  47. where `response` is a dict representing the decoded JSON body.
  48. """
  49. pass
  50. class TwistedHttpClient(HttpClient):
  51. """ Wrapper around the twisted HTTP client api.
  52. Attributes:
  53. agent (twisted.web.client.Agent): The twisted Agent used to send the
  54. requests.
  55. """
  56. def __init__(self):
  57. self.agent = Agent(reactor)
  58. @defer.inlineCallbacks
  59. def put_json(self, url, data):
  60. response = yield self._create_put_request(
  61. url,
  62. data,
  63. headers_dict={"Content-Type": ["application/json"]}
  64. )
  65. body = yield readBody(response)
  66. defer.returnValue((response.code, body))
  67. @defer.inlineCallbacks
  68. def get_json(self, url, args=None):
  69. if args:
  70. # generates a list of strings of form "k=v".
  71. qs = urllib.urlencode(args, True)
  72. url = "%s?%s" % (url, qs)
  73. response = yield self._create_get_request(url)
  74. body = yield readBody(response)
  75. defer.returnValue(json.loads(body))
  76. def _create_put_request(self, url, json_data, headers_dict={}):
  77. """ Wrapper of _create_request to issue a PUT request
  78. """
  79. if "Content-Type" not in headers_dict:
  80. raise defer.error(
  81. RuntimeError("Must include Content-Type header for PUTs"))
  82. return self._create_request(
  83. "PUT",
  84. url,
  85. producer=_JsonProducer(json_data),
  86. headers_dict=headers_dict
  87. )
  88. def _create_get_request(self, url, headers_dict={}):
  89. """ Wrapper of _create_request to issue a GET request
  90. """
  91. return self._create_request(
  92. "GET",
  93. url,
  94. headers_dict=headers_dict
  95. )
  96. @defer.inlineCallbacks
  97. def do_request(self, method, url, data=None, qparams=None, jsonreq=True, headers={}):
  98. if qparams:
  99. url = "%s?%s" % (url, urllib.urlencode(qparams, True))
  100. if jsonreq:
  101. prod = _JsonProducer(data)
  102. headers['Content-Type'] = ["application/json"];
  103. else:
  104. prod = _RawProducer(data)
  105. if method in ["POST", "PUT"]:
  106. response = yield self._create_request(method, url,
  107. producer=prod,
  108. headers_dict=headers)
  109. else:
  110. response = yield self._create_request(method, url)
  111. body = yield readBody(response)
  112. defer.returnValue(json.loads(body))
  113. @defer.inlineCallbacks
  114. def _create_request(self, method, url, producer=None, headers_dict={}):
  115. """ Creates and sends a request to the given url
  116. """
  117. headers_dict["User-Agent"] = ["Synapse Cmd Client"]
  118. retries_left = 5
  119. print "%s to %s with headers %s" % (method, url, headers_dict)
  120. if self.verbose and producer:
  121. if "password" in producer.data:
  122. temp = producer.data["password"]
  123. producer.data["password"] = "[REDACTED]"
  124. print json.dumps(producer.data, indent=4)
  125. producer.data["password"] = temp
  126. else:
  127. print json.dumps(producer.data, indent=4)
  128. while True:
  129. try:
  130. response = yield self.agent.request(
  131. method,
  132. url.encode("UTF8"),
  133. Headers(headers_dict),
  134. producer
  135. )
  136. break
  137. except Exception as e:
  138. print "uh oh: %s" % e
  139. if retries_left:
  140. yield self.sleep(2 ** (5 - retries_left))
  141. retries_left -= 1
  142. else:
  143. raise e
  144. if self.verbose:
  145. print "Status %s %s" % (response.code, response.phrase)
  146. print pformat(list(response.headers.getAllRawHeaders()))
  147. defer.returnValue(response)
  148. def sleep(self, seconds):
  149. d = defer.Deferred()
  150. reactor.callLater(seconds, d.callback, seconds)
  151. return d
  152. class _RawProducer(object):
  153. def __init__(self, data):
  154. self.data = data
  155. self.body = data
  156. self.length = len(self.body)
  157. def startProducing(self, consumer):
  158. consumer.write(self.body)
  159. return defer.succeed(None)
  160. def pauseProducing(self):
  161. pass
  162. def stopProducing(self):
  163. pass
  164. class _JsonProducer(object):
  165. """ Used by the twisted http client to create the HTTP body from json
  166. """
  167. def __init__(self, jsn):
  168. self.data = jsn
  169. self.body = json.dumps(jsn).encode("utf8")
  170. self.length = len(self.body)
  171. def startProducing(self, consumer):
  172. consumer.write(self.body)
  173. return defer.succeed(None)
  174. def pauseProducing(self):
  175. pass
  176. def stopProducing(self):
  177. pass