Well-Known support (#118)

Switches the FederationHTTPClient to one that uses MatrixFederationAgent from Synapse, that provides caching, a connection pool and well-known support (isn't twisted neat).

Closes #94

setup.py

@@ -47,6 +47,9 @@ setup(
 
         "phonenumbers",
         "pyopenssl",
+        "attrs>=19.1.0",
+        "netaddr>=0.7.0",
+        "sortedcontainers>=2.1.0",
     ],
     # make sure we package the sql files
     include_package_data=True,

sydent/hs_federation/verifier.py

@@ -65,7 +65,7 @@ class Verifier(object):
                 defer.returnValue(self.cache[server_name]['verify_keys'])
 
         client = FederationHttpClient(self.sydent)
-        result = yield client.get_json("https://%s/_matrix/key/v2/server/" % server_name)
+        result = yield client.get_json("matrix://%s/_matrix/key/v2/server/" % server_name)
         if 'verify_keys' not in result:
             raise SignatureVerifyException("No key found in response")
 
@@ -81,11 +81,11 @@ class Verifier(object):
     def verifyServerSignedJson(self, signed_json, acceptable_server_names=None):
         """Given a signed json object, try to verify any one
         of the signatures on it
+
         XXX: This contains a fairly noddy version of the home server
-        SRV lookup and signature verification. It only looks at
-        the first SRV result. It does no caching (just fetches the
-        signature each time and does not contact any other servers
-        to do perspectives checks.
+        SRV lookup and signature verification. It does no caching (just
+        fetches the signature each time and does not contact any other
+        servers to do perspective checks).
 
         :param acceptable_server_names: If provided and not None,
         only signatures from servers in this list will be accepted.
@@ -131,7 +131,7 @@ class Verifier(object):
 
         :param request: The request object to authenticate
         :param content: The content of the request, if any
-        :type content: bytes or None
+        :type content: bytes, None
 
         :returns: The origin of the server whose signature was validated
         """

sydent/http/httpclient.py

@@ -18,40 +18,28 @@ import json
 import logging
 
 from StringIO import StringIO
-from twisted.internet import defer, reactor, ssl
-from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS
-from twisted.internet._sslverify import _defaultCurveName, ClientTLSOptions
+from twisted.internet import defer, reactor
 from twisted.web.client import FileBodyProducer, Agent, readBody
 from twisted.web.http_headers import Headers
-import twisted.names.client
-from twisted.names.error import DNSNameError
-from OpenSSL import SSL, crypto
+from sydent.http.matrixfederationagent import MatrixFederationAgent
+
+from sydent.http.federation_tls_options import ClientTLSOptionsFactory
 
 logger = logging.getLogger(__name__)
 
-class SimpleHttpClient(object):
-    """
-    A simple, no-frills HTTP client based on the class of the same name
-    from synapse
+class HTTPClient(object):
+    """A base HTTP class that contains methods for making GET and POST HTTP
+    requests.
     """
-    def __init__(self, sydent, endpoint_factory=None):
-        self.sydent = sydent
-        if endpoint_factory is None:
-            # The default endpoint factory in Twisted 14.0.0 (which we require) uses the
-            # BrowserLikePolicyForHTTPS context factory which will do regular cert validation
-            # 'like a browser'
-            self.agent = Agent(
-                reactor,
-                connectTimeout=15,
-            )
-        else:
-            self.agent = Agent.usingEndpointFactory(
-                reactor,
-                endpoint_factory,
-            )
-
     @defer.inlineCallbacks
     def get_json(self, uri):
+        """Make a GET request to an endpoint returning JSON and parse result
+
+        :param uri: The URI to make a GET request to.
+        :type uri: str
+        :returns a deferred containing JSON parsed into a Python object.
+        :rtype: Deferred[any]
+        """
         logger.debug("HTTP GET %s", uri)
 
         response = yield self.agent.request(
@@ -59,10 +47,31 @@ class SimpleHttpClient(object):
             uri.encode("ascii"),
         )
         body = yield readBody(response)
-        defer.returnValue(json.loads(body))
+        try:
+            json_body = json.loads(body)
+        except Exception as e:
+            logger.exception("Error parsing JSON from %s", uri)
+            raise
+        defer.returnValue(json_body)
 
     @defer.inlineCallbacks
     def post_json_get_nothing(self, uri, post_json, opts):
+        """Make a GET request to an endpoint returning JSON and parse result
+
+        :param uri: The URI to make a GET request to.
+        :type uri: str
+
+        :param post_json: A Python object that will be converted to a JSON
+            string and POSTed to the given URI.
+        :type post_json: any
+
+        :opts: A dictionary of request options. Currently only opts.headers
+            is supported.
+        :type opts: Dict[str,any]
+
+        :returns a response from the remote server.
+        :rtype: Deferred[twisted.web.iweb.IResponse]
+        """
         json_str = json.dumps(post_json)
 
         headers = opts.get('headers', Headers({
@@ -79,115 +88,27 @@ class SimpleHttpClient(object):
         )
         defer.returnValue(response)
 
-class SRVClientEndpoint(object):
-    def __init__(self, reactor, service, domain, protocol="tcp",
-                 default_port=None, endpoint=HostnameEndpoint,
-                 endpoint_kw_args={}):
-        self.reactor = reactor
-        self.domain = domain
-
-        self.endpoint = endpoint
-        self.endpoint_kw_args = endpoint_kw_args
-
-    @defer.inlineCallbacks
-    def lookup_server(self):
-        service_name = "%s.%s.%s" % ('_matrix', '_tcp', self.domain)
-
-        default = self.domain, 8448
-
-        try:
-            answers, _, _ = yield twisted.names.client.lookupService(service_name)
-        except DNSNameError:
-            logger.info("DNSNameError doing SRV lookup for %s - using default", service_name)
-            defer.returnValue(default)
-
-        for answer in answers:
-            if answer.type != twisted.names.dns.SRV or not answer.payload:
-                continue
-
-            # XXX we just use the first
-            logger.info("Got SRV answer: %r / %d for %s", str(answer.payload.target), answer.payload.port, service_name)
-            defer.returnValue((str(answer.payload.target), answer.payload.port))
-
-        logger.info("No valid answers found in response from %s (%r)", self.domain, answers)
-        defer.returnValue(default)
-
-    @defer.inlineCallbacks
-    def connect(self, protocolFactory):
-        server = yield self.lookup_server()
-        logger.info("Connecting to %s:%s", server[0], server[1])
-        endpoint = self.endpoint(
-            self.reactor, server[0], server[1], **self.endpoint_kw_args
-        )
-        connection = yield endpoint.connect(protocolFactory)
-        defer.returnValue(connection)
-
-def matrix_federation_endpoint(reactor, destination, ssl_context_factory=None,
-                               timeout=None):
-    """Construct an endpoint for the given matrix destination.
-
-    :param reactor: Twisted reactor.
-    :param destination: The name of the server to connect to.
-    :type destination: bytes
-    :param ssl_context_factory: Factory which generates SSL contexts to use for TLS.
-    :type ssl_context_factory: twisted.internet.ssl.ContextFactory
-    :param timeout (int): connection timeout in seconds
-    :type timeout: int
+class SimpleHttpClient(HTTPClient):
+    """A simple, no-frills HTTP client based on the class of the same name
+    from Synapse.
     """
-
-    domain_port = destination.split(":")
-    domain = domain_port[0]
-    port = int(domain_port[1]) if domain_port[1:] else None
-
-    endpoint_kw_args = {}
-
-    if timeout is not None:
-        endpoint_kw_args.update(timeout=timeout)
-
-    if ssl_context_factory is None:
-        transport_endpoint = HostnameEndpoint
-        default_port = 8008
-    else:
-        def transport_endpoint(reactor, host, port, timeout):
-            return wrapClientTLS(
-                ssl_context_factory,
-                HostnameEndpoint(reactor, host, port, timeout=timeout))
-        default_port = 8448
-
-    if port is None:
-        return SRVClientEndpoint(
-            reactor, "matrix", domain, protocol="tcp",
-            default_port=default_port, endpoint=transport_endpoint,
-            endpoint_kw_args=endpoint_kw_args
-        )
-    else:
-        return transport_endpoint(
-            reactor, domain, port, **endpoint_kw_args
-        )
-
-class FederationEndpointFactory(object):
-    def endpointForURI(self, uri):
-        destination = uri.netloc
-        context_factory = FederationContextFactory()
-
-        return matrix_federation_endpoint(
-            reactor, destination, timeout=10,
-            ssl_context_factory=context_factory,
+    def __init__(self, sydent):
+        self.sydent = sydent
+        # The default endpoint factory in Twisted 14.0.0 (which we require) uses the
+        # BrowserLikePolicyForHTTPS context factory which will do regular cert validation
+        # 'like a browser'
+        self.agent = Agent(
+            reactor,
+            connectTimeout=15,
         )
 
-class FederationContextFactory(object):
-    def getContext(self):
-        context = SSL.Context(SSL.SSLv23_METHOD)
-        try:
-            _ecCurve = crypto.get_elliptic_curve(_defaultCurveName)
-            context.set_tmp_ecdh(_ecCurve)
-        except Exception:
-            logger.exception("Failed to enable elliptic curve for TLS")
-        context.set_options(SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3)
-
-        context.set_cipher_list("!ADH:HIGH+kEDH:!AECDH:HIGH+kEECDH")
-        return context
-
-class FederationHttpClient(SimpleHttpClient):
+class FederationHttpClient(HTTPClient):
+    """HTTP client for federation requests to homeservers. Uses a
+    MatrixFederationAgent.
+    """
     def __init__(self, sydent):
-        super(FederationHttpClient, self).__init__(sydent, FederationEndpointFactory())
+        self.sydent = sydent
+        self.agent = MatrixFederationAgent(
+            reactor,
+            ClientTLSOptionsFactory(),
+        )