123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242 |
- # -*- coding: utf-8 -*-
- # Copyright 2019 New Vector Ltd
- #
- # Licensed under the Apache License, Version 2.0 (the "License");
- # you may not use this file except in compliance with the License.
- # You may obtain a copy of the License at
- #
- # http://www.apache.org/licenses/LICENSE-2.0
- #
- # Unless required by applicable law or agreed to in writing, software
- # distributed under the License is distributed on an "AS IS" BASIS,
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- # See the License for the specific language governing permissions and
- # limitations under the License.
- import logging
- from mock import Mock
- import treq
- from service_identity import VerificationError
- from zope.interface import implementer
- from twisted.internet import defer
- from twisted.internet._sslverify import ClientTLSOptions, OpenSSLCertificateOptions
- from twisted.internet.protocol import Factory
- from twisted.protocols.tls import TLSMemoryBIOFactory
- from twisted.web._newclient import ResponseNeverReceived
- from twisted.web.client import Agent
- from twisted.web.http import HTTPChannel
- from twisted.web.http_headers import Headers
- from twisted.web.iweb import IPolicyForHTTPS
- from synapse.config.homeserver import HomeServerConfig
- from synapse.crypto.context_factory import ClientTLSOptionsFactory
- from synapse.http.federation.matrix_federation_agent import MatrixFederationAgent
- from synapse.http.federation.srv_resolver import Server
- from synapse.http.federation.well_known_resolver import (
- WellKnownResolver,
- _cache_period_from_headers,
- )
- from synapse.logging.context import LoggingContext
- from synapse.util.caches.ttlcache import TTLCache
- from tests import unittest
- from tests.http import TestServerTLSConnectionFactory, get_test_ca_cert_file
- from tests.server import FakeTransport, ThreadedMemoryReactorClock
- from tests.utils import default_config
- logger = logging.getLogger(__name__)
- test_server_connection_factory = None
- def get_connection_factory():
- # this needs to happen once, but not until we are ready to run the first test
- global test_server_connection_factory
- if test_server_connection_factory is None:
- test_server_connection_factory = TestServerTLSConnectionFactory(
- sanlist=[
- b"DNS:testserv",
- b"DNS:target-server",
- b"DNS:xn--bcher-kva.com",
- b"IP:1.2.3.4",
- b"IP:::1",
- ]
- )
- return test_server_connection_factory
- class MatrixFederationAgentTests(unittest.TestCase):
- def setUp(self):
- self.reactor = ThreadedMemoryReactorClock()
- self.mock_resolver = Mock()
- config_dict = default_config("test", parse=False)
- config_dict["federation_custom_ca_list"] = [get_test_ca_cert_file()]
- self._config = config = HomeServerConfig()
- config.parse_config_dict(config_dict, "", "")
- self.tls_factory = ClientTLSOptionsFactory(config)
- self.well_known_cache = TTLCache("test_cache", timer=self.reactor.seconds)
- self.had_well_known_cache = TTLCache("test_cache", timer=self.reactor.seconds)
- self.well_known_resolver = WellKnownResolver(
- self.reactor,
- Agent(self.reactor, contextFactory=self.tls_factory),
- well_known_cache=self.well_known_cache,
- had_well_known_cache=self.had_well_known_cache,
- )
- self.agent = MatrixFederationAgent(
- reactor=self.reactor,
- tls_client_options_factory=self.tls_factory,
- _srv_resolver=self.mock_resolver,
- _well_known_resolver=self.well_known_resolver,
- )
- def _make_connection(self, client_factory, expected_sni):
- """Builds a test server, and completes the outgoing client connection
- Returns:
- HTTPChannel: the test server
- """
- # build the test server
- server_tls_protocol = _build_test_server(get_connection_factory())
- # now, tell the client protocol factory to build the client protocol (it will be a
- # _WrappingProtocol, around a TLSMemoryBIOProtocol, around an
- # HTTP11ClientProtocol) and wire the output of said protocol up to the server via
- # a FakeTransport.
- #
- # Normally this would be done by the TCP socket code in Twisted, but we are
- # stubbing that out here.
- client_protocol = client_factory.buildProtocol(None)
- client_protocol.makeConnection(
- FakeTransport(server_tls_protocol, self.reactor, client_protocol)
- )
- # tell the server tls protocol to send its stuff back to the client, too
- server_tls_protocol.makeConnection(
- FakeTransport(client_protocol, self.reactor, server_tls_protocol)
- )
- # grab a hold of the TLS connection, in case it gets torn down
- server_tls_connection = server_tls_protocol._tlsConnection
- # fish the test server back out of the server-side TLS protocol.
- http_protocol = server_tls_protocol.wrappedProtocol
- # give the reactor a pump to get the TLS juices flowing.
- self.reactor.pump((0.1,))
- # check the SNI
- server_name = server_tls_connection.get_servername()
- self.assertEqual(
- server_name,
- expected_sni,
- "Expected SNI %s but got %s" % (expected_sni, server_name),
- )
- return http_protocol
- @defer.inlineCallbacks
- def _make_get_request(self, uri):
- """
- Sends a simple GET request via the agent, and checks its logcontext management
- """
- with LoggingContext("one") as context:
- fetch_d = self.agent.request(b"GET", uri)
- # Nothing happened yet
- self.assertNoResult(fetch_d)
- # should have reset logcontext to the sentinel
- _check_logcontext(LoggingContext.sentinel)
- try:
- fetch_res = yield fetch_d
- return fetch_res
- except Exception as e:
- logger.info("Fetch of %s failed: %s", uri.decode("ascii"), e)
- raise
- finally:
- _check_logcontext(context)
- def _handle_well_known_connection(
- self, client_factory, expected_sni, content, response_headers={}
- ):
- """Handle an outgoing HTTPs connection: wire it up to a server, check that the
- request is for a .well-known, and send the response.
- Args:
- client_factory (IProtocolFactory): outgoing connection
- expected_sni (bytes): SNI that we expect the outgoing connection to send
- content (bytes): content to send back as the .well-known
- Returns:
- HTTPChannel: server impl
- """
- # make the connection for .well-known
- well_known_server = self._make_connection(
- client_factory, expected_sni=expected_sni
- )
- # check the .well-known request and send a response
- self.assertEqual(len(well_known_server.requests), 1)
- request = well_known_server.requests[0]
- self._send_well_known_response(request, content, headers=response_headers)
- return well_known_server
- def _send_well_known_response(self, request, content, headers={}):
- """Check that an incoming request looks like a valid .well-known request, and
- send back the response.
- """
- self.assertEqual(request.method, b"GET")
- self.assertEqual(request.path, b"/.well-known/matrix/server")
- self.assertEqual(request.requestHeaders.getRawHeaders(b"host"), [b"testserv"])
- # send back a response
- for k, v in headers.items():
- request.setHeader(k, v)
- request.write(content)
- request.finish()
- self.reactor.pump((0.1,))
- def test_get(self):
- """
- happy-path test of a GET request with an explicit port
- """
- self.reactor.lookups["testserv"] = "1.2.3.4"
- test_d = self._make_get_request(b"matrix://testserv:8448/foo/bar")
- # Nothing happened yet
- self.assertNoResult(test_d)
- # Make sure treq is trying to connect
- clients = self.reactor.tcpClients
- self.assertEqual(len(clients), 1)
- (host, port, client_factory, _timeout, _bindAddress) = clients[0]
- self.assertEqual(host, "1.2.3.4")
- self.assertEqual(port, 8448)
- # make a test server, and wire up the client
- http_server = self._make_connection(client_factory, expected_sni=b"testserv")
- self.assertEqual(len(http_server.requests), 1)
- request = http_server.requests[0]
- self.assertEqual(request.method, b"GET")
- self.assertEqual(request.path, b"/foo/bar")
- self.assertEqual(
- request.requestHeaders.getRawHeaders(b"host"), [b"testserv:8448"]
- )
- content = request.content.read()
- self.assertEqual(content, b"")
- # Deferred is still without a result
- self.assertNoResult(test_d)
- # send the headers
- request.responseHeaders.setRawHeaders(b"Content-Type", [b"application/json"])
- request.write("")
- self.reactor.pump((0.1,))
- response = self.successResultOf(test_d)
- # that should give us a Response object
- self.assertEqual(response.code, 200)
- # Send the body
- request.write('{ "a": 1 }'.encode("ascii"))
- request.finish()
- self.reactor.pump((0.1,))
- # check it can be read
- json = self.successResultOf(treq.json_content(response))
- self.assertEqual(json, {"a": 1})
- def test_get_ip_address(self):
- """
- Test the behaviour when the server name contains an explicit IP (with no port)
- """
- # there will be a getaddrinfo on the IP
- self.reactor.lookups["1.2.3.4"] = "1.2.3.4"
- test_d = self._make_get_request(b"matrix://1.2.3.4/foo/bar")
- # Nothing happened yet
- self.assertNoResult(test_d)
- # Make sure treq is trying to connect
- clients = self.reactor.tcpClients
- self.assertEqual(len(clients), 1)
- (host, port, client_factory, _timeout, _bindAddress) = clients[0]
- self.assertEqual(host, "1.2.3.4")
- self.assertEqual(port, 8448)
- # make a test server, and wire up the client
- http_server = self._make_connection(client_factory, expected_sni=None)
- self.assertEqual(len(http_server.requests), 1)
- request = http_server.requests[0]
- self.assertEqual(request.method, b"GET")
- self.assertEqual(request.path, b"/foo/bar")
- self.assertEqual(request.requestHeaders.getRawHeaders(b"host"), [b"1.2.3.4"])
- # finish the request
- request.finish()
- self.reactor.pump((0.1,))
- self.successResultOf(test_d)
- def test_get_ipv6_address(self):
- """
- Test the behaviour when the server name contains an explicit IPv6 address
- (with no port)
- """
- # there will be a getaddrinfo on the IP
- self.reactor.lookups["::1"] = "::1"
- test_d = self._make_get_request(b"matrix://[::1]/foo/bar")
- # Nothing happened yet
- self.assertNoResult(test_d)
- # Make sure treq is trying to connect
- clients = self.reactor.tcpClients
- self.assertEqual(len(clients), 1)
- (host, port, client_factory, _timeout, _bindAddress) = clients[0]
- self.assertEqual(host, "::1")
- self.assertEqual(port, 8448)
- # make a test server, and wire up the client
- http_server = self._make_connection(client_factory, expected_sni=None)
- self.assertEqual(len(http_server.requests), 1)
- request = http_server.requests[0]
- self.assertEqual(request.method, b"GET")
- self.assertEqual(request.path, b"/foo/bar")
- self.assertEqual(request.requestHeaders.getRawHeaders(b"host"), [b"[::1]"])
- # finish the request
- request.finish()
- self.reactor.pump((0.1,))
- self.successResultOf(test_d)
- def test_get_ipv6_address_with_port(self):
- """
- Test the behaviour when the server name contains an explicit IPv6 address
- (with explicit port)
- """
- # there will be a getaddrinfo on the IP
- self.reactor.lookups["::1"] = "::1"
- test_d = self._make_get_request(b"matrix://[::1]:80/foo/bar")
- # Nothing happened yet
- self.assertNoResult(test_d)
- # Make sure treq is trying to connect
- clients = self.reactor.tcpClients
- self.assertEqual(len(clients), 1)
- (host, port, client_factory, _timeout, _bindAddress) = clients[0]
- self.assertEqual(host, "::1")
- self.assertEqual(port, 80)
- # make a test server, and wire up the client
- http_server = self._make_connection(client_factory, expected_sni=None)
- self.assertEqual(len(http_server.requests), 1)
- request = http_server.requests[0]
- self.assertEqual(request.method, b"GET")
- self.assertEqual(request.path, b"/foo/bar")
- self.assertEqual(request.requestHeaders.getRawHeaders(b"host"), [b"[::1]:80"])
- # finish the request
- request.finish()
- self.reactor.pump((0.1,))
- self.successResultOf(test_d)
- def test_get_hostname_bad_cert(self):
- """
- Test the behaviour when the certificate on the server doesn't match the hostname
- """
- self.mock_resolver.resolve_service.side_effect = lambda _: []
- self.reactor.lookups["testserv1"] = "1.2.3.4"
- test_d = self._make_get_request(b"matrix://testserv1/foo/bar")
- # Nothing happened yet
- self.assertNoResult(test_d)
- # No SRV record lookup yet
- self.mock_resolver.resolve_service.assert_not_called()
- # there should be an attempt to connect on port 443 for the .well-known
- clients = self.reactor.tcpClients
- self.assertEqual(len(clients), 1)
- (host, port, client_factory, _timeout, _bindAddress) = clients[0]
- self.assertEqual(host, "1.2.3.4")
- self.assertEqual(port, 443)
- # fonx the connection
- client_factory.clientConnectionFailed(None, Exception("nope"))
- # attemptdelay on the hostnameendpoint is 0.3, so takes that long before the
- # .well-known request fails.
- self.reactor.pump((0.4,))
- # now there should be a SRV lookup
- self.mock_resolver.resolve_service.assert_called_once_with(
- b"_matrix._tcp.testserv1"
- )
- # we should fall back to a direct connection
- self.assertEqual(len(clients), 2)
- (host, port, client_factory, _timeout, _bindAddress) = clients[1]
- self.assertEqual(host, "1.2.3.4")
- self.assertEqual(port, 8448)
- # make a test server, and wire up the client
- http_server = self._make_connection(client_factory, expected_sni=b"testserv1")
- # there should be no requests
- self.assertEqual(len(http_server.requests), 0)
- # ... and the request should have failed
- e = self.failureResultOf(test_d, ResponseNeverReceived)
- failure_reason = e.value.reasons[0]
- self.assertIsInstance(failure_reason.value, VerificationError)
- def test_get_ip_address_bad_cert(self):
- """
- Test the behaviour when the server name contains an explicit IP, but
- the server cert doesn't cover it
- """
- # there will be a getaddrinfo on the IP
- self.reactor.lookups["1.2.3.5"] = "1.2.3.5"
- test_d = self._make_get_request(b"matrix://1.2.3.5/foo/bar")
- # Nothing happened yet
- self.assertNoResult(test_d)
- # Make sure treq is trying to connect
- clients = self.reactor.tcpClients
- self.assertEqual(len(clients), 1)
- (host, port, client_factory, _timeout, _bindAddress) = clients[0]
- self.assertEqual(host, "1.2.3.5")
- self.assertEqual(port, 8448)
- # make a test server, and wire up the client
- http_server = self._make_connection(client_factory, expected_sni=None)
- # there should be no requests
- self.assertEqual(len(http_server.requests), 0)
- # ... and the request should have failed
- e = self.failureResultOf(test_d, ResponseNeverReceived)
- failure_reason = e.value.reasons[0]
- self.assertIsInstance(failure_reason.value, VerificationError)
- def test_get_no_srv_no_well_known(self):
- """
- Test the behaviour when the server name has no port, no SRV, and no well-known
- """
- self.mock_resolver.resolve_service.side_effect = lambda _: []
- self.reactor.lookups["testserv"] = "1.2.3.4"
- test_d = self._make_get_request(b"matrix://testserv/foo/bar")
- # Nothing happened yet
- self.assertNoResult(test_d)
- # No SRV record lookup yet
- self.mock_resolver.resolve_service.assert_not_called()
- # there should be an attempt to connect on port 443 for the .well-known
- clients = self.reactor.tcpClients
- self.assertEqual(len(clients), 1)
- (host, port, client_factory, _timeout, _bindAddress) = clients[0]
- self.assertEqual(host, "1.2.3.4")
- self.assertEqual(port, 443)
- # fonx the connection
- client_factory.clientConnectionFailed(None, Exception("nope"))
- # attemptdelay on the hostnameendpoint is 0.3, so takes that long before the
- # .well-known request fails.
- self.reactor.pump((0.4,))
- # now there should be a SRV lookup
- self.mock_resolver.resolve_service.assert_called_once_with(
- b"_matrix._tcp.testserv"
- )
- # we should fall back to a direct connection
- self.assertEqual(len(clients), 2)
- (host, port, client_factory, _timeout, _bindAddress) = clients[1]
- self.assertEqual(host, "1.2.3.4")
- self.assertEqual(port, 8448)
- # make a test server, and wire up the client
- http_server = self._make_connection(client_factory, expected_sni=b"testserv")
- self.assertEqual(len(http_server.requests), 1)
- request = http_server.requests[0]
- self.assertEqual(request.method, b"GET")
- self.assertEqual(request.path, b"/foo/bar")
- self.assertEqual(request.requestHeaders.getRawHeaders(b"host"), [b"testserv"])
- # finish the request
- request.finish()
- self.reactor.pump((0.1,))
- self.successResultOf(test_d)
- def test_get_well_known(self):
- """Test the behaviour when the .well-known delegates elsewhere
- """
- self.mock_resolver.resolve_service.side_effect = lambda _: []
- self.reactor.lookups["testserv"] = "1.2.3.4"
- self.reactor.lookups["target-server"] = "1::f"
- test_d = self._make_get_request(b"matrix://testserv/foo/bar")
- # Nothing happened yet
- self.assertNoResult(test_d)
- # there should be an attempt to connect on port 443 for the .well-known
- clients = self.reactor.tcpClients
- self.assertEqual(len(clients), 1)
- (host, port, client_factory, _timeout, _bindAddress) = clients[0]
- self.assertEqual(host, "1.2.3.4")
- self.assertEqual(port, 443)
- self._handle_well_known_connection(
- client_factory,
- expected_sni=b"testserv",
- content=b'{ "m.server": "target-server" }',
- )
- # there should be a SRV lookup
- self.mock_resolver.resolve_service.assert_called_once_with(
- b"_matrix._tcp.target-server"
- )
- # now we should get a connection to the target server
- self.assertEqual(len(clients), 2)
- (host, port, client_factory, _timeout, _bindAddress) = clients[1]
- self.assertEqual(host, "1::f")
- self.assertEqual(port, 8448)
- # make a test server, and wire up the client
- http_server = self._make_connection(
- client_factory, expected_sni=b"target-server"
- )
- self.assertEqual(len(http_server.requests), 1)
- request = http_server.requests[0]
- self.assertEqual(request.method, b"GET")
- self.assertEqual(request.path, b"/foo/bar")
- self.assertEqual(
- request.requestHeaders.getRawHeaders(b"host"), [b"target-server"]
- )
- # finish the request
- request.finish()
- self.reactor.pump((0.1,))
- self.successResultOf(test_d)
- self.assertEqual(self.well_known_cache[b"testserv"], b"target-server")
- # check the cache expires
- self.reactor.pump((48 * 3600,))
- self.well_known_cache.expire()
- self.assertNotIn(b"testserv", self.well_known_cache)
- def test_get_well_known_redirect(self):
- """Test the behaviour when the server name has no port and no SRV record, but
- the .well-known has a 300 redirect
- """
- self.mock_resolver.resolve_service.side_effect = lambda _: []
- self.reactor.lookups["testserv"] = "1.2.3.4"
- self.reactor.lookups["target-server"] = "1::f"
- test_d = self._make_get_request(b"matrix://testserv/foo/bar")
- # Nothing happened yet
- self.assertNoResult(test_d)
- # there should be an attempt to connect on port 443 for the .well-known
- clients = self.reactor.tcpClients
- self.assertEqual(len(clients), 1)
- (host, port, client_factory, _timeout, _bindAddress) = clients.pop()
- self.assertEqual(host, "1.2.3.4")
- self.assertEqual(port, 443)
- redirect_server = self._make_connection(
- client_factory, expected_sni=b"testserv"
- )
- # send a 302 redirect
- self.assertEqual(len(redirect_server.requests), 1)
- request = redirect_server.requests[0]
- request.redirect(b"https://testserv/even_better_known")
- request.finish()
- self.reactor.pump((0.1,))
- # now there should be another connection
- clients = self.reactor.tcpClients
- self.assertEqual(len(clients), 1)
- (host, port, client_factory, _timeout, _bindAddress) = clients.pop()
- self.assertEqual(host, "1.2.3.4")
- self.assertEqual(port, 443)
- well_known_server = self._make_connection(
- client_factory, expected_sni=b"testserv"
- )
- self.assertEqual(len(well_known_server.requests), 1, "No request after 302")
- request = well_known_server.requests[0]
- self.assertEqual(request.method, b"GET")
- self.assertEqual(request.path, b"/even_better_known")
- request.write(b'{ "m.server": "target-server" }')
- request.finish()
- self.reactor.pump((0.1,))
- # there should be a SRV lookup
- self.mock_resolver.resolve_service.assert_called_once_with(
- b"_matrix._tcp.target-server"
- )
- # now we should get a connection to the target server
- self.assertEqual(len(clients), 1)
- (host, port, client_factory, _timeout, _bindAddress) = clients[0]
- self.assertEqual(host, "1::f")
- self.assertEqual(port, 8448)
- # make a test server, and wire up the client
- http_server = self._make_connection(
- client_factory, expected_sni=b"target-server"
- )
- self.assertEqual(len(http_server.requests), 1)
- request = http_server.requests[0]
- self.assertEqual(request.method, b"GET")
- self.assertEqual(request.path, b"/foo/bar")
- self.assertEqual(
- request.requestHeaders.getRawHeaders(b"host"), [b"target-server"]
- )
- # finish the request
- request.finish()
- self.reactor.pump((0.1,))
- self.successResultOf(test_d)
- self.assertEqual(self.well_known_cache[b"testserv"], b"target-server")
- # check the cache expires
- self.reactor.pump((48 * 3600,))
- self.well_known_cache.expire()
- self.assertNotIn(b"testserv", self.well_known_cache)
- def test_get_invalid_well_known(self):
- """
- Test the behaviour when the server name has an *invalid* well-known (and no SRV)
- """
- self.mock_resolver.resolve_service.side_effect = lambda _: []
- self.reactor.lookups["testserv"] = "1.2.3.4"
- test_d = self._make_get_request(b"matrix://testserv/foo/bar")
- # Nothing happened yet
- self.assertNoResult(test_d)
- # No SRV record lookup yet
- self.mock_resolver.resolve_service.assert_not_called()
- # there should be an attempt to connect on port 443 for the .well-known
- clients = self.reactor.tcpClients
- self.assertEqual(len(clients), 1)
- (host, port, client_factory, _timeout, _bindAddress) = clients.pop()
- self.assertEqual(host, "1.2.3.4")
- self.assertEqual(port, 443)
- self._handle_well_known_connection(
- client_factory, expected_sni=b"testserv", content=b"NOT JSON"
- )
- # now there should be a SRV lookup
- self.mock_resolver.resolve_service.assert_called_once_with(
- b"_matrix._tcp.testserv"
- )
- # we should fall back to a direct connection
- self.assertEqual(len(clients), 1)
- (host, port, client_factory, _timeout, _bindAddress) = clients.pop()
- self.assertEqual(host, "1.2.3.4")
- self.assertEqual(port, 8448)
- # make a test server, and wire up the client
- http_server = self._make_connection(client_factory, expected_sni=b"testserv")
- self.assertEqual(len(http_server.requests), 1)
- request = http_server.requests[0]
- self.assertEqual(request.method, b"GET")
- self.assertEqual(request.path, b"/foo/bar")
- self.assertEqual(request.requestHeaders.getRawHeaders(b"host"), [b"testserv"])
- # finish the request
- request.finish()
- self.reactor.pump((0.1,))
- self.successResultOf(test_d)
- def test_get_well_known_unsigned_cert(self):
- """Test the behaviour when the .well-known server presents a cert
- not signed by a CA
- """
- # we use the same test server as the other tests, but use an agent with
- # the config left to the default, which will not trust it (since the
- # presented cert is signed by a test CA)
- self.mock_resolver.resolve_service.side_effect = lambda _: []
- self.reactor.lookups["testserv"] = "1.2.3.4"
- config = default_config("test", parse=True)
- # Build a new agent and WellKnownResolver with a different tls factory
- tls_factory = ClientTLSOptionsFactory(config)
- agent = MatrixFederationAgent(
- reactor=self.reactor,
- tls_client_options_factory=tls_factory,
- _srv_resolver=self.mock_resolver,
- _well_known_resolver=WellKnownResolver(
- self.reactor,
- Agent(self.reactor, contextFactory=tls_factory),
- well_known_cache=self.well_known_cache,
- had_well_known_cache=self.had_well_known_cache,
- ),
- )
- test_d = agent.request(b"GET", b"matrix://testserv/foo/bar")
- # Nothing happened yet
- self.assertNoResult(test_d)
- # there should be an attempt to connect on port 443 for the .well-known
- clients = self.reactor.tcpClients
- self.assertEqual(len(clients), 1)
- (host, port, client_factory, _timeout, _bindAddress) = clients[0]
- self.assertEqual(host, "1.2.3.4")
- self.assertEqual(port, 443)
- http_proto = self._make_connection(client_factory, expected_sni=b"testserv")
- # there should be no requests
- self.assertEqual(len(http_proto.requests), 0)
- # and there should be a SRV lookup instead
- self.mock_resolver.resolve_service.assert_called_once_with(
- b"_matrix._tcp.testserv"
- )
- def test_get_hostname_srv(self):
- """
- Test the behaviour when there is a single SRV record
- """
- self.mock_resolver.resolve_service.side_effect = lambda _: [
- Server(host=b"srvtarget", port=8443)
- ]
- self.reactor.lookups["srvtarget"] = "1.2.3.4"
- test_d = self._make_get_request(b"matrix://testserv/foo/bar")
- # Nothing happened yet
- self.assertNoResult(test_d)
- # the request for a .well-known will have failed with a DNS lookup error.
- self.mock_resolver.resolve_service.assert_called_once_with(
- b"_matrix._tcp.testserv"
- )
- # Make sure treq is trying to connect
- clients = self.reactor.tcpClients
- self.assertEqual(len(clients), 1)
- (host, port, client_factory, _timeout, _bindAddress) = clients[0]
- self.assertEqual(host, "1.2.3.4")
- self.assertEqual(port, 8443)
- # make a test server, and wire up the client
- http_server = self._make_connection(client_factory, expected_sni=b"testserv")
- self.assertEqual(len(http_server.requests), 1)
- request = http_server.requests[0]
- self.assertEqual(request.method, b"GET")
- self.assertEqual(request.path, b"/foo/bar")
- self.assertEqual(request.requestHeaders.getRawHeaders(b"host"), [b"testserv"])
- # finish the request
- request.finish()
- self.reactor.pump((0.1,))
- self.successResultOf(test_d)
- def test_get_well_known_srv(self):
- """Test the behaviour when the .well-known redirects to a place where there
- is a SRV.
- """
- self.reactor.lookups["testserv"] = "1.2.3.4"
- self.reactor.lookups["srvtarget"] = "5.6.7.8"
- test_d = self._make_get_request(b"matrix://testserv/foo/bar")
- # Nothing happened yet
- self.assertNoResult(test_d)
- # there should be an attempt to connect on port 443 for the .well-known
- clients = self.reactor.tcpClients
- self.assertEqual(len(clients), 1)
- (host, port, client_factory, _timeout, _bindAddress) = clients[0]
- self.assertEqual(host, "1.2.3.4")
- self.assertEqual(port, 443)
- self.mock_resolver.resolve_service.side_effect = lambda _: [
- Server(host=b"srvtarget", port=8443)
- ]
- self._handle_well_known_connection(
- client_factory,
- expected_sni=b"testserv",
- content=b'{ "m.server": "target-server" }',
- )
- # there should be a SRV lookup
- self.mock_resolver.resolve_service.assert_called_once_with(
- b"_matrix._tcp.target-server"
- )
- # now we should get a connection to the target of the SRV record
- self.assertEqual(len(clients), 2)
- (host, port, client_factory, _timeout, _bindAddress) = clients[1]
- self.assertEqual(host, "5.6.7.8")
- self.assertEqual(port, 8443)
- # make a test server, and wire up the client
- http_server = self._make_connection(
- client_factory, expected_sni=b"target-server"
- )
- self.assertEqual(len(http_server.requests), 1)
- request = http_server.requests[0]
- self.assertEqual(request.method, b"GET")
- self.assertEqual(request.path, b"/foo/bar")
- self.assertEqual(
- request.requestHeaders.getRawHeaders(b"host"), [b"target-server"]
- )
- # finish the request
- request.finish()
- self.reactor.pump((0.1,))
- self.successResultOf(test_d)
- def test_idna_servername(self):
- """test the behaviour when the server name has idna chars in"""
- self.mock_resolver.resolve_service.side_effect = lambda _: []
- # the resolver is always called with the IDNA hostname as a native string.
- self.reactor.lookups["xn--bcher-kva.com"] = "1.2.3.4"
- # this is idna for bücher.com
- test_d = self._make_get_request(b"matrix://xn--bcher-kva.com/foo/bar")
- # Nothing happened yet
- self.assertNoResult(test_d)
- # No SRV record lookup yet
- self.mock_resolver.resolve_service.assert_not_called()
- # there should be an attempt to connect on port 443 for the .well-known
- clients = self.reactor.tcpClients
- self.assertEqual(len(clients), 1)
- (host, port, client_factory, _timeout, _bindAddress) = clients[0]
- self.assertEqual(host, "1.2.3.4")
- self.assertEqual(port, 443)
- # fonx the connection
- client_factory.clientConnectionFailed(None, Exception("nope"))
- # attemptdelay on the hostnameendpoint is 0.3, so takes that long before the
- # .well-known request fails.
- self.reactor.pump((0.4,))
- # now there should have been a SRV lookup
- self.mock_resolver.resolve_service.assert_called_once_with(
- b"_matrix._tcp.xn--bcher-kva.com"
- )
- # We should fall back to port 8448
- clients = self.reactor.tcpClients
- self.assertEqual(len(clients), 2)
- (host, port, client_factory, _timeout, _bindAddress) = clients[1]
- self.assertEqual(host, "1.2.3.4")
- self.assertEqual(port, 8448)
- # make a test server, and wire up the client
- http_server = self._make_connection(
- client_factory, expected_sni=b"xn--bcher-kva.com"
- )
- self.assertEqual(len(http_server.requests), 1)
- request = http_server.requests[0]
- self.assertEqual(request.method, b"GET")
- self.assertEqual(request.path, b"/foo/bar")
- self.assertEqual(
- request.requestHeaders.getRawHeaders(b"host"), [b"xn--bcher-kva.com"]
- )
- # finish the request
- request.finish()
- self.reactor.pump((0.1,))
- self.successResultOf(test_d)
- def test_idna_srv_target(self):
- """test the behaviour when the target of a SRV record has idna chars"""
- self.mock_resolver.resolve_service.side_effect = lambda _: [
- Server(host=b"xn--trget-3qa.com", port=8443) # târget.com
- ]
- self.reactor.lookups["xn--trget-3qa.com"] = "1.2.3.4"
- test_d = self._make_get_request(b"matrix://xn--bcher-kva.com/foo/bar")
- # Nothing happened yet
- self.assertNoResult(test_d)
- self.mock_resolver.resolve_service.assert_called_once_with(
- b"_matrix._tcp.xn--bcher-kva.com"
- )
- # Make sure treq is trying to connect
- clients = self.reactor.tcpClients
- self.assertEqual(len(clients), 1)
- (host, port, client_factory, _timeout, _bindAddress) = clients[0]
- self.assertEqual(host, "1.2.3.4")
- self.assertEqual(port, 8443)
- # make a test server, and wire up the client
- http_server = self._make_connection(
- client_factory, expected_sni=b"xn--bcher-kva.com"
- )
- self.assertEqual(len(http_server.requests), 1)
- request = http_server.requests[0]
- self.assertEqual(request.method, b"GET")
- self.assertEqual(request.path, b"/foo/bar")
- self.assertEqual(
- request.requestHeaders.getRawHeaders(b"host"), [b"xn--bcher-kva.com"]
- )
- # finish the request
- request.finish()
- self.reactor.pump((0.1,))
- self.successResultOf(test_d)
- def test_well_known_cache(self):
- self.reactor.lookups["testserv"] = "1.2.3.4"
- fetch_d = self.well_known_resolver.get_well_known(b"testserv")
- # there should be an attempt to connect on port 443 for the .well-known
- clients = self.reactor.tcpClients
- self.assertEqual(len(clients), 1)
- (host, port, client_factory, _timeout, _bindAddress) = clients.pop(0)
- self.assertEqual(host, "1.2.3.4")
- self.assertEqual(port, 443)
- well_known_server = self._handle_well_known_connection(
- client_factory,
- expected_sni=b"testserv",
- response_headers={b"Cache-Control": b"max-age=1000"},
- content=b'{ "m.server": "target-server" }',
- )
- r = self.successResultOf(fetch_d)
- self.assertEqual(r.delegated_server, b"target-server")
- # close the tcp connection
- well_known_server.loseConnection()
- # repeat the request: it should hit the cache
- fetch_d = self.well_known_resolver.get_well_known(b"testserv")
- r = self.successResultOf(fetch_d)
- self.assertEqual(r.delegated_server, b"target-server")
- # expire the cache
- self.reactor.pump((1000.0,))
- # now it should connect again
- fetch_d = self.well_known_resolver.get_well_known(b"testserv")
- self.assertEqual(len(clients), 1)
- (host, port, client_factory, _timeout, _bindAddress) = clients.pop(0)
- self.assertEqual(host, "1.2.3.4")
- self.assertEqual(port, 443)
- self._handle_well_known_connection(
- client_factory,
- expected_sni=b"testserv",
- content=b'{ "m.server": "other-server" }',
- )
- r = self.successResultOf(fetch_d)
- self.assertEqual(r.delegated_server, b"other-server")
- def test_well_known_cache_with_temp_failure(self):
- """Test that we refetch well-known before the cache expires, and that
- it ignores transient errors.
- """
- self.reactor.lookups["testserv"] = "1.2.3.4"
- fetch_d = self.well_known_resolver.get_well_known(b"testserv")
- # there should be an attempt to connect on port 443 for the .well-known
- clients = self.reactor.tcpClients
- self.assertEqual(len(clients), 1)
- (host, port, client_factory, _timeout, _bindAddress) = clients.pop(0)
- self.assertEqual(host, "1.2.3.4")
- self.assertEqual(port, 443)
- well_known_server = self._handle_well_known_connection(
- client_factory,
- expected_sni=b"testserv",
- response_headers={b"Cache-Control": b"max-age=1000"},
- content=b'{ "m.server": "target-server" }',
- )
- r = self.successResultOf(fetch_d)
- self.assertEqual(r.delegated_server, b"target-server")
- # close the tcp connection
- well_known_server.loseConnection()
- # Get close to the cache expiry, this will cause the resolver to do
- # another lookup.
- self.reactor.pump((900.0,))
- fetch_d = self.well_known_resolver.get_well_known(b"testserv")
- # The resolver may retry a few times, so fonx all requests that come along
- attempts = 0
- while self.reactor.tcpClients:
- clients = self.reactor.tcpClients
- (host, port, client_factory, _timeout, _bindAddress) = clients.pop(0)
- attempts += 1
- # fonx the connection attempt, this will be treated as a temporary
- # failure.
- client_factory.clientConnectionFailed(None, Exception("nope"))
- # There's a few sleeps involved, so we have to pump the reactor a
- # bit.
- self.reactor.pump((1.0, 1.0))
- # We expect to see more than one attempt as there was previously a valid
- # well known.
- self.assertGreater(attempts, 1)
- # Resolver should return cached value, despite the lookup failing.
- r = self.successResultOf(fetch_d)
- self.assertEqual(r.delegated_server, b"target-server")
- # Expire both caches and repeat the request
- self.reactor.pump((10000.0,))
- # Repated the request, this time it should fail if the lookup fails.
- fetch_d = self.well_known_resolver.get_well_known(b"testserv")
- clients = self.reactor.tcpClients
- (host, port, client_factory, _timeout, _bindAddress) = clients.pop(0)
- client_factory.clientConnectionFailed(None, Exception("nope"))
- self.reactor.pump((0.4,))
- r = self.successResultOf(fetch_d)
- self.assertEqual(r.delegated_server, None)
- def test_srv_fallbacks(self):
- """Test that other SRV results are tried if the first one fails.
- """
- self.mock_resolver.resolve_service.side_effect = lambda _: [
- Server(host=b"target.com", port=8443),
- Server(host=b"target.com", port=8444),
- ]
- self.reactor.lookups["target.com"] = "1.2.3.4"
- test_d = self._make_get_request(b"matrix://testserv/foo/bar")
- # Nothing happened yet
- self.assertNoResult(test_d)
- self.mock_resolver.resolve_service.assert_called_once_with(
- b"_matrix._tcp.testserv"
- )
- # We should see an attempt to connect to the first server
- clients = self.reactor.tcpClients
- self.assertEqual(len(clients), 1)
- (host, port, client_factory, _timeout, _bindAddress) = clients.pop(0)
- self.assertEqual(host, "1.2.3.4")
- self.assertEqual(port, 8443)
- # Fonx the connection
- client_factory.clientConnectionFailed(None, Exception("nope"))
- # There's a 300ms delay in HostnameEndpoint
- self.reactor.pump((0.4,))
- # Hasn't failed yet
- self.assertNoResult(test_d)
- # We shouldnow see an attempt to connect to the second server
- clients = self.reactor.tcpClients
- self.assertEqual(len(clients), 1)
- (host, port, client_factory, _timeout, _bindAddress) = clients.pop(0)
- self.assertEqual(host, "1.2.3.4")
- self.assertEqual(port, 8444)
- # make a test server, and wire up the client
- http_server = self._make_connection(client_factory, expected_sni=b"testserv")
- self.assertEqual(len(http_server.requests), 1)
- request = http_server.requests[0]
- self.assertEqual(request.method, b"GET")
- self.assertEqual(request.path, b"/foo/bar")
- self.assertEqual(request.requestHeaders.getRawHeaders(b"host"), [b"testserv"])
- # finish the request
- request.finish()
- self.reactor.pump((0.1,))
- self.successResultOf(test_d)
- class TestCachePeriodFromHeaders(unittest.TestCase):
- def test_cache_control(self):
- # uppercase
- self.assertEqual(
- _cache_period_from_headers(
- Headers({b"Cache-Control": [b"foo, Max-Age = 100, bar"]})
- ),
- 100,
- )
- # missing value
- self.assertIsNone(
- _cache_period_from_headers(Headers({b"Cache-Control": [b"max-age=, bar"]}))
- )
- # hackernews: bogus due to semicolon
- self.assertIsNone(
- _cache_period_from_headers(
- Headers({b"Cache-Control": [b"private; max-age=0"]})
- )
- )
- # github
- self.assertEqual(
- _cache_period_from_headers(
- Headers({b"Cache-Control": [b"max-age=0, private, must-revalidate"]})
- ),
- 0,
- )
- # google
- self.assertEqual(
- _cache_period_from_headers(
- Headers({b"cache-control": [b"private, max-age=0"]})
- ),
- 0,
- )
- def test_expires(self):
- self.assertEqual(
- _cache_period_from_headers(
- Headers({b"Expires": [b"Wed, 30 Jan 2019 07:35:33 GMT"]}),
- time_now=lambda: 1548833700,
- ),
- 33,
- )
- # cache-control overrides expires
- self.assertEqual(
- _cache_period_from_headers(
- Headers(
- {
- b"cache-control": [b"max-age=10"],
- b"Expires": [b"Wed, 30 Jan 2019 07:35:33 GMT"],
- }
- ),
- time_now=lambda: 1548833700,
- ),
- 10,
- )
- # invalid expires means immediate expiry
- self.assertEqual(_cache_period_from_headers(Headers({b"Expires": [b"0"]})), 0)
- def _check_logcontext(context):
- current = LoggingContext.current_context()
- if current is not context:
- raise AssertionError("Expected logcontext %s but was %s" % (context, current))
- def _build_test_server(connection_creator):
- """Construct a test server
- This builds an HTTP channel, wrapped with a TLSMemoryBIOProtocol
- Args:
- connection_creator (IOpenSSLServerConnectionCreator): thing to build
- SSL connections
- sanlist (list[bytes]): list of the SAN entries for the cert returned
- by the server
- Returns:
- TLSMemoryBIOProtocol
- """
- server_factory = Factory.forProtocol(HTTPChannel)
- # Request.finish expects the factory to have a 'log' method.
- server_factory.log = _log_request
- server_tls_factory = TLSMemoryBIOFactory(
- connection_creator, isClient=False, wrappedFactory=server_factory
- )
- return server_tls_factory.buildProtocol(None)
- def _log_request(request):
- """Implements Factory.log, which is expected by Request.finish"""
- logger.info("Completed request %s", request)
- @implementer(IPolicyForHTTPS)
- class TrustingTLSPolicyForHTTPS(object):
- """An IPolicyForHTTPS which checks that the certificate belongs to the
- right server, but doesn't check the certificate chain."""
- def creatorForNetloc(self, hostname, port):
- certificateOptions = OpenSSLCertificateOptions()
- return ClientTLSOptions(hostname, certificateOptions.getContext())
|