test_matrix_federation_agent.py 57 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586
  1. # Copyright 2019 New Vector Ltd
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. import base64
  15. import logging
  16. import os
  17. from typing import Any, Awaitable, Callable, Generator, List, Optional, cast
  18. from unittest.mock import Mock, patch
  19. import treq
  20. from netaddr import IPSet
  21. from service_identity import VerificationError
  22. from zope.interface import implementer
  23. from twisted.internet import defer
  24. from twisted.internet._sslverify import ClientTLSOptions, OpenSSLCertificateOptions
  25. from twisted.internet.defer import Deferred
  26. from twisted.internet.endpoints import _WrappingProtocol
  27. from twisted.internet.interfaces import (
  28. IOpenSSLClientConnectionCreator,
  29. IProtocolFactory,
  30. )
  31. from twisted.internet.protocol import Factory, Protocol
  32. from twisted.protocols.tls import TLSMemoryBIOFactory, TLSMemoryBIOProtocol
  33. from twisted.web._newclient import ResponseNeverReceived
  34. from twisted.web.client import Agent
  35. from twisted.web.http import HTTPChannel, Request
  36. from twisted.web.http_headers import Headers
  37. from twisted.web.iweb import IPolicyForHTTPS, IResponse
  38. from synapse.config.homeserver import HomeServerConfig
  39. from synapse.crypto.context_factory import FederationPolicyForHTTPS
  40. from synapse.http.federation.matrix_federation_agent import MatrixFederationAgent
  41. from synapse.http.federation.srv_resolver import Server
  42. from synapse.http.federation.well_known_resolver import (
  43. WELL_KNOWN_MAX_SIZE,
  44. WellKnownResolver,
  45. _cache_period_from_headers,
  46. )
  47. from synapse.logging.context import (
  48. SENTINEL_CONTEXT,
  49. LoggingContext,
  50. LoggingContextOrSentinel,
  51. current_context,
  52. )
  53. from synapse.types import ISynapseReactor
  54. from synapse.util.caches.ttlcache import TTLCache
  55. from tests import unittest
  56. from tests.http import (
  57. TestServerTLSConnectionFactory,
  58. dummy_address,
  59. get_test_ca_cert_file,
  60. )
  61. from tests.server import FakeTransport, ThreadedMemoryReactorClock
  62. from tests.utils import checked_cast, default_config
  63. logger = logging.getLogger(__name__)
  64. # Once Async Mocks or lambdas are supported this can go away.
  65. def generate_resolve_service(
  66. result: List[Server],
  67. ) -> Callable[[Any], Awaitable[List[Server]]]:
  68. async def resolve_service(_: Any) -> List[Server]:
  69. return result
  70. return resolve_service
  71. class MatrixFederationAgentTests(unittest.TestCase):
  72. def setUp(self) -> None:
  73. self.reactor = ThreadedMemoryReactorClock()
  74. self.mock_resolver = Mock()
  75. config_dict = default_config("test", parse=False)
  76. config_dict["federation_custom_ca_list"] = [get_test_ca_cert_file()]
  77. self._config = config = HomeServerConfig()
  78. config.parse_config_dict(config_dict, "", "")
  79. self.tls_factory = FederationPolicyForHTTPS(config)
  80. self.well_known_cache: TTLCache[bytes, Optional[bytes]] = TTLCache(
  81. "test_cache", timer=self.reactor.seconds
  82. )
  83. self.had_well_known_cache: TTLCache[bytes, bool] = TTLCache(
  84. "test_cache", timer=self.reactor.seconds
  85. )
  86. self.well_known_resolver = WellKnownResolver(
  87. self.reactor,
  88. Agent(self.reactor, contextFactory=self.tls_factory),
  89. b"test-agent",
  90. well_known_cache=self.well_known_cache,
  91. had_well_known_cache=self.had_well_known_cache,
  92. )
  93. def _make_connection(
  94. self,
  95. client_factory: IProtocolFactory,
  96. ssl: bool = True,
  97. expected_sni: Optional[bytes] = None,
  98. tls_sanlist: Optional[List[bytes]] = None,
  99. ) -> HTTPChannel:
  100. """Builds a test server, and completes the outgoing client connection
  101. Args:
  102. client_factory: the the factory that the
  103. application is trying to use to make the outbound connection. We will
  104. invoke it to build the client Protocol
  105. ssl: If true, we will expect an ssl connection and wrap
  106. server_factory with a TLSMemoryBIOFactory
  107. False is set only for when proxy expect http connection.
  108. Otherwise federation requests use always https.
  109. expected_sni: the expected SNI value
  110. tls_sanlist: list of SAN entries for the TLS cert presented by the server.
  111. Returns:
  112. the server Protocol returned by server_factory
  113. """
  114. # build the test server
  115. server_factory = _get_test_protocol_factory()
  116. if ssl:
  117. server_factory = _wrap_server_factory_for_tls(server_factory, tls_sanlist)
  118. server_protocol = server_factory.buildProtocol(dummy_address)
  119. assert server_protocol is not None
  120. # now, tell the client protocol factory to build the client protocol (it will be a
  121. # _WrappingProtocol, around a TLSMemoryBIOProtocol, around an
  122. # HTTP11ClientProtocol) and wire the output of said protocol up to the server via
  123. # a FakeTransport.
  124. #
  125. # Normally this would be done by the TCP socket code in Twisted, but we are
  126. # stubbing that out here.
  127. # NB: we use a checked_cast here to workaround https://github.com/Shoobx/mypy-zope/issues/91)
  128. client_protocol = checked_cast(
  129. _WrappingProtocol, client_factory.buildProtocol(dummy_address)
  130. )
  131. client_protocol.makeConnection(
  132. FakeTransport(server_protocol, self.reactor, client_protocol)
  133. )
  134. # tell the server protocol to send its stuff back to the client, too
  135. server_protocol.makeConnection(
  136. FakeTransport(client_protocol, self.reactor, server_protocol)
  137. )
  138. if ssl:
  139. assert isinstance(server_protocol, TLSMemoryBIOProtocol)
  140. # fish the test server back out of the server-side TLS protocol.
  141. http_protocol = server_protocol.wrappedProtocol
  142. # grab a hold of the TLS connection, in case it gets torn down
  143. tls_connection = server_protocol._tlsConnection
  144. else:
  145. http_protocol = server_protocol
  146. tls_connection = None
  147. assert isinstance(http_protocol, HTTPChannel)
  148. # give the reactor a pump to get the TLS juices flowing (if needed)
  149. self.reactor.advance(0)
  150. # check the SNI
  151. if expected_sni is not None:
  152. server_name = tls_connection.get_servername()
  153. self.assertEqual(
  154. server_name,
  155. expected_sni,
  156. f"Expected SNI {expected_sni!s} but got {server_name!s}",
  157. )
  158. return http_protocol
  159. @defer.inlineCallbacks
  160. def _make_get_request(
  161. self, uri: bytes
  162. ) -> Generator["Deferred[object]", object, IResponse]:
  163. """
  164. Sends a simple GET request via the agent, and checks its logcontext management
  165. """
  166. with LoggingContext("one") as context:
  167. fetch_d: Deferred[IResponse] = self.agent.request(b"GET", uri)
  168. # Nothing happened yet
  169. self.assertNoResult(fetch_d)
  170. # should have reset logcontext to the sentinel
  171. _check_logcontext(SENTINEL_CONTEXT)
  172. fetch_res: IResponse
  173. try:
  174. fetch_res = yield fetch_d # type: ignore[misc, assignment]
  175. return fetch_res
  176. except Exception as e:
  177. logger.info("Fetch of %s failed: %s", uri.decode("ascii"), e)
  178. raise
  179. finally:
  180. _check_logcontext(context)
  181. def _handle_well_known_connection(
  182. self,
  183. client_factory: IProtocolFactory,
  184. expected_sni: bytes,
  185. content: bytes,
  186. response_headers: Optional[dict] = None,
  187. ) -> HTTPChannel:
  188. """Handle an outgoing HTTPs connection: wire it up to a server, check that the
  189. request is for a .well-known, and send the response.
  190. Args:
  191. client_factory: outgoing connection
  192. expected_sni: SNI that we expect the outgoing connection to send
  193. content: content to send back as the .well-known
  194. Returns:
  195. server impl
  196. """
  197. # make the connection for .well-known
  198. well_known_server = self._make_connection(
  199. client_factory, expected_sni=expected_sni
  200. )
  201. # check the .well-known request and send a response
  202. self.assertEqual(len(well_known_server.requests), 1)
  203. request = well_known_server.requests[0]
  204. self.assertEqual(
  205. request.requestHeaders.getRawHeaders(b"user-agent"), [b"test-agent"]
  206. )
  207. self._send_well_known_response(request, content, headers=response_headers or {})
  208. return well_known_server
  209. def _send_well_known_response(
  210. self,
  211. request: Request,
  212. content: bytes,
  213. headers: Optional[dict] = None,
  214. ) -> None:
  215. """Check that an incoming request looks like a valid .well-known request, and
  216. send back the response.
  217. """
  218. self.assertEqual(request.method, b"GET")
  219. self.assertEqual(request.path, b"/.well-known/matrix/server")
  220. self.assertEqual(request.requestHeaders.getRawHeaders(b"host"), [b"testserv"])
  221. # send back a response
  222. for k, v in (headers or {}).items():
  223. request.setHeader(k, v)
  224. request.write(content)
  225. request.finish()
  226. self.reactor.pump((0.1,))
  227. def _make_agent(self) -> MatrixFederationAgent:
  228. """
  229. If a proxy server is set, the MatrixFederationAgent must be created again
  230. because it is created too early during setUp
  231. """
  232. return MatrixFederationAgent(
  233. reactor=cast(ISynapseReactor, self.reactor),
  234. tls_client_options_factory=self.tls_factory,
  235. user_agent=b"test-agent", # Note that this is unused since _well_known_resolver is provided.
  236. ip_whitelist=IPSet(),
  237. ip_blacklist=IPSet(),
  238. _srv_resolver=self.mock_resolver,
  239. _well_known_resolver=self.well_known_resolver,
  240. )
  241. def test_get(self) -> None:
  242. """happy-path test of a GET request with an explicit port"""
  243. self._do_get()
  244. @patch.dict(
  245. os.environ,
  246. {"https_proxy": "proxy.com", "no_proxy": "testserv"},
  247. )
  248. def test_get_bypass_proxy(self) -> None:
  249. """test of a GET request with an explicit port and bypass proxy"""
  250. self._do_get()
  251. def _do_get(self) -> None:
  252. """test of a GET request with an explicit port"""
  253. self.agent = self._make_agent()
  254. self.reactor.lookups["testserv"] = "1.2.3.4"
  255. test_d = self._make_get_request(b"matrix://testserv:8448/foo/bar")
  256. # Nothing happened yet
  257. self.assertNoResult(test_d)
  258. # Make sure treq is trying to connect
  259. clients = self.reactor.tcpClients
  260. self.assertEqual(len(clients), 1)
  261. (host, port, client_factory, _timeout, _bindAddress) = clients[0]
  262. self.assertEqual(host, "1.2.3.4")
  263. self.assertEqual(port, 8448)
  264. # make a test server, and wire up the client
  265. http_server = self._make_connection(client_factory, expected_sni=b"testserv")
  266. self.assertEqual(len(http_server.requests), 1)
  267. request = http_server.requests[0]
  268. self.assertEqual(request.method, b"GET")
  269. self.assertEqual(request.path, b"/foo/bar")
  270. self.assertEqual(
  271. request.requestHeaders.getRawHeaders(b"host"), [b"testserv:8448"]
  272. )
  273. self.assertEqual(
  274. request.requestHeaders.getRawHeaders(b"user-agent"), [b"test-agent"]
  275. )
  276. content = request.content.read()
  277. self.assertEqual(content, b"")
  278. # Deferred is still without a result
  279. self.assertNoResult(test_d)
  280. # send the headers
  281. request.responseHeaders.setRawHeaders(b"Content-Type", [b"application/json"])
  282. request.write("")
  283. self.reactor.pump((0.1,))
  284. response = self.successResultOf(test_d)
  285. # that should give us a Response object
  286. self.assertEqual(response.code, 200)
  287. # Send the body
  288. request.write(b'{ "a": 1 }')
  289. request.finish()
  290. self.reactor.pump((0.1,))
  291. # check it can be read
  292. json = self.successResultOf(treq.json_content(response))
  293. self.assertEqual(json, {"a": 1})
  294. @patch.dict(
  295. os.environ, {"https_proxy": "http://proxy.com", "no_proxy": "unused.com"}
  296. )
  297. def test_get_via_http_proxy(self) -> None:
  298. """test for federation request through a http proxy"""
  299. self._do_get_via_proxy(expect_proxy_ssl=False, expected_auth_credentials=None)
  300. @patch.dict(
  301. os.environ,
  302. {"https_proxy": "http://user:pass@proxy.com", "no_proxy": "unused.com"},
  303. )
  304. def test_get_via_http_proxy_with_auth(self) -> None:
  305. """test for federation request through a http proxy with authentication"""
  306. self._do_get_via_proxy(
  307. expect_proxy_ssl=False, expected_auth_credentials=b"user:pass"
  308. )
  309. @patch.dict(
  310. os.environ, {"https_proxy": "https://proxy.com", "no_proxy": "unused.com"}
  311. )
  312. def test_get_via_https_proxy(self) -> None:
  313. """test for federation request through a https proxy"""
  314. self._do_get_via_proxy(expect_proxy_ssl=True, expected_auth_credentials=None)
  315. @patch.dict(
  316. os.environ,
  317. {"https_proxy": "https://user:pass@proxy.com", "no_proxy": "unused.com"},
  318. )
  319. def test_get_via_https_proxy_with_auth(self) -> None:
  320. """test for federation request through a https proxy with authentication"""
  321. self._do_get_via_proxy(
  322. expect_proxy_ssl=True, expected_auth_credentials=b"user:pass"
  323. )
  324. def _do_get_via_proxy(
  325. self,
  326. expect_proxy_ssl: bool = False,
  327. expected_auth_credentials: Optional[bytes] = None,
  328. ) -> None:
  329. """Send a https federation request via an agent and check that it is correctly
  330. received at the proxy and client. The proxy can use either http or https.
  331. Args:
  332. expect_proxy_ssl: True if we expect the request to connect to the proxy via https.
  333. expected_auth_credentials: credentials we expect to be presented to authenticate at the proxy
  334. """
  335. self.agent = self._make_agent()
  336. self.reactor.lookups["testserv"] = "1.2.3.4"
  337. self.reactor.lookups["proxy.com"] = "9.9.9.9"
  338. test_d = self._make_get_request(b"matrix://testserv:8448/foo/bar")
  339. # Nothing happened yet
  340. self.assertNoResult(test_d)
  341. # Make sure treq is trying to connect
  342. clients = self.reactor.tcpClients
  343. self.assertEqual(len(clients), 1)
  344. (host, port, client_factory, _timeout, _bindAddress) = clients[0]
  345. # make sure we are connecting to the proxy
  346. self.assertEqual(host, "9.9.9.9")
  347. self.assertEqual(port, 1080)
  348. # make a test server to act as the proxy, and wire up the client
  349. proxy_server = self._make_connection(
  350. client_factory,
  351. ssl=expect_proxy_ssl,
  352. tls_sanlist=[b"DNS:proxy.com"] if expect_proxy_ssl else None,
  353. expected_sni=b"proxy.com" if expect_proxy_ssl else None,
  354. )
  355. assert isinstance(proxy_server, HTTPChannel)
  356. # now there should be a pending CONNECT request
  357. self.assertEqual(len(proxy_server.requests), 1)
  358. request = proxy_server.requests[0]
  359. self.assertEqual(request.method, b"CONNECT")
  360. self.assertEqual(request.path, b"testserv:8448")
  361. # Check whether auth credentials have been supplied to the proxy
  362. proxy_auth_header_values = request.requestHeaders.getRawHeaders(
  363. b"Proxy-Authorization"
  364. )
  365. if expected_auth_credentials is not None:
  366. # Compute the correct header value for Proxy-Authorization
  367. encoded_credentials = base64.b64encode(expected_auth_credentials)
  368. expected_header_value = b"Basic " + encoded_credentials
  369. # Validate the header's value
  370. self.assertIn(expected_header_value, proxy_auth_header_values)
  371. else:
  372. # Check that the Proxy-Authorization header has not been supplied to the proxy
  373. self.assertIsNone(proxy_auth_header_values)
  374. # tell the proxy server not to close the connection
  375. proxy_server.persistent = True
  376. request.finish()
  377. # now we make another test server to act as the upstream HTTP server.
  378. server_ssl_protocol = _wrap_server_factory_for_tls(
  379. _get_test_protocol_factory()
  380. ).buildProtocol(dummy_address)
  381. # Tell the HTTP server to send outgoing traffic back via the proxy's transport.
  382. proxy_server_transport = proxy_server.transport
  383. assert proxy_server_transport is not None
  384. server_ssl_protocol.makeConnection(proxy_server_transport)
  385. # ... and replace the protocol on the proxy's transport with the
  386. # TLSMemoryBIOProtocol for the test server, so that incoming traffic
  387. # to the proxy gets sent over to the HTTP(s) server.
  388. # See also comment at `_do_https_request_via_proxy`
  389. # in ../test_proxyagent.py for more details
  390. if expect_proxy_ssl:
  391. assert isinstance(proxy_server_transport, TLSMemoryBIOProtocol)
  392. proxy_server_transport.wrappedProtocol = server_ssl_protocol
  393. else:
  394. assert isinstance(proxy_server_transport, FakeTransport)
  395. client_protocol = proxy_server_transport.other
  396. assert isinstance(client_protocol, Protocol)
  397. c2s_transport = checked_cast(FakeTransport, client_protocol.transport)
  398. c2s_transport.other = server_ssl_protocol
  399. self.reactor.advance(0)
  400. server_name = server_ssl_protocol._tlsConnection.get_servername()
  401. expected_sni = b"testserv"
  402. self.assertEqual(
  403. server_name,
  404. expected_sni,
  405. f"Expected SNI {expected_sni!s} but got {server_name!s}",
  406. )
  407. # now there should be a pending request
  408. http_server = server_ssl_protocol.wrappedProtocol
  409. assert isinstance(http_server, HTTPChannel)
  410. self.assertEqual(len(http_server.requests), 1)
  411. request = http_server.requests[0]
  412. self.assertEqual(request.method, b"GET")
  413. self.assertEqual(request.path, b"/foo/bar")
  414. self.assertEqual(
  415. request.requestHeaders.getRawHeaders(b"host"), [b"testserv:8448"]
  416. )
  417. self.assertEqual(
  418. request.requestHeaders.getRawHeaders(b"user-agent"), [b"test-agent"]
  419. )
  420. # Check that the destination server DID NOT receive proxy credentials
  421. self.assertIsNone(request.requestHeaders.getRawHeaders(b"Proxy-Authorization"))
  422. content = request.content.read()
  423. self.assertEqual(content, b"")
  424. # Deferred is still without a result
  425. self.assertNoResult(test_d)
  426. # send the headers
  427. request.responseHeaders.setRawHeaders(b"Content-Type", [b"application/json"])
  428. request.write("")
  429. self.reactor.pump((0.1,))
  430. response = self.successResultOf(test_d)
  431. # that should give us a Response object
  432. self.assertEqual(response.code, 200)
  433. # Send the body
  434. request.write('{ "a": 1 }'.encode("ascii"))
  435. request.finish()
  436. self.reactor.pump((0.1,))
  437. # check it can be read
  438. json = self.successResultOf(treq.json_content(response))
  439. self.assertEqual(json, {"a": 1})
  440. def test_get_ip_address(self) -> None:
  441. """
  442. Test the behaviour when the server name contains an explicit IP (with no port)
  443. """
  444. self.agent = self._make_agent()
  445. # there will be a getaddrinfo on the IP
  446. self.reactor.lookups["1.2.3.4"] = "1.2.3.4"
  447. test_d = self._make_get_request(b"matrix://1.2.3.4/foo/bar")
  448. # Nothing happened yet
  449. self.assertNoResult(test_d)
  450. # Make sure treq is trying to connect
  451. clients = self.reactor.tcpClients
  452. self.assertEqual(len(clients), 1)
  453. (host, port, client_factory, _timeout, _bindAddress) = clients[0]
  454. self.assertEqual(host, "1.2.3.4")
  455. self.assertEqual(port, 8448)
  456. # make a test server, and wire up the client
  457. http_server = self._make_connection(client_factory, expected_sni=None)
  458. self.assertEqual(len(http_server.requests), 1)
  459. request = http_server.requests[0]
  460. self.assertEqual(request.method, b"GET")
  461. self.assertEqual(request.path, b"/foo/bar")
  462. self.assertEqual(request.requestHeaders.getRawHeaders(b"host"), [b"1.2.3.4"])
  463. # finish the request
  464. request.finish()
  465. self.reactor.pump((0.1,))
  466. self.successResultOf(test_d)
  467. def test_get_ipv6_address(self) -> None:
  468. """
  469. Test the behaviour when the server name contains an explicit IPv6 address
  470. (with no port)
  471. """
  472. self.agent = self._make_agent()
  473. # there will be a getaddrinfo on the IP
  474. self.reactor.lookups["::1"] = "::1"
  475. test_d = self._make_get_request(b"matrix://[::1]/foo/bar")
  476. # Nothing happened yet
  477. self.assertNoResult(test_d)
  478. # Make sure treq is trying to connect
  479. clients = self.reactor.tcpClients
  480. self.assertEqual(len(clients), 1)
  481. (host, port, client_factory, _timeout, _bindAddress) = clients[0]
  482. self.assertEqual(host, "::1")
  483. self.assertEqual(port, 8448)
  484. # make a test server, and wire up the client
  485. http_server = self._make_connection(client_factory, expected_sni=None)
  486. self.assertEqual(len(http_server.requests), 1)
  487. request = http_server.requests[0]
  488. self.assertEqual(request.method, b"GET")
  489. self.assertEqual(request.path, b"/foo/bar")
  490. self.assertEqual(request.requestHeaders.getRawHeaders(b"host"), [b"[::1]"])
  491. # finish the request
  492. request.finish()
  493. self.reactor.pump((0.1,))
  494. self.successResultOf(test_d)
  495. def test_get_ipv6_address_with_port(self) -> None:
  496. """
  497. Test the behaviour when the server name contains an explicit IPv6 address
  498. (with explicit port)
  499. """
  500. self.agent = self._make_agent()
  501. # there will be a getaddrinfo on the IP
  502. self.reactor.lookups["::1"] = "::1"
  503. test_d = self._make_get_request(b"matrix://[::1]:80/foo/bar")
  504. # Nothing happened yet
  505. self.assertNoResult(test_d)
  506. # Make sure treq is trying to connect
  507. clients = self.reactor.tcpClients
  508. self.assertEqual(len(clients), 1)
  509. (host, port, client_factory, _timeout, _bindAddress) = clients[0]
  510. self.assertEqual(host, "::1")
  511. self.assertEqual(port, 80)
  512. # make a test server, and wire up the client
  513. http_server = self._make_connection(client_factory, expected_sni=None)
  514. self.assertEqual(len(http_server.requests), 1)
  515. request = http_server.requests[0]
  516. self.assertEqual(request.method, b"GET")
  517. self.assertEqual(request.path, b"/foo/bar")
  518. self.assertEqual(request.requestHeaders.getRawHeaders(b"host"), [b"[::1]:80"])
  519. # finish the request
  520. request.finish()
  521. self.reactor.pump((0.1,))
  522. self.successResultOf(test_d)
  523. def test_get_hostname_bad_cert(self) -> None:
  524. """
  525. Test the behaviour when the certificate on the server doesn't match the hostname
  526. """
  527. self.agent = self._make_agent()
  528. self.mock_resolver.resolve_service.side_effect = generate_resolve_service([])
  529. self.reactor.lookups["testserv1"] = "1.2.3.4"
  530. test_d = self._make_get_request(b"matrix://testserv1/foo/bar")
  531. # Nothing happened yet
  532. self.assertNoResult(test_d)
  533. # No SRV record lookup yet
  534. self.mock_resolver.resolve_service.assert_not_called()
  535. # there should be an attempt to connect on port 443 for the .well-known
  536. clients = self.reactor.tcpClients
  537. self.assertEqual(len(clients), 1)
  538. (host, port, client_factory, _timeout, _bindAddress) = clients[0]
  539. self.assertEqual(host, "1.2.3.4")
  540. self.assertEqual(port, 443)
  541. # fonx the connection
  542. client_factory.clientConnectionFailed(None, Exception("nope"))
  543. # attemptdelay on the hostnameendpoint is 0.3, so takes that long before the
  544. # .well-known request fails.
  545. self.reactor.pump((0.4,))
  546. # now there should be a SRV lookup
  547. self.mock_resolver.resolve_service.assert_called_once_with(
  548. b"_matrix._tcp.testserv1"
  549. )
  550. # we should fall back to a direct connection
  551. self.assertEqual(len(clients), 2)
  552. (host, port, client_factory, _timeout, _bindAddress) = clients[1]
  553. self.assertEqual(host, "1.2.3.4")
  554. self.assertEqual(port, 8448)
  555. # make a test server, and wire up the client
  556. http_server = self._make_connection(client_factory, expected_sni=b"testserv1")
  557. # there should be no requests
  558. self.assertEqual(len(http_server.requests), 0)
  559. # ... and the request should have failed
  560. e = self.failureResultOf(test_d, ResponseNeverReceived)
  561. failure_reason = e.value.reasons[0]
  562. self.assertIsInstance(failure_reason.value, VerificationError)
  563. def test_get_ip_address_bad_cert(self) -> None:
  564. """
  565. Test the behaviour when the server name contains an explicit IP, but
  566. the server cert doesn't cover it
  567. """
  568. self.agent = self._make_agent()
  569. # there will be a getaddrinfo on the IP
  570. self.reactor.lookups["1.2.3.5"] = "1.2.3.5"
  571. test_d = self._make_get_request(b"matrix://1.2.3.5/foo/bar")
  572. # Nothing happened yet
  573. self.assertNoResult(test_d)
  574. # Make sure treq is trying to connect
  575. clients = self.reactor.tcpClients
  576. self.assertEqual(len(clients), 1)
  577. (host, port, client_factory, _timeout, _bindAddress) = clients[0]
  578. self.assertEqual(host, "1.2.3.5")
  579. self.assertEqual(port, 8448)
  580. # make a test server, and wire up the client
  581. http_server = self._make_connection(client_factory, expected_sni=None)
  582. # there should be no requests
  583. self.assertEqual(len(http_server.requests), 0)
  584. # ... and the request should have failed
  585. e = self.failureResultOf(test_d, ResponseNeverReceived)
  586. failure_reason = e.value.reasons[0]
  587. self.assertIsInstance(failure_reason.value, VerificationError)
  588. def test_get_no_srv_no_well_known(self) -> None:
  589. """
  590. Test the behaviour when the server name has no port, no SRV, and no well-known
  591. """
  592. self.agent = self._make_agent()
  593. self.mock_resolver.resolve_service.side_effect = generate_resolve_service([])
  594. self.reactor.lookups["testserv"] = "1.2.3.4"
  595. test_d = self._make_get_request(b"matrix://testserv/foo/bar")
  596. # Nothing happened yet
  597. self.assertNoResult(test_d)
  598. # No SRV record lookup yet
  599. self.mock_resolver.resolve_service.assert_not_called()
  600. # there should be an attempt to connect on port 443 for the .well-known
  601. clients = self.reactor.tcpClients
  602. self.assertEqual(len(clients), 1)
  603. (host, port, client_factory, _timeout, _bindAddress) = clients[0]
  604. self.assertEqual(host, "1.2.3.4")
  605. self.assertEqual(port, 443)
  606. # fonx the connection
  607. client_factory.clientConnectionFailed(None, Exception("nope"))
  608. # attemptdelay on the hostnameendpoint is 0.3, so takes that long before the
  609. # .well-known request fails.
  610. self.reactor.pump((0.4,))
  611. # now there should be a SRV lookup
  612. self.mock_resolver.resolve_service.assert_called_once_with(
  613. b"_matrix._tcp.testserv"
  614. )
  615. # we should fall back to a direct connection
  616. self.assertEqual(len(clients), 2)
  617. (host, port, client_factory, _timeout, _bindAddress) = clients[1]
  618. self.assertEqual(host, "1.2.3.4")
  619. self.assertEqual(port, 8448)
  620. # make a test server, and wire up the client
  621. http_server = self._make_connection(client_factory, expected_sni=b"testserv")
  622. self.assertEqual(len(http_server.requests), 1)
  623. request = http_server.requests[0]
  624. self.assertEqual(request.method, b"GET")
  625. self.assertEqual(request.path, b"/foo/bar")
  626. self.assertEqual(request.requestHeaders.getRawHeaders(b"host"), [b"testserv"])
  627. # finish the request
  628. request.finish()
  629. self.reactor.pump((0.1,))
  630. self.successResultOf(test_d)
  631. def test_get_well_known(self) -> None:
  632. """Test the behaviour when the .well-known delegates elsewhere"""
  633. self.agent = self._make_agent()
  634. self.mock_resolver.resolve_service.side_effect = generate_resolve_service([])
  635. self.reactor.lookups["testserv"] = "1.2.3.4"
  636. self.reactor.lookups["target-server"] = "1::f"
  637. test_d = self._make_get_request(b"matrix://testserv/foo/bar")
  638. # Nothing happened yet
  639. self.assertNoResult(test_d)
  640. # there should be an attempt to connect on port 443 for the .well-known
  641. clients = self.reactor.tcpClients
  642. self.assertEqual(len(clients), 1)
  643. (host, port, client_factory, _timeout, _bindAddress) = clients[0]
  644. self.assertEqual(host, "1.2.3.4")
  645. self.assertEqual(port, 443)
  646. self._handle_well_known_connection(
  647. client_factory,
  648. expected_sni=b"testserv",
  649. content=b'{ "m.server": "target-server" }',
  650. )
  651. # there should be a SRV lookup
  652. self.mock_resolver.resolve_service.assert_called_once_with(
  653. b"_matrix._tcp.target-server"
  654. )
  655. # now we should get a connection to the target server
  656. self.assertEqual(len(clients), 2)
  657. (host, port, client_factory, _timeout, _bindAddress) = clients[1]
  658. self.assertEqual(host, "1::f")
  659. self.assertEqual(port, 8448)
  660. # make a test server, and wire up the client
  661. http_server = self._make_connection(
  662. client_factory, expected_sni=b"target-server"
  663. )
  664. self.assertEqual(len(http_server.requests), 1)
  665. request = http_server.requests[0]
  666. self.assertEqual(request.method, b"GET")
  667. self.assertEqual(request.path, b"/foo/bar")
  668. self.assertEqual(
  669. request.requestHeaders.getRawHeaders(b"host"), [b"target-server"]
  670. )
  671. # finish the request
  672. request.finish()
  673. self.reactor.pump((0.1,))
  674. self.successResultOf(test_d)
  675. self.assertEqual(self.well_known_cache[b"testserv"], b"target-server")
  676. # check the cache expires
  677. self.reactor.pump((48 * 3600,))
  678. self.well_known_cache.expire()
  679. self.assertNotIn(b"testserv", self.well_known_cache)
  680. def test_get_well_known_redirect(self) -> None:
  681. """Test the behaviour when the server name has no port and no SRV record, but
  682. the .well-known has a 300 redirect
  683. """
  684. self.agent = self._make_agent()
  685. self.mock_resolver.resolve_service.side_effect = generate_resolve_service([])
  686. self.reactor.lookups["testserv"] = "1.2.3.4"
  687. self.reactor.lookups["target-server"] = "1::f"
  688. test_d = self._make_get_request(b"matrix://testserv/foo/bar")
  689. # Nothing happened yet
  690. self.assertNoResult(test_d)
  691. # there should be an attempt to connect on port 443 for the .well-known
  692. clients = self.reactor.tcpClients
  693. self.assertEqual(len(clients), 1)
  694. (host, port, client_factory, _timeout, _bindAddress) = clients.pop()
  695. self.assertEqual(host, "1.2.3.4")
  696. self.assertEqual(port, 443)
  697. redirect_server = self._make_connection(
  698. client_factory, expected_sni=b"testserv"
  699. )
  700. # send a 302 redirect
  701. self.assertEqual(len(redirect_server.requests), 1)
  702. request = redirect_server.requests[0]
  703. request.redirect(b"https://testserv/even_better_known")
  704. request.finish()
  705. self.reactor.pump((0.1,))
  706. # now there should be another connection
  707. clients = self.reactor.tcpClients
  708. self.assertEqual(len(clients), 1)
  709. (host, port, client_factory, _timeout, _bindAddress) = clients.pop()
  710. self.assertEqual(host, "1.2.3.4")
  711. self.assertEqual(port, 443)
  712. well_known_server = self._make_connection(
  713. client_factory, expected_sni=b"testserv"
  714. )
  715. self.assertEqual(len(well_known_server.requests), 1, "No request after 302")
  716. request = well_known_server.requests[0]
  717. self.assertEqual(request.method, b"GET")
  718. self.assertEqual(request.path, b"/even_better_known")
  719. request.write(b'{ "m.server": "target-server" }')
  720. request.finish()
  721. self.reactor.pump((0.1,))
  722. # there should be a SRV lookup
  723. self.mock_resolver.resolve_service.assert_called_once_with(
  724. b"_matrix._tcp.target-server"
  725. )
  726. # now we should get a connection to the target server
  727. self.assertEqual(len(clients), 1)
  728. (host, port, client_factory, _timeout, _bindAddress) = clients[0]
  729. self.assertEqual(host, "1::f")
  730. self.assertEqual(port, 8448)
  731. # make a test server, and wire up the client
  732. http_server = self._make_connection(
  733. client_factory, expected_sni=b"target-server"
  734. )
  735. self.assertEqual(len(http_server.requests), 1)
  736. request = http_server.requests[0]
  737. self.assertEqual(request.method, b"GET")
  738. self.assertEqual(request.path, b"/foo/bar")
  739. self.assertEqual(
  740. request.requestHeaders.getRawHeaders(b"host"), [b"target-server"]
  741. )
  742. # finish the request
  743. request.finish()
  744. self.reactor.pump((0.1,))
  745. self.successResultOf(test_d)
  746. self.assertEqual(self.well_known_cache[b"testserv"], b"target-server")
  747. # check the cache expires
  748. self.reactor.pump((48 * 3600,))
  749. self.well_known_cache.expire()
  750. self.assertNotIn(b"testserv", self.well_known_cache)
  751. def test_get_invalid_well_known(self) -> None:
  752. """
  753. Test the behaviour when the server name has an *invalid* well-known (and no SRV)
  754. """
  755. self.agent = self._make_agent()
  756. self.mock_resolver.resolve_service.side_effect = generate_resolve_service([])
  757. self.reactor.lookups["testserv"] = "1.2.3.4"
  758. test_d = self._make_get_request(b"matrix://testserv/foo/bar")
  759. # Nothing happened yet
  760. self.assertNoResult(test_d)
  761. # No SRV record lookup yet
  762. self.mock_resolver.resolve_service.assert_not_called()
  763. # there should be an attempt to connect on port 443 for the .well-known
  764. clients = self.reactor.tcpClients
  765. self.assertEqual(len(clients), 1)
  766. (host, port, client_factory, _timeout, _bindAddress) = clients.pop()
  767. self.assertEqual(host, "1.2.3.4")
  768. self.assertEqual(port, 443)
  769. self._handle_well_known_connection(
  770. client_factory, expected_sni=b"testserv", content=b"NOT JSON"
  771. )
  772. # now there should be a SRV lookup
  773. self.mock_resolver.resolve_service.assert_called_once_with(
  774. b"_matrix._tcp.testserv"
  775. )
  776. # we should fall back to a direct connection
  777. self.assertEqual(len(clients), 1)
  778. (host, port, client_factory, _timeout, _bindAddress) = clients.pop()
  779. self.assertEqual(host, "1.2.3.4")
  780. self.assertEqual(port, 8448)
  781. # make a test server, and wire up the client
  782. http_server = self._make_connection(client_factory, expected_sni=b"testserv")
  783. self.assertEqual(len(http_server.requests), 1)
  784. request = http_server.requests[0]
  785. self.assertEqual(request.method, b"GET")
  786. self.assertEqual(request.path, b"/foo/bar")
  787. self.assertEqual(request.requestHeaders.getRawHeaders(b"host"), [b"testserv"])
  788. # finish the request
  789. request.finish()
  790. self.reactor.pump((0.1,))
  791. self.successResultOf(test_d)
  792. def test_get_well_known_unsigned_cert(self) -> None:
  793. """Test the behaviour when the .well-known server presents a cert
  794. not signed by a CA
  795. """
  796. # we use the same test server as the other tests, but use an agent with
  797. # the config left to the default, which will not trust it (since the
  798. # presented cert is signed by a test CA)
  799. self.mock_resolver.resolve_service.side_effect = generate_resolve_service([])
  800. self.reactor.lookups["testserv"] = "1.2.3.4"
  801. config = default_config("test", parse=True)
  802. # Build a new agent and WellKnownResolver with a different tls factory
  803. tls_factory = FederationPolicyForHTTPS(config)
  804. agent = MatrixFederationAgent(
  805. reactor=self.reactor,
  806. tls_client_options_factory=tls_factory,
  807. user_agent=b"test-agent", # This is unused since _well_known_resolver is passed below.
  808. ip_whitelist=IPSet(),
  809. ip_blacklist=IPSet(),
  810. _srv_resolver=self.mock_resolver,
  811. _well_known_resolver=WellKnownResolver(
  812. cast(ISynapseReactor, self.reactor),
  813. Agent(self.reactor, contextFactory=tls_factory),
  814. b"test-agent",
  815. well_known_cache=self.well_known_cache,
  816. had_well_known_cache=self.had_well_known_cache,
  817. ),
  818. )
  819. test_d = agent.request(b"GET", b"matrix://testserv/foo/bar")
  820. # Nothing happened yet
  821. self.assertNoResult(test_d)
  822. # there should be an attempt to connect on port 443 for the .well-known
  823. clients = self.reactor.tcpClients
  824. self.assertEqual(len(clients), 1)
  825. (host, port, client_factory, _timeout, _bindAddress) = clients[0]
  826. self.assertEqual(host, "1.2.3.4")
  827. self.assertEqual(port, 443)
  828. http_proto = self._make_connection(client_factory, expected_sni=b"testserv")
  829. # there should be no requests
  830. self.assertEqual(len(http_proto.requests), 0)
  831. # and there should be a SRV lookup instead
  832. self.mock_resolver.resolve_service.assert_called_once_with(
  833. b"_matrix._tcp.testserv"
  834. )
  835. def test_get_hostname_srv(self) -> None:
  836. """
  837. Test the behaviour when there is a single SRV record
  838. """
  839. self.agent = self._make_agent()
  840. self.mock_resolver.resolve_service.side_effect = generate_resolve_service(
  841. [Server(host=b"srvtarget", port=8443)]
  842. )
  843. self.reactor.lookups["srvtarget"] = "1.2.3.4"
  844. test_d = self._make_get_request(b"matrix://testserv/foo/bar")
  845. # Nothing happened yet
  846. self.assertNoResult(test_d)
  847. # the request for a .well-known will have failed with a DNS lookup error.
  848. self.mock_resolver.resolve_service.assert_called_once_with(
  849. b"_matrix._tcp.testserv"
  850. )
  851. # Make sure treq is trying to connect
  852. clients = self.reactor.tcpClients
  853. self.assertEqual(len(clients), 1)
  854. (host, port, client_factory, _timeout, _bindAddress) = clients[0]
  855. self.assertEqual(host, "1.2.3.4")
  856. self.assertEqual(port, 8443)
  857. # make a test server, and wire up the client
  858. http_server = self._make_connection(client_factory, expected_sni=b"testserv")
  859. self.assertEqual(len(http_server.requests), 1)
  860. request = http_server.requests[0]
  861. self.assertEqual(request.method, b"GET")
  862. self.assertEqual(request.path, b"/foo/bar")
  863. self.assertEqual(request.requestHeaders.getRawHeaders(b"host"), [b"testserv"])
  864. # finish the request
  865. request.finish()
  866. self.reactor.pump((0.1,))
  867. self.successResultOf(test_d)
  868. def test_get_well_known_srv(self) -> None:
  869. """Test the behaviour when the .well-known redirects to a place where there
  870. is a SRV.
  871. """
  872. self.agent = self._make_agent()
  873. self.reactor.lookups["testserv"] = "1.2.3.4"
  874. self.reactor.lookups["srvtarget"] = "5.6.7.8"
  875. test_d = self._make_get_request(b"matrix://testserv/foo/bar")
  876. # Nothing happened yet
  877. self.assertNoResult(test_d)
  878. # there should be an attempt to connect on port 443 for the .well-known
  879. clients = self.reactor.tcpClients
  880. self.assertEqual(len(clients), 1)
  881. (host, port, client_factory, _timeout, _bindAddress) = clients[0]
  882. self.assertEqual(host, "1.2.3.4")
  883. self.assertEqual(port, 443)
  884. self.mock_resolver.resolve_service.side_effect = generate_resolve_service(
  885. [Server(host=b"srvtarget", port=8443)]
  886. )
  887. self._handle_well_known_connection(
  888. client_factory,
  889. expected_sni=b"testserv",
  890. content=b'{ "m.server": "target-server" }',
  891. )
  892. # there should be a SRV lookup
  893. self.mock_resolver.resolve_service.assert_called_once_with(
  894. b"_matrix._tcp.target-server"
  895. )
  896. # now we should get a connection to the target of the SRV record
  897. self.assertEqual(len(clients), 2)
  898. (host, port, client_factory, _timeout, _bindAddress) = clients[1]
  899. self.assertEqual(host, "5.6.7.8")
  900. self.assertEqual(port, 8443)
  901. # make a test server, and wire up the client
  902. http_server = self._make_connection(
  903. client_factory, expected_sni=b"target-server"
  904. )
  905. self.assertEqual(len(http_server.requests), 1)
  906. request = http_server.requests[0]
  907. self.assertEqual(request.method, b"GET")
  908. self.assertEqual(request.path, b"/foo/bar")
  909. self.assertEqual(
  910. request.requestHeaders.getRawHeaders(b"host"), [b"target-server"]
  911. )
  912. # finish the request
  913. request.finish()
  914. self.reactor.pump((0.1,))
  915. self.successResultOf(test_d)
  916. def test_idna_servername(self) -> None:
  917. """test the behaviour when the server name has idna chars in"""
  918. self.agent = self._make_agent()
  919. self.mock_resolver.resolve_service.side_effect = generate_resolve_service([])
  920. # the resolver is always called with the IDNA hostname as a native string.
  921. self.reactor.lookups["xn--bcher-kva.com"] = "1.2.3.4"
  922. # this is idna for bücher.com
  923. test_d = self._make_get_request(b"matrix://xn--bcher-kva.com/foo/bar")
  924. # Nothing happened yet
  925. self.assertNoResult(test_d)
  926. # No SRV record lookup yet
  927. self.mock_resolver.resolve_service.assert_not_called()
  928. # there should be an attempt to connect on port 443 for the .well-known
  929. clients = self.reactor.tcpClients
  930. self.assertEqual(len(clients), 1)
  931. (host, port, client_factory, _timeout, _bindAddress) = clients[0]
  932. self.assertEqual(host, "1.2.3.4")
  933. self.assertEqual(port, 443)
  934. # fonx the connection
  935. client_factory.clientConnectionFailed(None, Exception("nope"))
  936. # attemptdelay on the hostnameendpoint is 0.3, so takes that long before the
  937. # .well-known request fails.
  938. self.reactor.pump((0.4,))
  939. # now there should have been a SRV lookup
  940. self.mock_resolver.resolve_service.assert_called_once_with(
  941. b"_matrix._tcp.xn--bcher-kva.com"
  942. )
  943. # We should fall back to port 8448
  944. clients = self.reactor.tcpClients
  945. self.assertEqual(len(clients), 2)
  946. (host, port, client_factory, _timeout, _bindAddress) = clients[1]
  947. self.assertEqual(host, "1.2.3.4")
  948. self.assertEqual(port, 8448)
  949. # make a test server, and wire up the client
  950. http_server = self._make_connection(
  951. client_factory, expected_sni=b"xn--bcher-kva.com"
  952. )
  953. self.assertEqual(len(http_server.requests), 1)
  954. request = http_server.requests[0]
  955. self.assertEqual(request.method, b"GET")
  956. self.assertEqual(request.path, b"/foo/bar")
  957. self.assertEqual(
  958. request.requestHeaders.getRawHeaders(b"host"), [b"xn--bcher-kva.com"]
  959. )
  960. # finish the request
  961. request.finish()
  962. self.reactor.pump((0.1,))
  963. self.successResultOf(test_d)
  964. def test_idna_srv_target(self) -> None:
  965. """test the behaviour when the target of a SRV record has idna chars"""
  966. self.agent = self._make_agent()
  967. self.mock_resolver.resolve_service.side_effect = generate_resolve_service(
  968. [Server(host=b"xn--trget-3qa.com", port=8443)] # târget.com
  969. )
  970. self.reactor.lookups["xn--trget-3qa.com"] = "1.2.3.4"
  971. test_d = self._make_get_request(b"matrix://xn--bcher-kva.com/foo/bar")
  972. # Nothing happened yet
  973. self.assertNoResult(test_d)
  974. self.mock_resolver.resolve_service.assert_called_once_with(
  975. b"_matrix._tcp.xn--bcher-kva.com"
  976. )
  977. # Make sure treq is trying to connect
  978. clients = self.reactor.tcpClients
  979. self.assertEqual(len(clients), 1)
  980. (host, port, client_factory, _timeout, _bindAddress) = clients[0]
  981. self.assertEqual(host, "1.2.3.4")
  982. self.assertEqual(port, 8443)
  983. # make a test server, and wire up the client
  984. http_server = self._make_connection(
  985. client_factory, expected_sni=b"xn--bcher-kva.com"
  986. )
  987. self.assertEqual(len(http_server.requests), 1)
  988. request = http_server.requests[0]
  989. self.assertEqual(request.method, b"GET")
  990. self.assertEqual(request.path, b"/foo/bar")
  991. self.assertEqual(
  992. request.requestHeaders.getRawHeaders(b"host"), [b"xn--bcher-kva.com"]
  993. )
  994. # finish the request
  995. request.finish()
  996. self.reactor.pump((0.1,))
  997. self.successResultOf(test_d)
  998. def test_well_known_cache(self) -> None:
  999. self.reactor.lookups["testserv"] = "1.2.3.4"
  1000. fetch_d = defer.ensureDeferred(
  1001. self.well_known_resolver.get_well_known(b"testserv")
  1002. )
  1003. # there should be an attempt to connect on port 443 for the .well-known
  1004. clients = self.reactor.tcpClients
  1005. self.assertEqual(len(clients), 1)
  1006. (host, port, client_factory, _timeout, _bindAddress) = clients.pop(0)
  1007. self.assertEqual(host, "1.2.3.4")
  1008. self.assertEqual(port, 443)
  1009. well_known_server = self._handle_well_known_connection(
  1010. client_factory,
  1011. expected_sni=b"testserv",
  1012. response_headers={b"Cache-Control": b"max-age=1000"},
  1013. content=b'{ "m.server": "target-server" }',
  1014. )
  1015. r = self.successResultOf(fetch_d)
  1016. self.assertEqual(r.delegated_server, b"target-server")
  1017. # close the tcp connection
  1018. well_known_server.loseConnection()
  1019. # repeat the request: it should hit the cache
  1020. fetch_d = defer.ensureDeferred(
  1021. self.well_known_resolver.get_well_known(b"testserv")
  1022. )
  1023. r = self.successResultOf(fetch_d)
  1024. self.assertEqual(r.delegated_server, b"target-server")
  1025. # expire the cache
  1026. self.reactor.pump((1000.0,))
  1027. # now it should connect again
  1028. fetch_d = defer.ensureDeferred(
  1029. self.well_known_resolver.get_well_known(b"testserv")
  1030. )
  1031. self.assertEqual(len(clients), 1)
  1032. (host, port, client_factory, _timeout, _bindAddress) = clients.pop(0)
  1033. self.assertEqual(host, "1.2.3.4")
  1034. self.assertEqual(port, 443)
  1035. self._handle_well_known_connection(
  1036. client_factory,
  1037. expected_sni=b"testserv",
  1038. content=b'{ "m.server": "other-server" }',
  1039. )
  1040. r = self.successResultOf(fetch_d)
  1041. self.assertEqual(r.delegated_server, b"other-server")
  1042. def test_well_known_cache_with_temp_failure(self) -> None:
  1043. """Test that we refetch well-known before the cache expires, and that
  1044. it ignores transient errors.
  1045. """
  1046. self.reactor.lookups["testserv"] = "1.2.3.4"
  1047. fetch_d = defer.ensureDeferred(
  1048. self.well_known_resolver.get_well_known(b"testserv")
  1049. )
  1050. # there should be an attempt to connect on port 443 for the .well-known
  1051. clients = self.reactor.tcpClients
  1052. self.assertEqual(len(clients), 1)
  1053. (host, port, client_factory, _timeout, _bindAddress) = clients.pop(0)
  1054. self.assertEqual(host, "1.2.3.4")
  1055. self.assertEqual(port, 443)
  1056. well_known_server = self._handle_well_known_connection(
  1057. client_factory,
  1058. expected_sni=b"testserv",
  1059. response_headers={b"Cache-Control": b"max-age=1000"},
  1060. content=b'{ "m.server": "target-server" }',
  1061. )
  1062. r = self.successResultOf(fetch_d)
  1063. self.assertEqual(r.delegated_server, b"target-server")
  1064. # close the tcp connection
  1065. well_known_server.loseConnection()
  1066. # Get close to the cache expiry, this will cause the resolver to do
  1067. # another lookup.
  1068. self.reactor.pump((900.0,))
  1069. fetch_d = defer.ensureDeferred(
  1070. self.well_known_resolver.get_well_known(b"testserv")
  1071. )
  1072. # The resolver may retry a few times, so fonx all requests that come along
  1073. attempts = 0
  1074. while self.reactor.tcpClients:
  1075. clients = self.reactor.tcpClients
  1076. (host, port, client_factory, _timeout, _bindAddress) = clients.pop(0)
  1077. attempts += 1
  1078. # fonx the connection attempt, this will be treated as a temporary
  1079. # failure.
  1080. client_factory.clientConnectionFailed(None, Exception("nope"))
  1081. # There's a few sleeps involved, so we have to pump the reactor a
  1082. # bit.
  1083. self.reactor.pump((1.0, 1.0))
  1084. # We expect to see more than one attempt as there was previously a valid
  1085. # well known.
  1086. self.assertGreater(attempts, 1)
  1087. # Resolver should return cached value, despite the lookup failing.
  1088. r = self.successResultOf(fetch_d)
  1089. self.assertEqual(r.delegated_server, b"target-server")
  1090. # Expire both caches and repeat the request
  1091. self.reactor.pump((10000.0,))
  1092. # Repeat the request, this time it should fail if the lookup fails.
  1093. fetch_d = defer.ensureDeferred(
  1094. self.well_known_resolver.get_well_known(b"testserv")
  1095. )
  1096. clients = self.reactor.tcpClients
  1097. (host, port, client_factory, _timeout, _bindAddress) = clients.pop(0)
  1098. client_factory.clientConnectionFailed(None, Exception("nope"))
  1099. self.reactor.pump((0.4,))
  1100. r = self.successResultOf(fetch_d)
  1101. self.assertEqual(r.delegated_server, None)
  1102. def test_well_known_too_large(self) -> None:
  1103. """A well-known query that returns a result which is too large should be rejected."""
  1104. self.reactor.lookups["testserv"] = "1.2.3.4"
  1105. fetch_d = defer.ensureDeferred(
  1106. self.well_known_resolver.get_well_known(b"testserv")
  1107. )
  1108. # there should be an attempt to connect on port 443 for the .well-known
  1109. clients = self.reactor.tcpClients
  1110. self.assertEqual(len(clients), 1)
  1111. (host, port, client_factory, _timeout, _bindAddress) = clients.pop(0)
  1112. self.assertEqual(host, "1.2.3.4")
  1113. self.assertEqual(port, 443)
  1114. self._handle_well_known_connection(
  1115. client_factory,
  1116. expected_sni=b"testserv",
  1117. response_headers={b"Cache-Control": b"max-age=1000"},
  1118. content=b'{ "m.server": "' + (b"a" * WELL_KNOWN_MAX_SIZE) + b'" }',
  1119. )
  1120. # The result is successful, but disabled delegation.
  1121. r = self.successResultOf(fetch_d)
  1122. self.assertIsNone(r.delegated_server)
  1123. def test_srv_fallbacks(self) -> None:
  1124. """Test that other SRV results are tried if the first one fails."""
  1125. self.agent = self._make_agent()
  1126. self.mock_resolver.resolve_service.side_effect = generate_resolve_service(
  1127. [
  1128. Server(host=b"target.com", port=8443),
  1129. Server(host=b"target.com", port=8444),
  1130. ]
  1131. )
  1132. self.reactor.lookups["target.com"] = "1.2.3.4"
  1133. test_d = self._make_get_request(b"matrix://testserv/foo/bar")
  1134. # Nothing happened yet
  1135. self.assertNoResult(test_d)
  1136. self.mock_resolver.resolve_service.assert_called_once_with(
  1137. b"_matrix._tcp.testserv"
  1138. )
  1139. # We should see an attempt to connect to the first server
  1140. clients = self.reactor.tcpClients
  1141. self.assertEqual(len(clients), 1)
  1142. (host, port, client_factory, _timeout, _bindAddress) = clients.pop(0)
  1143. self.assertEqual(host, "1.2.3.4")
  1144. self.assertEqual(port, 8443)
  1145. # Fonx the connection
  1146. client_factory.clientConnectionFailed(None, Exception("nope"))
  1147. # There's a 300ms delay in HostnameEndpoint
  1148. self.reactor.pump((0.4,))
  1149. # Hasn't failed yet
  1150. self.assertNoResult(test_d)
  1151. # We shouldnow see an attempt to connect to the second server
  1152. clients = self.reactor.tcpClients
  1153. self.assertEqual(len(clients), 1)
  1154. (host, port, client_factory, _timeout, _bindAddress) = clients.pop(0)
  1155. self.assertEqual(host, "1.2.3.4")
  1156. self.assertEqual(port, 8444)
  1157. # make a test server, and wire up the client
  1158. http_server = self._make_connection(client_factory, expected_sni=b"testserv")
  1159. self.assertEqual(len(http_server.requests), 1)
  1160. request = http_server.requests[0]
  1161. self.assertEqual(request.method, b"GET")
  1162. self.assertEqual(request.path, b"/foo/bar")
  1163. self.assertEqual(request.requestHeaders.getRawHeaders(b"host"), [b"testserv"])
  1164. # finish the request
  1165. request.finish()
  1166. self.reactor.pump((0.1,))
  1167. self.successResultOf(test_d)
  1168. class TestCachePeriodFromHeaders(unittest.TestCase):
  1169. def test_cache_control(self) -> None:
  1170. # uppercase
  1171. self.assertEqual(
  1172. _cache_period_from_headers(
  1173. Headers({b"Cache-Control": [b"foo, Max-Age = 100, bar"]})
  1174. ),
  1175. 100,
  1176. )
  1177. # missing value
  1178. self.assertIsNone(
  1179. _cache_period_from_headers(Headers({b"Cache-Control": [b"max-age=, bar"]}))
  1180. )
  1181. # hackernews: bogus due to semicolon
  1182. self.assertIsNone(
  1183. _cache_period_from_headers(
  1184. Headers({b"Cache-Control": [b"private; max-age=0"]})
  1185. )
  1186. )
  1187. # github
  1188. self.assertEqual(
  1189. _cache_period_from_headers(
  1190. Headers({b"Cache-Control": [b"max-age=0, private, must-revalidate"]})
  1191. ),
  1192. 0,
  1193. )
  1194. # google
  1195. self.assertEqual(
  1196. _cache_period_from_headers(
  1197. Headers({b"cache-control": [b"private, max-age=0"]})
  1198. ),
  1199. 0,
  1200. )
  1201. def test_expires(self) -> None:
  1202. self.assertEqual(
  1203. _cache_period_from_headers(
  1204. Headers({b"Expires": [b"Wed, 30 Jan 2019 07:35:33 GMT"]}),
  1205. time_now=lambda: 1548833700,
  1206. ),
  1207. 33,
  1208. )
  1209. # cache-control overrides expires
  1210. self.assertEqual(
  1211. _cache_period_from_headers(
  1212. Headers(
  1213. {
  1214. b"cache-control": [b"max-age=10"],
  1215. b"Expires": [b"Wed, 30 Jan 2019 07:35:33 GMT"],
  1216. }
  1217. ),
  1218. time_now=lambda: 1548833700,
  1219. ),
  1220. 10,
  1221. )
  1222. # invalid expires means immediate expiry
  1223. self.assertEqual(_cache_period_from_headers(Headers({b"Expires": [b"0"]})), 0)
  1224. def _check_logcontext(context: LoggingContextOrSentinel) -> None:
  1225. current = current_context()
  1226. if current is not context:
  1227. raise AssertionError("Expected logcontext %s but was %s" % (context, current))
  1228. def _wrap_server_factory_for_tls(
  1229. factory: IProtocolFactory, sanlist: Optional[List[bytes]] = None
  1230. ) -> TLSMemoryBIOFactory:
  1231. """Wrap an existing Protocol Factory with a test TLSMemoryBIOFactory
  1232. The resultant factory will create a TLS server which presents a certificate
  1233. signed by our test CA, valid for the domains in `sanlist`
  1234. Args:
  1235. factory: protocol factory to wrap
  1236. sanlist: list of domains the cert should be valid for
  1237. Returns:
  1238. interfaces.IProtocolFactory
  1239. """
  1240. if sanlist is None:
  1241. sanlist = [
  1242. b"DNS:testserv",
  1243. b"DNS:target-server",
  1244. b"DNS:xn--bcher-kva.com",
  1245. b"IP:1.2.3.4",
  1246. b"IP:::1",
  1247. ]
  1248. connection_creator = TestServerTLSConnectionFactory(sanlist=sanlist)
  1249. return TLSMemoryBIOFactory(
  1250. connection_creator, isClient=False, wrappedFactory=factory
  1251. )
  1252. def _get_test_protocol_factory() -> IProtocolFactory:
  1253. """Get a protocol Factory which will build an HTTPChannel
  1254. Returns:
  1255. interfaces.IProtocolFactory
  1256. """
  1257. server_factory = Factory.forProtocol(HTTPChannel)
  1258. # Request.finish expects the factory to have a 'log' method.
  1259. server_factory.log = _log_request
  1260. return server_factory
  1261. def _log_request(request: str) -> None:
  1262. """Implements Factory.log, which is expected by Request.finish"""
  1263. logger.info(f"Completed request {request}")
  1264. @implementer(IPolicyForHTTPS)
  1265. class TrustingTLSPolicyForHTTPS:
  1266. """An IPolicyForHTTPS which checks that the certificate belongs to the
  1267. right server, but doesn't check the certificate chain."""
  1268. def creatorForNetloc(
  1269. self, hostname: bytes, port: int
  1270. ) -> IOpenSSLClientConnectionCreator:
  1271. certificateOptions = OpenSSLCertificateOptions()
  1272. return ClientTLSOptions(hostname, certificateOptions.getContext())