1
0

test_proxyagent.py 33 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892
  1. # Copyright 2019 The Matrix.org Foundation C.I.C.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. import base64
  15. import logging
  16. import os
  17. from typing import List, Optional
  18. from unittest.mock import patch
  19. import treq
  20. from netaddr import IPSet
  21. from parameterized import parameterized
  22. from twisted.internet import interfaces # noqa: F401
  23. from twisted.internet.endpoints import (
  24. HostnameEndpoint,
  25. _WrapperEndpoint,
  26. _WrappingProtocol,
  27. )
  28. from twisted.internet.interfaces import IProtocol, IProtocolFactory
  29. from twisted.internet.protocol import Factory, Protocol
  30. from twisted.protocols.tls import TLSMemoryBIOFactory, TLSMemoryBIOProtocol
  31. from twisted.web.http import HTTPChannel
  32. from synapse.http.client import BlacklistingReactorWrapper
  33. from synapse.http.connectproxyclient import ProxyCredentials
  34. from synapse.http.proxyagent import ProxyAgent, parse_proxy
  35. from tests.http import (
  36. TestServerTLSConnectionFactory,
  37. dummy_address,
  38. get_test_https_policy,
  39. )
  40. from tests.server import FakeTransport, ThreadedMemoryReactorClock
  41. from tests.unittest import TestCase
  42. from tests.utils import checked_cast
  43. logger = logging.getLogger(__name__)
  44. HTTPFactory = Factory.forProtocol(HTTPChannel)
  45. class ProxyParserTests(TestCase):
  46. """
  47. Values for test
  48. [
  49. proxy_string,
  50. expected_scheme,
  51. expected_hostname,
  52. expected_port,
  53. expected_credentials,
  54. ]
  55. """
  56. @parameterized.expand(
  57. [
  58. # host
  59. [b"localhost", b"http", b"localhost", 1080, None],
  60. [b"localhost:9988", b"http", b"localhost", 9988, None],
  61. # host+scheme
  62. [b"https://localhost", b"https", b"localhost", 1080, None],
  63. [b"https://localhost:1234", b"https", b"localhost", 1234, None],
  64. # ipv4
  65. [b"1.2.3.4", b"http", b"1.2.3.4", 1080, None],
  66. [b"1.2.3.4:9988", b"http", b"1.2.3.4", 9988, None],
  67. # ipv4+scheme
  68. [b"https://1.2.3.4", b"https", b"1.2.3.4", 1080, None],
  69. [b"https://1.2.3.4:9988", b"https", b"1.2.3.4", 9988, None],
  70. # ipv6 - without brackets is broken
  71. # [
  72. # b"2001:0db8:85a3:0000:0000:8a2e:0370:effe",
  73. # b"http",
  74. # b"2001:0db8:85a3:0000:0000:8a2e:0370:effe",
  75. # 1080,
  76. # None,
  77. # ],
  78. # [
  79. # b"2001:0db8:85a3:0000:0000:8a2e:0370:1234",
  80. # b"http",
  81. # b"2001:0db8:85a3:0000:0000:8a2e:0370:1234",
  82. # 1080,
  83. # None,
  84. # ],
  85. # [b"::1", b"http", b"::1", 1080, None],
  86. # [b"::ffff:0.0.0.0", b"http", b"::ffff:0.0.0.0", 1080, None],
  87. # ipv6 - with brackets
  88. [
  89. b"[2001:0db8:85a3:0000:0000:8a2e:0370:effe]",
  90. b"http",
  91. b"2001:0db8:85a3:0000:0000:8a2e:0370:effe",
  92. 1080,
  93. None,
  94. ],
  95. [
  96. b"[2001:0db8:85a3:0000:0000:8a2e:0370:1234]",
  97. b"http",
  98. b"2001:0db8:85a3:0000:0000:8a2e:0370:1234",
  99. 1080,
  100. None,
  101. ],
  102. [b"[::1]", b"http", b"::1", 1080, None],
  103. [b"[::ffff:0.0.0.0]", b"http", b"::ffff:0.0.0.0", 1080, None],
  104. # ipv6+port
  105. [
  106. b"[2001:0db8:85a3:0000:0000:8a2e:0370:effe]:9988",
  107. b"http",
  108. b"2001:0db8:85a3:0000:0000:8a2e:0370:effe",
  109. 9988,
  110. None,
  111. ],
  112. [
  113. b"[2001:0db8:85a3:0000:0000:8a2e:0370:1234]:9988",
  114. b"http",
  115. b"2001:0db8:85a3:0000:0000:8a2e:0370:1234",
  116. 9988,
  117. None,
  118. ],
  119. [b"[::1]:9988", b"http", b"::1", 9988, None],
  120. [b"[::ffff:0.0.0.0]:9988", b"http", b"::ffff:0.0.0.0", 9988, None],
  121. # ipv6+scheme
  122. [
  123. b"https://[2001:0db8:85a3:0000:0000:8a2e:0370:effe]",
  124. b"https",
  125. b"2001:0db8:85a3:0000:0000:8a2e:0370:effe",
  126. 1080,
  127. None,
  128. ],
  129. [
  130. b"https://[2001:0db8:85a3:0000:0000:8a2e:0370:1234]",
  131. b"https",
  132. b"2001:0db8:85a3:0000:0000:8a2e:0370:1234",
  133. 1080,
  134. None,
  135. ],
  136. [b"https://[::1]", b"https", b"::1", 1080, None],
  137. [b"https://[::ffff:0.0.0.0]", b"https", b"::ffff:0.0.0.0", 1080, None],
  138. # ipv6+scheme+port
  139. [
  140. b"https://[2001:0db8:85a3:0000:0000:8a2e:0370:effe]:9988",
  141. b"https",
  142. b"2001:0db8:85a3:0000:0000:8a2e:0370:effe",
  143. 9988,
  144. None,
  145. ],
  146. [
  147. b"https://[2001:0db8:85a3:0000:0000:8a2e:0370:1234]:9988",
  148. b"https",
  149. b"2001:0db8:85a3:0000:0000:8a2e:0370:1234",
  150. 9988,
  151. None,
  152. ],
  153. [b"https://[::1]:9988", b"https", b"::1", 9988, None],
  154. # with credentials
  155. [
  156. b"https://user:pass@1.2.3.4:9988",
  157. b"https",
  158. b"1.2.3.4",
  159. 9988,
  160. b"user:pass",
  161. ],
  162. [b"user:pass@1.2.3.4:9988", b"http", b"1.2.3.4", 9988, b"user:pass"],
  163. [
  164. b"https://user:pass@proxy.local:9988",
  165. b"https",
  166. b"proxy.local",
  167. 9988,
  168. b"user:pass",
  169. ],
  170. [
  171. b"user:pass@proxy.local:9988",
  172. b"http",
  173. b"proxy.local",
  174. 9988,
  175. b"user:pass",
  176. ],
  177. ]
  178. )
  179. def test_parse_proxy(
  180. self,
  181. proxy_string: bytes,
  182. expected_scheme: bytes,
  183. expected_hostname: bytes,
  184. expected_port: int,
  185. expected_credentials: Optional[bytes],
  186. ) -> None:
  187. """
  188. Tests that a given proxy URL will be broken into the components.
  189. Args:
  190. proxy_string: The proxy connection string.
  191. expected_scheme: Expected value of proxy scheme.
  192. expected_hostname: Expected value of proxy hostname.
  193. expected_port: Expected value of proxy port.
  194. expected_credentials: Expected value of credentials.
  195. Must be in form '<username>:<password>' or None
  196. """
  197. proxy_cred = None
  198. if expected_credentials:
  199. proxy_cred = ProxyCredentials(expected_credentials)
  200. self.assertEqual(
  201. (
  202. expected_scheme,
  203. expected_hostname,
  204. expected_port,
  205. proxy_cred,
  206. ),
  207. parse_proxy(proxy_string),
  208. )
  209. class MatrixFederationAgentTests(TestCase):
  210. def setUp(self) -> None:
  211. self.reactor = ThreadedMemoryReactorClock()
  212. def _make_connection(
  213. self,
  214. client_factory: IProtocolFactory,
  215. server_factory: IProtocolFactory,
  216. ssl: bool = False,
  217. expected_sni: Optional[bytes] = None,
  218. tls_sanlist: Optional[List[bytes]] = None,
  219. ) -> IProtocol:
  220. """Builds a test server, and completes the outgoing client connection
  221. Args:
  222. client_factory: the the factory that the
  223. application is trying to use to make the outbound connection. We will
  224. invoke it to build the client Protocol
  225. server_factory: a factory to build the
  226. server-side protocol
  227. ssl: If true, we will expect an ssl connection and wrap
  228. server_factory with a TLSMemoryBIOFactory
  229. expected_sni: the expected SNI value
  230. tls_sanlist: list of SAN entries for the TLS cert presented by the server.
  231. Defaults to [b'DNS:test.com']
  232. Returns:
  233. the server Protocol returned by server_factory
  234. """
  235. if ssl:
  236. server_factory = _wrap_server_factory_for_tls(server_factory, tls_sanlist)
  237. server_protocol = server_factory.buildProtocol(dummy_address)
  238. assert server_protocol is not None
  239. # now, tell the client protocol factory to build the client protocol,
  240. # and wire the output of said protocol up to the server via
  241. # a FakeTransport.
  242. #
  243. # Normally this would be done by the TCP socket code in Twisted, but we are
  244. # stubbing that out here.
  245. client_protocol = client_factory.buildProtocol(dummy_address)
  246. assert client_protocol is not None
  247. client_protocol.makeConnection(
  248. FakeTransport(server_protocol, self.reactor, client_protocol)
  249. )
  250. # tell the server protocol to send its stuff back to the client, too
  251. server_protocol.makeConnection(
  252. FakeTransport(client_protocol, self.reactor, server_protocol)
  253. )
  254. if ssl:
  255. assert isinstance(server_protocol, TLSMemoryBIOProtocol)
  256. http_protocol = server_protocol.wrappedProtocol
  257. tls_connection = server_protocol._tlsConnection
  258. else:
  259. http_protocol = server_protocol
  260. tls_connection = None
  261. # give the reactor a pump to get the TLS juices flowing (if needed)
  262. self.reactor.advance(0)
  263. if expected_sni is not None:
  264. server_name = tls_connection.get_servername()
  265. self.assertEqual(
  266. server_name,
  267. expected_sni,
  268. f"Expected SNI {expected_sni!s} but got {server_name!s}",
  269. )
  270. return http_protocol
  271. def _test_request_direct_connection(
  272. self,
  273. agent: ProxyAgent,
  274. scheme: bytes,
  275. hostname: bytes,
  276. path: bytes,
  277. ) -> None:
  278. """Runs a test case for a direct connection not going through a proxy.
  279. Args:
  280. agent: the proxy agent being tested
  281. scheme: expected to be either "http" or "https"
  282. hostname: the hostname to connect to in the test
  283. path: the path to connect to in the test
  284. """
  285. is_https = scheme == b"https"
  286. self.reactor.lookups[hostname.decode()] = "1.2.3.4"
  287. d = agent.request(b"GET", scheme + b"://" + hostname + b"/" + path)
  288. # there should be a pending TCP connection
  289. clients = self.reactor.tcpClients
  290. self.assertEqual(len(clients), 1)
  291. (host, port, client_factory, _timeout, _bindAddress) = clients[0]
  292. self.assertEqual(host, "1.2.3.4")
  293. self.assertEqual(port, 443 if is_https else 80)
  294. # make a test server, and wire up the client
  295. http_server = self._make_connection(
  296. client_factory,
  297. _get_test_protocol_factory(),
  298. ssl=is_https,
  299. expected_sni=hostname if is_https else None,
  300. )
  301. assert isinstance(http_server, HTTPChannel)
  302. # the FakeTransport is async, so we need to pump the reactor
  303. self.reactor.advance(0)
  304. # now there should be a pending request
  305. self.assertEqual(len(http_server.requests), 1)
  306. request = http_server.requests[0]
  307. self.assertEqual(request.method, b"GET")
  308. self.assertEqual(request.path, b"/" + path)
  309. self.assertEqual(request.requestHeaders.getRawHeaders(b"host"), [hostname])
  310. request.write(b"result")
  311. request.finish()
  312. self.reactor.advance(0)
  313. resp = self.successResultOf(d)
  314. body = self.successResultOf(treq.content(resp))
  315. self.assertEqual(body, b"result")
  316. def test_http_request(self) -> None:
  317. agent = ProxyAgent(self.reactor)
  318. self._test_request_direct_connection(agent, b"http", b"test.com", b"")
  319. def test_https_request(self) -> None:
  320. agent = ProxyAgent(self.reactor, contextFactory=get_test_https_policy())
  321. self._test_request_direct_connection(agent, b"https", b"test.com", b"abc")
  322. def test_http_request_use_proxy_empty_environment(self) -> None:
  323. agent = ProxyAgent(self.reactor, use_proxy=True)
  324. self._test_request_direct_connection(agent, b"http", b"test.com", b"")
  325. @patch.dict(os.environ, {"http_proxy": "proxy.com:8888", "NO_PROXY": "test.com"})
  326. def test_http_request_via_uppercase_no_proxy(self) -> None:
  327. agent = ProxyAgent(self.reactor, use_proxy=True)
  328. self._test_request_direct_connection(agent, b"http", b"test.com", b"")
  329. @patch.dict(
  330. os.environ, {"http_proxy": "proxy.com:8888", "no_proxy": "test.com,unused.com"}
  331. )
  332. def test_http_request_via_no_proxy(self) -> None:
  333. agent = ProxyAgent(self.reactor, use_proxy=True)
  334. self._test_request_direct_connection(agent, b"http", b"test.com", b"")
  335. @patch.dict(
  336. os.environ, {"https_proxy": "proxy.com", "no_proxy": "test.com,unused.com"}
  337. )
  338. def test_https_request_via_no_proxy(self) -> None:
  339. agent = ProxyAgent(
  340. self.reactor,
  341. contextFactory=get_test_https_policy(),
  342. use_proxy=True,
  343. )
  344. self._test_request_direct_connection(agent, b"https", b"test.com", b"abc")
  345. @patch.dict(os.environ, {"http_proxy": "proxy.com:8888", "no_proxy": "*"})
  346. def test_http_request_via_no_proxy_star(self) -> None:
  347. agent = ProxyAgent(self.reactor, use_proxy=True)
  348. self._test_request_direct_connection(agent, b"http", b"test.com", b"")
  349. @patch.dict(os.environ, {"https_proxy": "proxy.com", "no_proxy": "*"})
  350. def test_https_request_via_no_proxy_star(self) -> None:
  351. agent = ProxyAgent(
  352. self.reactor,
  353. contextFactory=get_test_https_policy(),
  354. use_proxy=True,
  355. )
  356. self._test_request_direct_connection(agent, b"https", b"test.com", b"abc")
  357. @patch.dict(os.environ, {"http_proxy": "proxy.com:8888", "no_proxy": "unused.com"})
  358. def test_http_request_via_proxy(self) -> None:
  359. """
  360. Tests that requests can be made through a proxy.
  361. """
  362. self._do_http_request_via_proxy(
  363. expect_proxy_ssl=False, expected_auth_credentials=None
  364. )
  365. @patch.dict(
  366. os.environ,
  367. {"http_proxy": "bob:pinkponies@proxy.com:8888", "no_proxy": "unused.com"},
  368. )
  369. def test_http_request_via_proxy_with_auth(self) -> None:
  370. """
  371. Tests that authenticated requests can be made through a proxy.
  372. """
  373. self._do_http_request_via_proxy(
  374. expect_proxy_ssl=False, expected_auth_credentials=b"bob:pinkponies"
  375. )
  376. @patch.dict(
  377. os.environ, {"http_proxy": "https://proxy.com:8888", "no_proxy": "unused.com"}
  378. )
  379. def test_http_request_via_https_proxy(self) -> None:
  380. self._do_http_request_via_proxy(
  381. expect_proxy_ssl=True, expected_auth_credentials=None
  382. )
  383. @patch.dict(
  384. os.environ,
  385. {
  386. "http_proxy": "https://bob:pinkponies@proxy.com:8888",
  387. "no_proxy": "unused.com",
  388. },
  389. )
  390. def test_http_request_via_https_proxy_with_auth(self) -> None:
  391. self._do_http_request_via_proxy(
  392. expect_proxy_ssl=True, expected_auth_credentials=b"bob:pinkponies"
  393. )
  394. @patch.dict(os.environ, {"https_proxy": "proxy.com", "no_proxy": "unused.com"})
  395. def test_https_request_via_proxy(self) -> None:
  396. """Tests that TLS-encrypted requests can be made through a proxy"""
  397. self._do_https_request_via_proxy(
  398. expect_proxy_ssl=False, expected_auth_credentials=None
  399. )
  400. @patch.dict(
  401. os.environ,
  402. {"https_proxy": "bob:pinkponies@proxy.com", "no_proxy": "unused.com"},
  403. )
  404. def test_https_request_via_proxy_with_auth(self) -> None:
  405. """Tests that authenticated, TLS-encrypted requests can be made through a proxy"""
  406. self._do_https_request_via_proxy(
  407. expect_proxy_ssl=False, expected_auth_credentials=b"bob:pinkponies"
  408. )
  409. @patch.dict(
  410. os.environ, {"https_proxy": "https://proxy.com", "no_proxy": "unused.com"}
  411. )
  412. def test_https_request_via_https_proxy(self) -> None:
  413. """Tests that TLS-encrypted requests can be made through a proxy"""
  414. self._do_https_request_via_proxy(
  415. expect_proxy_ssl=True, expected_auth_credentials=None
  416. )
  417. @patch.dict(
  418. os.environ,
  419. {"https_proxy": "https://bob:pinkponies@proxy.com", "no_proxy": "unused.com"},
  420. )
  421. def test_https_request_via_https_proxy_with_auth(self) -> None:
  422. """Tests that authenticated, TLS-encrypted requests can be made through a proxy"""
  423. self._do_https_request_via_proxy(
  424. expect_proxy_ssl=True, expected_auth_credentials=b"bob:pinkponies"
  425. )
  426. def _do_http_request_via_proxy(
  427. self,
  428. expect_proxy_ssl: bool = False,
  429. expected_auth_credentials: Optional[bytes] = None,
  430. ) -> None:
  431. """Send a http request via an agent and check that it is correctly received at
  432. the proxy. The proxy can use either http or https.
  433. Args:
  434. expect_proxy_ssl: True if we expect the request to connect via https to proxy
  435. expected_auth_credentials: credentials to authenticate at proxy
  436. """
  437. if expect_proxy_ssl:
  438. agent = ProxyAgent(
  439. self.reactor, use_proxy=True, contextFactory=get_test_https_policy()
  440. )
  441. else:
  442. agent = ProxyAgent(self.reactor, use_proxy=True)
  443. self.reactor.lookups["proxy.com"] = "1.2.3.5"
  444. d = agent.request(b"GET", b"http://test.com")
  445. # there should be a pending TCP connection
  446. clients = self.reactor.tcpClients
  447. self.assertEqual(len(clients), 1)
  448. (host, port, client_factory, _timeout, _bindAddress) = clients[0]
  449. self.assertEqual(host, "1.2.3.5")
  450. self.assertEqual(port, 8888)
  451. # make a test server, and wire up the client
  452. http_server = self._make_connection(
  453. client_factory,
  454. _get_test_protocol_factory(),
  455. ssl=expect_proxy_ssl,
  456. tls_sanlist=[b"DNS:proxy.com"] if expect_proxy_ssl else None,
  457. expected_sni=b"proxy.com" if expect_proxy_ssl else None,
  458. )
  459. assert isinstance(http_server, HTTPChannel)
  460. # the FakeTransport is async, so we need to pump the reactor
  461. self.reactor.advance(0)
  462. # now there should be a pending request
  463. self.assertEqual(len(http_server.requests), 1)
  464. request = http_server.requests[0]
  465. # Check whether auth credentials have been supplied to the proxy
  466. proxy_auth_header_values = request.requestHeaders.getRawHeaders(
  467. b"Proxy-Authorization"
  468. )
  469. if expected_auth_credentials is not None:
  470. # Compute the correct header value for Proxy-Authorization
  471. encoded_credentials = base64.b64encode(expected_auth_credentials)
  472. expected_header_value = b"Basic " + encoded_credentials
  473. # Validate the header's value
  474. self.assertIn(expected_header_value, proxy_auth_header_values)
  475. else:
  476. # Check that the Proxy-Authorization header has not been supplied to the proxy
  477. self.assertIsNone(proxy_auth_header_values)
  478. self.assertEqual(request.method, b"GET")
  479. self.assertEqual(request.path, b"http://test.com")
  480. self.assertEqual(request.requestHeaders.getRawHeaders(b"host"), [b"test.com"])
  481. request.write(b"result")
  482. request.finish()
  483. self.reactor.advance(0)
  484. resp = self.successResultOf(d)
  485. body = self.successResultOf(treq.content(resp))
  486. self.assertEqual(body, b"result")
  487. def _do_https_request_via_proxy(
  488. self,
  489. expect_proxy_ssl: bool = False,
  490. expected_auth_credentials: Optional[bytes] = None,
  491. ) -> None:
  492. """Send a https request via an agent and check that it is correctly received at
  493. the proxy and client. The proxy can use either http or https.
  494. Args:
  495. expect_proxy_ssl: True if we expect the request to connect via https to proxy
  496. expected_auth_credentials: credentials to authenticate at proxy
  497. """
  498. agent = ProxyAgent(
  499. self.reactor,
  500. contextFactory=get_test_https_policy(),
  501. use_proxy=True,
  502. )
  503. self.reactor.lookups["proxy.com"] = "1.2.3.5"
  504. d = agent.request(b"GET", b"https://test.com/abc")
  505. # there should be a pending TCP connection
  506. clients = self.reactor.tcpClients
  507. self.assertEqual(len(clients), 1)
  508. (host, port, client_factory, _timeout, _bindAddress) = clients[0]
  509. self.assertEqual(host, "1.2.3.5")
  510. self.assertEqual(port, 1080)
  511. # make a test server to act as the proxy, and wire up the client
  512. proxy_server = self._make_connection(
  513. client_factory,
  514. _get_test_protocol_factory(),
  515. ssl=expect_proxy_ssl,
  516. tls_sanlist=[b"DNS:proxy.com"] if expect_proxy_ssl else None,
  517. expected_sni=b"proxy.com" if expect_proxy_ssl else None,
  518. )
  519. assert isinstance(proxy_server, HTTPChannel)
  520. # now there should be a pending CONNECT request
  521. self.assertEqual(len(proxy_server.requests), 1)
  522. request = proxy_server.requests[0]
  523. self.assertEqual(request.method, b"CONNECT")
  524. self.assertEqual(request.path, b"test.com:443")
  525. # Check whether auth credentials have been supplied to the proxy
  526. proxy_auth_header_values = request.requestHeaders.getRawHeaders(
  527. b"Proxy-Authorization"
  528. )
  529. if expected_auth_credentials is not None:
  530. # Compute the correct header value for Proxy-Authorization
  531. encoded_credentials = base64.b64encode(expected_auth_credentials)
  532. expected_header_value = b"Basic " + encoded_credentials
  533. # Validate the header's value
  534. self.assertIn(expected_header_value, proxy_auth_header_values)
  535. else:
  536. # Check that the Proxy-Authorization header has not been supplied to the proxy
  537. self.assertIsNone(proxy_auth_header_values)
  538. # tell the proxy server not to close the connection
  539. proxy_server.persistent = True
  540. request.finish()
  541. # now we make another test server to act as the upstream HTTP server.
  542. server_ssl_protocol = _wrap_server_factory_for_tls(
  543. _get_test_protocol_factory()
  544. ).buildProtocol(dummy_address)
  545. # Tell the HTTP server to send outgoing traffic back via the proxy's transport.
  546. proxy_server_transport = proxy_server.transport
  547. assert proxy_server_transport is not None
  548. server_ssl_protocol.makeConnection(proxy_server_transport)
  549. # ... and replace the protocol on the proxy's transport with the
  550. # TLSMemoryBIOProtocol for the test server, so that incoming traffic
  551. # to the proxy gets sent over to the HTTP(s) server.
  552. #
  553. # This needs a bit of gut-wrenching, which is different depending on whether
  554. # the proxy is using TLS or not.
  555. #
  556. # (an alternative, possibly more elegant, approach would be to use a custom
  557. # Protocol to implement the proxy, which starts out by forwarding to an
  558. # HTTPChannel (to implement the CONNECT command) and can then be switched
  559. # into a mode where it forwards its traffic to another Protocol.)
  560. if expect_proxy_ssl:
  561. assert isinstance(proxy_server_transport, TLSMemoryBIOProtocol)
  562. proxy_server_transport.wrappedProtocol = server_ssl_protocol
  563. else:
  564. assert isinstance(proxy_server_transport, FakeTransport)
  565. client_protocol = proxy_server_transport.other
  566. assert isinstance(client_protocol, Protocol)
  567. c2s_transport = checked_cast(FakeTransport, client_protocol.transport)
  568. c2s_transport.other = server_ssl_protocol
  569. self.reactor.advance(0)
  570. server_name = server_ssl_protocol._tlsConnection.get_servername()
  571. expected_sni = b"test.com"
  572. self.assertEqual(
  573. server_name,
  574. expected_sni,
  575. f"Expected SNI {expected_sni!s} but got {server_name!s}",
  576. )
  577. # now there should be a pending request
  578. http_server = server_ssl_protocol.wrappedProtocol
  579. assert isinstance(http_server, HTTPChannel)
  580. self.assertEqual(len(http_server.requests), 1)
  581. request = http_server.requests[0]
  582. self.assertEqual(request.method, b"GET")
  583. self.assertEqual(request.path, b"/abc")
  584. self.assertEqual(request.requestHeaders.getRawHeaders(b"host"), [b"test.com"])
  585. # Check that the destination server DID NOT receive proxy credentials
  586. proxy_auth_header_values = request.requestHeaders.getRawHeaders(
  587. b"Proxy-Authorization"
  588. )
  589. self.assertIsNone(proxy_auth_header_values)
  590. request.write(b"result")
  591. request.finish()
  592. self.reactor.advance(0)
  593. resp = self.successResultOf(d)
  594. body = self.successResultOf(treq.content(resp))
  595. self.assertEqual(body, b"result")
  596. @patch.dict(os.environ, {"http_proxy": "proxy.com:8888"})
  597. def test_http_request_via_proxy_with_blacklist(self) -> None:
  598. # The blacklist includes the configured proxy IP.
  599. agent = ProxyAgent(
  600. BlacklistingReactorWrapper(
  601. self.reactor, ip_whitelist=None, ip_blacklist=IPSet(["1.0.0.0/8"])
  602. ),
  603. self.reactor,
  604. use_proxy=True,
  605. )
  606. self.reactor.lookups["proxy.com"] = "1.2.3.5"
  607. d = agent.request(b"GET", b"http://test.com")
  608. # there should be a pending TCP connection
  609. clients = self.reactor.tcpClients
  610. self.assertEqual(len(clients), 1)
  611. (host, port, client_factory, _timeout, _bindAddress) = clients[0]
  612. self.assertEqual(host, "1.2.3.5")
  613. self.assertEqual(port, 8888)
  614. # make a test server, and wire up the client
  615. http_server = self._make_connection(
  616. client_factory, _get_test_protocol_factory()
  617. )
  618. assert isinstance(http_server, HTTPChannel)
  619. # the FakeTransport is async, so we need to pump the reactor
  620. self.reactor.advance(0)
  621. # now there should be a pending request
  622. self.assertEqual(len(http_server.requests), 1)
  623. request = http_server.requests[0]
  624. self.assertEqual(request.method, b"GET")
  625. self.assertEqual(request.path, b"http://test.com")
  626. self.assertEqual(request.requestHeaders.getRawHeaders(b"host"), [b"test.com"])
  627. request.write(b"result")
  628. request.finish()
  629. self.reactor.advance(0)
  630. resp = self.successResultOf(d)
  631. body = self.successResultOf(treq.content(resp))
  632. self.assertEqual(body, b"result")
  633. @patch.dict(os.environ, {"HTTPS_PROXY": "proxy.com"})
  634. def test_https_request_via_uppercase_proxy_with_blacklist(self) -> None:
  635. # The blacklist includes the configured proxy IP.
  636. agent = ProxyAgent(
  637. BlacklistingReactorWrapper(
  638. self.reactor, ip_whitelist=None, ip_blacklist=IPSet(["1.0.0.0/8"])
  639. ),
  640. self.reactor,
  641. contextFactory=get_test_https_policy(),
  642. use_proxy=True,
  643. )
  644. self.reactor.lookups["proxy.com"] = "1.2.3.5"
  645. d = agent.request(b"GET", b"https://test.com/abc")
  646. # there should be a pending TCP connection
  647. clients = self.reactor.tcpClients
  648. self.assertEqual(len(clients), 1)
  649. (host, port, client_factory, _timeout, _bindAddress) = clients[0]
  650. self.assertEqual(host, "1.2.3.5")
  651. self.assertEqual(port, 1080)
  652. # make a test HTTP server, and wire up the client
  653. proxy_server = self._make_connection(
  654. client_factory, _get_test_protocol_factory()
  655. )
  656. assert isinstance(proxy_server, HTTPChannel)
  657. # fish the transports back out so that we can do the old switcheroo
  658. # To help mypy out with the various Protocols and wrappers and mocks, we do
  659. # some explicit casting. Without the casts, we hit the bug I reported at
  660. # https://github.com/Shoobx/mypy-zope/issues/91 .
  661. # We also double-checked these casts at runtime (test-time) because I found it
  662. # quite confusing to deduce these types in the first place!
  663. s2c_transport = checked_cast(FakeTransport, proxy_server.transport)
  664. client_protocol = checked_cast(_WrappingProtocol, s2c_transport.other)
  665. c2s_transport = checked_cast(FakeTransport, client_protocol.transport)
  666. # the FakeTransport is async, so we need to pump the reactor
  667. self.reactor.advance(0)
  668. # now there should be a pending CONNECT request
  669. self.assertEqual(len(proxy_server.requests), 1)
  670. request = proxy_server.requests[0]
  671. self.assertEqual(request.method, b"CONNECT")
  672. self.assertEqual(request.path, b"test.com:443")
  673. # tell the proxy server not to close the connection
  674. proxy_server.persistent = True
  675. # this just stops the http Request trying to do a chunked response
  676. # request.setHeader(b"Content-Length", b"0")
  677. request.finish()
  678. # now we can replace the proxy channel with a new, SSL-wrapped HTTP channel
  679. ssl_factory = _wrap_server_factory_for_tls(_get_test_protocol_factory())
  680. ssl_protocol = ssl_factory.buildProtocol(dummy_address)
  681. assert isinstance(ssl_protocol, TLSMemoryBIOProtocol)
  682. http_server = ssl_protocol.wrappedProtocol
  683. assert isinstance(http_server, HTTPChannel)
  684. ssl_protocol.makeConnection(
  685. FakeTransport(client_protocol, self.reactor, ssl_protocol)
  686. )
  687. c2s_transport.other = ssl_protocol
  688. self.reactor.advance(0)
  689. server_name = ssl_protocol._tlsConnection.get_servername()
  690. expected_sni = b"test.com"
  691. self.assertEqual(
  692. server_name,
  693. expected_sni,
  694. f"Expected SNI {expected_sni!s} but got {server_name!s}",
  695. )
  696. # now there should be a pending request
  697. self.assertEqual(len(http_server.requests), 1)
  698. request = http_server.requests[0]
  699. self.assertEqual(request.method, b"GET")
  700. self.assertEqual(request.path, b"/abc")
  701. self.assertEqual(request.requestHeaders.getRawHeaders(b"host"), [b"test.com"])
  702. request.write(b"result")
  703. request.finish()
  704. self.reactor.advance(0)
  705. resp = self.successResultOf(d)
  706. body = self.successResultOf(treq.content(resp))
  707. self.assertEqual(body, b"result")
  708. @patch.dict(os.environ, {"http_proxy": "proxy.com:8888"})
  709. def test_proxy_with_no_scheme(self) -> None:
  710. http_proxy_agent = ProxyAgent(self.reactor, use_proxy=True)
  711. proxy_ep = checked_cast(HostnameEndpoint, http_proxy_agent.http_proxy_endpoint)
  712. self.assertEqual(proxy_ep._hostStr, "proxy.com")
  713. self.assertEqual(proxy_ep._port, 8888)
  714. @patch.dict(os.environ, {"http_proxy": "socks://proxy.com:8888"})
  715. def test_proxy_with_unsupported_scheme(self) -> None:
  716. with self.assertRaises(ValueError):
  717. ProxyAgent(self.reactor, use_proxy=True)
  718. @patch.dict(os.environ, {"http_proxy": "http://proxy.com:8888"})
  719. def test_proxy_with_http_scheme(self) -> None:
  720. http_proxy_agent = ProxyAgent(self.reactor, use_proxy=True)
  721. proxy_ep = checked_cast(HostnameEndpoint, http_proxy_agent.http_proxy_endpoint)
  722. self.assertEqual(proxy_ep._hostStr, "proxy.com")
  723. self.assertEqual(proxy_ep._port, 8888)
  724. @patch.dict(os.environ, {"http_proxy": "https://proxy.com:8888"})
  725. def test_proxy_with_https_scheme(self) -> None:
  726. https_proxy_agent = ProxyAgent(self.reactor, use_proxy=True)
  727. proxy_ep = checked_cast(_WrapperEndpoint, https_proxy_agent.http_proxy_endpoint)
  728. self.assertEqual(proxy_ep._wrappedEndpoint._hostStr, "proxy.com")
  729. self.assertEqual(proxy_ep._wrappedEndpoint._port, 8888)
  730. def _wrap_server_factory_for_tls(
  731. factory: IProtocolFactory, sanlist: Optional[List[bytes]] = None
  732. ) -> TLSMemoryBIOFactory:
  733. """Wrap an existing Protocol Factory with a test TLSMemoryBIOFactory
  734. The resultant factory will create a TLS server which presents a certificate
  735. signed by our test CA, valid for the domains in `sanlist`
  736. Args:
  737. factory: protocol factory to wrap
  738. sanlist: list of domains the cert should be valid for
  739. Returns:
  740. interfaces.IProtocolFactory
  741. """
  742. if sanlist is None:
  743. sanlist = [b"DNS:test.com"]
  744. connection_creator = TestServerTLSConnectionFactory(sanlist=sanlist)
  745. return TLSMemoryBIOFactory(
  746. connection_creator, isClient=False, wrappedFactory=factory
  747. )
  748. def _get_test_protocol_factory() -> IProtocolFactory:
  749. """Get a protocol Factory which will build an HTTPChannel
  750. Returns:
  751. interfaces.IProtocolFactory
  752. """
  753. server_factory = Factory.forProtocol(HTTPChannel)
  754. # Request.finish expects the factory to have a 'log' method.
  755. server_factory.log = _log_request
  756. return server_factory
  757. def _log_request(request: str) -> None:
  758. """Implements Factory.log, which is expected by Request.finish"""
  759. logger.info(f"Completed request {request}")