test_proxyagent.py 34 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889
  1. # Copyright 2019 The Matrix.org Foundation C.I.C.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. import base64
  15. import logging
  16. import os
  17. from typing import List, Optional
  18. from unittest.mock import patch
  19. import treq
  20. from netaddr import IPSet
  21. from parameterized import parameterized
  22. from twisted.internet import interfaces # noqa: F401
  23. from twisted.internet.endpoints import (
  24. HostnameEndpoint,
  25. _WrapperEndpoint,
  26. _WrappingProtocol,
  27. )
  28. from twisted.internet.interfaces import IProtocol, IProtocolFactory
  29. from twisted.internet.protocol import Factory, Protocol
  30. from twisted.protocols.tls import TLSMemoryBIOProtocol
  31. from twisted.web.http import HTTPChannel
  32. from synapse.http.client import BlocklistingReactorWrapper
  33. from synapse.http.connectproxyclient import BasicProxyCredentials
  34. from synapse.http.proxyagent import ProxyAgent, parse_proxy
  35. from tests.http import dummy_address, get_test_https_policy, wrap_server_factory_for_tls
  36. from tests.server import FakeTransport, ThreadedMemoryReactorClock
  37. from tests.unittest import TestCase
  38. from tests.utils import checked_cast
  39. logger = logging.getLogger(__name__)
  40. HTTPFactory = Factory.forProtocol(HTTPChannel)
  41. class ProxyParserTests(TestCase):
  42. """
  43. Values for test
  44. [
  45. proxy_string,
  46. expected_scheme,
  47. expected_hostname,
  48. expected_port,
  49. expected_credentials,
  50. ]
  51. """
  52. @parameterized.expand(
  53. [
  54. # host
  55. [b"localhost", b"http", b"localhost", 1080, None],
  56. [b"localhost:9988", b"http", b"localhost", 9988, None],
  57. # host+scheme
  58. [b"https://localhost", b"https", b"localhost", 1080, None],
  59. [b"https://localhost:1234", b"https", b"localhost", 1234, None],
  60. # ipv4
  61. [b"1.2.3.4", b"http", b"1.2.3.4", 1080, None],
  62. [b"1.2.3.4:9988", b"http", b"1.2.3.4", 9988, None],
  63. # ipv4+scheme
  64. [b"https://1.2.3.4", b"https", b"1.2.3.4", 1080, None],
  65. [b"https://1.2.3.4:9988", b"https", b"1.2.3.4", 9988, None],
  66. # ipv6 - without brackets is broken
  67. # [
  68. # b"2001:0db8:85a3:0000:0000:8a2e:0370:effe",
  69. # b"http",
  70. # b"2001:0db8:85a3:0000:0000:8a2e:0370:effe",
  71. # 1080,
  72. # None,
  73. # ],
  74. # [
  75. # b"2001:0db8:85a3:0000:0000:8a2e:0370:1234",
  76. # b"http",
  77. # b"2001:0db8:85a3:0000:0000:8a2e:0370:1234",
  78. # 1080,
  79. # None,
  80. # ],
  81. # [b"::1", b"http", b"::1", 1080, None],
  82. # [b"::ffff:0.0.0.0", b"http", b"::ffff:0.0.0.0", 1080, None],
  83. # ipv6 - with brackets
  84. [
  85. b"[2001:0db8:85a3:0000:0000:8a2e:0370:effe]",
  86. b"http",
  87. b"2001:0db8:85a3:0000:0000:8a2e:0370:effe",
  88. 1080,
  89. None,
  90. ],
  91. [
  92. b"[2001:0db8:85a3:0000:0000:8a2e:0370:1234]",
  93. b"http",
  94. b"2001:0db8:85a3:0000:0000:8a2e:0370:1234",
  95. 1080,
  96. None,
  97. ],
  98. [b"[::1]", b"http", b"::1", 1080, None],
  99. [b"[::ffff:0.0.0.0]", b"http", b"::ffff:0.0.0.0", 1080, None],
  100. # ipv6+port
  101. [
  102. b"[2001:0db8:85a3:0000:0000:8a2e:0370:effe]:9988",
  103. b"http",
  104. b"2001:0db8:85a3:0000:0000:8a2e:0370:effe",
  105. 9988,
  106. None,
  107. ],
  108. [
  109. b"[2001:0db8:85a3:0000:0000:8a2e:0370:1234]:9988",
  110. b"http",
  111. b"2001:0db8:85a3:0000:0000:8a2e:0370:1234",
  112. 9988,
  113. None,
  114. ],
  115. [b"[::1]:9988", b"http", b"::1", 9988, None],
  116. [b"[::ffff:0.0.0.0]:9988", b"http", b"::ffff:0.0.0.0", 9988, None],
  117. # ipv6+scheme
  118. [
  119. b"https://[2001:0db8:85a3:0000:0000:8a2e:0370:effe]",
  120. b"https",
  121. b"2001:0db8:85a3:0000:0000:8a2e:0370:effe",
  122. 1080,
  123. None,
  124. ],
  125. [
  126. b"https://[2001:0db8:85a3:0000:0000:8a2e:0370:1234]",
  127. b"https",
  128. b"2001:0db8:85a3:0000:0000:8a2e:0370:1234",
  129. 1080,
  130. None,
  131. ],
  132. [b"https://[::1]", b"https", b"::1", 1080, None],
  133. [b"https://[::ffff:0.0.0.0]", b"https", b"::ffff:0.0.0.0", 1080, None],
  134. # ipv6+scheme+port
  135. [
  136. b"https://[2001:0db8:85a3:0000:0000:8a2e:0370:effe]:9988",
  137. b"https",
  138. b"2001:0db8:85a3:0000:0000:8a2e:0370:effe",
  139. 9988,
  140. None,
  141. ],
  142. [
  143. b"https://[2001:0db8:85a3:0000:0000:8a2e:0370:1234]:9988",
  144. b"https",
  145. b"2001:0db8:85a3:0000:0000:8a2e:0370:1234",
  146. 9988,
  147. None,
  148. ],
  149. [b"https://[::1]:9988", b"https", b"::1", 9988, None],
  150. # with credentials
  151. [
  152. b"https://user:pass@1.2.3.4:9988",
  153. b"https",
  154. b"1.2.3.4",
  155. 9988,
  156. b"user:pass",
  157. ],
  158. [b"user:pass@1.2.3.4:9988", b"http", b"1.2.3.4", 9988, b"user:pass"],
  159. [
  160. b"https://user:pass@proxy.local:9988",
  161. b"https",
  162. b"proxy.local",
  163. 9988,
  164. b"user:pass",
  165. ],
  166. [
  167. b"user:pass@proxy.local:9988",
  168. b"http",
  169. b"proxy.local",
  170. 9988,
  171. b"user:pass",
  172. ],
  173. ]
  174. )
  175. def test_parse_proxy(
  176. self,
  177. proxy_string: bytes,
  178. expected_scheme: bytes,
  179. expected_hostname: bytes,
  180. expected_port: int,
  181. expected_credentials: Optional[bytes],
  182. ) -> None:
  183. """
  184. Tests that a given proxy URL will be broken into the components.
  185. Args:
  186. proxy_string: The proxy connection string.
  187. expected_scheme: Expected value of proxy scheme.
  188. expected_hostname: Expected value of proxy hostname.
  189. expected_port: Expected value of proxy port.
  190. expected_credentials: Expected value of credentials.
  191. Must be in form '<username>:<password>' or None
  192. """
  193. proxy_cred = None
  194. if expected_credentials:
  195. proxy_cred = BasicProxyCredentials(expected_credentials)
  196. self.assertEqual(
  197. (
  198. expected_scheme,
  199. expected_hostname,
  200. expected_port,
  201. proxy_cred,
  202. ),
  203. parse_proxy(proxy_string),
  204. )
  205. class TestBasicProxyCredentials(TestCase):
  206. def test_long_user_pass_string_encoded_without_newlines(self) -> None:
  207. """Reproduces https://github.com/matrix-org/synapse/pull/16504."""
  208. proxy_connection_string = b"looooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooonguser:pass@proxy.local:9988"
  209. _, _, _, creds = parse_proxy(proxy_connection_string)
  210. assert creds is not None # for mypy's benefit
  211. self.assertIsInstance(creds, BasicProxyCredentials)
  212. auth_value = creds.as_proxy_authorization_value()
  213. self.assertNotIn(b"\n", auth_value)
  214. self.assertEqual(
  215. creds.as_proxy_authorization_value(),
  216. b"Basic bG9vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vbmd1c2VyOnBhc3M=",
  217. )
  218. basic_auth_payload = creds.as_proxy_authorization_value().split(b" ")[1]
  219. self.assertEqual(
  220. base64.b64decode(basic_auth_payload),
  221. b"looooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooonguser:pass",
  222. )
  223. class MatrixFederationAgentTests(TestCase):
  224. def setUp(self) -> None:
  225. self.reactor = ThreadedMemoryReactorClock()
  226. def _make_connection(
  227. self,
  228. client_factory: IProtocolFactory,
  229. server_factory: IProtocolFactory,
  230. ssl: bool = False,
  231. expected_sni: Optional[bytes] = None,
  232. tls_sanlist: Optional[List[bytes]] = None,
  233. ) -> IProtocol:
  234. """Builds a test server, and completes the outgoing client connection
  235. Args:
  236. client_factory: the the factory that the
  237. application is trying to use to make the outbound connection. We will
  238. invoke it to build the client Protocol
  239. server_factory: a factory to build the
  240. server-side protocol
  241. ssl: If true, we will expect an ssl connection and wrap
  242. server_factory with a TLSMemoryBIOFactory
  243. expected_sni: the expected SNI value
  244. tls_sanlist: list of SAN entries for the TLS cert presented by the server.
  245. Defaults to [b'DNS:test.com']
  246. Returns:
  247. the server Protocol returned by server_factory
  248. """
  249. if ssl:
  250. server_factory = wrap_server_factory_for_tls(
  251. server_factory, self.reactor, tls_sanlist or [b"DNS:test.com"]
  252. )
  253. server_protocol = server_factory.buildProtocol(dummy_address)
  254. assert server_protocol is not None
  255. # now, tell the client protocol factory to build the client protocol,
  256. # and wire the output of said protocol up to the server via
  257. # a FakeTransport.
  258. #
  259. # Normally this would be done by the TCP socket code in Twisted, but we are
  260. # stubbing that out here.
  261. client_protocol = client_factory.buildProtocol(dummy_address)
  262. assert client_protocol is not None
  263. client_protocol.makeConnection(
  264. FakeTransport(server_protocol, self.reactor, client_protocol)
  265. )
  266. # tell the server protocol to send its stuff back to the client, too
  267. server_protocol.makeConnection(
  268. FakeTransport(client_protocol, self.reactor, server_protocol)
  269. )
  270. if ssl:
  271. assert isinstance(server_protocol, TLSMemoryBIOProtocol)
  272. http_protocol = server_protocol.wrappedProtocol
  273. tls_connection = server_protocol._tlsConnection
  274. else:
  275. http_protocol = server_protocol
  276. tls_connection = None
  277. # give the reactor a pump to get the TLS juices flowing (if needed)
  278. self.reactor.advance(0)
  279. if expected_sni is not None:
  280. server_name = tls_connection.get_servername()
  281. self.assertEqual(
  282. server_name,
  283. expected_sni,
  284. f"Expected SNI {expected_sni!s} but got {server_name!s}",
  285. )
  286. return http_protocol
  287. def _test_request_direct_connection(
  288. self,
  289. agent: ProxyAgent,
  290. scheme: bytes,
  291. hostname: bytes,
  292. path: bytes,
  293. ) -> None:
  294. """Runs a test case for a direct connection not going through a proxy.
  295. Args:
  296. agent: the proxy agent being tested
  297. scheme: expected to be either "http" or "https"
  298. hostname: the hostname to connect to in the test
  299. path: the path to connect to in the test
  300. """
  301. is_https = scheme == b"https"
  302. self.reactor.lookups[hostname.decode()] = "1.2.3.4"
  303. d = agent.request(b"GET", scheme + b"://" + hostname + b"/" + path)
  304. # there should be a pending TCP connection
  305. clients = self.reactor.tcpClients
  306. self.assertEqual(len(clients), 1)
  307. (host, port, client_factory, _timeout, _bindAddress) = clients[0]
  308. self.assertEqual(host, "1.2.3.4")
  309. self.assertEqual(port, 443 if is_https else 80)
  310. # make a test server, and wire up the client
  311. http_server = self._make_connection(
  312. client_factory,
  313. _get_test_protocol_factory(),
  314. ssl=is_https,
  315. expected_sni=hostname if is_https else None,
  316. )
  317. assert isinstance(http_server, HTTPChannel)
  318. # the FakeTransport is async, so we need to pump the reactor
  319. self.reactor.advance(0)
  320. # now there should be a pending request
  321. self.assertEqual(len(http_server.requests), 1)
  322. request = http_server.requests[0]
  323. self.assertEqual(request.method, b"GET")
  324. self.assertEqual(request.path, b"/" + path)
  325. self.assertEqual(request.requestHeaders.getRawHeaders(b"host"), [hostname])
  326. request.write(b"result")
  327. request.finish()
  328. self.reactor.advance(0)
  329. resp = self.successResultOf(d)
  330. body = self.successResultOf(treq.content(resp))
  331. self.assertEqual(body, b"result")
  332. def test_http_request(self) -> None:
  333. agent = ProxyAgent(self.reactor)
  334. self._test_request_direct_connection(agent, b"http", b"test.com", b"")
  335. def test_https_request(self) -> None:
  336. agent = ProxyAgent(self.reactor, contextFactory=get_test_https_policy())
  337. self._test_request_direct_connection(agent, b"https", b"test.com", b"abc")
  338. def test_http_request_use_proxy_empty_environment(self) -> None:
  339. agent = ProxyAgent(self.reactor, use_proxy=True)
  340. self._test_request_direct_connection(agent, b"http", b"test.com", b"")
  341. @patch.dict(os.environ, {"http_proxy": "proxy.com:8888", "NO_PROXY": "test.com"})
  342. def test_http_request_via_uppercase_no_proxy(self) -> None:
  343. agent = ProxyAgent(self.reactor, use_proxy=True)
  344. self._test_request_direct_connection(agent, b"http", b"test.com", b"")
  345. @patch.dict(
  346. os.environ, {"http_proxy": "proxy.com:8888", "no_proxy": "test.com,unused.com"}
  347. )
  348. def test_http_request_via_no_proxy(self) -> None:
  349. agent = ProxyAgent(self.reactor, use_proxy=True)
  350. self._test_request_direct_connection(agent, b"http", b"test.com", b"")
  351. @patch.dict(
  352. os.environ, {"https_proxy": "proxy.com", "no_proxy": "test.com,unused.com"}
  353. )
  354. def test_https_request_via_no_proxy(self) -> None:
  355. agent = ProxyAgent(
  356. self.reactor,
  357. contextFactory=get_test_https_policy(),
  358. use_proxy=True,
  359. )
  360. self._test_request_direct_connection(agent, b"https", b"test.com", b"abc")
  361. @patch.dict(os.environ, {"http_proxy": "proxy.com:8888", "no_proxy": "*"})
  362. def test_http_request_via_no_proxy_star(self) -> None:
  363. agent = ProxyAgent(self.reactor, use_proxy=True)
  364. self._test_request_direct_connection(agent, b"http", b"test.com", b"")
  365. @patch.dict(os.environ, {"https_proxy": "proxy.com", "no_proxy": "*"})
  366. def test_https_request_via_no_proxy_star(self) -> None:
  367. agent = ProxyAgent(
  368. self.reactor,
  369. contextFactory=get_test_https_policy(),
  370. use_proxy=True,
  371. )
  372. self._test_request_direct_connection(agent, b"https", b"test.com", b"abc")
  373. @patch.dict(os.environ, {"http_proxy": "proxy.com:8888", "no_proxy": "unused.com"})
  374. def test_http_request_via_proxy(self) -> None:
  375. """
  376. Tests that requests can be made through a proxy.
  377. """
  378. self._do_http_request_via_proxy(
  379. expect_proxy_ssl=False, expected_auth_credentials=None
  380. )
  381. @patch.dict(
  382. os.environ,
  383. {"http_proxy": "bob:pinkponies@proxy.com:8888", "no_proxy": "unused.com"},
  384. )
  385. def test_http_request_via_proxy_with_auth(self) -> None:
  386. """
  387. Tests that authenticated requests can be made through a proxy.
  388. """
  389. self._do_http_request_via_proxy(
  390. expect_proxy_ssl=False, expected_auth_credentials=b"bob:pinkponies"
  391. )
  392. @patch.dict(
  393. os.environ, {"http_proxy": "https://proxy.com:8888", "no_proxy": "unused.com"}
  394. )
  395. def test_http_request_via_https_proxy(self) -> None:
  396. self._do_http_request_via_proxy(
  397. expect_proxy_ssl=True, expected_auth_credentials=None
  398. )
  399. @patch.dict(
  400. os.environ,
  401. {
  402. "http_proxy": "https://bob:pinkponies@proxy.com:8888",
  403. "no_proxy": "unused.com",
  404. },
  405. )
  406. def test_http_request_via_https_proxy_with_auth(self) -> None:
  407. self._do_http_request_via_proxy(
  408. expect_proxy_ssl=True, expected_auth_credentials=b"bob:pinkponies"
  409. )
  410. @patch.dict(os.environ, {"https_proxy": "proxy.com", "no_proxy": "unused.com"})
  411. def test_https_request_via_proxy(self) -> None:
  412. """Tests that TLS-encrypted requests can be made through a proxy"""
  413. self._do_https_request_via_proxy(
  414. expect_proxy_ssl=False, expected_auth_credentials=None
  415. )
  416. @patch.dict(
  417. os.environ,
  418. {"https_proxy": "bob:pinkponies@proxy.com", "no_proxy": "unused.com"},
  419. )
  420. def test_https_request_via_proxy_with_auth(self) -> None:
  421. """Tests that authenticated, TLS-encrypted requests can be made through a proxy"""
  422. self._do_https_request_via_proxy(
  423. expect_proxy_ssl=False, expected_auth_credentials=b"bob:pinkponies"
  424. )
  425. @patch.dict(
  426. os.environ, {"https_proxy": "https://proxy.com", "no_proxy": "unused.com"}
  427. )
  428. def test_https_request_via_https_proxy(self) -> None:
  429. """Tests that TLS-encrypted requests can be made through a proxy"""
  430. self._do_https_request_via_proxy(
  431. expect_proxy_ssl=True, expected_auth_credentials=None
  432. )
  433. @patch.dict(
  434. os.environ,
  435. {"https_proxy": "https://bob:pinkponies@proxy.com", "no_proxy": "unused.com"},
  436. )
  437. def test_https_request_via_https_proxy_with_auth(self) -> None:
  438. """Tests that authenticated, TLS-encrypted requests can be made through a proxy"""
  439. self._do_https_request_via_proxy(
  440. expect_proxy_ssl=True, expected_auth_credentials=b"bob:pinkponies"
  441. )
  442. def _do_http_request_via_proxy(
  443. self,
  444. expect_proxy_ssl: bool = False,
  445. expected_auth_credentials: Optional[bytes] = None,
  446. ) -> None:
  447. """Send a http request via an agent and check that it is correctly received at
  448. the proxy. The proxy can use either http or https.
  449. Args:
  450. expect_proxy_ssl: True if we expect the request to connect via https to proxy
  451. expected_auth_credentials: credentials to authenticate at proxy
  452. """
  453. if expect_proxy_ssl:
  454. agent = ProxyAgent(
  455. self.reactor, use_proxy=True, contextFactory=get_test_https_policy()
  456. )
  457. else:
  458. agent = ProxyAgent(self.reactor, use_proxy=True)
  459. self.reactor.lookups["proxy.com"] = "1.2.3.5"
  460. d = agent.request(b"GET", b"http://test.com")
  461. # there should be a pending TCP connection
  462. clients = self.reactor.tcpClients
  463. self.assertEqual(len(clients), 1)
  464. (host, port, client_factory, _timeout, _bindAddress) = clients[0]
  465. self.assertEqual(host, "1.2.3.5")
  466. self.assertEqual(port, 8888)
  467. # make a test server, and wire up the client
  468. http_server = self._make_connection(
  469. client_factory,
  470. _get_test_protocol_factory(),
  471. ssl=expect_proxy_ssl,
  472. tls_sanlist=[b"DNS:proxy.com"] if expect_proxy_ssl else None,
  473. expected_sni=b"proxy.com" if expect_proxy_ssl else None,
  474. )
  475. assert isinstance(http_server, HTTPChannel)
  476. # the FakeTransport is async, so we need to pump the reactor
  477. self.reactor.advance(0)
  478. # now there should be a pending request
  479. self.assertEqual(len(http_server.requests), 1)
  480. request = http_server.requests[0]
  481. # Check whether auth credentials have been supplied to the proxy
  482. proxy_auth_header_values = request.requestHeaders.getRawHeaders(
  483. b"Proxy-Authorization"
  484. )
  485. if expected_auth_credentials is not None:
  486. # Compute the correct header value for Proxy-Authorization
  487. encoded_credentials = base64.b64encode(expected_auth_credentials)
  488. expected_header_value = b"Basic " + encoded_credentials
  489. # Validate the header's value
  490. self.assertIn(expected_header_value, proxy_auth_header_values)
  491. else:
  492. # Check that the Proxy-Authorization header has not been supplied to the proxy
  493. self.assertIsNone(proxy_auth_header_values)
  494. self.assertEqual(request.method, b"GET")
  495. self.assertEqual(request.path, b"http://test.com")
  496. self.assertEqual(request.requestHeaders.getRawHeaders(b"host"), [b"test.com"])
  497. request.write(b"result")
  498. request.finish()
  499. self.reactor.advance(0)
  500. resp = self.successResultOf(d)
  501. body = self.successResultOf(treq.content(resp))
  502. self.assertEqual(body, b"result")
  503. def _do_https_request_via_proxy(
  504. self,
  505. expect_proxy_ssl: bool = False,
  506. expected_auth_credentials: Optional[bytes] = None,
  507. ) -> None:
  508. """Send a https request via an agent and check that it is correctly received at
  509. the proxy and client. The proxy can use either http or https.
  510. Args:
  511. expect_proxy_ssl: True if we expect the request to connect via https to proxy
  512. expected_auth_credentials: credentials to authenticate at proxy
  513. """
  514. agent = ProxyAgent(
  515. self.reactor,
  516. contextFactory=get_test_https_policy(),
  517. use_proxy=True,
  518. )
  519. self.reactor.lookups["proxy.com"] = "1.2.3.5"
  520. d = agent.request(b"GET", b"https://test.com/abc")
  521. # there should be a pending TCP connection
  522. clients = self.reactor.tcpClients
  523. self.assertEqual(len(clients), 1)
  524. (host, port, client_factory, _timeout, _bindAddress) = clients[0]
  525. self.assertEqual(host, "1.2.3.5")
  526. self.assertEqual(port, 1080)
  527. # make a test server to act as the proxy, and wire up the client
  528. proxy_server = self._make_connection(
  529. client_factory,
  530. _get_test_protocol_factory(),
  531. ssl=expect_proxy_ssl,
  532. tls_sanlist=[b"DNS:proxy.com"] if expect_proxy_ssl else None,
  533. expected_sni=b"proxy.com" if expect_proxy_ssl else None,
  534. )
  535. assert isinstance(proxy_server, HTTPChannel)
  536. # now there should be a pending CONNECT request
  537. self.assertEqual(len(proxy_server.requests), 1)
  538. request = proxy_server.requests[0]
  539. self.assertEqual(request.method, b"CONNECT")
  540. self.assertEqual(request.path, b"test.com:443")
  541. # Check whether auth credentials have been supplied to the proxy
  542. proxy_auth_header_values = request.requestHeaders.getRawHeaders(
  543. b"Proxy-Authorization"
  544. )
  545. if expected_auth_credentials is not None:
  546. # Compute the correct header value for Proxy-Authorization
  547. encoded_credentials = base64.b64encode(expected_auth_credentials)
  548. expected_header_value = b"Basic " + encoded_credentials
  549. # Validate the header's value
  550. self.assertIn(expected_header_value, proxy_auth_header_values)
  551. else:
  552. # Check that the Proxy-Authorization header has not been supplied to the proxy
  553. self.assertIsNone(proxy_auth_header_values)
  554. # tell the proxy server not to close the connection
  555. proxy_server.persistent = True
  556. request.finish()
  557. # now we make another test server to act as the upstream HTTP server.
  558. server_ssl_protocol = wrap_server_factory_for_tls(
  559. _get_test_protocol_factory(), self.reactor, sanlist=[b"DNS:test.com"]
  560. ).buildProtocol(dummy_address)
  561. # Tell the HTTP server to send outgoing traffic back via the proxy's transport.
  562. proxy_server_transport = proxy_server.transport
  563. assert proxy_server_transport is not None
  564. server_ssl_protocol.makeConnection(proxy_server_transport)
  565. # ... and replace the protocol on the proxy's transport with the
  566. # TLSMemoryBIOProtocol for the test server, so that incoming traffic
  567. # to the proxy gets sent over to the HTTP(s) server.
  568. #
  569. # This needs a bit of gut-wrenching, which is different depending on whether
  570. # the proxy is using TLS or not.
  571. #
  572. # (an alternative, possibly more elegant, approach would be to use a custom
  573. # Protocol to implement the proxy, which starts out by forwarding to an
  574. # HTTPChannel (to implement the CONNECT command) and can then be switched
  575. # into a mode where it forwards its traffic to another Protocol.)
  576. if expect_proxy_ssl:
  577. assert isinstance(proxy_server_transport, TLSMemoryBIOProtocol)
  578. proxy_server_transport.wrappedProtocol = server_ssl_protocol
  579. else:
  580. assert isinstance(proxy_server_transport, FakeTransport)
  581. client_protocol = proxy_server_transport.other
  582. assert isinstance(client_protocol, Protocol)
  583. c2s_transport = checked_cast(FakeTransport, client_protocol.transport)
  584. c2s_transport.other = server_ssl_protocol
  585. self.reactor.advance(0)
  586. server_name = server_ssl_protocol._tlsConnection.get_servername()
  587. expected_sni = b"test.com"
  588. self.assertEqual(
  589. server_name,
  590. expected_sni,
  591. f"Expected SNI {expected_sni!s} but got {server_name!s}",
  592. )
  593. # now there should be a pending request
  594. http_server = server_ssl_protocol.wrappedProtocol
  595. assert isinstance(http_server, HTTPChannel)
  596. self.assertEqual(len(http_server.requests), 1)
  597. request = http_server.requests[0]
  598. self.assertEqual(request.method, b"GET")
  599. self.assertEqual(request.path, b"/abc")
  600. self.assertEqual(request.requestHeaders.getRawHeaders(b"host"), [b"test.com"])
  601. # Check that the destination server DID NOT receive proxy credentials
  602. proxy_auth_header_values = request.requestHeaders.getRawHeaders(
  603. b"Proxy-Authorization"
  604. )
  605. self.assertIsNone(proxy_auth_header_values)
  606. request.write(b"result")
  607. request.finish()
  608. self.reactor.advance(0)
  609. resp = self.successResultOf(d)
  610. body = self.successResultOf(treq.content(resp))
  611. self.assertEqual(body, b"result")
  612. @patch.dict(os.environ, {"http_proxy": "proxy.com:8888"})
  613. def test_http_request_via_proxy_with_blocklist(self) -> None:
  614. # The blocklist includes the configured proxy IP.
  615. agent = ProxyAgent(
  616. BlocklistingReactorWrapper(
  617. self.reactor, ip_allowlist=None, ip_blocklist=IPSet(["1.0.0.0/8"])
  618. ),
  619. self.reactor,
  620. use_proxy=True,
  621. )
  622. self.reactor.lookups["proxy.com"] = "1.2.3.5"
  623. d = agent.request(b"GET", b"http://test.com")
  624. # there should be a pending TCP connection
  625. clients = self.reactor.tcpClients
  626. self.assertEqual(len(clients), 1)
  627. (host, port, client_factory, _timeout, _bindAddress) = clients[0]
  628. self.assertEqual(host, "1.2.3.5")
  629. self.assertEqual(port, 8888)
  630. # make a test server, and wire up the client
  631. http_server = self._make_connection(
  632. client_factory, _get_test_protocol_factory()
  633. )
  634. assert isinstance(http_server, HTTPChannel)
  635. # the FakeTransport is async, so we need to pump the reactor
  636. self.reactor.advance(0)
  637. # now there should be a pending request
  638. self.assertEqual(len(http_server.requests), 1)
  639. request = http_server.requests[0]
  640. self.assertEqual(request.method, b"GET")
  641. self.assertEqual(request.path, b"http://test.com")
  642. self.assertEqual(request.requestHeaders.getRawHeaders(b"host"), [b"test.com"])
  643. request.write(b"result")
  644. request.finish()
  645. self.reactor.advance(0)
  646. resp = self.successResultOf(d)
  647. body = self.successResultOf(treq.content(resp))
  648. self.assertEqual(body, b"result")
  649. @patch.dict(os.environ, {"HTTPS_PROXY": "proxy.com"})
  650. def test_https_request_via_uppercase_proxy_with_blocklist(self) -> None:
  651. # The blocklist includes the configured proxy IP.
  652. agent = ProxyAgent(
  653. BlocklistingReactorWrapper(
  654. self.reactor, ip_allowlist=None, ip_blocklist=IPSet(["1.0.0.0/8"])
  655. ),
  656. self.reactor,
  657. contextFactory=get_test_https_policy(),
  658. use_proxy=True,
  659. )
  660. self.reactor.lookups["proxy.com"] = "1.2.3.5"
  661. d = agent.request(b"GET", b"https://test.com/abc")
  662. # there should be a pending TCP connection
  663. clients = self.reactor.tcpClients
  664. self.assertEqual(len(clients), 1)
  665. (host, port, client_factory, _timeout, _bindAddress) = clients[0]
  666. self.assertEqual(host, "1.2.3.5")
  667. self.assertEqual(port, 1080)
  668. # make a test HTTP server, and wire up the client
  669. proxy_server = self._make_connection(
  670. client_factory, _get_test_protocol_factory()
  671. )
  672. assert isinstance(proxy_server, HTTPChannel)
  673. # fish the transports back out so that we can do the old switcheroo
  674. # To help mypy out with the various Protocols and wrappers and mocks, we do
  675. # some explicit casting. Without the casts, we hit the bug I reported at
  676. # https://github.com/Shoobx/mypy-zope/issues/91 .
  677. # We also double-checked these casts at runtime (test-time) because I found it
  678. # quite confusing to deduce these types in the first place!
  679. s2c_transport = checked_cast(FakeTransport, proxy_server.transport)
  680. client_protocol = checked_cast(_WrappingProtocol, s2c_transport.other)
  681. c2s_transport = checked_cast(FakeTransport, client_protocol.transport)
  682. # the FakeTransport is async, so we need to pump the reactor
  683. self.reactor.advance(0)
  684. # now there should be a pending CONNECT request
  685. self.assertEqual(len(proxy_server.requests), 1)
  686. request = proxy_server.requests[0]
  687. self.assertEqual(request.method, b"CONNECT")
  688. self.assertEqual(request.path, b"test.com:443")
  689. # tell the proxy server not to close the connection
  690. proxy_server.persistent = True
  691. # this just stops the http Request trying to do a chunked response
  692. # request.setHeader(b"Content-Length", b"0")
  693. request.finish()
  694. # now we can replace the proxy channel with a new, SSL-wrapped HTTP channel
  695. ssl_factory = wrap_server_factory_for_tls(
  696. _get_test_protocol_factory(), self.reactor, sanlist=[b"DNS:test.com"]
  697. )
  698. ssl_protocol = ssl_factory.buildProtocol(dummy_address)
  699. assert isinstance(ssl_protocol, TLSMemoryBIOProtocol)
  700. http_server = ssl_protocol.wrappedProtocol
  701. assert isinstance(http_server, HTTPChannel)
  702. ssl_protocol.makeConnection(
  703. FakeTransport(client_protocol, self.reactor, ssl_protocol)
  704. )
  705. c2s_transport.other = ssl_protocol
  706. self.reactor.advance(0)
  707. server_name = ssl_protocol._tlsConnection.get_servername()
  708. expected_sni = b"test.com"
  709. self.assertEqual(
  710. server_name,
  711. expected_sni,
  712. f"Expected SNI {expected_sni!s} but got {server_name!s}",
  713. )
  714. # now there should be a pending request
  715. self.assertEqual(len(http_server.requests), 1)
  716. request = http_server.requests[0]
  717. self.assertEqual(request.method, b"GET")
  718. self.assertEqual(request.path, b"/abc")
  719. self.assertEqual(request.requestHeaders.getRawHeaders(b"host"), [b"test.com"])
  720. request.write(b"result")
  721. request.finish()
  722. self.reactor.advance(0)
  723. resp = self.successResultOf(d)
  724. body = self.successResultOf(treq.content(resp))
  725. self.assertEqual(body, b"result")
  726. @patch.dict(os.environ, {"http_proxy": "proxy.com:8888"})
  727. def test_proxy_with_no_scheme(self) -> None:
  728. http_proxy_agent = ProxyAgent(self.reactor, use_proxy=True)
  729. proxy_ep = checked_cast(HostnameEndpoint, http_proxy_agent.http_proxy_endpoint)
  730. self.assertEqual(proxy_ep._hostStr, "proxy.com")
  731. self.assertEqual(proxy_ep._port, 8888)
  732. @patch.dict(os.environ, {"http_proxy": "socks://proxy.com:8888"})
  733. def test_proxy_with_unsupported_scheme(self) -> None:
  734. with self.assertRaises(ValueError):
  735. ProxyAgent(self.reactor, use_proxy=True)
  736. @patch.dict(os.environ, {"http_proxy": "http://proxy.com:8888"})
  737. def test_proxy_with_http_scheme(self) -> None:
  738. http_proxy_agent = ProxyAgent(self.reactor, use_proxy=True)
  739. proxy_ep = checked_cast(HostnameEndpoint, http_proxy_agent.http_proxy_endpoint)
  740. self.assertEqual(proxy_ep._hostStr, "proxy.com")
  741. self.assertEqual(proxy_ep._port, 8888)
  742. @patch.dict(os.environ, {"http_proxy": "https://proxy.com:8888"})
  743. def test_proxy_with_https_scheme(self) -> None:
  744. https_proxy_agent = ProxyAgent(self.reactor, use_proxy=True)
  745. proxy_ep = checked_cast(_WrapperEndpoint, https_proxy_agent.http_proxy_endpoint)
  746. self.assertEqual(proxy_ep._wrappedEndpoint._hostStr, "proxy.com")
  747. self.assertEqual(proxy_ep._wrappedEndpoint._port, 8888)
  748. def _get_test_protocol_factory() -> IProtocolFactory:
  749. """Get a protocol Factory which will build an HTTPChannel
  750. Returns:
  751. interfaces.IProtocolFactory
  752. """
  753. server_factory = Factory.forProtocol(HTTPChannel)
  754. # Request.finish expects the factory to have a 'log' method.
  755. server_factory.log = _log_request
  756. return server_factory
  757. def _log_request(request: str) -> None:
  758. """Implements Factory.log, which is expected by Request.finish"""
  759. logger.info(f"Completed request {request}")