httpd.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473
  1. #!/usr/bin/env python3
  2. # -*- coding: utf-8 -*-
  3. #***************************************************************************
  4. # _ _ ____ _
  5. # Project ___| | | | _ \| |
  6. # / __| | | | |_) | |
  7. # | (__| |_| | _ <| |___
  8. # \___|\___/|_| \_\_____|
  9. #
  10. # Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
  11. #
  12. # This software is licensed as described in the file COPYING, which
  13. # you should have received as part of this distribution. The terms
  14. # are also available at https://curl.se/docs/copyright.html.
  15. #
  16. # You may opt to use, copy, modify, merge, publish, distribute and/or sell
  17. # copies of the Software, and permit persons to whom the Software is
  18. # furnished to do so, under the terms of the COPYING file.
  19. #
  20. # This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
  21. # KIND, either express or implied.
  22. #
  23. # SPDX-License-Identifier: curl
  24. #
  25. ###########################################################################
  26. #
  27. import inspect
  28. import logging
  29. import os
  30. import subprocess
  31. from datetime import timedelta, datetime
  32. from json import JSONEncoder
  33. import time
  34. from typing import List, Union, Optional
  35. import copy
  36. from .curl import CurlClient, ExecResult
  37. from .env import Env
  38. log = logging.getLogger(__name__)
  39. class Httpd:
  40. MODULES = [
  41. 'log_config', 'logio', 'unixd', 'version', 'watchdog',
  42. 'authn_core', 'authn_file',
  43. 'authz_user', 'authz_core', 'authz_host',
  44. 'auth_basic', 'auth_digest',
  45. 'alias', 'env', 'filter', 'headers', 'mime', 'setenvif',
  46. 'socache_shmcb',
  47. 'rewrite', 'http2', 'ssl', 'proxy', 'proxy_http', 'proxy_connect',
  48. 'brotli',
  49. 'mpm_event',
  50. ]
  51. COMMON_MODULES_DIRS = [
  52. '/usr/lib/apache2/modules', # debian
  53. '/usr/libexec/apache2/', # macos
  54. ]
  55. MOD_CURLTEST = None
  56. def __init__(self, env: Env, proxy_auth: bool = False):
  57. self.env = env
  58. self._cmd = env.apachectl
  59. self._apache_dir = os.path.join(env.gen_dir, 'apache')
  60. self._run_dir = os.path.join(self._apache_dir, 'run')
  61. self._lock_dir = os.path.join(self._apache_dir, 'locks')
  62. self._docs_dir = os.path.join(self._apache_dir, 'docs')
  63. self._conf_dir = os.path.join(self._apache_dir, 'conf')
  64. self._conf_file = os.path.join(self._conf_dir, 'test.conf')
  65. self._logs_dir = os.path.join(self._apache_dir, 'logs')
  66. self._error_log = os.path.join(self._logs_dir, 'error_log')
  67. self._tmp_dir = os.path.join(self._apache_dir, 'tmp')
  68. self._basic_passwords = os.path.join(self._conf_dir, 'basic.passwords')
  69. self._digest_passwords = os.path.join(self._conf_dir, 'digest.passwords')
  70. self._mods_dir = None
  71. self._auth_digest = True
  72. self._proxy_auth_basic = proxy_auth
  73. self._extra_configs = {}
  74. self._loaded_extra_configs = None
  75. assert env.apxs
  76. p = subprocess.run(args=[env.apxs, '-q', 'libexecdir'],
  77. capture_output=True, text=True)
  78. if p.returncode != 0:
  79. raise Exception(f'{env.apxs} failed to query libexecdir: {p}')
  80. self._mods_dir = p.stdout.strip()
  81. if self._mods_dir is None:
  82. raise Exception(f'apache modules dir cannot be found')
  83. if not os.path.exists(self._mods_dir):
  84. raise Exception(f'apache modules dir does not exist: {self._mods_dir}')
  85. self._process = None
  86. self._rmf(self._error_log)
  87. self._init_curltest()
  88. @property
  89. def docs_dir(self):
  90. return self._docs_dir
  91. def clear_logs(self):
  92. self._rmf(self._error_log)
  93. def exists(self):
  94. return os.path.exists(self._cmd)
  95. def set_extra_config(self, domain: str, lines: Optional[Union[str, List[str]]]):
  96. if lines is None:
  97. self._extra_configs.pop(domain, None)
  98. else:
  99. self._extra_configs[domain] = lines
  100. def clear_extra_configs(self):
  101. self._extra_configs = {}
  102. def set_proxy_auth(self, active: bool):
  103. self._proxy_auth_basic = active
  104. def _run(self, args, intext=''):
  105. env = {}
  106. for key, val in os.environ.items():
  107. env[key] = val
  108. env['APACHE_RUN_DIR'] = self._run_dir
  109. env['APACHE_RUN_USER'] = os.environ['USER']
  110. env['APACHE_LOCK_DIR'] = self._lock_dir
  111. env['APACHE_CONFDIR'] = self._apache_dir
  112. p = subprocess.run(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE,
  113. cwd=self.env.gen_dir,
  114. input=intext.encode() if intext else None,
  115. env=env)
  116. start = datetime.now()
  117. return ExecResult(args=args, exit_code=p.returncode,
  118. stdout=p.stdout.decode().splitlines(),
  119. stderr=p.stderr.decode().splitlines(),
  120. duration=datetime.now() - start)
  121. def _apachectl(self, cmd: str):
  122. args = [self.env.apachectl,
  123. "-d", self._apache_dir,
  124. "-f", self._conf_file,
  125. "-k", cmd]
  126. return self._run(args=args)
  127. def start(self):
  128. if self._process:
  129. self.stop()
  130. self._write_config()
  131. with open(self._error_log, 'a') as fd:
  132. fd.write('start of server\n')
  133. with open(os.path.join(self._apache_dir, 'xxx'), 'a') as fd:
  134. fd.write('start of server\n')
  135. r = self._apachectl('start')
  136. if r.exit_code != 0:
  137. log.error(f'failed to start httpd: {r}')
  138. return False
  139. self._loaded_extra_configs = copy.deepcopy(self._extra_configs)
  140. return self.wait_live(timeout=timedelta(seconds=5))
  141. def stop(self):
  142. r = self._apachectl('stop')
  143. self._loaded_extra_configs = None
  144. if r.exit_code == 0:
  145. return self.wait_dead(timeout=timedelta(seconds=5))
  146. log.fatal(f'stopping httpd failed: {r}')
  147. return r.exit_code == 0
  148. def restart(self):
  149. self.stop()
  150. return self.start()
  151. def reload(self):
  152. self._write_config()
  153. r = self._apachectl("graceful")
  154. self._loaded_extra_configs = None
  155. if r.exit_code != 0:
  156. log.error(f'failed to reload httpd: {r}')
  157. self._loaded_extra_configs = copy.deepcopy(self._extra_configs)
  158. return self.wait_live(timeout=timedelta(seconds=5))
  159. def reload_if_config_changed(self):
  160. if self._loaded_extra_configs == self._extra_configs:
  161. return True
  162. return self.reload()
  163. def wait_dead(self, timeout: timedelta):
  164. curl = CurlClient(env=self.env, run_dir=self._tmp_dir)
  165. try_until = datetime.now() + timeout
  166. while datetime.now() < try_until:
  167. r = curl.http_get(url=f'http://{self.env.domain1}:{self.env.http_port}/')
  168. if r.exit_code != 0:
  169. return True
  170. time.sleep(.1)
  171. log.debug(f"Server still responding after {timeout}")
  172. return False
  173. def wait_live(self, timeout: timedelta):
  174. curl = CurlClient(env=self.env, run_dir=self._tmp_dir,
  175. timeout=timeout.total_seconds())
  176. try_until = datetime.now() + timeout
  177. while datetime.now() < try_until:
  178. r = curl.http_get(url=f'http://{self.env.domain1}:{self.env.http_port}/')
  179. if r.exit_code == 0:
  180. return True
  181. time.sleep(.1)
  182. log.debug(f"Server still not responding after {timeout}")
  183. return False
  184. def _rmf(self, path):
  185. if os.path.exists(path):
  186. return os.remove(path)
  187. def _mkpath(self, path):
  188. if not os.path.exists(path):
  189. return os.makedirs(path)
  190. def _write_config(self):
  191. domain1 = self.env.domain1
  192. domain1brotli = self.env.domain1brotli
  193. creds1 = self.env.get_credentials(domain1)
  194. domain2 = self.env.domain2
  195. creds2 = self.env.get_credentials(domain2)
  196. proxy_domain = self.env.proxy_domain
  197. proxy_creds = self.env.get_credentials(proxy_domain)
  198. self._mkpath(self._conf_dir)
  199. self._mkpath(self._logs_dir)
  200. self._mkpath(self._tmp_dir)
  201. self._mkpath(os.path.join(self._docs_dir, 'two'))
  202. with open(os.path.join(self._docs_dir, 'data.json'), 'w') as fd:
  203. data = {
  204. 'server': f'{domain1}',
  205. }
  206. fd.write(JSONEncoder().encode(data))
  207. with open(os.path.join(self._docs_dir, 'two/data.json'), 'w') as fd:
  208. data = {
  209. 'server': f'{domain2}',
  210. }
  211. fd.write(JSONEncoder().encode(data))
  212. if self._proxy_auth_basic:
  213. with open(self._basic_passwords, 'w') as fd:
  214. fd.write('proxy:$apr1$FQfeInbs$WQZbODJlVg60j0ogEIlTW/\n')
  215. if self._auth_digest:
  216. with open(self._digest_passwords, 'w') as fd:
  217. fd.write('test:restricted area:57123e269fd73d71ae0656594e938e2f\n')
  218. self._mkpath(os.path.join(self.docs_dir, 'restricted/digest'))
  219. with open(os.path.join(self.docs_dir, 'restricted/digest/data.json'), 'w') as fd:
  220. fd.write('{"area":"digest"}\n')
  221. with open(self._conf_file, 'w') as fd:
  222. for m in self.MODULES:
  223. if os.path.exists(os.path.join(self._mods_dir, f'mod_{m}.so')):
  224. fd.write(f'LoadModule {m}_module "{self._mods_dir}/mod_{m}.so"\n')
  225. if Httpd.MOD_CURLTEST is not None:
  226. fd.write(f'LoadModule curltest_module \"{Httpd.MOD_CURLTEST}\"\n')
  227. conf = [ # base server config
  228. f'ServerRoot "{self._apache_dir}"',
  229. f'DefaultRuntimeDir logs',
  230. f'PidFile httpd.pid',
  231. f'ErrorLog {self._error_log}',
  232. f'LogLevel {self._get_log_level()}',
  233. f'StartServers 4',
  234. f'ReadBufferSize 16000',
  235. f'H2MinWorkers 16',
  236. f'H2MaxWorkers 256',
  237. f'Listen {self.env.http_port}',
  238. f'Listen {self.env.https_port}',
  239. f'Listen {self.env.proxy_port}',
  240. f'Listen {self.env.proxys_port}',
  241. f'TypesConfig "{self._conf_dir}/mime.types',
  242. f'SSLSessionCache "shmcb:ssl_gcache_data(32000)"',
  243. ]
  244. if 'base' in self._extra_configs:
  245. conf.extend(self._extra_configs['base'])
  246. conf.extend([ # plain http host for domain1
  247. f'<VirtualHost *:{self.env.http_port}>',
  248. f' ServerName {domain1}',
  249. f' ServerAlias localhost',
  250. f' DocumentRoot "{self._docs_dir}"',
  251. f' Protocols h2c http/1.1',
  252. f' H2Direct on',
  253. ])
  254. conf.extend(self._curltest_conf(domain1))
  255. conf.extend([
  256. f'</VirtualHost>',
  257. f'',
  258. ])
  259. conf.extend([ # https host for domain1, h1 + h2
  260. f'<VirtualHost *:{self.env.https_port}>',
  261. f' ServerName {domain1}',
  262. f' ServerAlias localhost',
  263. f' Protocols h2 http/1.1',
  264. f' SSLEngine on',
  265. f' SSLCertificateFile {creds1.cert_file}',
  266. f' SSLCertificateKeyFile {creds1.pkey_file}',
  267. f' DocumentRoot "{self._docs_dir}"',
  268. ])
  269. conf.extend(self._curltest_conf(domain1))
  270. if domain1 in self._extra_configs:
  271. conf.extend(self._extra_configs[domain1])
  272. conf.extend([
  273. f'</VirtualHost>',
  274. f'',
  275. ])
  276. # Alternate to domain1 with BROTLI compression
  277. conf.extend([ # https host for domain1, h1 + h2
  278. f'<VirtualHost *:{self.env.https_port}>',
  279. f' ServerName {domain1brotli}',
  280. f' Protocols h2 http/1.1',
  281. f' SSLEngine on',
  282. f' SSLCertificateFile {creds1.cert_file}',
  283. f' SSLCertificateKeyFile {creds1.pkey_file}',
  284. f' DocumentRoot "{self._docs_dir}"',
  285. f' SetOutputFilter BROTLI_COMPRESS',
  286. ])
  287. conf.extend(self._curltest_conf(domain1))
  288. if domain1 in self._extra_configs:
  289. conf.extend(self._extra_configs[domain1])
  290. conf.extend([
  291. f'</VirtualHost>',
  292. f'',
  293. ])
  294. conf.extend([ # plain http host for domain2
  295. f'<VirtualHost *:{self.env.http_port}>',
  296. f' ServerName {domain2}',
  297. f' ServerAlias localhost',
  298. f' DocumentRoot "{self._docs_dir}"',
  299. f' Protocols h2c http/1.1',
  300. ])
  301. conf.extend(self._curltest_conf(domain2))
  302. conf.extend([
  303. f'</VirtualHost>',
  304. f'',
  305. ])
  306. conf.extend([ # https host for domain2, no h2
  307. f'<VirtualHost *:{self.env.https_port}>',
  308. f' ServerName {domain2}',
  309. f' Protocols http/1.1',
  310. f' SSLEngine on',
  311. f' SSLCertificateFile {creds2.cert_file}',
  312. f' SSLCertificateKeyFile {creds2.pkey_file}',
  313. f' DocumentRoot "{self._docs_dir}/two"',
  314. ])
  315. conf.extend(self._curltest_conf(domain2))
  316. if domain2 in self._extra_configs:
  317. conf.extend(self._extra_configs[domain2])
  318. conf.extend([
  319. f'</VirtualHost>',
  320. f'',
  321. ])
  322. conf.extend([ # http forward proxy
  323. f'<VirtualHost *:{self.env.proxy_port}>',
  324. f' ServerName {proxy_domain}',
  325. f' Protocols h2c http/1.1',
  326. f' ProxyRequests On',
  327. f' H2ProxyRequests On',
  328. f' ProxyVia On',
  329. f' AllowCONNECT {self.env.http_port} {self.env.https_port}',
  330. ])
  331. conf.extend(self._get_proxy_conf())
  332. conf.extend([
  333. f'</VirtualHost>',
  334. f'',
  335. ])
  336. conf.extend([ # https forward proxy
  337. f'<VirtualHost *:{self.env.proxys_port}>',
  338. f' ServerName {proxy_domain}',
  339. f' Protocols h2 http/1.1',
  340. f' SSLEngine on',
  341. f' SSLCertificateFile {proxy_creds.cert_file}',
  342. f' SSLCertificateKeyFile {proxy_creds.pkey_file}',
  343. f' ProxyRequests On',
  344. f' H2ProxyRequests On',
  345. f' ProxyVia On',
  346. f' AllowCONNECT {self.env.http_port} {self.env.https_port}',
  347. ])
  348. conf.extend(self._get_proxy_conf())
  349. conf.extend([
  350. f'</VirtualHost>',
  351. f'',
  352. ])
  353. fd.write("\n".join(conf))
  354. with open(os.path.join(self._conf_dir, 'mime.types'), 'w') as fd:
  355. fd.write("\n".join([
  356. 'text/html html',
  357. 'application/json json',
  358. ''
  359. ]))
  360. def _get_proxy_conf(self):
  361. if self._proxy_auth_basic:
  362. return [
  363. f' <Proxy "*">',
  364. f' AuthType Basic',
  365. f' AuthName "Restricted Proxy"',
  366. f' AuthBasicProvider file',
  367. f' AuthUserFile "{self._basic_passwords}"',
  368. f' Require user proxy',
  369. f' </Proxy>',
  370. ]
  371. else:
  372. return [
  373. f' <Proxy "*">',
  374. f' Require ip 127.0.0.1',
  375. f' </Proxy>',
  376. ]
  377. def _get_log_level(self):
  378. if self.env.verbose > 3:
  379. return 'trace2'
  380. if self.env.verbose > 2:
  381. return 'trace1'
  382. if self.env.verbose > 1:
  383. return 'debug'
  384. return 'info'
  385. def _curltest_conf(self, servername) -> List[str]:
  386. lines = []
  387. if Httpd.MOD_CURLTEST is not None:
  388. lines.extend([
  389. f' Redirect 302 /data.json.302 /data.json',
  390. f' Redirect 301 /curltest/echo301 /curltest/echo',
  391. f' Redirect 302 /curltest/echo302 /curltest/echo',
  392. f' Redirect 303 /curltest/echo303 /curltest/echo',
  393. f' Redirect 307 /curltest/echo307 /curltest/echo',
  394. f' <Location /curltest/sslinfo>',
  395. f' SSLOptions StdEnvVars',
  396. f' SetHandler curltest-sslinfo',
  397. f' </Location>',
  398. f' <Location /curltest/echo>',
  399. f' SetHandler curltest-echo',
  400. f' </Location>',
  401. f' <Location /curltest/put>',
  402. f' SetHandler curltest-put',
  403. f' </Location>',
  404. f' <Location /curltest/tweak>',
  405. f' SetHandler curltest-tweak',
  406. f' </Location>',
  407. f' Redirect 302 /tweak /curltest/tweak',
  408. f' <Location /curltest/1_1>',
  409. f' SetHandler curltest-1_1-required',
  410. f' </Location>',
  411. f' <Location /curltest/shutdown_unclean>',
  412. f' SetHandler curltest-tweak',
  413. f' SetEnv force-response-1.0 1',
  414. f' </Location>',
  415. f' SetEnvIf Request_URI "/shutdown_unclean" ssl-unclean=1',
  416. ])
  417. if self._auth_digest:
  418. lines.extend([
  419. f' <Directory {self.docs_dir}/restricted/digest>',
  420. f' AuthType Digest',
  421. f' AuthName "restricted area"',
  422. f' AuthDigestDomain "https://{servername}"',
  423. f' AuthBasicProvider file',
  424. f' AuthUserFile "{self._digest_passwords}"',
  425. f' Require valid-user',
  426. f' </Directory>',
  427. ])
  428. return lines
  429. def _init_curltest(self):
  430. if Httpd.MOD_CURLTEST is not None:
  431. return
  432. local_dir = os.path.dirname(inspect.getfile(Httpd))
  433. p = subprocess.run([self.env.apxs, '-c', 'mod_curltest.c'],
  434. capture_output=True,
  435. cwd=os.path.join(local_dir, 'mod_curltest'))
  436. rv = p.returncode
  437. if rv != 0:
  438. log.error(f"compiling mod_curltest failed: {p.stderr}")
  439. raise Exception(f"compiling mod_curltest failed: {p.stderr}")
  440. Httpd.MOD_CURLTEST = os.path.join(
  441. local_dir, 'mod_curltest/.libs/mod_curltest.so')