downloads.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439
  1. #!/usr/bin/env python3
  2. # -*- coding: UTF-8 -*-
  3. # Copyright (c) 2019 The ungoogled-chromium Authors. All rights reserved.
  4. # Use of this source code is governed by a BSD-style license that can be
  5. # found in the LICENSE file.
  6. """
  7. Module for the downloading, checking, and unpacking of necessary files into the source tree.
  8. """
  9. import argparse
  10. import configparser
  11. import enum
  12. import hashlib
  13. import shutil
  14. import ssl
  15. import subprocess
  16. import sys
  17. import urllib.request
  18. from pathlib import Path
  19. from _common import ENCODING, USE_REGISTRY, ExtractorEnum, get_logger, \
  20. get_chromium_version, add_common_params
  21. from _extraction import extract_tar_file, extract_with_7z, extract_with_winrar
  22. sys.path.insert(0, str(Path(__file__).parent / 'third_party'))
  23. import schema #pylint: disable=wrong-import-position, wrong-import-order
  24. sys.path.pop(0)
  25. # Constants
  26. class HashesURLEnum(str, enum.Enum):
  27. """Enum for supported hash URL schemes"""
  28. CHROMIUM = 'chromium'
  29. class HashMismatchError(BaseException):
  30. """Exception for computed hashes not matching expected hashes"""
  31. class DownloadInfo: #pylint: disable=too-few-public-methods
  32. """Representation of an downloads.ini file for downloading files"""
  33. _hashes = ('md5', 'sha1', 'sha256', 'sha512')
  34. hash_url_delimiter = '|'
  35. _nonempty_keys = ('url', 'download_filename')
  36. _optional_keys = (
  37. 'version',
  38. 'strip_leading_dirs',
  39. )
  40. _passthrough_properties = (*_nonempty_keys, *_optional_keys, 'extractor', 'output_path')
  41. _ini_vars = {
  42. '_chromium_version': get_chromium_version(),
  43. }
  44. @staticmethod
  45. def _is_hash_url(value):
  46. return value.count(DownloadInfo.hash_url_delimiter) == 2 and value.split(
  47. DownloadInfo.hash_url_delimiter)[0] in iter(HashesURLEnum)
  48. _schema = schema.Schema({
  49. schema.Optional(schema.And(str, len)): {
  50. **{x: schema.And(str, len)
  51. for x in _nonempty_keys},
  52. 'output_path': (lambda x: str(Path(x).relative_to(''))),
  53. **{schema.Optional(x): schema.And(str, len)
  54. for x in _optional_keys},
  55. schema.Optional('extractor'): schema.Or(ExtractorEnum.TAR, ExtractorEnum.SEVENZIP,
  56. ExtractorEnum.WINRAR),
  57. schema.Optional(schema.Or(*_hashes)): schema.And(str, len),
  58. schema.Optional('hash_url'): lambda x: DownloadInfo._is_hash_url(x), #pylint: disable=unnecessary-lambda
  59. }
  60. })
  61. class _DownloadsProperties: #pylint: disable=too-few-public-methods
  62. def __init__(self, section_dict, passthrough_properties, hashes):
  63. self._section_dict = section_dict
  64. self._passthrough_properties = passthrough_properties
  65. self._hashes = hashes
  66. def has_hash_url(self):
  67. """
  68. Returns a boolean indicating whether the current
  69. download has a hash URL"""
  70. return 'hash_url' in self._section_dict
  71. def __getattr__(self, name):
  72. if name in self._passthrough_properties:
  73. return self._section_dict.get(name, fallback=None)
  74. if name == 'hashes':
  75. hashes_dict = {}
  76. for hash_name in (*self._hashes, 'hash_url'):
  77. value = self._section_dict.get(hash_name, fallback=None)
  78. if value:
  79. if hash_name == 'hash_url':
  80. value = value.split(DownloadInfo.hash_url_delimiter)
  81. hashes_dict[hash_name] = value
  82. return hashes_dict
  83. raise AttributeError('"{}" has no attribute "{}"'.format(type(self).__name__, name))
  84. def _parse_data(self, path):
  85. """
  86. Parses an INI file located at path
  87. Raises schema.SchemaError if validation fails
  88. """
  89. def _section_generator(data):
  90. for section in data:
  91. if section == configparser.DEFAULTSECT:
  92. continue
  93. yield section, dict(
  94. filter(lambda x: x[0] not in self._ini_vars, data.items(section)))
  95. new_data = configparser.ConfigParser(defaults=self._ini_vars)
  96. with path.open(encoding=ENCODING) as ini_file:
  97. new_data.read_file(ini_file, source=str(path))
  98. try:
  99. self._schema.validate(dict(_section_generator(new_data)))
  100. except schema.SchemaError as exc:
  101. get_logger().error('downloads.ini failed schema validation (located in %s)', path)
  102. raise exc
  103. return new_data
  104. def __init__(self, ini_paths):
  105. """Reads an iterable of pathlib.Path to download.ini files"""
  106. self._data = configparser.ConfigParser()
  107. for path in ini_paths:
  108. self._data.read_dict(self._parse_data(path))
  109. def __getitem__(self, section):
  110. """
  111. Returns an object with keys as attributes and
  112. values already pre-processed strings
  113. """
  114. return self._DownloadsProperties(self._data[section], self._passthrough_properties,
  115. self._hashes)
  116. def __contains__(self, item):
  117. """
  118. Returns True if item is a name of a section; False otherwise.
  119. """
  120. return self._data.has_section(item)
  121. def __iter__(self):
  122. """Returns an iterator over the section names"""
  123. return iter(self._data.sections())
  124. def properties_iter(self):
  125. """Iterator for the download properties sorted by output path"""
  126. return sorted(map(lambda x: (x, self[x]), self),
  127. key=(lambda x: str(Path(x[1].output_path))))
  128. class _UrlRetrieveReportHook: #pylint: disable=too-few-public-methods
  129. """Hook for urllib.request.urlretrieve to log progress information to console"""
  130. def __init__(self):
  131. self._max_len_printed = 0
  132. self._last_percentage = None
  133. def __call__(self, block_count, block_size, total_size):
  134. # Use total_blocks to handle case total_size < block_size
  135. # total_blocks is ceiling of total_size / block_size
  136. # Ceiling division from: https://stackoverflow.com/a/17511341
  137. total_blocks = -(-total_size // block_size)
  138. if total_blocks > 0:
  139. # Do not needlessly update the console. Since the console is
  140. # updated synchronously, we don't want updating the console to
  141. # bottleneck downloading. Thus, only refresh the output when the
  142. # displayed value should change.
  143. percentage = round(block_count / total_blocks, ndigits=3)
  144. if percentage == self._last_percentage:
  145. return
  146. self._last_percentage = percentage
  147. print('\r' + ' ' * self._max_len_printed, end='')
  148. status_line = 'Progress: {:.1%} of {:,d} B'.format(percentage, total_size)
  149. else:
  150. downloaded_estimate = block_count * block_size
  151. status_line = 'Progress: {:,d} B of unknown size'.format(downloaded_estimate)
  152. self._max_len_printed = len(status_line)
  153. print('\r' + status_line, end='')
  154. def _download_via_urllib(url, file_path, show_progress, disable_ssl_verification):
  155. reporthook = None
  156. if show_progress:
  157. reporthook = _UrlRetrieveReportHook()
  158. if disable_ssl_verification:
  159. # TODO: Remove this or properly implement disabling SSL certificate verification
  160. orig_https_context = ssl._create_default_https_context #pylint: disable=protected-access
  161. ssl._create_default_https_context = ssl._create_unverified_context #pylint: disable=protected-access
  162. try:
  163. urllib.request.urlretrieve(url, str(file_path), reporthook=reporthook)
  164. finally:
  165. # Try to reduce damage of hack by reverting original HTTPS context ASAP
  166. if disable_ssl_verification:
  167. ssl._create_default_https_context = orig_https_context #pylint: disable=protected-access
  168. if show_progress:
  169. print()
  170. def _download_if_needed(file_path, url, show_progress, disable_ssl_verification):
  171. """
  172. Downloads a file from url to the specified path file_path if necessary.
  173. If show_progress is True, download progress is printed to the console.
  174. """
  175. if file_path.exists():
  176. get_logger().info('%s already exists. Skipping download.', file_path)
  177. return
  178. # File name for partially download file
  179. tmp_file_path = file_path.with_name(file_path.name + '.partial')
  180. if tmp_file_path.exists():
  181. get_logger().debug('Resuming downloading URL %s ...', url)
  182. else:
  183. get_logger().debug('Downloading URL %s ...', url)
  184. # Perform download
  185. if shutil.which('curl'):
  186. get_logger().debug('Using curl')
  187. try:
  188. subprocess.run(['curl', '-fL', '-o', str(tmp_file_path), '-C', '-', url], check=True)
  189. except subprocess.CalledProcessError as exc:
  190. get_logger().error('curl failed. Re-run the download command to resume downloading.')
  191. raise exc
  192. else:
  193. get_logger().debug('Using urllib')
  194. _download_via_urllib(url, tmp_file_path, show_progress, disable_ssl_verification)
  195. # Download complete; rename file
  196. tmp_file_path.rename(file_path)
  197. def _chromium_hashes_generator(hashes_path):
  198. with hashes_path.open(encoding=ENCODING) as hashes_file:
  199. hash_lines = hashes_file.read().splitlines()
  200. for hash_name, hash_hex, _ in map(lambda x: x.lower().split(' '), hash_lines):
  201. if hash_name in hashlib.algorithms_available:
  202. yield hash_name, hash_hex
  203. else:
  204. get_logger().warning('Skipping unknown hash algorithm: %s', hash_name)
  205. def _get_hash_pairs(download_properties, cache_dir):
  206. """Generator of (hash_name, hash_hex) for the given download"""
  207. for entry_type, entry_value in download_properties.hashes.items():
  208. if entry_type == 'hash_url':
  209. hash_processor, hash_filename, _ = entry_value
  210. if hash_processor == 'chromium':
  211. yield from _chromium_hashes_generator(cache_dir / hash_filename)
  212. else:
  213. raise ValueError('Unknown hash_url processor: %s' % hash_processor)
  214. else:
  215. yield entry_type, entry_value
  216. def retrieve_downloads(download_info, cache_dir, show_progress, disable_ssl_verification=False):
  217. """
  218. Retrieve downloads into the downloads cache.
  219. download_info is the DowloadInfo of downloads to retrieve.
  220. cache_dir is the pathlib.Path to the downloads cache.
  221. show_progress is a boolean indicating if download progress is printed to the console.
  222. disable_ssl_verification is a boolean indicating if certificate verification
  223. should be disabled for downloads using HTTPS.
  224. Raises FileNotFoundError if the downloads path does not exist.
  225. Raises NotADirectoryError if the downloads path is not a directory.
  226. """
  227. if not cache_dir.exists():
  228. raise FileNotFoundError(cache_dir)
  229. if not cache_dir.is_dir():
  230. raise NotADirectoryError(cache_dir)
  231. for download_name, download_properties in download_info.properties_iter():
  232. get_logger().info('Downloading "%s" to "%s" ...', download_name,
  233. download_properties.download_filename)
  234. download_path = cache_dir / download_properties.download_filename
  235. _download_if_needed(download_path, download_properties.url, show_progress,
  236. disable_ssl_verification)
  237. if download_properties.has_hash_url():
  238. get_logger().info('Downloading hashes for "%s"', download_name)
  239. _, hash_filename, hash_url = download_properties.hashes['hash_url']
  240. _download_if_needed(cache_dir / hash_filename, hash_url, show_progress,
  241. disable_ssl_verification)
  242. def check_downloads(download_info, cache_dir):
  243. """
  244. Check integrity of the downloads cache.
  245. download_info is the DownloadInfo of downloads to unpack.
  246. cache_dir is the pathlib.Path to the downloads cache.
  247. Raises source_retrieval.HashMismatchError when the computed and expected hashes do not match.
  248. """
  249. for download_name, download_properties in download_info.properties_iter():
  250. get_logger().info('Verifying hashes for "%s" ...', download_name)
  251. download_path = cache_dir / download_properties.download_filename
  252. with download_path.open('rb') as file_obj:
  253. archive_data = file_obj.read()
  254. for hash_name, hash_hex in _get_hash_pairs(download_properties, cache_dir):
  255. get_logger().debug('Verifying %s hash...', hash_name)
  256. hasher = hashlib.new(hash_name, data=archive_data)
  257. if not hasher.hexdigest().lower() == hash_hex.lower():
  258. raise HashMismatchError(download_path)
  259. def unpack_downloads(download_info, cache_dir, output_dir, skip_unused, extractors=None):
  260. """
  261. Unpack downloads in the downloads cache to output_dir. Assumes all downloads are retrieved.
  262. download_info is the DownloadInfo of downloads to unpack.
  263. cache_dir is the pathlib.Path directory containing the download cache
  264. output_dir is the pathlib.Path directory to unpack the downloads to.
  265. extractors is a dictionary of PlatformEnum to a command or path to the
  266. extractor binary. Defaults to 'tar' for tar, and '_use_registry' for 7-Zip and WinRAR.
  267. May raise undetermined exceptions during archive unpacking.
  268. """
  269. for download_name, download_properties in download_info.properties_iter():
  270. download_path = cache_dir / download_properties.download_filename
  271. get_logger().info('Unpacking "%s" to %s ...', download_name,
  272. download_properties.output_path)
  273. extractor_name = download_properties.extractor or ExtractorEnum.TAR
  274. if extractor_name == ExtractorEnum.SEVENZIP:
  275. extractor_func = extract_with_7z
  276. elif extractor_name == ExtractorEnum.WINRAR:
  277. extractor_func = extract_with_winrar
  278. elif extractor_name == ExtractorEnum.TAR:
  279. extractor_func = extract_tar_file
  280. else:
  281. raise NotImplementedError(extractor_name)
  282. if download_properties.strip_leading_dirs is None:
  283. strip_leading_dirs_path = None
  284. else:
  285. strip_leading_dirs_path = Path(download_properties.strip_leading_dirs)
  286. extractor_func(archive_path=download_path,
  287. output_dir=output_dir / Path(download_properties.output_path),
  288. relative_to=strip_leading_dirs_path,
  289. skip_unused=skip_unused,
  290. extractors=extractors)
  291. def _add_common_args(parser):
  292. parser.add_argument(
  293. '-i',
  294. '--ini',
  295. type=Path,
  296. nargs='+',
  297. help='The downloads INI to parse for downloads. Can be specified multiple times.')
  298. parser.add_argument('-c',
  299. '--cache',
  300. type=Path,
  301. required=True,
  302. help='Path to the directory to cache downloads.')
  303. def _retrieve_callback(args):
  304. retrieve_downloads(DownloadInfo(args.ini), args.cache, args.show_progress,
  305. args.disable_ssl_verification)
  306. try:
  307. check_downloads(DownloadInfo(args.ini), args.cache)
  308. except HashMismatchError as exc:
  309. get_logger().error('File checksum does not match: %s', exc)
  310. sys.exit(1)
  311. def _unpack_callback(args):
  312. extractors = {
  313. ExtractorEnum.SEVENZIP: args.sevenz_path,
  314. ExtractorEnum.WINRAR: args.winrar_path,
  315. ExtractorEnum.TAR: args.tar_path,
  316. }
  317. unpack_downloads(DownloadInfo(args.ini), args.cache, args.output, args.skip_unused, extractors)
  318. def main():
  319. """CLI Entrypoint"""
  320. parser = argparse.ArgumentParser(description=__doc__)
  321. add_common_params(parser)
  322. subparsers = parser.add_subparsers(title='Download actions', dest='action')
  323. # retrieve
  324. retrieve_parser = subparsers.add_parser(
  325. 'retrieve',
  326. help='Retrieve and check download files',
  327. description=('Retrieves and checks downloads without unpacking. '
  328. 'The downloader will attempt to use CLI command "curl". '
  329. 'If it is not present, Python\'s urllib will be used. However, only '
  330. 'the CLI-based downloaders can be resumed if the download is aborted.'))
  331. _add_common_args(retrieve_parser)
  332. retrieve_parser.add_argument('--hide-progress-bar',
  333. action='store_false',
  334. dest='show_progress',
  335. help='Hide the download progress.')
  336. retrieve_parser.add_argument(
  337. '--disable-ssl-verification',
  338. action='store_true',
  339. help='Disables certification verification for downloads using HTTPS.')
  340. retrieve_parser.set_defaults(callback=_retrieve_callback)
  341. # unpack
  342. unpack_parser = subparsers.add_parser(
  343. 'unpack',
  344. help='Unpack download files',
  345. description='Verifies hashes of and unpacks download files into the specified directory.')
  346. _add_common_args(unpack_parser)
  347. unpack_parser.add_argument('--tar-path',
  348. default='tar',
  349. help=('(Linux and macOS only) Command or path to the BSD or GNU tar '
  350. 'binary for extraction. Default: %(default)s'))
  351. unpack_parser.add_argument(
  352. '--7z-path',
  353. dest='sevenz_path',
  354. default=USE_REGISTRY,
  355. help=('Command or path to 7-Zip\'s "7z" binary. If "_use_registry" is '
  356. 'specified, determine the path from the registry. Default: %(default)s'))
  357. unpack_parser.add_argument(
  358. '--winrar-path',
  359. dest='winrar_path',
  360. default=USE_REGISTRY,
  361. help=('Command or path to WinRAR\'s "winrar" binary. If "_use_registry" is '
  362. 'specified, determine the path from the registry. Default: %(default)s'))
  363. unpack_parser.add_argument('output', type=Path, help='The directory to unpack to.')
  364. unpack_parser.add_argument('--skip-unused',
  365. action='store_true',
  366. help='Skip extraction of unused directories (CONTINGENT_PATHS).')
  367. unpack_parser.set_defaults(callback=_unpack_callback)
  368. args = parser.parse_args()
  369. args.callback(args)
  370. if __name__ == '__main__':
  371. main()