update_lists.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387
  1. #!/usr/bin/env python3
  2. # Copyright (c) 2019 The ungoogled-chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. """
  6. Update binary pruning and domain substitution lists automatically.
  7. It will download and unpack into the source tree as necessary.
  8. No binary pruning or domain substitution will be applied to the source tree after
  9. the process has finished.
  10. """
  11. import argparse
  12. import os
  13. import sys
  14. from itertools import repeat
  15. from multiprocessing import Pool
  16. from pathlib import Path, PurePosixPath
  17. sys.path.insert(0, str(Path(__file__).resolve().parent.parent / 'utils'))
  18. from _common import get_logger
  19. from domain_substitution import DomainRegexList, TREE_ENCODINGS
  20. from prune_binaries import CONTINGENT_PATHS
  21. sys.path.pop(0)
  22. # Encoding for output files
  23. _ENCODING = 'UTF-8'
  24. # NOTE: Include patterns have precedence over exclude patterns
  25. # pathlib.Path.match() paths to include in binary pruning
  26. PRUNING_INCLUDE_PATTERNS = [
  27. 'components/domain_reliability/baked_in_configs/*',
  28. # Removals for patches/core/ungoogled-chromium/remove-unused-preferences-fields.patch
  29. 'components/safe_browsing/core/common/safe_browsing_prefs.cc',
  30. 'components/safe_browsing/core/common/safe_browsing_prefs.h',
  31. 'components/signin/public/base/signin_pref_names.cc',
  32. 'components/signin/public/base/signin_pref_names.h',
  33. ]
  34. # pathlib.Path.match() paths to exclude from binary pruning
  35. PRUNING_EXCLUDE_PATTERNS = [
  36. 'chrome/common/win/eventlog_messages.mc', # TODO: False positive textfile
  37. # Exclusions for DOM distiller (contains model data only)
  38. 'components/dom_distiller/core/data/distillable_page_model_new.bin',
  39. 'components/dom_distiller/core/data/long_page_model.bin',
  40. # Exclusions for GeoLanguage data
  41. # Details: https://docs.google.com/document/d/18WqVHz5F9vaUiE32E8Ge6QHmku2QSJKvlqB9JjnIM-g/edit
  42. # Introduced with: https://chromium.googlesource.com/chromium/src/+/6647da61
  43. 'components/language/content/browser/ulp_language_code_locator/geolanguage-data_rank0.bin',
  44. 'components/language/content/browser/ulp_language_code_locator/geolanguage-data_rank1.bin',
  45. 'components/language/content/browser/ulp_language_code_locator/geolanguage-data_rank2.bin',
  46. # Exclusion for required prebuilt object for Windows arm64 builds
  47. 'third_party/crashpad/crashpad/util/misc/capture_context_win_arm64.obj',
  48. 'third_party/icu/common/icudtl.dat', # Exclusion for ICU data
  49. # Exclusion for Android
  50. 'build/android/chromium-debug.keystore',
  51. 'third_party/icu/android/icudtl.dat',
  52. 'third_party/icu/common/icudtb.dat',
  53. # Exclusion for performance tracing
  54. 'third_party/perfetto/src/trace_processor/importers/proto/atoms.descriptor',
  55. # Exclusions for safe file extensions
  56. '*.avif',
  57. '*.ttf',
  58. '*.png',
  59. '*.jpg',
  60. '*.webp',
  61. '*.gif',
  62. '*.ico',
  63. '*.mp3',
  64. '*.wav',
  65. '*.flac',
  66. '*.icns',
  67. '*.woff',
  68. '*.woff2',
  69. '*makefile',
  70. '*.profdata',
  71. '*.xcf',
  72. '*.cur',
  73. '*.pdf',
  74. '*.ai',
  75. '*.h',
  76. '*.c',
  77. '*.cpp',
  78. '*.cc',
  79. '*.mk',
  80. '*.bmp',
  81. '*.py',
  82. '*.xml',
  83. '*.html',
  84. '*.js',
  85. '*.json',
  86. '*.txt',
  87. '*.xtb'
  88. ]
  89. # NOTE: Domain substitution path prefix exclusion has precedence over inclusion patterns
  90. # Paths to exclude by prefixes of the POSIX representation for domain substitution
  91. DOMAIN_EXCLUDE_PREFIXES = [
  92. 'components/test/',
  93. 'net/http/transport_security_state_static.json',
  94. 'net/http/transport_security_state_static_pins.json',
  95. # Exclusions for Visual Studio Project generation with GN (PR #445)
  96. 'tools/gn/',
  97. # Exclusions for files covered with other patches/unnecessary
  98. 'components/search_engines/prepopulated_engines.json',
  99. 'third_party/blink/renderer/core/dom/document.cc',
  100. ]
  101. # pathlib.Path.match() patterns to include in domain substitution
  102. DOMAIN_INCLUDE_PATTERNS = [
  103. '*.h', '*.hh', '*.hpp', '*.hxx', '*.cc', '*.cpp', '*.cxx', '*.c', '*.h', '*.json', '*.js',
  104. '*.html', '*.htm', '*.css', '*.py*', '*.grd*', '*.sql', '*.idl', '*.mk', '*.gyp*', 'makefile',
  105. '*.ts', '*.txt', '*.xml', '*.mm', '*.jinja*', '*.gn', '*.gni'
  106. ]
  107. # Binary-detection constant
  108. _TEXTCHARS = bytearray({7, 8, 9, 10, 12, 13, 27} | set(range(0x20, 0x100)) - {0x7f})
  109. class UnusedPatterns: #pylint: disable=too-few-public-methods
  110. """Tracks unused prefixes and patterns"""
  111. _all_names = ('pruning_include_patterns', 'pruning_exclude_patterns', 'domain_include_patterns',
  112. 'domain_exclude_prefixes')
  113. def __init__(self):
  114. # Initialize all tracked patterns and prefixes in sets
  115. # Users will discard elements that are used
  116. for name in self._all_names:
  117. setattr(self, name, set(globals()[name.upper()]))
  118. def log_unused(self, error=True):
  119. """
  120. Logs unused patterns and prefixes
  121. Returns True if there are unused patterns or prefixes; False otherwise
  122. """
  123. have_unused = False
  124. log = get_logger().error if error else get_logger().info
  125. for name in self._all_names:
  126. current_set = getattr(self, name, None)
  127. if current_set:
  128. log('Unused from %s: %s', name.upper(), current_set)
  129. have_unused = True
  130. return have_unused
  131. def _is_binary(bytes_data):
  132. """
  133. Returns True if the data seems to be binary data (i.e. not human readable); False otherwise
  134. """
  135. # From: https://stackoverflow.com/a/7392391
  136. return bool(bytes_data.translate(None, _TEXTCHARS))
  137. def _dir_empty(path):
  138. """
  139. Returns True if the directory is empty; False otherwise
  140. path is a pathlib.Path or string to a directory to test.
  141. """
  142. try:
  143. next(os.scandir(str(path)))
  144. except StopIteration:
  145. return True
  146. return False
  147. def should_prune(path, relative_path, used_pep_set, used_pip_set):
  148. """
  149. Returns True if a path should be pruned from the source tree; False otherwise
  150. path is the pathlib.Path to the file from the current working directory.
  151. relative_path is the pathlib.Path to the file from the source tree
  152. used_pep_set is a list of PRUNING_EXCLUDE_PATTERNS that have been matched
  153. used_pip_set is a list of PRUNING_INCLUDE_PATTERNS that have been matched
  154. """
  155. # Match against include patterns
  156. for pattern in filter(relative_path.match, PRUNING_INCLUDE_PATTERNS):
  157. used_pip_set.add(pattern)
  158. return True
  159. # Match against exclude patterns
  160. for pattern in filter(Path(str(relative_path).lower()).match, PRUNING_EXCLUDE_PATTERNS):
  161. used_pep_set.add(pattern)
  162. return False
  163. # Do binary data detection
  164. with path.open('rb') as file_obj:
  165. if _is_binary(file_obj.read()):
  166. return True
  167. # Passed all filtering; do not prune
  168. return False
  169. def _check_regex_match(file_path, search_regex):
  170. """
  171. Returns True if a regex pattern matches a file; False otherwise
  172. file_path is a pathlib.Path to the file to test
  173. search_regex is a compiled regex object to search for domain names
  174. """
  175. with file_path.open("rb") as file_obj:
  176. file_bytes = file_obj.read()
  177. content = None
  178. for encoding in TREE_ENCODINGS:
  179. try:
  180. content = file_bytes.decode(encoding)
  181. break
  182. except UnicodeDecodeError:
  183. continue
  184. if not search_regex.search(content) is None:
  185. return True
  186. return False
  187. def should_domain_substitute(path, relative_path, search_regex, used_dep_set, used_dip_set):
  188. """
  189. Returns True if a path should be domain substituted in the source tree; False otherwise
  190. path is the pathlib.Path to the file from the current working directory.
  191. relative_path is the pathlib.Path to the file from the source tree.
  192. used_dep_set is a list of DOMAIN_EXCLUDE_PREFIXES that have been matched
  193. used_dip_set is a list of DOMAIN_INCLUDE_PATTERNS that have been matched
  194. """
  195. relative_path_posix = relative_path.as_posix().lower()
  196. for include_pattern in DOMAIN_INCLUDE_PATTERNS:
  197. if PurePosixPath(relative_path_posix).match(include_pattern):
  198. used_dip_set.add(include_pattern)
  199. for exclude_prefix in DOMAIN_EXCLUDE_PREFIXES:
  200. if relative_path_posix.startswith(exclude_prefix):
  201. used_dep_set.add(exclude_prefix)
  202. return False
  203. return _check_regex_match(path, search_regex)
  204. return False
  205. def compute_lists_proc(path, source_tree, search_regex):
  206. """
  207. Adds the path to appropriate lists to be used by compute_lists.
  208. path is the pathlib.Path to the file from the current working directory.
  209. source_tree is a pathlib.Path to the source tree
  210. search_regex is a compiled regex object to search for domain names
  211. """
  212. used_pep_set = set() # PRUNING_EXCLUDE_PATTERNS
  213. used_pip_set = set() # PRUNING_INCLUDE_PATTERNS
  214. used_dep_set = set() # DOMAIN_EXCLUDE_PREFIXES
  215. used_dip_set = set() # DOMAIN_INCLUDE_PATTERNS
  216. pruning_set = set()
  217. domain_substitution_set = set()
  218. symlink_set = set()
  219. if path.is_file():
  220. relative_path = path.relative_to(source_tree)
  221. if not any(cpath in str(relative_path.as_posix()) for cpath in CONTINGENT_PATHS):
  222. if path.is_symlink():
  223. try:
  224. resolved_relative_posix = path.resolve().relative_to(source_tree).as_posix()
  225. symlink_set.add((resolved_relative_posix, relative_path.as_posix()))
  226. except ValueError:
  227. # Symlink leads out of the source tree
  228. pass
  229. elif not any(skip in ('.git', '__pycache__', 'uc_staging') for skip in path.parts):
  230. try:
  231. if should_prune(path, relative_path, used_pep_set, used_pip_set):
  232. pruning_set.add(relative_path.as_posix())
  233. elif should_domain_substitute(path, relative_path, search_regex, used_dep_set,
  234. used_dip_set):
  235. domain_substitution_set.add(relative_path.as_posix())
  236. except: #pylint: disable=bare-except
  237. get_logger().exception('Unhandled exception while processing %s', relative_path)
  238. return (used_pep_set, used_pip_set, used_dep_set, used_dip_set, pruning_set,
  239. domain_substitution_set, symlink_set)
  240. def compute_lists(source_tree, search_regex, processes): # pylint: disable=too-many-locals
  241. """
  242. Compute the binary pruning and domain substitution lists of the source tree.
  243. Returns a tuple of three items in the following order:
  244. 1. The sorted binary pruning list
  245. 2. The sorted domain substitution list
  246. 3. An UnusedPatterns object
  247. source_tree is a pathlib.Path to the source tree
  248. search_regex is a compiled regex object to search for domain names
  249. processes is the maximum number of worker processes to create
  250. """
  251. pruning_set = set()
  252. domain_substitution_set = set()
  253. symlink_set = set() # POSIX resolved path -> set of POSIX symlink paths
  254. source_tree = source_tree.resolve()
  255. unused_patterns = UnusedPatterns()
  256. # Launch multiple processes iterating over the source tree
  257. with Pool(processes) as procpool:
  258. returned_data = procpool.starmap(
  259. compute_lists_proc,
  260. zip(source_tree.rglob('*'), repeat(source_tree), repeat(search_regex)))
  261. # Handle the returned data
  262. for (used_pep_set, used_pip_set, used_dep_set, used_dip_set, returned_pruning_set,
  263. returned_domain_sub_set, returned_symlink_set) in returned_data:
  264. # pragma pylint: disable=no-member
  265. unused_patterns.pruning_exclude_patterns.difference_update(used_pep_set)
  266. unused_patterns.pruning_include_patterns.difference_update(used_pip_set)
  267. unused_patterns.domain_exclude_prefixes.difference_update(used_dep_set)
  268. unused_patterns.domain_include_patterns.difference_update(used_dip_set)
  269. # pragma pylint: enable=no-member
  270. pruning_set.update(returned_pruning_set)
  271. domain_substitution_set.update(returned_domain_sub_set)
  272. symlink_set.update(returned_symlink_set)
  273. # Prune symlinks for pruned files
  274. for (resolved, symlink) in symlink_set:
  275. if resolved in pruning_set:
  276. pruning_set.add(symlink)
  277. return sorted(pruning_set), sorted(domain_substitution_set), unused_patterns
  278. def main(args_list=None):
  279. """CLI entrypoint"""
  280. parser = argparse.ArgumentParser(description=__doc__)
  281. parser.add_argument('--pruning',
  282. metavar='PATH',
  283. type=Path,
  284. default='pruning.list',
  285. help='The path to store pruning.list. Default: %(default)s')
  286. parser.add_argument('--domain-substitution',
  287. metavar='PATH',
  288. type=Path,
  289. default='domain_substitution.list',
  290. help='The path to store domain_substitution.list. Default: %(default)s')
  291. parser.add_argument('--domain-regex',
  292. metavar='PATH',
  293. type=Path,
  294. default='domain_regex.list',
  295. help='The path to domain_regex.list. Default: %(default)s')
  296. parser.add_argument('-t',
  297. '--tree',
  298. metavar='PATH',
  299. type=Path,
  300. required=True,
  301. help='The path to the source tree to use.')
  302. parser.add_argument(
  303. '--processes',
  304. metavar='NUM',
  305. type=int,
  306. default=None,
  307. help=
  308. 'The maximum number of worker processes to create. Defaults to the number of system CPUs.')
  309. parser.add_argument('--domain-exclude-prefix',
  310. metavar='PREFIX',
  311. type=str,
  312. action='append',
  313. help='Additional exclusion for domain_substitution.list.')
  314. parser.add_argument('--no-error-unused',
  315. action='store_false',
  316. dest='error_unused',
  317. help='Do not treat unused patterns/prefixes as an error.')
  318. args = parser.parse_args(args_list)
  319. if args.domain_exclude_prefix is not None:
  320. DOMAIN_EXCLUDE_PREFIXES.extend(args.domain_exclude_prefix)
  321. if args.tree.exists() and not _dir_empty(args.tree):
  322. get_logger().info('Using existing source tree at %s', args.tree)
  323. else:
  324. get_logger().error('No source tree found. Aborting.')
  325. sys.exit(1)
  326. get_logger().info('Computing lists...')
  327. pruning_set, domain_substitution_set, unused_patterns = compute_lists(
  328. args.tree,
  329. DomainRegexList(args.domain_regex).search_regex, args.processes)
  330. with args.pruning.open('w', encoding=_ENCODING) as file_obj:
  331. file_obj.writelines('%s\n' % line for line in pruning_set)
  332. with args.domain_substitution.open('w', encoding=_ENCODING) as file_obj:
  333. file_obj.writelines('%s\n' % line for line in domain_substitution_set)
  334. if unused_patterns.log_unused(args.error_unused) and args.error_unused:
  335. get_logger().error('Please update or remove unused patterns and/or prefixes. '
  336. 'The lists have still been updated with the remaining valid entries.')
  337. sys.exit(1)
  338. if __name__ == "__main__":
  339. main()