update_lists.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377
  1. #!/usr/bin/env python3
  2. # Copyright (c) 2019 The ungoogled-chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. """
  6. Update binary pruning and domain substitution lists automatically.
  7. It will download and unpack into the source tree as necessary.
  8. No binary pruning or domain substitution will be applied to the source tree after
  9. the process has finished.
  10. """
  11. import argparse
  12. import os
  13. import sys
  14. from itertools import repeat
  15. from multiprocessing import Pool
  16. from pathlib import Path, PurePosixPath
  17. sys.path.insert(0, str(Path(__file__).resolve().parent.parent / 'utils'))
  18. from _common import get_logger
  19. from domain_substitution import DomainRegexList, TREE_ENCODINGS
  20. sys.path.pop(0)
  21. # Encoding for output files
  22. _ENCODING = 'UTF-8'
  23. # NOTE: Include patterns have precedence over exclude patterns
  24. # pathlib.Path.match() paths to include in binary pruning
  25. PRUNING_INCLUDE_PATTERNS = [
  26. 'components/domain_reliability/baked_in_configs/*',
  27. # Removals for patches/core/ungoogled-chromium/remove-unused-preferences-fields.patch
  28. 'components/safe_browsing/core/common/safe_browsing_prefs.cc',
  29. 'components/safe_browsing/core/common/safe_browsing_prefs.h',
  30. 'components/signin/public/base/signin_pref_names.cc',
  31. 'components/signin/public/base/signin_pref_names.h',
  32. ]
  33. # pathlib.Path.match() paths to exclude from binary pruning
  34. PRUNING_EXCLUDE_PATTERNS = [
  35. 'chrome/common/win/eventlog_messages.mc', # TODO: False positive textfile
  36. # TabRanker example preprocessor config
  37. # Details in chrome/browser/resource_coordinator/tab_ranker/README.md
  38. 'chrome/browser/resource_coordinator/tab_ranker/example_preprocessor_config.pb',
  39. 'chrome/browser/resource_coordinator/tab_ranker/pairwise_preprocessor_config.pb',
  40. # Exclusions for DOM distiller (contains model data only)
  41. 'components/dom_distiller/core/data/distillable_page_model_new.bin',
  42. 'components/dom_distiller/core/data/long_page_model.bin',
  43. # Exclusions for GeoLanguage data
  44. # Details: https://docs.google.com/document/d/18WqVHz5F9vaUiE32E8Ge6QHmku2QSJKvlqB9JjnIM-g/edit
  45. # Introduced with: https://chromium.googlesource.com/chromium/src/+/6647da61
  46. 'components/language/content/browser/ulp_language_code_locator/geolanguage-data_rank0.bin',
  47. 'components/language/content/browser/ulp_language_code_locator/geolanguage-data_rank1.bin',
  48. 'components/language/content/browser/ulp_language_code_locator/geolanguage-data_rank2.bin',
  49. # Exclusion for required prebuilt object for Windows arm64 builds
  50. 'third_party/crashpad/crashpad/util/misc/capture_context_win_arm64.obj',
  51. 'third_party/icu/common/icudtl.dat', # Exclusion for ICU data
  52. # Exclusion for Android
  53. 'build/android/chromium-debug.keystore',
  54. 'third_party/icu/android/icudtl.dat',
  55. 'third_party/icu/common/icudtb.dat',
  56. # Exclusion for performance tracing
  57. 'third_party/perfetto/src/trace_processor/importers/proto/atoms.descriptor',
  58. # Exclusions for safe file extensions
  59. '*.avif',
  60. '*.ttf',
  61. '*.png',
  62. '*.jpg',
  63. '*.webp',
  64. '*.gif',
  65. '*.ico',
  66. '*.mp3',
  67. '*.wav',
  68. '*.flac',
  69. '*.icns',
  70. '*.woff',
  71. '*.woff2',
  72. '*makefile',
  73. '*.xcf',
  74. '*.cur',
  75. '*.pdf',
  76. '*.ai',
  77. '*.h',
  78. '*.c',
  79. '*.cpp',
  80. '*.cc',
  81. '*.mk',
  82. '*.bmp',
  83. '*.py',
  84. '*.xml',
  85. '*.html',
  86. '*.js',
  87. '*.json',
  88. '*.txt',
  89. '*.xtb'
  90. ]
  91. # NOTE: Domain substitution path prefix exclusion has precedence over inclusion patterns
  92. # Paths to exclude by prefixes of the POSIX representation for domain substitution
  93. DOMAIN_EXCLUDE_PREFIXES = [
  94. 'components/test/',
  95. 'net/http/transport_security_state_static.json',
  96. # Exclusions for Visual Studio Project generation with GN (PR #445)
  97. 'tools/gn/src/gn/visual_studio_writer.cc',
  98. # Exclusions for files covered with other patches/unnecessary
  99. 'components/search_engines/prepopulated_engines.json',
  100. 'third_party/blink/renderer/core/dom/document.cc',
  101. ]
  102. # pathlib.Path.match() patterns to include in domain substitution
  103. DOMAIN_INCLUDE_PATTERNS = [
  104. '*.h', '*.hh', '*.hpp', '*.hxx', '*.cc', '*.cpp', '*.cxx', '*.c', '*.h', '*.json', '*.js',
  105. '*.html', '*.htm', '*.css', '*.py*', '*.grd*', '*.sql', '*.idl', '*.mk', '*.gyp*', 'makefile',
  106. '*.ts', '*.txt', '*.xml', '*.mm', '*.jinja*', '*.gn', '*.gni'
  107. ]
  108. # Binary-detection constant
  109. _TEXTCHARS = bytearray({7, 8, 9, 10, 12, 13, 27} | set(range(0x20, 0x100)) - {0x7f})
  110. class UnusedPatterns: #pylint: disable=too-few-public-methods
  111. """Tracks unused prefixes and patterns"""
  112. _all_names = ('pruning_include_patterns', 'pruning_exclude_patterns', 'domain_include_patterns',
  113. 'domain_exclude_prefixes')
  114. def __init__(self):
  115. # Initialize all tracked patterns and prefixes in sets
  116. # Users will discard elements that are used
  117. for name in self._all_names:
  118. setattr(self, name, set(globals()[name.upper()]))
  119. def log_unused(self):
  120. """
  121. Logs unused patterns and prefixes
  122. Returns True if there are unused patterns or prefixes; False otherwise
  123. """
  124. have_unused = False
  125. for name in self._all_names:
  126. current_set = getattr(self, name, None)
  127. if current_set:
  128. get_logger().error('Unused from %s: %s', name.upper(), current_set)
  129. have_unused = True
  130. return have_unused
  131. def _is_binary(bytes_data):
  132. """
  133. Returns True if the data seems to be binary data (i.e. not human readable); False otherwise
  134. """
  135. # From: https://stackoverflow.com/a/7392391
  136. return bool(bytes_data.translate(None, _TEXTCHARS))
  137. def _dir_empty(path):
  138. """
  139. Returns True if the directory is empty; False otherwise
  140. path is a pathlib.Path or string to a directory to test.
  141. """
  142. try:
  143. next(os.scandir(str(path)))
  144. except StopIteration:
  145. return True
  146. return False
  147. def should_prune(path, relative_path, used_pep_set, used_pip_set):
  148. """
  149. Returns True if a path should be pruned from the source tree; False otherwise
  150. path is the pathlib.Path to the file from the current working directory.
  151. relative_path is the pathlib.Path to the file from the source tree
  152. used_pep_set is a list of PRUNING_EXCLUDE_PATTERNS that have been matched
  153. used_pip_set is a list of PRUNING_INCLUDE_PATTERNS that have been matched
  154. """
  155. # Match against include patterns
  156. for pattern in filter(relative_path.match, PRUNING_INCLUDE_PATTERNS):
  157. used_pip_set.add(pattern)
  158. return True
  159. # Match against exclude patterns
  160. for pattern in filter(Path(str(relative_path).lower()).match, PRUNING_EXCLUDE_PATTERNS):
  161. used_pep_set.add(pattern)
  162. return False
  163. # Do binary data detection
  164. with path.open('rb') as file_obj:
  165. if _is_binary(file_obj.read()):
  166. return True
  167. # Passed all filtering; do not prune
  168. return False
  169. def _check_regex_match(file_path, search_regex):
  170. """
  171. Returns True if a regex pattern matches a file; False otherwise
  172. file_path is a pathlib.Path to the file to test
  173. search_regex is a compiled regex object to search for domain names
  174. """
  175. with file_path.open("rb") as file_obj:
  176. file_bytes = file_obj.read()
  177. content = None
  178. for encoding in TREE_ENCODINGS:
  179. try:
  180. content = file_bytes.decode(encoding)
  181. break
  182. except UnicodeDecodeError:
  183. continue
  184. if not search_regex.search(content) is None:
  185. return True
  186. return False
  187. def should_domain_substitute(path, relative_path, search_regex, used_dep_set, used_dip_set):
  188. """
  189. Returns True if a path should be domain substituted in the source tree; False otherwise
  190. path is the pathlib.Path to the file from the current working directory.
  191. relative_path is the pathlib.Path to the file from the source tree.
  192. used_dep_set is a list of DOMAIN_EXCLUDE_PREFIXES that have been matched
  193. used_dip_set is a list of DOMAIN_INCLUDE_PATTERNS that have been matched
  194. """
  195. relative_path_posix = relative_path.as_posix().lower()
  196. for include_pattern in DOMAIN_INCLUDE_PATTERNS:
  197. if PurePosixPath(relative_path_posix).match(include_pattern):
  198. used_dip_set.add(include_pattern)
  199. for exclude_prefix in DOMAIN_EXCLUDE_PREFIXES:
  200. if relative_path_posix.startswith(exclude_prefix):
  201. used_dep_set.add(exclude_prefix)
  202. return False
  203. return _check_regex_match(path, search_regex)
  204. return False
  205. def compute_lists_proc(path, source_tree, search_regex):
  206. """
  207. Adds the path to appropriate lists to be used by compute_lists.
  208. path is the pathlib.Path to the file from the current working directory.
  209. source_tree is a pathlib.Path to the source tree
  210. search_regex is a compiled regex object to search for domain names
  211. """
  212. used_pep_set = set() # PRUNING_EXCLUDE_PATTERNS
  213. used_pip_set = set() # PRUNING_INCLUDE_PATTERNS
  214. used_dep_set = set() # DOMAIN_EXCLUDE_PREFIXES
  215. used_dip_set = set() # DOMAIN_INCLUDE_PATTERNS
  216. pruning_set = set()
  217. domain_substitution_set = set()
  218. symlink_set = set()
  219. if path.is_file():
  220. relative_path = path.relative_to(source_tree)
  221. if path.is_symlink():
  222. try:
  223. resolved_relative_posix = path.resolve().relative_to(source_tree).as_posix()
  224. symlink_set.add((resolved_relative_posix, relative_path.as_posix()))
  225. except ValueError:
  226. # Symlink leads out of the source tree
  227. pass
  228. else:
  229. try:
  230. if should_prune(path, relative_path, used_pep_set, used_pip_set):
  231. pruning_set.add(relative_path.as_posix())
  232. elif should_domain_substitute(path, relative_path, search_regex, used_dep_set,
  233. used_dip_set):
  234. domain_substitution_set.add(relative_path.as_posix())
  235. except: #pylint: disable=bare-except
  236. get_logger().exception('Unhandled exception while processing %s', relative_path)
  237. return (used_pep_set, used_pip_set, used_dep_set, used_dip_set, pruning_set,
  238. domain_substitution_set, symlink_set)
  239. def compute_lists(source_tree, search_regex, processes):
  240. """
  241. Compute the binary pruning and domain substitution lists of the source tree.
  242. Returns a tuple of three items in the following order:
  243. 1. The sorted binary pruning list
  244. 2. The sorted domain substitution list
  245. 3. An UnusedPatterns object
  246. source_tree is a pathlib.Path to the source tree
  247. search_regex is a compiled regex object to search for domain names
  248. processes is the maximum number of worker processes to create
  249. """
  250. pruning_set = set()
  251. domain_substitution_set = set()
  252. symlink_set = set() # POSIX resolved path -> set of POSIX symlink paths
  253. source_tree = source_tree.resolve()
  254. unused_patterns = UnusedPatterns()
  255. # Launch multiple processes iterating over the source tree
  256. with Pool(processes) as procpool:
  257. returned_data = procpool.starmap(
  258. compute_lists_proc,
  259. zip(source_tree.rglob('*'), repeat(source_tree), repeat(search_regex)))
  260. # Handle the returned data
  261. for (used_pep_set, used_pip_set, used_dep_set, used_dip_set, returned_pruning_set,
  262. returned_domain_sub_set, returned_symlink_set) in returned_data:
  263. unused_patterns.pruning_exclude_patterns.difference_update(used_pep_set)
  264. unused_patterns.pruning_include_patterns.difference_update(used_pip_set)
  265. unused_patterns.domain_exclude_prefixes.difference_update(used_dep_set)
  266. unused_patterns.domain_include_patterns.difference_update(used_dip_set)
  267. pruning_set.update(returned_pruning_set)
  268. domain_substitution_set.update(returned_domain_sub_set)
  269. symlink_set.update(returned_symlink_set)
  270. # Prune symlinks for pruned files
  271. for (resolved, symlink) in symlink_set:
  272. if resolved in pruning_set:
  273. pruning_set.add(symlink)
  274. return sorted(pruning_set), sorted(domain_substitution_set), unused_patterns
  275. def main(args_list=None):
  276. """CLI entrypoint"""
  277. parser = argparse.ArgumentParser(description=__doc__)
  278. parser.add_argument(
  279. '--pruning',
  280. metavar='PATH',
  281. type=Path,
  282. default='pruning.list',
  283. help='The path to store pruning.list. Default: %(default)s')
  284. parser.add_argument(
  285. '--domain-substitution',
  286. metavar='PATH',
  287. type=Path,
  288. default='domain_substitution.list',
  289. help='The path to store domain_substitution.list. Default: %(default)s')
  290. parser.add_argument(
  291. '--domain-regex',
  292. metavar='PATH',
  293. type=Path,
  294. default='domain_regex.list',
  295. help='The path to domain_regex.list. Default: %(default)s')
  296. parser.add_argument(
  297. '-t',
  298. '--tree',
  299. metavar='PATH',
  300. type=Path,
  301. required=True,
  302. help='The path to the source tree to use.')
  303. parser.add_argument(
  304. '--processes',
  305. metavar='NUM',
  306. type=int,
  307. default=None,
  308. help=
  309. 'The maximum number of worker processes to create. Defaults to the number of system CPUs.')
  310. args = parser.parse_args(args_list)
  311. if args.tree.exists() and not _dir_empty(args.tree):
  312. get_logger().info('Using existing source tree at %s', args.tree)
  313. else:
  314. get_logger().error('No source tree found. Aborting.')
  315. exit(1)
  316. get_logger().info('Computing lists...')
  317. pruning_set, domain_substitution_set, unused_patterns = compute_lists(
  318. args.tree,
  319. DomainRegexList(args.domain_regex).search_regex, args.processes)
  320. with args.pruning.open('w', encoding=_ENCODING) as file_obj:
  321. file_obj.writelines('%s\n' % line for line in pruning_set)
  322. with args.domain_substitution.open('w', encoding=_ENCODING) as file_obj:
  323. file_obj.writelines('%s\n' % line for line in domain_substitution_set)
  324. if unused_patterns.log_unused():
  325. get_logger().error('Please update or remove unused patterns and/or prefixes. '
  326. 'The lists have still been updated with the remaining valid entries.')
  327. exit(1)
  328. if __name__ == "__main__":
  329. main()