prune_binaries.py 6.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162
  1. #!/usr/bin/env python3
  2. # -*- coding: UTF-8 -*-
  3. # Copyright (c) 2019 The ungoogled-chromium Authors. All rights reserved.
  4. # Use of this source code is governed by a BSD-style license that can be
  5. # found in the LICENSE file.
  6. """Prune binaries from the source tree"""
  7. import argparse
  8. import itertools
  9. import sys
  10. import os
  11. import stat
  12. from pathlib import Path
  13. from _common import ENCODING, get_logger, add_common_params
  14. # List of paths to prune if they exist, excluded from domain_substitution and pruning lists
  15. # These allow the lists to be compatible between cloned and tarball sources
  16. CONTINGENT_PATHS = (
  17. # Overridable git sources
  18. 'third_party/angle/third_party/VK-GL-CTS/src/',
  19. 'third_party/instrumented_libs/',
  20. # CIPD sources
  21. 'buildtools/linux64/',
  22. 'buildtools/reclient/',
  23. 'third_party/apache-linux/',
  24. 'third_party/checkstyle/',
  25. 'third_party/google-java-format/',
  26. 'third_party/libei/',
  27. 'third_party/ninja/',
  28. 'third_party/screen-ai/',
  29. 'third_party/siso/',
  30. 'third_party/updater/chrome_linux64/',
  31. 'third_party/updater/chromium_linux64/',
  32. 'tools/luci-go/',
  33. 'tools/resultdb/',
  34. 'tools/skia_goldctl/linux/',
  35. # GCS sources
  36. 'base/tracing/test/data',
  37. 'build/linux/debian_bullseye_amd64-sysroot/',
  38. 'build/linux/debian_bullseye_i386-sysroot/',
  39. 'buildtools/linux64-format/',
  40. 'third_party/blink/renderer/core/css/perftest_data/',
  41. 'third_party/js_code_coverage/',
  42. 'third_party/llvm-build/Release+Asserts/',
  43. 'third_party/node/linux/',
  44. 'third_party/opus/tests/resources/',
  45. 'third_party/rust-toolchain/',
  46. 'third_party/subresource-filter-ruleset/data',
  47. 'third_party/test_fonts/test_fonts',
  48. 'third_party/tfhub_models/testdata/',
  49. 'tools/perf/page_sets/maps_perf_test/dataset/',
  50. )
  51. def prune_files(unpack_root, prune_list):
  52. """
  53. Delete files under unpack_root listed in prune_list. Returns an iterable of unremovable files.
  54. unpack_root is a pathlib.Path to the directory to be pruned
  55. prune_list is an iterable of files to be removed.
  56. """
  57. unremovable_files = set()
  58. for relative_file in prune_list:
  59. file_path = unpack_root / relative_file
  60. try:
  61. file_path.unlink()
  62. # read-only files can't be deleted on Windows
  63. # so remove the flag and try again.
  64. except PermissionError:
  65. os.chmod(file_path, stat.S_IWRITE)
  66. file_path.unlink()
  67. except FileNotFoundError:
  68. unremovable_files.add(Path(relative_file).as_posix())
  69. return unremovable_files
  70. def _prune_path(path):
  71. """
  72. Delete all files and directories in path.
  73. path is a pathlib.Path to the directory to be pruned
  74. """
  75. for node in sorted(path.rglob('*'), key=lambda l: len(str(l)), reverse=True):
  76. if node.is_file() or node.is_symlink():
  77. try:
  78. node.unlink()
  79. except PermissionError:
  80. node.chmod(stat.S_IWRITE)
  81. node.unlink()
  82. elif node.is_dir() and not any(node.iterdir()):
  83. try:
  84. node.rmdir()
  85. except PermissionError:
  86. node.chmod(stat.S_IWRITE)
  87. node.rmdir()
  88. def prune_dirs(unpack_root, keep_contingent_paths, sysroot):
  89. """
  90. Delete all files and directories in pycache and CONTINGENT_PATHS directories.
  91. unpack_root is a pathlib.Path to the source tree
  92. keep_contingent_paths is a boolean that determines if the contingent paths should be pruned
  93. sysroot is a string that optionally defines a sysroot to exempt from pruning
  94. """
  95. for pycache in unpack_root.rglob('__pycache__'):
  96. _prune_path(pycache)
  97. if keep_contingent_paths:
  98. get_logger().info('Keeping Contingent Paths')
  99. else:
  100. get_logger().info('Removing Contingent Paths')
  101. for cpath in CONTINGENT_PATHS:
  102. if sysroot and f'{sysroot}-sysroot' in cpath:
  103. get_logger().info('%s: %s', 'Exempt', cpath)
  104. continue
  105. get_logger().info('%s: %s', 'Exists' if Path(cpath).exists() else 'Absent', cpath)
  106. _prune_path(unpack_root / cpath)
  107. def _callback(args):
  108. if not args.directory.exists():
  109. get_logger().error('Specified directory does not exist: %s', args.directory)
  110. sys.exit(1)
  111. if not args.pruning_list.exists():
  112. get_logger().error('Could not find the pruning list: %s', args.pruning_list)
  113. prune_dirs(args.directory, args.keep_contingent_paths, args.sysroot)
  114. prune_list = tuple(filter(len, args.pruning_list.read_text(encoding=ENCODING).splitlines()))
  115. unremovable_files = prune_files(args.directory, prune_list)
  116. if unremovable_files:
  117. file_list = '\n'.join(f for f in itertools.islice(unremovable_files, 5))
  118. if len(unremovable_files) > 5:
  119. file_list += '\n... and ' + str(len(unremovable_files) - 5) + ' more'
  120. get_logger().debug('files that could not be pruned:\n%s',
  121. '\n'.join(f for f in unremovable_files))
  122. get_logger().error('%d files could not be pruned:\n%s', len(unremovable_files), file_list)
  123. sys.exit(1)
  124. def main():
  125. """CLI Entrypoint"""
  126. parser = argparse.ArgumentParser()
  127. parser.add_argument('directory', type=Path, help='The directory to apply binary pruning.')
  128. parser.add_argument('pruning_list', type=Path, help='Path to pruning.list')
  129. parser.add_argument('--keep-contingent-paths',
  130. action='store_true',
  131. help=('Skip pruning the contingent paths. '
  132. 'Useful when building with the Google tooling is desired.'))
  133. parser.add_argument('--sysroot',
  134. choices=('amd64', 'i386'),
  135. help=('Skip pruning the sysroot for the specified architecture. '
  136. 'Not needed when --keep-contingent-paths is used.'))
  137. add_common_params(parser)
  138. parser.set_defaults(callback=_callback)
  139. args = parser.parse_args()
  140. args.callback(args)
  141. if __name__ == '__main__':
  142. main()