prune_binaries.py 4.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135
  1. #!/usr/bin/env python3
  2. # -*- coding: UTF-8 -*-
  3. # Copyright (c) 2019 The ungoogled-chromium Authors. All rights reserved.
  4. # Use of this source code is governed by a BSD-style license that can be
  5. # found in the LICENSE file.
  6. """Prune binaries from the source tree"""
  7. import argparse
  8. import sys
  9. import os
  10. import stat
  11. from pathlib import Path
  12. from _common import ENCODING, get_logger, add_common_params
  13. # List of paths to prune if they exist, excluded from domain_substitution and pruning lists
  14. # These allow the lists to be compatible between cloned and tarball sources
  15. CONTINGENT_PATHS = (
  16. # Sources
  17. 'third_party/angle/third_party/VK-GL-CTS/src/',
  18. 'third_party/js_code_coverage/',
  19. 'third_party/llvm/',
  20. 'third_party/rust-src/',
  21. # Binaries
  22. 'buildtools/linux64/',
  23. 'buildtools/reclient/',
  24. 'third_party/android_rust_toolchain/',
  25. 'third_party/apache-linux/',
  26. 'third_party/checkstyle/',
  27. 'third_party/dawn/third_party/ninja/',
  28. 'third_party/dawn/tools/golang/',
  29. 'third_party/depot_tools/external_bin/',
  30. 'third_party/devtools-frontend/src/third_party/esbuild/',
  31. 'third_party/google-java-format/',
  32. 'third_party/libei/',
  33. 'third_party/llvm-build-tools/',
  34. 'third_party/ninja/',
  35. 'third_party/screen-ai/',
  36. 'third_party/siso/',
  37. 'third_party/updater/chrome_linux64/',
  38. 'third_party/updater/chromium_linux64/',
  39. 'tools/luci-go/',
  40. 'tools/resultdb/',
  41. 'tools/skia_goldctl/linux/',
  42. )
  43. def prune_files(unpack_root, prune_list):
  44. """
  45. Delete files under unpack_root listed in prune_list. Returns an iterable of unremovable files.
  46. unpack_root is a pathlib.Path to the directory to be pruned
  47. prune_list is an iterable of files to be removed.
  48. """
  49. unremovable_files = set()
  50. for relative_file in prune_list:
  51. file_path = unpack_root / relative_file
  52. try:
  53. file_path.unlink()
  54. # read-only files can't be deleted on Windows
  55. # so remove the flag and try again.
  56. except PermissionError:
  57. os.chmod(file_path, stat.S_IWRITE)
  58. file_path.unlink()
  59. except FileNotFoundError:
  60. unremovable_files.add(Path(relative_file).as_posix())
  61. return unremovable_files
  62. def _prune_path(path):
  63. """
  64. Delete all files and directories in path.
  65. path is a pathlib.Path to the directory to be pruned
  66. """
  67. for node in sorted(path.rglob('*'), key=lambda l: len(str(l)), reverse=True):
  68. if node.is_file() or node.is_symlink():
  69. try:
  70. node.unlink()
  71. except PermissionError:
  72. node.chmod(stat.S_IWRITE)
  73. node.unlink()
  74. elif node.is_dir() and not any(node.iterdir()):
  75. try:
  76. node.rmdir()
  77. except PermissionError:
  78. node.chmod(stat.S_IWRITE)
  79. node.rmdir()
  80. def prune_dirs(unpack_root):
  81. """
  82. Delete all files and directories in pycache and CONTINGENT_PATHS directories.
  83. unpack_root is a pathlib.Path to the source tree
  84. """
  85. for pycache in unpack_root.rglob('__pycache__'):
  86. _prune_path(pycache)
  87. get_logger().info('Removing Contingent Paths')
  88. for cpath in CONTINGENT_PATHS:
  89. get_logger().info('%s: %s', 'Exists' if Path(cpath).exists() else 'Absent', cpath)
  90. _prune_path(unpack_root / cpath)
  91. def _callback(args):
  92. if not args.directory.exists():
  93. get_logger().error('Specified directory does not exist: %s', args.directory)
  94. sys.exit(1)
  95. if not args.pruning_list.exists():
  96. get_logger().error('Could not find the pruning list: %s', args.pruning_list)
  97. prune_dirs(args.directory)
  98. prune_list = tuple(filter(len, args.pruning_list.read_text(encoding=ENCODING).splitlines()))
  99. unremovable_files = prune_files(args.directory, prune_list)
  100. if unremovable_files:
  101. get_logger().error('%d files could not be pruned.', len(unremovable_files))
  102. get_logger().debug('Files could not be pruned:\n%s',
  103. '\n'.join(f for f in unremovable_files))
  104. sys.exit(1)
  105. def main():
  106. """CLI Entrypoint"""
  107. parser = argparse.ArgumentParser()
  108. parser.add_argument('directory', type=Path, help='The directory to apply binary pruning.')
  109. parser.add_argument('pruning_list', type=Path, help='Path to pruning.list')
  110. add_common_params(parser)
  111. parser.set_defaults(callback=_callback)
  112. args = parser.parse_args()
  113. args.callback(args)
  114. if __name__ == '__main__':
  115. main()