validate_patches.py 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707
  1. #!/usr/bin/env python3
  2. # -*- coding: utf-8 -*-
  3. # Copyright (c) 2020 The ungoogled-chromium Authors. All rights reserved.
  4. # Use of this source code is governed by a BSD-style license that can be
  5. # found in the LICENSE file.
  6. """
  7. Validates that all patches apply cleanly against the source tree.
  8. The required source tree files can be retrieved from Google directly.
  9. """
  10. import argparse
  11. import ast
  12. import base64
  13. import email.utils
  14. import json
  15. import logging
  16. import sys
  17. import tempfile
  18. from pathlib import Path
  19. sys.path.insert(0, str(Path(__file__).resolve().parent / 'third_party'))
  20. import unidiff
  21. from unidiff.constants import LINE_TYPE_EMPTY, LINE_TYPE_NO_NEWLINE
  22. sys.path.pop(0)
  23. sys.path.insert(0, str(Path(__file__).resolve().parent.parent / 'utils'))
  24. from domain_substitution import TREE_ENCODINGS
  25. from _common import ENCODING, get_logger, get_chromium_version, parse_series, add_common_params
  26. from patches import dry_run_check
  27. sys.path.pop(0)
  28. try:
  29. import requests
  30. import requests.adapters
  31. import urllib3.util
  32. class _VerboseRetry(urllib3.util.Retry):
  33. """A more verbose version of HTTP Adatper about retries"""
  34. def sleep_for_retry(self, response=None):
  35. """Sleeps for Retry-After, and logs the sleep time"""
  36. if response:
  37. retry_after = self.get_retry_after(response)
  38. if retry_after:
  39. get_logger().info(
  40. 'Got HTTP status %s with Retry-After header. Retrying after %s seconds...',
  41. response.status, retry_after)
  42. else:
  43. get_logger().info(
  44. 'Could not find Retry-After header for HTTP response %s. Status reason: %s',
  45. response.status, response.reason)
  46. return super().sleep_for_retry(response)
  47. def _sleep_backoff(self):
  48. """Log info about backoff sleep"""
  49. get_logger().info('Running HTTP request sleep backoff')
  50. super()._sleep_backoff()
  51. def _get_requests_session():
  52. session = requests.Session()
  53. http_adapter = requests.adapters.HTTPAdapter(
  54. max_retries=_VerboseRetry(total=10,
  55. read=10,
  56. connect=10,
  57. backoff_factor=8,
  58. status_forcelist=urllib3.Retry.RETRY_AFTER_STATUS_CODES,
  59. raise_on_status=False))
  60. session.mount('http://', http_adapter)
  61. session.mount('https://', http_adapter)
  62. return session
  63. except ImportError:
  64. def _get_requests_session():
  65. raise RuntimeError('The Python module "requests" is required for remote'
  66. 'file downloading. It can be installed from PyPI.')
  67. _ROOT_DIR = Path(__file__).resolve().parent.parent
  68. _SRC_PATH = Path('src')
  69. class _PatchValidationError(Exception):
  70. """Raised when patch validation fails"""
  71. class _UnexpectedSyntaxError(RuntimeError):
  72. """Raised when unexpected syntax is used in DEPS"""
  73. class _NotInRepoError(RuntimeError):
  74. """Raised when the remote file is not present in the given repo"""
  75. class _DepsNodeVisitor(ast.NodeVisitor):
  76. _valid_syntax_types = (ast.mod, ast.expr_context, ast.boolop, ast.Assign, ast.Add, ast.Name,
  77. ast.Dict, ast.Str, ast.NameConstant, ast.List, ast.BinOp)
  78. _allowed_callables = ('Var', )
  79. def visit_Call(self, node): #pylint: disable=invalid-name
  80. """Override Call syntax handling"""
  81. if node.func.id not in self._allowed_callables:
  82. raise _UnexpectedSyntaxError('Unexpected call of "%s" at line %s, column %s' %
  83. (node.func.id, node.lineno, node.col_offset))
  84. def generic_visit(self, node):
  85. for ast_type in self._valid_syntax_types:
  86. if isinstance(node, ast_type):
  87. super().generic_visit(node)
  88. return
  89. raise _UnexpectedSyntaxError('Unexpected {} at line {}, column {}'.format(
  90. type(node).__name__, node.lineno, node.col_offset))
  91. def _validate_deps(deps_text):
  92. """Returns True if the DEPS file passes validation; False otherwise"""
  93. try:
  94. _DepsNodeVisitor().visit(ast.parse(deps_text))
  95. except _UnexpectedSyntaxError as exc:
  96. get_logger().error('%s', exc)
  97. return False
  98. return True
  99. def _deps_var(deps_globals):
  100. """Return a function that implements DEPS's Var() function"""
  101. def _var_impl(var_name):
  102. """Implementation of Var() in DEPS"""
  103. return deps_globals['vars'][var_name]
  104. return _var_impl
  105. def _parse_deps(deps_text):
  106. """Returns a dict of parsed DEPS data"""
  107. deps_globals = {'__builtins__': None}
  108. deps_globals['Var'] = _deps_var(deps_globals)
  109. exec(deps_text, deps_globals) #pylint: disable=exec-used
  110. return deps_globals
  111. def _download_googlesource_file(download_session, repo_url, version, relative_path):
  112. """
  113. Returns the contents of the text file with path within the given
  114. googlesource.com repo as a string.
  115. """
  116. if 'googlesource.com' not in repo_url:
  117. raise ValueError('Repository URL is not a googlesource.com URL: {}'.format(repo_url))
  118. full_url = repo_url + '/+/{}/{}?format=TEXT'.format(version, str(relative_path))
  119. get_logger().debug('Downloading: %s', full_url)
  120. response = download_session.get(full_url)
  121. if response.status_code == 404:
  122. raise _NotInRepoError()
  123. response.raise_for_status()
  124. # Assume all files that need patching are compatible with UTF-8
  125. return base64.b64decode(response.text, validate=True).decode('UTF-8')
  126. def _get_dep_value_url(deps_globals, dep_value):
  127. """Helper for _process_deps_entries"""
  128. if isinstance(dep_value, str):
  129. url = dep_value
  130. elif isinstance(dep_value, dict):
  131. if 'url' not in dep_value:
  132. # Ignore other types like CIPD since
  133. # it probably isn't necessary
  134. return None
  135. url = dep_value['url']
  136. else:
  137. raise NotImplementedError()
  138. if '{' in url:
  139. # Probably a Python format string
  140. url = url.format(**deps_globals['vars'])
  141. if url.count('@') != 1:
  142. raise _PatchValidationError('Invalid number of @ symbols in URL: {}'.format(url))
  143. return url
  144. def _process_deps_entries(deps_globals, child_deps_tree, child_path, deps_use_relative_paths):
  145. """Helper for _get_child_deps_tree"""
  146. for dep_path_str, dep_value in deps_globals.get('deps', dict()).items():
  147. url = _get_dep_value_url(deps_globals, dep_value)
  148. if url is None:
  149. continue
  150. dep_path = Path(dep_path_str)
  151. if not deps_use_relative_paths:
  152. try:
  153. dep_path = Path(dep_path_str).relative_to(child_path)
  154. except ValueError:
  155. # Not applicable to the current DEPS tree path
  156. continue
  157. grandchild_deps_tree = None # Delaying creation of dict() until it's needed
  158. for recursedeps_item in deps_globals.get('recursedeps', tuple()):
  159. if isinstance(recursedeps_item, str):
  160. if recursedeps_item == str(dep_path):
  161. grandchild_deps_tree = 'DEPS'
  162. else: # Some sort of iterable
  163. recursedeps_item_path, recursedeps_item_depsfile = recursedeps_item
  164. if recursedeps_item_path == str(dep_path):
  165. grandchild_deps_tree = recursedeps_item_depsfile
  166. if grandchild_deps_tree is None:
  167. # This dep is not recursive; i.e. it is fully loaded
  168. grandchild_deps_tree = dict()
  169. child_deps_tree[dep_path] = (*url.split('@'), grandchild_deps_tree)
  170. def _get_child_deps_tree(download_session, current_deps_tree, child_path, deps_use_relative_paths):
  171. """Helper for _download_source_file"""
  172. repo_url, version, child_deps_tree = current_deps_tree[child_path]
  173. if isinstance(child_deps_tree, str):
  174. # Load unloaded DEPS
  175. deps_globals = _parse_deps(
  176. _download_googlesource_file(download_session, repo_url, version, child_deps_tree))
  177. child_deps_tree = dict()
  178. current_deps_tree[child_path] = (repo_url, version, child_deps_tree)
  179. deps_use_relative_paths = deps_globals.get('use_relative_paths', False)
  180. _process_deps_entries(deps_globals, child_deps_tree, child_path, deps_use_relative_paths)
  181. return child_deps_tree, deps_use_relative_paths
  182. def _get_last_chromium_modification():
  183. """Returns the last modification date of the chromium-browser-official tar file"""
  184. with _get_requests_session() as session:
  185. response = session.head(
  186. 'https://storage.googleapis.com/chromium-browser-official/chromium-{}.tar.xz'.format(
  187. get_chromium_version()))
  188. response.raise_for_status()
  189. return email.utils.parsedate_to_datetime(response.headers['Last-Modified'])
  190. def _get_gitiles_git_log_date(log_entry):
  191. """Helper for _get_gitiles_git_log_date"""
  192. return email.utils.parsedate_to_datetime(log_entry['committer']['time'])
  193. def _get_gitiles_commit_before_date(repo_url, target_branch, target_datetime):
  194. """Returns the hexadecimal hash of the closest commit before target_datetime"""
  195. json_log_url = '{repo}/+log/{branch}?format=JSON'.format(repo=repo_url, branch=target_branch)
  196. with _get_requests_session() as session:
  197. response = session.get(json_log_url)
  198. response.raise_for_status()
  199. git_log = json.loads(response.text[5:]) # Trim closing delimiters for various structures
  200. assert len(git_log) == 2 # 'log' and 'next' entries
  201. assert 'log' in git_log
  202. assert git_log['log']
  203. git_log = git_log['log']
  204. # Check boundary conditions
  205. if _get_gitiles_git_log_date(git_log[0]) < target_datetime:
  206. # Newest commit is older than target datetime
  207. return git_log[0]['commit']
  208. if _get_gitiles_git_log_date(git_log[-1]) > target_datetime:
  209. # Oldest commit is newer than the target datetime; assume oldest is close enough.
  210. get_logger().warning('Oldest entry in gitiles log for repo "%s" is newer than target; '
  211. 'continuing with oldest entry...')
  212. return git_log[-1]['commit']
  213. # Do binary search
  214. low_index = 0
  215. high_index = len(git_log) - 1
  216. mid_index = high_index
  217. while low_index != high_index:
  218. mid_index = low_index + (high_index - low_index) // 2
  219. if _get_gitiles_git_log_date(git_log[mid_index]) > target_datetime:
  220. low_index = mid_index + 1
  221. else:
  222. high_index = mid_index
  223. return git_log[mid_index]['commit']
  224. class _FallbackRepoManager:
  225. """Retrieves fallback repos and caches data needed for determining repos"""
  226. _GN_REPO_URL = 'https://gn.googlesource.com/gn.git'
  227. def __init__(self):
  228. self._cache_gn_version = None
  229. @property
  230. def gn_version(self):
  231. """
  232. Returns the version of the GN repo for the Chromium version used by this code
  233. """
  234. if not self._cache_gn_version:
  235. # Because there seems to be no reference to the logic for generating the
  236. # chromium-browser-official tar file, it's possible that it is being generated
  237. # by an internal script that manually injects the GN repository files.
  238. # Therefore, assume that the GN version used in the chromium-browser-official tar
  239. # files correspond to the latest commit in the master branch of the GN repository
  240. # at the time of the tar file's generation. We can get an approximation for the
  241. # generation time by using the last modification date of the tar file on
  242. # Google's file server.
  243. self._cache_gn_version = _get_gitiles_commit_before_date(
  244. self._GN_REPO_URL, 'master', _get_last_chromium_modification())
  245. return self._cache_gn_version
  246. def get_fallback(self, current_relative_path, current_node, root_deps_tree):
  247. """
  248. Helper for _download_source_file
  249. It returns a new (repo_url, version, new_relative_path) to attempt a file download with
  250. """
  251. assert len(current_node) == 3
  252. # GN special processing
  253. try:
  254. new_relative_path = current_relative_path.relative_to('tools/gn')
  255. except ValueError:
  256. pass
  257. else:
  258. if current_node is root_deps_tree[_SRC_PATH]:
  259. get_logger().info('Redirecting to GN repo version %s for path: %s', self.gn_version,
  260. current_relative_path)
  261. return (self._GN_REPO_URL, self.gn_version, new_relative_path)
  262. return None, None, None
  263. def _get_target_file_deps_node(download_session, root_deps_tree, target_file):
  264. """
  265. Helper for _download_source_file
  266. Returns the corresponding repo containing target_file based on the DEPS tree
  267. """
  268. # The "deps" from the current DEPS file
  269. current_deps_tree = root_deps_tree
  270. current_node = None
  271. # Path relative to the current node (i.e. DEPS file)
  272. current_relative_path = Path('src', target_file)
  273. previous_relative_path = None
  274. deps_use_relative_paths = False
  275. child_path = None
  276. while current_relative_path != previous_relative_path:
  277. previous_relative_path = current_relative_path
  278. for child_path in current_deps_tree:
  279. try:
  280. current_relative_path = previous_relative_path.relative_to(child_path)
  281. except ValueError:
  282. # previous_relative_path does not start with child_path
  283. continue
  284. current_node = current_deps_tree[child_path]
  285. # current_node will match with current_deps_tree after the following statement
  286. current_deps_tree, deps_use_relative_paths = _get_child_deps_tree(
  287. download_session, current_deps_tree, child_path, deps_use_relative_paths)
  288. break
  289. assert not current_node is None
  290. return current_node, current_relative_path
  291. def _download_source_file(download_session, root_deps_tree, fallback_repo_manager, target_file):
  292. """
  293. Downloads the source tree file from googlesource.com
  294. download_session is an active requests.Session() object
  295. deps_dir is a pathlib.Path to the directory containing a DEPS file.
  296. """
  297. current_node, current_relative_path = _get_target_file_deps_node(download_session,
  298. root_deps_tree, target_file)
  299. # Attempt download with potential fallback logic
  300. repo_url, version, _ = current_node
  301. try:
  302. # Download with DEPS-provided repo
  303. return _download_googlesource_file(download_session, repo_url, version,
  304. current_relative_path)
  305. except _NotInRepoError:
  306. pass
  307. get_logger().debug(
  308. 'Path "%s" (relative: "%s") not found using DEPS tree; finding fallback repo...',
  309. target_file, current_relative_path)
  310. repo_url, version, current_relative_path = fallback_repo_manager.get_fallback(
  311. current_relative_path, current_node, root_deps_tree)
  312. if not repo_url:
  313. get_logger().error('No fallback repo found for "%s" (relative: "%s")', target_file,
  314. current_relative_path)
  315. raise _NotInRepoError()
  316. try:
  317. # Download with fallback repo
  318. return _download_googlesource_file(download_session, repo_url, version,
  319. current_relative_path)
  320. except _NotInRepoError:
  321. pass
  322. get_logger().error('File "%s" (relative: "%s") not found in fallback repo "%s", version "%s"',
  323. target_file, current_relative_path, repo_url, version)
  324. raise _NotInRepoError()
  325. def _initialize_deps_tree():
  326. """
  327. Initializes and returns a dependency tree for DEPS files
  328. The DEPS tree is a dict has the following format:
  329. key - pathlib.Path relative to the DEPS file's path
  330. value - tuple(repo_url, version, recursive dict here)
  331. repo_url is the URL to the dependency's repository root
  332. If the recursive dict is a string, then it is a string to the DEPS file to load
  333. if needed
  334. download_session is an active requests.Session() object
  335. """
  336. root_deps_tree = {
  337. _SRC_PATH: ('https://chromium.googlesource.com/chromium/src.git', get_chromium_version(),
  338. 'DEPS')
  339. }
  340. return root_deps_tree
  341. def _retrieve_remote_files(file_iter):
  342. """
  343. Retrieves all file paths in file_iter from Google
  344. file_iter is an iterable of strings that are relative UNIX paths to
  345. files in the Chromium source.
  346. Returns a dict of relative UNIX path strings to a list of lines in the file as strings
  347. """
  348. files = dict()
  349. root_deps_tree = _initialize_deps_tree()
  350. try:
  351. total_files = len(file_iter)
  352. except TypeError:
  353. total_files = None
  354. logger = get_logger()
  355. if total_files is None:
  356. logger.info('Downloading remote files...')
  357. else:
  358. logger.info('Downloading %d remote files...', total_files)
  359. last_progress = 0
  360. file_count = 0
  361. fallback_repo_manager = _FallbackRepoManager()
  362. with _get_requests_session() as download_session:
  363. download_session.stream = False # To ensure connection to Google can be reused
  364. for file_path in file_iter:
  365. if total_files:
  366. file_count += 1
  367. current_progress = file_count * 100 // total_files // 5 * 5
  368. if current_progress != last_progress:
  369. last_progress = current_progress
  370. logger.info('%d%% downloaded', current_progress)
  371. else:
  372. current_progress = file_count // 20 * 20
  373. if current_progress != last_progress:
  374. last_progress = current_progress
  375. logger.info('%d files downloaded', current_progress)
  376. try:
  377. files[file_path] = _download_source_file(download_session, root_deps_tree,
  378. fallback_repo_manager,
  379. file_path).split('\n')
  380. except _NotInRepoError:
  381. get_logger().warning('Could not find "%s" remotely. Skipping...', file_path)
  382. return files
  383. def _retrieve_local_files(file_iter, source_dir):
  384. """
  385. Retrieves all file paths in file_iter from the local source tree
  386. file_iter is an iterable of strings that are relative UNIX paths to
  387. files in the Chromium source.
  388. Returns a dict of relative UNIX path strings to a list of lines in the file as strings
  389. """
  390. files = dict()
  391. for file_path in file_iter:
  392. try:
  393. raw_content = (source_dir / file_path).read_bytes()
  394. except FileNotFoundError:
  395. get_logger().warning('Missing file from patches: %s', file_path)
  396. continue
  397. for encoding in TREE_ENCODINGS:
  398. try:
  399. content = raw_content.decode(encoding)
  400. break
  401. except UnicodeDecodeError:
  402. continue
  403. if not content:
  404. raise UnicodeDecodeError('Unable to decode with any encoding: %s' % file_path)
  405. files[file_path] = content.split('\n')
  406. if not files:
  407. get_logger().error('All files used by patches are missing!')
  408. return files
  409. def _modify_file_lines(patched_file, file_lines):
  410. """Helper for _apply_file_unidiff"""
  411. # Cursor for keeping track of the current line during hunk application
  412. # NOTE: The cursor is based on the line list index, not the line number!
  413. line_cursor = None
  414. for hunk in patched_file:
  415. # Validate hunk will match
  416. if not hunk.is_valid():
  417. raise _PatchValidationError('Hunk is not valid: {}'.format(repr(hunk)))
  418. line_cursor = hunk.target_start - 1
  419. for line in hunk:
  420. normalized_line = line.value.rstrip('\n')
  421. if line.is_added:
  422. file_lines[line_cursor:line_cursor] = (normalized_line, )
  423. line_cursor += 1
  424. elif line.is_removed:
  425. if normalized_line != file_lines[line_cursor]:
  426. raise _PatchValidationError(
  427. "Line '{}' does not match removal line '{}' from patch".format(
  428. file_lines[line_cursor], normalized_line))
  429. del file_lines[line_cursor]
  430. elif line.is_context:
  431. if not normalized_line and line_cursor == len(file_lines):
  432. # We reached the end of the file
  433. break
  434. if normalized_line != file_lines[line_cursor]:
  435. raise _PatchValidationError(
  436. "Line '{}' does not match context line '{}' from patch".format(
  437. file_lines[line_cursor], normalized_line))
  438. line_cursor += 1
  439. else:
  440. assert line.line_type in (LINE_TYPE_EMPTY, LINE_TYPE_NO_NEWLINE)
  441. def _apply_file_unidiff(patched_file, files_under_test):
  442. """Applies the unidiff.PatchedFile to the source files under testing"""
  443. patched_file_path = Path(patched_file.path)
  444. if patched_file.is_added_file:
  445. if patched_file_path in files_under_test:
  446. assert files_under_test[patched_file_path] is None
  447. assert len(patched_file) == 1 # Should be only one hunk
  448. assert patched_file[0].removed == 0
  449. assert patched_file[0].target_start == 1
  450. files_under_test[patched_file_path] = [x.value.rstrip('\n') for x in patched_file[0]]
  451. elif patched_file.is_removed_file:
  452. # Remove lines to see if file to be removed matches patch
  453. _modify_file_lines(patched_file, files_under_test[patched_file_path])
  454. files_under_test[patched_file_path] = None
  455. else: # Patching an existing file
  456. assert patched_file.is_modified_file
  457. _modify_file_lines(patched_file, files_under_test[patched_file_path])
  458. def _dry_check_patched_file(patched_file, orig_file_content):
  459. """Run "patch --dry-check" on a unidiff.PatchedFile for diagnostics"""
  460. with tempfile.TemporaryDirectory() as tmpdirname:
  461. tmp_dir = Path(tmpdirname)
  462. # Write file to patch
  463. patched_file_path = tmp_dir / patched_file.path
  464. patched_file_path.parent.mkdir(parents=True, exist_ok=True)
  465. patched_file_path.write_text(orig_file_content)
  466. # Write patch
  467. patch_path = tmp_dir / 'broken_file.patch'
  468. patch_path.write_text(str(patched_file))
  469. # Dry run
  470. _, dry_stdout, _ = dry_run_check(patch_path, tmp_dir)
  471. return dry_stdout
  472. def _test_patches(series_iter, patch_cache, files_under_test):
  473. """
  474. Tests the patches specified in the iterable series_iter
  475. Returns a boolean indicating if any of the patches have failed
  476. """
  477. for patch_path_str in series_iter:
  478. for patched_file in patch_cache[patch_path_str]:
  479. orig_file_content = None
  480. if get_logger().isEnabledFor(logging.DEBUG):
  481. orig_file_content = files_under_test.get(Path(patched_file.path))
  482. if orig_file_content:
  483. orig_file_content = ' '.join(orig_file_content)
  484. try:
  485. _apply_file_unidiff(patched_file, files_under_test)
  486. except _PatchValidationError as exc:
  487. get_logger().warning('Patch failed validation: %s', patch_path_str)
  488. get_logger().debug('Specifically, file "%s" failed validation: %s',
  489. patched_file.path, exc)
  490. if get_logger().isEnabledFor(logging.DEBUG):
  491. # _PatchValidationError cannot be thrown when a file is added
  492. assert patched_file.is_modified_file or patched_file.is_removed_file
  493. assert orig_file_content is not None
  494. get_logger().debug(
  495. 'Output of "patch --dry-run" for this patch on this file:\n%s',
  496. _dry_check_patched_file(patched_file, orig_file_content))
  497. return True
  498. except: #pylint: disable=bare-except
  499. get_logger().warning('Patch failed validation: %s', patch_path_str)
  500. get_logger().debug('Specifically, file "%s" caused exception while applying:',
  501. patched_file.path,
  502. exc_info=True)
  503. return True
  504. return False
  505. def _load_all_patches(series_iter, patches_dir):
  506. """
  507. Returns a tuple of the following:
  508. - boolean indicating success or failure of reading files
  509. - dict of relative UNIX path strings to unidiff.PatchSet
  510. """
  511. had_failure = False
  512. unidiff_dict = dict()
  513. for relative_path in series_iter:
  514. if relative_path in unidiff_dict:
  515. continue
  516. unidiff_dict[relative_path] = unidiff.PatchSet.from_filename(str(patches_dir /
  517. relative_path),
  518. encoding=ENCODING)
  519. if not (patches_dir / relative_path).read_text(encoding=ENCODING).endswith('\n'):
  520. had_failure = True
  521. get_logger().warning('Patch file does not end with newline: %s',
  522. str(patches_dir / relative_path))
  523. return had_failure, unidiff_dict
  524. def _get_required_files(patch_cache):
  525. """Returns an iterable of pathlib.Path files needed from the source tree for patching"""
  526. new_files = set() # Files introduced by patches
  527. file_set = set()
  528. for patch_set in patch_cache.values():
  529. for patched_file in patch_set:
  530. if patched_file.is_added_file:
  531. new_files.add(patched_file.path)
  532. elif patched_file.path not in new_files:
  533. file_set.add(Path(patched_file.path))
  534. return file_set
  535. def _get_files_under_test(args, required_files, parser):
  536. """
  537. Helper for main to get files_under_test
  538. Exits the program if --cache-remote debugging option is used
  539. """
  540. if args.local:
  541. files_under_test = _retrieve_local_files(required_files, args.local)
  542. else: # --remote and --cache-remote
  543. files_under_test = _retrieve_remote_files(required_files)
  544. if args.cache_remote:
  545. for file_path, file_content in files_under_test.items():
  546. if not (args.cache_remote / file_path).parent.exists():
  547. (args.cache_remote / file_path).parent.mkdir(parents=True)
  548. with (args.cache_remote / file_path).open('w', encoding=ENCODING) as cache_file:
  549. cache_file.write('\n'.join(file_content))
  550. parser.exit()
  551. return files_under_test
  552. def main():
  553. """CLI Entrypoint"""
  554. parser = argparse.ArgumentParser(description=__doc__)
  555. parser.add_argument('-s',
  556. '--series',
  557. type=Path,
  558. metavar='FILE',
  559. default=str(Path('patches', 'series')),
  560. help='The series file listing patches to apply. Default: %(default)s')
  561. parser.add_argument('-p',
  562. '--patches',
  563. type=Path,
  564. metavar='DIRECTORY',
  565. default='patches',
  566. help='The patches directory to read from. Default: %(default)s')
  567. add_common_params(parser)
  568. file_source_group = parser.add_mutually_exclusive_group(required=True)
  569. file_source_group.add_argument(
  570. '-l',
  571. '--local',
  572. type=Path,
  573. metavar='DIRECTORY',
  574. help=
  575. 'Use a local source tree. It must be UNMODIFIED, otherwise the results will not be valid.')
  576. file_source_group.add_argument(
  577. '-r',
  578. '--remote',
  579. action='store_true',
  580. help=('Download the required source tree files from Google. '
  581. 'This feature requires the Python module "requests". If you do not want to '
  582. 'install this, consider using --local instead.'))
  583. file_source_group.add_argument(
  584. '-c',
  585. '--cache-remote',
  586. type=Path,
  587. metavar='DIRECTORY',
  588. help='(For debugging) Store the required remote files in an empty local directory')
  589. args = parser.parse_args()
  590. if args.cache_remote and not args.cache_remote.exists():
  591. if args.cache_remote.parent.exists():
  592. args.cache_remote.mkdir()
  593. else:
  594. parser.error('Parent of cache path {} does not exist'.format(args.cache_remote))
  595. if not args.series.is_file():
  596. parser.error('--series path is not a file or not found: {}'.format(args.series))
  597. if not args.patches.is_dir():
  598. parser.error('--patches path is not a directory or not found: {}'.format(args.patches))
  599. series_iterable = tuple(parse_series(args.series))
  600. had_failure, patch_cache = _load_all_patches(series_iterable, args.patches)
  601. required_files = _get_required_files(patch_cache)
  602. files_under_test = _get_files_under_test(args, required_files, parser)
  603. had_failure |= _test_patches(series_iterable, patch_cache, files_under_test)
  604. if had_failure:
  605. get_logger().error('***FAILED VALIDATION; SEE ABOVE***')
  606. if not args.verbose:
  607. get_logger().info('(For more error details, re-run with the "-v" flag)')
  608. parser.exit(status=1)
  609. else:
  610. get_logger().info('Passed validation (%d patches total)', len(series_iterable))
  611. if __name__ == '__main__':
  612. main()