Add timestamp manipulation to domain substitution

Also add preliminary testing code for utils

Fixes #849

.cirrus.yml

@@ -12,6 +12,7 @@ code_check_task:
     utils_script:
         - python3 -m yapf --style '.style.yapf' -e '*/third_party/*' -rpd utils
         - ./devutils/run_utils_pylint.py --hide-fixme
+        - ./devutils/run_utils_tests.sh
     devutils_script:
         - python3 -m yapf --style '.style.yapf' -e '*/third_party/*' -rpd devutils
         - ./devutils/run_devutils_pylint.py --hide-fixme

.gitignore

@@ -2,9 +2,12 @@
 __pycache__/
 *.py[cod]
 
+# Python testing files
+.coverage
+
 # Ignore macOS Finder meta
 .DS_Store
 .tm_properties
 
 # Ignore optional build / cache directory
-/build
+/build

devutils/run_utils_pylint.py

@@ -37,7 +37,8 @@ def main():
     ]
 
     ignore_prefixes = [
-        ('third_party', ),
+        ('third_party',),
+        ('tests',),
     ]
 
     sys.path.insert(1, str(Path(__file__).resolve().parent.parent / 'utils' / 'third_party'))

devutils/run_utils_tests.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -eux
+
+_root_dir=$(dirname $(dirname $(readlink -f $0)))
+cd ${_root_dir}/utils
+python3 -m pytest -c ${_root_dir}/utils/pytest.ini

utils/.coveragerc

@@ -0,0 +1,22 @@
+[run]
+branch = True
+parallel = True
+omit = tests/*
+
+[report]
+# Regexes for lines to exclude from consideration
+exclude_lines =
+    # Have to re-enable the standard pragma
+    pragma: no cover
+
+    # Don't complain about missing debug-only code:
+    def __repr__
+    if self\.debug
+
+    # Don't complain if tests don't hit defensive assertion code:
+    raise AssertionError
+    raise NotImplementedError
+
+    # Don't complain if non-runnable code isn't run:
+    if 0:
+    if __name__ == .__main__.:

utils/domain_substitution.py

@@ -8,14 +8,16 @@
 Substitute domain names in the source tree with blockable strings.
 """
 
+from pathlib import Path
 import argparse
 import collections
+import contextlib
 import io
+import os
 import re
 import tarfile
 import tempfile
 import zlib
-from pathlib import Path
 
 from _extraction import extract_tar_file
 from _common import ENCODING, get_logger, add_common_params
@@ -28,6 +30,10 @@ _INDEX_LIST = 'cache_index.list'
 _INDEX_HASH_DELIMITER = '|'
 _ORIG_DIR = 'orig'
 
+# Constants for timestamp manipulation
+# Delta between all file timestamps in nanoseconds
+_TIMESTAMP_DELTA = 1*10**9
+
 
 class DomainRegexList:
     """Representation of a domain_regex.list file"""
@@ -145,6 +151,24 @@ def _validate_file_index(index_file, resolved_tree, cache_index_files):
         cache_index_files.add(relative_path)
     return all_hashes_valid
 
+@contextlib.contextmanager
+def _update_timestamp(path: os.PathLike, set_new: bool) -> None:
+    """
+    Context manager to set the timestamp of the path to plus or
+    minus a fixed delta, regardless of modifications within the context.
+
+    if set_new is True, the delta is added. Otherwise, the delta is subtracted.
+    """
+    stats = os.stat(path)
+    if set_new:
+        new_timestamp = (stats.st_atime_ns + _TIMESTAMP_DELTA, stats.st_mtime_ns + _TIMESTAMP_DELTA)
+    else:
+        new_timestamp = (stats.st_atime_ns - _TIMESTAMP_DELTA, stats.st_mtime_ns - _TIMESTAMP_DELTA)
+    try:
+        yield
+    finally:
+        os.utime(path, ns=new_timestamp)
+
 
 # Public Methods
 
@@ -194,7 +218,8 @@ def apply_substitution(regex_path, files_path, source_tree, domainsub_cache):
             if path.is_symlink():
                 get_logger().warning('Skipping path that has become a symlink: %s', path)
                 continue
-            crc32_hash, orig_content = _substitute_path(path, regex_pairs)
+            with _update_timestamp(path, set_new=True):
+                crc32_hash, orig_content = _substitute_path(path, regex_pairs)
             if crc32_hash is None:
                 get_logger().info('Path has no substitutions: %s', relative_path)
                 continue
@@ -261,7 +286,8 @@ def revert_substitution(domainsub_cache, source_tree):
         # Move original files over substituted ones
         get_logger().debug('Moving original files over substituted ones...')
         for relative_path in cache_index_files:
-            (extract_path / _ORIG_DIR / relative_path).replace(resolved_tree / relative_path)
+            with _update_timestamp(resolved_tree / relative_path, set_new=False):
+                (extract_path / _ORIG_DIR / relative_path).replace(resolved_tree / relative_path)
 
         # Quick check for unused files in cache
         orig_has_unused = False

utils/pytest.ini

@@ -0,0 +1,7 @@
+[pytest]
+testpaths = tests
+#filterwarnings =
+#	error
+#	ignore::DeprecationWarning
+#addopts = --cov-report term-missing --hypothesis-show-statistics -p no:warnings
+addopts = --cov=. --cov-config=.coveragerc --cov-report term-missing -p no:warnings

utils/tests/test_domain_substitution.py

@@ -0,0 +1,35 @@
+# -*- coding: UTF-8 -*-
+
+# Copyright (c) 2019 The ungoogled-chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import os
+import tempfile
+from pathlib import Path
+
+from .. import domain_substitution
+
+def test_update_timestamp():
+    with tempfile.TemporaryDirectory() as tmpdirname:
+        path = Path(tmpdirname, 'tmp_update_timestamp')
+        path.touch()
+        orig_stats: os.stat_result = path.stat()
+
+        # Add delta to timestamp
+        with domain_substitution._update_timestamp(path, set_new=True):
+            with path.open('w') as fileobj:
+                fileobj.write('foo')
+
+        new_stats: os.stat_result = path.stat()
+        assert orig_stats.st_atime_ns != new_stats.st_atime_ns
+        assert orig_stats.st_mtime_ns != new_stats.st_mtime_ns
+
+        # Remove delta from timestamp
+        with domain_substitution._update_timestamp(path, set_new=False):
+            with path.open('w') as fileobj:
+                fileobj.write('bar')
+
+        new_stats: os.stat_result = path.stat()
+        assert orig_stats.st_atime_ns == new_stats.st_atime_ns
+        assert orig_stats.st_mtime_ns == new_stats.st_mtime_ns