WIP refactoring of new build system into utilikit

developer_utilities/run_pylint.py

@@ -23,9 +23,6 @@ if __name__ == "__main__":
     from pylint import epylint as lint
     import pathlib
 
-    lint.lint(filename=str(pathlib.Path(__file__).parent.parent / "utilities"),
+    lint.lint(filename=str(pathlib.Path(__file__).parent.parent / "utilikit"),
               options=["--disable=logging-format-interpolation",
-                       "--disable=fixme",
-                       "--disable=locally-disabled",
-                       "--disable=duplicate-code",
-                       "--ignore=_external"])
+                       "--disable=locally-disabled"])

resources/configs/archlinux/metadata.ini

@@ -1,4 +1,4 @@
 [config]
 display_name = Arch Linux
-parent = linux_dynamic
+parents = linux_dynamic
 visible = true

resources/configs/common_debian/metadata.ini

@@ -1,3 +1,3 @@
 [config]
 display_name = Common Debian
-parent = linux_dynamic
+parents = linux_dynamic

resources/configs/debian_stretch/metadata.ini

@@ -1,4 +1,4 @@
 [config]
 display_name = Debian 9.0 (stretch)
-parent = common_debian
+parents = common_debian
 visible = true

resources/configs/linux_dynamic/metadata.ini

@@ -1,4 +1,4 @@
 [config]
 display_name = Linux Dynamic
-parent = common
+parents = common
 visible = true

resources/configs/linux_static/metadata.ini

@@ -1,4 +1,4 @@
 [config]
 display_name = Linux Static
-parent = common
+parents = common
 visible = true

resources/configs/macos/extra_deps.ini

@@ -8,7 +8,7 @@ download_name = google-toolbox-for-mac-{version}.tar.gz
 strip_leading_dirs = google-toolbox-for-mac-{version}
 
 [third_party/llvm-build/Release+Asserts]
-version = 3.9.0
+version = 4.0.0
 url = http://llvm.org/releases/{version}/clang+llvm-{version}-x86_64-apple-darwin.tar.xz
 download_name = clang+llvm-{version}-x86_64-apple-darwin.tar.xz
 strip_leading_dirs = clang+llvm-{version}-x86_64-apple-darwin

resources/configs/macos/metadata.ini

@@ -1,4 +1,4 @@
 [config]
 display_name = macOS
-parent = common
+parents = common
 visible = true

resources/configs/windows/metadata.ini

@@ -1,4 +1,4 @@
 [config]
 display_name = Windows
-parent = common
+parents = common
 visible = true

utilities/build_files_generator/debian.py → utilikit/_build_files_generators/debian.py

@@ -2,7 +2,7 @@
 
 # ungoogled-chromium: Modifications to Google Chromium for removing Google
 # integration and enhancing privacy, control, and transparency
-# Copyright (C) 2016  Eloston
+# Copyright (C) 2017  Eloston
 #
 # This file is part of ungoogled-chromium.
 #
@@ -21,7 +21,6 @@
 
 """Debian-specific build files generation code"""
 
-import pathlib
 import string
 import locale
 import datetime
@@ -29,12 +28,11 @@ import re
 import distutils.dir_util
 import os
 
-from . import ROOT_DIR
+from .. import _common
+from .. import substitute_domains as _substitute_domains
 
 # Private definitions
 
-_DPKG_DIR = ROOT_DIR / pathlib.Path("resources", "packaging", "debian")
-
 class _BuildFileStringTemplate(string.Template):
     """
     Custom string substitution class
@@ -80,26 +78,32 @@ def _get_parsed_gn_flags(gn_flags):
 
 # Public definitions
 
-def generate_build_files(resources_parser, output_dir, build_output,
-                         distribution_version):
+def generate_build_files(resources, output_dir, build_output,
+                         distribution_version, disable_domain_substitution):
     """
     Generates the `debian` directory in `output_dir` using resources from
-    `resources_parser`
+    `resources`
     """
     build_file_subs = dict(
-        changelog_version="{}-{}".format(*resources_parser.get_version()),
+        changelog_version="{}-{}".format(*resources.read_version()),
         changelog_datetime=_get_dpkg_changelog_datetime(),
         build_output=build_output,
         distribution_version=distribution_version,
-        gn_flags=_get_parsed_gn_flags(resources_parser.get_gn_flags())
+        gn_flags=_get_parsed_gn_flags(resources.read_gn_flags())
     )
 
     debian_dir = output_dir / "debian"
-    distutils.dir_util.copy_tree(str(_DPKG_DIR), str(debian_dir))
-    distutils.dir_util.copy_tree(str(resources_parser.PATCHES),
-                                 str(debian_dir / resources_parser.PATCHES.name))
-    (debian_dir / resources_parser.PATCHES.name / "series").write_bytes(
-        resources_parser.PATCH_ORDER.read_bytes())
+    dpkg_dir = _common.get_resources_dir() / "packaging" / "debian"
+    distutils.dir_util.copy_tree(str(dpkg_dir), str(debian_dir))
+    distutils.dir_util.copy_tree(str(_common.PATCHES_DIR),
+                                 str(debian_dir / _common.PATCHES_DIR))
+    patch_order = resources.read_patch_order()
+    if not disable_domain_substitution:
+        _substitute_domains.substitute_domains(
+            _substitute_domains.get_parsed_domain_regexes(resources.read_domain_regex_list()),
+            patch_order, debian_dir / _common.PATCHES_DIR, log_warnings=False)
+    _common.write_list(debian_dir / _common.PATCHES_DIR / "series",
+                       patch_order)
 
     for old_path in debian_dir.glob("*.in"):
         new_path = debian_dir / old_path.stem

utilikit/_common.py

@@ -0,0 +1,373 @@
+# -*- coding: UTF-8 -*-
+
+# ungoogled-chromium: Modifications to Google Chromium for removing Google integration
+# and enhancing privacy, control, and transparency
+# Copyright (C) 2017  Eloston
+#
+# This file is part of ungoogled-chromium.
+#
+# ungoogled-chromium is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# ungoogled-chromium is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with ungoogled-chromium.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Common code"""
+
+import os
+import pathlib
+import sys
+import tarfile
+import zipfile
+import abc
+import configparser
+
+# Constants
+
+_ENV_PREFIX = "UTILIKIT_"
+
+CONFIGS_DIR = "configs"
+PACKAGING_DIR = "packaging"
+PATCHES_DIR = "patches"
+
+CLEANING_LIST = "cleaning_list"
+DOMAIN_REGEX_LIST = "domain_regex_list"
+DOMAIN_SUBSTITUTION_LIST = "domain_substitution_list"
+EXTRA_DEPS_INI = "extra_deps.ini"
+GN_FLAGS = "gn_flags"
+METADATA_INI = "metadata.ini"
+PATCH_ORDER = "patch_order"
+VERSION_INI = "version.ini"
+
+# Classes
+
+class ResourceABC(metaclass=abc.ABCMeta):
+    """Abstract class for resource directories and configs"""
+
+    @abc.abstractmethod
+    def _read_list_generator(self, list_name, binary=False):
+        pass
+
+    @abc.abstractmethod
+    def _read_list(self, list_name, binary=False):
+        pass
+
+    @abc.abstractmethod
+    def _read_ini(self, ini_name):
+        pass
+
+    @abc.abstractmethod
+    def _read_dict_list(self, dict_list_name, binary=False):
+        pass
+
+    @abc.abstractmethod
+    def get_patches_dir(self):
+        """Returns the directory containing patches"""
+        pass
+
+    def read_cleaning_list(self, use_generator=False):
+        """Reads cleaning_list"""
+        if use_generator:
+            return self._read_list_generator(CLEANING_LIST)
+        else:
+            return self._read_list(CLEANING_LIST)
+
+    def read_domain_regex_list(self, binary=True):
+        """Returns domain_regex_list as a list"""
+        return self._read_list(DOMAIN_REGEX_LIST, binary)
+
+    def read_domain_substitution_list(self, use_generator=False):
+        """Returns domain_substitution_list as a list"""
+        if use_generator:
+            return self._read_list_generator(DOMAIN_SUBSTITUTION_LIST)
+        else:
+            return self._read_list(DOMAIN_SUBSTITUTION_LIST)
+
+    def read_extra_deps(self):
+        """Returns extra_deps.ini as a dictionary"""
+        extra_deps_config = self._read_ini(EXTRA_DEPS_INI)
+        tmp_dict = dict()
+        for section in extra_deps_config:
+            if section == "DEFAULT":
+                continue
+            else:
+                tmp_dict[section] = dict()
+                for keyname in extra_deps_config[section]:
+                    if keyname not in ["version", "url", "download_name", "strip_leading_dirs"]:
+                        raise KeyError(keyname)
+                    tmp_dict[section][keyname] = extra_deps_config[section][keyname]
+        return tmp_dict
+
+    def read_gn_flags(self):
+        """Returns gn_flags as a dictionary"""
+        return self._read_dict_list(GN_FLAGS)
+
+    def read_patch_order(self):
+        """Returns patch_order as a list"""
+        return self._read_list(PATCH_ORDER)
+
+    def read_version(self):
+        """Reads version.ini and returns a tuple"""
+        result = self._read_ini(VERSION_INI)["main"]
+        return (result["chromium_version"], result["release_revision"])
+
+class StandaloneResourceDirectory(ResourceABC):
+    """Represents a standalone resource directory (i.e. without metadata, e.g. exported)"""
+
+    def __init__(self, path):
+        self.path = path
+        self.name = path.name
+        self.display_name = path.name
+        self.visible = True
+
+    def _read_list_generator(self, list_name, binary=False):
+        return read_list_generator(self.path / list_name, binary=binary)
+
+    def _read_list(self, list_name, binary=False):
+        return read_list(self.path / list_name, binary=binary)
+
+    def _read_ini(self, ini_name):
+        return read_ini(self.path / ini_name)
+
+    def _read_dict_list(self, dict_list_name, binary=False):
+        return read_dict_list(self.path / dict_list_name, binary=binary)
+
+    def get_patches_dir(self):
+        """Returns the directory containing patches"""
+        return self.path / PATCHES_DIR
+
+class LinkedResourceDirectory(StandaloneResourceDirectory):
+    """Represents a single directory in resources/configs"""
+
+    def __init__(self, name): #pylint: disable=super-init-not-called
+        self.name = name
+        self.path = get_resources_dir() / CONFIGS_DIR / name
+        self.visible = False
+        self.display_name = name
+
+        self.parents = list()
+
+        self._read_metadata()
+
+    def _read_metadata(self):
+        """Reads metadata.ini"""
+        metadata_config = self._read_ini(METADATA_INI)
+        for section in metadata_config:
+            if section == "DEFAULT":
+                continue
+            elif section == "config":
+                for keyname in metadata_config["config"]:
+                    if keyname == "display_name":
+                        self.display_name = metadata_config[section][keyname]
+                    elif keyname == "parents":
+                        for name in metadata_config[section][keyname].split(","):
+                            self.parents.append(name.strip())
+                    elif keyname == "visible":
+                        self.visible = metadata_config[section][keyname]
+                    else:
+                        raise NameError("Unknown key name: {}. Configuration: {}".format(
+                            keyname, self.path.name))
+            else:
+                raise NameError("Unknown section name: {}. Configuration: {}".format(
+                    section, self.path.name))
+
+    def get_patches_dir(self):
+        """Returns the directory containing patches"""
+        return get_resources_dir() / PATCHES_DIR
+
+class ResourceConfig(ResourceABC):
+    """Represents a complete configuration in resources/configs"""
+
+    _loaded_directories = dict()
+
+    def __init__(self, name):
+        load_order = [name]
+        index = 0
+        while index < len(load_order):
+            name = load_order[index]
+            if name not in self._loaded_directories:
+                self._loaded_directories[name] = LinkedResourceDirectory(name)
+            for parent in reversed(self._loaded_directories[name].parents):
+                load_order[:] = [x for x in load_order if not x == parent]
+                load_order.append(parent)
+            index += 1
+
+        load_order.reverse()
+        self._load_order = load_order
+        self.name = name
+        self.display_name = self._loaded_directories[name].display_name
+        self.visible = self._loaded_directories[name].visible
+        self.target_path = self._loaded_directories[name].path
+
+    def _linked_resource_generator(self):
+        for name in self._load_order:
+            yield self._loaded_directories[name]
+
+    def _read_list_generator(self, list_name, binary=False):
+        for directory in self._linked_resource_generator():
+            yield from directory._read_list_generator(list_name, binary=binary) #pylint: disable=protected-access
+
+    def _read_list(self, list_name, binary=False):
+        return list(self._read_list_generator(list_name, binary=binary))
+
+    def _read_ini(self, ini_name):
+        result = dict()
+        for directory in self._linked_resource_generator():
+            result.update(directory._read_ini(ini_name)) #pylint: disable=protected-access
+        return result
+
+    def _read_dict_list(self, dict_list_name, binary=False):
+        result = dict()
+        for directory in self._linked_resource_generator():
+            result.update(directory._read_dict_list(dict_list_name, binary)) #pylint: disable=protected-access
+        return result
+
+    def get_patches_dir(self):
+        """Returns the directory containing patches"""
+        return get_resources_dir() / PATCHES_DIR
+
+# Methods
+
+def get_resources_dir():
+    """Returns the path to the root of the resources directory"""
+    env_value = os.environ.get(_ENV_PREFIX + "RESOURCES")
+    if env_value:
+        path = pathlib.Path(env_value)
+        if not path.is_dir():
+            raise NotADirectoryError(env_value)
+        return path
+    # Assume that this is a clone of the repository
+    return pathlib.Path(__file__).absolute().parent.parent / "resources"
+
+def get_resource_obj():
+    """Returns a resource object"""
+    config_type = os.environ.get(_ENV_PREFIX + "CONFIG_TYPE")
+    if not config_type:
+        raise ValueError(_ENV_PREFIX + "CONFIG_TYPE environment variable must be defined")
+    if config_type == "custom":
+        custom_path = pathlib.Path(os.environ.get(_ENV_PREFIX + "CUSTOM_CONFIG_PATH"))
+        if not custom_path.is_dir():
+            raise NotADirectoryError(str(custom_path))
+        return StandaloneResourceDirectory(custom_path)
+    else:
+        return ResourceConfig(config_type)
+
+def get_downloads_dir():
+    """Returns the downloads directory path"""
+    env_value = os.environ.get(_ENV_PREFIX + "DOWNLOADS_DIR")
+    if env_value:
+        path = pathlib.Path(env_value)
+        if not path.is_dir():
+            raise NotADirectoryError(env_value)
+        return path
+    return pathlib.Path(__file__).absolute().parent.parent / "build" / "downloads"
+
+def get_sandbox_dir():
+    """Returns the sandbox directory path"""
+    env_value = os.environ.get(_ENV_PREFIX + "SANDBOX_DIR")
+    if env_value:
+        path = pathlib.Path(env_value)
+        if not path.is_dir():
+            raise NotADirectoryError(env_value)
+        return path
+    return pathlib.Path(__file__).absolute().parent.parent / "build" / "sandbox"
+
+def read_list_generator(list_path, binary=False, allow_nonexistant=True):
+    """Generator to read a list. Ignores `binary` if reading from stdin"""
+    def _line_generator(file_obj):
+        for line in file_obj.read().splitlines():
+            if len(line) > 0:
+                yield line
+    if binary:
+        mode = "rb"
+    else:
+        mode = "r"
+    if str(list_path) == "-":
+        yield from _line_generator(sys.stdin)
+    else:
+        if list_path.is_file():
+            with list_path.open(mode) as file_obj:
+                yield from _line_generator(file_obj)
+        elif allow_nonexistant:
+            yield from iter(list())
+        else:
+            raise FileNotFoundError(str(list_path))
+
+def read_list(list_path, binary=False, allow_nonexistant=True):
+    """Reads a list. Ignores `binary` if reading from stdin"""
+    return list(read_list_generator(list_path, binary, allow_nonexistant))
+
+def read_ini(ini_path, allow_nonexistant=True):
+    """Returns a configparser object"""
+    if not ini_path.is_file():
+        if allow_nonexistant:
+            return dict()
+        else:
+            raise FileNotFoundError(str(ini_path))
+    config = configparser.ConfigParser()
+    config.read(str(ini_path))
+    return config
+
+def read_dict_list(dict_list_path, binary=False, allow_nonexistant=True):
+    """
+    Reads a text document that is a list of key-value pairs delimited by an equals sign
+
+    The last occurence of any given key will be the assigned value.
+
+    Blank lines are ignored
+    """
+    if not dict_list_path.is_file():
+        if allow_nonexistant:
+            return dict()
+        else:
+            raise FileNotFoundError(str(dict_list_path))
+    if binary:
+        delimiter = b"="
+    else:
+        delimiter = "=" #pylint: disable=redefined-variable-type
+    tmp_dict = dict()
+    for entry in read_list_generator(dict_list_path, binary):
+        key, value = entry.split(delimiter)
+        tmp_dict[key] = value
+    return tmp_dict
+
+def write_list(path, list_obj):
+    """Writes a list to `path`"""
+    with path.open("w") as file_obj:
+        file_obj.write("\n".join(list_obj))
+
+def write_dict_list(path, dict_obj):
+    """Writes a dictionary as a list to `path`"""
+    write_list(path, [key + "=" + value for key, value in dict_obj.items()])
+
+def write_ini(path, dict_obj):
+    """Writes a dictionary as an ini file to `path`"""
+    config = configparser.ConfigParser()
+    for section in dict_obj:
+        config.add_section(section)
+        for option, value in config[section].items():
+            config.set(section, option, value)
+    with path.open("w") as file_obj:
+        config.write(file_obj)
+
+def write_tar(output_filename, path_generator, mode="w:xz"):
+    """Writes out a .tar.xz package"""
+    with tarfile.open(output_filename, mode=mode) as tar_obj:
+        for arcname, real_path in path_generator:
+            print("Including '{}'".format(arcname))
+            tar_obj.add(str(real_path), arcname=arcname)
+
+def write_zip(output_filename, path_generator):
+    """Writes out a .zip package"""
+    with zipfile.ZipFile(output_filename, mode="w",
+                         compression=zipfile.ZIP_DEFLATED) as zip_file:
+        for arcname, real_path in path_generator:
+            print("Including '{}'".format(arcname))
+            zip_file.write(str(real_path), arcname)

utilities/archive_packager.py → utilikit/archive_packager.py

@@ -3,7 +3,7 @@
 
 # ungoogled-chromium: Modifications to Google Chromium for removing Google integration
 # and enhancing privacy, control, and transparency
-# Copyright (C) 2016  Eloston
+# Copyright (C) 2017  Eloston
 #
 # This file is part of ungoogled-chromium.
 #
@@ -24,10 +24,23 @@
 
 import sys
 import pathlib
-import tarfile
-import zipfile
 import argparse
 
+def fix_relative_import():
+    """Allow relative imports to work from anywhere"""
+    import os.path
+    parent_path = os.path.dirname(os.path.realpath(os.path.abspath(__file__)))
+    sys.path.insert(0, os.path.dirname(parent_path))
+    global __package__ #pylint: disable=global-variable-undefined
+    __package__ = os.path.basename(parent_path) #pylint: disable=redefined-builtin
+    __import__(__package__)
+    sys.path.pop(0)
+
+if __name__ == "__main__" and (__package__ is None or __package__ == ""):
+    fix_relative_import()
+
+from ._common import write_tar, write_zip #pylint: disable=wrong-import-position
+
 def file_list_generator(root_dir_name, files_cfg_path, build_output_dir, include_files, target_cpu):
     """
     Generator for files to be included in the archive
@@ -53,21 +66,6 @@ def file_list_generator(root_dir_name, files_cfg_path, build_output_dir, include
     for include_path in include_files:
         yield (str(root_dir_name / pathlib.Path(include_path.name)), include_path)
 
-def write_tar(output_filename, path_generator, mode="w:xz"):
-    """Writes out a .tar.xz package"""
-    with tarfile.open(output_filename, mode=mode) as tar_obj:
-        for arcname, real_path in path_generator:
-            print("Including '{}'".format(arcname))
-            tar_obj.add(str(real_path), arcname=arcname)
-
-def write_zip(output_filename, path_generator):
-    """Writes out a .zip package"""
-    with zipfile.ZipFile(output_filename, mode="w",
-                         compression=zipfile.ZIP_DEFLATED) as zip_file:
-        for arcname, real_path in path_generator:
-            print("Including '{}'".format(arcname))
-            zip_file.write(str(real_path), arcname)
-
 def _parse_args(args_list):
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--files-cfg", metavar="FILE", required=True,

utilities/build_gn.py → utilikit/build_gn.py

@@ -3,7 +3,7 @@
 
 # ungoogled-chromium: Modifications to Google Chromium for removing Google integration
 # and enhancing privacy, control, and transparency
-# Copyright (C) 2016  Eloston
+# Copyright (C) 2017  Eloston
 #
 # This file is part of ungoogled-chromium.
 #
@@ -27,12 +27,27 @@ import sys
 import pathlib
 import argparse
 
-def build_gn(output_path, gn_flags_path, src_root, python2_command):
+def fix_relative_import():
+    """Allow relative imports to work from anywhere"""
+    import os.path
+    parent_path = os.path.dirname(os.path.realpath(os.path.abspath(__file__)))
+    sys.path.insert(0, os.path.dirname(parent_path))
+    global __package__ #pylint: disable=global-variable-undefined
+    __package__ = os.path.basename(parent_path) #pylint: disable=redefined-builtin
+    __import__(__package__)
+    sys.path.pop(0)
+
+if __name__ == "__main__" and (__package__ is None or __package__ == ""):
+    fix_relative_import()
+
+from . import _common #pylint: disable=wrong-import-position
+
+def build_gn(output_path, gn_flags, src_root, python2_command):
     """
     Build the GN tool to out/gn_tool in the build sandbox
     """
-    with gn_flags_path.open() as file_obj:
-        gn_args_string = file_obj.read().replace("\n", " ")
+    gn_args_string = " ".join(
+        [flag + "=" + value for flag, value in gn_flags.items()])
     if output_path.exists():
         print("gn already exists in " + str(output_path))
     else:
@@ -49,28 +64,45 @@ def build_gn(output_path, gn_flags_path, src_root, python2_command):
 def main(args_list):
     """Entry point"""
     parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--ignore-environment", action="store_true",
+                        help="Ignore all 'UTILIKIT_*' environment variables.")
     parser.add_argument("--output-path", required=True, metavar="DIRECTORY",
                         help="The directory to output the GN binary")
-    parser.add_argument("--gn-flags-path", required=True, metavar="FILE",
-                        help="The GN flags to bootstrap GN with")
-    parser.add_argument("--sandbox-root", metavar="DIRECTORY", default=".",
+    parser.add_argument("--gn-flags-path", metavar="FILE",
+                        help=("The GN flags to bootstrap GN with. "
+                              "Required if --ignore-environment is set"))
+    parser.add_argument("--sandbox-root", metavar="DIRECTORY",
                         help=("The build sandbox root directory. "
-                              "Defaults to the current directory"))
+                              "Required if --ignore-environment is set"))
     parser.add_argument("--python2-command", metavar="COMMAND",
                         help="The Python 2 command to use. Defaults to the file's shebang")
     args = parser.parse_args(args_list)
+    gn_flags = dict()
+    if args.ignore_environment:
+        error_template = "--{} required since --ignore-environment is set"
+        if not args.gn_flags_path:
+            parser.error(error_template.format("gn-flags-path"))
+        if not args.sandbox_root:
+            parser.error(error_template.format("sandbox-root"))
+    else:
+        resources = _common.get_resource_obj()
+        gn_flags = resources.read_gn_flags()
+        sandbox_root = _common.get_sandbox_dir()
     output_path = pathlib.Path(args.output_path)
-    gn_flags_path = pathlib.Path(args.gn_flags_path)
-    if not gn_flags_path.is_file():
-        parser.error("--gn-flags-path is not a file: " + args.gn_flags_path)
-    src_root = pathlib.Path(args.sandbox_root)
-    if not src_root.is_dir():
-        parser.error("--sandbox-root is not a directory: " + args.sandbox_root)
+    if args.gn_flags_path:
+        gn_flags_path = pathlib.Path(args.gn_flags_path)
+        if not gn_flags_path.is_file():
+            parser.error("--gn-flags-path is not a file: " + args.gn_flags_path)
+        gn_flags = _common.read_dict_list(gn_flags_path)
+    if args.sandbox_root:
+        sandbox_root = pathlib.Path(args.sandbox_root)
+        if not sandbox_root.is_dir():
+            parser.error("--sandbox-root is not a directory: " + args.sandbox_root)
     python2_command = None
     if args.python2_command:
         python2_command = pathlib.Path(args.python2_command)
 
-    build_gn(output_path, gn_flags_path, src_root, python2_command)
+    build_gn(output_path, gn_flags, sandbox_root, python2_command)
 
     return 0
 

utilities/check_requirements.py → utilikit/check_requirements.py

@@ -3,7 +3,7 @@
 
 # ungoogled-chromium: Modifications to Google Chromium for removing Google integration
 # and enhancing privacy, control, and transparency
-# Copyright (C) 2016  Eloston
+# Copyright (C) 2017  Eloston
 #
 # This file is part of ungoogled-chromium.
 #

utilikit/clean_sources.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+# -*- coding: UTF-8 -*-
+
+# ungoogled-chromium: Modifications to Google Chromium for removing Google integration
+# and enhancing privacy, control, and transparency
+# Copyright (C) 2017  Eloston
+#
+# This file is part of ungoogled-chromium.
+#
+# ungoogled-chromium is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# ungoogled-chromium is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with ungoogled-chromium.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Runs source cleaner"""
+
+import pathlib
+import sys
+import argparse
+
+def fix_relative_import():
+    """Allow relative imports to work from anywhere"""
+    import os.path
+    parent_path = os.path.dirname(os.path.realpath(os.path.abspath(__file__)))
+    sys.path.insert(0, os.path.dirname(parent_path))
+    global __package__ #pylint: disable=global-variable-undefined
+    __package__ = os.path.basename(parent_path) #pylint: disable=redefined-builtin
+    __import__(__package__)
+    sys.path.pop(0)
+
+if __name__ == "__main__" and (__package__ is None or __package__ == ""):
+    fix_relative_import()
+
+from . import _common #pylint: disable=wrong-import-position
+
+def clean_sources(cleaning_list_iter, root_dir):
+    """Delete files given by iterable cleaning_list_iter relative to root_dir"""
+    for entry in cleaning_list_iter:
+        tmp_path = root_dir / entry
+        try:
+            tmp_path.unlink()
+        except FileNotFoundError:
+            print("No such file: " + str(tmp_path))
+
+def main(args_list):
+    """Entry point"""
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--ignore-environment", action="store_true",
+                        help="Ignore all 'UTILIKIT_*' environment variables.")
+    parser.add_argument("--cleaning-list", metavar="FILE",
+                        help=("The cleaning list file. "
+                              "Required if --ignore-environment is set"))
+    parser.add_argument("--root-dir", metavar="DIRECTORY",
+                        help=("The root directory."
+                              "Required if --ignore-environment is set"))
+    args = parser.parse_args(args_list)
+    if args.ignore_environment:
+        error_template = "--{} required since --ignore-environment is set"
+        if not args.cleaning_list:
+            parser.error(error_template.format("cleaning-list"))
+        if not args.root_dir:
+            parser.error(error_template.format("root-dir"))
+    else:
+        resources = _common.get_resource_obj()
+        cleaning_list = resources.read_cleaning_list(use_generator=True)
+        root_dir = _common.get_sandbox_dir()
+    if args.cleaning_list:
+        cleaning_list_path = pathlib.Path(args.cleaning_list)
+        if not cleaning_list_path.exists():
+            parser.error("Specified list does not exist: " + args.cleaning_list)
+        cleaning_list = _common.read_list_generator(cleaning_list_path)
+    if args.root_dir:
+        root_dir = pathlib.Path(args.root_dir)
+        if not root_dir.is_dir():
+            parser.error("Specified root directory does not exist: " + args.root_dir)
+
+    clean_sources(cleaning_list, root_dir)
+
+    return 0
+
+if __name__ == "__main__":
+    exit(main(sys.argv[1:]))

utilikit/export_resources.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+# -*- coding: UTF-8 -*-
+
+# ungoogled-chromium: Modifications to Google Chromium for removing Google integration
+# and enhancing privacy, control, and transparency
+# Copyright (C) 2017  Eloston
+#
+# This file is part of ungoogled-chromium.
+#
+# ungoogled-chromium is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# ungoogled-chromium is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with ungoogled-chromium.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Exports resources for a specific configuration"""
+
+import pathlib
+import sys
+import argparse
+
+def fix_relative_import():
+    """Allow relative imports to work from anywhere"""
+    import os.path
+    parent_path = os.path.dirname(os.path.realpath(os.path.abspath(__file__)))
+    sys.path.insert(0, os.path.dirname(parent_path))
+    global __package__ #pylint: disable=global-variable-undefined
+    __package__ = os.path.basename(parent_path) #pylint: disable=redefined-builtin
+    __import__(__package__)
+    sys.path.pop(0)
+
+if __name__ == "__main__" and (__package__ is None or __package__ == ""):
+    fix_relative_import()
+
+from . import _common #pylint: disable=wrong-import-position
+from . import substitute_domains as _substitute_domains #pylint: disable=wrong-import-position
+
+def _parse_args(args_list):
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--domain-substitute-patches", action="store_true", default=False,
+                        help="Apply domain substituion over patches")
+    parser.add_argument("output_dir", help="The directory to output resources to. ")
+    parser.add_argument("target_config", help="The target configuration to assemble")
+    args = parser.parse_args(args_list)
+    output_dir = pathlib.Path(args.output_dir)
+    if not output_dir.is_dir():
+        raise NotADirectoryError(args.output_dir)
+    return args.target_config, output_dir, args.domain_substitute_patches
+
+def main(args): #pylint: disable=too-many-locals
+    """Entry point"""
+    target_config, output_dir, domain_substitute_patches = _parse_args(args)
+
+    resources = _common.ResourceConfig(target_config)
+
+    patch_order = resources.read_patch_order()
+
+    _common.write_list(output_dir / _common.CLEANING_LIST, resources.read_cleaning_list())
+    _common.write_list(output_dir / _common.DOMAIN_REGEX_LIST,
+                       resources.read_domain_regex_list(binary=False))
+    _common.write_list(output_dir / _common.DOMAIN_SUBSTITUTION_LIST,
+                       resources.read_domain_substitution_list())
+    _common.write_ini(output_dir / _common.EXTRA_DEPS_INI, resources.read_extra_deps())
+    _common.write_dict_list(output_dir / _common.GN_FLAGS, resources.read_gn_flags())
+    _common.write_list(output_dir / _common.PATCH_ORDER, patch_order)
+    _common.write_ini(output_dir / _common.VERSION_INI,
+                      resources._read_ini(_common.VERSION_INI)) #pylint: disable=protected-access
+
+    output_patches_dir = output_dir / "patches"
+    output_patches_dir.mkdir(exist_ok=True)
+
+    for patch_name in patch_order:
+        input_path = resources.get_patches_dir() / patch_name
+        output_path = output_patches_dir / pathlib.Path(patch_name)
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        output_path.write_bytes(input_path.read_bytes())
+
+    if domain_substitute_patches:
+        _substitute_domains.substitute_domains(
+            _substitute_domains.get_parsed_domain_regexes(resources.read_domain_regex_list()),
+            patch_order, output_patches_dir, log_warnings=False)
+
+    return 0
+
+if __name__ == "__main__":
+    exit(main(sys.argv[1:]))

utilikit/generate_build_files.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+# -*- coding: UTF-8 -*-
+
+# ungoogled-chromium: Modifications to Google Chromium for removing Google
+# integration and enhancing privacy, control, and transparency
+# Copyright (C) 2017  Eloston
+#
+# This file is part of ungoogled-chromium.
+#
+# ungoogled-chromium is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# ungoogled-chromium is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with ungoogled-chromium.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Simple build files generator using assembled resources"""
+
+import pathlib
+import argparse
+import sys
+
+def fix_relative_import():
+    """Allow relative imports to work from anywhere"""
+    import os.path
+    parent_path = os.path.dirname(os.path.realpath(os.path.abspath(__file__)))
+    sys.path.insert(0, os.path.dirname(parent_path))
+    global __package__ #pylint: disable=global-variable-undefined
+    __package__ = os.path.basename(parent_path) #pylint: disable=redefined-builtin
+    __import__(__package__)
+    sys.path.pop(0)
+
+if __name__ == "__main__" and (__package__ is None or __package__ == ""):
+    fix_relative_import()
+
+from . import _common #pylint: disable=wrong-import-position
+
+def _add_subparsers(subparsers):
+    """Adds argument subparsers"""
+    subparsers.required = True # Workaround: http://bugs.python.org/issue9253#msg186387
+    def _debian_callback(resources, output_dir, args):
+        from ._build_files_generators import debian
+        debian.generate_build_files(resources, output_dir, args.build_output,
+                                    args.distro_version, args.disable_domain_substitution)
+    debian_subparser = subparsers.add_parser("debian", help="Generator for Debian and derivatives")
+    debian_subparser.add_argument("--build-output", metavar="DIRECTORY", default="out/Default",
+                                  help="The Chromium build output directory")
+    debian_subparser.add_argument("--distro-version", default="stable",
+                                  help=("The target distribution version (for use in "
+                                        "'debian/changelog'"))
+    debian_subparser.add_argument("--disable-domain-substitution", action="store_true",
+                                  help="Disable use of domain substitution")
+    debian_subparser.set_defaults(callback=_debian_callback)
+
+def _main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--ignore-environment", action="store_true",
+                        help="Ignore all 'UTILIKIT_*' environment variables.")
+    parser.add_argument("--resources", metavar="DIRECTORY",
+                        help=("The assembled resources directory. "
+                              "Required if --ignore-environment is set"))
+    parser.add_argument("--output-dir", metavar="DIRECTORY",
+                        help=("The directory to output build files to. "
+                              "Required if --ignore-environment is set"))
+
+    _add_subparsers(parser.add_subparsers(title="Build file types", dest="files_type"))
+
+    args = parser.parse_args()
+
+    if args.ignore_environment:
+        error_template = "--{} required since --ignore-environment is set"
+        if not args.resources:
+            parser.error(error_template.format("resources"))
+        if not args.output_dir:
+            parser.error(error_template.format("output-dir"))
+    else:
+        resources = _common.get_resource_obj()
+
+    if args.resources:
+        resources_path = pathlib.Path(args.resources)
+        if not resources_path.is_dir():
+            parser.error("--resources value is not a directory: " + args.resources)
+        resources = _common.StandaloneResourceDirectory(resources_path)
+
+    if args.output_dir:
+        output_dir = pathlib.Path(args.output_dir)
+        if not output_dir.is_dir():
+            parser.error("--output-dir value is not a directory: " + args.output_dir)
+
+    args.callback(resources, output_dir, args)
+
+if __name__ == "__main__":
+    _main()

utilities/prepare_dependencies.py → utilikit/prepare_sources.py

@@ -20,11 +20,10 @@
 # You should have received a copy of the GNU General Public License
 # along with ungoogled-chromium.  If not, see <http://www.gnu.org/licenses/>.
 
-"""Downloads and extracts the main source or extra dependencies"""
+"""Downloads the main source and extra dependencies"""
 
 import pathlib
 import sys
-import configparser
 import shutil
 import os
 import tarfile
@@ -32,23 +31,20 @@ import urllib.request
 import hashlib
 import argparse
 
-def read_extra_deps(deps_path):
-    """Reads extra_deps.ini"""
-    config = configparser.ConfigParser()
-    config.read(str(deps_path))
-    return config
+def fix_relative_import():
+    """Allow relative imports to work from anywhere"""
+    import os.path #pylint: disable=redefined-outer-name
+    parent_path = os.path.dirname(os.path.realpath(os.path.abspath(__file__)))
+    sys.path.insert(0, os.path.dirname(parent_path))
+    global __package__ #pylint: disable=global-variable-undefined
+    __package__ = os.path.basename(parent_path) #pylint: disable=redefined-builtin
+    __import__(__package__)
+    sys.path.pop(0)
 
-def _read_list(list_path):
-    """
-    Reads a text document that is a simple new-line delimited list
+if __name__ == "__main__" and (__package__ is None or __package__ == ""):
+    fix_relative_import()
 
-    Blank lines are ignored
-    """
-    if not list_path.exists():
-        return list()
-    with list_path.open() as file_obj:
-        tmp_list = file_obj.read().splitlines()
-        return [x for x in tmp_list if len(x) > 0]
+from . import _common #pylint: disable=wrong-import-position
 
 def _extract_tar_file(tar_path, destination_dir, ignore_files, relative_to):
     """Improved one-time tar extraction function"""
@@ -185,45 +181,57 @@ def download_main_source(version, downloads_dir, root_dir, source_cleaning_list)
 def main(args_list):
     """Entry point"""
     parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("--mode", choices=["main_source", "extra_deps"],
-                        help="The dependency to download and unpack")
-    parser.add_argument("--downloads-dir", required=True, metavar="DIRECTORY",
-                        help="The directory to store downloaded archive files")
-    parser.add_argument("--root-dir", required=True, metavar="DIRECTORY",
-                        help="The root directory of the source tree")
+    parser.add_argument("--ignore-environment", action="store_true",
+                        help="Ignore all 'UTILIKIT_*' environment variables.")
+    parser.add_argument("--downloads-dir", metavar="DIRECTORY",
+                        help=("The directory to store downloaded archive files. "
+                              "Required if --ignore-environment is set"))
+    parser.add_argument("--root-dir", metavar="DIRECTORY",
+                        help=("The root directory of the source tree. "
+                              "Required if --ignore-environment is set"))
     parser.add_argument("--chromium-version", metavar="X.X.X.X",
-                        help=("The Chromium version to download. Required if"
-                              "mode is 'main_source'"))
+                        help=("The Chromium version to download. "
+                              "Required if --ignore-environment is set"))
     parser.add_argument("--source-cleaning-list", metavar="FILE",
-                        help=("The path to the source cleaning list. If not"
-                              "specified, the source is not cleaned during"
-                              " unpacking. Used only when mode is"
-                              " 'main_source'"))
+                        help=("The path to the source cleaning list. If not "
+                              "specified, the source is not cleaned during "
+                              "unpacking. Use '-' to read stdin."))
     parser.add_argument("--extra-deps-path", metavar="INI_FILE",
-                        help=("The path to the extra deps ini file. Required if"
-                              " mode is 'extra_deps'"))
+                        help="The path to the extra deps ini file.")
     args = parser.parse_args(args_list)
-    downloads_dir = pathlib.Path(args.downloads_dir)
-    if not downloads_dir.is_dir():
-        parser.error("--downloads-dir value '{}' is not a directory".format(args.downloads_dir))
-    root_dir = pathlib.Path(args.root_dir)
-    if not root_dir.is_dir():
-        parser.error("--root-dir value '{}' is not a directory".format(args.root_dir))
-    if args.mode == "main_source":
+    source_cleaning_list = list()
+    extra_deps = dict()
+    if args.ignore_environment:
+        error_template = "--{} required since --ignore-environment is set"
+        if not args.downloads_dir:
+            parser.error(error_template.format("downloads-dir"))
+        if not args.root_dir:
+            parser.error(error_template.format("root-dir"))
         if not args.chromium_version:
-            parser.error("--chromium-version required when --mode is 'main_source'")
-        source_cleaning_list = list()
-        if args.source_cleaning_list:
-            source_cleaning_list = _read_list(pathlib.Path(args.source_cleaning_list))
-            print("Parsed source cleaning list")
-        else:
-            print("Disabling source cleaning because no source cleaning list was provided.")
-        download_main_source(args.chromium_version, downloads_dir, root_dir, source_cleaning_list)
-    elif args.mode == "extra_deps":
-        if not args.extra_deps_path:
-            parser.error("--extra-deps-path required when --mode is 'extra_deps'")
-        extra_deps_path = pathlib.Path(args.extra_deps_path)
-        download_extra_deps(read_extra_deps(extra_deps_path), root_dir, downloads_dir)
+            parser.error(error_template.format("chromium-version"))
+    else:
+        resources = _common.get_resource_obj()
+        source_cleaning_list = resources.read_cleaning_list() #pylint: disable=redefined-variable-type
+        chromium_version = resources.read_version()[0]
+        extra_deps = resources.read_extra_deps()
+        root_dir = _common.get_sandbox_dir()
+        downloads_dir = _common.get_downloads_dir()
+    if args.downloads_dir:
+        downloads_dir = pathlib.Path(args.downloads_dir)
+        if not downloads_dir.is_dir():
+            parser.error("--downloads-dir value '{}' is not a directory".format(args.downloads_dir))
+    if args.root_dir:
+        root_dir = pathlib.Path(args.root_dir)
+        if not root_dir.is_dir():
+            parser.error("--root-dir value '{}' is not a directory".format(args.root_dir))
+    if args.chromium_version:
+        chromium_version = args.chromium_version
+    if args.source_cleaning_list:
+        source_cleaning_list = _common.read_list(pathlib.Path(args.source_cleaning_list))
+    if args.extra_deps_path:
+        extra_deps = _common.read_ini(pathlib.Path(args.extra_deps_path))
+    download_main_source(chromium_version, downloads_dir, root_dir, source_cleaning_list)
+    download_extra_deps(extra_deps, root_dir, downloads_dir)
 
     return 0
 

utilikit/substitute_domains.py

@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+# -*- coding: UTF-8 -*-
+
+# ungoogled-chromium: Modifications to Google Chromium for removing Google integration
+# and enhancing privacy, control, and transparency
+# Copyright (C) 2017  Eloston
+#
+# This file is part of ungoogled-chromium.
+#
+# ungoogled-chromium is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# ungoogled-chromium is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with ungoogled-chromium.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Runs domain substitution"""
+
+import pathlib
+import sys
+import re
+import argparse
+
+def fix_relative_import():
+    """Allow relative imports to work from anywhere"""
+    import os.path
+    parent_path = os.path.dirname(os.path.realpath(os.path.abspath(__file__)))
+    sys.path.insert(0, os.path.dirname(parent_path))
+    global __package__ #pylint: disable=global-variable-undefined
+    __package__ = os.path.basename(parent_path) #pylint: disable=redefined-builtin
+    __import__(__package__)
+    sys.path.pop(0)
+
+if __name__ == "__main__" and (__package__ is None or __package__ == ""):
+    fix_relative_import()
+
+from . import _common #pylint: disable=wrong-import-position
+
+def get_parsed_domain_regexes(domain_regex_list):
+    """Parses and compiles domain regular expressions"""
+    domain_regexes = list()
+    for expression in domain_regex_list:
+        expression = expression.split(b'#')
+        domain_regexes.append((re.compile(expression[0]), expression[1]))
+    return domain_regexes
+
+def substitute_domains(regex_list, file_list, root_dir, log_warnings=True):
+    """Runs domain substitution with regex_list over files file_list"""
+
+    for path in file_list:
+        try:
+            with (root_dir / path).open(mode="r+b") as file_obj:
+                content = file_obj.read()
+                file_subs = 0
+                for regex_pair in regex_list:
+                    compiled_regex, replacement_regex = regex_pair
+                    content, number_of_subs = compiled_regex.subn(replacement_regex, content)
+                    file_subs += number_of_subs
+                if file_subs > 0:
+                    file_obj.seek(0)
+                    file_obj.write(content)
+                    file_obj.truncate()
+                elif log_warnings:
+                    print("File {} has no matches".format(path))
+        except Exception as exc:
+            print("Exception thrown for path {}".format(path))
+            raise exc
+
+def _parse_args(args_list):
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--ignore-environment", action="store_true",
+                        help="Ignore all 'UTILIKIT_*' environment variables.")
+    parser.add_argument("--domain-regex-list", metavar="FILE",
+                        help=("Path to the domain regular expression list "
+                              "Required if --ignore-environment is set"))
+    parser.add_argument("--domain-substitution-list", metavar="FILE",
+                        help=("Path to the domain substitution list. "
+                              "Use '-' to read from stdin. "
+                              "Required if --ignore-environment is set"))
+    parser.add_argument("--root-dir", metavar="DIRECTORY",
+                        help=("The directory to operate relative to. "
+                              "Required if --ignore-environment is set"))
+    args = parser.parse_args(args_list)
+    if args.ignore_environment:
+        error_template = "--{} required since --ignore-environment is set"
+        if not args.domain_regex_list:
+            parser.error(error_template.format("domain-regex-list"))
+        if not args.domain_substitution_list:
+            parser.error(error_template.format("domain-substitution-list"))
+        if not args.root_dir:
+            parser.error(error_template.format("root-dir"))
+    else:
+        resources = _common.get_resource_obj()
+        domain_regex_list = resources.read_domain_regex_list()
+        domain_substitution_list = resources.read_domain_substitution_list(use_generator=True)
+        root_dir = _common.get_sandbox_dir()
+    if args.domain_regex_list:
+        domain_regex_list_path = pathlib.Path(args.domain_regex_list)
+        if not domain_regex_list_path.exists():
+            parser.error("--domain-regex-list path does not exist: " + args.domain_regex_list)
+        domain_regex_list = _common.read_list(domain_regex_list_path, binary=True)
+    if args.domain_substitution_list:
+        domain_substitution_list_path = pathlib.Path(args.domain_substitution_list)
+        if not args.domain_substitution_list == "-" and not domain_substitution_list_path.exists():
+            parser.error("--domain-substitution-list path does not exist: " +
+                         args.domain_substitution_list)
+        domain_substitution_list = _common.read_list_generator(domain_substitution_list_path)
+    if args.root_dir:
+        root_dir = pathlib.Path(args.root_dir)
+        if not root_dir.is_dir():
+            parser.error("--root-dir is not a directory: " + args.root_dir)
+    return domain_regex_list, domain_substitution_list, root_dir
+
+def main(args):
+    """Entry point"""
+
+    domain_regex_list, domain_substitution_list, root_dir = _parse_args(args)
+    substitute_domains(get_parsed_domain_regexes(domain_regex_list),
+                       domain_substitution_list, root_dir)
+
+    return 0
+
+if __name__ == "__main__":
+    exit(main(sys.argv[1:]))

utilities/assemble_resources.py

@@ -1,242 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: UTF-8 -*-
-
-# ungoogled-chromium: Modifications to Google Chromium for removing Google integration
-# and enhancing privacy, control, and transparency
-# Copyright (C) 2016  Eloston
-#
-# This file is part of ungoogled-chromium.
-#
-# ungoogled-chromium is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# ungoogled-chromium is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with ungoogled-chromium.  If not, see <http://www.gnu.org/licenses/>.
-
-"""Assembles resources for a specific configuration"""
-
-import pathlib
-import configparser
-import sys
-import argparse
-
-# TODO: Should probably be customizable via an environment variable
-ROOT_DIR = pathlib.Path(__file__).absolute().parent.parent
-VERSION = ROOT_DIR / "version.ini"
-RESOURCES = ROOT_DIR / "resources"
-CONFIGS = RESOURCES / "configs"
-PACKAGING = RESOURCES / "packaging"
-PATCHES = RESOURCES / "patches"
-
-CLEANING_LIST = "cleaning_list"
-DOMAIN_REGEX_LIST = "domain_regex_list"
-DOMAIN_SUBSTITUTION_LIST = "domain_substitution_list"
-EXTRA_DEPS = "extra_deps.ini"
-GN_FLAGS = "gn_flags"
-METADATA = "metadata.ini"
-PATCH_ORDER = "patch_order"
-
-class ConfigurationReader:
-    """A reader for a configuration directory"""
-
-    @staticmethod
-    def _read_ini(ini_path):
-        """Returns a configparser object"""
-        if not ini_path.exists():
-            return dict()
-        config = configparser.ConfigParser()
-        config.read(str(ini_path))
-        return config
-
-    @staticmethod
-    def _read_list(list_path, is_binary=False):
-        """
-        Reads a text document that is a simple new-line delimited list
-
-        Blank lines are ignored
-        """
-        if not list_path.exists():
-            return list()
-        if is_binary:
-            file_mode = "rb"
-        else:
-            file_mode = "r"
-        with list_path.open(file_mode) as file_obj:
-            tmp_list = file_obj.read().splitlines()
-            return [x for x in tmp_list if len(x) > 0]
-
-    def __init__(self, path):
-        self.path = path
-        self.name = path.name
-
-        self.visible = False
-        self.parent = None
-        self.display_name = self.name
-
-    def _read_dict_list(self, dict_list_path, is_binary=False):
-        """
-        Reads a text document that is a list of key-value pairs delimited by an equals sign
-
-        Blank lines are ignored
-        """
-        if not dict_list_path.exists():
-            return dict()
-        if is_binary:
-            delimiter = b"="
-        else:
-            delimiter = "=" #pylint: disable=redefined-variable-type
-        tmp_dict = dict()
-        for entry in self._read_list(dict_list_path, is_binary):
-            key, value = entry.split(delimiter)
-            tmp_dict[key] = value
-        return tmp_dict
-
-    def read_metadata(self, config_dict):
-        """Reads metadata.ini"""
-        metadata_config = self._read_ini(self.path / METADATA)
-        for section in metadata_config:
-            if section == "DEFAULT":
-                continue
-            elif section == "config":
-                for keyname in metadata_config["config"]:
-                    if keyname == "display_name":
-                        self.display_name = metadata_config[section][keyname]
-                    elif keyname == "parent":
-                        self.parent = config_dict[metadata_config[section][keyname]]
-                    elif keyname == "visible":
-                        self.visible = metadata_config[section][keyname]
-                    else:
-                        raise NameError("Unknown key name: {}. Configuration: {}".format(
-                            keyname, self.path.name))
-            else:
-                raise NameError("Unknown section name: {}. Configuration: {}".format(
-                    section, self.path.name))
-
-    def read_cleaning_list(self):
-        """Reads cleaning_list"""
-        return self._read_list(self.path / CLEANING_LIST)
-
-    def read_domain_regex_list(self):
-        """Reads domain_regex_list"""
-        return self._read_list(self.path / DOMAIN_REGEX_LIST)
-
-    def read_domain_substitution_list(self):
-        """Reads domain_substitution_list"""
-        return self._read_list(self.path / DOMAIN_SUBSTITUTION_LIST)
-
-    def read_extra_deps(self):
-        """Reads extra_deps.ini"""
-        extra_deps_config = self._read_ini(self.path / EXTRA_DEPS)
-        tmp_dict = dict()
-        for section in extra_deps_config:
-            if section == "DEFAULT":
-                continue
-            else:
-                tmp_dict[section] = dict()
-                for keyname in extra_deps_config[section]:
-                    if keyname not in ["version", "url", "download_name", "strip_leading_dirs"]:
-                        raise KeyError(keyname)
-                    tmp_dict[section][keyname] = extra_deps_config[section][keyname]
-        return tmp_dict
-
-    def read_gn_flags(self):
-        """Reads gn_flags"""
-        return self._read_dict_list(self.path / GN_FLAGS)
-
-    def read_patch_order(self):
-        """Reads patch_order"""
-        return self._read_list(self.path / PATCH_ORDER)
-
-def _parse_args(args_list):
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("target_config", help="The target configuration to assemble")
-    parser.add_argument("--output-dir", metavar="DIRECTORY", required=True,
-                        help="The directory to output resources to. ")
-    args = parser.parse_args(args_list)
-    output_dir = pathlib.Path(args.output_dir)
-    if not output_dir.is_dir():
-        raise NotADirectoryError(args.output_dir)
-    return args.target_config, output_dir
-
-def _get_config_dict():
-    config_dict = dict()
-    for filepath in CONFIGS.iterdir():
-        if filepath.is_dir():
-            config_dict[filepath.name] = ConfigurationReader(filepath)
-    for config_obj in config_dict.values():
-        config_obj.read_metadata(config_dict)
-    return config_dict
-
-def _traverse_down_to_node(last_node):
-    current_config = last_node
-    stack = list()
-    while not current_config is None:
-        stack.append(current_config)
-        current_config = current_config.parent
-    yield from reversed(stack)
-
-def _write_list(path, list_obj):
-    with path.open("w") as file_obj:
-        file_obj.write("\n".join(list_obj))
-
-def _write_dict_list(path, dict_obj):
-    _write_list(path, [key + "=" + value for key, value in dict_obj.items()])
-
-def _write_ini(path, dict_obj):
-    config = configparser.ConfigParser()
-    for section in dict_obj:
-        config.add_section(section)
-        for option, value in config[section].items():
-            config.set(section, option, value)
-    with path.open("w") as file_obj:
-        config.write(file_obj)
-
-def main(args): #pylint: disable=too-many-locals
-    """Entry point"""
-    target_config, output_dir = _parse_args(args)
-
-    cleaning_list = list()
-    domain_regex_list = list()
-    domain_substitution_list = list()
-    extra_deps = dict()
-    gn_flags = dict()
-    patch_order = list()
-    for config_obj in _traverse_down_to_node(_get_config_dict()[target_config]):
-        cleaning_list.extend(config_obj.read_cleaning_list())
-        domain_regex_list.extend(config_obj.read_domain_regex_list())
-        domain_substitution_list.extend(config_obj.read_domain_substitution_list())
-        for key, value in config_obj.read_extra_deps().items():
-            extra_deps[key] = value
-        for key, value in config_obj.read_gn_flags().items():
-            gn_flags[key] = value
-        patch_order.extend(config_obj.read_patch_order())
-
-    _write_list(output_dir / CLEANING_LIST, cleaning_list)
-    _write_list(output_dir / DOMAIN_REGEX_LIST, domain_regex_list)
-    _write_list(output_dir / DOMAIN_SUBSTITUTION_LIST, domain_substitution_list)
-    _write_ini(output_dir / EXTRA_DEPS, extra_deps)
-    _write_dict_list(output_dir / GN_FLAGS, gn_flags)
-    _write_list(output_dir / PATCH_ORDER, patch_order)
-
-    (output_dir / VERSION.name).write_bytes(VERSION.read_bytes())
-
-    output_patches_dir = output_dir / "patches"
-    output_patches_dir.mkdir(exist_ok=True)
-
-    for patch_name in patch_order:
-        input_path = PATCHES / pathlib.Path(patch_name)
-        output_path = output_patches_dir / pathlib.Path(patch_name)
-        output_path.parent.mkdir(parents=True, exist_ok=True)
-        output_path.write_bytes(input_path.read_bytes())
-
-    return 0
-
-if __name__ == "__main__":
-    exit(main(sys.argv[1:]))

utilities/build_files_generator/__init__.py

@@ -1,128 +0,0 @@
-# -*- coding: UTF-8 -*-
-
-# ungoogled-chromium: Modifications to Google Chromium for removing Google
-# integration and enhancing privacy, control, and transparency
-# Copyright (C) 2016  Eloston
-#
-# This file is part of ungoogled-chromium.
-#
-# ungoogled-chromium is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# ungoogled-chromium is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with ungoogled-chromium.  If not, see <http://www.gnu.org/licenses/>.
-
-"""Common code for all build file generators"""
-
-import configparser
-import pathlib
-
-# TODO: Should probably be customizable via an environment variable
-ROOT_DIR = pathlib.Path(__file__).absolute().parent.parent.parent
-
-class ResourcesParser: # pylint: disable=too-many-instance-attributes
-    """Parses a resources directory"""
-
-    @staticmethod
-    def _read_ini(ini_path):
-        """Returns a configparser object"""
-        if not ini_path.exists():
-            return dict()
-        config = configparser.ConfigParser()
-        config.read(str(ini_path))
-        return config
-
-    @staticmethod
-    def _read_list(list_path, is_binary=False):
-        """
-        Reads a text document that is a simple new-line delimited list
-
-        Blank lines are ignored
-        """
-        if not list_path.exists():
-            return list()
-        if is_binary:
-            file_mode = "rb"
-        else:
-            file_mode = "r"
-        with list_path.open(file_mode) as file_obj:
-            tmp_list = file_obj.read().splitlines()
-            return [x for x in tmp_list if len(x) > 0]
-
-    def __init__(self, resources_dir):
-        # pylint: disable=invalid-name
-        self.CLEANING_LIST = resources_dir / "cleaning_list"
-        self.DOMAIN_SUBSTITUTION_LIST = resources_dir / "domain_substitution_list"
-        self.GN_FLAGS = resources_dir / "gn_flags"
-        self.DOMAIN_REGEX_LIST = resources_dir / "domain_regex_list"
-        self.EXTRA_DEPS_INI = resources_dir / "extra_deps.ini"
-        self.PATCHES = resources_dir / "patches"
-        self.PATCH_ORDER = resources_dir / "patch_order"
-        self.VERSION_INI = resources_dir / "version.ini"
-
-    def _read_dict_list(self, dict_list_path, is_binary=False):
-        """
-        Reads a text document that is a list of key-value pairs delimited by an equals sign
-
-        Blank lines are ignored
-        """
-        if not dict_list_path.exists():
-            return dict()
-        if is_binary:
-            delimiter = b"="
-        else:
-            delimiter = "=" #pylint: disable=redefined-variable-type
-        tmp_dict = dict()
-        for entry in self._read_list(dict_list_path, is_binary):
-            key, value = entry.split(delimiter)
-            tmp_dict[key] = value
-        return tmp_dict
-
-    def get_version(self):
-        """Returns a tuple of (chromium_version, release_revision)"""
-        version_config = self._read_ini(self.VERSION_INI)
-        return (version_config["main"]["chromium_version"],
-                version_config["main"]["release_revision"])
-
-    def get_cleaning_list(self):
-        """Reads cleaning_list"""
-        return self._read_list(self.CLEANING_LIST)
-
-    def get_domain_regex_list(self):
-        """Reads domain_regex_list"""
-        return self._read_list(self.DOMAIN_REGEX_LIST)
-
-    def get_domain_substitution_list(self):
-        """Reads domain_substitution_list"""
-        return self._read_list(self.DOMAIN_SUBSTITUTION_LIST)
-
-    def get_extra_deps(self):
-        """Reads extra_deps.ini"""
-        extra_deps_config = self._read_ini(self.EXTRA_DEPS_INI)
-        tmp_dict = dict()
-        for section in extra_deps_config:
-            if section == "DEFAULT":
-                continue
-            else:
-                tmp_dict[section] = dict()
-                # TODO: Syntax validity shouldn't be checked here
-                for keyname in extra_deps_config[section]:
-                    if keyname not in ["version", "url", "download_name", "strip_leading_dirs"]:
-                        raise KeyError(keyname)
-                    tmp_dict[section][keyname] = extra_deps_config[section][keyname]
-        return tmp_dict
-
-    def get_gn_flags(self):
-        """Reads gn_flags"""
-        return self._read_dict_list(self.GN_FLAGS)
-
-    def get_patch_order(self):
-        """Reads patch_order"""
-        return self._read_list(self.PATCH_ORDER)

utilities/build_files_generator/__main__.py

@@ -1,68 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: UTF-8 -*-
-
-# ungoogled-chromium: Modifications to Google Chromium for removing Google
-# integration and enhancing privacy, control, and transparency
-# Copyright (C) 2016  Eloston
-#
-# This file is part of ungoogled-chromium.
-#
-# ungoogled-chromium is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# ungoogled-chromium is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with ungoogled-chromium.  If not, see <http://www.gnu.org/licenses/>.
-
-"""Entry point for the build files generator"""
-
-import pathlib
-import argparse
-
-from . import ResourcesParser
-
-def _add_subparsers(subparsers):
-    """Adds argument subparsers"""
-    subparsers.required = True # Workaround: http://bugs.python.org/issue9253#msg186387
-    def _debian_callback(resources_parser, output_dir, args):
-        from . import debian
-        debian.generate_build_files(resources_parser, output_dir, args.build_output,
-                                    args.distro_version)
-    debian_subparser = subparsers.add_parser("debian", help="Generator for Debian and derivatives")
-    debian_subparser.add_argument("--build-output", metavar="DIRECTORY", default="out/Default",
-                                  help="The Chromium build output directory")
-    debian_subparser.add_argument("--distro-version", default="stable",
-                                  help=("The target distribution version (for use in "
-                                        "'debian/changelog'"))
-    debian_subparser.set_defaults(callback=_debian_callback)
-
-def _main():
-    parser = argparse.ArgumentParser(
-        description="Simple build files generator using assembled resources")
-    parser.add_argument("--resources-dir", required=True, metavar="DIRECTORY",
-                        help="The assembled resources directory")
-    parser.add_argument("--output-dir", metavar="DIRECTORY", default=".",
-                        help="The directory to output build files to")
-
-    _add_subparsers(parser.add_subparsers(title="Build file types", dest="files_type"))
-
-    args = parser.parse_args()
-
-    resources_dir = pathlib.Path(args.resources_dir)
-    if not resources_dir.is_dir():
-        parser.error("--resources-dir value is not a directory: " + args.resources_dir)
-
-    output_dir = pathlib.Path(args.output_dir)
-    if not output_dir.is_dir():
-        parser.error("--output-dir value is not a directory: " + args.output_dir)
-
-    resources_parser = ResourcesParser(resources_dir)
-    args.callback(resources_parser, output_dir, args)
-
-_main()

utilities/clean_sources.py

@@ -1,66 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: UTF-8 -*-
-
-# ungoogled-chromium: Modifications to Google Chromium for removing Google integration
-# and enhancing privacy, control, and transparency
-# Copyright (C) 2016  Eloston
-#
-# This file is part of ungoogled-chromium.
-#
-# ungoogled-chromium is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# ungoogled-chromium is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with ungoogled-chromium.  If not, see <http://www.gnu.org/licenses/>.
-
-"""Runs source cleaner"""
-
-import pathlib
-import sys
-import argparse
-
-def read_cleaning_list(list_path):
-    """Reads cleaning_list"""
-    if not list_path.exists():
-        return list()
-    with list_path.open() as file_obj:
-        tmp_list = file_obj.read().splitlines()
-        return [x for x in tmp_list if len(x) > 0]
-
-def clean_sources(cleaning_list_iter, root_dir):
-    """Delete files given by iterable cleaning_list_iter relative to root_dir"""
-    for entry in cleaning_list_iter:
-        tmp_path = root_dir / entry
-        try:
-            tmp_path.unlink()
-        except FileNotFoundError:
-            print("No such file: " + str(tmp_path))
-
-def main(args_list):
-    """Entry point"""
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("--cleaning-list", required=True, metavar="FILE",
-                        help="The cleaning list file")
-    parser.add_argument("--root-dir", metavar="DIRECTORY", required=True,
-                        help="The root directory.")
-    args = parser.parse_args(args_list)
-    cleaning_list_path = pathlib.Path(args.cleaning_list)
-    if not cleaning_list_path.exists():
-        parser.error("Specified list does not exist: " + args.cleaning_list)
-    root_dir = pathlib.Path(args.root_dir)
-    if not root_dir.is_dir():
-        parser.error("Specified root directory does not exist: " + args.root_dir)
-
-    clean_sources(read_cleaning_list(cleaning_list_path), root_dir)
-
-    return 0
-
-if __name__ == "__main__":
-    exit(main(sys.argv[1:]))

utilities/substitute_domains.py

@@ -1,109 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: UTF-8 -*-
-
-# ungoogled-chromium: Modifications to Google Chromium for removing Google integration
-# and enhancing privacy, control, and transparency
-# Copyright (C) 2016  Eloston
-#
-# This file is part of ungoogled-chromium.
-#
-# ungoogled-chromium is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# ungoogled-chromium is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with ungoogled-chromium.  If not, see <http://www.gnu.org/licenses/>.
-
-"""Runs domain substitution"""
-
-import pathlib
-import sys
-import re
-import argparse
-
-def _line_generator(file_obj):
-    for line in file_obj.read().splitlines():
-        if len(line) > 0:
-            yield line
-
-def _read_list(list_path, binary=False):
-    """Reads a list. Ignores `binary` if reading from stdin"""
-    if binary:
-        mode = "rb"
-    else:
-        mode = "r"
-    if str(list_path) == "-":
-        yield from _line_generator(sys.stdin)
-    else:
-        with list_path.open(mode) as file_obj:
-            yield from _line_generator(file_obj)
-
-def get_parsed_domain_regexes(domain_regex_list_path):
-    """Parses and compiles domain regular expressions"""
-    domain_regexes = list()
-    for expression in _read_list(domain_regex_list_path, binary=True):
-        expression = expression.split(b'#')
-        domain_regexes.append((re.compile(expression[0]), expression[1]))
-    return domain_regexes
-
-def substitute_domains(regex_list, file_list, root_dir, log_warnings=True):
-    """Runs domain substitution with regex_list over files file_list"""
-
-    for path in file_list:
-        try:
-            with (root_dir / path).open(mode="r+b") as file_obj:
-                content = file_obj.read()
-                file_subs = 0
-                for regex_pair in regex_list:
-                    compiled_regex, replacement_regex = regex_pair
-                    content, number_of_subs = compiled_regex.subn(replacement_regex, content)
-                    file_subs += number_of_subs
-                if file_subs > 0:
-                    file_obj.seek(0)
-                    file_obj.write(content)
-                    file_obj.truncate()
-                elif log_warnings:
-                    print("File {} has no matches".format(path))
-        except Exception as exc:
-            print("Exception thrown for path {}".format(path))
-            raise exc
-
-def _parse_args(args_list):
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("--domain-regex-list", required=True, metavar="FILE",
-                        help="Path to the domain regular expression list")
-    parser.add_argument("--domain-substitution-list", metavar="FILE", default="-",
-                        help="Path to the domain substitution list. Default is to read from stdin")
-    parser.add_argument("--root-dir", metavar="DIRECTORY", required=True,
-                        help="The directory to operate relative to.")
-    args = parser.parse_args(args_list)
-    domain_regex_list_path = pathlib.Path(args.domain_regex_list)
-    if not domain_regex_list_path.exists():
-        parser.error("--domain-regex-list path does not exist: " + args.domain_regex_list)
-    domain_substitution_list_path = pathlib.Path(args.domain_substitution_list)
-    if not args.domain_substitution_list == "-" and not domain_substitution_list_path.exists():
-        parser.error("--domain-substitution-list path does not exist: " +
-                     args.domain_substitution_list)
-    root_dir = pathlib.Path(args.root_dir)
-    if not root_dir.is_dir():
-        parser.error("--root-dir is not a directory: " + args.root_dir)
-    return domain_regex_list_path, domain_substitution_list_path, root_dir
-
-def main(args):
-    """Entry point"""
-
-    domain_regex_list_path, domain_substitution_list_path, root_dir = _parse_args(args)
-    substitute_domains(get_parsed_domain_regexes(domain_regex_list_path),
-                       _read_list(domain_substitution_list_path),
-                       root_dir)
-
-    return 0
-
-if __name__ == "__main__":
-    exit(main(sys.argv[1:]))