From 1cf68e71977e55ec3872ab06c8928a3de9ea504b Mon Sep 17 00:00:00 2001 From: Joachim Metz Date: Fri, 15 May 2026 13:31:24 +0200 Subject: [PATCH 1/7] Changes to use black Python formatter --- data/rpm_templates/zstd.spec | 2 +- data/templates/.pylintrc | 7 +- data/templates/check_dependencies.py | 11 +- data/templates/docs/conf.py | 147 +- .../github_actions/lint.yml/header.yml | 51 + .../github_actions/lint.yml/yamllint.yml | 17 + data/templates/github_actions/test_tox.yml | 31 - data/templates/pyproject.toml/black.toml | 5 + data/templates/tox.ini/testenv_black | 16 + .../tox.ini/{testenv_lint => testenv_pylint} | 2 +- ...estenv_lint-with_yaml => testenv_yamllint} | 5 +- l2tdevtools/dependencies.py | 1243 +++++++++-------- .../dependency_writers/github_actions.py | 81 ++ l2tdevtools/dependency_writers/setup.py | 4 + l2tdevtools/dependency_writers/tox_ini.py | 23 +- tools/update-dependencies.py | 1 + 16 files changed, 911 insertions(+), 735 deletions(-) create mode 100644 data/templates/github_actions/lint.yml/header.yml create mode 100644 data/templates/github_actions/lint.yml/yamllint.yml create mode 100644 data/templates/pyproject.toml/black.toml create mode 100644 data/templates/tox.ini/testenv_black rename data/templates/tox.ini/{testenv_lint => testenv_pylint} (93%) rename data/templates/tox.ini/{testenv_lint-with_yaml => testenv_yamllint} (67%) diff --git a/data/rpm_templates/zstd.spec b/data/rpm_templates/zstd.spec index 983d87c5..d01fa03e 100644 --- a/data/rpm_templates/zstd.spec +++ b/data/rpm_templates/zstd.spec @@ -12,7 +12,7 @@ Group: Development/Libraries Prefix: %{{_prefix}} Vendor: Sergey Dryabzhinsky Url: https://github.com/sergey-dryabzhinsky/python-zstd -BuildRequires: gcc, python3-devel, python3-setuptools +BuildRequires: gcc, python3-devel, python3-pip, python3-setuptools %description Python bindings to Yann Collet ZSTD compression library. diff --git a/data/templates/.pylintrc b/data/templates/.pylintrc index 3a1c509c..5c04739d 100644 --- a/data/templates/.pylintrc +++ b/data/templates/.pylintrc @@ -358,12 +358,11 @@ indent-after-paren=4 # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 # tab). -# indent-string=' ' -indent-string=' ' +indent-string=' ' # Maximum number of characters on a single line. # max-line-length=100 -max-line-length=80 +max-line-length=88 # Maximum number of lines in a module. max-module-lines=1000 @@ -599,7 +598,7 @@ spelling-store-unknown-words=no # This flag controls whether inconsistent-quotes generates a warning when the # character used as a quote delimiter is used inconsistently within a module. -check-quote-consistency=no +check-quote-consistency=yes # This flag controls whether the implicit-str-concat should generate a warning # on implicit string concatenation in sequences defined over several lines. diff --git a/data/templates/check_dependencies.py b/data/templates/check_dependencies.py index 97e95d3a..6814c134 100755 --- a/data/templates/check_dependencies.py +++ b/data/templates/check_dependencies.py @@ -4,13 +4,12 @@ import sys # Change PYTHONPATH to include dependencies. -sys.path.insert(0, '.') +sys.path.insert(0, ".") import utils.dependencies # pylint: disable=wrong-import-position +if __name__ == "__main__": + dependency_helper = utils.dependencies.DependencyHelper() -if __name__ == '__main__': - dependency_helper = utils.dependencies.DependencyHelper() - - if not dependency_helper.CheckDependencies(): - sys.exit(1) + if not dependency_helper.CheckDependencies(): + sys.exit(1) diff --git a/data/templates/docs/conf.py b/data/templates/docs/conf.py index d5ac8a67..339912b1 100644 --- a/data/templates/docs/conf.py +++ b/data/templates/docs/conf.py @@ -9,30 +9,29 @@ from docutils import transforms # Change PYTHONPATH to include ${python_module_name} module and dependencies. -sys.path.insert(0, os.path.abspath('..')) +sys.path.insert(0, os.path.abspath("..")) import ${python_module_name} # pylint: disable=wrong-import-position import utils.dependencies # pylint: disable=wrong-import-position - # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. -needs_sphinx = '2.0.1' +needs_sphinx = "2.0.1" # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'recommonmark', - 'sphinx.ext.autodoc', - 'sphinx.ext.coverage', - 'sphinx.ext.doctest', - 'sphinx.ext.napoleon', - 'sphinx.ext.viewcode', - 'sphinx_markdown_tables', - 'sphinx_rtd_theme', + "recommonmark", + "sphinx.ext.autodoc", + "sphinx.ext.coverage", + "sphinx.ext.doctest", + "sphinx.ext.napoleon", + "sphinx.ext.viewcode", + "sphinx_markdown_tables", + "sphinx_rtd_theme", ] # We cannot install architecture dependent Python modules on readthedocs, @@ -40,8 +39,9 @@ pip_installed_modules = set() dependency_helper = utils.dependencies.DependencyHelper( - dependencies_file=os.path.join('..', 'dependencies.ini'), - test_dependencies_file=os.path.join('..', 'test_dependencies.ini')) + dependencies_file=os.path.join("..", "dependencies.ini"), + test_dependencies_file=os.path.join("..", "test_dependencies.ini"), +) modules_to_mock = set(dependency_helper.dependencies.keys()) modules_to_mock = modules_to_mock.difference(pip_installed_modules) @@ -57,39 +57,38 @@ # General information about the project. # pylint: disable=redefined-builtin -project = '${name_description}' -copyright = 'The ${name_description} authors' +project = "${name_description}" +copyright = "The ${name_description} authors" version = ${python_module_name}.__version__ release = ${python_module_name}.__version__ # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ['_build'] +exclude_patterns = ["_build"] # The master toctree document. -master_doc = 'index' +master_doc = "index" # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # -- Options for HTML output ---------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'sphinx_rtd_theme' +html_theme = "sphinx_rtd_theme" # Output file base name for HTML help builder. -htmlhelp_basename = '${htmlhelp_basename}doc' +htmlhelp_basename = "${htmlhelp_basename}doc" # -- Options linkcheck ---------------------------------------------------- -linkcheck_ignore = [ -] +linkcheck_ignore = [] # -- Code to rewrite links for readthedocs -------------------------------- @@ -97,74 +96,74 @@ # This function is a Sphinx core event callback, the format of which is detailed # here: https://www.sphinx-doc.org/en/master/extdev/appapi.html#events + # pylint: disable=unused-argument def RunSphinxAPIDoc(app): - """Runs sphinx-apidoc to auto-generate documentation. + """Runs sphinx-apidoc to auto-generate documentation. - Args: - app (sphinx.application.Sphinx): Sphinx application. Required by the - the Sphinx event callback API. - """ - current_directory = os.path.abspath(os.path.dirname(__file__)) - module_path = os.path.join(current_directory, '..', '${python_module_name}') - api_directory = os.path.join(current_directory, 'sources', 'api') - apidoc.main(['-o', api_directory, module_path, '--force']) + Args: + app (sphinx.application.Sphinx): Sphinx application. Required by the + the Sphinx event callback API. + """ + current_directory = os.path.abspath(os.path.dirname(__file__)) + module_path = os.path.join(current_directory, "..", "${python_module_name}") + api_directory = os.path.join(current_directory, "sources", "api") + apidoc.main(["-o", api_directory, module_path, "--force"]) class MarkdownLinkFixer(transforms.Transform): - """Transform definition to parse .md references to internal pages.""" + """Transform definition to parse .md references to internal pages.""" - default_priority = 1000 + default_priority = 1000 - _URI_PREFIXES = [] + _URI_PREFIXES = [] - def _FixLinks(self, node): - """Corrects links to .md files not part of the documentation. + def _FixLinks(self, node): + """Corrects links to .md files not part of the documentation. - Args: - node (docutils.nodes.Node): docutils node. + Args: + node (docutils.nodes.Node): docutils node. - Returns: - docutils.nodes.Node: docutils node, with correct URIs outside - of Markdown pages outside the documentation. - """ - if isinstance(node, nodes.reference) and 'refuri' in node: - reference_uri = node['refuri'] - for uri_prefix in self._URI_PREFIXES: - if (reference_uri.startswith(uri_prefix) and not ( - reference_uri.endswith('.asciidoc') or - reference_uri.endswith('.md'))): - node['refuri'] = reference_uri + '.md' - break + Returns: + docutils.nodes.Node: docutils node, with correct URIs outside + of Markdown pages outside the documentation. + """ + if isinstance(node, nodes.reference) and "refuri" in node: + reference_uri = node["refuri"] + for uri_prefix in self._URI_PREFIXES: + if reference_uri.startswith(uri_prefix) and not ( + reference_uri.endswith(".asciidoc") or reference_uri.endswith(".md") + ): + node["refuri"] = reference_uri + ".md" + break - return node + return node - def _Traverse(self, node): - """Traverses the document tree rooted at node. + def _Traverse(self, node): + """Traverses the document tree rooted at node. - Args: - node (docutils.nodes.Node): docutils node. - """ - self._FixLinks(node) + Args: + node (docutils.nodes.Node): docutils node. + """ + self._FixLinks(node) - for child_node in node.children: - self._Traverse(child_node) + for child_node in node.children: + self._Traverse(child_node) - # pylint: disable=arguments-differ - def apply(self): - """Applies this transform on document tree.""" - self._Traverse(self.document) + # pylint: disable=arguments-differ + def apply(self): + """Applies this transform on document tree.""" + self._Traverse(self.document) # pylint: invalid-name def setup(app): - """Called at Sphinx initialization. - - Args: - app (sphinx.application.Sphinx): Sphinx application. - """ - # Triggers sphinx-apidoc to generate API documentation. - app.connect('builder-inited', RunSphinxAPIDoc) - app.add_config_value( - 'recommonmark_config', {'enable_auto_toc_tree': True}, True) - app.add_transform(MarkdownLinkFixer) + """Called at Sphinx initialization. + + Args: + app (sphinx.application.Sphinx): Sphinx application. + """ + # Triggers sphinx-apidoc to generate API documentation. + app.connect("builder-inited", RunSphinxAPIDoc) + app.add_config_value("recommonmark_config", {"enable_auto_toc_tree": True}, True) + app.add_transform(MarkdownLinkFixer) diff --git a/data/templates/github_actions/lint.yml/header.yml b/data/templates/github_actions/lint.yml/header.yml new file mode 100644 index 00000000..3d9790ff --- /dev/null +++ b/data/templates/github_actions/lint.yml/header.yml @@ -0,0 +1,51 @@ +# Check source. +name: lint +on: + pull_request: + branches: + - main + push: + branches: + - main +permissions: read-all +jobs: + black: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - name: Check format of Python code + uses: psf/black@stable + with: + options: "--check" + src: "." + pylint: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.14'] + container: + image: ubuntu:26.04 + steps: + - uses: actions/checkout@v6 + - name: Set up container + env: + DEBIAN_FRONTEND: noninteractive + run: | + apt-get update -q + apt-get install -y libterm-readline-gnu-perl locales software-properties-common + locale-gen en_US.UTF-8 + ln -f -s /usr/share/zoneinfo/UTC /etc/localtime + - name: Install dependencies + env: + DEBIAN_FRONTEND: noninteractive + run: | + add-apt-repository -y universe + add-apt-repository -y ppa:deadsnakes/ppa + add-apt-repository -y ppa:gift/dev + apt-get update -q + apt-get install -y build-essential git ${dpkg_dev_dependencies} python$${{ matrix.python-version }} python$${{ matrix.python-version }}-dev python$${{ matrix.python-version }}-venv ${dpkg_dependencies} + - name: Run linter + env: + LANG: en_US.UTF-8 + run: | + tox -e pylint diff --git a/data/templates/github_actions/lint.yml/yamllint.yml b/data/templates/github_actions/lint.yml/yamllint.yml new file mode 100644 index 00000000..c211b8c0 --- /dev/null +++ b/data/templates/github_actions/lint.yml/yamllint.yml @@ -0,0 +1,17 @@ + yamllint: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.12'] + steps: + - uses: actions/checkout@v6 + - name: Set up Python $${{ matrix.python-version }} + uses: actions/setup-python@v6 + with: + python-version: $${{ matrix.python-version }} + - name: Install tox + run: | + python -m pip install tox + - name: Run linter + run: | + tox -e yamllint diff --git a/data/templates/github_actions/test_tox.yml b/data/templates/github_actions/test_tox.yml index cd01a318..a499104d 100644 --- a/data/templates/github_actions/test_tox.yml +++ b/data/templates/github_actions/test_tox.yml @@ -85,34 +85,3 @@ jobs: uses: codecov/codecov-action@v6 with: token: $${{ secrets.CODECOV_TOKEN }} - lint: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ['3.14'] - container: - image: ubuntu:26.04 - steps: - - uses: actions/checkout@v6 - - name: Set up container - env: - DEBIAN_FRONTEND: noninteractive - run: | - apt-get update -q - apt-get install -y libterm-readline-gnu-perl locales software-properties-common - locale-gen en_US.UTF-8 - ln -f -s /usr/share/zoneinfo/UTC /etc/localtime - - name: Install dependencies - env: - DEBIAN_FRONTEND: noninteractive - run: | - add-apt-repository -y universe - add-apt-repository -y ppa:deadsnakes/ppa - add-apt-repository -y ppa:gift/dev - apt-get update -q - apt-get install -y build-essential git ${dpkg_dev_dependencies} python$${{ matrix.python-version }} python$${{ matrix.python-version }}-dev python$${{ matrix.python-version }}-venv ${dpkg_dependencies} - - name: Run linter - env: - LANG: en_US.UTF-8 - run: | - tox -e lint diff --git a/data/templates/pyproject.toml/black.toml b/data/templates/pyproject.toml/black.toml new file mode 100644 index 00000000..b05f6de8 --- /dev/null +++ b/data/templates/pyproject.toml/black.toml @@ -0,0 +1,5 @@ + +[tool.black] +line-length = 88 +target-version = ["py310"] +include = "\\.pyi?$$" diff --git a/data/templates/tox.ini/testenv_black b/data/templates/tox.ini/testenv_black new file mode 100644 index 00000000..7792ffa7 --- /dev/null +++ b/data/templates/tox.ini/testenv_black @@ -0,0 +1,16 @@ + +[testenv:black] +skipsdist = True +pip_pre = True +passenv = + CFLAGS + CPPFLAGS + LDFLAGS +setenv = + PYTHONPATH = {toxinidir} +deps = + black + setuptools >= 65 +commands = + black --version + black --check . diff --git a/data/templates/tox.ini/testenv_lint b/data/templates/tox.ini/testenv_pylint similarity index 93% rename from data/templates/tox.ini/testenv_lint rename to data/templates/tox.ini/testenv_pylint index 11fac4af..45bb9203 100644 --- a/data/templates/tox.ini/testenv_lint +++ b/data/templates/tox.ini/testenv_pylint @@ -1,5 +1,5 @@ -[testenv:lint] +[testenv:pylint] skipsdist = True pip_pre = True passenv = diff --git a/data/templates/tox.ini/testenv_lint-with_yaml b/data/templates/tox.ini/testenv_yamllint similarity index 67% rename from data/templates/tox.ini/testenv_lint-with_yaml rename to data/templates/tox.ini/testenv_yamllint index efcd264c..03225b49 100644 --- a/data/templates/tox.ini/testenv_lint-with_yaml +++ b/data/templates/tox.ini/testenv_yamllint @@ -1,5 +1,5 @@ -[testenv:lint] +[testenv:yamllint] skipsdist = True pip_pre = True passenv = @@ -9,11 +9,8 @@ passenv = setenv = PYTHONPATH = {toxinidir} deps = - pylint >= 3.3.0, < 3.4.0 setuptools >= 65 yamllint >= 1.26.0 commands = - pylint --version yamllint -v - pylint --rcfile=.pylintrc ${paths_to_lint_python} yamllint -c .yamllint.yaml ${paths_to_lint_yaml} diff --git a/l2tdevtools/dependencies.py b/l2tdevtools/dependencies.py index 405009ab..6c9732a7 100644 --- a/l2tdevtools/dependencies.py +++ b/l2tdevtools/dependencies.py @@ -6,620 +6,651 @@ class DependencyDefinition: - """Dependency definition. - - Attributes: - dpkg_name (str): name of the dpkg package that provides the dependency. - is_optional (bool): True if the dependency is optional. - l2tbinaries_name (str): name of the l2tbinaries package that provides - the dependency. - maximum_version (str): maximum supported version, a greater or equal - version is not supported. - minimum_version (str): minimum supported version, a lesser version is - not supported. - name (str): name of (the Python module that provides) the dependency. - pypi_name (str): name of the PyPI package that provides the dependency. - python2_only (bool): True if the dependency is only supported by Python 2. - python3_only (bool): True if the dependency is only supported by Python 3. - rpm_name (str): name of the rpm package that provides the dependency. - skip_check (bool): True if the dependency should be skipped by the - CheckDependencies or CheckTestDependencies methods of DependencyHelper. - skip_requires (bool): True if the dependency should be excluded from - pyproject.toml dependencies. - version_property (str): name of the version attribute or function. - """ - - def __init__(self, name): - """Initializes a dependency configuration. - - Args: - name (str): name of the dependency. + """Dependency definition. + + Attributes: + dpkg_name (str): name of the dpkg package that provides the dependency. + is_optional (bool): True if the dependency is optional. + l2tbinaries_name (str): name of the l2tbinaries package that provides + the dependency. + maximum_version (str): maximum supported version, a greater or equal + version is not supported. + minimum_version (str): minimum supported version, a lesser version is + not supported. + name (str): name of (the Python module that provides) the dependency. + pypi_name (str): name of the PyPI package that provides the dependency. + python2_only (bool): True if the dependency is only supported by Python 2. + python3_only (bool): True if the dependency is only supported by Python 3. + rpm_name (str): name of the rpm package that provides the dependency. + skip_check (bool): True if the dependency should be skipped by the + CheckDependencies or CheckTestDependencies methods of DependencyHelper. + skip_requires (bool): True if the dependency should be excluded from + pyproject.toml dependencies. + version_property (str): name of the version attribute or function. """ - super().__init__() - self.dpkg_name = None - self.is_optional = False - self.l2tbinaries_name = None - self.maximum_version = None - self.minimum_version = None - self.name = name - self.pypi_name = None - self.python2_only = False - self.python3_only = False - self.rpm_name = None - self.skip_check = None - self.skip_requires = None - self.version_property = None + def __init__(self, name): + """Initializes a dependency configuration. + + Args: + name (str): name of the dependency. + """ + super().__init__() + self.dpkg_name = None + self.is_optional = False + self.l2tbinaries_name = None + self.maximum_version = None + self.minimum_version = None + self.name = name + self.pypi_name = None + self.python2_only = False + self.python3_only = False + self.rpm_name = None + self.skip_check = None + self.skip_requires = None + self.version_property = None -class DependencyDefinitionReader: - """Dependency definition reader.""" - - _VALUE_NAMES = frozenset([ - 'dpkg_name', - 'is_optional', - 'l2tbinaries_name', - 'maximum_version', - 'minimum_version', - 'pypi_name', - 'python2_only', - 'python3_only', - 'rpm_name', - 'skip_check', - 'skip_requires', - 'version_property']) - - def _GetConfigValue(self, config_parser, section_name, value_name): - """Retrieves a value from the config parser. - - Args: - config_parser (ConfigParser): configuration parser. - section_name (str): name of the section that contains the value. - value_name (str): name of the value. - - Returns: - object: configuration value or None if the value does not exists. - """ - try: - return config_parser.get(section_name, value_name) - except configparser.NoOptionError: - return None - - def Read(self, file_object): - """Reads dependency definitions. - - Args: - file_object (file): file-like object to read from. - - Yields: - DependencyDefinition: dependency definition. - """ - config_parser = configparser.ConfigParser(interpolation=None) - config_parser.read_file(file_object) - for section_name in config_parser.sections(): - dependency_definition = DependencyDefinition(section_name) - for value_name in self._VALUE_NAMES: - value = self._GetConfigValue(config_parser, section_name, value_name) - setattr(dependency_definition, value_name, value) - - yield dependency_definition +class DependencyDefinitionReader: + """Dependency definition reader.""" + + _VALUE_NAMES = frozenset( + [ + "dpkg_name", + "is_optional", + "l2tbinaries_name", + "maximum_version", + "minimum_version", + "pypi_name", + "python2_only", + "python3_only", + "rpm_name", + "skip_check", + "skip_requires", + "version_property", + ] + ) + + def _GetConfigValue(self, config_parser, section_name, value_name): + """Retrieves a value from the config parser. + + Args: + config_parser (ConfigParser): configuration parser. + section_name (str): name of the section that contains the value. + value_name (str): name of the value. + + Returns: + object: configuration value or None if the value does not exists. + """ + try: + return config_parser.get(section_name, value_name) + except configparser.NoOptionError: + return None + + def Read(self, file_object): + """Reads dependency definitions. + + Args: + file_object (file): file-like object to read from. + + Yields: + DependencyDefinition: dependency definition. + """ + config_parser = configparser.ConfigParser(interpolation=None) + config_parser.read_file(file_object) + + for section_name in config_parser.sections(): + dependency_definition = DependencyDefinition(section_name) + for value_name in self._VALUE_NAMES: + value = self._GetConfigValue(config_parser, section_name, value_name) + setattr(dependency_definition, value_name, value) + + yield dependency_definition class DependencyHelper: - """Dependency helper. - - Attributes: - dependencies (dict[str, DependencyDefinition]): dependencies. - """ - - _VERSION_NUMBERS_REGEX = re.compile(r'[0-9.]+') - _VERSION_SPLIT_REGEX = re.compile(r'\.|\-') - - def __init__( - self, dependencies_file='dependencies.ini', - test_dependencies_file='test_dependencies.ini'): - """Initializes a dependency helper. - - Args: - dependencies_file (Optional[str]): path to the dependencies configuration - file. - test_dependencies_file (Optional[str]): path to the test dependencies - configuration file. - """ - super().__init__() - self._test_dependencies = {} - self.dependencies = {} - - dependency_reader = DependencyDefinitionReader() - - with open(dependencies_file, 'r', encoding='utf-8') as file_object: - for dependency in dependency_reader.Read(file_object): - self.dependencies[dependency.name] = dependency - - if os.path.exists(test_dependencies_file): - with open(test_dependencies_file, 'r', encoding='utf-8') as file_object: - for dependency in dependency_reader.Read(file_object): - self._test_dependencies[dependency.name] = dependency - - def _CheckPythonModule(self, dependency): - """Checks the availability of a Python module. - - Args: - dependency (DependencyDefinition): dependency definition. - - Returns: - tuple: containing: - - bool: True if the Python module is available and conforms to - the minimum required version, False otherwise. - str: status message. - """ - module_object = self._ImportPythonModule(dependency.name) - if not module_object: - return False, f'missing: {dependency.name:s}' - - if not dependency.version_property: - return True, dependency.name - - return self._CheckPythonModuleVersion( - dependency.name, module_object, dependency.version_property, - dependency.minimum_version, dependency.maximum_version) - - def _CheckPythonModuleVersion( - self, module_name, module_object, version_property, minimum_version, - maximum_version): - """Checks the version of a Python module. - - Args: - module_object (module): Python module. - module_name (str): name of the Python module. - version_property (str): version attribute or function. - minimum_version (str): minimum version. - maximum_version (str): maximum version. - - Returns: - tuple: containing: - - bool: True if the Python module is available and conforms to - the minimum required version, False otherwise. - str: status message. - """ - module_version = None - if not version_property.endswith('()'): - module_version = getattr(module_object, version_property, None) - else: - version_method = getattr( - module_object, version_property[:-2], None) - if version_method: - module_version = version_method() - - if not module_version: - return False, ( - f'unable to determine version information for: {module_name:s}') - - # Make sure the module version is a string. - module_version = f'{module_version!s}' - - # Split the version string and convert every digit into an integer. - # A string compare of both version strings will yield an incorrect result. - - # Strip any semantic suffixes such as a1, b1, pre, post, rc, dev. - module_version = self._VERSION_NUMBERS_REGEX.findall(module_version)[0] - - if module_version[-1] == '.': - module_version = module_version[:-1] - - try: - module_version_map = list( - map(int, self._VERSION_SPLIT_REGEX.split(module_version))) - except ValueError: - return False, ( - f'unable to parse module version: {module_name:s} {module_version:s}') - - if minimum_version: - try: - minimum_version_map = list( - map(int, self._VERSION_SPLIT_REGEX.split(minimum_version))) - except ValueError: - return False, ( - f'unable to parse minimum version: {module_name:s} ' - f'{minimum_version:s}') - - if module_version_map < minimum_version_map: - return False, ( - f'{module_name:s} version: {module_version!s} is too old, ' - f'{minimum_version!s} or later required') - - if maximum_version: - try: - maximum_version_map = list( - map(int, self._VERSION_SPLIT_REGEX.split(maximum_version))) - except ValueError: - return False, ( - f'unable to parse maximum version: {module_name:s} ' - f'{maximum_version:s}') - - if module_version_map > maximum_version_map: - return False, ( - f'{module_name:s} version: {module_version!s} is too recent, ' - f'{maximum_version!s} or earlier required') - - return True, f'{module_name:s} version: {module_version!s}' - - def _ImportPythonModule(self, module_name): - """Imports a Python module. - - Args: - module_name (str): name of the module. - - Returns: - module: Python module or None if the module cannot be imported. - """ - try: - module_object = list(map(__import__, [module_name]))[0] - except ImportError: - return None - - # If the module name contains dots get the upper most module object. - if '.' in module_name: - for submodule_name in module_name.split('.')[1:]: - module_object = getattr(module_object, submodule_name, None) - - return module_object - - def _PrintCheckDependencyStatus( - self, dependency, result, status_message, verbose_output=True): - """Prints the check dependency status. - - Args: - dependency (DependencyDefinition): dependency definition. - result (bool): True if the Python module is available and conforms to - the minimum required version, False otherwise. - status_message (str): status message. - verbose_output (Optional[bool]): True if output should be verbose. - """ - if not result or dependency.is_optional: - if dependency.is_optional: - status_indicator = '[OPTIONAL]' - else: - status_indicator = '[FAILURE]' - - print(f'{status_indicator:s}\t{status_message:s}') - - elif verbose_output: - print(f'[OK]\t\t{status_message:s}') - - def CheckDependencies(self, verbose_output=True): - """Checks the availability of the dependencies. - - Args: - verbose_output (Optional[bool]): True if output should be verbose. - - Returns: - bool: True if the dependencies are available, False otherwise. - """ - print('Checking availability and versions of dependencies.') - check_result = True - - for _, dependency in sorted(self.dependencies.items()): - if dependency.skip_check: - continue - - result, status_message = self._CheckPythonModule(dependency) - - if not result and not dependency.is_optional: - check_result = False - - self._PrintCheckDependencyStatus( - dependency, result, status_message, verbose_output=verbose_output) - - if check_result and not verbose_output: - print('[OK]') - - print('') - return check_result - - def CheckTestDependencies(self, verbose_output=True): - """Checks the availability of the dependencies when running tests. - - Args: - verbose_output (Optional[bool]): True if output should be verbose. - - Returns: - bool: True if the dependencies are available, False otherwise. - """ - if not self.CheckDependencies(verbose_output=verbose_output): - return False - - print('Checking availability and versions of test dependencies.') - check_result = True - - for dependency in sorted( - self._test_dependencies.values(), - key=lambda dependency: dependency.name): - if dependency.skip_check: - continue - - result, status_message = self._CheckPythonModule(dependency) - - if not result and not dependency.is_optional: - check_result = False - - self._PrintCheckDependencyStatus( - dependency, result, status_message, verbose_output=verbose_output) - - if check_result and not verbose_output: - print('[OK]') - - print('') - return check_result + """Dependency helper. - # The following functions should not be included in utils/dependencies.py - - def _GetDPKGDepends(self, dependencies, exclude_version=False): - """Retrieves the DPKG control file installation requirements. - - Args: - dependencies (dict[str, DependencyDefinition]): dependencies. - exclude_version (Optional[bool]): True if the version should be excluded - from the dependency definitions. - - Returns: - list[str]: dependency definitions for requires for DPKG control file. - """ - requires = [] - for dependency in sorted( - dependencies.values(), key=lambda dependency: dependency.name): - module_name = dependency.dpkg_name or dependency.name - if 'python2' in module_name: - module_name = module_name.replace('python2', 'python3') - elif 'python3' not in module_name: - module_name = module_name.replace('python', 'python3') - - if exclude_version or not dependency.minimum_version: - requires_string = module_name - else: - requires_string = f'{module_name:s} (>= {dependency.minimum_version:s})' - - requires.append(requires_string) - - return sorted(requires) - - def _GetInstallRequires(self, dependencies, exclude_version=False): - """Retrieves the setup.py installation requirements. - - Args: - dependencies (dict[str, DependencyDefinition]): dependencies. - exclude_version (Optional[bool]): True if the version should be excluded - from the dependency definitions. - - Returns: - list[str]: dependency definitions for install_requires in setup.py. - """ - install_requires = [] - for dependency in sorted( - dependencies.values(), key=lambda dependency: dependency.name): - if dependency.skip_requires: - continue - - module_name = dependency.pypi_name or dependency.name - - # Use the sqlite3 module provided by the standard library. - if module_name == 'pysqlite': - continue - - requires_part = [] - if not exclude_version: - if dependency.minimum_version: - requires_part.append(f'>= {dependency.minimum_version!s}') - if dependency.maximum_version: - requires_part.append(f'< {dependency.maximum_version!s}') - - requires_string = module_name - if requires_part: - requires_string = ' '.join([requires_string, ','.join(requires_part)]) - - if module_name in ('pyxattr', 'xattr'): - requires_string = ( - f'{requires_string:s} ; platform_system != \\"Windows\\"') - - install_requires.append(requires_string) - - return sorted(install_requires) - - def _GetL2TBinaries(self, dependencies): - """Retrieves the l2tbinaries requirements. - - Args: - dependencies (dict[str, DependencyDefinition]): dependencies. - - Returns: - list[str]: dependency definitions for l2tbinaries. - """ - requires = [] - for dependency in sorted( - dependencies.values(), key=lambda dependency: dependency.name): - if dependency.l2tbinaries_name: - module_name = dependency.l2tbinaries_name - else: - module_name = dependency.name - - requires.append(module_name) - - return sorted(requires) - - def _GetRPMRequires(self, dependencies, exclude_version=False): - """Retrieves the setup.cfg RPM installation requirements. - - Args: + Attributes: dependencies (dict[str, DependencyDefinition]): dependencies. - exclude_version (Optional[bool]): True if the version should be excluded - from the dependency definitions. - - Returns: - list[str]: dependency definitions for requires for setup.cfg. - """ - requires = [] - for dependency in sorted( - dependencies.values(), key=lambda dependency: dependency.name): - module_name = dependency.rpm_name or dependency.name - if module_name.startswith('python-'): - module_name = module_name.replace('python-', 'python3-') - else: - module_name = module_name.replace('python2-', 'python3-') - if module_name.endswith('-python'): - module_name = module_name.replace('-python', '-python3') - else: - module_name = module_name.replace('-python2', '-python3') - - if exclude_version or not dependency.minimum_version: - requires_string = module_name - else: - requires_string = f'{module_name:s} >= {dependency.minimum_version:s}' - - requires.append(requires_string) - - return sorted(requires) - - def GetDPKGDepends(self, exclude_version=False, test_dependencies=False): - """Retrieves the DPKG control file installation requirements. - - Args: - exclude_version (Optional[bool]): True if the version should be excluded - from the dependency definitions. - test_dependencies (Optional[bool]): True if the test dependencies should - returned instead of the regular dependencies. - - Returns: - list[str]: dependency definitions for requires for DPKG control file. - """ - if test_dependencies: - dependencies = self._test_dependencies - else: - dependencies = self.dependencies - - return self._GetDPKGDepends(dependencies, exclude_version=exclude_version) - - def GetL2TBinaries(self, test_dependencies=False): - """Retrieves the l2tbinaries requirements. - - Args: - test_dependencies (Optional[bool]): True if the test dependencies should - returned instead of the regular dependencies. - - Returns: - list[str]: dependency definitions for l2tbinaries. - """ - if test_dependencies: - dependencies = self._test_dependencies - else: - dependencies = self.dependencies - - return self._GetL2TBinaries(dependencies) - - def GetInstallRequires(self, exclude_version=False, test_dependencies=False): - """Retrieves the setup.py installation requirements. - - Args: - exclude_version (Optional[bool]): True if the version should be excluded - from the dependency definitions. - test_dependencies (Optional[bool]): True if the test dependencies should - returned instead of the regular dependencies. - - Returns: - list[str]: dependency definitions for install_requires in setup.py. - """ - if test_dependencies: - dependencies = self._test_dependencies - else: - dependencies = self.dependencies - - return self._GetInstallRequires( - dependencies, exclude_version=exclude_version) - - def GetPylintRcExtensionPkgs(self): - """Retrieves the .pylintrc extension packages. - - Returns: - list[str]: name of packages for extension-pkg-whitelist in .pylintrc. - """ - # Do not move to a class constant due to how DependencyHelper is used - # to generate utils.DependencyHelper of the individual projects. - names = ( - 'pybde', - 'pycaes', - 'pycreg', - 'pyesedb', - 'pyevt', - 'pyevtx', - 'pyewf', - 'pyexe', - 'pyfcrypto', - 'pyfmos', - 'pyfsapfs', - 'pyfsext', - 'pyfsfat', - 'pygzipf', - 'pyfshfs', - 'pyfsntfs', - 'pyfsxfs', - 'pyfvde', - 'pyfwevt', - 'pyfwnt', - 'pyfwps', - 'pyfwsi', - 'pyhmac', - 'pylnk', - 'pyluksde', - 'pymodi', - 'pymsiecf', - 'pyolecf', - 'pyphdi', - 'pyqcow', - 'pyregf', - 'pyscca', - 'pysigscan', - 'pysmdev', - 'pysmraw', - 'pytsk3', - 'pyvhdi', - 'pyvmdk', - 'pyvshadow', - 'pyvsapm', - 'pyvsbsdl', - 'pyvsgpt', - 'pyvslvm', - 'pyvsmbr', - 'pywrc', - 'xattr', - 'yara', - 'zstd') - - extension_packages = [] - for dependency in sorted( - self.dependencies.values(), key=lambda dependency: dependency.name): - if dependency.name not in names: - continue - - extension_packages.append(dependency.name) - - return sorted(extension_packages) - - def GetRPMRequires(self, exclude_version=False, test_dependencies=False): - """Retrieves the setup.cfg RPM installation requirements. - - Args: - exclude_version (Optional[bool]): True if the version should be excluded - from the dependency definitions. - test_dependencies (Optional[bool]): True if the test dependencies should - returned instead of the regular dependencies. - - Returns: - list[str]: dependency definitions for requires for setup.cfg. """ - if test_dependencies: - dependencies = self._test_dependencies - else: - dependencies = self.dependencies - return self._GetRPMRequires(dependencies, exclude_version=exclude_version) + _VERSION_NUMBERS_REGEX = re.compile(r"[0-9.]+") + _VERSION_SPLIT_REGEX = re.compile(r"\.|\-") + + def __init__( + self, + dependencies_file="dependencies.ini", + test_dependencies_file="test_dependencies.ini", + ): + """Initializes a dependency helper. + + Args: + dependencies_file (Optional[str]): path to the dependencies configuration + file. + test_dependencies_file (Optional[str]): path to the test dependencies + configuration file. + """ + super().__init__() + self._test_dependencies = {} + self.dependencies = {} + + dependency_reader = DependencyDefinitionReader() + + with open(dependencies_file, "r", encoding="utf-8") as file_object: + for dependency in dependency_reader.Read(file_object): + self.dependencies[dependency.name] = dependency + + if os.path.exists(test_dependencies_file): + with open(test_dependencies_file, "r", encoding="utf-8") as file_object: + for dependency in dependency_reader.Read(file_object): + self._test_dependencies[dependency.name] = dependency + + def _CheckPythonModule(self, dependency): + """Checks the availability of a Python module. + + Args: + dependency (DependencyDefinition): dependency definition. + + Returns: + tuple: containing: + + bool: True if the Python module is available and conforms to + the minimum required version, False otherwise. + str: status message. + """ + module_object = self._ImportPythonModule(dependency.name) + if not module_object: + return False, f"missing: {dependency.name:s}" + + if not dependency.version_property: + return True, dependency.name + + return self._CheckPythonModuleVersion( + dependency.name, + module_object, + dependency.version_property, + dependency.minimum_version, + dependency.maximum_version, + ) + + def _CheckPythonModuleVersion( + self, + module_name, + module_object, + version_property, + minimum_version, + maximum_version, + ): + """Checks the version of a Python module. + + Args: + module_object (module): Python module. + module_name (str): name of the Python module. + version_property (str): version attribute or function. + minimum_version (str): minimum version. + maximum_version (str): maximum version. + + Returns: + tuple: containing: + + bool: True if the Python module is available and conforms to + the minimum required version, False otherwise. + str: status message. + """ + module_version = None + if not version_property.endswith("()"): + module_version = getattr(module_object, version_property, None) + else: + version_method = getattr(module_object, version_property[:-2], None) + if version_method: + module_version = version_method() + + if not module_version: + return False, ( + f"unable to determine version information for: {module_name:s}" + ) + + # Make sure the module version is a string. + module_version = f"{module_version!s}" + + # Split the version string and convert every digit into an integer. + # A string compare of both version strings will yield an incorrect result. + + # Strip any semantic suffixes such as a1, b1, pre, post, rc, dev. + module_version = self._VERSION_NUMBERS_REGEX.findall(module_version)[0] + + if module_version[-1] == ".": + module_version = module_version[:-1] + + try: + module_version_map = list( + map(int, self._VERSION_SPLIT_REGEX.split(module_version)) + ) + except ValueError: + return False, ( + f"unable to parse module version: {module_name:s} {module_version:s}" + ) + + if minimum_version: + try: + minimum_version_map = list( + map(int, self._VERSION_SPLIT_REGEX.split(minimum_version)) + ) + except ValueError: + return False, ( + f"unable to parse minimum version: {module_name:s} " + f"{minimum_version:s}" + ) + + if module_version_map < minimum_version_map: + return False, ( + f"{module_name:s} version: {module_version!s} is too old, " + f"{minimum_version!s} or later required" + ) + + if maximum_version: + try: + maximum_version_map = list( + map(int, self._VERSION_SPLIT_REGEX.split(maximum_version)) + ) + except ValueError: + return False, ( + f"unable to parse maximum version: {module_name:s} " + f"{maximum_version:s}" + ) + + if module_version_map > maximum_version_map: + return False, ( + f"{module_name:s} version: {module_version!s} is too recent, " + f"{maximum_version!s} or earlier required" + ) + + return True, f"{module_name:s} version: {module_version!s}" + + def _ImportPythonModule(self, module_name): + """Imports a Python module. + + Args: + module_name (str): name of the module. + + Returns: + module: Python module or None if the module cannot be imported. + """ + try: + module_object = list(map(__import__, [module_name]))[0] + except ImportError: + return None + + # If the module name contains dots get the upper most module object. + if "." in module_name: + for submodule_name in module_name.split(".")[1:]: + module_object = getattr(module_object, submodule_name, None) + + return module_object + + def _PrintCheckDependencyStatus( + self, dependency, result, status_message, verbose_output=True + ): + """Prints the check dependency status. + + Args: + dependency (DependencyDefinition): dependency definition. + result (bool): True if the Python module is available and conforms to + the minimum required version, False otherwise. + status_message (str): status message. + verbose_output (Optional[bool]): True if output should be verbose. + """ + if not result or dependency.is_optional: + if dependency.is_optional: + status_indicator = "[OPTIONAL]" + else: + status_indicator = "[FAILURE]" + + print(f"{status_indicator:s}\t{status_message:s}") + + elif verbose_output: + print(f"[OK]\t\t{status_message:s}") + + def CheckDependencies(self, verbose_output=True): + """Checks the availability of the dependencies. + + Args: + verbose_output (Optional[bool]): True if output should be verbose. + + Returns: + bool: True if the dependencies are available, False otherwise. + """ + print("Checking availability and versions of dependencies.") + check_result = True + + for _, dependency in sorted(self.dependencies.items()): + if dependency.skip_check: + continue + + result, status_message = self._CheckPythonModule(dependency) + + if not result and not dependency.is_optional: + check_result = False + + self._PrintCheckDependencyStatus( + dependency, result, status_message, verbose_output=verbose_output + ) + + if check_result and not verbose_output: + print("[OK]") + + print("") + return check_result + + def CheckTestDependencies(self, verbose_output=True): + """Checks the availability of the dependencies when running tests. + + Args: + verbose_output (Optional[bool]): True if output should be verbose. + + Returns: + bool: True if the dependencies are available, False otherwise. + """ + if not self.CheckDependencies(verbose_output=verbose_output): + return False + + print("Checking availability and versions of test dependencies.") + check_result = True + + for dependency in sorted( + self._test_dependencies.values(), key=lambda dependency: dependency.name + ): + if dependency.skip_check: + continue + + result, status_message = self._CheckPythonModule(dependency) + + if not result and not dependency.is_optional: + check_result = False + + self._PrintCheckDependencyStatus( + dependency, result, status_message, verbose_output=verbose_output + ) + + if check_result and not verbose_output: + print("[OK]") + + print("") + return check_result + + # The following functions should not be included in utils/dependencies.py + + def _GetDPKGDepends(self, dependencies, exclude_version=False): + """Retrieves the DPKG control file installation requirements. + + Args: + dependencies (dict[str, DependencyDefinition]): dependencies. + exclude_version (Optional[bool]): True if the version should be excluded + from the dependency definitions. + + Returns: + list[str]: dependency definitions for requires for DPKG control file. + """ + requires = [] + for dependency in sorted( + dependencies.values(), key=lambda dependency: dependency.name + ): + module_name = dependency.dpkg_name or dependency.name + if "python2" in module_name: + module_name = module_name.replace("python2", "python3") + elif "python3" not in module_name: + module_name = module_name.replace("python", "python3") + + if exclude_version or not dependency.minimum_version: + requires_string = module_name + else: + requires_string = f"{module_name:s} (>= {dependency.minimum_version:s})" + + requires.append(requires_string) + + return sorted(requires) + + def _GetInstallRequires(self, dependencies, exclude_version=False): + """Retrieves the setup.py installation requirements. + + Args: + dependencies (dict[str, DependencyDefinition]): dependencies. + exclude_version (Optional[bool]): True if the version should be excluded + from the dependency definitions. + + Returns: + list[str]: dependency definitions for install_requires in setup.py. + """ + install_requires = [] + for dependency in sorted( + dependencies.values(), key=lambda dependency: dependency.name + ): + if dependency.skip_requires: + continue + + module_name = dependency.pypi_name or dependency.name + + # Use the sqlite3 module provided by the standard library. + if module_name == "pysqlite": + continue + + requires_part = [] + if not exclude_version: + if dependency.minimum_version: + requires_part.append(f">= {dependency.minimum_version!s}") + if dependency.maximum_version: + requires_part.append(f"< {dependency.maximum_version!s}") + + requires_string = module_name + if requires_part: + requires_string = " ".join([requires_string, ",".join(requires_part)]) + + if module_name in ("pyxattr", "xattr"): + requires_string = ( + f'{requires_string:s} ; platform_system != \\"Windows\\"' + ) + + install_requires.append(requires_string) + + return sorted(install_requires) + + def _GetL2TBinaries(self, dependencies): + """Retrieves the l2tbinaries requirements. + + Args: + dependencies (dict[str, DependencyDefinition]): dependencies. + + Returns: + list[str]: dependency definitions for l2tbinaries. + """ + requires = [] + for dependency in sorted( + dependencies.values(), key=lambda dependency: dependency.name + ): + if dependency.l2tbinaries_name: + module_name = dependency.l2tbinaries_name + else: + module_name = dependency.name + + requires.append(module_name) + + return sorted(requires) + + def _GetRPMRequires(self, dependencies, exclude_version=False): + """Retrieves the setup.cfg RPM installation requirements. + + Args: + dependencies (dict[str, DependencyDefinition]): dependencies. + exclude_version (Optional[bool]): True if the version should be excluded + from the dependency definitions. + + Returns: + list[str]: dependency definitions for requires for setup.cfg. + """ + requires = [] + for dependency in sorted( + dependencies.values(), key=lambda dependency: dependency.name + ): + module_name = dependency.rpm_name or dependency.name + if module_name.startswith("python-"): + module_name = module_name.replace("python-", "python3-") + else: + module_name = module_name.replace("python2-", "python3-") + if module_name.endswith("-python"): + module_name = module_name.replace("-python", "-python3") + else: + module_name = module_name.replace("-python2", "-python3") + + if exclude_version or not dependency.minimum_version: + requires_string = module_name + else: + requires_string = f"{module_name:s} >= {dependency.minimum_version:s}" + + requires.append(requires_string) + + return sorted(requires) + + def GetDPKGDepends(self, exclude_version=False, test_dependencies=False): + """Retrieves the DPKG control file installation requirements. + + Args: + exclude_version (Optional[bool]): True if the version should be excluded + from the dependency definitions. + test_dependencies (Optional[bool]): True if the test dependencies should + returned instead of the regular dependencies. + + Returns: + list[str]: dependency definitions for requires for DPKG control file. + """ + if test_dependencies: + dependencies = self._test_dependencies + else: + dependencies = self.dependencies + + return self._GetDPKGDepends(dependencies, exclude_version=exclude_version) + + def GetL2TBinaries(self, test_dependencies=False): + """Retrieves the l2tbinaries requirements. + + Args: + test_dependencies (Optional[bool]): True if the test dependencies should + returned instead of the regular dependencies. + + Returns: + list[str]: dependency definitions for l2tbinaries. + """ + if test_dependencies: + dependencies = self._test_dependencies + else: + dependencies = self.dependencies + + return self._GetL2TBinaries(dependencies) + + def GetInstallRequires(self, exclude_version=False, test_dependencies=False): + """Retrieves the setup.py installation requirements. + + Args: + exclude_version (Optional[bool]): True if the version should be excluded + from the dependency definitions. + test_dependencies (Optional[bool]): True if the test dependencies should + returned instead of the regular dependencies. + + Returns: + list[str]: dependency definitions for install_requires in setup.py. + """ + if test_dependencies: + dependencies = self._test_dependencies + else: + dependencies = self.dependencies + + return self._GetInstallRequires(dependencies, exclude_version=exclude_version) + + def GetPylintRcExtensionPkgs(self): + """Retrieves the .pylintrc extension packages. + + Returns: + list[str]: name of packages for extension-pkg-whitelist in .pylintrc. + """ + # Do not move to a class constant due to how DependencyHelper is used + # to generate utils.DependencyHelper of the individual projects. + names = ( + "pybde", + "pycaes", + "pycreg", + "pyesedb", + "pyevt", + "pyevtx", + "pyewf", + "pyexe", + "pyfcrypto", + "pyfmos", + "pyfsapfs", + "pyfsext", + "pyfsfat", + "pygzipf", + "pyfshfs", + "pyfsntfs", + "pyfsxfs", + "pyfvde", + "pyfwevt", + "pyfwnt", + "pyfwps", + "pyfwsi", + "pyhmac", + "pylnk", + "pyluksde", + "pymodi", + "pymsiecf", + "pyolecf", + "pyphdi", + "pyqcow", + "pyregf", + "pyscca", + "pysigscan", + "pysmdev", + "pysmraw", + "pytsk3", + "pyvhdi", + "pyvmdk", + "pyvshadow", + "pyvsapm", + "pyvsbsdl", + "pyvsgpt", + "pyvslvm", + "pyvsmbr", + "pywrc", + "xattr", + "yara", + "zstd", + ) + + extension_packages = [] + for dependency in sorted( + self.dependencies.values(), key=lambda dependency: dependency.name + ): + if dependency.name not in names: + continue + + extension_packages.append(dependency.name) + + return sorted(extension_packages) + + def GetRPMRequires(self, exclude_version=False, test_dependencies=False): + """Retrieves the setup.cfg RPM installation requirements. + + Args: + exclude_version (Optional[bool]): True if the version should be excluded + from the dependency definitions. + test_dependencies (Optional[bool]): True if the test dependencies should + returned instead of the regular dependencies. + + Returns: + list[str]: dependency definitions for requires for setup.cfg. + """ + if test_dependencies: + dependencies = self._test_dependencies + else: + dependencies = self.dependencies + + return self._GetRPMRequires(dependencies, exclude_version=exclude_version) diff --git a/l2tdevtools/dependency_writers/github_actions.py b/l2tdevtools/dependency_writers/github_actions.py index 440b2226..8a7878cf 100644 --- a/l2tdevtools/dependency_writers/github_actions.py +++ b/l2tdevtools/dependency_writers/github_actions.py @@ -1,10 +1,91 @@ """Writer for GitHub actions workflow files.""" +import glob import os from l2tdevtools.dependency_writers import interface +class GitHubActionsLintYmlWriter(interface.DependencyFileWriter): + """lint.yml GitHub actions workflow file writer.""" + + _TEMPLATE_DIRECTORY = os.path.join( + 'data', 'templates', 'github_actions', 'lint.yml') + + PATH = os.path.join('.github', 'workflows', 'lint.yml') + + def _GenerateFromTemplate(self, template_filename, template_mappings): + """Generates file context based on a template file. + + Args: + template_filename (str): path of the template file. + template_mappings (dict[str, str]): template mappings, where the key + maps to the name of a template variable. + + Returns: + str: output based on the template string. + + Raises: + RuntimeError: if the template cannot be formatted. + """ + template_filename = os.path.join( + self._l2tdevtools_path, self._TEMPLATE_DIRECTORY, template_filename) + return super()._GenerateFromTemplate( + template_filename, template_mappings) + + def Write(self): + """Writes a lint.yml GitHub actions workflow file .""" + dpkg_dependencies = self._GetDPKGPythonDependencies() + test_dependencies = self._GetDPKGTestDependencies(dpkg_dependencies) + dpkg_dependencies.extend(test_dependencies) + dpkg_dependencies.extend(['python3-pip', 'python3-setuptools', 'tox']) + + dpkg_dev_dependencies = self._GetDPKGDevDependencies() + + template_mappings = { + 'dpkg_dependencies': ' '.join(sorted(set(dpkg_dependencies))), + 'dpkg_dev_dependencies': ' '.join(sorted(set(dpkg_dev_dependencies)))} + + python_module_name = self._project_definition.name + + if self._project_definition.name.endswith('-kb'): + python_module_name = ''.join([python_module_name[:-3], 'rc']) + + paths_to_lint_yaml = [] + + if os.path.isdir(python_module_name): + if glob.glob(os.path.join( + python_module_name, '**', '*.yaml'), recursive=True): + paths_to_lint_yaml.append(python_module_name) + + if os.path.isdir('data'): + if glob.glob(os.path.join('data', '**', '*.yaml'), recursive=True): + paths_to_lint_yaml.append('data') + + if os.path.isdir('test_data'): + if glob.glob(os.path.join('test_data', '**', '*.yaml'), recursive=True): + paths_to_lint_yaml.append('test_data') + + if os.path.isdir('tests'): + if glob.glob(os.path.join('tests', '**', '*.yaml'), recursive=True): + paths_to_lint_yaml.append('tests') + + file_content = [] + + template_data = self._GenerateFromTemplate('header.yml', template_mappings) + file_content.append(template_data) + + if paths_to_lint_yaml: + template_data = self._GenerateFromTemplate( + 'yamllint.yml', template_mappings) + file_content.append(template_data) + + file_content = ''.join(file_content) + + with open(self.PATH, 'w', encoding='utf-8') as file_object: + file_object.write(file_content) + + class GitHubActionsTestDockerYmlWriter(interface.DependencyFileWriter): """test_docker.yml GitHub actions workflow file writer.""" diff --git a/l2tdevtools/dependency_writers/setup.py b/l2tdevtools/dependency_writers/setup.py index 062c702c..7a89b4b5 100644 --- a/l2tdevtools/dependency_writers/setup.py +++ b/l2tdevtools/dependency_writers/setup.py @@ -153,6 +153,10 @@ def Write(self): file_content.append(f'Homepage = "{url:s}"\n') file_content.append(f'Repository = "{url:s}"\n') + template_data = self._GenerateFromTemplate( + 'black.toml', template_mappings) + file_content.append(template_data) + template_data = self._GenerateFromTemplate( 'setuptools.packages.toml', template_mappings) file_content.append(template_data) diff --git a/l2tdevtools/dependency_writers/tox_ini.py b/l2tdevtools/dependency_writers/tox_ini.py index 1b64bdd5..0d371ec1 100644 --- a/l2tdevtools/dependency_writers/tox_ini.py +++ b/l2tdevtools/dependency_writers/tox_ini.py @@ -73,11 +73,14 @@ def Write(self): if os.path.isdir('tools'): paths_to_lint_python.append('tools') - envlist = ['py3{10,11,12,13,14}', 'coverage'] + envlist = ['py3{10,11,12,13,14}', 'black', 'coverage'] if os.path.isdir('docs'): envlist.append('docs') - envlist.extend(['lint', 'wheel']) + envlist.extend(['pylint', 'wheel']) + + if paths_to_lint_yaml: + envlist.append('yamllint') template_mappings = { 'envlist': ','.join(envlist), @@ -91,20 +94,24 @@ def Write(self): template_data = self._GenerateFromTemplate('header', template_mappings) file_content.append(template_data) + template_data = self._GenerateFromTemplate( + 'testenv_black', template_mappings) + file_content.append(template_data) + if os.path.isdir('docs'): template_data = self._GenerateFromTemplate( 'testenv_docs', template_mappings) file_content.append(template_data) - if paths_to_lint_yaml: - template_name = 'testenv_lint-with_yaml' - else: - template_name = 'testenv_lint' - template_data = self._GenerateFromTemplate( - template_name, template_mappings) + 'testenv_pylint', template_mappings) file_content.append(template_data) + if paths_to_lint_yaml: + template_data = self._GenerateFromTemplate( + 'testenv_yamllint', template_mappings) + file_content.append(template_data) + file_content = ''.join(file_content) with open(self.PATH, 'w', encoding='utf-8') as file_object: diff --git a/tools/update-dependencies.py b/tools/update-dependencies.py index 20c136b8..26b57e6f 100755 --- a/tools/update-dependencies.py +++ b/tools/update-dependencies.py @@ -49,6 +49,7 @@ def Main(): writer.Write() for writer_class in ( + github_actions.GitHubActionsLintYmlWriter, github_actions.GitHubActionsTestDockerYmlWriter, github_actions.GitHubActionsTestDocsYmlWriter, github_actions.GitHubActionsTestMacOSYmlWriter, From 5264d0869c35ca342bb072eed7421736f0c6edaa Mon Sep 17 00:00:00 2001 From: Joachim Metz Date: Fri, 15 May 2026 19:01:45 +0200 Subject: [PATCH 2/7] Changes to use black Python formatter --- .github/workflows/lint.yml | 68 ++++++++++++++++++++++++++++++++++ .github/workflows/test_tox.yml | 31 ---------------- .pylintrc | 7 ++-- pyproject.toml | 7 +++- tox.ini | 37 ++++++++++++++++-- 5 files changed, 110 insertions(+), 40 deletions(-) create mode 100644 .github/workflows/lint.yml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 00000000..9558379b --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,68 @@ +# Check source. +name: lint +on: + pull_request: + branches: + - main + push: + branches: + - main +permissions: read-all +jobs: + black: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - name: Check format of Python code + uses: psf/black@stable + with: + options: "--check" + src: "." + pylint: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.14'] + container: + image: ubuntu:26.04 + steps: + - uses: actions/checkout@v6 + - name: Set up container + env: + DEBIAN_FRONTEND: noninteractive + run: | + apt-get update -q + apt-get install -y libterm-readline-gnu-perl locales software-properties-common + locale-gen en_US.UTF-8 + ln -f -s /usr/share/zoneinfo/UTC /etc/localtime + - name: Install dependencies + env: + DEBIAN_FRONTEND: noninteractive + run: | + add-apt-repository -y universe + add-apt-repository -y ppa:deadsnakes/ppa + add-apt-repository -y ppa:gift/dev + apt-get update -q + apt-get install -y build-essential git pkg-config python${{ matrix.python-version }} python${{ matrix.python-version }}-dev python${{ matrix.python-version }}-venv python3-pip python3-setuptools tox + - name: Run linter + env: + LANG: en_US.UTF-8 + run: | + tox -e pylint + yamllint: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.12'] + steps: + - uses: actions/checkout@v6 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + - name: Install tox + run: | + python -m pip install tox + - name: Run linter + run: | + tox -e yamllint diff --git a/.github/workflows/test_tox.yml b/.github/workflows/test_tox.yml index 2e708214..a6cd86e6 100644 --- a/.github/workflows/test_tox.yml +++ b/.github/workflows/test_tox.yml @@ -81,34 +81,3 @@ jobs: uses: codecov/codecov-action@v6 with: token: ${{ secrets.CODECOV_TOKEN }} - lint: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ['3.14'] - container: - image: ubuntu:26.04 - steps: - - uses: actions/checkout@v6 - - name: Set up container - env: - DEBIAN_FRONTEND: noninteractive - run: | - apt-get update -q - apt-get install -y libterm-readline-gnu-perl locales software-properties-common - locale-gen en_US.UTF-8 - ln -f -s /usr/share/zoneinfo/UTC /etc/localtime - - name: Install dependencies - env: - DEBIAN_FRONTEND: noninteractive - run: | - add-apt-repository -y universe - add-apt-repository -y ppa:deadsnakes/ppa - add-apt-repository -y ppa:gift/dev - apt-get update -q - apt-get install -y build-essential git pkg-config python${{ matrix.python-version }} python${{ matrix.python-version }}-dev python${{ matrix.python-version }}-venv python3-pip python3-setuptools tox - - name: Run linter - env: - LANG: en_US.UTF-8 - run: | - tox -e lint diff --git a/.pylintrc b/.pylintrc index ca6099d2..34bcab36 100644 --- a/.pylintrc +++ b/.pylintrc @@ -358,12 +358,11 @@ indent-after-paren=4 # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 # tab). -# indent-string=' ' -indent-string=' ' +indent-string=' ' # Maximum number of characters on a single line. # max-line-length=100 -max-line-length=80 +max-line-length=88 # Maximum number of lines in a module. max-module-lines=1000 @@ -599,7 +598,7 @@ spelling-store-unknown-words=no # This flag controls whether inconsistent-quotes generates a warning when the # character used as a quote delimiter is used inconsistently within a module. -check-quote-consistency=no +check-quote-consistency=yes # This flag controls whether the implicit-str-concat should generate a warning # on implicit string concatenation in sequences defined over several lines. diff --git a/pyproject.toml b/pyproject.toml index ac4a40e6..ab70e87f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "l2tdevtools" -version = "20260508" +version = "20260515" description = "Development tools for the log2timeline projects" maintainers = [ { name = "Log2Timeline maintainers", email = "log2timeline-maintainers@googlegroups.com" }, @@ -22,5 +22,10 @@ requires-python = ">=3.10" Homepage = "https://github.com/log2timeline/l2tdevtools" Repository = "https://github.com/log2timeline/l2tdevtools" +[tool.black] +line-length = 88 +target-version = ["py310"] +include = "\\.pyi?$" + [tool.setuptools] package-dir = {"l2tdevtools" = "l2tdevtools"} diff --git a/tox.ini b/tox.ini index 457d960d..e36fc0b6 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py3{10,11,12,13,14},coverage,lint,wheel +envlist = py3{10,11,12,13,14},black,coverage,pylint,wheel,yamllint [testenv] allowlist_externals = ./run_tests.py @@ -23,7 +23,23 @@ commands = coverage: coverage xml wheel: python -m build --no-isolation --wheel -[testenv:lint] +[testenv:black] +skipsdist = True +pip_pre = True +passenv = + CFLAGS + CPPFLAGS + LDFLAGS +setenv = + PYTHONPATH = {toxinidir} +deps = + black + setuptools >= 65 +commands = + black --version + black --check . + +[testenv:pylint] skipsdist = True pip_pre = True passenv = @@ -35,9 +51,22 @@ setenv = deps = pylint >= 3.3.0, < 3.4.0 setuptools >= 65 - yamllint >= 1.26.0 commands = pylint --version - yamllint -v pylint --rcfile=.pylintrc l2tdevtools tests tools + +[testenv:yamllint] +skipsdist = True +pip_pre = True +passenv = + CFLAGS + CPPFLAGS + LDFLAGS +setenv = + PYTHONPATH = {toxinidir} +deps = + setuptools >= 65 + yamllint >= 1.26.0 +commands = + yamllint -v yamllint -c .yamllint.yaml data From 42eb1b2adb39b302c7c257b15648f677e6391256 Mon Sep 17 00:00:00 2001 From: Joachim Metz Date: Fri, 15 May 2026 19:17:34 +0200 Subject: [PATCH 3/7] Changes to use black Python formatter --- l2tdevtools/__init__.py | 2 +- l2tdevtools/build_helpers/dpkg.py | 2295 +++++++++-------- l2tdevtools/build_helpers/factory.py | 117 +- l2tdevtools/build_helpers/interface.py | 199 +- l2tdevtools/build_helpers/rpm.py | 1394 +++++----- l2tdevtools/build_helpers/source.py | 179 +- l2tdevtools/build_helpers/wheel.py | 424 +-- .../dependency_writers/appveyor_scripts.py | 34 +- .../dependency_writers/appveyor_yml.py | 120 +- .../dependency_writers/check_dependencies.py | 37 +- .../dependency_writers/dependencies_py.py | 62 +- l2tdevtools/dependency_writers/dpkg.py | 368 +-- l2tdevtools/dependency_writers/gift_copr.py | 249 +- l2tdevtools/dependency_writers/gift_ppa.py | 257 +- .../dependency_writers/github_actions.py | 293 ++- l2tdevtools/dependency_writers/interface.py | 325 +-- .../dependency_writers/jenkins_scripts.py | 62 +- .../dependency_writers/linux_scripts.py | 253 +- l2tdevtools/dependency_writers/pylint_rc.py | 22 +- l2tdevtools/dependency_writers/setup.py | 354 +-- l2tdevtools/dependency_writers/sphinx_docs.py | 79 +- l2tdevtools/dependency_writers/tox_ini.py | 160 +- l2tdevtools/download_helper.py | 94 +- l2tdevtools/download_helpers/github.py | 334 +-- l2tdevtools/download_helpers/interface.py | 159 +- l2tdevtools/download_helpers/project.py | 282 +- l2tdevtools/download_helpers/pypi.py | 297 +-- l2tdevtools/download_helpers/zlib.py | 139 +- l2tdevtools/dpkg_files.py | 1414 +++++----- l2tdevtools/helpers/project.py | 253 +- l2tdevtools/lib/definitions.py | 5 +- l2tdevtools/lib/errors.py | 4 +- l2tdevtools/presets.py | 142 +- l2tdevtools/project_config.py | 147 +- l2tdevtools/projects.py | 736 +++--- l2tdevtools/review_helpers/cli.py | 93 +- l2tdevtools/review_helpers/git.py | 615 ++--- l2tdevtools/review_helpers/github.py | 297 +-- l2tdevtools/review_helpers/pylint.py | 192 +- l2tdevtools/review_helpers/review.py | 446 ++-- l2tdevtools/review_helpers/url_lib.py | 60 +- l2tdevtools/source_helper.py | 792 +++--- l2tdevtools/spec_file.py | 1518 ++++++----- l2tdevtools/versions.py | 54 +- pyproject.toml | 6 + run_tests.py | 10 +- test_data/linter_fail.py | 12 +- test_data/linter_pass.py | 10 +- tests/build_helpers/dpkg.py | 130 +- tests/build_helpers/factory.py | 40 +- tests/build_helpers/interface.py | 30 +- tests/build_helpers/rpm.py | 4 +- tests/build_helpers/source.py | 4 +- tests/build_helpers/test_lib.py | 250 +- tests/build_helpers/wheel.py | 254 +- tests/dependencies.py | 283 +- tests/dependency_writers/appveyor_yml.py | 34 +- tests/dependency_writers/dependencies_py.py | 34 +- tests/dependency_writers/dpkg.py | 64 +- tests/dependency_writers/gift_copr.py | 197 +- tests/dependency_writers/gift_ppa.py | 199 +- tests/dependency_writers/interface.py | 128 +- tests/dependency_writers/jenkins_scripts.py | 64 +- tests/dependency_writers/linux_scripts.py | 199 +- tests/dependency_writers/pylint_rc.py | 34 +- tests/dependency_writers/setup.py | 34 +- tests/dependency_writers/sphinx_docs.py | 94 +- tests/dependency_writers/tox_ini.py | 34 +- tests/download_helper.py | 4 +- tests/download_helpers/github.py | 327 +-- tests/download_helpers/interface.py | 67 +- tests/download_helpers/pypi.py | 97 +- tests/dpkg_files.py | 39 +- tests/helpers/project.py | 22 +- tests/presets.py | 14 +- tests/projects.py | 115 +- tests/review_helpers/cli.py | 44 +- tests/review_helpers/git.py | 14 +- tests/review_helpers/github.py | 79 +- tests/review_helpers/pylint.py | 30 +- tests/review_helpers/review.py | 54 +- tests/review_helpers/test_lib.py | 38 +- tests/schema_extractor.py | 54 +- tests/source_helper.py | 16 +- tests/spec_file.py | 75 +- tests/test_lib.py | 74 +- tests/update.py | 165 +- tools/build.py | 1082 ++++---- tools/dpkg-generate.py | 243 +- tools/manage.py | 2005 +++++++------- tools/review.py | 213 +- tools/schema_extractor.py | 198 +- tools/stats.py | 906 ++++--- tools/update-dependencies.py | 180 +- tools/update.py | 1488 ++++++----- 95 files changed, 13116 insertions(+), 12033 deletions(-) diff --git a/l2tdevtools/__init__.py b/l2tdevtools/__init__.py index 343677d7..88eab66b 100644 --- a/l2tdevtools/__init__.py +++ b/l2tdevtools/__init__.py @@ -1,3 +1,3 @@ """The log2timeline devtools module.""" -__version__ = '20180708' +__version__ = "20180708" diff --git a/l2tdevtools/build_helpers/dpkg.py b/l2tdevtools/build_helpers/dpkg.py index dee6a0ee..f334f508 100644 --- a/l2tdevtools/build_helpers/dpkg.py +++ b/l2tdevtools/build_helpers/dpkg.py @@ -18,1136 +18,1241 @@ class DPKGBuildHelper(interface.BuildHelper): - """Helper to build dpkg packages (.deb). - - Attributes: - architecture (str): dpkg target architecture. - distribution (str): dpkg target distributions. - version_suffix (str): dpkg version suffix. - """ - - _BUILD_DEPENDENCIES = frozenset([ - 'git', - 'build-essential', - 'autotools-dev', - 'autoconf', - 'automake', - 'autopoint', - 'dh-autoreconf', - 'libtool', - 'gettext', - 'flex', - 'byacc', - 'debhelper', - 'devscripts', - 'dpkg-dev', - 'fakeroot', - 'pkg-config', - 'quilt', - 'python3-all', - 'python3-all-dev', - 'python3-setuptools', - ]) - - _BUILD_DEPENDENCY_PACKAGE_NAMES = { - 'bzip2': 'libbz2-dev', - 'fuse': 'libfuse-dev', - 'libcrypto': 'libssl-dev', - 'liblzma': 'liblzma-dev', - 'sqlite': 'libsqlite3-dev', - 'zeromq': 'libzmq3-dev', - 'zlib': 'zlib1g-dev' - } - - def __init__( - self, project_definition, l2tdevtools_path, dependency_definitions): - """Initializes a build helper. - - Args: - project_definition (ProjectDefinition): definition of the project - to build. - l2tdevtools_path (str): path to the l2tdevtools directory. - dependency_definitions (dict[str, ProjectDefinition]): definitions of all - projects, which is used to determine the properties of dependencies. - """ - super().__init__( - project_definition, l2tdevtools_path, dependency_definitions) - self._build_host_distribution = self._GetBuildHostDistribution() - self._prep_script = 'prep-dpkg.sh' - self._post_script = 'post-dpkg.sh' - - self.architecture = None - self.distribution = None - self.version_suffix = None - - def _BuildPrepare( - self, source_directory, project_name, project_version, version_suffix, - distribution, architecture): - """Make the necessary preparations before building the dpkg packages. - - Args: - source_directory (str): name of the source directory. - project_name (str): name of the project. - project_version (str): version of the project. - version_suffix (str): version suffix. - distribution (str): distribution. - architecture (str): architecture. - - Returns: - bool: True if the preparations were successful, False otherwise. - """ - # Script to run before building, e.g. to change the dpkg packaging files. - if os.path.exists(self._prep_script): - command = ( - f'sh ../{self._prep_script:s} {project_name:s} {project_version!s} ' - f'{version_suffix:s} {distribution:s} {architecture:s}') - exit_code = subprocess.call( - f'(cd {source_directory:s} && {command:s})', shell=True) - if exit_code != 0: - logging.error(f'Running: "{command:s}" failed.') - return False - - return True - - def _BuildFinalize( - self, source_directory, project_name, project_version, version_suffix, - distribution, architecture): - """Make the necessary finalizations after building the dpkg packages. - - Args: - source_directory (str): name of the source directory. - project_name (str): name of the project. - project_version (str): version of the project. - version_suffix (str): version suffix. - distribution (str): distribution. - architecture (str): architecture. - - Returns: - bool: True if the finalizations were successful, False otherwise. - """ - # Script to run after building, e.g. to automatically upload the dpkg - # package files to an apt repository. - if os.path.exists(self._post_script): - command = ( - f'sh ../{self._post_script:s} {project_name:s} {project_version!s} ' - f'{version_suffix:s} {distribution:s} {architecture:s}') - exit_code = subprocess.call( - f'(cd {source_directory:s} && {command:s})', shell=True) - if exit_code != 0: - logging.error(f'Running: "{command:s}" failed.') - return False - - return True - - def _CheckIsInstalled(self, package_name): - """Checks if a package is installed. - - Args: - package_name (str): name of the package. - - Returns: - bool: True if the package is installed, False otherwise. - """ - exit_code = subprocess.call( - f'dpkg-query -s {package_name:s} >/dev/null 2>&1', shell=True) - return exit_code == 0 - - def _CreateOriginalSourcePackage( - self, source_package_path, project_name, project_version): - """Creates the .orig.tar.gz source package. - - Args: - source_package_path (str): path of the source package file. - project_name (str): project name. - project_version (str): version of the project. - """ - if self._project_definition.dpkg_source_name: - project_name = self._project_definition.dpkg_source_name - - deb_orig_source_package_filename = ( - f'{project_name:s}_{project_version!s}.orig.tar.gz') - if os.path.exists(deb_orig_source_package_filename): - return - - if source_package_path.endswith('.zip'): - self._CreateOriginalSourcePackageFromZip( - source_package_path, deb_orig_source_package_filename) - else: - shutil.copy(source_package_path, deb_orig_source_package_filename) - - def _CreateOriginalSourcePackageFromZip( - self, source_package_path, orig_source_package_filename): - """Creates the .orig.tar.gz source package from a .zip file. - - Args: - source_package_path (str): path of the source package file. - orig_source_package_filename (str): name of the .orig.tar.gz source - package file. - """ - posix_epoch = datetime.datetime(1970, 1, 1) - - with zipfile.ZipFile(source_package_path, 'r') as zip_file: - with tarfile.open( - name=orig_source_package_filename, mode='w:gz') as tar_file: - for filename in zip_file.namelist(): - with zip_file.open(filename) as file_object: - zip_info = zip_file.getinfo(filename) - - tar_info = tarfile.TarInfo(filename) - - if zip_info.is_dir(): - tar_info.mode = 0o755 - else: - tar_info.mode = 0o644 - - tar_info.uid = os.getuid() - tar_info.gid = os.getgid() - tar_info.size = zip_info.file_size - - # Populate modification times from zip file into tar archive, - # as launchpad refuses to build packages containing files with - # timestamps too far in the past. - date_time = zip_info.date_time - modification_time = datetime.datetime(*date_time) - modification_time = int( - (modification_time - posix_epoch).total_seconds()) - - tar_info.mtime = modification_time - - tar_file.addfile(tar_info, fileobj=file_object) - - def _CreatePackagingFiles(self, source_directory, project_version): - """Creates packaging files. - - Args: - source_directory (str): name of the source directory. - project_version (str): project version. - - Returns: - bool: True if successful, False otherwise. - """ - debian_directory = os.path.join(source_directory, 'debian') - - # If there is a debian directory remove it and recreate it from - # the dpkg directory. - if os.path.exists(debian_directory): - logging.info(f'Removing: {debian_directory:s}') - shutil.rmtree(debian_directory, ignore_errors=True) - - dpkg_directory = os.path.join(source_directory, 'dpkg') - - if not os.path.exists(dpkg_directory): - dpkg_directory = os.path.join(source_directory, 'config', 'dpkg') - - if os.path.exists(dpkg_directory): - shutil.copytree(dpkg_directory, debian_directory) - - else: - build_configuration = self._DetermineBuildConfiguration(source_directory) # pylint: disable=assignment-from-none - - os.chdir(source_directory) - - try: - build_files_generator = dpkg_files.DPKGBuildFilesGenerator( - self._project_definition, project_version, self._data_path, - self._dependency_definitions, - build_configuration=build_configuration) - - build_files_generator.GenerateFiles('debian') - - finally: - os.chdir('..') - - if not os.path.exists(debian_directory): - logging.error(f'Missing debian sub directory in: {source_directory:s}') - return False - - return True - - # pylint: disable=redundant-returns-doc,unused-argument - def _DetermineBuildConfiguration(self, source_directory): - """Determines the build configuration of a project. - - Args: - source_directory (str): path of the source directory. - - Returns: - DPKGBuildConfiguration: dpkg build configuration or None if the build - configuration could not be determined. - """ - return None - - def _GetBuildHostDistribution(self): - """Determines the Debian/Ubuntu distribution name of the build host. - - This information can be found in the configuration file "/etc/lsb-release" - if preset. Otherwise this information can be obtained by invoking - "/usr/bin/lsb_release -sc". + """Helper to build dpkg packages (.deb). - Returns: - str: Debian/Ubuntu distribution name or None if the value could not - be determined. + Attributes: + architecture (str): dpkg target architecture. + distribution (str): dpkg target distributions. + version_suffix (str): dpkg version suffix. """ - distribution_name = None - lsb_release_path = '/etc/lsb-release' - if os.path.exists(lsb_release_path): - lsb_release = self._ReadLSBReleaseConfigurationFile(lsb_release_path) - distribution_name = lsb_release.get('distrib_codename', None) - - if not distribution_name: - output = self._RunLSBReleaseCommand(option='-sc') - if output: - distribution_name = output.strip() or None - - return distribution_name - - def _ReadLSBReleaseConfigurationFile(self, path): - """Reads a lsb-release configuration (/etc/lsb-release) file. - - This function ignores comment lines, lines that are not UTF-8 formatted - and lines that do not consist of "key=value". - - Args: - path (str): path of the lsb-release configuration file. - - Returns: - dict[str, str]: key value pairs that are stored in the lsb-release - configuration file. - """ - lsb_release_values = {} - with open(path, 'rb') as file_object: - for line in file_object.readlines(): - try: - line = line.decode('utf-8').strip() - except UnicodeDecodeError: - continue - - if not line.startswith('#') and '=' in line: - line = line.lower() - key, _, value = line.partition('=') - lsb_release_values[key.strip()] = value.strip() or None - - return lsb_release_values - - def _RemoveOlderDPKGPackages(self, project_name, project_version): - """Removes previous versions of dpkg packages. - - Args: - project_name (str): project name. - project_version (str): project version. - """ - if self._project_definition.dpkg_source_name: - project_name = self._project_definition.dpkg_source_name - - filenames_to_ignore = re.compile( - f'^{project_name:s}[-_].*{project_version!s}') - - # Remove files of previous versions in the format: - # *[-_][0-9]*-[1-9]_.* - for filename in glob.glob( - f'{project_name:s}*[-_][0-9]*-[1-9]_{self.architecture:s}.*'): - if not filenames_to_ignore.match(filename): - logging.info(f'Removing: {filename:s}') - try: - os.remove(filename) - except PermissionError as exception: - logging.info( - f'Unable to remove: {filename:s} with error: {exception!s}') - - # Remove files of previous versions in the format: - # [-_][0-9]*-[1-9].* - for filename in glob.glob(f'{project_name:s}[-_][0-9]*-[1-9].*'): - if not filenames_to_ignore.match(filename): - logging.info(f'Removing: {filename:s}') - try: - os.remove(filename) - except PermissionError as exception: - logging.info( - f'Unable to remove: {filename:s} with error: {exception!s}') - - def _RemoveOlderOriginalSourcePackage( - self, project_name, project_version, version_suffix=None, - distribution=None): - """Removes previous versions of original source package. - - Args: - project_name (str): name of the project. - project_version (str): version of the project. - version_suffix (str): version suffix. - distribution (str): distribution. - """ - if self._project_definition.dpkg_source_name: - project_name = self._project_definition.dpkg_source_name - - filenames_to_ignore = re.compile( - f'^{project_name:s}_{project_version!s}.orig.tar.gz') - - # Remove files of previous versions in the format: - # _[0-9]*.orig.tar.gz - for filename in glob.glob(f'{project_name:s}_[0-9]*.orig.tar.gz'): - if not filenames_to_ignore.match(filename): - logging.info(f'Removing: {filename:s}') - try: - os.remove(filename) - except PermissionError as exception: - logging.info( - f'Unable to remove: {filename:s} with error: {exception!s}') - - # Remove files of previous versions in the format: - # _[0-9]*~.orig.tar.gz - if version_suffix and distribution: - for filename in glob.glob( - f'{project_name:s}_[0-9]*{version_suffix:s}~{distribution:s}' - f'.orig.tar.gz'): - if not filenames_to_ignore.match(filename): - logging.info(f'Removing: {filename:s}') - try: - os.remove(filename) - except PermissionError as exception: - logging.info( - f'Unable to remove: {filename:s} with error: {exception!s}') - - def _RemoveOlderSourceDPKGPackages(self, project_name, project_version): - """Removes previous versions of source dpkg packages. - - Args: - project_name (str): project name. - project_version (str): project version. - """ - if self._project_definition.dpkg_source_name: - project_name = self._project_definition.dpkg_source_name - - filenames_to_ignore = re.compile( - f'^{project_name:s}[-_].*{project_version!s}') - - # Remove files of previous versions in the format: - # [-_][0-9]*-[1-9]~_.* - for filename in glob.glob( - f'{project_name:s}[-_][0-9]*-[1-9]{self.version_suffix:s}~' - f'{self.distribution:s}_{self.architecture:s}.*'): - if not filenames_to_ignore.match(filename): - logging.info(f'Removing: {filename:s}') - try: - os.remove(filename) - except PermissionError as exception: - logging.info( - f'Unable to remove: {filename:s} with error: {exception!s}') - - # Remove files of previous versions in the format: - # [-_][0-9]*-[1-9]i~.* - filenames_glob = ( - f'{project_name:s}[-_][0-9]*-[1-9]{self.version_suffix:s}' - f'~{self.distribution:s}.*') - filenames = glob.glob(filenames_glob) - - for filename in filenames: - if not filenames_to_ignore.match(filename): - logging.info(f'Removing: {filename:s}') - try: - os.remove(filename) - except PermissionError as exception: - logging.info( - f'Unable to remove: {filename:s} with error: {exception!s}') - - def _RunLSBReleaseCommand(self, option='-a'): - """Runs the lsb-release command (/usr/bin/lsb_release). - - Args: - option (Optional[str]): option to pass to the lsb-release command. - - Returns: - str: output of the command or None if there is no lsb-release command - available, or running the command failed, or the output cannot be - decoded. - """ - output = None - path = '/usr/bin/lsb_release' - - if os.path.exists(path): - with subprocess.Popen( - [path, option], stderr=subprocess.PIPE, - stdout=subprocess.PIPE) as process: - - output, _ = process.communicate() - if not process or process.returncode != 0: - output = None - - elif isinstance(output, bytes): - try: - output = output.decode('utf-8') - except UnicodeDecodeError: - output = None - - return output - - def CheckBuildDependencies(self): - """Checks if the build dependencies are met. - - Returns: - list[str]: build dependency names that are not met or an empty list. - """ - missing_packages = [] - for package_name in self._BUILD_DEPENDENCIES: - if not self._CheckIsInstalled(package_name): - missing_packages.append(package_name) - - for name in self._project_definition.build_dependencies: - package_name = self._BUILD_DEPENDENCY_PACKAGE_NAMES.get(name, name) - if package_name not in self._project_definition.dpkg_build_dependencies: - self._project_definition.dpkg_build_dependencies.append(package_name) - - for package_name in self._project_definition.dpkg_build_dependencies: - if not self._CheckIsInstalled(package_name): - missing_packages.append(package_name) - - return missing_packages - - def CheckProjectConfiguration(self): - """Checks if the project configuration is correct. - - This functions checks if all build dependencies are defined as dpkg build - dependencies. - - Returns: - bool: True if the project configuration is correct, False otherwise. - """ - result = True - for name in self._project_definition.build_dependencies: - package_name = self._BUILD_DEPENDENCY_PACKAGE_NAMES.get(name, name) - if package_name not in self._project_definition.dpkg_build_dependencies: - logging.warning( - f'Build dependency: {name:s} not defined as dpkg build dependency: ' - f'{package_name:s}') - result = False - - return result + _BUILD_DEPENDENCIES = frozenset( + [ + "git", + "build-essential", + "autotools-dev", + "autoconf", + "automake", + "autopoint", + "dh-autoreconf", + "libtool", + "gettext", + "flex", + "byacc", + "debhelper", + "devscripts", + "dpkg-dev", + "fakeroot", + "pkg-config", + "quilt", + "python3-all", + "python3-all-dev", + "python3-setuptools", + ] + ) + + _BUILD_DEPENDENCY_PACKAGE_NAMES = { + "bzip2": "libbz2-dev", + "fuse": "libfuse-dev", + "libcrypto": "libssl-dev", + "liblzma": "liblzma-dev", + "sqlite": "libsqlite3-dev", + "zeromq": "libzmq3-dev", + "zlib": "zlib1g-dev", + } + + def __init__(self, project_definition, l2tdevtools_path, dependency_definitions): + """Initializes a build helper. + + Args: + project_definition (ProjectDefinition): definition of the project + to build. + l2tdevtools_path (str): path to the l2tdevtools directory. + dependency_definitions (dict[str, ProjectDefinition]): definitions of all + projects, which is used to determine the properties of dependencies. + """ + super().__init__(project_definition, l2tdevtools_path, dependency_definitions) + self._build_host_distribution = self._GetBuildHostDistribution() + self._prep_script = "prep-dpkg.sh" + self._post_script = "post-dpkg.sh" + + self.architecture = None + self.distribution = None + self.version_suffix = None + + def _BuildPrepare( + self, + source_directory, + project_name, + project_version, + version_suffix, + distribution, + architecture, + ): + """Make the necessary preparations before building the dpkg packages. + + Args: + source_directory (str): name of the source directory. + project_name (str): name of the project. + project_version (str): version of the project. + version_suffix (str): version suffix. + distribution (str): distribution. + architecture (str): architecture. + + Returns: + bool: True if the preparations were successful, False otherwise. + """ + # Script to run before building, e.g. to change the dpkg packaging files. + if os.path.exists(self._prep_script): + command = ( + f"sh ../{self._prep_script:s} {project_name:s} {project_version!s} " + f"{version_suffix:s} {distribution:s} {architecture:s}" + ) + exit_code = subprocess.call( + f"(cd {source_directory:s} && {command:s})", shell=True + ) + if exit_code != 0: + logging.error(f'Running: "{command:s}" failed.') + return False + + return True + + def _BuildFinalize( + self, + source_directory, + project_name, + project_version, + version_suffix, + distribution, + architecture, + ): + """Make the necessary finalizations after building the dpkg packages. + + Args: + source_directory (str): name of the source directory. + project_name (str): name of the project. + project_version (str): version of the project. + version_suffix (str): version suffix. + distribution (str): distribution. + architecture (str): architecture. + + Returns: + bool: True if the finalizations were successful, False otherwise. + """ + # Script to run after building, e.g. to automatically upload the dpkg + # package files to an apt repository. + if os.path.exists(self._post_script): + command = ( + f"sh ../{self._post_script:s} {project_name:s} {project_version!s} " + f"{version_suffix:s} {distribution:s} {architecture:s}" + ) + exit_code = subprocess.call( + f"(cd {source_directory:s} && {command:s})", shell=True + ) + if exit_code != 0: + logging.error(f'Running: "{command:s}" failed.') + return False + + return True + + def _CheckIsInstalled(self, package_name): + """Checks if a package is installed. + + Args: + package_name (str): name of the package. + + Returns: + bool: True if the package is installed, False otherwise. + """ + exit_code = subprocess.call( + f"dpkg-query -s {package_name:s} >/dev/null 2>&1", shell=True + ) + return exit_code == 0 + + def _CreateOriginalSourcePackage( + self, source_package_path, project_name, project_version + ): + """Creates the .orig.tar.gz source package. + + Args: + source_package_path (str): path of the source package file. + project_name (str): project name. + project_version (str): version of the project. + """ + if self._project_definition.dpkg_source_name: + project_name = self._project_definition.dpkg_source_name + + deb_orig_source_package_filename = ( + f"{project_name:s}_{project_version!s}.orig.tar.gz" + ) + if os.path.exists(deb_orig_source_package_filename): + return + + if source_package_path.endswith(".zip"): + self._CreateOriginalSourcePackageFromZip( + source_package_path, deb_orig_source_package_filename + ) + else: + shutil.copy(source_package_path, deb_orig_source_package_filename) + + def _CreateOriginalSourcePackageFromZip( + self, source_package_path, orig_source_package_filename + ): + """Creates the .orig.tar.gz source package from a .zip file. + + Args: + source_package_path (str): path of the source package file. + orig_source_package_filename (str): name of the .orig.tar.gz source + package file. + """ + posix_epoch = datetime.datetime(1970, 1, 1) + + with zipfile.ZipFile(source_package_path, "r") as zip_file: + with tarfile.open( + name=orig_source_package_filename, mode="w:gz" + ) as tar_file: + for filename in zip_file.namelist(): + with zip_file.open(filename) as file_object: + zip_info = zip_file.getinfo(filename) + + tar_info = tarfile.TarInfo(filename) + + if zip_info.is_dir(): + tar_info.mode = 0o755 + else: + tar_info.mode = 0o644 + + tar_info.uid = os.getuid() + tar_info.gid = os.getgid() + tar_info.size = zip_info.file_size + + # Populate modification times from zip file into tar archive, + # as launchpad refuses to build packages containing files with + # timestamps too far in the past. + date_time = zip_info.date_time + modification_time = datetime.datetime(*date_time) + modification_time = int( + (modification_time - posix_epoch).total_seconds() + ) + + tar_info.mtime = modification_time + + tar_file.addfile(tar_info, fileobj=file_object) + + def _CreatePackagingFiles(self, source_directory, project_version): + """Creates packaging files. + + Args: + source_directory (str): name of the source directory. + project_version (str): project version. + + Returns: + bool: True if successful, False otherwise. + """ + debian_directory = os.path.join(source_directory, "debian") + + # If there is a debian directory remove it and recreate it from + # the dpkg directory. + if os.path.exists(debian_directory): + logging.info(f"Removing: {debian_directory:s}") + shutil.rmtree(debian_directory, ignore_errors=True) + + dpkg_directory = os.path.join(source_directory, "dpkg") + + if not os.path.exists(dpkg_directory): + dpkg_directory = os.path.join(source_directory, "config", "dpkg") + + if os.path.exists(dpkg_directory): + shutil.copytree(dpkg_directory, debian_directory) + + else: + build_configuration = self._DetermineBuildConfiguration( + source_directory + ) # pylint: disable=assignment-from-none + + os.chdir(source_directory) + + try: + build_files_generator = dpkg_files.DPKGBuildFilesGenerator( + self._project_definition, + project_version, + self._data_path, + self._dependency_definitions, + build_configuration=build_configuration, + ) + + build_files_generator.GenerateFiles("debian") + + finally: + os.chdir("..") + + if not os.path.exists(debian_directory): + logging.error(f"Missing debian sub directory in: {source_directory:s}") + return False + + return True + + # pylint: disable=redundant-returns-doc,unused-argument + def _DetermineBuildConfiguration(self, source_directory): + """Determines the build configuration of a project. + + Args: + source_directory (str): path of the source directory. + + Returns: + DPKGBuildConfiguration: dpkg build configuration or None if the build + configuration could not be determined. + """ + return None + + def _GetBuildHostDistribution(self): + """Determines the Debian/Ubuntu distribution name of the build host. + + This information can be found in the configuration file "/etc/lsb-release" + if preset. Otherwise this information can be obtained by invoking + "/usr/bin/lsb_release -sc". + + Returns: + str: Debian/Ubuntu distribution name or None if the value could not + be determined. + """ + distribution_name = None + + lsb_release_path = "/etc/lsb-release" + if os.path.exists(lsb_release_path): + lsb_release = self._ReadLSBReleaseConfigurationFile(lsb_release_path) + distribution_name = lsb_release.get("distrib_codename", None) + + if not distribution_name: + output = self._RunLSBReleaseCommand(option="-sc") + if output: + distribution_name = output.strip() or None + + return distribution_name + + def _ReadLSBReleaseConfigurationFile(self, path): + """Reads a lsb-release configuration (/etc/lsb-release) file. + + This function ignores comment lines, lines that are not UTF-8 formatted + and lines that do not consist of "key=value". + + Args: + path (str): path of the lsb-release configuration file. + + Returns: + dict[str, str]: key value pairs that are stored in the lsb-release + configuration file. + """ + lsb_release_values = {} + with open(path, "rb") as file_object: + for line in file_object.readlines(): + try: + line = line.decode("utf-8").strip() + except UnicodeDecodeError: + continue + + if not line.startswith("#") and "=" in line: + line = line.lower() + key, _, value = line.partition("=") + lsb_release_values[key.strip()] = value.strip() or None + + return lsb_release_values + + def _RemoveOlderDPKGPackages(self, project_name, project_version): + """Removes previous versions of dpkg packages. + + Args: + project_name (str): project name. + project_version (str): project version. + """ + if self._project_definition.dpkg_source_name: + project_name = self._project_definition.dpkg_source_name + + filenames_to_ignore = re.compile(f"^{project_name:s}[-_].*{project_version!s}") + + # Remove files of previous versions in the format: + # *[-_][0-9]*-[1-9]_.* + for filename in glob.glob( + f"{project_name:s}*[-_][0-9]*-[1-9]_{self.architecture:s}.*" + ): + if not filenames_to_ignore.match(filename): + logging.info(f"Removing: {filename:s}") + try: + os.remove(filename) + except PermissionError as exception: + logging.info( + f"Unable to remove: {filename:s} with error: {exception!s}" + ) + + # Remove files of previous versions in the format: + # [-_][0-9]*-[1-9].* + for filename in glob.glob(f"{project_name:s}[-_][0-9]*-[1-9].*"): + if not filenames_to_ignore.match(filename): + logging.info(f"Removing: {filename:s}") + try: + os.remove(filename) + except PermissionError as exception: + logging.info( + f"Unable to remove: {filename:s} with error: {exception!s}" + ) + + def _RemoveOlderOriginalSourcePackage( + self, project_name, project_version, version_suffix=None, distribution=None + ): + """Removes previous versions of original source package. + + Args: + project_name (str): name of the project. + project_version (str): version of the project. + version_suffix (str): version suffix. + distribution (str): distribution. + """ + if self._project_definition.dpkg_source_name: + project_name = self._project_definition.dpkg_source_name + + filenames_to_ignore = re.compile( + f"^{project_name:s}_{project_version!s}.orig.tar.gz" + ) + + # Remove files of previous versions in the format: + # _[0-9]*.orig.tar.gz + for filename in glob.glob(f"{project_name:s}_[0-9]*.orig.tar.gz"): + if not filenames_to_ignore.match(filename): + logging.info(f"Removing: {filename:s}") + try: + os.remove(filename) + except PermissionError as exception: + logging.info( + f"Unable to remove: {filename:s} with error: {exception!s}" + ) + + # Remove files of previous versions in the format: + # _[0-9]*~.orig.tar.gz + if version_suffix and distribution: + for filename in glob.glob( + f"{project_name:s}_[0-9]*{version_suffix:s}~{distribution:s}" + f".orig.tar.gz" + ): + if not filenames_to_ignore.match(filename): + logging.info(f"Removing: {filename:s}") + try: + os.remove(filename) + except PermissionError as exception: + logging.info( + f"Unable to remove: {filename:s} with error: {exception!s}" + ) + + def _RemoveOlderSourceDPKGPackages(self, project_name, project_version): + """Removes previous versions of source dpkg packages. + + Args: + project_name (str): project name. + project_version (str): project version. + """ + if self._project_definition.dpkg_source_name: + project_name = self._project_definition.dpkg_source_name + + filenames_to_ignore = re.compile(f"^{project_name:s}[-_].*{project_version!s}") + + # Remove files of previous versions in the format: + # [-_][0-9]*-[1-9]~_.* + for filename in glob.glob( + f"{project_name:s}[-_][0-9]*-[1-9]{self.version_suffix:s}~" + f"{self.distribution:s}_{self.architecture:s}.*" + ): + if not filenames_to_ignore.match(filename): + logging.info(f"Removing: {filename:s}") + try: + os.remove(filename) + except PermissionError as exception: + logging.info( + f"Unable to remove: {filename:s} with error: {exception!s}" + ) + + # Remove files of previous versions in the format: + # [-_][0-9]*-[1-9]i~.* + filenames_glob = ( + f"{project_name:s}[-_][0-9]*-[1-9]{self.version_suffix:s}" + f"~{self.distribution:s}.*" + ) + filenames = glob.glob(filenames_glob) + + for filename in filenames: + if not filenames_to_ignore.match(filename): + logging.info(f"Removing: {filename:s}") + try: + os.remove(filename) + except PermissionError as exception: + logging.info( + f"Unable to remove: {filename:s} with error: {exception!s}" + ) + + def _RunLSBReleaseCommand(self, option="-a"): + """Runs the lsb-release command (/usr/bin/lsb_release). + + Args: + option (Optional[str]): option to pass to the lsb-release command. + + Returns: + str: output of the command or None if there is no lsb-release command + available, or running the command failed, or the output cannot be + decoded. + """ + output = None + path = "/usr/bin/lsb_release" + + if os.path.exists(path): + with subprocess.Popen( + [path, option], stderr=subprocess.PIPE, stdout=subprocess.PIPE + ) as process: + + output, _ = process.communicate() + if not process or process.returncode != 0: + output = None + + elif isinstance(output, bytes): + try: + output = output.decode("utf-8") + except UnicodeDecodeError: + output = None + + return output + + def CheckBuildDependencies(self): + """Checks if the build dependencies are met. + + Returns: + list[str]: build dependency names that are not met or an empty list. + """ + missing_packages = [] + for package_name in self._BUILD_DEPENDENCIES: + if not self._CheckIsInstalled(package_name): + missing_packages.append(package_name) + + for name in self._project_definition.build_dependencies: + package_name = self._BUILD_DEPENDENCY_PACKAGE_NAMES.get(name, name) + if package_name not in self._project_definition.dpkg_build_dependencies: + self._project_definition.dpkg_build_dependencies.append(package_name) + + for package_name in self._project_definition.dpkg_build_dependencies: + if not self._CheckIsInstalled(package_name): + missing_packages.append(package_name) + + return missing_packages + + def CheckProjectConfiguration(self): + """Checks if the project configuration is correct. + + This functions checks if all build dependencies are defined as dpkg build + dependencies. + + Returns: + bool: True if the project configuration is correct, False otherwise. + """ + result = True + for name in self._project_definition.build_dependencies: + package_name = self._BUILD_DEPENDENCY_PACKAGE_NAMES.get(name, name) + if package_name not in self._project_definition.dpkg_build_dependencies: + logging.warning( + f"Build dependency: {name:s} not defined as dpkg build dependency: " + f"{package_name:s}" + ) + result = False + + return result class ConfigureMakeDPKGBuildHelper(DPKGBuildHelper): - """Helper to build dpkg packages (.deb).""" - - def __init__( - self, project_definition, l2tdevtools_path, dependency_definitions): - """Initializes a build helper. - - Args: - project_definition (ProjectDefinition): definition of the project - to build. - l2tdevtools_path (str): path to the l2tdevtools directory. - dependency_definitions (dict[str, ProjectDefinition]): definitions of all - projects, which is used to determine the properties of dependencies. - """ - super().__init__( - project_definition, l2tdevtools_path, dependency_definitions) - self.architecture = platform.machine() - self.distribution = '' - self.version_suffix = '' - - if self.architecture == 'i686': - self.architecture = 'i386' - elif self.architecture == 'x86_64': - self.architecture = 'amd64' - - def Build(self, source_helper_object): - """Builds the dpkg packages. - - Args: - source_helper_object (SourceHelper): source helper. - - Returns: - bool: True if successful, False otherwise. - """ - project_name = source_helper_object.project_name - - source_package_path = source_helper_object.GetSourcePackagePath() - if not source_package_path: - logging.info(f'Missing source package of: {project_name:s}') - return False - - source_directory = source_helper_object.GetSourceDirectoryPath() - if not source_directory: - logging.info(f'Missing source directory of: {project_name:s}') - return False - - project_version = source_helper_object.GetProjectVersion() - - # dpkg-buildpackage wants an source package filename without - # the status indication and orig indication. - self._CreateOriginalSourcePackage( - source_package_path, project_name, project_version) - - source_package_filename = source_helper_object.GetSourcePackageFilename() - logging.info(f'Building deb of: {source_package_filename:s}') - - if not self._CreatePackagingFiles(source_directory, project_version): - return False - - # If there is a temporary packaging directory remove it. - temporary_directory = os.path.join(source_directory, 'tmp') - if os.path.exists(temporary_directory): - logging.info(f'Removing: {temporary_directory:s}') - shutil.rmtree(temporary_directory, ignore_errors=True) - - if not self._BuildPrepare( - source_directory, project_name, project_version, self.version_suffix, - self.distribution, self.architecture): - return False - - log_file_path = os.path.join('..', self.LOG_FILENAME) - command = f'dpkg-buildpackage -uc -us -rfakeroot > {log_file_path:s} 2>&1' - exit_code = subprocess.call( - f'(cd {source_directory:s} && {command:s})', shell=True) - if exit_code != 0: - logging.error(f'Running: "{command:s}" failed.') - return False - - if not self._BuildFinalize( - source_directory, project_name, project_version, self.version_suffix, - self.distribution, self.architecture): - return False - - return True - - def CheckBuildRequired(self, source_helper_object): - """Checks if a build is required. - - Args: - source_helper_object (SourceHelper): source helper. - - Returns: - bool: True if a build is required, False otherwise. - """ - project_name = source_helper_object.project_name - project_version = source_helper_object.GetProjectVersion() - - return not os.path.exists( - f'{project_name:s}_{project_version!s}-1_{self.architecture:s}.deb') - - def Clean(self, source_helper_object): - """Cleans the dpkg packages in the current directory. - - Args: - source_helper_object (SourceHelper): source helper. - """ - project_name = source_helper_object.project_name - project_version = source_helper_object.GetProjectVersion() - - self._RemoveOlderSourceDirectories(project_name, project_version) - self._RemoveOlderSourcePackages(project_name, project_version) - self._RemoveOlderOriginalSourcePackage(project_name, project_version) - self._RemoveOlderDPKGPackages(project_name, project_version) + """Helper to build dpkg packages (.deb).""" + + def __init__(self, project_definition, l2tdevtools_path, dependency_definitions): + """Initializes a build helper. + + Args: + project_definition (ProjectDefinition): definition of the project + to build. + l2tdevtools_path (str): path to the l2tdevtools directory. + dependency_definitions (dict[str, ProjectDefinition]): definitions of all + projects, which is used to determine the properties of dependencies. + """ + super().__init__(project_definition, l2tdevtools_path, dependency_definitions) + self.architecture = platform.machine() + self.distribution = "" + self.version_suffix = "" + + if self.architecture == "i686": + self.architecture = "i386" + elif self.architecture == "x86_64": + self.architecture = "amd64" + + def Build(self, source_helper_object): + """Builds the dpkg packages. + + Args: + source_helper_object (SourceHelper): source helper. + + Returns: + bool: True if successful, False otherwise. + """ + project_name = source_helper_object.project_name + + source_package_path = source_helper_object.GetSourcePackagePath() + if not source_package_path: + logging.info(f"Missing source package of: {project_name:s}") + return False + + source_directory = source_helper_object.GetSourceDirectoryPath() + if not source_directory: + logging.info(f"Missing source directory of: {project_name:s}") + return False + + project_version = source_helper_object.GetProjectVersion() + + # dpkg-buildpackage wants an source package filename without + # the status indication and orig indication. + self._CreateOriginalSourcePackage( + source_package_path, project_name, project_version + ) + + source_package_filename = source_helper_object.GetSourcePackageFilename() + logging.info(f"Building deb of: {source_package_filename:s}") + + if not self._CreatePackagingFiles(source_directory, project_version): + return False + + # If there is a temporary packaging directory remove it. + temporary_directory = os.path.join(source_directory, "tmp") + if os.path.exists(temporary_directory): + logging.info(f"Removing: {temporary_directory:s}") + shutil.rmtree(temporary_directory, ignore_errors=True) + + if not self._BuildPrepare( + source_directory, + project_name, + project_version, + self.version_suffix, + self.distribution, + self.architecture, + ): + return False + + log_file_path = os.path.join("..", self.LOG_FILENAME) + command = f"dpkg-buildpackage -uc -us -rfakeroot > {log_file_path:s} 2>&1" + exit_code = subprocess.call( + f"(cd {source_directory:s} && {command:s})", shell=True + ) + if exit_code != 0: + logging.error(f'Running: "{command:s}" failed.') + return False + + if not self._BuildFinalize( + source_directory, + project_name, + project_version, + self.version_suffix, + self.distribution, + self.architecture, + ): + return False + + return True + + def CheckBuildRequired(self, source_helper_object): + """Checks if a build is required. + + Args: + source_helper_object (SourceHelper): source helper. + + Returns: + bool: True if a build is required, False otherwise. + """ + project_name = source_helper_object.project_name + project_version = source_helper_object.GetProjectVersion() + + return not os.path.exists( + f"{project_name:s}_{project_version!s}-1_{self.architecture:s}.deb" + ) + + def Clean(self, source_helper_object): + """Cleans the dpkg packages in the current directory. + + Args: + source_helper_object (SourceHelper): source helper. + """ + project_name = source_helper_object.project_name + project_version = source_helper_object.GetProjectVersion() + + self._RemoveOlderSourceDirectories(project_name, project_version) + self._RemoveOlderSourcePackages(project_name, project_version) + self._RemoveOlderOriginalSourcePackage(project_name, project_version) + self._RemoveOlderDPKGPackages(project_name, project_version) class ConfigureMakeSourceDPKGBuildHelper(DPKGBuildHelper): - """Helper to build source dpkg packages (.deb).""" - - def __init__( - self, project_definition, l2tdevtools_path, dependency_definitions): - """Initializes a build helper. - - Args: - project_definition (ProjectDefinition): definition of the project - to build. - l2tdevtools_path (str): path to the l2tdevtools directory. - dependency_definitions (dict[str, ProjectDefinition]): definitions of all - projects, which is used to determine the properties of dependencies. - """ - super().__init__( - project_definition, l2tdevtools_path, dependency_definitions) - self._prep_script = 'prep-dpkg-source.sh' - self._post_script = 'post-dpkg-source.sh' - self.architecture = 'source' - self.distribution = definitions.DEFAULT_UBUNTU_DISTRIBUTION - self.version_suffix = 'ppa1' - - def Build(self, source_helper_object): - """Builds the dpkg packages. - - Args: - source_helper_object (SourceHelper): source helper. - - Returns: - bool: True if successful, False otherwise. - """ - project_name = source_helper_object.project_name - - source_package_path = source_helper_object.GetSourcePackagePath() - if not source_package_path: - logging.info(f'Missing source package of: {project_name:s}') - return False - - source_directory = source_helper_object.GetSourceDirectoryPath() - if not source_directory: - logging.info(f'Missing source directory of: {project_name:s}') - return False - - project_version = source_helper_object.GetProjectVersion() - - self._CreateOriginalSourcePackage( - source_package_path, project_name, project_version) - - source_package_filename = source_helper_object.GetSourcePackageFilename() - logging.info( - f'Building source deb of: {source_package_filename:s} for: ' - f'{self.distribution:s}') - - if not self._CreatePackagingFiles(source_directory, project_version): - return False - - # If there is a temporary packaging directory remove it. - temporary_directory = os.path.join(source_directory, 'tmp') - if os.path.exists(temporary_directory): - logging.info(f'Removing: {temporary_directory:s}') - shutil.rmtree(temporary_directory, ignore_errors=True) - - if not self._BuildPrepare( - source_directory, project_name, project_version, self.version_suffix, - self.distribution, self.architecture): - return False - - log_file_path = os.path.join('..', self.LOG_FILENAME) - command = f'debuild -S -sa > {log_file_path:s} 2>&1' - exit_code = subprocess.call( - f'(cd {source_directory:s} && {command:s})', shell=True) - if exit_code != 0: - logging.error( - f'Failed to run: "(cd {source_directory:s} && {command:s})" with ' - f'exit code {exit_code:d}') - return False - - if not self._BuildFinalize( - source_directory, project_name, project_version, self.version_suffix, - self.distribution, self.architecture): - return False - - return True - - def CheckBuildRequired(self, source_helper_object): - """Checks if a build is required. - - Args: - source_helper_object (SourceHelper): source helper. - - Returns: - bool: True if a build is required, False otherwise. - """ - project_name = source_helper_object.project_name - project_version = source_helper_object.GetProjectVersion() - - return not os.path.exists( - f'{project_name:s}_{project_version!s}-1{self.version_suffix:s}' - f'~{self.distribution:s}_{self.architecture:s}.changes') - - def Clean(self, source_helper_object): - """Cleans the source dpkg packages in the current directory. - - Args: - source_helper_object (SourceHelper): source helper. - """ - project_name = source_helper_object.project_name - project_version = source_helper_object.GetProjectVersion() - - self._RemoveOlderSourceDirectories(project_name, project_version) - self._RemoveOlderSourcePackages(project_name, project_version) - self._RemoveOlderOriginalSourcePackage(project_name, project_version) - self._RemoveOlderSourceDPKGPackages(project_name, project_version) + """Helper to build source dpkg packages (.deb).""" + + def __init__(self, project_definition, l2tdevtools_path, dependency_definitions): + """Initializes a build helper. + + Args: + project_definition (ProjectDefinition): definition of the project + to build. + l2tdevtools_path (str): path to the l2tdevtools directory. + dependency_definitions (dict[str, ProjectDefinition]): definitions of all + projects, which is used to determine the properties of dependencies. + """ + super().__init__(project_definition, l2tdevtools_path, dependency_definitions) + self._prep_script = "prep-dpkg-source.sh" + self._post_script = "post-dpkg-source.sh" + self.architecture = "source" + self.distribution = definitions.DEFAULT_UBUNTU_DISTRIBUTION + self.version_suffix = "ppa1" + + def Build(self, source_helper_object): + """Builds the dpkg packages. + + Args: + source_helper_object (SourceHelper): source helper. + + Returns: + bool: True if successful, False otherwise. + """ + project_name = source_helper_object.project_name + + source_package_path = source_helper_object.GetSourcePackagePath() + if not source_package_path: + logging.info(f"Missing source package of: {project_name:s}") + return False + + source_directory = source_helper_object.GetSourceDirectoryPath() + if not source_directory: + logging.info(f"Missing source directory of: {project_name:s}") + return False + + project_version = source_helper_object.GetProjectVersion() + + self._CreateOriginalSourcePackage( + source_package_path, project_name, project_version + ) + + source_package_filename = source_helper_object.GetSourcePackageFilename() + logging.info( + f"Building source deb of: {source_package_filename:s} for: " + f"{self.distribution:s}" + ) + + if not self._CreatePackagingFiles(source_directory, project_version): + return False + + # If there is a temporary packaging directory remove it. + temporary_directory = os.path.join(source_directory, "tmp") + if os.path.exists(temporary_directory): + logging.info(f"Removing: {temporary_directory:s}") + shutil.rmtree(temporary_directory, ignore_errors=True) + + if not self._BuildPrepare( + source_directory, + project_name, + project_version, + self.version_suffix, + self.distribution, + self.architecture, + ): + return False + + log_file_path = os.path.join("..", self.LOG_FILENAME) + command = f"debuild -S -sa > {log_file_path:s} 2>&1" + exit_code = subprocess.call( + f"(cd {source_directory:s} && {command:s})", shell=True + ) + if exit_code != 0: + logging.error( + f'Failed to run: "(cd {source_directory:s} && {command:s})" with ' + f"exit code {exit_code:d}" + ) + return False + + if not self._BuildFinalize( + source_directory, + project_name, + project_version, + self.version_suffix, + self.distribution, + self.architecture, + ): + return False + + return True + + def CheckBuildRequired(self, source_helper_object): + """Checks if a build is required. + + Args: + source_helper_object (SourceHelper): source helper. + + Returns: + bool: True if a build is required, False otherwise. + """ + project_name = source_helper_object.project_name + project_version = source_helper_object.GetProjectVersion() + + return not os.path.exists( + f"{project_name:s}_{project_version!s}-1{self.version_suffix:s}" + f"~{self.distribution:s}_{self.architecture:s}.changes" + ) + + def Clean(self, source_helper_object): + """Cleans the source dpkg packages in the current directory. + + Args: + source_helper_object (SourceHelper): source helper. + """ + project_name = source_helper_object.project_name + project_version = source_helper_object.GetProjectVersion() + + self._RemoveOlderSourceDirectories(project_name, project_version) + self._RemoveOlderSourcePackages(project_name, project_version) + self._RemoveOlderOriginalSourcePackage(project_name, project_version) + self._RemoveOlderSourceDPKGPackages(project_name, project_version) class PybuildDPKGBuildHelperBase(DPKGBuildHelper): - """Shared functionality for dh-pybuild build system dpkg build helpers.""" + """Shared functionality for dh-pybuild build system dpkg build helpers.""" - def _DetermineBuildConfiguration(self, source_directory): - """Determines the build configuration of a project that has setup.py. + def _DetermineBuildConfiguration(self, source_directory): + """Determines the build configuration of a project that has setup.py. - Args: - source_directory (str): path of the source directory. + Args: + source_directory (str): path of the source directory. - Returns: - DPKGBuildConfiguration: dpkg build configuration or None if the build - configuration could not be determined. - """ - if os.path.isfile(os.path.join(source_directory, 'pyproject.toml')): - command = ( - f'{sys.executable:s} -m pip install --no-deps -t installroot . ' - f'> /dev/null 2>&1') + Returns: + DPKGBuildConfiguration: dpkg build configuration or None if the build + configuration could not be determined. + """ + if os.path.isfile(os.path.join(source_directory, "pyproject.toml")): + command = ( + f"{sys.executable:s} -m pip install --no-deps -t installroot . " + f"> /dev/null 2>&1" + ) - elif os.path.isfile(os.path.join(source_directory, 'setup.py')): - command = ( - f'{sys.executable:s} setup.py install --root=installroot ' - f'> /dev/null 2>&1') + elif os.path.isfile(os.path.join(source_directory, "setup.py")): + command = ( + f"{sys.executable:s} setup.py install --root=installroot " + f"> /dev/null 2>&1" + ) - else: - return None + else: + return None - installroot_path = os.path.join(source_directory, 'installroot') + installroot_path = os.path.join(source_directory, "installroot") - exit_code = subprocess.call( - f'(cd {source_directory:s} && {command:s})', shell=True) - if exit_code != 0: - logging.error(f'Running: "{command:s}" failed.') - build_configuration = None + exit_code = subprocess.call( + f"(cd {source_directory:s} && {command:s})", shell=True + ) + if exit_code != 0: + logging.error(f'Running: "{command:s}" failed.') + build_configuration = None - else: - build_configuration = dpkg_files.DPKGBuildConfiguration() + else: + build_configuration = dpkg_files.DPKGBuildConfiguration() - if os.path.exists(os.path.join(installroot_path, 'usr', 'bin')): - build_configuration.has_bin_directory = True + if os.path.exists(os.path.join(installroot_path, "usr", "bin")): + build_configuration.has_bin_directory = True - dist_packages = os.path.join( - installroot_path, 'usr', 'local', 'lib', 'python3.*', 'dist-packages') - dist_packages = glob.glob(dist_packages) - if dist_packages: - dist_packages = dist_packages[0] + dist_packages = os.path.join( + installroot_path, "usr", "local", "lib", "python3.*", "dist-packages" + ) + dist_packages = glob.glob(dist_packages) + if dist_packages: + dist_packages = dist_packages[0] - for directory_entry in os.listdir(dist_packages): - directory_entry_path = os.path.join(dist_packages, directory_entry) + for directory_entry in os.listdir(dist_packages): + directory_entry_path = os.path.join(dist_packages, directory_entry) - if directory_entry.endswith('.dist-info'): - # pylint: disable=simplifiable-if-statement - if os.path.isdir(directory_entry_path): - build_configuration.has_dist_info_directory = True - else: - build_configuration.has_dist_info_file = True + if directory_entry.endswith(".dist-info"): + # pylint: disable=simplifiable-if-statement + if os.path.isdir(directory_entry_path): + build_configuration.has_dist_info_directory = True + else: + build_configuration.has_dist_info_file = True - elif directory_entry.endswith('.egg-info'): - # pylint: disable=simplifiable-if-statement - if os.path.isdir(directory_entry_path): - build_configuration.has_egg_info_directory = True - else: - build_configuration.has_egg_info_file = True + elif directory_entry.endswith(".egg-info"): + # pylint: disable=simplifiable-if-statement + if os.path.isdir(directory_entry_path): + build_configuration.has_egg_info_directory = True + else: + build_configuration.has_egg_info_file = True - elif os.path.isdir(directory_entry_path): - if directory_entry != '__pycache__': - build_configuration.module_directories.append(directory_entry) + elif os.path.isdir(directory_entry_path): + if directory_entry != "__pycache__": + build_configuration.module_directories.append( + directory_entry + ) - # TODO: determine depth of module directories. + # TODO: determine depth of module directories. - elif directory_entry.endswith('.py'): - build_configuration.has_module_source_files = True + elif directory_entry.endswith(".py"): + build_configuration.has_module_source_files = True - elif directory_entry.endswith('.so'): - build_configuration.has_module_shared_object = True + elif directory_entry.endswith(".so"): + build_configuration.has_module_shared_object = True - bin_scripts = os.path.join( - installroot_path, 'usr', 'local', 'bin', '*.py') - bin_scripts = glob.glob(bin_scripts) - if bin_scripts: - build_configuration.has_bin_directory = True + bin_scripts = os.path.join(installroot_path, "usr", "local", "bin", "*.py") + bin_scripts = glob.glob(bin_scripts) + if bin_scripts: + build_configuration.has_bin_directory = True - if os.path.exists(installroot_path): - shutil.rmtree(installroot_path, ignore_errors=True) + if os.path.exists(installroot_path): + shutil.rmtree(installroot_path, ignore_errors=True) - return build_configuration + return build_configuration class PybuildDPKGBuildHelper(PybuildDPKGBuildHelperBase): - """Helper to build dpkg packages (.deb) using dh-pybuild build system.""" - - def __init__( - self, project_definition, l2tdevtools_path, dependency_definitions): - """Initializes a build helper. - - Args: - project_definition (ProjectDefinition): definition of the project - to build. - l2tdevtools_path (str): path to the l2tdevtools directory. - dependency_definitions (dict[str, ProjectDefinition]): definitions of all - projects, which is used to determine the properties of dependencies. - """ - super().__init__( - project_definition, l2tdevtools_path, dependency_definitions) - self.architecture = platform.machine() - self.distribution = '' - self.version_suffix = '' - - if not project_definition.architecture_dependent: - self.architecture = 'all' - elif self.architecture == 'i686': - self.architecture = 'i386' - elif self.architecture == 'x86_64': - self.architecture = 'amd64' - - def _GetFilenameSafeProjectInformation(self, source_helper_object): - """Determines the filename safe project name and version. - - Args: - source_helper_object (SourceHelper): source helper. - - Returns: - tuple: containing: - - * str: filename safe project name. - * str: version. - """ - if self._project_definition.dpkg_name: - project_name = self._project_definition.dpkg_name - else: - project_name = source_helper_object.project_name - if not project_name.startswith('python3-'): - project_name = f'python3-{project_name:s}' - - project_version = source_helper_object.GetProjectVersion() - if project_version and project_version.startswith('1!'): - # Remove setuptools epoch. - project_version = project_version[2:] - - return project_name, project_version - - def Build(self, source_helper_object): - """Builds the dpkg packages. - - Args: - source_helper_object (SourceHelper): source helper. - - Returns: - bool: True if successful, False otherwise. - """ - project_name = source_helper_object.project_name - - source_package_path = source_helper_object.GetSourcePackagePath() - if not source_package_path: - logging.info(f'Missing source package of: {project_name:s}') - return False - - source_directory = source_helper_object.GetSourceDirectoryPath() - if not source_directory: - logging.info(f'Missing source directory of: {project_name:s}') - return False - - project_name, project_version = self._GetFilenameSafeProjectInformation( - source_helper_object) - - # dpkg-buildpackage wants an source package filename without - # the status indication and orig indication. - - # Note that we need to pass the original project name to - # _CreateOriginalSourcePackage. - self._CreateOriginalSourcePackage( - source_package_path, source_helper_object.project_name, project_version) - - source_package_filename = source_helper_object.GetSourcePackageFilename() - logging.info(f'Building deb of: {source_package_filename:s}') - - if not self._CreatePackagingFiles(source_directory, project_version): - return False - - # If there is a temporary packaging directory remove it. - temporary_directory = os.path.join(source_directory, 'tmp') - if os.path.exists(temporary_directory): - logging.info(f'Removing: {temporary_directory:s}') - shutil.rmtree(temporary_directory, ignore_errors=True) - - if not self._BuildPrepare( - source_directory, project_name, project_version, self.version_suffix, - self.distribution, self.architecture): - return False - - log_file_path = os.path.join('..', self.LOG_FILENAME) - command = f'dpkg-buildpackage -uc -us -rfakeroot > {log_file_path:s} 2>&1' - exit_code = subprocess.call( - f'(cd {source_directory:s} && {command:s})', shell=True) - if exit_code != 0: - logging.error( - f'Failed to run: "(cd {source_directory:s} && {command:s})" with ' - f'exit code {exit_code:d}') - return False - - if not self._BuildFinalize( - source_directory, project_name, project_version, self.version_suffix, - self.distribution, self.architecture): - return False - - return True - - def CheckBuildRequired(self, source_helper_object): - """Checks if a build is required. - - Args: - source_helper_object (SourceHelper): source helper. - - Returns: - bool: True if a build is required, False otherwise. - """ - project_name, project_version = self._GetFilenameSafeProjectInformation( - source_helper_object) - - return not os.path.exists( - f'{project_name:s}_{project_version!s}-1_{self.architecture:s}.deb') - - def Clean(self, source_helper_object): - """Cleans the dpkg packages in the current directory. - - Args: - source_helper_object (SourceHelper): source helper. - """ - project_name, project_version = self._GetFilenameSafeProjectInformation( - source_helper_object) - - self._RemoveOlderSourceDirectories( - source_helper_object.project_name, project_version) - self._RemoveOlderSourcePackages( - source_helper_object.project_name, project_version) - - self._RemoveOlderOriginalSourcePackage( - source_helper_object.project_name, project_version) - - self._RemoveOlderDPKGPackages(project_name, project_version) - - if project_name.startswith('python-'): - project_name = f'python3-{project_name[7:]:s}' - self._RemoveOlderDPKGPackages(project_name, project_version) - - elif (project_name.startswith('python2-') or - project_name.startswith('python3-')): - project_name = f'python3-{project_name[8:]:s}' - self._RemoveOlderDPKGPackages(project_name, project_version) + """Helper to build dpkg packages (.deb) using dh-pybuild build system.""" + + def __init__(self, project_definition, l2tdevtools_path, dependency_definitions): + """Initializes a build helper. + + Args: + project_definition (ProjectDefinition): definition of the project + to build. + l2tdevtools_path (str): path to the l2tdevtools directory. + dependency_definitions (dict[str, ProjectDefinition]): definitions of all + projects, which is used to determine the properties of dependencies. + """ + super().__init__(project_definition, l2tdevtools_path, dependency_definitions) + self.architecture = platform.machine() + self.distribution = "" + self.version_suffix = "" + + if not project_definition.architecture_dependent: + self.architecture = "all" + elif self.architecture == "i686": + self.architecture = "i386" + elif self.architecture == "x86_64": + self.architecture = "amd64" + + def _GetFilenameSafeProjectInformation(self, source_helper_object): + """Determines the filename safe project name and version. + + Args: + source_helper_object (SourceHelper): source helper. + + Returns: + tuple: containing: + + * str: filename safe project name. + * str: version. + """ + if self._project_definition.dpkg_name: + project_name = self._project_definition.dpkg_name + else: + project_name = source_helper_object.project_name + if not project_name.startswith("python3-"): + project_name = f"python3-{project_name:s}" + + project_version = source_helper_object.GetProjectVersion() + if project_version and project_version.startswith("1!"): + # Remove setuptools epoch. + project_version = project_version[2:] + + return project_name, project_version + + def Build(self, source_helper_object): + """Builds the dpkg packages. + + Args: + source_helper_object (SourceHelper): source helper. + + Returns: + bool: True if successful, False otherwise. + """ + project_name = source_helper_object.project_name + + source_package_path = source_helper_object.GetSourcePackagePath() + if not source_package_path: + logging.info(f"Missing source package of: {project_name:s}") + return False + + source_directory = source_helper_object.GetSourceDirectoryPath() + if not source_directory: + logging.info(f"Missing source directory of: {project_name:s}") + return False + + project_name, project_version = self._GetFilenameSafeProjectInformation( + source_helper_object + ) + + # dpkg-buildpackage wants an source package filename without + # the status indication and orig indication. + + # Note that we need to pass the original project name to + # _CreateOriginalSourcePackage. + self._CreateOriginalSourcePackage( + source_package_path, source_helper_object.project_name, project_version + ) + + source_package_filename = source_helper_object.GetSourcePackageFilename() + logging.info(f"Building deb of: {source_package_filename:s}") + + if not self._CreatePackagingFiles(source_directory, project_version): + return False + + # If there is a temporary packaging directory remove it. + temporary_directory = os.path.join(source_directory, "tmp") + if os.path.exists(temporary_directory): + logging.info(f"Removing: {temporary_directory:s}") + shutil.rmtree(temporary_directory, ignore_errors=True) + + if not self._BuildPrepare( + source_directory, + project_name, + project_version, + self.version_suffix, + self.distribution, + self.architecture, + ): + return False + + log_file_path = os.path.join("..", self.LOG_FILENAME) + command = f"dpkg-buildpackage -uc -us -rfakeroot > {log_file_path:s} 2>&1" + exit_code = subprocess.call( + f"(cd {source_directory:s} && {command:s})", shell=True + ) + if exit_code != 0: + logging.error( + f'Failed to run: "(cd {source_directory:s} && {command:s})" with ' + f"exit code {exit_code:d}" + ) + return False + + if not self._BuildFinalize( + source_directory, + project_name, + project_version, + self.version_suffix, + self.distribution, + self.architecture, + ): + return False + + return True + + def CheckBuildRequired(self, source_helper_object): + """Checks if a build is required. + + Args: + source_helper_object (SourceHelper): source helper. + + Returns: + bool: True if a build is required, False otherwise. + """ + project_name, project_version = self._GetFilenameSafeProjectInformation( + source_helper_object + ) + + return not os.path.exists( + f"{project_name:s}_{project_version!s}-1_{self.architecture:s}.deb" + ) + + def Clean(self, source_helper_object): + """Cleans the dpkg packages in the current directory. + + Args: + source_helper_object (SourceHelper): source helper. + """ + project_name, project_version = self._GetFilenameSafeProjectInformation( + source_helper_object + ) + + self._RemoveOlderSourceDirectories( + source_helper_object.project_name, project_version + ) + self._RemoveOlderSourcePackages( + source_helper_object.project_name, project_version + ) + + self._RemoveOlderOriginalSourcePackage( + source_helper_object.project_name, project_version + ) + + self._RemoveOlderDPKGPackages(project_name, project_version) + + if project_name.startswith("python-"): + project_name = f"python3-{project_name[7:]:s}" + self._RemoveOlderDPKGPackages(project_name, project_version) + + elif project_name.startswith("python2-") or project_name.startswith("python3-"): + project_name = f"python3-{project_name[8:]:s}" + self._RemoveOlderDPKGPackages(project_name, project_version) class PybuildSourceDPKGBuildHelper(PybuildDPKGBuildHelperBase): - """Helper to build source dpkg packages using setup.py build system.""" - - def __init__( - self, project_definition, l2tdevtools_path, dependency_definitions): - """Initializes a build helper. - - Args: - project_definition (ProjectDefinition): definition of the project - to build. - l2tdevtools_path (str): path to the l2tdevtools directory. - dependency_definitions (dict[str, ProjectDefinition]): definitions of all - projects, which is used to determine the properties of dependencies. - """ - super().__init__( - project_definition, l2tdevtools_path, dependency_definitions) - self._prep_script = 'prep-dpkg-source.sh' - self._post_script = 'post-dpkg-source.sh' - self.architecture = 'source' - self.distribution = definitions.DEFAULT_UBUNTU_DISTRIBUTION - self.version_suffix = 'ppa1' - - def _GetFilenameSafeProjectInformation(self, source_helper_object): - """Determines the filename safe project name and version. - - Args: - source_helper_object (SourceHelper): source helper. - - Returns: - tuple: containing: - - * str: filename safe project name. - * str: version. - """ - if self._project_definition.dpkg_source_name: - project_name = self._project_definition.dpkg_source_name - else: - project_name = source_helper_object.project_name - if not project_name.startswith('python3-'): - project_name = f'python3-{project_name:s}' - - project_version = source_helper_object.GetProjectVersion() - if project_version and project_version.startswith('1!'): - # Remove setuptools epoch. - project_version = project_version[2:] - - return project_name, project_version - - def Build(self, source_helper_object): - """Builds the dpkg packages. - - Args: - source_helper_object (SourceHelper): source helper. - - Returns: - bool: True if successful, False otherwise. - """ - project_name = source_helper_object.project_name - - source_package_path = source_helper_object.GetSourcePackagePath() - if not source_package_path: - logging.info(f'Missing source package of: {project_name:s}') - return False - - source_directory = source_helper_object.GetSourceDirectoryPath() - if not source_directory: - logging.info(f'Missing source directory of: {project_name:s}') - return False - - project_name, project_version = self._GetFilenameSafeProjectInformation( - source_helper_object) - - # Note that we need to pass the original project name to - # _CreateOriginalSourcePackage. - self._CreateOriginalSourcePackage( - source_package_path, source_helper_object.project_name, project_version) - - source_package_filename = source_helper_object.GetSourcePackageFilename() - logging.info( - f'Building source deb of: {source_package_filename:s} for: ' - f'{self.distribution:s}') - - if not self._CreatePackagingFiles(source_directory, project_version): - return False - - # If there is a temporary packaging directory remove it. - temporary_directory = os.path.join(source_directory, 'tmp') - if os.path.exists(temporary_directory): - logging.info(f'Removing: {temporary_directory:s}') - shutil.rmtree(temporary_directory, ignore_errors=True) - - if not self._BuildPrepare( - source_directory, project_name, project_version, self.version_suffix, - self.distribution, self.architecture): - return False - - log_file_path = os.path.join('..', self.LOG_FILENAME) - command = f'debuild -S -sa > {log_file_path:s} 2>&1' - exit_code = subprocess.call( - f'(cd {source_directory:s} && {command:s})', shell=True) - if exit_code != 0: - logging.error( - f'Failed to run: "(cd {source_directory:s} && {command:s})" with ' - f'exit code {exit_code:d}') - - if not self._BuildFinalize( - source_directory, project_name, project_version, self.version_suffix, - self.distribution, self.architecture): - return False - - return True - - def CheckBuildRequired(self, source_helper_object): - """Checks if a build is required. - - Args: - source_helper_object (SourceHelper): source helper. - - Returns: - bool: True if a build is required, False otherwise. - """ - project_name, project_version = self._GetFilenameSafeProjectInformation( - source_helper_object) - - return not os.path.exists( - f'{project_name:s}_{project_version!s}-1{self.version_suffix:s}' - f'~{self.distribution:s}_{self.architecture:s}.changes') - - def Clean(self, source_helper_object): - """Cleans the dpkg packages in the current directory. - - Args: - source_helper_object (SourceHelper): source helper. - """ - project_version = source_helper_object.GetProjectVersion() - - self._RemoveOlderSourceDirectories( - source_helper_object.project_name, project_version) - self._RemoveOlderSourcePackages( - source_helper_object.project_name, project_version) - - self._RemoveOlderOriginalSourcePackage( - source_helper_object.project_name, project_version, - version_suffix=self.version_suffix, distribution=self.distribution) - - self._RemoveOlderSourceDPKGPackages( - source_helper_object.project_name, project_version) + """Helper to build source dpkg packages using setup.py build system.""" + + def __init__(self, project_definition, l2tdevtools_path, dependency_definitions): + """Initializes a build helper. + + Args: + project_definition (ProjectDefinition): definition of the project + to build. + l2tdevtools_path (str): path to the l2tdevtools directory. + dependency_definitions (dict[str, ProjectDefinition]): definitions of all + projects, which is used to determine the properties of dependencies. + """ + super().__init__(project_definition, l2tdevtools_path, dependency_definitions) + self._prep_script = "prep-dpkg-source.sh" + self._post_script = "post-dpkg-source.sh" + self.architecture = "source" + self.distribution = definitions.DEFAULT_UBUNTU_DISTRIBUTION + self.version_suffix = "ppa1" + + def _GetFilenameSafeProjectInformation(self, source_helper_object): + """Determines the filename safe project name and version. + + Args: + source_helper_object (SourceHelper): source helper. + + Returns: + tuple: containing: + + * str: filename safe project name. + * str: version. + """ + if self._project_definition.dpkg_source_name: + project_name = self._project_definition.dpkg_source_name + else: + project_name = source_helper_object.project_name + if not project_name.startswith("python3-"): + project_name = f"python3-{project_name:s}" + + project_version = source_helper_object.GetProjectVersion() + if project_version and project_version.startswith("1!"): + # Remove setuptools epoch. + project_version = project_version[2:] + + return project_name, project_version + + def Build(self, source_helper_object): + """Builds the dpkg packages. + + Args: + source_helper_object (SourceHelper): source helper. + + Returns: + bool: True if successful, False otherwise. + """ + project_name = source_helper_object.project_name + + source_package_path = source_helper_object.GetSourcePackagePath() + if not source_package_path: + logging.info(f"Missing source package of: {project_name:s}") + return False + + source_directory = source_helper_object.GetSourceDirectoryPath() + if not source_directory: + logging.info(f"Missing source directory of: {project_name:s}") + return False + + project_name, project_version = self._GetFilenameSafeProjectInformation( + source_helper_object + ) + + # Note that we need to pass the original project name to + # _CreateOriginalSourcePackage. + self._CreateOriginalSourcePackage( + source_package_path, source_helper_object.project_name, project_version + ) + + source_package_filename = source_helper_object.GetSourcePackageFilename() + logging.info( + f"Building source deb of: {source_package_filename:s} for: " + f"{self.distribution:s}" + ) + + if not self._CreatePackagingFiles(source_directory, project_version): + return False + + # If there is a temporary packaging directory remove it. + temporary_directory = os.path.join(source_directory, "tmp") + if os.path.exists(temporary_directory): + logging.info(f"Removing: {temporary_directory:s}") + shutil.rmtree(temporary_directory, ignore_errors=True) + + if not self._BuildPrepare( + source_directory, + project_name, + project_version, + self.version_suffix, + self.distribution, + self.architecture, + ): + return False + + log_file_path = os.path.join("..", self.LOG_FILENAME) + command = f"debuild -S -sa > {log_file_path:s} 2>&1" + exit_code = subprocess.call( + f"(cd {source_directory:s} && {command:s})", shell=True + ) + if exit_code != 0: + logging.error( + f'Failed to run: "(cd {source_directory:s} && {command:s})" with ' + f"exit code {exit_code:d}" + ) + + if not self._BuildFinalize( + source_directory, + project_name, + project_version, + self.version_suffix, + self.distribution, + self.architecture, + ): + return False + + return True + + def CheckBuildRequired(self, source_helper_object): + """Checks if a build is required. + + Args: + source_helper_object (SourceHelper): source helper. + + Returns: + bool: True if a build is required, False otherwise. + """ + project_name, project_version = self._GetFilenameSafeProjectInformation( + source_helper_object + ) + + return not os.path.exists( + f"{project_name:s}_{project_version!s}-1{self.version_suffix:s}" + f"~{self.distribution:s}_{self.architecture:s}.changes" + ) + + def Clean(self, source_helper_object): + """Cleans the dpkg packages in the current directory. + + Args: + source_helper_object (SourceHelper): source helper. + """ + project_version = source_helper_object.GetProjectVersion() + + self._RemoveOlderSourceDirectories( + source_helper_object.project_name, project_version + ) + self._RemoveOlderSourcePackages( + source_helper_object.project_name, project_version + ) + + self._RemoveOlderOriginalSourcePackage( + source_helper_object.project_name, + project_version, + version_suffix=self.version_suffix, + distribution=self.distribution, + ) + + self._RemoveOlderSourceDPKGPackages( + source_helper_object.project_name, project_version + ) diff --git a/l2tdevtools/build_helpers/factory.py b/l2tdevtools/build_helpers/factory.py index 85c00b68..54976e3b 100644 --- a/l2tdevtools/build_helpers/factory.py +++ b/l2tdevtools/build_helpers/factory.py @@ -7,69 +7,76 @@ class BuildHelperFactory: - """Factory class for build helpers.""" + """Factory class for build helpers.""" - _BUILD_HELPER_CLASSES = { - 'dpkg': dpkg.PybuildDPKGBuildHelper, - 'dpkg-source': dpkg.PybuildSourceDPKGBuildHelper, - 'rpm': rpm.PyprojectRPMBuildHelper, - 'srpm': rpm.PyprojectSRPMBuildHelper, - 'wheel': wheel.BuildWheelBuildHelper, - } + _BUILD_HELPER_CLASSES = { + "dpkg": dpkg.PybuildDPKGBuildHelper, + "dpkg-source": dpkg.PybuildSourceDPKGBuildHelper, + "rpm": rpm.PyprojectRPMBuildHelper, + "srpm": rpm.PyprojectSRPMBuildHelper, + "wheel": wheel.BuildWheelBuildHelper, + } - _CONFIGURE_MAKE_BUILD_HELPER_CLASSES = { - 'dpkg': dpkg.ConfigureMakeDPKGBuildHelper, - 'dpkg-source': dpkg.ConfigureMakeSourceDPKGBuildHelper, - 'rpm': rpm.ConfigureMakeRPMBuildHelper, - 'source': source.ConfigureMakeSourceBuildHelper, - 'srpm': rpm.ConfigureMakeSRPMBuildHelper, - 'wheel': wheel.ConfigureMakeWheelBuildHelper, - } + _CONFIGURE_MAKE_BUILD_HELPER_CLASSES = { + "dpkg": dpkg.ConfigureMakeDPKGBuildHelper, + "dpkg-source": dpkg.ConfigureMakeSourceDPKGBuildHelper, + "rpm": rpm.ConfigureMakeRPMBuildHelper, + "source": source.ConfigureMakeSourceBuildHelper, + "srpm": rpm.ConfigureMakeSRPMBuildHelper, + "wheel": wheel.ConfigureMakeWheelBuildHelper, + } - _SETUP_PY_BUILD_HELPER_CLASSES = { - 'dpkg': dpkg.PybuildDPKGBuildHelper, - 'dpkg-source': dpkg.PybuildSourceDPKGBuildHelper, - 'rpm': rpm.PyprojectRPMBuildHelper, - 'source': source.SetupPySourceBuildHelper, - 'srpm': rpm.PyprojectSRPMBuildHelper, - 'wheel': wheel.BuildWheelBuildHelper, - } + _SETUP_PY_BUILD_HELPER_CLASSES = { + "dpkg": dpkg.PybuildDPKGBuildHelper, + "dpkg-source": dpkg.PybuildSourceDPKGBuildHelper, + "rpm": rpm.PyprojectRPMBuildHelper, + "source": source.SetupPySourceBuildHelper, + "srpm": rpm.PyprojectSRPMBuildHelper, + "wheel": wheel.BuildWheelBuildHelper, + } - @classmethod - def NewBuildHelper( - cls, project_definition, build_target, l2tdevtools_path, - dependency_definitions): - """Creates a new build helper. + @classmethod + def NewBuildHelper( + cls, project_definition, build_target, l2tdevtools_path, dependency_definitions + ): + """Creates a new build helper. - Args: - project_definition (ProjectDefinition): definition of the project - to build. - build_target (str): build target. - l2tdevtools_path (str): path to the l2tdevtools directory. - dependency_definitions (dict[str, ProjectDefinition]): definitions of all - projects, which is used to determine the properties of dependencies. + Args: + project_definition (ProjectDefinition): definition of the project + to build. + build_target (str): build target. + l2tdevtools_path (str): path to the l2tdevtools directory. + dependency_definitions (dict[str, ProjectDefinition]): definitions of all + projects, which is used to determine the properties of dependencies. - Returns: - BuildHelper: build helper or None if build system is not supported. - """ - if project_definition.build_system == 'configure_make': - build_helper_class = cls._CONFIGURE_MAKE_BUILD_HELPER_CLASSES.get( - build_target, None) + Returns: + BuildHelper: build helper or None if build system is not supported. + """ + if project_definition.build_system == "configure_make": + build_helper_class = cls._CONFIGURE_MAKE_BUILD_HELPER_CLASSES.get( + build_target, None + ) - elif project_definition.build_system in ( - 'flit', 'hatchling', 'poetry', 'scikit', 'setuptools'): - build_helper_class = cls._BUILD_HELPER_CLASSES.get( - build_target, None) + elif project_definition.build_system in ( + "flit", + "hatchling", + "poetry", + "scikit", + "setuptools", + ): + build_helper_class = cls._BUILD_HELPER_CLASSES.get(build_target, None) - elif project_definition.build_system == 'setup_py': - build_helper_class = cls._SETUP_PY_BUILD_HELPER_CLASSES.get( - build_target, None) + elif project_definition.build_system == "setup_py": + build_helper_class = cls._SETUP_PY_BUILD_HELPER_CLASSES.get( + build_target, None + ) - else: - build_helper_class = None + else: + build_helper_class = None - if not build_helper_class: - return None + if not build_helper_class: + return None - return build_helper_class( - project_definition, l2tdevtools_path, dependency_definitions) + return build_helper_class( + project_definition, l2tdevtools_path, dependency_definitions + ) diff --git a/l2tdevtools/build_helpers/interface.py b/l2tdevtools/build_helpers/interface.py index bbb5343e..9935ae83 100644 --- a/l2tdevtools/build_helpers/interface.py +++ b/l2tdevtools/build_helpers/interface.py @@ -8,103 +8,102 @@ class BuildHelper: - """Helper to build projects from source.""" - - LOG_FILENAME = 'build.log' - - def __init__( - self, project_definition, l2tdevtools_path, dependency_definitions): - """Initializes a build helper. - - Args: - project_definition (ProjectDefinition): definition of the project - to build. - l2tdevtools_path (str): path to the l2tdevtools directory. - dependency_definitions (dict[str, ProjectDefinition]): definitions of all - projects, which is used to determine the properties of dependencies. - """ - super().__init__() - self._data_path = os.path.join(l2tdevtools_path, 'data') - self._dependency_definitions = dependency_definitions - self._project_definition = project_definition - - def _RemoveOlderSourceDirectories(self, project_name, project_version): - """Removes previous versions of source directories. - - Args: - project_name (str): name of the project. - project_version (str): version of the project. - """ - filenames_to_ignore = re.compile(f'^{project_name:s}-.*{project_version!s}') - - # Remove previous versions of source directories in the format: - # -[0-9]* - filenames = glob.glob(f'{project_name:s}-[0-9]*') - for filename in filenames: - if os.path.isdir(filename) and not filenames_to_ignore.match(filename): - logging.info(f'Removing: {filename:s}') - shutil.rmtree(filename, ignore_errors=True) - - def _RemoveOlderSourcePackages(self, project_name, project_version): - """Removes previous versions of source packages. - - Args: - project_name (str): name of the project. - project_version (str): version of the project. - """ - filenames_to_ignore = re.compile(f'^{project_name:s}-.*{project_version!s}') - - # Remove previous versions of source packages in the format: - # -[0-9]*.tar.gz - filenames = glob.glob(f'{project_name:s}-[0-9]*.tar.gz') - for filename in filenames: - if not filenames_to_ignore.match(filename): - logging.info(f'Removing: {filename:s}') - os.remove(filename) - - # Remove previous versions of source packages in the format: - # -[0-9]*.tgz - filenames = glob.glob(f'{project_name:s}-[0-9]*.tgz') - for filename in filenames: - if not filenames_to_ignore.match(filename): - logging.info(f'Removing: {filename:s}') - os.remove(filename) - - # Remove previous versions of source packages in the format: - # -[0-9]*.zip - filenames = glob.glob(f'{project_name:s}-[0-9]*.zip') - for filename in filenames: - if not filenames_to_ignore.match(filename): - logging.info(f'Removing: {filename:s}') - os.remove(filename) - - def CheckBuildDependencies(self): - """Checks if the build dependencies are met. - - Returns: - list[str]: build dependency names that are not met or an empty list. - """ - build_dependencies = self._project_definition.build_dependencies - if not build_dependencies: - build_dependencies = [] - return list(build_dependencies) - - # pylint: disable=unused-argument - def CheckBuildRequired(self, source_helper_object): - """Checks if a build is required. - - Args: - source_helper_object (SourceHelper): source helper. - - Returns: - bool: True if a build is required, False otherwise. - """ - return True - - def CheckProjectConfiguration(self): - """Checks if the project configuration is correct. - - Returns: - bool: True if the project configuration is correct, False otherwise. - """ - return True + """Helper to build projects from source.""" + + LOG_FILENAME = "build.log" + + def __init__(self, project_definition, l2tdevtools_path, dependency_definitions): + """Initializes a build helper. + + Args: + project_definition (ProjectDefinition): definition of the project + to build. + l2tdevtools_path (str): path to the l2tdevtools directory. + dependency_definitions (dict[str, ProjectDefinition]): definitions of all + projects, which is used to determine the properties of dependencies. + """ + super().__init__() + self._data_path = os.path.join(l2tdevtools_path, "data") + self._dependency_definitions = dependency_definitions + self._project_definition = project_definition + + def _RemoveOlderSourceDirectories(self, project_name, project_version): + """Removes previous versions of source directories. + + Args: + project_name (str): name of the project. + project_version (str): version of the project. + """ + filenames_to_ignore = re.compile(f"^{project_name:s}-.*{project_version!s}") + + # Remove previous versions of source directories in the format: + # -[0-9]* + filenames = glob.glob(f"{project_name:s}-[0-9]*") + for filename in filenames: + if os.path.isdir(filename) and not filenames_to_ignore.match(filename): + logging.info(f"Removing: {filename:s}") + shutil.rmtree(filename, ignore_errors=True) + + def _RemoveOlderSourcePackages(self, project_name, project_version): + """Removes previous versions of source packages. + + Args: + project_name (str): name of the project. + project_version (str): version of the project. + """ + filenames_to_ignore = re.compile(f"^{project_name:s}-.*{project_version!s}") + + # Remove previous versions of source packages in the format: + # -[0-9]*.tar.gz + filenames = glob.glob(f"{project_name:s}-[0-9]*.tar.gz") + for filename in filenames: + if not filenames_to_ignore.match(filename): + logging.info(f"Removing: {filename:s}") + os.remove(filename) + + # Remove previous versions of source packages in the format: + # -[0-9]*.tgz + filenames = glob.glob(f"{project_name:s}-[0-9]*.tgz") + for filename in filenames: + if not filenames_to_ignore.match(filename): + logging.info(f"Removing: {filename:s}") + os.remove(filename) + + # Remove previous versions of source packages in the format: + # -[0-9]*.zip + filenames = glob.glob(f"{project_name:s}-[0-9]*.zip") + for filename in filenames: + if not filenames_to_ignore.match(filename): + logging.info(f"Removing: {filename:s}") + os.remove(filename) + + def CheckBuildDependencies(self): + """Checks if the build dependencies are met. + + Returns: + list[str]: build dependency names that are not met or an empty list. + """ + build_dependencies = self._project_definition.build_dependencies + if not build_dependencies: + build_dependencies = [] + return list(build_dependencies) + + # pylint: disable=unused-argument + def CheckBuildRequired(self, source_helper_object): + """Checks if a build is required. + + Args: + source_helper_object (SourceHelper): source helper. + + Returns: + bool: True if a build is required, False otherwise. + """ + return True + + def CheckProjectConfiguration(self): + """Checks if the project configuration is correct. + + Returns: + bool: True if the project configuration is correct, False otherwise. + """ + return True diff --git a/l2tdevtools/build_helpers/rpm.py b/l2tdevtools/build_helpers/rpm.py index 12f7ebdf..edf59be0 100644 --- a/l2tdevtools/build_helpers/rpm.py +++ b/l2tdevtools/build_helpers/rpm.py @@ -13,752 +13,812 @@ class BaseRPMBuildHelper(interface.BuildHelper): - """Helper to build RPM packages (.rpm).""" - - _BUILD_DEPENDENCIES = frozenset([ - 'git', - 'binutils', - 'autoconf', - 'automake', - 'libtool', - 'gettext-devel', - 'make', - 'pkgconf', - 'gcc', - 'gcc-c++', - 'flex', - 'byacc', - 'rpm-build', - 'python3-dateutil', - 'python3-devel', - 'python3-setuptools', - 'python3-test', - ]) - - _BUILD_DEPENDENCY_PACKAGE_NAMES = { - 'bzip2': ['bzip2-devel'], - 'fuse': ['fuse3-devel'], - 'libcrypto': ['openssl-devel'], - 'liblzma': ['xz-devel'], - 'pytest-runner': ['python3-pytest-runner'], - 'sqlite': ['sqlite-devel'], - 'zeromq': ['libzmq3-devel'], - 'zlib': ['zlib-ng-devel'] - } - - def __init__( - self, project_definition, l2tdevtools_path, dependency_definitions): - """Initializes a build helper. - - Args: - project_definition (ProjectDefinition): definition of the project - to build. - l2tdevtools_path (str): path to the l2tdevtools directory. - dependency_definitions (dict[str, ProjectDefinition]): definitions of all - projects, which is used to determine the properties of dependencies. - """ - super().__init__( - project_definition, l2tdevtools_path, dependency_definitions) - self.architecture = platform.machine() - - self.rpmbuild_path = os.path.join('~', 'rpmbuild') - self.rpmbuild_path = os.path.expanduser(self.rpmbuild_path) - - self._rpmbuild_rpms_path = os.path.join(self.rpmbuild_path, 'RPMS') - self._rpmbuild_sources_path = os.path.join(self.rpmbuild_path, 'SOURCES') - self._rpmbuild_specs_path = os.path.join(self.rpmbuild_path, 'SPECS') - self._rpmbuild_srpms_path = os.path.join(self.rpmbuild_path, 'SRPMS') - - def _BuildFromSpecFile(self, spec_filename, rpmbuild_flags='-ba'): - """Builds the rpms directly from a spec file. - - Args: - spec_filename (str): name of the spec file as stored in the rpmbuild - SPECS sub directory. - rpmbuild_flags (Optional(str)): rpmbuild flags. - - Returns: - bool: True if successful, False otherwise. - """ - spec_filename = os.path.join('SPECS', spec_filename) - - current_path = os.getcwd() - os.chdir(self.rpmbuild_path) - - command = ( - f'rpmbuild {rpmbuild_flags:s} {spec_filename:s} > ' - f'{self.LOG_FILENAME:s} 2>&1') - exit_code = subprocess.call(command, shell=True) - if exit_code != 0: - logging.error(f'Running: "{command:s}" failed.') - - os.chdir(current_path) - - return exit_code == 0 - - def _BuildFromSourcePackage( - self, source_package_filename, rpmbuild_flags='-ta'): - """Builds the rpms directly from the source package file. - - For this to work the source package needs to contain a valid rpm .spec file. - - Args: - source_package_filename (str): name of the source package file. - rpmbuild_flags (Optional(str)): rpmbuild flags. - - Returns: - bool: True if successful, False otherwise. - """ - command = ( - f'rpmbuild {rpmbuild_flags:s} {source_package_filename:s} > ' - f'{self.LOG_FILENAME:s} 2>&1') - exit_code = subprocess.call(command, shell=True) - if exit_code != 0: - logging.error(f'Running: "{command:s}" failed.') - return False - - return True - - def _CheckIsInstalled(self, package_name): - """Checks if a package is installed. - - Args: - package_name (str): name of the package. - - Returns: - bool: True if the package is installed, False otherwise. - """ - exit_code = subprocess.call( - f'rpm -qi {package_name:s} >/dev/null 2>&1', shell=True) - return exit_code == 0 - - def _CopySourcePackageToRPMBuildSources(self, source_package_path): - """Copies the source package to the rpmbuild SOURCES directory. - - Args: - source_package_path (str): path of the source package file. - """ - source_package_filename = os.path.basename(source_package_path) - rpm_source_package_path = os.path.join( - self._rpmbuild_sources_path, source_package_filename) - - if not os.path.exists(rpm_source_package_path): - self._CreateRPMbuildDirectories() - - shutil.copy(source_package_path, rpm_source_package_path) - - def _CreateRPMbuildDirectories(self): - """Creates the rpmbuild and sub directories.""" - if not os.path.exists(self.rpmbuild_path): - os.mkdir(self.rpmbuild_path) - - if not os.path.exists(self._rpmbuild_sources_path): - os.mkdir(self._rpmbuild_sources_path) + """Helper to build RPM packages (.rpm).""" + + _BUILD_DEPENDENCIES = frozenset( + [ + "git", + "binutils", + "autoconf", + "automake", + "libtool", + "gettext-devel", + "make", + "pkgconf", + "gcc", + "gcc-c++", + "flex", + "byacc", + "rpm-build", + "python3-dateutil", + "python3-devel", + "python3-setuptools", + "python3-test", + ] + ) + + _BUILD_DEPENDENCY_PACKAGE_NAMES = { + "bzip2": ["bzip2-devel"], + "fuse": ["fuse3-devel"], + "libcrypto": ["openssl-devel"], + "liblzma": ["xz-devel"], + "pytest-runner": ["python3-pytest-runner"], + "sqlite": ["sqlite-devel"], + "zeromq": ["libzmq3-devel"], + "zlib": ["zlib-ng-devel"], + } + + def __init__(self, project_definition, l2tdevtools_path, dependency_definitions): + """Initializes a build helper. + + Args: + project_definition (ProjectDefinition): definition of the project + to build. + l2tdevtools_path (str): path to the l2tdevtools directory. + dependency_definitions (dict[str, ProjectDefinition]): definitions of all + projects, which is used to determine the properties of dependencies. + """ + super().__init__(project_definition, l2tdevtools_path, dependency_definitions) + self.architecture = platform.machine() + + self.rpmbuild_path = os.path.join("~", "rpmbuild") + self.rpmbuild_path = os.path.expanduser(self.rpmbuild_path) + + self._rpmbuild_rpms_path = os.path.join(self.rpmbuild_path, "RPMS") + self._rpmbuild_sources_path = os.path.join(self.rpmbuild_path, "SOURCES") + self._rpmbuild_specs_path = os.path.join(self.rpmbuild_path, "SPECS") + self._rpmbuild_srpms_path = os.path.join(self.rpmbuild_path, "SRPMS") + + def _BuildFromSpecFile(self, spec_filename, rpmbuild_flags="-ba"): + """Builds the rpms directly from a spec file. + + Args: + spec_filename (str): name of the spec file as stored in the rpmbuild + SPECS sub directory. + rpmbuild_flags (Optional(str)): rpmbuild flags. + + Returns: + bool: True if successful, False otherwise. + """ + spec_filename = os.path.join("SPECS", spec_filename) + + current_path = os.getcwd() + os.chdir(self.rpmbuild_path) + + command = ( + f"rpmbuild {rpmbuild_flags:s} {spec_filename:s} > " + f"{self.LOG_FILENAME:s} 2>&1" + ) + exit_code = subprocess.call(command, shell=True) + if exit_code != 0: + logging.error(f'Running: "{command:s}" failed.') + + os.chdir(current_path) + + return exit_code == 0 + + def _BuildFromSourcePackage(self, source_package_filename, rpmbuild_flags="-ta"): + """Builds the rpms directly from the source package file. + + For this to work the source package needs to contain a valid rpm .spec file. + + Args: + source_package_filename (str): name of the source package file. + rpmbuild_flags (Optional(str)): rpmbuild flags. + + Returns: + bool: True if successful, False otherwise. + """ + command = ( + f"rpmbuild {rpmbuild_flags:s} {source_package_filename:s} > " + f"{self.LOG_FILENAME:s} 2>&1" + ) + exit_code = subprocess.call(command, shell=True) + if exit_code != 0: + logging.error(f'Running: "{command:s}" failed.') + return False + + return True + + def _CheckIsInstalled(self, package_name): + """Checks if a package is installed. + + Args: + package_name (str): name of the package. + + Returns: + bool: True if the package is installed, False otherwise. + """ + exit_code = subprocess.call( + f"rpm -qi {package_name:s} >/dev/null 2>&1", shell=True + ) + return exit_code == 0 + + def _CopySourcePackageToRPMBuildSources(self, source_package_path): + """Copies the source package to the rpmbuild SOURCES directory. + + Args: + source_package_path (str): path of the source package file. + """ + source_package_filename = os.path.basename(source_package_path) + rpm_source_package_path = os.path.join( + self._rpmbuild_sources_path, source_package_filename + ) + + if not os.path.exists(rpm_source_package_path): + self._CreateRPMbuildDirectories() + + shutil.copy(source_package_path, rpm_source_package_path) + + def _CreateRPMbuildDirectories(self): + """Creates the rpmbuild and sub directories.""" + if not os.path.exists(self.rpmbuild_path): + os.mkdir(self.rpmbuild_path) + + if not os.path.exists(self._rpmbuild_sources_path): + os.mkdir(self._rpmbuild_sources_path) + + if not os.path.exists(self._rpmbuild_specs_path): + os.mkdir(self._rpmbuild_specs_path) - if not os.path.exists(self._rpmbuild_specs_path): - os.mkdir(self._rpmbuild_specs_path) + def _CreateSpecFile(self, project_name, spec_file_data): + """Creates a spec file in the rpmbuild directory. - def _CreateSpecFile(self, project_name, spec_file_data): - """Creates a spec file in the rpmbuild directory. + Args: + project_name (str): name of the project. + spec_file_data (str): spec file data. + """ + spec_filename = os.path.join( + self._rpmbuild_specs_path, f"{project_name:s}.spec" + ) - Args: - project_name (str): name of the project. - spec_file_data (str): spec file data. - """ - spec_filename = os.path.join( - self._rpmbuild_specs_path, f'{project_name:s}.spec') + with open(spec_filename, "w", encoding="utf-8") as rpm_spec_file: + rpm_spec_file.write(spec_file_data) + + def _CopySourceFile(self, source_package_path): + """Copies the source file to the rpmbuild directory. + + Args: + source_package_path (str): path of the source package file. + """ + shutil.copy(source_package_path, self._rpmbuild_sources_path) + + def _GetFilenameSafeProjectInformation(self, source_helper_object): + """Determines the filename safe project name and version. + + Args: + source_helper_object (SourceHelper): source helper. - with open(spec_filename, 'w', encoding='utf-8') as rpm_spec_file: - rpm_spec_file.write(spec_file_data) + Returns: + tuple: containing: - def _CopySourceFile(self, source_package_path): - """Copies the source file to the rpmbuild directory. + * str: filename safe project name. + * str: version. + """ + project_name = source_helper_object.project_name - Args: - source_package_path (str): path of the source package file. - """ - shutil.copy(source_package_path, self._rpmbuild_sources_path) + project_version = source_helper_object.GetProjectVersion() + if project_version and project_version.startswith("1!"): + # Remove setuptools epoch. + project_version = project_version[2:] + + return project_name, project_version + + def _MoveFilesToCurrentDirectory(self, filenames_glob): + """Moves files into the current directory. + + Args: + filenames_glob (str): glob of the filenames to move. + """ + filenames = glob.glob(filenames_glob) + for filename in filenames: + logging.info(f"Moving: {filename:s}") - def _GetFilenameSafeProjectInformation(self, source_helper_object): - """Determines the filename safe project name and version. + local_filename = os.path.basename(filename) + if os.path.exists(local_filename): + os.remove(local_filename) - Args: - source_helper_object (SourceHelper): source helper. + shutil.move(filename, ".") - Returns: - tuple: containing: + def CheckBuildDependencies(self): + """Checks if the build dependencies are met. - * str: filename safe project name. - * str: version. - """ - project_name = source_helper_object.project_name + Returns: + list[str]: build dependency names that are not met or an empty list. + """ + missing_packages = [] + for package_name in self._BUILD_DEPENDENCIES: + if not self._CheckIsInstalled(package_name): + missing_packages.append(package_name) - project_version = source_helper_object.GetProjectVersion() - if project_version and project_version.startswith('1!'): - # Remove setuptools epoch. - project_version = project_version[2:] + for package_name in self._project_definition.build_dependencies: + dependencies = self._BUILD_DEPENDENCY_PACKAGE_NAMES.get( + package_name, [package_name] + ) + for dependency in dependencies: + if not self._CheckIsInstalled(dependency): + missing_packages.append(dependency) - return project_name, project_version - - def _MoveFilesToCurrentDirectory(self, filenames_glob): - """Moves files into the current directory. - - Args: - filenames_glob (str): glob of the filenames to move. - """ - filenames = glob.glob(filenames_glob) - for filename in filenames: - logging.info(f'Moving: {filename:s}') - - local_filename = os.path.basename(filename) - if os.path.exists(local_filename): - os.remove(local_filename) - - shutil.move(filename, '.') - - def CheckBuildDependencies(self): - """Checks if the build dependencies are met. - - Returns: - list[str]: build dependency names that are not met or an empty list. - """ - missing_packages = [] - for package_name in self._BUILD_DEPENDENCIES: - if not self._CheckIsInstalled(package_name): - missing_packages.append(package_name) - - for package_name in self._project_definition.build_dependencies: - dependencies = self._BUILD_DEPENDENCY_PACKAGE_NAMES.get( - package_name, [package_name]) - for dependency in dependencies: - if not self._CheckIsInstalled(dependency): - missing_packages.append(dependency) - - return missing_packages + return missing_packages class RPMBuildHelper(BaseRPMBuildHelper): - """Helper to build RPM packages (.rpm).""" - - def _RemoveBuildDirectory(self, project_name, project_version): - """Removes build directory. - - Args: - project_name (str): name of the project. - project_version (str): version of the project. - """ - filename = os.path.join( - self.rpmbuild_path, 'BUILD', f'{project_name:s}-{project_version!s}') - - if os.path.exists(filename): - try: - shutil.rmtree(filename) - logging.info(f'Removed: {filename:s}') - except OSError: - logging.warning(f'Unable to remove: {filename:s}') - - def _RemoveOlderBuildDirectory(self, project_name, project_version): - """Removes previous versions of build directories. - - Args: - project_name (str): name of the project. - project_version (str): version of the project. - """ - filenames_to_ignore = re.compile(f'{project_name:s}-{project_version!s}') - - filenames_glob = os.path.join( - self.rpmbuild_path, 'BUILD', f'{project_name:s}-*') - filenames = glob.glob(filenames_glob) - - for filename in filenames: - if not filenames_to_ignore.match(filename): - logging.info(f'Removing: {filename:s}') - shutil.rmtree(filename, ignore_errors=True) - - def _RemoveOlderRPMs(self, project_name, project_version): - """Removes previous versions of .rpm files. - - Args: - project_name (str): name of the project. - project_version (str): version of the project. - """ - filenames_to_ignore = ( - f'.*{project_name:s}-.*{project_version!s}-1.{self.architecture:s}.rpm') - filenames_to_ignore = re.compile(filenames_to_ignore) - - rpm_filenames_glob = f'*{project_name:s}-*-1.{self.architecture:s}.rpm' - filenames = glob.glob(rpm_filenames_glob) - - for filename in filenames: - if not filenames_to_ignore.match(filename): - logging.info(f'Removing: {filename:s}') - os.remove(filename) - - filenames_glob = os.path.join( - self.rpmbuild_path, 'RPMS', self.architecture, rpm_filenames_glob) - filenames = glob.glob(filenames_glob) - - for filename in filenames: - if not filenames_to_ignore.match(filename): - logging.info(f'Removing: {filename:s}') - os.remove(filename) - - def CheckBuildRequired(self, source_helper_object): - """Checks if a build is required. - - Args: - source_helper_object (SourceHelper): source helper. - - Returns: - bool: True if a build is required, False otherwise. - """ - project_name, project_version = self._GetFilenameSafeProjectInformation( - source_helper_object) - - rpm_filename = ( - f'{project_name:s}-{project_version!s}-1.{self.architecture:s}.rpm') - - return not os.path.exists(rpm_filename) + """Helper to build RPM packages (.rpm).""" + + def _RemoveBuildDirectory(self, project_name, project_version): + """Removes build directory. + + Args: + project_name (str): name of the project. + project_version (str): version of the project. + """ + filename = os.path.join( + self.rpmbuild_path, "BUILD", f"{project_name:s}-{project_version!s}" + ) + + if os.path.exists(filename): + try: + shutil.rmtree(filename) + logging.info(f"Removed: {filename:s}") + except OSError: + logging.warning(f"Unable to remove: {filename:s}") + + def _RemoveOlderBuildDirectory(self, project_name, project_version): + """Removes previous versions of build directories. + + Args: + project_name (str): name of the project. + project_version (str): version of the project. + """ + filenames_to_ignore = re.compile(f"{project_name:s}-{project_version!s}") + + filenames_glob = os.path.join( + self.rpmbuild_path, "BUILD", f"{project_name:s}-*" + ) + filenames = glob.glob(filenames_glob) + + for filename in filenames: + if not filenames_to_ignore.match(filename): + logging.info(f"Removing: {filename:s}") + shutil.rmtree(filename, ignore_errors=True) + + def _RemoveOlderRPMs(self, project_name, project_version): + """Removes previous versions of .rpm files. + + Args: + project_name (str): name of the project. + project_version (str): version of the project. + """ + filenames_to_ignore = ( + f".*{project_name:s}-.*{project_version!s}-1.{self.architecture:s}.rpm" + ) + filenames_to_ignore = re.compile(filenames_to_ignore) + + rpm_filenames_glob = f"*{project_name:s}-*-1.{self.architecture:s}.rpm" + filenames = glob.glob(rpm_filenames_glob) + + for filename in filenames: + if not filenames_to_ignore.match(filename): + logging.info(f"Removing: {filename:s}") + os.remove(filename) + + filenames_glob = os.path.join( + self.rpmbuild_path, "RPMS", self.architecture, rpm_filenames_glob + ) + filenames = glob.glob(filenames_glob) + + for filename in filenames: + if not filenames_to_ignore.match(filename): + logging.info(f"Removing: {filename:s}") + os.remove(filename) + + def CheckBuildRequired(self, source_helper_object): + """Checks if a build is required. + + Args: + source_helper_object (SourceHelper): source helper. + + Returns: + bool: True if a build is required, False otherwise. + """ + project_name, project_version = self._GetFilenameSafeProjectInformation( + source_helper_object + ) + + rpm_filename = ( + f"{project_name:s}-{project_version!s}-1.{self.architecture:s}.rpm" + ) + + return not os.path.exists(rpm_filename) class ConfigureMakeRPMBuildHelper(RPMBuildHelper): - """Helper to build RPM packages (.rpm).""" + """Helper to build RPM packages (.rpm).""" - def _MoveRPMs(self, project_name, project_version): - """Moves the rpms from the rpmbuild directory into the current directory. + def _MoveRPMs(self, project_name, project_version): + """Moves the rpms from the rpmbuild directory into the current directory. - Args: - project_name (str): name of the project. - project_version (str): version of the project. - """ - rpm_name = self._project_definition.rpm_name or project_name + Args: + project_name (str): name of the project. + project_version (str): version of the project. + """ + rpm_name = self._project_definition.rpm_name or project_name - filenames_glob = os.path.join( - self._rpmbuild_rpms_path, self.architecture, - f'{rpm_name:s}-*{project_version!s}-1.{self.architecture:s}.rpm') + filenames_glob = os.path.join( + self._rpmbuild_rpms_path, + self.architecture, + f"{rpm_name:s}-*{project_version!s}-1.{self.architecture:s}.rpm", + ) - self._MoveFilesToCurrentDirectory(filenames_glob) + self._MoveFilesToCurrentDirectory(filenames_glob) - def Build(self, source_helper_object): - """Builds the rpms. + def Build(self, source_helper_object): + """Builds the rpms. - Args: - source_helper_object (SourceHelper): source helper. + Args: + source_helper_object (SourceHelper): source helper. - Returns: - bool: True if successful, False otherwise. - """ - source_package_path = source_helper_object.GetSourcePackagePath() - if not source_package_path: - logging.info( - f'Missing source package of: {source_helper_object.project_name:s}') - return False + Returns: + bool: True if successful, False otherwise. + """ + source_package_path = source_helper_object.GetSourcePackagePath() + if not source_package_path: + logging.info( + f"Missing source package of: {source_helper_object.project_name:s}" + ) + return False - source_package_filename = source_helper_object.GetSourcePackageFilename() - logging.info(f'Building rpm of: {source_package_filename:s}') + source_package_filename = source_helper_object.GetSourcePackageFilename() + logging.info(f"Building rpm of: {source_package_filename:s}") - project_name, project_version = self._GetFilenameSafeProjectInformation( - source_helper_object) + project_name, project_version = self._GetFilenameSafeProjectInformation( + source_helper_object + ) - # rpmbuild wants the source package filename without the status indication. - rpm_source_package_filename = f'{project_name:s}-{project_version!s}.tar.gz' - if not os.path.exists(rpm_source_package_filename): - shutil.copyfile(source_package_path, rpm_source_package_filename) + # rpmbuild wants the source package filename without the status indication. + rpm_source_package_filename = f"{project_name:s}-{project_version!s}.tar.gz" + if not os.path.exists(rpm_source_package_filename): + shutil.copyfile(source_package_path, rpm_source_package_filename) - build_successful = self._BuildFromSourcePackage( - rpm_source_package_filename, rpmbuild_flags='-tb') + build_successful = self._BuildFromSourcePackage( + rpm_source_package_filename, rpmbuild_flags="-tb" + ) - if build_successful: - self._MoveRPMs(project_name, project_version) + if build_successful: + self._MoveRPMs(project_name, project_version) - setup_name = self._project_definition.setup_name or project_name - self._RemoveBuildDirectory(setup_name, project_version) + setup_name = self._project_definition.setup_name or project_name + self._RemoveBuildDirectory(setup_name, project_version) - return build_successful + return build_successful - def Clean(self, source_helper_object): - """Cleans the rpmbuild directory. + def Clean(self, source_helper_object): + """Cleans the rpmbuild directory. - Args: - source_helper_object (SourceHelper): source helper. - """ - project_name, project_version = self._GetFilenameSafeProjectInformation( - source_helper_object) + Args: + source_helper_object (SourceHelper): source helper. + """ + project_name, project_version = self._GetFilenameSafeProjectInformation( + source_helper_object + ) - self._RemoveOlderSourceDirectories(project_name, project_version) - self._RemoveOlderSourcePackages(project_name, project_version) + self._RemoveOlderSourceDirectories(project_name, project_version) + self._RemoveOlderSourcePackages(project_name, project_version) - setup_name = self._project_definition.setup_name or project_name - self._RemoveOlderBuildDirectory(setup_name, project_version) + setup_name = self._project_definition.setup_name or project_name + self._RemoveOlderBuildDirectory(setup_name, project_version) - self._RemoveOlderRPMs(project_name, project_version) + self._RemoveOlderRPMs(project_name, project_version) class PyprojectRPMBuildHelper(RPMBuildHelper): - """Helper to build RPM packages (.rpm).""" - - def __init__( - self, project_definition, l2tdevtools_path, dependency_definitions): - """Initializes a build helper. - - Args: - project_definition (ProjectDefinition): definition of the project - to build. - l2tdevtools_path (str): path to the l2tdevtools directory. - dependency_definitions (dict[str, ProjectDefinition]): definitions of all - projects, which is used to determine the properties of dependencies. - """ - super().__init__( - project_definition, l2tdevtools_path, dependency_definitions) - if not project_definition.architecture_dependent: - self.architecture = 'noarch' - - def _GenerateSpecFile( - self, project_name, project_version, source_package_filename, - source_helper_object): - """Generates the rpm spec file. - - Args: - project_name (str): name of the project. - project_version (str): version of the project. - source_package_filename (str): name of the source package file. - source_helper_object (SourceHelper): source helper. - - Returns: - str: path of the generated rpm spec file or None if not available. - """ - source_directory = source_helper_object.GetSourceDirectoryPath() - if not source_directory: - logging.info( - f'Missing source directory of: {source_helper_object.project_name:s}') - return None - - spec_file_generator = spec_file.RPMSpecFileGenerator(self._data_path) - - if project_name.startswith('python-'): - project_name = project_name[7:] - - output_file_path = os.path.join( - self._rpmbuild_specs_path, f'{project_name:s}.spec') - - try: - result = spec_file_generator.Generate( - self._project_definition, source_directory, source_package_filename, - project_name, project_version, output_file_path) - except (FileNotFoundError, TypeError, ValueError) as exception: - logging.warning( - f'Unable to gerenate rpm spec file with error: {exception!s}') - result = False - - if not result: - return None - - return output_file_path - - def _MoveRPMs(self, project_name, project_version): - """Moves the rpms from the rpmbuild directory into the current directory. - - Args: - project_name (str): name of the project. - project_version (str): version of the project. - """ - rpm_name = self._project_definition.rpm_name or project_name - if (rpm_name.startswith('python-') or rpm_name.startswith('python2-') or - rpm_name.startswith('python3-')): - _, _, rpm_name = rpm_name.partition('-') - - # TODO: add support for rpm_python_prefix. - python_rpm_name = f'python*-{rpm_name:s}' - - filenames_glob = os.path.join( - self._rpmbuild_rpms_path, self.architecture, - f'{python_rpm_name:s}-*{project_version!s}-1.{self.architecture:s}.rpm') - - self._MoveFilesToCurrentDirectory(filenames_glob) - - filenames_glob = os.path.join( - self._rpmbuild_rpms_path, self.architecture, - f'{rpm_name:s}-*{project_version!s}-1.{self.architecture:s}.rpm') - - self._MoveFilesToCurrentDirectory(filenames_glob) - - def Build(self, source_helper_object): - """Builds the rpms. - - Args: - source_helper_object (SourceHelper): source helper. - - Returns: - bool: True if successful, False otherwise. - """ - source_package_path = source_helper_object.GetSourcePackagePath() - if not source_package_path: - logging.info( - f'Missing source package of: {source_helper_object.project_name:s}') - return False - - source_package_filename = source_helper_object.GetSourcePackageFilename() - logging.info(f'Building rpm of: {source_package_filename:s}') - - project_name, project_version = self._GetFilenameSafeProjectInformation( - source_helper_object) - - self._CopySourcePackageToRPMBuildSources(source_package_path) - - rpm_spec_file_path = self._GenerateSpecFile( - project_name, project_version, source_package_filename, - source_helper_object) - if not rpm_spec_file_path: - logging.error('Unable to generate rpm spec file.') - return False - - build_successful = self._BuildFromSpecFile( - rpm_spec_file_path, rpmbuild_flags='-bb') - - if build_successful: - self._MoveRPMs(project_name, project_version) - - setup_name = self._project_definition.setup_name or project_name - self._RemoveBuildDirectory(setup_name, project_version) - - return build_successful - - def Clean(self, source_helper_object): - """Cleans the build and dist directory. - - Args: - source_helper_object (SourceHelper): source helper. - """ - # Remove previous versions build directories. - for filename in ('build', 'dist'): - if os.path.exists(filename): - logging.info(f'Removing: {filename:s}') - shutil.rmtree(filename, ignore_errors=True) + """Helper to build RPM packages (.rpm).""" + + def __init__(self, project_definition, l2tdevtools_path, dependency_definitions): + """Initializes a build helper. + + Args: + project_definition (ProjectDefinition): definition of the project + to build. + l2tdevtools_path (str): path to the l2tdevtools directory. + dependency_definitions (dict[str, ProjectDefinition]): definitions of all + projects, which is used to determine the properties of dependencies. + """ + super().__init__(project_definition, l2tdevtools_path, dependency_definitions) + if not project_definition.architecture_dependent: + self.architecture = "noarch" + + def _GenerateSpecFile( + self, + project_name, + project_version, + source_package_filename, + source_helper_object, + ): + """Generates the rpm spec file. + + Args: + project_name (str): name of the project. + project_version (str): version of the project. + source_package_filename (str): name of the source package file. + source_helper_object (SourceHelper): source helper. + + Returns: + str: path of the generated rpm spec file or None if not available. + """ + source_directory = source_helper_object.GetSourceDirectoryPath() + if not source_directory: + logging.info( + f"Missing source directory of: {source_helper_object.project_name:s}" + ) + return None + + spec_file_generator = spec_file.RPMSpecFileGenerator(self._data_path) + + if project_name.startswith("python-"): + project_name = project_name[7:] + + output_file_path = os.path.join( + self._rpmbuild_specs_path, f"{project_name:s}.spec" + ) + + try: + result = spec_file_generator.Generate( + self._project_definition, + source_directory, + source_package_filename, + project_name, + project_version, + output_file_path, + ) + except (FileNotFoundError, TypeError, ValueError) as exception: + logging.warning( + f"Unable to gerenate rpm spec file with error: {exception!s}" + ) + result = False + + if not result: + return None + + return output_file_path + + def _MoveRPMs(self, project_name, project_version): + """Moves the rpms from the rpmbuild directory into the current directory. + + Args: + project_name (str): name of the project. + project_version (str): version of the project. + """ + rpm_name = self._project_definition.rpm_name or project_name + if ( + rpm_name.startswith("python-") + or rpm_name.startswith("python2-") + or rpm_name.startswith("python3-") + ): + _, _, rpm_name = rpm_name.partition("-") + + # TODO: add support for rpm_python_prefix. + python_rpm_name = f"python*-{rpm_name:s}" + + filenames_glob = os.path.join( + self._rpmbuild_rpms_path, + self.architecture, + f"{python_rpm_name:s}-*{project_version!s}-1.{self.architecture:s}.rpm", + ) + + self._MoveFilesToCurrentDirectory(filenames_glob) + + filenames_glob = os.path.join( + self._rpmbuild_rpms_path, + self.architecture, + f"{rpm_name:s}-*{project_version!s}-1.{self.architecture:s}.rpm", + ) + + self._MoveFilesToCurrentDirectory(filenames_glob) + + def Build(self, source_helper_object): + """Builds the rpms. + + Args: + source_helper_object (SourceHelper): source helper. + + Returns: + bool: True if successful, False otherwise. + """ + source_package_path = source_helper_object.GetSourcePackagePath() + if not source_package_path: + logging.info( + f"Missing source package of: {source_helper_object.project_name:s}" + ) + return False + + source_package_filename = source_helper_object.GetSourcePackageFilename() + logging.info(f"Building rpm of: {source_package_filename:s}") + + project_name, project_version = self._GetFilenameSafeProjectInformation( + source_helper_object + ) + + self._CopySourcePackageToRPMBuildSources(source_package_path) + + rpm_spec_file_path = self._GenerateSpecFile( + project_name, project_version, source_package_filename, source_helper_object + ) + if not rpm_spec_file_path: + logging.error("Unable to generate rpm spec file.") + return False + + build_successful = self._BuildFromSpecFile( + rpm_spec_file_path, rpmbuild_flags="-bb" + ) + + if build_successful: + self._MoveRPMs(project_name, project_version) + + setup_name = self._project_definition.setup_name or project_name + self._RemoveBuildDirectory(setup_name, project_version) + + return build_successful + + def Clean(self, source_helper_object): + """Cleans the build and dist directory. + + Args: + source_helper_object (SourceHelper): source helper. + """ + # Remove previous versions build directories. + for filename in ("build", "dist"): + if os.path.exists(filename): + logging.info(f"Removing: {filename:s}") + shutil.rmtree(filename, ignore_errors=True) - # Remove previous versions of rpms. - project_name, project_version = self._GetFilenameSafeProjectInformation( - source_helper_object) + # Remove previous versions of rpms. + project_name, project_version = self._GetFilenameSafeProjectInformation( + source_helper_object + ) - # The setup.py directory name can differ from the project name. - setup_name = self._project_definition.setup_name or project_name + # The setup.py directory name can differ from the project name. + setup_name = self._project_definition.setup_name or project_name - self._RemoveOlderSourceDirectories(setup_name, project_version) - self._RemoveOlderSourcePackages(project_name, project_version) + self._RemoveOlderSourceDirectories(setup_name, project_version) + self._RemoveOlderSourcePackages(project_name, project_version) - self._RemoveOlderBuildDirectory(setup_name, project_version) + self._RemoveOlderBuildDirectory(setup_name, project_version) - self._RemoveOlderRPMs(project_name, project_version) + self._RemoveOlderRPMs(project_name, project_version) class SRPMBuildHelper(BaseRPMBuildHelper): - """Helper to build source RPM packages (.src.rpm).""" + """Helper to build source RPM packages (.src.rpm).""" - def _MoveRPMs(self, project_name, project_version): - """Moves the rpms from the rpmbuild directory into the current directory. + def _MoveRPMs(self, project_name, project_version): + """Moves the rpms from the rpmbuild directory into the current directory. - Args: - project_name (str): name of the project. - project_version (str): version of the project. - """ - srpm_name = self._project_definition.srpm_name or project_name - if (srpm_name.startswith('python-') or srpm_name.startswith('python2-') or - srpm_name.startswith('python3-')): - _, _, srpm_name = srpm_name.partition('-') + Args: + project_name (str): name of the project. + project_version (str): version of the project. + """ + srpm_name = self._project_definition.srpm_name or project_name + if ( + srpm_name.startswith("python-") + or srpm_name.startswith("python2-") + or srpm_name.startswith("python3-") + ): + _, _, srpm_name = srpm_name.partition("-") - filenames_glob = os.path.join( - self._rpmbuild_srpms_path, - f'{srpm_name:s}-*{project_version!s}-1.src.rpm') + filenames_glob = os.path.join( + self._rpmbuild_srpms_path, f"{srpm_name:s}-*{project_version!s}-1.src.rpm" + ) - self._MoveFilesToCurrentDirectory(filenames_glob) + self._MoveFilesToCurrentDirectory(filenames_glob) - def _RemoveOlderSourceRPMs(self, project_name, project_version): - """Removes previous versions of .src.rpm files. + def _RemoveOlderSourceRPMs(self, project_name, project_version): + """Removes previous versions of .src.rpm files. - Args: - project_name (str): name of the project. - project_version (str): version of the project. - """ - filenames_to_ignore = re.compile( - f'{project_name:s}-.*{project_version!s}-1.src.rpm') + Args: + project_name (str): name of the project. + project_version (str): version of the project. + """ + filenames_to_ignore = re.compile( + f"{project_name:s}-.*{project_version!s}-1.src.rpm" + ) - src_rpm_filenames_glob = f'{project_name:s}-*-1.src.rpm' + src_rpm_filenames_glob = f"{project_name:s}-*-1.src.rpm" - for filename in glob.glob(src_rpm_filenames_glob): - if not filenames_to_ignore.match(filename): - logging.info(f'Removing: {filename:s}') - os.remove(filename) + for filename in glob.glob(src_rpm_filenames_glob): + if not filenames_to_ignore.match(filename): + logging.info(f"Removing: {filename:s}") + os.remove(filename) - filenames_glob = os.path.join( - self.rpmbuild_path, 'SRPMS', src_rpm_filenames_glob) + filenames_glob = os.path.join( + self.rpmbuild_path, "SRPMS", src_rpm_filenames_glob + ) - for filename in glob.glob(filenames_glob): - if not filenames_to_ignore.match(filename): - logging.info(f'Removing: {filename:s}') - os.remove(filename) + for filename in glob.glob(filenames_glob): + if not filenames_to_ignore.match(filename): + logging.info(f"Removing: {filename:s}") + os.remove(filename) - def CheckBuildRequired(self, source_helper_object): - """Checks if a build is required. + def CheckBuildRequired(self, source_helper_object): + """Checks if a build is required. - Args: - source_helper_object (SourceHelper): source helper. + Args: + source_helper_object (SourceHelper): source helper. - Returns: - bool: True if a build is required, False otherwise. - """ - project_name, project_version = self._GetFilenameSafeProjectInformation( - source_helper_object) + Returns: + bool: True if a build is required, False otherwise. + """ + project_name, project_version = self._GetFilenameSafeProjectInformation( + source_helper_object + ) - return not os.path.exists(f'{project_name:s}-{project_version!s}-1.src.rpm') + return not os.path.exists(f"{project_name:s}-{project_version!s}-1.src.rpm") - def Clean(self, source_helper_object): - """Cleans the rpmbuild directory. + def Clean(self, source_helper_object): + """Cleans the rpmbuild directory. - Args: - source_helper_object (SourceHelper): source helper. - """ - project_name, project_version = self._GetFilenameSafeProjectInformation( - source_helper_object) + Args: + source_helper_object (SourceHelper): source helper. + """ + project_name, project_version = self._GetFilenameSafeProjectInformation( + source_helper_object + ) - self._RemoveOlderSourceDirectories(project_name, project_version) - self._RemoveOlderSourcePackages(project_name, project_version) + self._RemoveOlderSourceDirectories(project_name, project_version) + self._RemoveOlderSourcePackages(project_name, project_version) - self._RemoveOlderSourceRPMs(project_name, project_version) + self._RemoveOlderSourceRPMs(project_name, project_version) class ConfigureMakeSRPMBuildHelper(SRPMBuildHelper): - """Helper to build source RPM packages (.src.rpm).""" + """Helper to build source RPM packages (.src.rpm).""" - def Build(self, source_helper_object): - """Builds the source rpm. + def Build(self, source_helper_object): + """Builds the source rpm. - Args: - source_helper_object (SourceHelper): source helper. + Args: + source_helper_object (SourceHelper): source helper. - Returns: - bool: True if successful, False otherwise. - """ - source_package_path = source_helper_object.GetSourcePackagePath() - if not source_package_path: - logging.info( - f'Missing source package of: {source_helper_object.project_name:s}') - return False + Returns: + bool: True if successful, False otherwise. + """ + source_package_path = source_helper_object.GetSourcePackagePath() + if not source_package_path: + logging.info( + f"Missing source package of: {source_helper_object.project_name:s}" + ) + return False - source_package_filename = source_helper_object.GetSourcePackageFilename() - logging.info(f'Building source rpm of: {source_package_filename:s}') + source_package_filename = source_helper_object.GetSourcePackageFilename() + logging.info(f"Building source rpm of: {source_package_filename:s}") - project_name, project_version = self._GetFilenameSafeProjectInformation( - source_helper_object) + project_name, project_version = self._GetFilenameSafeProjectInformation( + source_helper_object + ) - # rpmbuild wants the source package filename without the status indication. - rpm_source_package_filename = f'{project_name:s}-{project_version!s}.tar.gz' - shutil.copyfile(source_package_path, rpm_source_package_filename) + # rpmbuild wants the source package filename without the status indication. + rpm_source_package_filename = f"{project_name:s}-{project_version!s}.tar.gz" + shutil.copyfile(source_package_path, rpm_source_package_filename) - build_successful = self._BuildFromSourcePackage( - rpm_source_package_filename, rpmbuild_flags='-ts') + build_successful = self._BuildFromSourcePackage( + rpm_source_package_filename, rpmbuild_flags="-ts" + ) - # TODO: test binary build of source package? + # TODO: test binary build of source package? - if build_successful: - self._MoveRPMs(project_name, project_version) + if build_successful: + self._MoveRPMs(project_name, project_version) - return build_successful + return build_successful class PyprojectSRPMBuildHelper(SRPMBuildHelper): - """Helper to build source RPM packages (.src.rpm).""" - - def __init__( - self, project_definition, l2tdevtools_path, dependency_definitions): - """Initializes a build helper. - - Args: - project_definition (ProjectDefinition): definition of the project - to build. - l2tdevtools_path (str): path to the l2tdevtools directory. - dependency_definitions (dict[str, ProjectDefinition]): definitions of all - projects, which is used to determine the properties of dependencies. - """ - super().__init__( - project_definition, l2tdevtools_path, dependency_definitions) - if not project_definition.architecture_dependent: - self.architecture = 'noarch' - - def _GenerateSpecFile( - self, project_name, project_version, source_package_filename, - source_helper_object): - """Generates the rpm spec file. - - Args: - project_name (str): name of the project. - project_version (str): version of the project. - source_package_filename (str): name of the source package file. - source_helper_object (SourceHelper): source helper. - - Returns: - str: path of the generated rpm spec file or None if not available. - """ - source_directory = source_helper_object.GetSourceDirectoryPath() - if not source_directory: - logging.info( - f'Missing source directory of: {source_helper_object.project_name:s}') - return None - - spec_file_generator = spec_file.RPMSpecFileGenerator(self._data_path) - - if project_name.startswith('python-'): - project_name = project_name[7:] - - output_file_path = os.path.join( - self._rpmbuild_specs_path, f'{project_name:s}.spec') - - try: - result = spec_file_generator.Generate( - self._project_definition, source_directory, source_package_filename, - project_name, project_version, output_file_path) - except (FileNotFoundError, TypeError, ValueError): - result = False - - if not result: - return None - - return output_file_path - - def Build(self, source_helper_object): - """Builds the source rpm. - - Args: - source_helper_object (SourceHelper): source helper. - - Returns: - bool: True if successful, False otherwise. - """ - source_package_path = source_helper_object.GetSourcePackagePath() - if not source_package_path: - logging.info( - f'Missing source package of: {source_helper_object.project_name:s}') - return False - - source_package_filename = source_helper_object.GetSourcePackageFilename() - logging.info(f'Building source rpm of: {source_package_filename:s}') - - project_name, project_version = self._GetFilenameSafeProjectInformation( - source_helper_object) - - self._CopySourcePackageToRPMBuildSources(source_package_path) - - rpm_spec_file_path = self._GenerateSpecFile( - project_name, project_version, source_package_filename, - source_helper_object) - if not rpm_spec_file_path: - logging.error('Unable to generate rpm spec file.') - return False - - build_successful = self._BuildFromSpecFile( - rpm_spec_file_path, rpmbuild_flags='-bs') - - # TODO: test binary build of source package? - - if build_successful: - self._MoveRPMs(project_name, project_version) - - return build_successful + """Helper to build source RPM packages (.src.rpm).""" + + def __init__(self, project_definition, l2tdevtools_path, dependency_definitions): + """Initializes a build helper. + + Args: + project_definition (ProjectDefinition): definition of the project + to build. + l2tdevtools_path (str): path to the l2tdevtools directory. + dependency_definitions (dict[str, ProjectDefinition]): definitions of all + projects, which is used to determine the properties of dependencies. + """ + super().__init__(project_definition, l2tdevtools_path, dependency_definitions) + if not project_definition.architecture_dependent: + self.architecture = "noarch" + + def _GenerateSpecFile( + self, + project_name, + project_version, + source_package_filename, + source_helper_object, + ): + """Generates the rpm spec file. + + Args: + project_name (str): name of the project. + project_version (str): version of the project. + source_package_filename (str): name of the source package file. + source_helper_object (SourceHelper): source helper. + + Returns: + str: path of the generated rpm spec file or None if not available. + """ + source_directory = source_helper_object.GetSourceDirectoryPath() + if not source_directory: + logging.info( + f"Missing source directory of: {source_helper_object.project_name:s}" + ) + return None + + spec_file_generator = spec_file.RPMSpecFileGenerator(self._data_path) + + if project_name.startswith("python-"): + project_name = project_name[7:] + + output_file_path = os.path.join( + self._rpmbuild_specs_path, f"{project_name:s}.spec" + ) + + try: + result = spec_file_generator.Generate( + self._project_definition, + source_directory, + source_package_filename, + project_name, + project_version, + output_file_path, + ) + except (FileNotFoundError, TypeError, ValueError): + result = False + + if not result: + return None + + return output_file_path + + def Build(self, source_helper_object): + """Builds the source rpm. + + Args: + source_helper_object (SourceHelper): source helper. + + Returns: + bool: True if successful, False otherwise. + """ + source_package_path = source_helper_object.GetSourcePackagePath() + if not source_package_path: + logging.info( + f"Missing source package of: {source_helper_object.project_name:s}" + ) + return False + + source_package_filename = source_helper_object.GetSourcePackageFilename() + logging.info(f"Building source rpm of: {source_package_filename:s}") + + project_name, project_version = self._GetFilenameSafeProjectInformation( + source_helper_object + ) + + self._CopySourcePackageToRPMBuildSources(source_package_path) + + rpm_spec_file_path = self._GenerateSpecFile( + project_name, project_version, source_package_filename, source_helper_object + ) + if not rpm_spec_file_path: + logging.error("Unable to generate rpm spec file.") + return False + + build_successful = self._BuildFromSpecFile( + rpm_spec_file_path, rpmbuild_flags="-bs" + ) + + # TODO: test binary build of source package? + + if build_successful: + self._MoveRPMs(project_name, project_version) + + return build_successful diff --git a/l2tdevtools/build_helpers/source.py b/l2tdevtools/build_helpers/source.py index 8cf57c04..00195f52 100644 --- a/l2tdevtools/build_helpers/source.py +++ b/l2tdevtools/build_helpers/source.py @@ -9,97 +9,100 @@ class SourceBuildHelper(interface.BuildHelper): - """Helper to build projects from source.""" + """Helper to build projects from source.""" class ConfigureMakeSourceBuildHelper(SourceBuildHelper): - """Helper to build projects from source using configure and make.""" - - def Build(self, source_helper_object): - """Builds the source. - - Args: - source_helper_object (SourceHelper): source helper. - - Returns: - bool: True if successful, False otherwise. - """ - project_name = source_helper_object.project_name - - source_package_path = source_helper_object.GetSourcePackagePath() - if not source_package_path: - logging.info(f'Missing source package of: {project_name:s}') - return False - - source_directory = source_helper_object.GetSourceDirectoryPath() - if not source_directory: - logging.info(f'Missing source directory of: {project_name:s}') - return False - - source_package_filename = source_helper_object.GetSourcePackageFilename() - logging.info(f'Building source of: {source_package_filename:s}') - - log_file_path = os.path.join('..', self.LOG_FILENAME) - command = f'./configure > {log_file_path:s} 2>&1' - exit_code = subprocess.call( - f'(cd {source_directory:s} && {command:s})', shell=True) - if exit_code != 0: - logging.error(f'Running: "{command:s}" failed.') - return False - - command = f'make >> {log_file_path:s} 2>&1' - exit_code = subprocess.call( - f'(cd {source_directory:s} && {command:s})', shell=True) - if exit_code != 0: - logging.error(f'Running: "{command:s}" failed.') - return False - - return True - - # pylint: disable=unused-argument - def Clean(self, source_helper_object): - """Cleans the source. - - Args: - source_helper_object (SourceHelper): source helper. - """ - # TODO: implement. - return + """Helper to build projects from source using configure and make.""" + + def Build(self, source_helper_object): + """Builds the source. + + Args: + source_helper_object (SourceHelper): source helper. + + Returns: + bool: True if successful, False otherwise. + """ + project_name = source_helper_object.project_name + + source_package_path = source_helper_object.GetSourcePackagePath() + if not source_package_path: + logging.info(f"Missing source package of: {project_name:s}") + return False + + source_directory = source_helper_object.GetSourceDirectoryPath() + if not source_directory: + logging.info(f"Missing source directory of: {project_name:s}") + return False + + source_package_filename = source_helper_object.GetSourcePackageFilename() + logging.info(f"Building source of: {source_package_filename:s}") + + log_file_path = os.path.join("..", self.LOG_FILENAME) + command = f"./configure > {log_file_path:s} 2>&1" + exit_code = subprocess.call( + f"(cd {source_directory:s} && {command:s})", shell=True + ) + if exit_code != 0: + logging.error(f'Running: "{command:s}" failed.') + return False + + command = f"make >> {log_file_path:s} 2>&1" + exit_code = subprocess.call( + f"(cd {source_directory:s} && {command:s})", shell=True + ) + if exit_code != 0: + logging.error(f'Running: "{command:s}" failed.') + return False + + return True + + # pylint: disable=unused-argument + def Clean(self, source_helper_object): + """Cleans the source. + + Args: + source_helper_object (SourceHelper): source helper. + """ + # TODO: implement. + return class SetupPySourceBuildHelper(SourceBuildHelper): - """Helper to build projects from source using setup.py.""" - - def Build(self, source_helper_object): - """Builds the source. - - Args: - source_helper_object (SourceHelper): source helper. - - Returns: - bool: True if successful, False otherwise. - """ - project_name = source_helper_object.project_name - - source_package_path = source_helper_object.GetSourcePackagePath() - if not source_package_path: - logging.info(f'Missing source package of: {project_name:s}') - return False - - source_directory = source_helper_object.GetSourceDirectoryPath() - if not source_directory: - logging.info(f'Missing source directory of: {project_name:s}') - return False - - source_package_filename = source_helper_object.GetSourcePackageFilename() - logging.info(f'Building source of: {source_package_filename:s}') - - log_file_path = os.path.join('..', self.LOG_FILENAME) - command = f'{sys.executable:s} setup.py build > {log_file_path:s} 2>&1' - exit_code = subprocess.call( - f'(cd {source_directory:s} && {command:s})', shell=True) - if exit_code != 0: - logging.error(f'Running: "{command:s}" failed.') - return False - - return True + """Helper to build projects from source using setup.py.""" + + def Build(self, source_helper_object): + """Builds the source. + + Args: + source_helper_object (SourceHelper): source helper. + + Returns: + bool: True if successful, False otherwise. + """ + project_name = source_helper_object.project_name + + source_package_path = source_helper_object.GetSourcePackagePath() + if not source_package_path: + logging.info(f"Missing source package of: {project_name:s}") + return False + + source_directory = source_helper_object.GetSourceDirectoryPath() + if not source_directory: + logging.info(f"Missing source directory of: {project_name:s}") + return False + + source_package_filename = source_helper_object.GetSourcePackageFilename() + logging.info(f"Building source of: {source_package_filename:s}") + + log_file_path = os.path.join("..", self.LOG_FILENAME) + command = f"{sys.executable:s} setup.py build > {log_file_path:s} 2>&1" + exit_code = subprocess.call( + f"(cd {source_directory:s} && {command:s})", shell=True + ) + if exit_code != 0: + logging.error(f'Running: "{command:s}" failed.') + return False + + return True diff --git a/l2tdevtools/build_helpers/wheel.py b/l2tdevtools/build_helpers/wheel.py index 092e2aa5..a1472423 100644 --- a/l2tdevtools/build_helpers/wheel.py +++ b/l2tdevtools/build_helpers/wheel.py @@ -13,222 +13,228 @@ class WheelBuildHelper(interface.BuildHelper): - """Helper to build Python wheel packages (.whl).""" + """Helper to build Python wheel packages (.whl).""" + + _NON_PYTHON_DEPENDENCIES = frozenset(["fuse", "libcrypto", "zlib"]) + + def __init__(self, project_definition, l2tdevtools_path, dependency_definitions): + """Initializes a build helper. + + Args: + project_definition (ProjectDefinition): definition of the project + to build. + l2tdevtools_path (str): path to the l2tdevtools directory. + dependency_definitions (dict[str, ProjectDefinition]): definitions of all + projects, which is used to determine the properties of dependencies. + """ + super().__init__(project_definition, l2tdevtools_path, dependency_definitions) + self.architecture = None + + # Note that platform.machine() does not indicate if a 32-bit version of + # Python is running on a 64-bit machine, hence we need to use + # platform.architecture() instead. + architecture = platform.architecture()[0] + if architecture == "32bit": + self.architecture = "win32" + elif architecture == "64bit": + self.architecture = "win_amd64" + + def _GetWheelFilenameProjectInformation(self, source_helper_object): + """Determines the project name and version used by the wheel filename. + + Args: + source_helper_object (SourceHelper): source helper. + + Returns: + tuple: containing: + + * str: project name used by the wheel filename. + * str: project version used by the wheel filename. + """ + if self._project_definition.wheel_name: + project_name = self._project_definition.wheel_name + elif self._project_definition.setup_name: + project_name = self._project_definition.setup_name + else: + project_name = source_helper_object.project_name + + project_version = source_helper_object.GetProjectVersion() + + return project_name, project_version + + def _MoveWheel(self, source_helper_object): + """Moves the wheel from the dist sub directory into the build directory. + + Args: + source_helper_object (SourceHelper): source helper. + + Returns: + bool: True if the move was successful, False otherwise. + """ + project_name, _ = self._GetWheelFilenameProjectInformation(source_helper_object) + + source_directory = source_helper_object.GetSourceDirectoryPath() + if not source_directory: + logging.info( + f"Missing source directory of: {source_helper_object.project_name:s}" + ) + return False + + filenames_glob = os.path.join( + source_directory, "dist", f"{project_name:s}-*-*-*.whl" + ) + filenames = glob.glob(filenames_glob) + + if len(filenames) != 1: + logging.error(f"Unable to find wheel file: {filenames_glob:s}") + return False + + _, _, wheel_filename = filenames[0].rpartition(os.path.sep) + if os.path.exists(wheel_filename): + logging.warning("Wheel file already exists.") + else: + logging.info(f"Moving: {filenames[0]:s}") + shutil.move(filenames[0], ".") + + return True + + def CheckBuildDependencies(self): + """Checks if the build dependencies are met. + + Returns: + list[str]: build dependency names that are not met or an empty list. + """ + missing_packages = [] + for package_name in self._project_definition.build_dependencies: + if package_name not in self._NON_PYTHON_DEPENDENCIES: + missing_packages.append(package_name) + + return missing_packages + + def CheckBuildRequired(self, source_helper_object): + """Checks if a build is required. + + Args: + source_helper_object (SourceHelper): source helper. + + Returns: + bool: True if a build is required, False otherwise. + """ + project_name, project_version = self._GetWheelFilenameProjectInformation( + source_helper_object + ) + + return not glob.glob(f"{project_name:s}-{project_version:s}-*-*-*.whl") + + def Clean(self, source_helper_object): + """Cleans the build and dist directory. + + Args: + source_helper_object (SourceHelper): source helper. + """ + # Remove previous versions of wheels. + project_name, project_version = self._GetWheelFilenameProjectInformation( + source_helper_object + ) - _NON_PYTHON_DEPENDENCIES = frozenset(['fuse', 'libcrypto', 'zlib']) + filenames_to_ignore = re.compile( + f"{project_name:s}-{project_version:s}-.*-.*-.*.whl" + ) - def __init__( - self, project_definition, l2tdevtools_path, dependency_definitions): - """Initializes a build helper. - - Args: - project_definition (ProjectDefinition): definition of the project - to build. - l2tdevtools_path (str): path to the l2tdevtools directory. - dependency_definitions (dict[str, ProjectDefinition]): definitions of all - projects, which is used to determine the properties of dependencies. - """ - super().__init__( - project_definition, l2tdevtools_path, dependency_definitions) - self.architecture = None - - # Note that platform.machine() does not indicate if a 32-bit version of - # Python is running on a 64-bit machine, hence we need to use - # platform.architecture() instead. - architecture = platform.architecture()[0] - if architecture == '32bit': - self.architecture = 'win32' - elif architecture == '64bit': - self.architecture = 'win_amd64' - - def _GetWheelFilenameProjectInformation(self, source_helper_object): - """Determines the project name and version used by the wheel filename. - - Args: - source_helper_object (SourceHelper): source helper. - - Returns: - tuple: containing: - - * str: project name used by the wheel filename. - * str: project version used by the wheel filename. - """ - if self._project_definition.wheel_name: - project_name = self._project_definition.wheel_name - elif self._project_definition.setup_name: - project_name = self._project_definition.setup_name - else: - project_name = source_helper_object.project_name - - project_version = source_helper_object.GetProjectVersion() - - return project_name, project_version - - def _MoveWheel(self, source_helper_object): - """Moves the wheel from the dist sub directory into the build directory. - - Args: - source_helper_object (SourceHelper): source helper. - - Returns: - bool: True if the move was successful, False otherwise. - """ - project_name, _ = self._GetWheelFilenameProjectInformation( - source_helper_object) - - source_directory = source_helper_object.GetSourceDirectoryPath() - if not source_directory: - logging.info( - f'Missing source directory of: {source_helper_object.project_name:s}') - return False - - filenames_glob = os.path.join( - source_directory, 'dist', f'{project_name:s}-*-*-*.whl') - filenames = glob.glob(filenames_glob) - - if len(filenames) != 1: - logging.error(f'Unable to find wheel file: {filenames_glob:s}') - return False - - _, _, wheel_filename = filenames[0].rpartition(os.path.sep) - if os.path.exists(wheel_filename): - logging.warning('Wheel file already exists.') - else: - logging.info(f'Moving: {filenames[0]:s}') - shutil.move(filenames[0], '.') - - return True - - def CheckBuildDependencies(self): - """Checks if the build dependencies are met. - - Returns: - list[str]: build dependency names that are not met or an empty list. - """ - missing_packages = [] - for package_name in self._project_definition.build_dependencies: - if package_name not in self._NON_PYTHON_DEPENDENCIES: - missing_packages.append(package_name) - - return missing_packages - - def CheckBuildRequired(self, source_helper_object): - """Checks if a build is required. - - Args: - source_helper_object (SourceHelper): source helper. - - Returns: - bool: True if a build is required, False otherwise. - """ - project_name, project_version = self._GetWheelFilenameProjectInformation( - source_helper_object) - - return not glob.glob(f'{project_name:s}-{project_version:s}-*-*-*.whl') - - def Clean(self, source_helper_object): - """Cleans the build and dist directory. - - Args: - source_helper_object (SourceHelper): source helper. - """ - # Remove previous versions of wheels. - project_name, project_version = self._GetWheelFilenameProjectInformation( - source_helper_object) - - filenames_to_ignore = re.compile( - f'{project_name:s}-{project_version:s}-.*-.*-.*.whl') - - for filename in glob.glob(f'{project_name:s}-*-*-*.whl'): - if not filenames_to_ignore.match(filename): - logging.info(f'Removing: {filename:s}') - os.remove(filename) + for filename in glob.glob(f"{project_name:s}-*-*-*.whl"): + if not filenames_to_ignore.match(filename): + logging.info(f"Removing: {filename:s}") + os.remove(filename) class BuildWheelBuildHelper(WheelBuildHelper): - """Helper to build Python wheel packages (.whl) using build.""" - - def Build(self, source_helper_object): - """Builds the wheel. - - Args: - source_helper_object (SourceHelper): source helper. - - Returns: - bool: True if successful, False otherwise. - """ - source_package_path = source_helper_object.GetSourcePackagePath() - if not source_package_path: - logging.info( - f'Missing source package of: {source_helper_object.project_name:s}') - return False - - source_directory = source_helper_object.GetSourceDirectoryPath() - if not source_directory: - logging.info( - f'Missing source directory of: {source_helper_object.project_name:s}') - return False - - source_package_filename = source_helper_object.GetSourcePackageFilename() - logging.info(f'Building wheel of: {source_package_filename:s}') - - log_file_path = os.path.join('..', self.LOG_FILENAME) - command = ( - f'\"{sys.executable:s}\" -m build --wheel > {log_file_path:s} 2>&1') - exit_code = subprocess.call( - f'(cd {source_directory:s} && {command:s})', shell=True) - if exit_code != 0: - logging.error(f'Running: "{command:s}" failed.') - return False - - return self._MoveWheel(source_helper_object) + """Helper to build Python wheel packages (.whl) using build.""" + + def Build(self, source_helper_object): + """Builds the wheel. + + Args: + source_helper_object (SourceHelper): source helper. + + Returns: + bool: True if successful, False otherwise. + """ + source_package_path = source_helper_object.GetSourcePackagePath() + if not source_package_path: + logging.info( + f"Missing source package of: {source_helper_object.project_name:s}" + ) + return False + + source_directory = source_helper_object.GetSourceDirectoryPath() + if not source_directory: + logging.info( + f"Missing source directory of: {source_helper_object.project_name:s}" + ) + return False + + source_package_filename = source_helper_object.GetSourcePackageFilename() + logging.info(f"Building wheel of: {source_package_filename:s}") + + log_file_path = os.path.join("..", self.LOG_FILENAME) + command = f'"{sys.executable:s}" -m build --wheel > {log_file_path:s} 2>&1' + exit_code = subprocess.call( + f"(cd {source_directory:s} && {command:s})", shell=True + ) + if exit_code != 0: + logging.error(f'Running: "{command:s}" failed.') + return False + + return self._MoveWheel(source_helper_object) class ConfigureMakeWheelBuildHelper(WheelBuildHelper): - """Helper to build Python wheel packages (.whl). + """Helper to build Python wheel packages (.whl). - Builds wheel packages for projects that use configure/make as their build - system. - """ - - def Build(self, source_helper_object): - """Builds the wheel. - - Args: - source_helper_object (SourceHelper): source helper. - - Returns: - bool: True if successful, False otherwise. - - Raises: - RuntimeError: if setup.py is missing and a wheel cannot be build. + Builds wheel packages for projects that use configure/make as their build + system. """ - source_package_path = source_helper_object.GetSourcePackagePath() - if not source_package_path: - logging.info( - f'Missing source package of: {source_helper_object.project_name:s}') - return False - - source_directory = source_helper_object.GetSourceDirectoryPath() - if not source_directory: - logging.info( - f'Missing source directory of: {source_helper_object.project_name:s}') - return False - - source_package_filename = source_helper_object.GetSourcePackageFilename() - logging.info(f'Building wheel of: {source_package_filename:s}') - - setup_py_path = os.path.join(source_directory, 'setup.py') - if not os.path.exists(setup_py_path): - raise RuntimeError('Missing setup.py cannot build wheel') - - log_file_path = os.path.join('..', self.LOG_FILENAME) - command = ( - f'\"{sys.executable:s}\" -m build --wheel > {log_file_path:s} 2>&1') - exit_code = subprocess.call( - f'(cd {source_directory:s} && {command:s})', shell=True) - if exit_code != 0: - logging.error(f'Running: "{command:s}" failed.') - return False - - return self._MoveWheel(source_helper_object) + + def Build(self, source_helper_object): + """Builds the wheel. + + Args: + source_helper_object (SourceHelper): source helper. + + Returns: + bool: True if successful, False otherwise. + + Raises: + RuntimeError: if setup.py is missing and a wheel cannot be build. + """ + source_package_path = source_helper_object.GetSourcePackagePath() + if not source_package_path: + logging.info( + f"Missing source package of: {source_helper_object.project_name:s}" + ) + return False + + source_directory = source_helper_object.GetSourceDirectoryPath() + if not source_directory: + logging.info( + f"Missing source directory of: {source_helper_object.project_name:s}" + ) + return False + + source_package_filename = source_helper_object.GetSourcePackageFilename() + logging.info(f"Building wheel of: {source_package_filename:s}") + + setup_py_path = os.path.join(source_directory, "setup.py") + if not os.path.exists(setup_py_path): + raise RuntimeError("Missing setup.py cannot build wheel") + + log_file_path = os.path.join("..", self.LOG_FILENAME) + command = f'"{sys.executable:s}" -m build --wheel > {log_file_path:s} 2>&1' + exit_code = subprocess.call( + f"(cd {source_directory:s} && {command:s})", shell=True + ) + if exit_code != 0: + logging.error(f'Running: "{command:s}" failed.') + return False + + return self._MoveWheel(source_helper_object) diff --git a/l2tdevtools/dependency_writers/appveyor_scripts.py b/l2tdevtools/dependency_writers/appveyor_scripts.py index 5032be07..dfee7c59 100644 --- a/l2tdevtools/dependency_writers/appveyor_scripts.py +++ b/l2tdevtools/dependency_writers/appveyor_scripts.py @@ -6,25 +6,25 @@ class AppVeyorInstallPS1ScriptWriter(interface.DependencyFileWriter): - """AppVeyor install.ps1 script file writer.""" + """AppVeyor install.ps1 script file writer.""" - _TEMPLATE_FILE = os.path.join( - 'data', 'templates', 'appveyor_scripts', 'install.ps1') + _TEMPLATE_FILE = os.path.join( + "data", "templates", "appveyor_scripts", "install.ps1" + ) - PATH = os.path.join('config', 'appveyor', 'install.ps1') + PATH = os.path.join("config", "appveyor", "install.ps1") - def Write(self): - """Writes an install.ps1 file.""" - dependencies = self._dependency_helper.GetL2TBinaries() - test_dependencies = self._dependency_helper.GetL2TBinaries( - test_dependencies=True) - dependencies.extend(test_dependencies) + def Write(self): + """Writes an install.ps1 file.""" + dependencies = self._dependency_helper.GetL2TBinaries() + test_dependencies = self._dependency_helper.GetL2TBinaries( + test_dependencies=True + ) + dependencies.extend(test_dependencies) - template_mappings = { - 'dependencies': ' '.join(sorted(set(dependencies))) - } - template_file = os.path.join(self._l2tdevtools_path, self._TEMPLATE_FILE) - file_content = self._GenerateFromTemplate(template_file, template_mappings) + template_mappings = {"dependencies": " ".join(sorted(set(dependencies)))} + template_file = os.path.join(self._l2tdevtools_path, self._TEMPLATE_FILE) + file_content = self._GenerateFromTemplate(template_file, template_mappings) - with open(self.PATH, 'w', encoding='utf-8') as file_object: - file_object.write(file_content) + with open(self.PATH, "w", encoding="utf-8") as file_object: + file_object.write(file_content) diff --git a/l2tdevtools/dependency_writers/appveyor_yml.py b/l2tdevtools/dependency_writers/appveyor_yml.py index eebdf69d..3327a8c1 100644 --- a/l2tdevtools/dependency_writers/appveyor_yml.py +++ b/l2tdevtools/dependency_writers/appveyor_yml.py @@ -6,83 +6,85 @@ class AppveyorYmlWriter(interface.DependencyFileWriter): - """Appveyor.yml file writer.""" + """Appveyor.yml file writer.""" - _TEMPLATE_DIRECTORY = os.path.join('data', 'templates', 'appveyor.yml') + _TEMPLATE_DIRECTORY = os.path.join("data", "templates", "appveyor.yml") - PATH = os.path.join('appveyor.yml') + PATH = os.path.join("appveyor.yml") - _PROJECTS_WITHOUT_BUILD = frozenset([ - 'dtformats', 'esedbrc', 'olecfrc', 'vstools', 'winevtrc', 'winregrc']) + _PROJECTS_WITHOUT_BUILD = frozenset( + ["dtformats", "esedbrc", "olecfrc", "vstools", "winevtrc", "winregrc"] + ) - def _GenerateFromTemplate(self, template_filename, template_mappings): - """Generates file context based on a template file. + def _GenerateFromTemplate(self, template_filename, template_mappings): + """Generates file context based on a template file. - Args: - template_filename (str): path of the template file. - template_mappings (dict[str, str]): template mappings, where the key - maps to the name of a template variable. + Args: + template_filename (str): path of the template file. + template_mappings (dict[str, str]): template mappings, where the key + maps to the name of a template variable. - Returns: - str: output based on the template string. + Returns: + str: output based on the template string. - Raises: - RuntimeError: if the template cannot be formatted. - """ - template_filename = os.path.join( - self._l2tdevtools_path, self._TEMPLATE_DIRECTORY, template_filename) - return super()._GenerateFromTemplate( - template_filename, template_mappings) + Raises: + RuntimeError: if the template cannot be formatted. + """ + template_filename = os.path.join( + self._l2tdevtools_path, self._TEMPLATE_DIRECTORY, template_filename + ) + return super()._GenerateFromTemplate(template_filename, template_mappings) - def Write(self): - """Writes an appveyor.yml file.""" - template_mappings = { - 'pypi_token': self._project_definition.pypi_token or ''} + def Write(self): + """Writes an appveyor.yml file.""" + template_mappings = {"pypi_token": self._project_definition.pypi_token or ""} - file_content = [] + file_content = [] - template_data = self._GenerateFromTemplate('environment', template_mappings) - file_content.append(template_data) - - if self._project_definition.name not in self._PROJECTS_WITHOUT_BUILD: - if self._project_definition.pypi_token: - template_data = self._GenerateFromTemplate( - 'pypi_token', template_mappings) + template_data = self._GenerateFromTemplate("environment", template_mappings) file_content.append(template_data) - template_data = self._GenerateFromTemplate('matrix', template_mappings) - file_content.append(template_data) - - template_data = self._GenerateFromTemplate('install', template_mappings) - file_content.append(template_data) + if self._project_definition.name not in self._PROJECTS_WITHOUT_BUILD: + if self._project_definition.pypi_token: + template_data = self._GenerateFromTemplate( + "pypi_token", template_mappings + ) + file_content.append(template_data) - if self._project_definition.name != 'l2tdevtools': - template_data = self._GenerateFromTemplate( - 'install_l2tdevtools', template_mappings) - file_content.append(template_data) + template_data = self._GenerateFromTemplate("matrix", template_mappings) + file_content.append(template_data) - if self._project_definition.name in self._PROJECTS_WITHOUT_BUILD: - template_filename = 'build_off' - else: - template_filename = 'build' + template_data = self._GenerateFromTemplate("install", template_mappings) + file_content.append(template_data) - template_data = self._GenerateFromTemplate( - template_filename, template_mappings) - file_content.append(template_data) + if self._project_definition.name != "l2tdevtools": + template_data = self._GenerateFromTemplate( + "install_l2tdevtools", template_mappings + ) + file_content.append(template_data) - template_data = self._GenerateFromTemplate('test_script', template_mappings) - file_content.append(template_data) + if self._project_definition.name in self._PROJECTS_WITHOUT_BUILD: + template_filename = "build_off" + else: + template_filename = "build" - if self._project_definition.name not in self._PROJECTS_WITHOUT_BUILD: - template_data = self._GenerateFromTemplate('artifacts', template_mappings) - file_content.append(template_data) + template_data = self._GenerateFromTemplate(template_filename, template_mappings) + file_content.append(template_data) - if self._project_definition.pypi_token: - template_data = self._GenerateFromTemplate( - 'deploy_script', template_mappings) + template_data = self._GenerateFromTemplate("test_script", template_mappings) file_content.append(template_data) - file_content = ''.join(file_content) + if self._project_definition.name not in self._PROJECTS_WITHOUT_BUILD: + template_data = self._GenerateFromTemplate("artifacts", template_mappings) + file_content.append(template_data) + + if self._project_definition.pypi_token: + template_data = self._GenerateFromTemplate( + "deploy_script", template_mappings + ) + file_content.append(template_data) + + file_content = "".join(file_content) - with open(self.PATH, 'w', encoding='utf-8') as file_object: - file_object.write(file_content) + with open(self.PATH, "w", encoding="utf-8") as file_object: + file_object.write(file_content) diff --git a/l2tdevtools/dependency_writers/check_dependencies.py b/l2tdevtools/dependency_writers/check_dependencies.py index f91782d4..70263382 100644 --- a/l2tdevtools/dependency_writers/check_dependencies.py +++ b/l2tdevtools/dependency_writers/check_dependencies.py @@ -6,28 +6,29 @@ class CheckDependenciesWriter(interface.DependencyFileWriter): - """Check dependencies script writer.""" + """Check dependencies script writer.""" - _TEMPLATE_DIRECTORY = os.path.join('data', 'templates') + _TEMPLATE_DIRECTORY = os.path.join("data", "templates") - _PROJECTS_WITH_PYTHON3_AS_DEFAULT = ('l2tscaffolder', 'plaso') + _PROJECTS_WITH_PYTHON3_AS_DEFAULT = ("l2tscaffolder", "plaso") - PATH = os.path.join('utils', 'check_dependencies.py') + PATH = os.path.join("utils", "check_dependencies.py") - def Write(self): - """Writes a check_dependencies.py file.""" - template_mappings = { - 'project_name': self._project_definition.name, - } + def Write(self): + """Writes a check_dependencies.py file.""" + template_mappings = { + "project_name": self._project_definition.name, + } - if self._project_definition.name == 'plaso': - template_file = 'check_dependencies-with_url.py' - else: - template_file = 'check_dependencies.py' + if self._project_definition.name == "plaso": + template_file = "check_dependencies-with_url.py" + else: + template_file = "check_dependencies.py" - template_file = os.path.join( - self._l2tdevtools_path, self._TEMPLATE_DIRECTORY, template_file) - file_content = self._GenerateFromTemplate(template_file, template_mappings) + template_file = os.path.join( + self._l2tdevtools_path, self._TEMPLATE_DIRECTORY, template_file + ) + file_content = self._GenerateFromTemplate(template_file, template_mappings) - with open(self.PATH, 'w', encoding='utf-8') as file_object: - file_object.write(file_content) + with open(self.PATH, "w", encoding="utf-8") as file_object: + file_object.write(file_content) diff --git a/l2tdevtools/dependency_writers/dependencies_py.py b/l2tdevtools/dependency_writers/dependencies_py.py index 92ea5c69..00429e7a 100644 --- a/l2tdevtools/dependency_writers/dependencies_py.py +++ b/l2tdevtools/dependency_writers/dependencies_py.py @@ -6,44 +6,46 @@ class DependenciesPyWriter(interface.DependencyFileWriter): - """Dependencies.py file writer.""" + """Dependencies.py file writer.""" - _TEMPLATE_FILE = os.path.join('data', 'templates', 'dependencies.py') + _TEMPLATE_FILE = os.path.join("data", "templates", "dependencies.py") - PATH = os.path.join('plaso', 'dependencies.py') + PATH = os.path.join("plaso", "dependencies.py") - def Write(self): - """Writes a dependencies.py file.""" - dependencies = [] - for dependency in sorted( - self._dependency_helper.dependencies.values(), - key=lambda dependency: dependency.name.lower()): - if not dependency.skip_check: - dependencies.append(dependency) + def Write(self): + """Writes a dependencies.py file.""" + dependencies = [] + for dependency in sorted( + self._dependency_helper.dependencies.values(), + key=lambda dependency: dependency.name.lower(), + ): + if not dependency.skip_check: + dependencies.append(dependency) - python_dependencies = [] - for dependency in dependencies: - if dependency.maximum_version: - maximum_version = f'\'{dependency.maximum_version:s}\'' - else: - maximum_version = 'None' + python_dependencies = [] + for dependency in dependencies: + if dependency.maximum_version: + maximum_version = f"'{dependency.maximum_version:s}'" + else: + maximum_version = "None" - version_property = dependency.version_property or '' - minimum_version = dependency.minimum_version or '' - not_optional = not dependency.is_optional + version_property = dependency.version_property or "" + minimum_version = dependency.minimum_version or "" + not_optional = not dependency.is_optional - python_dependency = ( - f' \'{dependency.name:s}\': (\'{version_property:s}\', ' - f'\'{minimum_version:s}\', {maximum_version:s}, {not_optional!s})') + python_dependency = ( + f" '{dependency.name:s}': ('{version_property:s}', " + f"'{minimum_version:s}', {maximum_version:s}, {not_optional!s})" + ) - python_dependencies.append(python_dependency) + python_dependencies.append(python_dependency) - python_dependencies = ',\n'.join(python_dependencies) + python_dependencies = ",\n".join(python_dependencies) - template_mappings = {'python_dependencies': python_dependencies} + template_mappings = {"python_dependencies": python_dependencies} - template_file = os.path.join(self._l2tdevtools_path, self._TEMPLATE_FILE) - file_content = self._GenerateFromTemplate(template_file, template_mappings) + template_file = os.path.join(self._l2tdevtools_path, self._TEMPLATE_FILE) + file_content = self._GenerateFromTemplate(template_file, template_mappings) - with open(self.PATH, 'w', encoding='utf-8') as file_object: - file_object.write(file_content) + with open(self.PATH, "w", encoding="utf-8") as file_object: + file_object.write(file_content) diff --git a/l2tdevtools/dependency_writers/dpkg.py b/l2tdevtools/dependency_writers/dpkg.py index 8de9dd63..9da17d30 100644 --- a/l2tdevtools/dependency_writers/dpkg.py +++ b/l2tdevtools/dependency_writers/dpkg.py @@ -6,185 +6,207 @@ class DPKGCompatWriter(interface.DependencyFileWriter): - """Dpkg compat file writer.""" + """Dpkg compat file writer.""" - PATH = os.path.join('config', 'dpkg', 'compat') + PATH = os.path.join("config", "dpkg", "compat") - _FILE_CONTENT = '9\n' + _FILE_CONTENT = "9\n" - def Write(self): - """Writes a dpkg control file.""" - with open(self.PATH, 'w', encoding='utf-8') as file_object: - file_object.write(self._FILE_CONTENT) + def Write(self): + """Writes a dpkg control file.""" + with open(self.PATH, "w", encoding="utf-8") as file_object: + file_object.write(self._FILE_CONTENT) class DPKGControlWriter(interface.DependencyFileWriter): - """Dpkg control file writer.""" - - PATH = os.path.join('config', 'dpkg', 'control') - - _PYTHON3_FILE_HEADER = [ - 'Source: {project_name:s}', - 'Section: python', - 'Priority: extra', - 'Maintainer: {maintainer:s}', - 'Build-Depends: debhelper (>= 9), dh-python, {build_dependencies:s}', - 'Standards-Version: 4.1.4', - 'X-Python3-Version: >= 3.10', - 'Homepage: {homepage_url:s}', - ''] - - _DATA_PACKAGE = [ - 'Package: {project_name:s}-data', - 'Architecture: all', - 'Depends: ${{misc:Depends}}', - 'Description: Data files for {name_description:s}', - '{description_long:s}', - ''] - - _PYTHON3_PACKAGE = [ - 'Package: python3-{python_module_name:s}', - 'Architecture: all', - 'Depends: {python3_dependencies:s}${{misc:Depends}}', - 'Description: Python 3 module of {python_module_description:s}', - '{description_long:s}', - ''] - - _TOOLS_PACKAGE = [ - 'Package: {project_name:s}-tools', - 'Architecture: all', - ('Depends: python3-{python_module_name:s} (>= ${{binary:Version}}), ' - '${{misc:Depends}}'), - 'Description: {tools_description:s}', - '{tool_description_long:s}', - ''] - - def Write(self): - """Writes a dpkg control file.""" - python_module_description = self._project_definition.name_description - python_module_name = self._project_definition.name - tools_description = ( - f'Tools of {self._project_definition.name_description:s}') - tool_description_long = self._project_definition.description_long - - if self._project_definition.name.endswith('-kb'): - python_module_name = ''.join([python_module_name[:-3], 'rc']) - - python_module_description, _, _ = python_module_description.partition( - ' knowledge base ') - python_module_description = ''.join([ - python_module_description, f' resources ({python_module_name:s})']) - - tools_description = ( - f'Tools for {self._project_definition.name_description:s}') - - tool_description_long, _, _ = ( - self._project_definition.name_description.rpartition(' (')) - tool_description_long = ( - f'{self._project_definition.name[0].upper()}' - f'{self._project_definition.name[1:]:s} ' - f'is a project to build a {tool_description_long:s}.') - - tool_description_long = '\n'.join( - [f' {line:s}' for line in tool_description_long.split('\n')]) - - file_content = [] - file_content.extend(self._PYTHON3_FILE_HEADER) - - data_dependency = '' - if self._project_definition.name in ('artifacts', 'plaso'): - data_dependency = ( - f'{self._project_definition.name:s}-data (>= ${{binary:Version}})') - - file_content.extend(self._DATA_PACKAGE) - - file_content.extend(self._PYTHON3_PACKAGE) - - if (os.path.isdir('scripts') or os.path.isdir('tools') or - os.path.isdir(os.path.join(python_module_name, 'scripts'))): - file_content.extend(self._TOOLS_PACKAGE) - - description_long = self._project_definition.description_long - description_long = '\n'.join( - [f' {line:s}' for line in description_long.split('\n')]) - - python3_dependencies = self._dependency_helper.GetDPKGDepends() - - if data_dependency: - python3_dependencies.insert(0, data_dependency) - - python3_dependencies = ', '.join(python3_dependencies) - if python3_dependencies: - python3_dependencies = f'{python3_dependencies:s}, ' - - build_dependencies = [ - 'python3-all (>= 3.10~)', 'python3-setuptools', - 'pybuild-plugin-pyproject'] - - build_dependencies = ', '.join(build_dependencies) - - template_mappings = { - 'build_dependencies': build_dependencies, - 'description_long': description_long, - 'description_short': self._project_definition.description_short, - 'homepage_url': self._project_definition.homepage_url, - 'maintainer': self._project_definition.maintainer, - 'name_description': self._project_definition.name_description, - 'project_name': self._project_definition.name, - 'python_module_description': python_module_description, - 'python_module_name': python_module_name, - 'python3_dependencies': python3_dependencies, - 'tools_description': tools_description, - 'tool_description_long': tool_description_long} - - file_content = '\n'.join(file_content) - file_content = file_content.format(**template_mappings) - - with open(self.PATH, 'w', encoding='utf-8') as file_object: - file_object.write(file_content) + """Dpkg control file writer.""" + + PATH = os.path.join("config", "dpkg", "control") + + _PYTHON3_FILE_HEADER = [ + "Source: {project_name:s}", + "Section: python", + "Priority: extra", + "Maintainer: {maintainer:s}", + "Build-Depends: debhelper (>= 9), dh-python, {build_dependencies:s}", + "Standards-Version: 4.1.4", + "X-Python3-Version: >= 3.10", + "Homepage: {homepage_url:s}", + "", + ] + + _DATA_PACKAGE = [ + "Package: {project_name:s}-data", + "Architecture: all", + "Depends: ${{misc:Depends}}", + "Description: Data files for {name_description:s}", + "{description_long:s}", + "", + ] + + _PYTHON3_PACKAGE = [ + "Package: python3-{python_module_name:s}", + "Architecture: all", + "Depends: {python3_dependencies:s}${{misc:Depends}}", + "Description: Python 3 module of {python_module_description:s}", + "{description_long:s}", + "", + ] + + _TOOLS_PACKAGE = [ + "Package: {project_name:s}-tools", + "Architecture: all", + ( + "Depends: python3-{python_module_name:s} (>= ${{binary:Version}}), " + "${{misc:Depends}}" + ), + "Description: {tools_description:s}", + "{tool_description_long:s}", + "", + ] + + def Write(self): + """Writes a dpkg control file.""" + python_module_description = self._project_definition.name_description + python_module_name = self._project_definition.name + tools_description = f"Tools of {self._project_definition.name_description:s}" + tool_description_long = self._project_definition.description_long + + if self._project_definition.name.endswith("-kb"): + python_module_name = "".join([python_module_name[:-3], "rc"]) + + python_module_description, _, _ = python_module_description.partition( + " knowledge base " + ) + python_module_description = "".join( + [python_module_description, f" resources ({python_module_name:s})"] + ) + + tools_description = ( + f"Tools for {self._project_definition.name_description:s}" + ) + + tool_description_long, _, _ = ( + self._project_definition.name_description.rpartition(" (") + ) + tool_description_long = ( + f"{self._project_definition.name[0].upper()}" + f"{self._project_definition.name[1:]:s} " + f"is a project to build a {tool_description_long:s}." + ) + + tool_description_long = "\n".join( + [f" {line:s}" for line in tool_description_long.split("\n")] + ) + + file_content = [] + file_content.extend(self._PYTHON3_FILE_HEADER) + + data_dependency = "" + if self._project_definition.name in ("artifacts", "plaso"): + data_dependency = ( + f"{self._project_definition.name:s}-data (>= ${{binary:Version}})" + ) + + file_content.extend(self._DATA_PACKAGE) + + file_content.extend(self._PYTHON3_PACKAGE) + + if ( + os.path.isdir("scripts") + or os.path.isdir("tools") + or os.path.isdir(os.path.join(python_module_name, "scripts")) + ): + file_content.extend(self._TOOLS_PACKAGE) + + description_long = self._project_definition.description_long + description_long = "\n".join( + [f" {line:s}" for line in description_long.split("\n")] + ) + + python3_dependencies = self._dependency_helper.GetDPKGDepends() + + if data_dependency: + python3_dependencies.insert(0, data_dependency) + + python3_dependencies = ", ".join(python3_dependencies) + if python3_dependencies: + python3_dependencies = f"{python3_dependencies:s}, " + + build_dependencies = [ + "python3-all (>= 3.10~)", + "python3-setuptools", + "pybuild-plugin-pyproject", + ] + + build_dependencies = ", ".join(build_dependencies) + + template_mappings = { + "build_dependencies": build_dependencies, + "description_long": description_long, + "description_short": self._project_definition.description_short, + "homepage_url": self._project_definition.homepage_url, + "maintainer": self._project_definition.maintainer, + "name_description": self._project_definition.name_description, + "project_name": self._project_definition.name, + "python_module_description": python_module_description, + "python_module_name": python_module_name, + "python3_dependencies": python3_dependencies, + "tools_description": tools_description, + "tool_description_long": tool_description_long, + } + + file_content = "\n".join(file_content) + file_content = file_content.format(**template_mappings) + + with open(self.PATH, "w", encoding="utf-8") as file_object: + file_object.write(file_content) class DPKGRulesWriter(interface.DependencyFileWriter): - """Dpkg rules file writer.""" - - PATH = os.path.join('config', 'dpkg', 'rules') - - _HEADER = [ - '#!/usr/bin/make -f', - '', - '%:', - '\tdh $@ --buildsystem=pybuild --with=python3', - '', - '.PHONY: override_dh_auto_test', - 'override_dh_auto_test:', - '', - ''] - - _DATA_PACKAGE = [ - '.PHONY: override_dh_auto_install', - 'override_dh_auto_install:', - '\tdh_auto_install', - '\tmkdir -p debian/tmp/usr/share/{project_name:s}', - ('\tmv -n debian/tmp/usr/lib/python*/dist-packages/{project_name:s}' - '/data/* debian/tmp/usr/share/{project_name:s}'), - '\trm -rf debian/tmp/usr/lib/python*/dist-packages/{project_name:s}/data', - '\tfind debian/tmp/usr/bin/ -type f -exec mv {{}} {{}}.py \\;', - '', - ''] - - def Write(self): - """Writes a dpkg control file.""" - template_mappings = { - 'project_name': self._project_definition.name} - - file_content = [] - file_content.extend(self._HEADER) - - if self._project_definition.name in ('artifacts', 'plaso'): - file_content.extend(self._DATA_PACKAGE) - - file_content = '\n'.join(file_content) - file_content = file_content.format(**template_mappings) - - with open(self.PATH, 'w', encoding='utf-8') as file_object: - file_object.write(file_content) + """Dpkg rules file writer.""" + + PATH = os.path.join("config", "dpkg", "rules") + + _HEADER = [ + "#!/usr/bin/make -f", + "", + "%:", + "\tdh $@ --buildsystem=pybuild --with=python3", + "", + ".PHONY: override_dh_auto_test", + "override_dh_auto_test:", + "", + "", + ] + + _DATA_PACKAGE = [ + ".PHONY: override_dh_auto_install", + "override_dh_auto_install:", + "\tdh_auto_install", + "\tmkdir -p debian/tmp/usr/share/{project_name:s}", + ( + "\tmv -n debian/tmp/usr/lib/python*/dist-packages/{project_name:s}" + "/data/* debian/tmp/usr/share/{project_name:s}" + ), + "\trm -rf debian/tmp/usr/lib/python*/dist-packages/{project_name:s}/data", + "\tfind debian/tmp/usr/bin/ -type f -exec mv {{}} {{}}.py \\;", + "", + "", + ] + + def Write(self): + """Writes a dpkg control file.""" + template_mappings = {"project_name": self._project_definition.name} + + file_content = [] + file_content.extend(self._HEADER) + + if self._project_definition.name in ("artifacts", "plaso"): + file_content.extend(self._DATA_PACKAGE) + + file_content = "\n".join(file_content) + file_content = file_content.format(**template_mappings) + + with open(self.PATH, "w", encoding="utf-8") as file_object: + file_object.write(file_content) diff --git a/l2tdevtools/dependency_writers/gift_copr.py b/l2tdevtools/dependency_writers/gift_copr.py index c739b64b..60df4f8a 100644 --- a/l2tdevtools/dependency_writers/gift_copr.py +++ b/l2tdevtools/dependency_writers/gift_copr.py @@ -6,164 +6,169 @@ class GIFTCOPRInstallScriptWriter(interface.DependencyFileWriter): - """GIFT COPR installation script file writer.""" + """GIFT COPR installation script file writer.""" - _TEMPLATE_FILE = os.path.join('data', 'templates', 'gift_copr_install.sh') + _TEMPLATE_FILE = os.path.join("data", "templates", "gift_copr_install.sh") - PATH = os.path.join('config', 'linux', 'gift_copr_install.sh') + PATH = os.path.join("config", "linux", "gift_copr_install.sh") - def _FormatRPMDebugDependencies(self, debug_dependencies): - """Formats RPM debug dependencies for the template. + def _FormatRPMDebugDependencies(self, debug_dependencies): + """Formats RPM debug dependencies for the template. - Args: - debug_dependencies (list[str]): RPM package names of debug dependencies. + Args: + debug_dependencies (list[str]): RPM package names of debug dependencies. - Returns: - str: formatted RPM debug dependencies. - """ - formatted_debug_dependencies = [] - if debug_dependencies: - for index, dependency in enumerate(sorted(debug_dependencies)): - if index == 0: - line = f'DEBUG_DEPENDENCIES="{dependency:s}' - else: - line = f' {dependency:s}' + Returns: + str: formatted RPM debug dependencies. + """ + formatted_debug_dependencies = [] + if debug_dependencies: + for index, dependency in enumerate(sorted(debug_dependencies)): + if index == 0: + line = f'DEBUG_DEPENDENCIES="{dependency:s}' + else: + line = f" {dependency:s}" - if index + 1 == len(debug_dependencies): - line = f'{line:s}";' + if index + 1 == len(debug_dependencies): + line = f'{line:s}";' - formatted_debug_dependencies.append(line) + formatted_debug_dependencies.append(line) - return '\n'.join(formatted_debug_dependencies) + return "\n".join(formatted_debug_dependencies) - def _FormatRPMDevelopmentDependencies(self, development_dependencies): - """Formats RPM development dependencies for the template. + def _FormatRPMDevelopmentDependencies(self, development_dependencies): + """Formats RPM development dependencies for the template. - Args: - development_dependencies (list[str]): RPM package names of development - dependencies. + Args: + development_dependencies (list[str]): RPM package names of development + dependencies. - Returns: - str: formatted RPM development dependencies. - """ - formatted_development_dependencies = [] - if development_dependencies: - for index, dependency in enumerate(sorted(development_dependencies)): - if index == 0: - line = f'DEVELOPMENT_DEPENDENCIES="{dependency:s}' - else: - line = f' {dependency:s}' + Returns: + str: formatted RPM development dependencies. + """ + formatted_development_dependencies = [] + if development_dependencies: + for index, dependency in enumerate(sorted(development_dependencies)): + if index == 0: + line = f'DEVELOPMENT_DEPENDENCIES="{dependency:s}' + else: + line = f" {dependency:s}" - if index + 1 == len(development_dependencies): - line = f'{line:s}";' + if index + 1 == len(development_dependencies): + line = f'{line:s}";' - formatted_development_dependencies.append(line) + formatted_development_dependencies.append(line) - return '\n'.join(formatted_development_dependencies) + return "\n".join(formatted_development_dependencies) - def _FormatRPMPythonDependencies(self, python_dependencies): - """Formats RPM Python dependencies for the template. + def _FormatRPMPythonDependencies(self, python_dependencies): + """Formats RPM Python dependencies for the template. - Args: - python_dependencies (list[str]): RPM package names of Python dependencies. + Args: + python_dependencies (list[str]): RPM package names of Python dependencies. - Returns: - str: formatted RPM Python dependencies. - """ - formatted_python_dependencies = [] + Returns: + str: formatted RPM Python dependencies. + """ + formatted_python_dependencies = [] - for index, dependency in enumerate(sorted(python_dependencies)): - if index == 0: - line = f'PYTHON3_DEPENDENCIES="{dependency:s}' - else: - line = f' {dependency:s}' + for index, dependency in enumerate(sorted(python_dependencies)): + if index == 0: + line = f'PYTHON3_DEPENDENCIES="{dependency:s}' + else: + line = f" {dependency:s}" - if index + 1 == len(python_dependencies): - line = f'{line:s}";' + if index + 1 == len(python_dependencies): + line = f'{line:s}";' - formatted_python_dependencies.append(line) + formatted_python_dependencies.append(line) - return '\n'.join(formatted_python_dependencies) + return "\n".join(formatted_python_dependencies) - def _FormatRPMTestDependencies(self, test_dependencies): - """Formats RPM test dependencies for the template. + def _FormatRPMTestDependencies(self, test_dependencies): + """Formats RPM test dependencies for the template. - Args: - test_dependencies (list[str]): RPM package names of test dependencies. + Args: + test_dependencies (list[str]): RPM package names of test dependencies. - Returns: - str: formatted RPM test dependencies. - """ - formatted_test_dependencies = [] - if test_dependencies: - for index, dependency in enumerate(sorted(test_dependencies)): - if index == 0: - line = f'TEST_DEPENDENCIES="{dependency:s}' - else: - line = f' {dependency:s}' + Returns: + str: formatted RPM test dependencies. + """ + formatted_test_dependencies = [] + if test_dependencies: + for index, dependency in enumerate(sorted(test_dependencies)): + if index == 0: + line = f'TEST_DEPENDENCIES="{dependency:s}' + else: + line = f" {dependency:s}" - if index + 1 == len(test_dependencies): - line = f'{line:s}";' + if index + 1 == len(test_dependencies): + line = f'{line:s}";' - formatted_test_dependencies.append(line) + formatted_test_dependencies.append(line) - return '\n'.join(formatted_test_dependencies) + return "\n".join(formatted_test_dependencies) - def _GetRPMDebugDependencies(self, python_dependencies): - """Retrieves RPM debug dependencies. + def _GetRPMDebugDependencies(self, python_dependencies): + """Retrieves RPM debug dependencies. - Args: - python_dependencies (list[str]): RPM package names of Python dependencies. + Args: + python_dependencies (list[str]): RPM package names of Python dependencies. - Returns: - list[str]: RPM package names of Python debug dependencies. - """ - debug_dependencies = [] - for dependency in sorted(python_dependencies): - if dependency.startswith('lib') and ( - dependency.endswith('python') or dependency.endswith('python2') or - dependency.endswith('python3')): - libyal_dependency, _, _ = dependency.partition('-') - debug_dependencies.extend([ - f'{libyal_dependency:s}-debuginfo', - f'{dependency:s}-debuginfo']) + Returns: + list[str]: RPM package names of Python debug dependencies. + """ + debug_dependencies = [] + for dependency in sorted(python_dependencies): + if dependency.startswith("lib") and ( + dependency.endswith("python") + or dependency.endswith("python2") + or dependency.endswith("python3") + ): + libyal_dependency, _, _ = dependency.partition("-") + debug_dependencies.extend( + [f"{libyal_dependency:s}-debuginfo", f"{dependency:s}-debuginfo"] + ) - return debug_dependencies + return debug_dependencies - def Write(self): - """Writes a gift_copr_install.sh file.""" - python_dependencies = self._GetRPMPythonDependencies() - test_dependencies = self._GetRPMTestDependencies(python_dependencies) + def Write(self): + """Writes a gift_copr_install.sh file.""" + python_dependencies = self._GetRPMPythonDependencies() + test_dependencies = self._GetRPMTestDependencies(python_dependencies) + + # TODO: replace by dev_dependencies.ini or equiv. + development_dependencies = ["pylint"] + + if self._project_definition.name == "plaso": + development_dependencies.append("python-sphinx") - # TODO: replace by dev_dependencies.ini or equiv. - development_dependencies = ['pylint'] + debug_dependencies = self._GetRPMDebugDependencies(python_dependencies) + + formatted_python_dependencies = self._FormatRPMPythonDependencies( + python_dependencies + ) - if self._project_definition.name == 'plaso': - development_dependencies.append('python-sphinx') + formatted_test_dependencies = self._FormatRPMTestDependencies(test_dependencies) - debug_dependencies = self._GetRPMDebugDependencies(python_dependencies) + formatted_development_dependencies = self._FormatRPMDevelopmentDependencies( + development_dependencies + ) - formatted_python_dependencies = self._FormatRPMPythonDependencies( - python_dependencies) + formatted_debug_dependencies = self._FormatRPMDebugDependencies( + debug_dependencies + ) - formatted_test_dependencies = self._FormatRPMTestDependencies( - test_dependencies) + template_mappings = { + "debug_dependencies": formatted_debug_dependencies, + "development_dependencies": formatted_development_dependencies, + "project_name": self._project_definition.name, + "python_dependencies": formatted_python_dependencies, + "test_dependencies": formatted_test_dependencies, + } - formatted_development_dependencies = self._FormatRPMDevelopmentDependencies( - development_dependencies) + template_file = os.path.join(self._l2tdevtools_path, self._TEMPLATE_FILE) + file_content = self._GenerateFromTemplate(template_file, template_mappings) - formatted_debug_dependencies = self._FormatRPMDebugDependencies( - debug_dependencies) - - template_mappings = { - 'debug_dependencies': formatted_debug_dependencies, - 'development_dependencies': formatted_development_dependencies, - 'project_name': self._project_definition.name, - 'python_dependencies': formatted_python_dependencies, - 'test_dependencies': formatted_test_dependencies} - - template_file = os.path.join(self._l2tdevtools_path, self._TEMPLATE_FILE) - file_content = self._GenerateFromTemplate(template_file, template_mappings) - - with open(self.PATH, 'w', encoding='utf-8') as file_object: - file_object.write(file_content) + with open(self.PATH, "w", encoding="utf-8") as file_object: + file_object.write(file_content) diff --git a/l2tdevtools/dependency_writers/gift_ppa.py b/l2tdevtools/dependency_writers/gift_ppa.py index 3c14f54f..08923710 100644 --- a/l2tdevtools/dependency_writers/gift_ppa.py +++ b/l2tdevtools/dependency_writers/gift_ppa.py @@ -6,172 +6,177 @@ class GIFTPPAInstallScriptWriter(interface.DependencyFileWriter): - """GIFT PPA installation script file writer.""" + """GIFT PPA installation script file writer.""" - _TEMPLATE_FILE = os.path.join('data', 'templates', 'gift_ppa_install.sh') + _TEMPLATE_FILE = os.path.join("data", "templates", "gift_ppa_install.sh") - # Path to GIFT PPA installation script. - PATH = os.path.join('config', 'linux', 'gift_ppa_install_py3.sh') + # Path to GIFT PPA installation script. + PATH = os.path.join("config", "linux", "gift_ppa_install_py3.sh") - def _FormatDPKGDebugDependencies(self, debug_dependencies): - """Formats DPKG debug dependencies for the template. + def _FormatDPKGDebugDependencies(self, debug_dependencies): + """Formats DPKG debug dependencies for the template. - Args: - debug_dependencies (list[str]): DPKG package names of debug dependencies. + Args: + debug_dependencies (list[str]): DPKG package names of debug dependencies. - Returns: - str: formatted DPKG debug dependencies. - """ - formatted_debug_dependencies = [] - if debug_dependencies: - for index, dependency in enumerate(sorted(debug_dependencies)): - if index == 0: - line = f'DEBUG_DEPENDENCIES="{dependency:s}' - else: - line = f' {dependency:s}' + Returns: + str: formatted DPKG debug dependencies. + """ + formatted_debug_dependencies = [] + if debug_dependencies: + for index, dependency in enumerate(sorted(debug_dependencies)): + if index == 0: + line = f'DEBUG_DEPENDENCIES="{dependency:s}' + else: + line = f" {dependency:s}" - if index + 1 == len(debug_dependencies): - line = f'{line:s}";' + if index + 1 == len(debug_dependencies): + line = f'{line:s}";' - formatted_debug_dependencies.append(line) + formatted_debug_dependencies.append(line) - return '\n'.join(formatted_debug_dependencies) + return "\n".join(formatted_debug_dependencies) - def _FormatDPKGDevelopmentDependencies(self, development_dependencies): - """Formats DPKG development dependencies for the template. + def _FormatDPKGDevelopmentDependencies(self, development_dependencies): + """Formats DPKG development dependencies for the template. - Args: - development_dependencies (list[str]): DPKG package names of development - dependencies. + Args: + development_dependencies (list[str]): DPKG package names of development + dependencies. - Returns: - str: formatted DPKG development dependencies. - """ - formatted_development_dependencies = [] - if development_dependencies: - for index, dependency in enumerate(sorted(development_dependencies)): - if index == 0: - line = f'DEVELOPMENT_DEPENDENCIES="{dependency:s}' - else: - line = f' {dependency:s}' + Returns: + str: formatted DPKG development dependencies. + """ + formatted_development_dependencies = [] + if development_dependencies: + for index, dependency in enumerate(sorted(development_dependencies)): + if index == 0: + line = f'DEVELOPMENT_DEPENDENCIES="{dependency:s}' + else: + line = f" {dependency:s}" - if index + 1 == len(development_dependencies): - line = f'{line:s}";' + if index + 1 == len(development_dependencies): + line = f'{line:s}";' - formatted_development_dependencies.append(line) + formatted_development_dependencies.append(line) - return '\n'.join(formatted_development_dependencies) + return "\n".join(formatted_development_dependencies) - def _FormatDPKGPythonDependencies(self, python_dependencies): - """Formats DPKG Python dependencies for the template. + def _FormatDPKGPythonDependencies(self, python_dependencies): + """Formats DPKG Python dependencies for the template. - Args: - python_dependencies (list[str]): DPKG package names of Python - dependencies. + Args: + python_dependencies (list[str]): DPKG package names of Python + dependencies. - Returns: - str: formatted DPKG Python dependencies. - """ - formatted_python_dependencies = [] + Returns: + str: formatted DPKG Python dependencies. + """ + formatted_python_dependencies = [] - for index, dependency in enumerate(sorted(python_dependencies)): - if index == 0: - line = f'PYTHON_DEPENDENCIES="{dependency:s}' - else: - line = f' {dependency:s}' + for index, dependency in enumerate(sorted(python_dependencies)): + if index == 0: + line = f'PYTHON_DEPENDENCIES="{dependency:s}' + else: + line = f" {dependency:s}" - if index + 1 == len(python_dependencies): - line = f'{line:s}";' + if index + 1 == len(python_dependencies): + line = f'{line:s}";' - formatted_python_dependencies.append(line) + formatted_python_dependencies.append(line) - return '\n'.join(formatted_python_dependencies) + return "\n".join(formatted_python_dependencies) - def _FormatDPKGTestDependencies(self, test_dependencies): - """Formats DPKG test dependencies for the template. + def _FormatDPKGTestDependencies(self, test_dependencies): + """Formats DPKG test dependencies for the template. - Args: - test_dependencies (list[str]): DPKG package names of test dependencies. + Args: + test_dependencies (list[str]): DPKG package names of test dependencies. - Returns: - str: formatted DPKG test dependencies. - """ - formatted_test_dependencies = [] - if test_dependencies: - for index, dependency in enumerate(sorted(test_dependencies)): - if index == 0: - line = f'TEST_DEPENDENCIES="{dependency:s}' - else: - line = f' {dependency:s}' + Returns: + str: formatted DPKG test dependencies. + """ + formatted_test_dependencies = [] + if test_dependencies: + for index, dependency in enumerate(sorted(test_dependencies)): + if index == 0: + line = f'TEST_DEPENDENCIES="{dependency:s}' + else: + line = f" {dependency:s}" - if index + 1 == len(test_dependencies): - line = f'{line:s}";' + if index + 1 == len(test_dependencies): + line = f'{line:s}";' - formatted_test_dependencies.append(line) + formatted_test_dependencies.append(line) - return '\n'.join(formatted_test_dependencies) + return "\n".join(formatted_test_dependencies) - def _GetDPKGDebugDependencies(self, python_dependencies): - """Retrieves DPKG debug dependencies. + def _GetDPKGDebugDependencies(self, python_dependencies): + """Retrieves DPKG debug dependencies. - Args: - python_dependencies (list[str]): DPKG package names of Python - dependencies. + Args: + python_dependencies (list[str]): DPKG package names of Python + dependencies. - Returns: - list[str]: DPKG package names of Python debug dependencies. - """ - debug_dependencies = [] - for dependency in sorted(python_dependencies): - if dependency.startswith('lib') and dependency.endswith('python3'): - dependency, _, _ = dependency.partition('-') - debug_dependencies.extend([ - f'{dependency:s}-dbg', - f'{dependency:s}-python3-dbg']) + Returns: + list[str]: DPKG package names of Python debug dependencies. + """ + debug_dependencies = [] + for dependency in sorted(python_dependencies): + if dependency.startswith("lib") and dependency.endswith("python3"): + dependency, _, _ = dependency.partition("-") + debug_dependencies.extend( + [f"{dependency:s}-dbg", f"{dependency:s}-python3-dbg"] + ) - return debug_dependencies + return debug_dependencies - def _Write(self): - """Writes a gift_ppa_install.sh file.""" - python_dependencies = self._GetDPKGPythonDependencies() - test_dependencies = self._GetDPKGTestDependencies(python_dependencies) + def _Write(self): + """Writes a gift_ppa_install.sh file.""" + python_dependencies = self._GetDPKGPythonDependencies() + test_dependencies = self._GetDPKGTestDependencies(python_dependencies) - # TODO: replace by dev_dependencies.ini or equiv. - test_dependencies.extend(['python3-setuptools']) + # TODO: replace by dev_dependencies.ini or equiv. + test_dependencies.extend(["python3-setuptools"]) - # TODO: replace by dev_dependencies.ini or equiv. - development_dependencies = ['pylint'] + # TODO: replace by dev_dependencies.ini or equiv. + development_dependencies = ["pylint"] - if self._project_definition.name == 'plaso': - development_dependencies.append('python-sphinx') + if self._project_definition.name == "plaso": + development_dependencies.append("python-sphinx") + + debug_dependencies = self._GetDPKGDebugDependencies(python_dependencies) - debug_dependencies = self._GetDPKGDebugDependencies(python_dependencies) + formatted_python_dependencies = self._FormatDPKGPythonDependencies( + python_dependencies + ) - formatted_python_dependencies = self._FormatDPKGPythonDependencies( - python_dependencies) + formatted_test_dependencies = self._FormatDPKGTestDependencies( + test_dependencies + ) - formatted_test_dependencies = self._FormatDPKGTestDependencies( - test_dependencies) + formatted_development_dependencies = self._FormatDPKGDevelopmentDependencies( + development_dependencies + ) - formatted_development_dependencies = ( - self._FormatDPKGDevelopmentDependencies(development_dependencies)) + formatted_debug_dependencies = self._FormatDPKGDebugDependencies( + debug_dependencies + ) - formatted_debug_dependencies = self._FormatDPKGDebugDependencies( - debug_dependencies) + template_mappings = { + "debug_dependencies": formatted_debug_dependencies, + "development_dependencies": formatted_development_dependencies, + "project_name": self._project_definition.name, + "python_dependencies": formatted_python_dependencies, + "test_dependencies": formatted_test_dependencies, + } - template_mappings = { - 'debug_dependencies': formatted_debug_dependencies, - 'development_dependencies': formatted_development_dependencies, - 'project_name': self._project_definition.name, - 'python_dependencies': formatted_python_dependencies, - 'test_dependencies': formatted_test_dependencies} + template_file = os.path.join(self._l2tdevtools_path, self._TEMPLATE_FILE) + file_content = self._GenerateFromTemplate(template_file, template_mappings) - template_file = os.path.join(self._l2tdevtools_path, self._TEMPLATE_FILE) - file_content = self._GenerateFromTemplate(template_file, template_mappings) + with open(self.PATH, "w", encoding="utf-8") as file_object: + file_object.write(file_content) - with open(self.PATH, 'w', encoding='utf-8') as file_object: - file_object.write(file_content) - - def Write(self): - """Writes a gift_ppa_install.sh file.""" - self._Write() + def Write(self): + """Writes a gift_ppa_install.sh file.""" + self._Write() diff --git a/l2tdevtools/dependency_writers/github_actions.py b/l2tdevtools/dependency_writers/github_actions.py index 8a7878cf..9ff19b38 100644 --- a/l2tdevtools/dependency_writers/github_actions.py +++ b/l2tdevtools/dependency_writers/github_actions.py @@ -7,190 +7,213 @@ class GitHubActionsLintYmlWriter(interface.DependencyFileWriter): - """lint.yml GitHub actions workflow file writer.""" + """lint.yml GitHub actions workflow file writer.""" - _TEMPLATE_DIRECTORY = os.path.join( - 'data', 'templates', 'github_actions', 'lint.yml') + _TEMPLATE_DIRECTORY = os.path.join( + "data", "templates", "github_actions", "lint.yml" + ) - PATH = os.path.join('.github', 'workflows', 'lint.yml') + PATH = os.path.join(".github", "workflows", "lint.yml") - def _GenerateFromTemplate(self, template_filename, template_mappings): - """Generates file context based on a template file. + def _GenerateFromTemplate(self, template_filename, template_mappings): + """Generates file context based on a template file. - Args: - template_filename (str): path of the template file. - template_mappings (dict[str, str]): template mappings, where the key - maps to the name of a template variable. + Args: + template_filename (str): path of the template file. + template_mappings (dict[str, str]): template mappings, where the key + maps to the name of a template variable. - Returns: - str: output based on the template string. + Returns: + str: output based on the template string. - Raises: - RuntimeError: if the template cannot be formatted. - """ - template_filename = os.path.join( - self._l2tdevtools_path, self._TEMPLATE_DIRECTORY, template_filename) - return super()._GenerateFromTemplate( - template_filename, template_mappings) + Raises: + RuntimeError: if the template cannot be formatted. + """ + template_filename = os.path.join( + self._l2tdevtools_path, self._TEMPLATE_DIRECTORY, template_filename + ) + return super()._GenerateFromTemplate(template_filename, template_mappings) - def Write(self): - """Writes a lint.yml GitHub actions workflow file .""" - dpkg_dependencies = self._GetDPKGPythonDependencies() - test_dependencies = self._GetDPKGTestDependencies(dpkg_dependencies) - dpkg_dependencies.extend(test_dependencies) - dpkg_dependencies.extend(['python3-pip', 'python3-setuptools', 'tox']) + def Write(self): + """Writes a lint.yml GitHub actions workflow file .""" + dpkg_dependencies = self._GetDPKGPythonDependencies() + test_dependencies = self._GetDPKGTestDependencies(dpkg_dependencies) + dpkg_dependencies.extend(test_dependencies) + dpkg_dependencies.extend(["python3-pip", "python3-setuptools", "tox"]) - dpkg_dev_dependencies = self._GetDPKGDevDependencies() + dpkg_dev_dependencies = self._GetDPKGDevDependencies() - template_mappings = { - 'dpkg_dependencies': ' '.join(sorted(set(dpkg_dependencies))), - 'dpkg_dev_dependencies': ' '.join(sorted(set(dpkg_dev_dependencies)))} + template_mappings = { + "dpkg_dependencies": " ".join(sorted(set(dpkg_dependencies))), + "dpkg_dev_dependencies": " ".join(sorted(set(dpkg_dev_dependencies))), + } - python_module_name = self._project_definition.name + python_module_name = self._project_definition.name - if self._project_definition.name.endswith('-kb'): - python_module_name = ''.join([python_module_name[:-3], 'rc']) + if self._project_definition.name.endswith("-kb"): + python_module_name = "".join([python_module_name[:-3], "rc"]) - paths_to_lint_yaml = [] + paths_to_lint_yaml = [] - if os.path.isdir(python_module_name): - if glob.glob(os.path.join( - python_module_name, '**', '*.yaml'), recursive=True): - paths_to_lint_yaml.append(python_module_name) + if os.path.isdir(python_module_name): + if glob.glob( + os.path.join(python_module_name, "**", "*.yaml"), recursive=True + ): + paths_to_lint_yaml.append(python_module_name) - if os.path.isdir('data'): - if glob.glob(os.path.join('data', '**', '*.yaml'), recursive=True): - paths_to_lint_yaml.append('data') + if os.path.isdir("data"): + if glob.glob(os.path.join("data", "**", "*.yaml"), recursive=True): + paths_to_lint_yaml.append("data") - if os.path.isdir('test_data'): - if glob.glob(os.path.join('test_data', '**', '*.yaml'), recursive=True): - paths_to_lint_yaml.append('test_data') + if os.path.isdir("test_data"): + if glob.glob(os.path.join("test_data", "**", "*.yaml"), recursive=True): + paths_to_lint_yaml.append("test_data") - if os.path.isdir('tests'): - if glob.glob(os.path.join('tests', '**', '*.yaml'), recursive=True): - paths_to_lint_yaml.append('tests') + if os.path.isdir("tests"): + if glob.glob(os.path.join("tests", "**", "*.yaml"), recursive=True): + paths_to_lint_yaml.append("tests") - file_content = [] + file_content = [] - template_data = self._GenerateFromTemplate('header.yml', template_mappings) - file_content.append(template_data) + template_data = self._GenerateFromTemplate("header.yml", template_mappings) + file_content.append(template_data) - if paths_to_lint_yaml: - template_data = self._GenerateFromTemplate( - 'yamllint.yml', template_mappings) - file_content.append(template_data) + if paths_to_lint_yaml: + template_data = self._GenerateFromTemplate( + "yamllint.yml", template_mappings + ) + file_content.append(template_data) - file_content = ''.join(file_content) + file_content = "".join(file_content) - with open(self.PATH, 'w', encoding='utf-8') as file_object: - file_object.write(file_content) + with open(self.PATH, "w", encoding="utf-8") as file_object: + file_object.write(file_content) class GitHubActionsTestDockerYmlWriter(interface.DependencyFileWriter): - """test_docker.yml GitHub actions workflow file writer.""" - - _TEMPLATE_FILE = os.path.join( - 'data', 'templates', 'github_actions', 'test_docker.yml') - - PATH = os.path.join('.github', 'workflows', 'test_docker.yml') - - def Write(self): - """Writes a test_docker.yml GitHub actions workflow file .""" - dpkg_dependencies = self._GetDPKGPythonDependencies() - test_dependencies = self._GetDPKGTestDependencies(dpkg_dependencies) - dpkg_dependencies.extend(test_dependencies) - dpkg_dependencies.extend([ - 'python3', 'python3-build', 'python3-dev', 'python3-pip', - 'python3-setuptools', 'python3-venv', 'python3-wheel']) - - rpm_dependencies = self._GetRPMPythonDependencies() - test_dependencies = self._GetRPMTestDependencies(rpm_dependencies) - rpm_dependencies.extend(test_dependencies) - rpm_dependencies.extend([ - 'python3', 'python3-build', 'python3-devel', 'python3-setuptools', - 'python3-wheel']) - - template_mappings = { - 'dpkg_dependencies': ' '.join(sorted(set(dpkg_dependencies))), - 'rpm_dependencies': ' '.join(sorted(set(rpm_dependencies)))} - - template_file = os.path.join(self._l2tdevtools_path, self._TEMPLATE_FILE) - file_content = self._GenerateFromTemplate(template_file, template_mappings) - - with open(self.PATH, 'w', encoding='utf-8') as file_object: - file_object.write(file_content) + """test_docker.yml GitHub actions workflow file writer.""" + + _TEMPLATE_FILE = os.path.join( + "data", "templates", "github_actions", "test_docker.yml" + ) + + PATH = os.path.join(".github", "workflows", "test_docker.yml") + + def Write(self): + """Writes a test_docker.yml GitHub actions workflow file .""" + dpkg_dependencies = self._GetDPKGPythonDependencies() + test_dependencies = self._GetDPKGTestDependencies(dpkg_dependencies) + dpkg_dependencies.extend(test_dependencies) + dpkg_dependencies.extend( + [ + "python3", + "python3-build", + "python3-dev", + "python3-pip", + "python3-setuptools", + "python3-venv", + "python3-wheel", + ] + ) + + rpm_dependencies = self._GetRPMPythonDependencies() + test_dependencies = self._GetRPMTestDependencies(rpm_dependencies) + rpm_dependencies.extend(test_dependencies) + rpm_dependencies.extend( + [ + "python3", + "python3-build", + "python3-devel", + "python3-setuptools", + "python3-wheel", + ] + ) + + template_mappings = { + "dpkg_dependencies": " ".join(sorted(set(dpkg_dependencies))), + "rpm_dependencies": " ".join(sorted(set(rpm_dependencies))), + } + + template_file = os.path.join(self._l2tdevtools_path, self._TEMPLATE_FILE) + file_content = self._GenerateFromTemplate(template_file, template_mappings) + + with open(self.PATH, "w", encoding="utf-8") as file_object: + file_object.write(file_content) class GitHubActionsTestDocsYmlWriter(interface.DependencyFileWriter): - """test_docs.yml GitHub actions workflow file writer.""" + """test_docs.yml GitHub actions workflow file writer.""" - _TEMPLATE_FILE = os.path.join( - 'data', 'templates', 'github_actions', 'test_docs.yml') + _TEMPLATE_FILE = os.path.join( + "data", "templates", "github_actions", "test_docs.yml" + ) - PATH = os.path.join('.github', 'workflows', 'test_docs.yml') + PATH = os.path.join(".github", "workflows", "test_docs.yml") - def Write(self): - """Writes a test_docs.yml GitHub actions workflow file .""" - dpkg_dependencies = self._GetDPKGPythonDependencies() - test_dependencies = self._GetDPKGTestDependencies(dpkg_dependencies) - dpkg_dependencies.extend(test_dependencies) - dpkg_dependencies.extend(['python3-pip', 'python3-setuptools', 'tox']) + def Write(self): + """Writes a test_docs.yml GitHub actions workflow file .""" + dpkg_dependencies = self._GetDPKGPythonDependencies() + test_dependencies = self._GetDPKGTestDependencies(dpkg_dependencies) + dpkg_dependencies.extend(test_dependencies) + dpkg_dependencies.extend(["python3-pip", "python3-setuptools", "tox"]) - dpkg_dev_dependencies = self._GetDPKGDevDependencies() + dpkg_dev_dependencies = self._GetDPKGDevDependencies() - template_mappings = { - 'dpkg_dependencies': ' '.join(sorted(set(dpkg_dependencies))), - 'dpkg_dev_dependencies': ' '.join(sorted(set(dpkg_dev_dependencies)))} + template_mappings = { + "dpkg_dependencies": " ".join(sorted(set(dpkg_dependencies))), + "dpkg_dev_dependencies": " ".join(sorted(set(dpkg_dev_dependencies))), + } - template_file = os.path.join(self._l2tdevtools_path, self._TEMPLATE_FILE) - file_content = self._GenerateFromTemplate(template_file, template_mappings) + template_file = os.path.join(self._l2tdevtools_path, self._TEMPLATE_FILE) + file_content = self._GenerateFromTemplate(template_file, template_mappings) - with open(self.PATH, 'w', encoding='utf-8') as file_object: - file_object.write(file_content) + with open(self.PATH, "w", encoding="utf-8") as file_object: + file_object.write(file_content) class GitHubActionsTestMacOSYmlWriter(interface.DependencyFileWriter): - """test_macos.yml GitHub actions workflow file writer.""" + """test_macos.yml GitHub actions workflow file writer.""" - _TEMPLATE_FILE = os.path.join( - 'data', 'templates', 'github_actions', 'test_macos.yml') + _TEMPLATE_FILE = os.path.join( + "data", "templates", "github_actions", "test_macos.yml" + ) - PATH = os.path.join('.github', 'workflows', 'test_macos.yml') + PATH = os.path.join(".github", "workflows", "test_macos.yml") - def Write(self): - """Writes a test_macos.yml GitHub actions workflow file .""" - template_mappings = {} + def Write(self): + """Writes a test_macos.yml GitHub actions workflow file .""" + template_mappings = {} - template_file = os.path.join(self._l2tdevtools_path, self._TEMPLATE_FILE) - file_content = self._GenerateFromTemplate(template_file, template_mappings) + template_file = os.path.join(self._l2tdevtools_path, self._TEMPLATE_FILE) + file_content = self._GenerateFromTemplate(template_file, template_mappings) - with open(self.PATH, 'w', encoding='utf-8') as file_object: - file_object.write(file_content) + with open(self.PATH, "w", encoding="utf-8") as file_object: + file_object.write(file_content) class GitHubActionsTestToxYmlWriter(interface.DependencyFileWriter): - """test_tox.yml GitHub actions workflow file writer.""" + """test_tox.yml GitHub actions workflow file writer.""" - _TEMPLATE_FILE = os.path.join( - 'data', 'templates', 'github_actions', 'test_tox.yml') + _TEMPLATE_FILE = os.path.join("data", "templates", "github_actions", "test_tox.yml") - PATH = os.path.join('.github', 'workflows', 'test_tox.yml') + PATH = os.path.join(".github", "workflows", "test_tox.yml") - def Write(self): - """Writes a test_tox.yml GitHub actions workflow file .""" - dpkg_dependencies = self._GetDPKGPythonDependencies() - test_dependencies = self._GetDPKGTestDependencies(dpkg_dependencies) - dpkg_dependencies.extend(test_dependencies) - dpkg_dependencies.extend(['python3-pip', 'python3-setuptools', 'tox']) + def Write(self): + """Writes a test_tox.yml GitHub actions workflow file .""" + dpkg_dependencies = self._GetDPKGPythonDependencies() + test_dependencies = self._GetDPKGTestDependencies(dpkg_dependencies) + dpkg_dependencies.extend(test_dependencies) + dpkg_dependencies.extend(["python3-pip", "python3-setuptools", "tox"]) - dpkg_dev_dependencies = self._GetDPKGDevDependencies() + dpkg_dev_dependencies = self._GetDPKGDevDependencies() - template_mappings = { - 'dpkg_dependencies': ' '.join(sorted(set(dpkg_dependencies))), - 'dpkg_dev_dependencies': ' '.join(sorted(set(dpkg_dev_dependencies)))} + template_mappings = { + "dpkg_dependencies": " ".join(sorted(set(dpkg_dependencies))), + "dpkg_dev_dependencies": " ".join(sorted(set(dpkg_dev_dependencies))), + } - template_file = os.path.join(self._l2tdevtools_path, self._TEMPLATE_FILE) - file_content = self._GenerateFromTemplate(template_file, template_mappings) + template_file = os.path.join(self._l2tdevtools_path, self._TEMPLATE_FILE) + file_content = self._GenerateFromTemplate(template_file, template_mappings) - with open(self.PATH, 'w', encoding='utf-8') as file_object: - file_object.write(file_content) + with open(self.PATH, "w", encoding="utf-8") as file_object: + file_object.write(file_content) diff --git a/l2tdevtools/dependency_writers/interface.py b/l2tdevtools/dependency_writers/interface.py index 8405bde1..87b62f32 100644 --- a/l2tdevtools/dependency_writers/interface.py +++ b/l2tdevtools/dependency_writers/interface.py @@ -5,172 +5,179 @@ class DependencyFileWriter: - """Base class for dependency file writers.""" + """Base class for dependency file writers.""" - def __init__( - self, l2tdevtools_path, project_definition, dependency_helper): - """Initializes a dependency file writer. + def __init__(self, l2tdevtools_path, project_definition, dependency_helper): + """Initializes a dependency file writer. - Args: - l2tdevtools_path (str): path to l2tdevtools. - project_definition (ProjectDefinition): project definition. - dependency_helper (DependencyHelper): dependency helper. - """ - super().__init__() - self._dependency_helper = dependency_helper - self._l2tdevtools_path = l2tdevtools_path - self._project_definition = project_definition + Args: + l2tdevtools_path (str): path to l2tdevtools. + project_definition (ProjectDefinition): project definition. + dependency_helper (DependencyHelper): dependency helper. + """ + super().__init__() + self._dependency_helper = dependency_helper + self._l2tdevtools_path = l2tdevtools_path + self._project_definition = project_definition - def _GenerateFromTemplate(self, template_filename, template_mappings): - """Generates file context based on a template file. + def _GenerateFromTemplate(self, template_filename, template_mappings): + """Generates file context based on a template file. - Args: - template_filename (str): path of the template file. - template_mappings (dict[str, str]): template mappings, where the key - maps to the name of a template variable. - - Returns: - str: output based on the template string. + Args: + template_filename (str): path of the template file. + template_mappings (dict[str, str]): template mappings, where the key + maps to the name of a template variable. - Raises: - RuntimeError: if the template cannot be formatted. - """ - template_string = self._ReadTemplateFile(template_filename) - - try: - return template_string.substitute(template_mappings) + Returns: + str: output based on the template string. + + Raises: + RuntimeError: if the template cannot be formatted. + """ + template_string = self._ReadTemplateFile(template_filename) + + try: + return template_string.substitute(template_mappings) + + except (KeyError, ValueError) as exception: + raise RuntimeError( + f"Unable to format template: {template_filename:s} " + f"with error: {exception!s}" + ) + + def _GetDPKGDevDependencies(self): + """Retrieves DPKG development dependencies. - except (KeyError, ValueError) as exception: - raise RuntimeError( - f'Unable to format template: {template_filename:s} ' - f'with error: {exception!s}') + Returns: + list[str]: DPKG package names of development dependencies. + """ + dpkg_dependencies = self._dependency_helper.GetDPKGDepends(exclude_version=True) + + dpkg_dev_dependencies = ["pkg-config"] + + # TODO: extract from configuration. + + if "python3-snappy" in dpkg_dependencies: + dpkg_dev_dependencies.append("libsnappy-dev") + + if "python3-yara" in dpkg_dependencies: + dpkg_dev_dependencies.append("libssl-dev") + + if "python3-xattr" in dpkg_dependencies: + dpkg_dev_dependencies.append("libffi-dev") + + return dpkg_dev_dependencies + + def _GetDPKGPythonDependencies(self): + """Retrieves DPKG Python dependencies. + + Returns: + list[str]: DPKG package names of Python dependencies. + """ + dpkg_dependencies = self._dependency_helper.GetDPKGDepends(exclude_version=True) + + return dpkg_dependencies + + def _GetDPKGTestDependencies(self, python_dependencies): + """Retrieves DPKG test dependencies. + + Args: + python_dependencies (list[str]): DPKG package names of Python + dependencies. + + Returns: + list[str]: DPKG package names of test dependencies. + """ + test_dependencies = self._dependency_helper.GetDPKGDepends( + exclude_version=True, test_dependencies=True + ) + + return [ + test_dependency + for test_dependency in sorted(test_dependencies) + if test_dependency not in python_dependencies + ] + + def _GetPyPIPythonDependencies(self, exclude_version=False): + """Retrieves PyPI Python dependencies. + + Args: + exclude_version (Optional[bool]): True if the version should be excluded + from the dependency definitions. + + Returns: + list[str]: PyPI package names of Python dependencies. + """ + return self._dependency_helper.GetInstallRequires( + exclude_version=exclude_version + ) + + def _GetPyPITestDependencies(self, python_dependencies, exclude_version=False): + """Retrieves PyPI test dependencies. + + Args: + python_dependencies (list[str]): PyPI package names of Python + dependencies. + exclude_version (Optional[bool]): True if the version should be excluded + from the dependency definitions. + + Returns: + list[str]: PyPI package names of test dependencies. + """ + test_dependencies = self._dependency_helper.GetInstallRequires( + exclude_version=exclude_version, test_dependencies=True + ) - def _GetDPKGDevDependencies(self): - """Retrieves DPKG development dependencies. + return [ + test_dependency + for test_dependency in test_dependencies + if test_dependency not in python_dependencies + ] + + def _GetRPMPythonDependencies(self): + """Retrieves RPM Python dependencies. + + Returns: + list[str]: RPM package names of Python dependencies. + """ + return self._dependency_helper.GetRPMRequires(exclude_version=True) + + def _GetRPMTestDependencies(self, python_dependencies): + """Retrieves RPM test dependencies. - Returns: - list[str]: DPKG package names of development dependencies. - """ - dpkg_dependencies = self._dependency_helper.GetDPKGDepends( - exclude_version=True) + Args: + python_dependencies (list[str]): RPM package names of Python dependencies. - dpkg_dev_dependencies = ['pkg-config'] + Returns: + list[str]: RPM package names of test dependencies. + """ + test_dependencies = self._dependency_helper.GetRPMRequires( + exclude_version=True, test_dependencies=True + ) - # TODO: extract from configuration. - - if 'python3-snappy' in dpkg_dependencies: - dpkg_dev_dependencies.append('libsnappy-dev') - - if 'python3-yara' in dpkg_dependencies: - dpkg_dev_dependencies.append('libssl-dev') - - if 'python3-xattr' in dpkg_dependencies: - dpkg_dev_dependencies.append('libffi-dev') - - return dpkg_dev_dependencies - - def _GetDPKGPythonDependencies(self): - """Retrieves DPKG Python dependencies. - - Returns: - list[str]: DPKG package names of Python dependencies. - """ - dpkg_dependencies = self._dependency_helper.GetDPKGDepends( - exclude_version=True) - - return dpkg_dependencies - - def _GetDPKGTestDependencies(self, python_dependencies): - """Retrieves DPKG test dependencies. - - Args: - python_dependencies (list[str]): DPKG package names of Python - dependencies. - - Returns: - list[str]: DPKG package names of test dependencies. - """ - test_dependencies = self._dependency_helper.GetDPKGDepends( - exclude_version=True, test_dependencies=True) - - return [ - test_dependency for test_dependency in sorted(test_dependencies) - if test_dependency not in python_dependencies] - - def _GetPyPIPythonDependencies(self, exclude_version=False): - """Retrieves PyPI Python dependencies. - - Args: - exclude_version (Optional[bool]): True if the version should be excluded - from the dependency definitions. - - Returns: - list[str]: PyPI package names of Python dependencies. - """ - return self._dependency_helper.GetInstallRequires( - exclude_version=exclude_version) - - def _GetPyPITestDependencies( - self, python_dependencies, exclude_version=False): - """Retrieves PyPI test dependencies. - - Args: - python_dependencies (list[str]): PyPI package names of Python - dependencies. - exclude_version (Optional[bool]): True if the version should be excluded - from the dependency definitions. - - Returns: - list[str]: PyPI package names of test dependencies. - """ - test_dependencies = self._dependency_helper.GetInstallRequires( - exclude_version=exclude_version, test_dependencies=True) - - return [ - test_dependency for test_dependency in test_dependencies - if test_dependency not in python_dependencies] - - def _GetRPMPythonDependencies(self): - """Retrieves RPM Python dependencies. - - Returns: - list[str]: RPM package names of Python dependencies. - """ - return self._dependency_helper.GetRPMRequires(exclude_version=True) - - def _GetRPMTestDependencies(self, python_dependencies): - """Retrieves RPM test dependencies. - - Args: - python_dependencies (list[str]): RPM package names of Python dependencies. - - Returns: - list[str]: RPM package names of test dependencies. - """ - test_dependencies = self._dependency_helper.GetRPMRequires( - exclude_version=True, test_dependencies=True) - - # TODO: replace by test_dependencies.ini or dev_dependencies.ini or equiv. - test_dependencies.extend(['python3-setuptools']) - - return [ - test_dependency for test_dependency in sorted(test_dependencies) - if test_dependency not in python_dependencies] - - def _ReadTemplateFile(self, filename): - """Reads a template string from file. - - Args: - filename (str): name of the file containing the template string. - - Returns: - string.Template: template string. - """ - with open(filename, 'r', encoding='utf-8') as file_object: - file_data = file_object.read() - - return string.Template(file_data) - - @abc.abstractmethod - def Write(self): - """Writes the file or files produced by the file writer.""" + # TODO: replace by test_dependencies.ini or dev_dependencies.ini or equiv. + test_dependencies.extend(["python3-setuptools"]) + + return [ + test_dependency + for test_dependency in sorted(test_dependencies) + if test_dependency not in python_dependencies + ] + + def _ReadTemplateFile(self, filename): + """Reads a template string from file. + + Args: + filename (str): name of the file containing the template string. + + Returns: + string.Template: template string. + """ + with open(filename, "r", encoding="utf-8") as file_object: + file_data = file_object.read() + + return string.Template(file_data) + + @abc.abstractmethod + def Write(self): + """Writes the file or files produced by the file writer.""" diff --git a/l2tdevtools/dependency_writers/jenkins_scripts.py b/l2tdevtools/dependency_writers/jenkins_scripts.py index cf94046f..3c2b7087 100644 --- a/l2tdevtools/dependency_writers/jenkins_scripts.py +++ b/l2tdevtools/dependency_writers/jenkins_scripts.py @@ -6,50 +6,50 @@ class LinuxRunEndToEndTestsScriptWriter(interface.DependencyFileWriter): - """Linux run end-to-end test script file writer.""" + """Linux run end-to-end test script file writer.""" - _TEMPLATE_FILE = os.path.join( - 'data', 'templates', 'jenkins_scripts', 'linux_run_end_to_end_tests.sh') + _TEMPLATE_FILE = os.path.join( + "data", "templates", "jenkins_scripts", "linux_run_end_to_end_tests.sh" + ) - PATH = os.path.join( - 'config', 'jenkins', 'linux', 'run_end_to_end_tests.sh') + PATH = os.path.join("config", "jenkins", "linux", "run_end_to_end_tests.sh") - def Write(self): - """Writes a Linux run_end_to_end_tests.sh file.""" - python_module_name = self._project_definition.name + def Write(self): + """Writes a Linux run_end_to_end_tests.sh file.""" + python_module_name = self._project_definition.name - if self._project_definition.name.endswith('-kb'): - python_module_name = ''.join([python_module_name[:-3], 'rc']) + if self._project_definition.name.endswith("-kb"): + python_module_name = "".join([python_module_name[:-3], "rc"]) - scripts_directory = os.path.join(python_module_name, 'scripts') + scripts_directory = os.path.join(python_module_name, "scripts") - template_mappings = { - 'project_name': self._project_definition.name, - 'scripts_directory': scripts_directory} + template_mappings = { + "project_name": self._project_definition.name, + "scripts_directory": scripts_directory, + } - template_file = os.path.join(self._l2tdevtools_path, self._TEMPLATE_FILE) - file_content = self._GenerateFromTemplate(template_file, template_mappings) + template_file = os.path.join(self._l2tdevtools_path, self._TEMPLATE_FILE) + file_content = self._GenerateFromTemplate(template_file, template_mappings) - with open(self.PATH, 'w', encoding='utf-8') as file_object: - file_object.write(file_content) + with open(self.PATH, "w", encoding="utf-8") as file_object: + file_object.write(file_content) class RunPython3EndToEndTestsScriptWriter(interface.DependencyFileWriter): - """Run Python 3 end-to-end test script file writer.""" + """Run Python 3 end-to-end test script file writer.""" - _TEMPLATE_FILE = os.path.join( - 'data', 'templates', 'jenkins_scripts', 'run_end_to_end_tests_py3.sh') + _TEMPLATE_FILE = os.path.join( + "data", "templates", "jenkins_scripts", "run_end_to_end_tests_py3.sh" + ) - PATH = os.path.join( - 'config', 'jenkins', 'linux', 'run_end_to_end_tests_py3.sh') + PATH = os.path.join("config", "jenkins", "linux", "run_end_to_end_tests_py3.sh") - def Write(self): - """Writes a run_end_to_end_tests.sh file.""" - template_mappings = { - 'project_name': self._project_definition.name} + def Write(self): + """Writes a run_end_to_end_tests.sh file.""" + template_mappings = {"project_name": self._project_definition.name} - template_file = os.path.join(self._l2tdevtools_path, self._TEMPLATE_FILE) - file_content = self._GenerateFromTemplate(template_file, template_mappings) + template_file = os.path.join(self._l2tdevtools_path, self._TEMPLATE_FILE) + file_content = self._GenerateFromTemplate(template_file, template_mappings) - with open(self.PATH, 'w', encoding='utf-8') as file_object: - file_object.write(file_content) + with open(self.PATH, "w", encoding="utf-8") as file_object: + file_object.write(file_content) diff --git a/l2tdevtools/dependency_writers/linux_scripts.py b/l2tdevtools/dependency_writers/linux_scripts.py index be7e9afb..e008a71d 100644 --- a/l2tdevtools/dependency_writers/linux_scripts.py +++ b/l2tdevtools/dependency_writers/linux_scripts.py @@ -6,171 +6,178 @@ class UbuntuInstallationScriptWriter(interface.DependencyFileWriter): - """Ubuntu installation script file writer.""" + """Ubuntu installation script file writer.""" - _TEMPLATE_FILE = os.path.join( - 'data', 'templates', 'linux_scripts', 'ubuntu_install_project.sh') + _TEMPLATE_FILE = os.path.join( + "data", "templates", "linux_scripts", "ubuntu_install_project.sh" + ) - PATH = os.path.join('config', 'linux') + PATH = os.path.join("config", "linux") - def _FormatDPKGDebugDependencies(self, debug_dependencies): - """Formats DPKG debug dependencies for the template. + def _FormatDPKGDebugDependencies(self, debug_dependencies): + """Formats DPKG debug dependencies for the template. - Args: - debug_dependencies (list[str]): DPKG package names of debug dependencies. + Args: + debug_dependencies (list[str]): DPKG package names of debug dependencies. - Returns: - str: formatted DPKG debug dependencies. - """ - formatted_debug_dependencies = [] - if debug_dependencies: - for index, dependency in enumerate(sorted(debug_dependencies)): - if index == 0: - line = f'DEBUG_DEPENDENCIES="{dependency:s}' - else: - line = f' {dependency:s}' + Returns: + str: formatted DPKG debug dependencies. + """ + formatted_debug_dependencies = [] + if debug_dependencies: + for index, dependency in enumerate(sorted(debug_dependencies)): + if index == 0: + line = f'DEBUG_DEPENDENCIES="{dependency:s}' + else: + line = f" {dependency:s}" - if index + 1 == len(debug_dependencies): - line = f'{line:s}";' + if index + 1 == len(debug_dependencies): + line = f'{line:s}";' - formatted_debug_dependencies.append(line) + formatted_debug_dependencies.append(line) - return '\n'.join(formatted_debug_dependencies) + return "\n".join(formatted_debug_dependencies) - def _FormatDPKGDevelopmentDependencies(self, development_dependencies): - """Formats DPKG development dependencies for the template. + def _FormatDPKGDevelopmentDependencies(self, development_dependencies): + """Formats DPKG development dependencies for the template. - Args: - development_dependencies (list[str]): DPKG package names of development - dependencies. + Args: + development_dependencies (list[str]): DPKG package names of development + dependencies. - Returns: - str: formatted DPKG development dependencies. - """ - formatted_development_dependencies = [] - if development_dependencies: - for index, dependency in enumerate(sorted(development_dependencies)): - if index == 0: - line = f'DEVELOPMENT_DEPENDENCIES="{dependency:s}' - else: - line = f' {dependency:s}' + Returns: + str: formatted DPKG development dependencies. + """ + formatted_development_dependencies = [] + if development_dependencies: + for index, dependency in enumerate(sorted(development_dependencies)): + if index == 0: + line = f'DEVELOPMENT_DEPENDENCIES="{dependency:s}' + else: + line = f" {dependency:s}" - if index + 1 == len(development_dependencies): - line = f'{line:s}";' + if index + 1 == len(development_dependencies): + line = f'{line:s}";' - formatted_development_dependencies.append(line) + formatted_development_dependencies.append(line) - return '\n'.join(formatted_development_dependencies) + return "\n".join(formatted_development_dependencies) - def _FormatDPKGPythonDependencies(self, python_dependencies): - """Formats DPKG Python dependencies for the template. + def _FormatDPKGPythonDependencies(self, python_dependencies): + """Formats DPKG Python dependencies for the template. - Args: - python_dependencies (list[str]): DPKG package names of Python - dependencies. + Args: + python_dependencies (list[str]): DPKG package names of Python + dependencies. - Returns: - str: formatted DPKG Python dependencies. - """ - formatted_python_dependencies = [] + Returns: + str: formatted DPKG Python dependencies. + """ + formatted_python_dependencies = [] - for index, dependency in enumerate(sorted(python_dependencies)): - if index == 0: - line = f'PYTHON_DEPENDENCIES="{dependency:s}' - else: - line = f' {dependency:s}' + for index, dependency in enumerate(sorted(python_dependencies)): + if index == 0: + line = f'PYTHON_DEPENDENCIES="{dependency:s}' + else: + line = f" {dependency:s}" - if index + 1 == len(python_dependencies): - line = f'{line:s}";' + if index + 1 == len(python_dependencies): + line = f'{line:s}";' - formatted_python_dependencies.append(line) + formatted_python_dependencies.append(line) - return '\n'.join(formatted_python_dependencies) + return "\n".join(formatted_python_dependencies) - def _FormatDPKGTestDependencies(self, test_dependencies): - """Formats DPKG test dependencies for the template. + def _FormatDPKGTestDependencies(self, test_dependencies): + """Formats DPKG test dependencies for the template. - Args: - test_dependencies (list[str]): DPKG package names of test dependencies. + Args: + test_dependencies (list[str]): DPKG package names of test dependencies. - Returns: - str: formatted DPKG test dependencies. - """ - formatted_test_dependencies = [] - if test_dependencies: - for index, dependency in enumerate(sorted(test_dependencies)): - if index == 0: - line = f'TEST_DEPENDENCIES="{dependency:s}' - else: - line = f' {dependency:s}' + Returns: + str: formatted DPKG test dependencies. + """ + formatted_test_dependencies = [] + if test_dependencies: + for index, dependency in enumerate(sorted(test_dependencies)): + if index == 0: + line = f'TEST_DEPENDENCIES="{dependency:s}' + else: + line = f" {dependency:s}" - if index + 1 == len(test_dependencies): - line = f'{line:s}";' + if index + 1 == len(test_dependencies): + line = f'{line:s}";' - formatted_test_dependencies.append(line) + formatted_test_dependencies.append(line) - return '\n'.join(formatted_test_dependencies) + return "\n".join(formatted_test_dependencies) - def _GetDPKGDebugDependencies(self, python_dependencies): - """Retrieves DPKG debug dependencies. + def _GetDPKGDebugDependencies(self, python_dependencies): + """Retrieves DPKG debug dependencies. - Args: - python_dependencies (list[str]): DPKG package names of Python - dependencies. + Args: + python_dependencies (list[str]): DPKG package names of Python + dependencies. - Returns: - list[str]: DPKG package names of Python debug dependencies. - """ - debug_dependencies = [] - for dependency in sorted(python_dependencies): - if dependency.startswith('lib') and dependency.endswith('python3'): - dependency, _, _ = dependency.partition('-') - debug_dependencies.extend([ - f'{dependency:s}-dbg', f'{dependency:s}-python3-dbg']) + Returns: + list[str]: DPKG package names of Python debug dependencies. + """ + debug_dependencies = [] + for dependency in sorted(python_dependencies): + if dependency.startswith("lib") and dependency.endswith("python3"): + dependency, _, _ = dependency.partition("-") + debug_dependencies.extend( + [f"{dependency:s}-dbg", f"{dependency:s}-python3-dbg"] + ) - return debug_dependencies + return debug_dependencies - def Write(self): - """Writes a ubuntu_install_project.sh file.""" - python_dependencies = self._GetDPKGPythonDependencies() + def Write(self): + """Writes a ubuntu_install_project.sh file.""" + python_dependencies = self._GetDPKGPythonDependencies() - test_dependencies = self._GetDPKGTestDependencies(python_dependencies) + test_dependencies = self._GetDPKGTestDependencies(python_dependencies) - # TODO: replace by dev_dependencies.ini or equiv. - test_dependencies.extend(['python3-setuptools']) + # TODO: replace by dev_dependencies.ini or equiv. + test_dependencies.extend(["python3-setuptools"]) - # TODO: replace by dev_dependencies.ini or equiv. - development_dependencies = ['pylint'] + # TODO: replace by dev_dependencies.ini or equiv. + development_dependencies = ["pylint"] - if self._project_definition.name == 'plaso': - development_dependencies.append('python-sphinx') + if self._project_definition.name == "plaso": + development_dependencies.append("python-sphinx") - debug_dependencies = self._GetDPKGDebugDependencies(python_dependencies) + debug_dependencies = self._GetDPKGDebugDependencies(python_dependencies) - formatted_python_dependencies = self._FormatDPKGPythonDependencies( - python_dependencies) + formatted_python_dependencies = self._FormatDPKGPythonDependencies( + python_dependencies + ) - formatted_test_dependencies = self._FormatDPKGTestDependencies( - test_dependencies) + formatted_test_dependencies = self._FormatDPKGTestDependencies( + test_dependencies + ) - formatted_development_dependencies = ( - self._FormatDPKGDevelopmentDependencies(development_dependencies)) + formatted_development_dependencies = self._FormatDPKGDevelopmentDependencies( + development_dependencies + ) - formatted_debug_dependencies = self._FormatDPKGDebugDependencies( - debug_dependencies) + formatted_debug_dependencies = self._FormatDPKGDebugDependencies( + debug_dependencies + ) - template_mappings = { - 'debug_dependencies': formatted_debug_dependencies, - 'development_dependencies': formatted_development_dependencies, - 'project_name': self._project_definition.name, - 'python_dependencies': formatted_python_dependencies, - 'test_dependencies': formatted_test_dependencies} + template_mappings = { + "debug_dependencies": formatted_debug_dependencies, + "development_dependencies": formatted_development_dependencies, + "project_name": self._project_definition.name, + "python_dependencies": formatted_python_dependencies, + "test_dependencies": formatted_test_dependencies, + } - template_file = os.path.join(self._l2tdevtools_path, self._TEMPLATE_FILE) - file_content = self._GenerateFromTemplate(template_file, template_mappings) + template_file = os.path.join(self._l2tdevtools_path, self._TEMPLATE_FILE) + file_content = self._GenerateFromTemplate(template_file, template_mappings) - script_name = f'ubuntu_install_{self._project_definition.name:s}.sh' - script_path = os.path.join('config', 'linux', script_name) + script_name = f"ubuntu_install_{self._project_definition.name:s}.sh" + script_path = os.path.join("config", "linux", script_name) - with open(script_path, 'w', encoding='utf-8') as file_object: - file_object.write(file_content) + with open(script_path, "w", encoding="utf-8") as file_object: + file_object.write(file_content) diff --git a/l2tdevtools/dependency_writers/pylint_rc.py b/l2tdevtools/dependency_writers/pylint_rc.py index a2e702ee..9632df7c 100644 --- a/l2tdevtools/dependency_writers/pylint_rc.py +++ b/l2tdevtools/dependency_writers/pylint_rc.py @@ -6,20 +6,20 @@ class PylintRcWriter(interface.DependencyFileWriter): - """Pylint.rc file writer.""" + """Pylint.rc file writer.""" - _TEMPLATE_FILE = os.path.join('data', 'templates', '.pylintrc') + _TEMPLATE_FILE = os.path.join("data", "templates", ".pylintrc") - PATH = '.pylintrc' + PATH = ".pylintrc" - def Write(self): - """Writes a .pylintrc file.""" - dependencies = self._dependency_helper.GetPylintRcExtensionPkgs() + def Write(self): + """Writes a .pylintrc file.""" + dependencies = self._dependency_helper.GetPylintRcExtensionPkgs() - template_mappings = {'extension_pkg_allow_list': ','.join(dependencies)} + template_mappings = {"extension_pkg_allow_list": ",".join(dependencies)} - template_file = os.path.join(self._l2tdevtools_path, self._TEMPLATE_FILE) - file_content = self._GenerateFromTemplate(template_file, template_mappings) + template_file = os.path.join(self._l2tdevtools_path, self._TEMPLATE_FILE) + file_content = self._GenerateFromTemplate(template_file, template_mappings) - with open(self.PATH, 'w', encoding='utf-8') as file_object: - file_object.write(file_content) + with open(self.PATH, "w", encoding="utf-8") as file_object: + file_object.write(file_content) diff --git a/l2tdevtools/dependency_writers/setup.py b/l2tdevtools/dependency_writers/setup.py index 7a89b4b5..c099da6a 100644 --- a/l2tdevtools/dependency_writers/setup.py +++ b/l2tdevtools/dependency_writers/setup.py @@ -9,172 +9,188 @@ class PyprojectTomlWriter(interface.DependencyFileWriter): - """Pyproject TOML script file writer.""" - - PATH = os.path.join('pyproject.toml') - - _PROJECTS_WITH_PACKAGE_DATA = ( - 'artifacts-kb', 'dfvfs', 'dfwinreg', 'dtformats', 'esedb-kb', 'mapi-kb', - 'olecf-kb', 'plaso', 'winreg-kb', 'winsps-kb') - - _TEMPLATE_DIRECTORY = os.path.join('data', 'templates', 'pyproject.toml') - - def _GenerateFromTemplate(self, template_filename, template_mappings): - """Generates file context based on a template file. - - Args: - template_filename (str): path of the template file. - template_mappings (dict[str, str]): template mappings, where the key - maps to the name of a template variable. - - Returns: - str: output based on the template string. - - Raises: - RuntimeError: if the template cannot be formatted. - """ - template_filename = os.path.join( - self._l2tdevtools_path, self._TEMPLATE_DIRECTORY, template_filename) - return super()._GenerateFromTemplate( - template_filename, template_mappings) - - def Write(self): - """Writes a pyproject.toml file.""" - if self._project_definition.status == 'experimental': - development_status = 'Development Status :: 2 - Pre-Alpha' - elif self._project_definition.status == 'alpha': - development_status = 'Development Status :: 3 - Alpha' - elif self._project_definition.status == 'beta': - development_status = 'Development Status :: 4 - Beta' - elif self._project_definition.status == 'stable': - development_status = 'Development Status :: 5 - Production/Stable' - else: - development_status = '' - - # TODO: handle license-files - - maintainer_name, _, maintainer_email = ( - self._project_definition.maintainer.partition('<')) - maintainer_name = maintainer_name.rstrip() - maintainer_email = maintainer_email[:-1] - - python_module_name = self._project_definition.name - - if self._project_definition.name.endswith('-kb'): - python_module_name = ''.join([python_module_name[:-3], 'rc']) - - package_data = [] - for data_file in glob.glob( - f'{python_module_name:s}/**/*.yaml', recursive=True): - data_file_directory = os.path.dirname( - data_file[len(f'{python_module_name:s}/'):]) - - data_file = '*.yaml' - if data_file_directory: - data_file = '/'.join([data_file_directory, data_file]) - - if data_file not in package_data: - package_data.append(data_file) - - readme_file = 'README.md' - if not os.path.isfile(readme_file): - readme_file = 'README' - - scripts_directory = 'scripts' - if not os.path.isdir(scripts_directory): - scripts_directory = 'tools' - if not os.path.isdir(scripts_directory): - scripts_directory = None - - if scripts_directory: - if glob.glob(f'{scripts_directory:s}/[a-z]*.py'): - logging.warning(( - 'Scripts are not supported by pyproject.toml, change them to ' - 'console_scripts entry points.')) - - console_scripts_directory = os.path.join(python_module_name, 'scripts') - if not os.path.isdir(console_scripts_directory): - console_scripts_directory = None - - console_scripts = [] - if console_scripts_directory: - console_scripts = glob.glob(f'{console_scripts_directory:s}/[a-z]*.py') - - date_time = datetime.datetime.now() - version = date_time.strftime('%Y%m%d') - - template_mappings = { - 'description_short': ( - self._project_definition.description_short.rstrip(".")), - 'development_status': development_status, - 'maintainer_email': maintainer_email, - 'maintainer_name': maintainer_name, - 'python_module_name': python_module_name, - 'readme_file': readme_file, - 'version': version} - - file_content = [] - - template_data = self._GenerateFromTemplate('header.toml', template_mappings) - file_content.append(template_data) - - template_data = self._GenerateFromTemplate( - 'project.toml', template_mappings) - file_content.append(template_data) - - python_dependencies = self._GetPyPIPythonDependencies() - if python_dependencies: - file_content.append('dependencies = [\n') - for dependency in python_dependencies: - dependency_string = str(dependency) - file_content.append(f' "{dependency_string:s}",\n') - - file_content.append(']\n') - - if console_scripts: - file_content.append('\n') - file_content.append('[project.scripts]\n') - - for console_script in sorted(console_scripts): - console_script = console_script.replace('.py', '') - module_name = console_script.replace(os.path.sep, '.') - name = os.path.basename(console_script) - file_content.append(f'{name:s} = "{module_name:s}:Main"\n') - - if self._project_definition.homepage_url: - file_content.append('\n') - file_content.append('[project.urls]\n') - - if os.path.isdir('docs'): - url = f'https://{python_module_name:s}.readthedocs.io/en/latest' - file_content.append(f'Documentation = "{url:s}"\n') - - url = self._project_definition.homepage_url - file_content.append(f'Homepage = "{url:s}"\n') - file_content.append(f'Repository = "{url:s}"\n') - - template_data = self._GenerateFromTemplate( - 'black.toml', template_mappings) - file_content.append(template_data) - - template_data = self._GenerateFromTemplate( - 'setuptools.packages.toml', template_mappings) - file_content.append(template_data) - - if package_data: - file_content.append('\n') - file_content.append('[tool.setuptools.package-data]\n') - - file_content.append(f'{python_module_name:s} = [\n') - for data_file in sorted(package_data): - if (self._project_definition.name == 'plaso' and - data_file == 'data/*.yaml'): - data_file = 'data/*.*' - file_content.append(f' "{data_file:s}",\n') - - file_content.append(']\n') - - file_content = ''.join(file_content) - - with open(self.PATH, 'w', encoding='utf-8') as file_object: - file_object.write(file_content) + """Pyproject TOML script file writer.""" + + PATH = os.path.join("pyproject.toml") + + _PROJECTS_WITH_PACKAGE_DATA = ( + "artifacts-kb", + "dfvfs", + "dfwinreg", + "dtformats", + "esedb-kb", + "mapi-kb", + "olecf-kb", + "plaso", + "winreg-kb", + "winsps-kb", + ) + + _TEMPLATE_DIRECTORY = os.path.join("data", "templates", "pyproject.toml") + + def _GenerateFromTemplate(self, template_filename, template_mappings): + """Generates file context based on a template file. + + Args: + template_filename (str): path of the template file. + template_mappings (dict[str, str]): template mappings, where the key + maps to the name of a template variable. + + Returns: + str: output based on the template string. + + Raises: + RuntimeError: if the template cannot be formatted. + """ + template_filename = os.path.join( + self._l2tdevtools_path, self._TEMPLATE_DIRECTORY, template_filename + ) + return super()._GenerateFromTemplate(template_filename, template_mappings) + + def Write(self): + """Writes a pyproject.toml file.""" + if self._project_definition.status == "experimental": + development_status = "Development Status :: 2 - Pre-Alpha" + elif self._project_definition.status == "alpha": + development_status = "Development Status :: 3 - Alpha" + elif self._project_definition.status == "beta": + development_status = "Development Status :: 4 - Beta" + elif self._project_definition.status == "stable": + development_status = "Development Status :: 5 - Production/Stable" + else: + development_status = "" + + # TODO: handle license-files + + maintainer_name, _, maintainer_email = ( + self._project_definition.maintainer.partition("<") + ) + maintainer_name = maintainer_name.rstrip() + maintainer_email = maintainer_email[:-1] + + python_module_name = self._project_definition.name + + if self._project_definition.name.endswith("-kb"): + python_module_name = "".join([python_module_name[:-3], "rc"]) + + package_data = [] + for data_file in glob.glob(f"{python_module_name:s}/**/*.yaml", recursive=True): + data_file_directory = os.path.dirname( + data_file[len(f"{python_module_name:s}/") :] + ) + + data_file = "*.yaml" + if data_file_directory: + data_file = "/".join([data_file_directory, data_file]) + + if data_file not in package_data: + package_data.append(data_file) + + readme_file = "README.md" + if not os.path.isfile(readme_file): + readme_file = "README" + + scripts_directory = "scripts" + if not os.path.isdir(scripts_directory): + scripts_directory = "tools" + if not os.path.isdir(scripts_directory): + scripts_directory = None + + if scripts_directory: + if glob.glob(f"{scripts_directory:s}/[a-z]*.py"): + logging.warning( + ( + "Scripts are not supported by pyproject.toml, change them to " + "console_scripts entry points." + ) + ) + + console_scripts_directory = os.path.join(python_module_name, "scripts") + if not os.path.isdir(console_scripts_directory): + console_scripts_directory = None + + console_scripts = [] + if console_scripts_directory: + console_scripts = glob.glob(f"{console_scripts_directory:s}/[a-z]*.py") + + date_time = datetime.datetime.now() + version = date_time.strftime("%Y%m%d") + + template_mappings = { + "description_short": ( + self._project_definition.description_short.rstrip(".") + ), + "development_status": development_status, + "maintainer_email": maintainer_email, + "maintainer_name": maintainer_name, + "python_module_name": python_module_name, + "readme_file": readme_file, + "version": version, + } + + file_content = [] + + template_data = self._GenerateFromTemplate("header.toml", template_mappings) + file_content.append(template_data) + + template_data = self._GenerateFromTemplate("project.toml", template_mappings) + file_content.append(template_data) + + python_dependencies = self._GetPyPIPythonDependencies() + if python_dependencies: + file_content.append("dependencies = [\n") + for dependency in python_dependencies: + dependency_string = str(dependency) + file_content.append(f' "{dependency_string:s}",\n') + + file_content.append("]\n") + + if console_scripts: + file_content.append("\n") + file_content.append("[project.scripts]\n") + + for console_script in sorted(console_scripts): + console_script = console_script.replace(".py", "") + module_name = console_script.replace(os.path.sep, ".") + name = os.path.basename(console_script) + file_content.append(f'{name:s} = "{module_name:s}:Main"\n') + + if self._project_definition.homepage_url: + file_content.append("\n") + file_content.append("[project.urls]\n") + + if os.path.isdir("docs"): + url = f"https://{python_module_name:s}.readthedocs.io/en/latest" + file_content.append(f'Documentation = "{url:s}"\n') + + url = self._project_definition.homepage_url + file_content.append(f'Homepage = "{url:s}"\n') + file_content.append(f'Repository = "{url:s}"\n') + + template_data = self._GenerateFromTemplate("black.toml", template_mappings) + file_content.append(template_data) + + template_data = self._GenerateFromTemplate( + "setuptools.packages.toml", template_mappings + ) + file_content.append(template_data) + + if package_data: + file_content.append("\n") + file_content.append("[tool.setuptools.package-data]\n") + + file_content.append(f"{python_module_name:s} = [\n") + for data_file in sorted(package_data): + if ( + self._project_definition.name == "plaso" + and data_file == "data/*.yaml" + ): + data_file = "data/*.*" + file_content.append(f' "{data_file:s}",\n') + + file_content.append("]\n") + + file_content = "".join(file_content) + + with open(self.PATH, "w", encoding="utf-8") as file_object: + file_object.write(file_content) diff --git a/l2tdevtools/dependency_writers/sphinx_docs.py b/l2tdevtools/dependency_writers/sphinx_docs.py index 4ac8b979..1574bf94 100644 --- a/l2tdevtools/dependency_writers/sphinx_docs.py +++ b/l2tdevtools/dependency_writers/sphinx_docs.py @@ -6,65 +6,66 @@ class ReadthedocsConfigurationWriter(interface.DependencyFileWriter): - """Readthedocs configuration file writer.""" + """Readthedocs configuration file writer.""" - _TEMPLATE_FILE = os.path.join('data', 'templates', 'readthedocs.yaml') + _TEMPLATE_FILE = os.path.join("data", "templates", "readthedocs.yaml") - PATH = os.path.join('.readthedocs.yaml') + PATH = os.path.join(".readthedocs.yaml") - def Write(self): - """Writes a .readthedocs.yaml file.""" - template_mappings = {} + def Write(self): + """Writes a .readthedocs.yaml file.""" + template_mappings = {} - template_file = os.path.join(self._l2tdevtools_path, self._TEMPLATE_FILE) - file_content = self._GenerateFromTemplate(template_file, template_mappings) + template_file = os.path.join(self._l2tdevtools_path, self._TEMPLATE_FILE) + file_content = self._GenerateFromTemplate(template_file, template_mappings) - with open(self.PATH, 'w', encoding='utf-8') as file_object: - file_object.write(file_content) + with open(self.PATH, "w", encoding="utf-8") as file_object: + file_object.write(file_content) class SphinxBuildConfigurationWriter(interface.DependencyFileWriter): - """Sphinx build configuration file writer.""" + """Sphinx build configuration file writer.""" - _TEMPLATE_FILE = os.path.join('data', 'templates', 'docs', 'conf.py') + _TEMPLATE_FILE = os.path.join("data", "templates", "docs", "conf.py") - PATH = os.path.join('docs', 'conf.py') + PATH = os.path.join("docs", "conf.py") - def Write(self): - """Writes a docs/conf.py file.""" - htmlhelp_basename = self._project_definition.name.replace('-', '') - python_module_name = self._project_definition.name + def Write(self): + """Writes a docs/conf.py file.""" + htmlhelp_basename = self._project_definition.name.replace("-", "") + python_module_name = self._project_definition.name - if self._project_definition.name.endswith('-kb'): - htmlhelp_basename = htmlhelp_basename.replace('-', '') - python_module_name = ''.join([python_module_name[:-3], 'rc']) + if self._project_definition.name.endswith("-kb"): + htmlhelp_basename = htmlhelp_basename.replace("-", "") + python_module_name = "".join([python_module_name[:-3], "rc"]) - template_mappings = { - 'htmlhelp_basename': htmlhelp_basename, - 'name_description': self._project_definition.name_description, - 'project_name': self._project_definition.name, - 'python_module_name': python_module_name} + template_mappings = { + "htmlhelp_basename": htmlhelp_basename, + "name_description": self._project_definition.name_description, + "project_name": self._project_definition.name, + "python_module_name": python_module_name, + } - template_file = os.path.join(self._l2tdevtools_path, self._TEMPLATE_FILE) - file_content = self._GenerateFromTemplate(template_file, template_mappings) + template_file = os.path.join(self._l2tdevtools_path, self._TEMPLATE_FILE) + file_content = self._GenerateFromTemplate(template_file, template_mappings) - with open(self.PATH, 'w', encoding='utf-8') as file_object: - file_object.write(file_content) + with open(self.PATH, "w", encoding="utf-8") as file_object: + file_object.write(file_content) class SphinxBuildRequirementsWriter(interface.DependencyFileWriter): - """Sphinx build requirements file writer.""" + """Sphinx build requirements file writer.""" - _TEMPLATE_FILE = os.path.join('data', 'templates', 'docs', 'requirements.txt') + _TEMPLATE_FILE = os.path.join("data", "templates", "docs", "requirements.txt") - PATH = os.path.join('docs', 'requirements.txt') + PATH = os.path.join("docs", "requirements.txt") - def Write(self): - """Writes a docs/requirements.txt file.""" - template_mappings = {} + def Write(self): + """Writes a docs/requirements.txt file.""" + template_mappings = {} - template_file = os.path.join(self._l2tdevtools_path, self._TEMPLATE_FILE) - file_content = self._GenerateFromTemplate(template_file, template_mappings) + template_file = os.path.join(self._l2tdevtools_path, self._TEMPLATE_FILE) + file_content = self._GenerateFromTemplate(template_file, template_mappings) - with open(self.PATH, 'w', encoding='utf-8') as file_object: - file_object.write(file_content) + with open(self.PATH, "w", encoding="utf-8") as file_object: + file_object.write(file_content) diff --git a/l2tdevtools/dependency_writers/tox_ini.py b/l2tdevtools/dependency_writers/tox_ini.py index 0d371ec1..f3eac98e 100644 --- a/l2tdevtools/dependency_writers/tox_ini.py +++ b/l2tdevtools/dependency_writers/tox_ini.py @@ -7,112 +7,114 @@ class ToxIniWriter(interface.DependencyFileWriter): - """Tox.ini file writer.""" + """Tox.ini file writer.""" - _TEMPLATE_DIRECTORY = os.path.join('data', 'templates', 'tox.ini') + _TEMPLATE_DIRECTORY = os.path.join("data", "templates", "tox.ini") - PATH = 'tox.ini' + PATH = "tox.ini" - def _GenerateFromTemplate(self, template_filename, template_mappings): - """Generates file context based on a template file. + def _GenerateFromTemplate(self, template_filename, template_mappings): + """Generates file context based on a template file. - Args: - template_filename (str): path of the template file. - template_mappings (dict[str, str]): template mappings, where the key - maps to the name of a template variable. + Args: + template_filename (str): path of the template file. + template_mappings (dict[str, str]): template mappings, where the key + maps to the name of a template variable. - Returns: - str: output based on the template string. + Returns: + str: output based on the template string. - Raises: - RuntimeError: if the template cannot be formatted. - """ - template_filename = os.path.join( - self._l2tdevtools_path, self._TEMPLATE_DIRECTORY, template_filename) - return super()._GenerateFromTemplate( - template_filename, template_mappings) + Raises: + RuntimeError: if the template cannot be formatted. + """ + template_filename = os.path.join( + self._l2tdevtools_path, self._TEMPLATE_DIRECTORY, template_filename + ) + return super()._GenerateFromTemplate(template_filename, template_mappings) - def Write(self): - """Writes a tox.ini file.""" - # TODO: Write test dependencies. - # python_dependencies = self._GetPyPIPythonDependencies() - # test_dependencies = self._GetPyPITestDependencies(python_dependencies) + def Write(self): + """Writes a tox.ini file.""" + # TODO: Write test dependencies. + # python_dependencies = self._GetPyPIPythonDependencies() + # test_dependencies = self._GetPyPITestDependencies(python_dependencies) - python_module_name = self._project_definition.name + python_module_name = self._project_definition.name - if self._project_definition.name.endswith('-kb'): - python_module_name = ''.join([python_module_name[:-3], 'rc']) + if self._project_definition.name.endswith("-kb"): + python_module_name = "".join([python_module_name[:-3], "rc"]) - paths_to_lint_python = [] - paths_to_lint_yaml = [] + paths_to_lint_python = [] + paths_to_lint_yaml = [] - if os.path.isdir(python_module_name): - paths_to_lint_python.append(python_module_name) + if os.path.isdir(python_module_name): + paths_to_lint_python.append(python_module_name) - if glob.glob(os.path.join( - python_module_name, '**', '*.yaml'), recursive=True): - paths_to_lint_yaml.append(python_module_name) + if glob.glob( + os.path.join(python_module_name, "**", "*.yaml"), recursive=True + ): + paths_to_lint_yaml.append(python_module_name) - if os.path.isdir('data'): - if glob.glob(os.path.join('data', '**', '*.yaml'), recursive=True): - paths_to_lint_yaml.append('data') + if os.path.isdir("data"): + if glob.glob(os.path.join("data", "**", "*.yaml"), recursive=True): + paths_to_lint_yaml.append("data") - if os.path.isdir('scripts'): - paths_to_lint_python.append('scripts') + if os.path.isdir("scripts"): + paths_to_lint_python.append("scripts") - if os.path.isdir('test_data'): - if glob.glob(os.path.join('test_data', '**', '*.yaml'), recursive=True): - paths_to_lint_yaml.append('test_data') + if os.path.isdir("test_data"): + if glob.glob(os.path.join("test_data", "**", "*.yaml"), recursive=True): + paths_to_lint_yaml.append("test_data") - if os.path.isdir('tests'): - paths_to_lint_python.append('tests') + if os.path.isdir("tests"): + paths_to_lint_python.append("tests") - if glob.glob(os.path.join('tests', '**', '*.yaml'), recursive=True): - paths_to_lint_yaml.append('tests') + if glob.glob(os.path.join("tests", "**", "*.yaml"), recursive=True): + paths_to_lint_yaml.append("tests") - if os.path.isdir('tools'): - paths_to_lint_python.append('tools') + if os.path.isdir("tools"): + paths_to_lint_python.append("tools") - envlist = ['py3{10,11,12,13,14}', 'black', 'coverage'] - if os.path.isdir('docs'): - envlist.append('docs') + envlist = ["py3{10,11,12,13,14}", "black", "coverage"] + if os.path.isdir("docs"): + envlist.append("docs") - envlist.extend(['pylint', 'wheel']) + envlist.extend(["pylint", "wheel"]) - if paths_to_lint_yaml: - envlist.append('yamllint') + if paths_to_lint_yaml: + envlist.append("yamllint") - template_mappings = { - 'envlist': ','.join(envlist), - 'paths_to_lint_python': ' '.join(sorted(paths_to_lint_python)), - 'paths_to_lint_yaml': ' '.join(sorted(paths_to_lint_yaml)), - 'project_name': self._project_definition.name, - 'python_module_name': python_module_name} + template_mappings = { + "envlist": ",".join(envlist), + "paths_to_lint_python": " ".join(sorted(paths_to_lint_python)), + "paths_to_lint_yaml": " ".join(sorted(paths_to_lint_yaml)), + "project_name": self._project_definition.name, + "python_module_name": python_module_name, + } - file_content = [] + file_content = [] - template_data = self._GenerateFromTemplate('header', template_mappings) - file_content.append(template_data) + template_data = self._GenerateFromTemplate("header", template_mappings) + file_content.append(template_data) - template_data = self._GenerateFromTemplate( - 'testenv_black', template_mappings) - file_content.append(template_data) + template_data = self._GenerateFromTemplate("testenv_black", template_mappings) + file_content.append(template_data) - if os.path.isdir('docs'): - template_data = self._GenerateFromTemplate( - 'testenv_docs', template_mappings) - file_content.append(template_data) + if os.path.isdir("docs"): + template_data = self._GenerateFromTemplate( + "testenv_docs", template_mappings + ) + file_content.append(template_data) - template_data = self._GenerateFromTemplate( - 'testenv_pylint', template_mappings) - file_content.append(template_data) + template_data = self._GenerateFromTemplate("testenv_pylint", template_mappings) + file_content.append(template_data) - if paths_to_lint_yaml: - template_data = self._GenerateFromTemplate( - 'testenv_yamllint', template_mappings) - file_content.append(template_data) + if paths_to_lint_yaml: + template_data = self._GenerateFromTemplate( + "testenv_yamllint", template_mappings + ) + file_content.append(template_data) - file_content = ''.join(file_content) + file_content = "".join(file_content) - with open(self.PATH, 'w', encoding='utf-8') as file_object: - file_object.write(file_content) + with open(self.PATH, "w", encoding="utf-8") as file_object: + file_object.write(file_content) diff --git a/l2tdevtools/download_helper.py b/l2tdevtools/download_helper.py index 5a53f0f8..e1d14cc2 100644 --- a/l2tdevtools/download_helper.py +++ b/l2tdevtools/download_helper.py @@ -5,47 +5,53 @@ class DownloadHelperFactory: - """Factory class for download helpers.""" - - @classmethod - def NewDownloadHelper(cls, project_definition): - """Creates a new download helper. - - Args: - project_definition (ProjectDefinition): project definition. - - Returns: - DownloadHelper: download helper. - - Raises: - ValueError: if no corresponding helper could be found for the download - URL. - """ - download_url = project_definition.download_url - - if download_url.endswith('/'): - download_url = download_url[:-1] - - # Unify http:// and https:// URLs for the download helper check. - if download_url.startswith('https://'): - url_suffix = download_url[8:] - download_url = f'http://{url_suffix:s}' - - # Remove URL arguments. - download_url, _, _ = download_url.partition('?') - - if download_url.startswith('http://pypi.org/project/'): - return pypi.PyPIDownloadHelper( - download_url, source_name=project_definition.pypi_source_name) - - if (download_url.startswith('http://github.com/') and - download_url.endswith('/releases')): - release_is_archive = project_definition.github_release_is_archive - release_prefix = project_definition.github_release_prefix - release_tag_prefix = project_definition.github_release_tag_prefix - return github.GitHubReleasesDownloadHelper( - download_url, release_is_archive=release_is_archive, - release_prefix=release_prefix, release_tag_prefix=release_tag_prefix) - - raise ValueError( - f'Unsupported download URL: {project_definition.download_url:s}') + """Factory class for download helpers.""" + + @classmethod + def NewDownloadHelper(cls, project_definition): + """Creates a new download helper. + + Args: + project_definition (ProjectDefinition): project definition. + + Returns: + DownloadHelper: download helper. + + Raises: + ValueError: if no corresponding helper could be found for the download + URL. + """ + download_url = project_definition.download_url + + if download_url.endswith("/"): + download_url = download_url[:-1] + + # Unify http:// and https:// URLs for the download helper check. + if download_url.startswith("https://"): + url_suffix = download_url[8:] + download_url = f"http://{url_suffix:s}" + + # Remove URL arguments. + download_url, _, _ = download_url.partition("?") + + if download_url.startswith("http://pypi.org/project/"): + return pypi.PyPIDownloadHelper( + download_url, source_name=project_definition.pypi_source_name + ) + + if download_url.startswith("http://github.com/") and download_url.endswith( + "/releases" + ): + release_is_archive = project_definition.github_release_is_archive + release_prefix = project_definition.github_release_prefix + release_tag_prefix = project_definition.github_release_tag_prefix + return github.GitHubReleasesDownloadHelper( + download_url, + release_is_archive=release_is_archive, + release_prefix=release_prefix, + release_tag_prefix=release_tag_prefix, + ) + + raise ValueError( + f"Unsupported download URL: {project_definition.download_url:s}" + ) diff --git a/l2tdevtools/download_helpers/github.py b/l2tdevtools/download_helpers/github.py index 8b06c10b..bc354ca7 100644 --- a/l2tdevtools/download_helpers/github.py +++ b/l2tdevtools/download_helpers/github.py @@ -6,167 +6,179 @@ class GitHubReleasesDownloadHelper(project.ProjectDownloadHelper): - """Helps in downloading a project with GitHub releases.""" - - def __init__( - self, download_url, release_is_archive=False, release_prefix=None, - release_tag_prefix=None): - """Initializes the download helper. - - Args: - download_url (str): download URL. - release_is_archive (Optional[bool]): source archive is the release. - release_prefix (Optional[str]): release prefix. - release_tag_prefix (Optional[str]): release tag prefix. - - Raises: - ValueError: if download URL is not supported. - """ - url_segments = download_url.split('/') - if len(url_segments) < 5 or url_segments[2] != 'github.com': - raise ValueError('Unsupported download URL.') - - super().__init__(download_url) - self._organization = url_segments[3] - self._release_is_archive = release_is_archive or False - self._release_prefix = release_prefix or '' - self._release_tag_prefix = release_tag_prefix or '' - self._repository = url_segments[4] - - def _GetAvailableVersions(self, version_strings): - """Determines the available versions from version string matched. - - This function will split the version string and convert every digit into - an integer. These lists of integers allow us to reliably compare versions. - A string compare of version strings will yield an incorrect result. - - Args: - version_strings (list[str]): version strings. - - Returns: - dict[str, [int]]: available versions where the key is the original - version string and the value the individual integers of - the digits in the version string. - """ - available_versions = {} - - release_tag_prefix_length = len(self._release_tag_prefix) - - for version_string in version_strings: - if not version_string: - continue - - if (self._release_tag_prefix and - version_string.startswith(self._release_tag_prefix)): - version_string = version_string[release_tag_prefix_length:] - - # Some versions contain '-' as the release number separator for the split - # we want this to be '.'. - comparable_version = version_string.replace('-', '.') - - # Convert the result of map() into a list for Python 3. - comparable_version = list(map(int, comparable_version.split('.'))) - available_versions[version_string] = comparable_version - - return available_versions - - # pylint: disable=unused-argument - def GetLatestVersion(self, project_name, version_definition): - """Retrieves the latest version number for a given project name. - - Args: - project_name (str): name of the project. - version_definition (ProjectVersionDefinition): project version definition - or None if not set. - - Returns: - str: latest version number or None if not available. - """ - earliest_version = None - latest_version = None - - if version_definition: - earliest_version = version_definition.GetEarliestVersion() - if earliest_version and earliest_version[0] == '==': - return '.'.join(earliest_version[1:]) - - latest_version = version_definition.GetLatestVersion() - - download_url = ( - f'https://github.com/{self._organization:s}/{self._repository:s}' - f'/releases') - - page_content = self.DownloadPageContent(download_url) - if not page_content: - return None - - # The format of the project download URL is: - # ]*>[^<]*') - matches = re.findall(expression_string, page_content, flags=re.IGNORECASE) - - if not matches: - return None - - available_versions = self._GetAvailableVersions(matches) - return self._GetLatestVersion( - earliest_version, latest_version, available_versions) - - def GetDownloadURL(self, project_name, project_version): - """Retrieves the download URL for a given project name and version. - - Args: - project_name (str): name of the project. - project_version (str): version of the project. - - Returns: - str: download URL of the project or None if not available. - """ - # TODO: add support for URL arguments '?after=release-2.2.0' - download_url = ( - f'https://github.com/{self._organization:s}/{self._repository:s}' - f'/releases') - - page_content = self.DownloadPageContent(download_url) - if not page_content: - return None + """Helps in downloading a project with GitHub releases.""" + + def __init__( + self, + download_url, + release_is_archive=False, + release_prefix=None, + release_tag_prefix=None, + ): + """Initializes the download helper. + + Args: + download_url (str): download URL. + release_is_archive (Optional[bool]): source archive is the release. + release_prefix (Optional[str]): release prefix. + release_tag_prefix (Optional[str]): release tag prefix. + + Raises: + ValueError: if download URL is not supported. + """ + url_segments = download_url.split("/") + if len(url_segments) < 5 or url_segments[2] != "github.com": + raise ValueError("Unsupported download URL.") + + super().__init__(download_url) + self._organization = url_segments[3] + self._release_is_archive = release_is_archive or False + self._release_prefix = release_prefix or "" + self._release_tag_prefix = release_tag_prefix or "" + self._repository = url_segments[4] + + def _GetAvailableVersions(self, version_strings): + """Determines the available versions from version string matched. + + This function will split the version string and convert every digit into + an integer. These lists of integers allow us to reliably compare versions. + A string compare of version strings will yield an incorrect result. + + Args: + version_strings (list[str]): version strings. + + Returns: + dict[str, [int]]: available versions where the key is the original + version string and the value the individual integers of + the digits in the version string. + """ + available_versions = {} + + release_tag_prefix_length = len(self._release_tag_prefix) + + for version_string in version_strings: + if not version_string: + continue + + if self._release_tag_prefix and version_string.startswith( + self._release_tag_prefix + ): + version_string = version_string[release_tag_prefix_length:] + + # Some versions contain '-' as the release number separator for the split + # we want this to be '.'. + comparable_version = version_string.replace("-", ".") + + # Convert the result of map() into a list for Python 3. + comparable_version = list(map(int, comparable_version.split("."))) + available_versions[version_string] = comparable_version + + return available_versions + + # pylint: disable=unused-argument + def GetLatestVersion(self, project_name, version_definition): + """Retrieves the latest version number for a given project name. + + Args: + project_name (str): name of the project. + version_definition (ProjectVersionDefinition): project version definition + or None if not set. + + Returns: + str: latest version number or None if not available. + """ + earliest_version = None + latest_version = None + + if version_definition: + earliest_version = version_definition.GetEarliestVersion() + if earliest_version and earliest_version[0] == "==": + return ".".join(earliest_version[1:]) + + latest_version = version_definition.GetLatestVersion() - # The format of the project download URL is: - # ]*>' - f'([^<]*)') - - matches = re.findall(expression_string, page_content, flags=re.IGNORECASE) - - download_url = None - if matches and len(matches) == 1: - version = matches[0][0] - - if self._release_is_archive: download_url = ( - f'https://github.com/{self._organization:s}/{self._repository:s}' - f'/archive/refs/tags/{version!s}.tar.gz') - else: - if self._release_prefix: - release = f'{self._release_prefix:s}{version:s}' - else: - release = matches[0][1].replace(" ", "-") - + f"https://github.com/{self._organization:s}/{self._repository:s}" + f"/releases" + ) + + page_content = self.DownloadPageContent(download_url) + if not page_content: + return None + + # The format of the project download URL is: + # ]*>[^<]*' + ) + matches = re.findall(expression_string, page_content, flags=re.IGNORECASE) + + if not matches: + return None + + available_versions = self._GetAvailableVersions(matches) + return self._GetLatestVersion( + earliest_version, latest_version, available_versions + ) + + def GetDownloadURL(self, project_name, project_version): + """Retrieves the download URL for a given project name and version. + + Args: + project_name (str): name of the project. + project_version (str): version of the project. + + Returns: + str: download URL of the project or None if not available. + """ + # TODO: add support for URL arguments '?after=release-2.2.0' download_url = ( - f'https://github.com/{self._organization:s}/{self._repository:s}' - f'/releases/download/{self._release_tag_prefix:s}{version!s}' - f'/{release:s}.tar.gz') - - return download_url - - def GetProjectIdentifier(self): - """Retrieves the project identifier for a given project name. - - Returns: - str: project identifier. - """ - return f'com.github.{self._organization:s}.{self._repository:s}' + f"https://github.com/{self._organization:s}/{self._repository:s}" + f"/releases" + ) + + page_content = self.DownloadPageContent(download_url) + if not page_content: + return None + + # The format of the project download URL is: + # ]*>' + f"([^<]*)" + ) + + matches = re.findall(expression_string, page_content, flags=re.IGNORECASE) + + download_url = None + if matches and len(matches) == 1: + version = matches[0][0] + + if self._release_is_archive: + download_url = ( + f"https://github.com/{self._organization:s}/{self._repository:s}" + f"/archive/refs/tags/{version!s}.tar.gz" + ) + else: + if self._release_prefix: + release = f"{self._release_prefix:s}{version:s}" + else: + release = matches[0][1].replace(" ", "-") + + download_url = ( + f"https://github.com/{self._organization:s}/{self._repository:s}" + f"/releases/download/{self._release_tag_prefix:s}{version!s}" + f"/{release:s}.tar.gz" + ) + + return download_url + + def GetProjectIdentifier(self): + """Retrieves the project identifier for a given project name. + + Returns: + str: project identifier. + """ + return f"com.github.{self._organization:s}.{self._repository:s}" diff --git a/l2tdevtools/download_helpers/interface.py b/l2tdevtools/download_helpers/interface.py index dc3f0606..6d16b289 100644 --- a/l2tdevtools/download_helpers/interface.py +++ b/l2tdevtools/download_helpers/interface.py @@ -9,88 +9,91 @@ class DownloadHelper: - """Helps in downloading files and web content.""" - - def __init__(self, download_url): - """Initializes a download helper. - - Args: - download_url (str): download URL. - """ - super().__init__() - self._cached_url = '' - self._cached_page_content = b'' - self._download_url = download_url - - def DownloadFile(self, download_url): - """Downloads a file from the URL and returns the filename. - - The filename is extracted from the last part of the URL. - - Args: - download_url (str): URL where to download the file. - - Returns: - str: filename if successful also if the file was already downloaded - or None if not available. - """ - _, _, filename = download_url.rpartition('/') - - if not os.path.exists(filename): - logging.info(f'Downloading: {download_url:s}') - - try: - with urllib_request.urlopen(download_url) as url_object: - if url_object.code != 200: - logging.warning( - f'Unable to download URL: {download_url:s} with status code: ' - f'{url_object.code:d}') + """Helps in downloading files and web content.""" + + def __init__(self, download_url): + """Initializes a download helper. + + Args: + download_url (str): download URL. + """ + super().__init__() + self._cached_url = "" + self._cached_page_content = b"" + self._download_url = download_url + + def DownloadFile(self, download_url): + """Downloads a file from the URL and returns the filename. + + The filename is extracted from the last part of the URL. + + Args: + download_url (str): URL where to download the file. + + Returns: + str: filename if successful also if the file was already downloaded + or None if not available. + """ + _, _, filename = download_url.rpartition("/") + + if not os.path.exists(filename): + logging.info(f"Downloading: {download_url:s}") + + try: + with urllib_request.urlopen(download_url) as url_object: + if url_object.code != 200: + logging.warning( + f"Unable to download URL: {download_url:s} with status " + f"code: {url_object.code:d}" + ) + return None + + page_content = url_object.read() + with open(filename, "wb") as file_object: + file_object.write(page_content) + + except (http.client.InvalidURL, urllib_error.URLError) as exception: + logging.warning( + f"Unable to download URL: {download_url:s} with error: " + f"{exception!s}" + ) + return None + + return filename + + def DownloadPageContent(self, download_url, encoding="utf-8"): + """Downloads the page content from the URL and caches it. + + Args: + download_url (str): URL where to download the page content. + encoding (Optional[str]): encoding of the page content, where None + represents no encoding (or binary data). + + Returns: + str: page content if successful or None if not available. + """ + if not download_url: return None - page_content = url_object.read() - with open(filename, 'wb') as file_object: - file_object.write(page_content) + if self._cached_url != download_url: + try: + with urllib_request.urlopen(download_url) as url_object: + if url_object.code != 200: + return None - except (http.client.InvalidURL, urllib_error.URLError) as exception: - logging.warning( - f'Unable to download URL: {download_url:s} with error: ' - f'{exception!s}') - return None + page_content = url_object.read() - return filename + except urllib_error.URLError as exception: + logging.warning( + f"Unable to download URL: {download_url:s} with error: " + f"{exception!s}" + ) + return None - def DownloadPageContent(self, download_url, encoding='utf-8'): - """Downloads the page content from the URL and caches it. + if encoding and isinstance(page_content, bytes): + page_content = page_content.decode(encoding) - Args: - download_url (str): URL where to download the page content. - encoding (Optional[str]): encoding of the page content, where None - represents no encoding (or binary data). + self._cached_page_content = page_content + self._cached_url = download_url - Returns: - str: page content if successful or None if not available. - """ - if not download_url: - return None - - if self._cached_url != download_url: - try: - with urllib_request.urlopen(download_url) as url_object: - if url_object.code != 200: - return None - - page_content = url_object.read() - - except urllib_error.URLError as exception: - logging.warning( - f'Unable to download URL: {download_url:s} with error: ' - f'{exception!s}') - return None - - if encoding and isinstance(page_content, bytes): - page_content = page_content.decode(encoding) - - self._cached_page_content = page_content - self._cached_url = download_url - - return self._cached_page_content + return self._cached_page_content diff --git a/l2tdevtools/download_helpers/project.py b/l2tdevtools/download_helpers/project.py index e8c17b55..1ce683b7 100644 --- a/l2tdevtools/download_helpers/project.py +++ b/l2tdevtools/download_helpers/project.py @@ -8,145 +8,143 @@ class ProjectDownloadHelper(interface.DownloadHelper): - """Helps in downloading a project.""" - - def __init__(self, download_url): - """Initializes a download helper. - - Args: - download_url (str): download URL. - """ - super().__init__(download_url) - self._project_name = None - - def _GetLatestVersion( - self, earliest_version, latest_version, available_versions, - with_epoch=False): - """Determines the latest version from a list of available versions. - - Args: - earliest_version (str): earliest version in the project version definition - or None if not set. - latest_version (str): latest version in the project version definition - or None if not set. - available_versions (dict[str, [int]]): available versions where - the key is the original version string and the value the individual - integers of the digits in the version string. - with_epoch (Optional[bool]): True if the available versions start with - an epoch number. - - Returns: - str: download URL of the project or None if not available. - """ - comparable_earliest_version = None - if earliest_version: - comparable_earliest_version = [ - int(digit) for digit in earliest_version[1:]] - - comparable_latest_version = None - if latest_version: - comparable_latest_version = [ - int(digit) for digit in latest_version[1:]] - - comparable_available_versions = [] - for version in available_versions.values(): - if with_epoch: - comparable_available_versions.append(version[1:]) - else: - comparable_available_versions.append(version) - - latest_match = None - for match in comparable_available_versions: - if earliest_version is not None: - if earliest_version[0] == '>' and match <= comparable_earliest_version: - continue - - if earliest_version[0] == '>=' and match < comparable_earliest_version: - continue - - if latest_version is not None: - if latest_version[0] == '<' and match >= comparable_latest_version: - continue - - if latest_version[0] == '<=' and match > comparable_latest_version: - continue - - if latest_match is None: - latest_match = match - - elif match > latest_match: - latest_match = match - - if not latest_match: - return None - - # Map the latest match value to its index within the dictionary and return - # the version string which is stored as the key in within the available - # versions dictionary. - latest_match = comparable_available_versions.index(latest_match) - - # Convert the result of dict.keys() into a list for Python 3. - return list(available_versions.keys())[latest_match] - - def Download(self, project_name, project_version): - """Downloads the project for a given project name and version. - - Args: - project_name (str): name of the project. - project_version (str): version of the project. - - Returns: - str: filename if successful also if the file was already downloaded - or None if not available. - """ - download_url = self.GetDownloadURL(project_name, project_version) - if not download_url: - logging.warning( - f'Unable to determine download URL for: {project_name:s}') - return None - - filename = self.DownloadFile(download_url) - - # GitHub archive package filenames can be: - # {project version}.tar.gz - # release-{project version}.tar.gz - # v{project version}.tar.gz - github_archive_filenames = [ - f'{project_version!s}.tar.gz', - f'release-{project_version!s}.tar.gz', - f'v{project_version!s}.tar.gz'] - - if filename in github_archive_filenames: - # The desired source package filename is: - # {project name}-{project version}.tar.gz - package_filename = f'{project_name:s}-{project_version!s}.tar.gz' - - if os.path.exists(package_filename): - os.remove(package_filename) - - os.rename(filename, package_filename) - filename = package_filename - - return filename - - # pylint: disable=redundant-returns-doc - @abc.abstractmethod - def GetDownloadURL(self, project_name, project_version): - """Retrieves the download URL for a given project name and version. - - Args: - project_name (str): name of the project. - project_version (str): version of the project. - - Returns: - str: download URL of the project or None on error. - """ - - # pylint: disable=redundant-returns-doc - @abc.abstractmethod - def GetProjectIdentifier(self): - """Retrieves the project identifier for a given project name. - - Returns: - str: project identifier. - """ + """Helps in downloading a project.""" + + def __init__(self, download_url): + """Initializes a download helper. + + Args: + download_url (str): download URL. + """ + super().__init__(download_url) + self._project_name = None + + def _GetLatestVersion( + self, earliest_version, latest_version, available_versions, with_epoch=False + ): + """Determines the latest version from a list of available versions. + + Args: + earliest_version (str): earliest version in the project version definition + or None if not set. + latest_version (str): latest version in the project version definition + or None if not set. + available_versions (dict[str, [int]]): available versions where + the key is the original version string and the value the individual + integers of the digits in the version string. + with_epoch (Optional[bool]): True if the available versions start with + an epoch number. + + Returns: + str: download URL of the project or None if not available. + """ + comparable_earliest_version = None + if earliest_version: + comparable_earliest_version = [int(digit) for digit in earliest_version[1:]] + + comparable_latest_version = None + if latest_version: + comparable_latest_version = [int(digit) for digit in latest_version[1:]] + + comparable_available_versions = [] + for version in available_versions.values(): + if with_epoch: + comparable_available_versions.append(version[1:]) + else: + comparable_available_versions.append(version) + + latest_match = None + for match in comparable_available_versions: + if earliest_version is not None: + if earliest_version[0] == ">" and match <= comparable_earliest_version: + continue + + if earliest_version[0] == ">=" and match < comparable_earliest_version: + continue + + if latest_version is not None: + if latest_version[0] == "<" and match >= comparable_latest_version: + continue + + if latest_version[0] == "<=" and match > comparable_latest_version: + continue + + if latest_match is None: + latest_match = match + + elif match > latest_match: + latest_match = match + + if not latest_match: + return None + + # Map the latest match value to its index within the dictionary and return + # the version string which is stored as the key in within the available + # versions dictionary. + latest_match = comparable_available_versions.index(latest_match) + + # Convert the result of dict.keys() into a list for Python 3. + return list(available_versions.keys())[latest_match] + + def Download(self, project_name, project_version): + """Downloads the project for a given project name and version. + + Args: + project_name (str): name of the project. + project_version (str): version of the project. + + Returns: + str: filename if successful also if the file was already downloaded + or None if not available. + """ + download_url = self.GetDownloadURL(project_name, project_version) + if not download_url: + logging.warning(f"Unable to determine download URL for: {project_name:s}") + return None + + filename = self.DownloadFile(download_url) + + # GitHub archive package filenames can be: + # {project version}.tar.gz + # release-{project version}.tar.gz + # v{project version}.tar.gz + github_archive_filenames = [ + f"{project_version!s}.tar.gz", + f"release-{project_version!s}.tar.gz", + f"v{project_version!s}.tar.gz", + ] + + if filename in github_archive_filenames: + # The desired source package filename is: + # {project name}-{project version}.tar.gz + package_filename = f"{project_name:s}-{project_version!s}.tar.gz" + + if os.path.exists(package_filename): + os.remove(package_filename) + + os.rename(filename, package_filename) + filename = package_filename + + return filename + + # pylint: disable=redundant-returns-doc + @abc.abstractmethod + def GetDownloadURL(self, project_name, project_version): + """Retrieves the download URL for a given project name and version. + + Args: + project_name (str): name of the project. + project_version (str): version of the project. + + Returns: + str: download URL of the project or None on error. + """ + + # pylint: disable=redundant-returns-doc + @abc.abstractmethod + def GetProjectIdentifier(self): + """Retrieves the project identifier for a given project name. + + Returns: + str: project identifier. + """ diff --git a/l2tdevtools/download_helpers/pypi.py b/l2tdevtools/download_helpers/pypi.py index 279471cf..d687b511 100644 --- a/l2tdevtools/download_helpers/pypi.py +++ b/l2tdevtools/download_helpers/pypi.py @@ -8,149 +8,154 @@ class PyPIDownloadHelper(project.ProjectDownloadHelper): - """Helps in downloading a PyPI code project.""" - - def __init__(self, download_url, source_name=None): - """Initializes the download helper. - - Args: - download_url (str): download URL. - source_name (Optional[str]): PyPI source pacakge name, where None - indicates the source package name is the same as the PyPI project - name. - - Raises: - ValueError: if download URL is not supported. - """ - url_segments = download_url.split('/') - if (len(url_segments) < 5 or url_segments[2] != 'pypi.org' or - url_segments[3] != 'project'): - raise ValueError(f'Unsupported download URL: {download_url:s}.') - - super().__init__(download_url) - self._project_name = url_segments[4] - self._source_name = source_name or self._project_name - - def _GetAvailableVersions(self, version_strings): - """Determines the available versions from version string matched. - - This function will split the version string and convert every digit into - an integer. These lists of integers allow us to reliable compare versions. - A string compare of version strings will yield an incorrect result. - - Args: - version_strings (list[str]): version strings. - - Returns: - dict[str, [int]]: available versions where the key is the original - version string and the value the individual integers of - the digits in the version string. - """ - available_versions = {} - - for version_string in version_strings: - if not version_string: - continue - - # We need to support PEP440 epoch versioning, which only newer versions - # of setuptools support. So this is a bit of hack to handle epochs - # but still use setuptools to handle most of the version matching. - epoch, _, epoch_version_string = version_string.partition('!') - if not epoch_version_string: - # Per PEP440, if there is no epoch specified, the epoch is 0. - epoch_version_string = epoch - epoch = 0 - else: - epoch = int(epoch, 10) - - version_object = packaging_version.parse(epoch_version_string) - - # Convert the result of map() into a list for Python 3. - comparable_version = version_object.base_version.split('.') - comparable_version = list(map(int, comparable_version)) - comparable_version.insert(0, epoch) - - # Add the epoch to the version string for comparison. - available_versions[version_string] = comparable_version - - return available_versions - - # pylint: disable=unused-argument - def GetLatestVersion(self, project_name, version_definition): - """Retrieves the latest version number for a given project name. - - Args: - project_name (str): name of the project. - version_definition (ProjectVersionDefinition): project version definition - or None. - - Returns: - str: latest version number or None if not available. - """ - earliest_version = None - latest_version = None - - if version_definition: - earliest_version = version_definition.GetEarliestVersion() - if earliest_version and earliest_version[0] == '==': - return '.'.join(earliest_version[1:]) - - latest_version = version_definition.GetLatestVersion() - - download_url = f'https://pypi.org/pypi/{self._project_name:s}/json' - - page_content = self.DownloadPageContent(download_url) - if not page_content: - return None - - expression_string = ( - f'"https://files.pythonhosted.org/packages/.*/.*/.*/' - f'{self._source_name:s}-([\\d\\.\\!]*(post\\d+)?)' - f'\\.(tar\\.bz2|tar\\.gz|zip)"') - - matches = re.findall(expression_string, page_content, flags=re.IGNORECASE) - if not matches: - return None - - available_versions = self._GetAvailableVersions([ - match[0] for match in matches]) - return self._GetLatestVersion( - earliest_version, latest_version, available_versions, with_epoch=True) - - def GetDownloadURL(self, project_name, project_version): - """Retrieves the download URL for a given project name and version. - - Args: - project_name (str): name of the project. - project_version (str): version of the project. - - Returns: - str: download URL of the project or None if not available. - """ - download_url = f'https://pypi.org/pypi/{self._project_name:s}/json' - - page_content = self.DownloadPageContent(download_url) - if not page_content: - return None - - # The format of the project download URL is: - # https://files.pythonhosted.org/packages/[0-9a-f]*/[0-9a-f]*/[0-9a-f]*/ - # {project name}-{version}.{extension} - expression_string = ( - f'(https://files.pythonhosted.org/packages/[0-9a-f]*/[0-9a-f]*/' - f'[0-9a-f]*/{self._source_name:s}-{project_version!s}[.]' - f'(tar[.]bz2|tar[.]gz|zip))') - matches = re.findall(expression_string, page_content, flags=re.IGNORECASE) - - if not matches: - return None - - return matches[0][0] - - def GetProjectIdentifier(self): - """Retrieves the project identifier for a given project name. - - Returns: - str: project identifier. - """ - return f'org.pypi.{self._project_name:s}' + """Helps in downloading a PyPI code project.""" + + def __init__(self, download_url, source_name=None): + """Initializes the download helper. + + Args: + download_url (str): download URL. + source_name (Optional[str]): PyPI source pacakge name, where None + indicates the source package name is the same as the PyPI project + name. + + Raises: + ValueError: if download URL is not supported. + """ + url_segments = download_url.split("/") + if ( + len(url_segments) < 5 + or url_segments[2] != "pypi.org" + or url_segments[3] != "project" + ): + raise ValueError(f"Unsupported download URL: {download_url:s}.") + + super().__init__(download_url) + self._project_name = url_segments[4] + self._source_name = source_name or self._project_name + + def _GetAvailableVersions(self, version_strings): + """Determines the available versions from version string matched. + + This function will split the version string and convert every digit into + an integer. These lists of integers allow us to reliable compare versions. + A string compare of version strings will yield an incorrect result. + + Args: + version_strings (list[str]): version strings. + + Returns: + dict[str, [int]]: available versions where the key is the original + version string and the value the individual integers of + the digits in the version string. + """ + available_versions = {} + + for version_string in version_strings: + if not version_string: + continue + + # We need to support PEP440 epoch versioning, which only newer versions + # of setuptools support. So this is a bit of hack to handle epochs + # but still use setuptools to handle most of the version matching. + epoch, _, epoch_version_string = version_string.partition("!") + if not epoch_version_string: + # Per PEP440, if there is no epoch specified, the epoch is 0. + epoch_version_string = epoch + epoch = 0 + else: + epoch = int(epoch, 10) + + version_object = packaging_version.parse(epoch_version_string) + + # Convert the result of map() into a list for Python 3. + comparable_version = version_object.base_version.split(".") + comparable_version = list(map(int, comparable_version)) + comparable_version.insert(0, epoch) + + # Add the epoch to the version string for comparison. + available_versions[version_string] = comparable_version + + return available_versions + + # pylint: disable=unused-argument + def GetLatestVersion(self, project_name, version_definition): + """Retrieves the latest version number for a given project name. + + Args: + project_name (str): name of the project. + version_definition (ProjectVersionDefinition): project version definition + or None. + + Returns: + str: latest version number or None if not available. + """ + earliest_version = None + latest_version = None + + if version_definition: + earliest_version = version_definition.GetEarliestVersion() + if earliest_version and earliest_version[0] == "==": + return ".".join(earliest_version[1:]) + + latest_version = version_definition.GetLatestVersion() + + download_url = f"https://pypi.org/pypi/{self._project_name:s}/json" + + page_content = self.DownloadPageContent(download_url) + if not page_content: + return None + + expression_string = ( + f'"https://files.pythonhosted.org/packages/.*/.*/.*/' + f"{self._source_name:s}-([\\d\\.\\!]*(post\\d+)?)" + f'\\.(tar\\.bz2|tar\\.gz|zip)"' + ) + + matches = re.findall(expression_string, page_content, flags=re.IGNORECASE) + if not matches: + return None + + available_versions = self._GetAvailableVersions([match[0] for match in matches]) + return self._GetLatestVersion( + earliest_version, latest_version, available_versions, with_epoch=True + ) + + def GetDownloadURL(self, project_name, project_version): + """Retrieves the download URL for a given project name and version. + + Args: + project_name (str): name of the project. + project_version (str): version of the project. + + Returns: + str: download URL of the project or None if not available. + """ + download_url = f"https://pypi.org/pypi/{self._project_name:s}/json" + + page_content = self.DownloadPageContent(download_url) + if not page_content: + return None + + # The format of the project download URL is: + # https://files.pythonhosted.org/packages/[0-9a-f]*/[0-9a-f]*/[0-9a-f]*/ + # {project name}-{version}.{extension} + expression_string = ( + f"(https://files.pythonhosted.org/packages/[0-9a-f]*/[0-9a-f]*/" + f"[0-9a-f]*/{self._source_name:s}-{project_version!s}[.]" + f"(tar[.]bz2|tar[.]gz|zip))" + ) + matches = re.findall(expression_string, page_content, flags=re.IGNORECASE) + + if not matches: + return None + + return matches[0][0] + + def GetProjectIdentifier(self): + """Retrieves the project identifier for a given project name. + + Returns: + str: project identifier. + """ + return f"org.pypi.{self._project_name:s}" diff --git a/l2tdevtools/download_helpers/zlib.py b/l2tdevtools/download_helpers/zlib.py index e57adbf7..096af9b5 100644 --- a/l2tdevtools/download_helpers/zlib.py +++ b/l2tdevtools/download_helpers/zlib.py @@ -6,72 +6,73 @@ class ZlibDownloadHelper(project.ProjectDownloadHelper): - """Helps in downloading the zlib project.""" - - def __init__(self, download_url): - """Initializes the download helper. - - Args: - download_url (str): download URL. - - Raises: - ValueError: if download URL is not supported. - """ - url_segments = download_url.split('/') - if len(url_segments) < 3 or url_segments[2] != 'www.zlib.net': - raise ValueError('Unsupported download URL.') - - super().__init__(download_url) - self._project_name = 'zlib' - - # pylint: disable=unused-argument - def GetLatestVersion(self, project_name, version_definition): - """Retrieves the latest version number for a given project name. - - Args: - project_name (str): name of the project. - version_definition (ProjectVersionDefinition): project version definition - or None. - - Returns: - str: latest version number or None if not available. - """ - download_url = 'http://www.zlib.net' - - page_content = self.DownloadPageContent(download_url) - if not page_content: - return None - - # The format of the project download URL is: - # http://zlib.net/{project name}-{version}.tar.gz - expression_string = ( - f'') - - _CHANGELOG_TEMPLATE = '\n'.join([ - '{source_package_name:s} ({project_version!s}-1) unstable; urgency=low', - '', - ' * Auto-generated', - '', - ' -- {maintainer_email_address:s} {date_time:s}', - '']) - - _CLEAN_TEMPLATE_PYTHON = '\n'.join([ - '{setup_name:s}/*.pyc', - '*.pyc', - '']) - - _COMPAT_TEMPLATE = '\n'.join([ - '10', - '']) - - _CONTROL_TEMPLATE_CONFIGURE_MAKE = [ - 'Source: {source_package_name:s}', - 'Section: libs', - 'Priority: extra', - 'Maintainer: {upstream_maintainer:s}', - ('Build-Depends: debhelper (>= 9){build_depends:s}'), - 'Standards-Version: 4.1.4', - 'Homepage: {upstream_homepage:s}', - '', - 'Package: {package_name:s}', - 'Architecture: {architecture:s}', - 'Depends: {depends:s}', - 'Description: {description_short:s}', - ' {description_long:s}', - ''] - - _CONTROL_TEMPLATE_SETUP_PY_PYTHON3 = [ - 'Source: {source_package_name:s}', - 'Section: python', - 'Priority: extra', - 'Maintainer: {upstream_maintainer:s}', - 'Build-Depends: debhelper (>= 9){build_depends:s}', - 'Standards-Version: 4.1.4', - 'X-Python3-Version: >= 3.10', - 'Homepage: {upstream_homepage:s}', - '', - 'Package: {python3_package_name:s}', - 'Architecture: {architecture:s}', - 'Depends: {python3_depends:s}', - 'Description: {description_short:s}', - ' {description_long:s}', - ''] - - _CONTROL_TEMPLATE_SETUP_PY_TOOLS = [ - 'Package: {source_package_name:s}-tools', - 'Architecture: all', - ('Depends: {python3_package_name:s} (>= ${{binary:Version}}), ' - 'python3 (>= 3.10~), ${{python3:Depends}}, ${{misc:Depends}}'), - 'Description: Tools of {description_name:s}', - ' {description_long:s}', - ''] - - _COPYRIGHT_TEMPLATE = '\n'.join([ - '']) - - _INSTALL_TEMPLATE_PYTHON_DATA = '\n'.join([ - 'data/* usr/share/{package_name:s}', - '']) - - _INSTALL_TEMPLATE_PYTHON3 = '\n'.join([ - 'usr/lib/python3*/dist-packages/{package_name:s}/', - 'usr/lib/python3*/dist-packages/{package_name:s}*.egg-info/*', - '']) - - _INSTALL_TEMPLATE_PYTHON_TOOLS = '\n'.join([ - 'usr/bin', - '']) - - _RULES_TEMPLATE_CONFIGURE_MAKE = '\n'.join([ - '#!/usr/bin/make -f', - '', - '# Uncomment this to turn on verbose mode.', - '# export DH_VERBOSE=1', - '', - '# This has to be exported to make some magic below work.', - 'export DH_OPTIONS', - '', - '%:', - '\tdh $@ {build_system:s}', - '', - '.PHONY: override_dh_auto_configure', - 'override_dh_auto_configure:', - '\tdh_auto_configure -- {configure_options:s} CFLAGS="-g"', - '', - '.PHONY: override_dh_auto_test', - 'override_dh_auto_test:', - '', - '.PHONY: override_dh_install', - 'override_dh_install:', - '\t# Create the {package_name:s} package.', - '\tdh_install', - '', - '.PHONY: override_dh_strip', - 'override_dh_strip:', - 'ifeq (,$(filter nostrip,$(DEB_BUILD_OPTIONS)))', - ' dh_strip -p{package_name:s} --dbg-package={package_name:s}-dbg', - 'endif', - '', - '.PHONY: override_dh_shlibdeps', - 'override_dh_shlibdeps:', - '\tdh_shlibdeps -L{package_name:s} -l${{CURDIR}}/debian/tmp/usr/lib', - '']) - - # Force the build system to setup.py here in case the package ships - # a Makefile or equivalent. - _RULES_TEMPLATE_SETUP_PY = '\n'.join([ - '#!/usr/bin/make -f', - '', - '%:', - '\tdh $@ --buildsystem=pybuild --with=python3', - '', - '.PHONY: override_dh_auto_test', - 'override_dh_auto_test:', - '', - '']) - - _SOURCE_FORMAT_TEMPLATE = '\n'.join([ - '3.0 (quilt)', - '']) - - _SOURCE_OPTIONS_TEMPLATE = '\n'.join([ - ('extend-diff-ignore = "(^|/)(\\.eggs|config\\.h|config\\.log|' - 'config\\.status|.*\\.egg-info|.*\\.egg-info/.*|.*\\.pxd|.*\\.pyx|' - 'Makefile|CMakeCache.txt|CMakeFiles.*)$"'), - '']) - - def __init__( - self, project_definition, project_version, data_path, - dependency_definitions, build_configuration=None): - """Initializes a dpkg build files generator. - - Args: - project_definition (ProjectDefinition): project definition. - project_version (str): version of the project. - data_path (str): path to the data directory which contains the dpkg - templates sub directory. - dependency_definitions (dict[str, ProjectDefinition]): definitions of all - projects, which is used to determine the properties of dependencies. - build_configuration (Optional[DPKGBuildConfiguration]): the dpgk build - configuration. - """ - super().__init__() - self._build_configuration = build_configuration - self._data_path = data_path - self._dependency_definitions = dependency_definitions - self._project_definition = project_definition - self._project_version = project_version - - def _GenerateFile( - self, template_filename, template_data, template_values, output_filename): - """Generates a file based on a template. - - Args: - template_filename (str): template filename or None if not defined. - If not defined template_data is used. - template_data (str): template data. - template_values (dict[str, str]): template values or None if not defined. - output_filename (str): name of the resulting file. - """ - if template_filename: - template_file_path = os.path.join( - self._data_path, 'dpkg_templates', template_filename) - with open(template_file_path, 'rb') as file_object: - template_data = file_object.read() - - template_data = template_data.decode('utf-8') - - if template_values: - template_data = template_data.format(**template_values) - - template_data = template_data.encode('utf-8') - - with open(output_filename, 'wb') as file_object: - file_object.write(template_data) - - def _GenerateChangelogFile(self, dpkg_path): - """Generates the dpkg build changelog file. - - Args: - dpkg_path (str): path to the dpkg files. - """ - source_package_name = self._GetSourcePackageName() - - timezone_minutes, _ = divmod(time.timezone, 60) - timezone_hours, timezone_minutes = divmod(timezone_minutes, 60) - - # If timezone_hours is -1 {0:02d} will format as -1 instead of -01 - # hence we detect the sign and force a leading zero. - if timezone_hours < 0: - timezone_hours = -timezone_hours - timezone_string = f'-{timezone_hours:02d}{timezone_minutes:02d}' - else: - timezone_string = f'+{timezone_hours:02d}{timezone_minutes:02d}' - - date_time_string = time.strftime('%a, %d %b %Y %H:%M:%S') - - template_values = { - 'date_time': f'{date_time_string:s} {timezone_string:s}', - 'maintainer_email_address': self._EMAIL_ADDRESS, - 'project_version': self._project_version, - 'source_package_name': source_package_name} - - output_filename = os.path.join(dpkg_path, 'changelog') - self._GenerateFile( - None, self._CHANGELOG_TEMPLATE, template_values, output_filename) - - def _GenerateCleanFile(self, dpkg_path): - """Generates the dpkg build clean file. - - Args: - dpkg_path (str): path to the dpkg files. - """ - # TODO: add support for configure_make - - if self._project_definition.build_system in ( - 'flit', 'hatchling', 'poetry', 'scikit', 'setup_py', 'setuptools'): - setup_name = self._GetPythonSetupName() - - template_values = { - 'setup_name': setup_name} - - output_filename = os.path.join(dpkg_path, 'clean') - self._GenerateFile( - None, self._CLEAN_TEMPLATE_PYTHON, template_values, output_filename) - - def _GenerateCompatFile(self, dpkg_path): - """Generates the dpkg build compat file. - - Args: - dpkg_path (str): path to the dpkg files. + """Dpkg build configuration. + + Attributes: + has_bin_directory (bool): True if the Python module creates + a /usr/bin directory. + has_dist_info_directory (bool): True if the Python module has + a .dist_info directory in the dist-packages directory. + has_egg_info_directory (bool): True if the Python module has + an .egg_info directory in the dist-packages directory. + has_egg_info_file (bool): True if the Python module has + an .egg_info file in the dist-packages directory. + has_module_source_files (bool): True if the Python module has + one or more source (*.py) files in the dist-packages directory. + has_module_shared_object (bool): True if the Python module has + one or more shared object (*.so) files in the dist-packages directory. + module_directories (list[str]): module directories in the dist-packages + directory. """ - output_filename = os.path.join(dpkg_path, 'compat') - self._GenerateFile(None, self._COMPAT_TEMPLATE, None, output_filename) - def _GenerateControlFile(self, dpkg_path): - """Generates the dpkg build control file. - - Args: - dpkg_path (str): path to the dpkg files. - """ - source_package_name = self._GetSourcePackageName() - - package_name = self._GetPackageName(self._project_definition) - python3_package_name = self._GetPython3PackageName() - - architecture = self._GetArchitecture() - - build_depends = [] - python3_build_depends = [] - - if self._project_definition.build_system == 'configure_make': - build_depends.append('autotools-dev') - - elif self._project_definition.build_system in ( - 'flit', 'hatchling', 'poetry', 'scikit', 'setup_py', 'setuptools'): - build_depends.append('dh-python') - - if self._project_definition.build_system == 'pyproject': - build_depends.append('pybuild-plugin-pyproject') - - python3_build_depends.append('python3-all (>= 3.10~)') - python3_build_depends.append('python3-setuptools') - - if self._project_definition.architecture_dependent: - python3_build_depends.append('python3-all-dev') - - for dependency in self._project_definition.dpkg_build_dependencies: - if self._project_definition.build_system in ( - 'flit', 'hatchling', 'poetry', 'scikit', 'setup_py', 'setuptools'): - if dependency.startswith('python-'): - python3_build_depends.append(f'python3-{dependency[7:]:s}') - continue - - if (dependency.startswith('python2-') or - dependency.startswith('python3-')): - python3_build_depends.append(f'python3-{dependency[8:]:s}') - continue - - build_depends.append(dependency) - - if self._project_definition.build_system in ( - 'flit', 'hatchling', 'poetry', 'scikit', 'setup_py', 'setuptools'): - build_depends.extend(python3_build_depends) - - if build_depends: - build_depends = ', '.join(build_depends) - build_depends = f', {build_depends:s}' - else: - build_depends = '' - - # description short needs to be a single line. - description_short = self._project_definition.description_short or '' - description_short = ' '.join(description_short.split('\n')) - - # description long needs a space at the start of every line after - # the first. - description_long = self._project_definition.description_long or '' - description_long = '\n '.join(description_long.split('\n')) - - depends = [] - python3_depends = [] - - for dependency in self._project_definition.dpkg_dependencies: - if dependency.startswith('python-'): - python3_depends.append(f'python3-{dependency[7:]:s}') - elif (dependency.startswith('python2-') or - dependency.startswith('python3-')): - python3_depends.append(f'python3-{dependency[8:]:s}') - else: - depends.append(dependency) - - depends.append('${shlibs:Depends}') - depends.append('${misc:Depends}') - depends = ', '.join(depends) - - python3_depends.append('${python3:Depends}') - python3_depends.append('${misc:Depends}') - python3_depends = ', '.join(python3_depends) - - template_values = { - 'architecture': architecture, - 'build_depends': build_depends, - 'depends': depends, - 'description_long': description_long, - 'description_name': self._project_definition.name, - 'description_short': description_short, - 'package_name': package_name, - 'python3_depends': python3_depends, - 'python3_package_name': python3_package_name, - 'source_package_name': source_package_name, - 'upstream_homepage': self._project_definition.homepage_url, - 'upstream_maintainer': self._project_definition.maintainer} - - control_template = [] - if self._project_definition.build_system == 'configure_make': - control_template.extend(self._CONTROL_TEMPLATE_CONFIGURE_MAKE) - - elif self._project_definition.build_system in ( - 'flit', 'hatchling', 'poetry', 'scikit', 'setup_py', 'setuptools'): - control_template.extend(self._CONTROL_TEMPLATE_SETUP_PY_PYTHON3) - - # TODO: add configuration setting to indicate tools should be packaged. - if package_name != 'psutil': - if (self._build_configuration and - self._build_configuration.has_bin_directory): - control_template.extend(self._CONTROL_TEMPLATE_SETUP_PY_TOOLS) - - control_template = '\n'.join(control_template) - - output_filename = os.path.join(dpkg_path, 'control') - self._GenerateFile( - self._project_definition.dpkg_template_control, control_template, - template_values, output_filename) - - def _GenerateCopyrightFile(self, dpkg_path): - """Generates the dpkg build copyright file. - - Args: - dpkg_path (str): path to the dpkg files. - """ - license_file = os.path.dirname(__file__) - license_file = os.path.dirname(license_file) - license_file = os.path.join( - license_file, 'data', 'licenses', - f'LICENSE.{self._project_definition.name:s}') + def __init__(self): + """Initializes a dpkg build configuration.""" + super().__init__() + self.has_bin_directory = False + self.has_dist_info_directory = False + self.has_egg_info_directory = False + self.has_egg_info_file = False + self.has_module_source_files = False + self.has_module_shared_object = False + self.module_directories = [] - filename = os.path.join(dpkg_path, 'copyright') - if os.path.exists(license_file): - shutil.copy(license_file, filename) - - else: - logging.warning(f'Missing license file: {license_file:s}') - with open(filename, 'wb') as file_object: - file_object.write(b'\n') - - def _GenerateInstallFiles(self, dpkg_path): - """Generates the dpkg build .install files. - - Args: - dpkg_path (str): path to the dpkg files. - """ - package_name = self._GetPackageName(self._project_definition) +class DPKGBuildFilesGenerator: + """Dpkg build files generator.""" + + _EMAIL_ADDRESS = "log2timeline development team " + + _CHANGELOG_TEMPLATE = "\n".join( + [ + "{source_package_name:s} ({project_version!s}-1) unstable; urgency=low", + "", + " * Auto-generated", + "", + " -- {maintainer_email_address:s} {date_time:s}", + "", + ] + ) + + _CLEAN_TEMPLATE_PYTHON = "\n".join(["{setup_name:s}/*.pyc", "*.pyc", ""]) + + _COMPAT_TEMPLATE = "\n".join(["10", ""]) + + _CONTROL_TEMPLATE_CONFIGURE_MAKE = [ + "Source: {source_package_name:s}", + "Section: libs", + "Priority: extra", + "Maintainer: {upstream_maintainer:s}", + ("Build-Depends: debhelper (>= 9){build_depends:s}"), + "Standards-Version: 4.1.4", + "Homepage: {upstream_homepage:s}", + "", + "Package: {package_name:s}", + "Architecture: {architecture:s}", + "Depends: {depends:s}", + "Description: {description_short:s}", + " {description_long:s}", + "", + ] + + _CONTROL_TEMPLATE_SETUP_PY_PYTHON3 = [ + "Source: {source_package_name:s}", + "Section: python", + "Priority: extra", + "Maintainer: {upstream_maintainer:s}", + "Build-Depends: debhelper (>= 9){build_depends:s}", + "Standards-Version: 4.1.4", + "X-Python3-Version: >= 3.10", + "Homepage: {upstream_homepage:s}", + "", + "Package: {python3_package_name:s}", + "Architecture: {architecture:s}", + "Depends: {python3_depends:s}", + "Description: {description_short:s}", + " {description_long:s}", + "", + ] + + _CONTROL_TEMPLATE_SETUP_PY_TOOLS = [ + "Package: {source_package_name:s}-tools", + "Architecture: all", + ( + "Depends: {python3_package_name:s} (>= ${{binary:Version}}), " + "python3 (>= 3.10~), ${{python3:Depends}}, ${{misc:Depends}}" + ), + "Description: Tools of {description_name:s}", + " {description_long:s}", + "", + ] + + _COPYRIGHT_TEMPLATE = "\n".join([""]) + + _INSTALL_TEMPLATE_PYTHON_DATA = "\n".join(["data/* usr/share/{package_name:s}", ""]) + + _INSTALL_TEMPLATE_PYTHON3 = "\n".join( + [ + "usr/lib/python3*/dist-packages/{package_name:s}/", + "usr/lib/python3*/dist-packages/{package_name:s}*.egg-info/*", + "", + ] + ) + + _INSTALL_TEMPLATE_PYTHON_TOOLS = "\n".join(["usr/bin", ""]) + + _RULES_TEMPLATE_CONFIGURE_MAKE = "\n".join( + [ + "#!/usr/bin/make -f", + "", + "# Uncomment this to turn on verbose mode.", + "# export DH_VERBOSE=1", + "", + "# This has to be exported to make some magic below work.", + "export DH_OPTIONS", + "", + "%:", + "\tdh $@ {build_system:s}", + "", + ".PHONY: override_dh_auto_configure", + "override_dh_auto_configure:", + '\tdh_auto_configure -- {configure_options:s} CFLAGS="-g"', + "", + ".PHONY: override_dh_auto_test", + "override_dh_auto_test:", + "", + ".PHONY: override_dh_install", + "override_dh_install:", + "\t# Create the {package_name:s} package.", + "\tdh_install", + "", + ".PHONY: override_dh_strip", + "override_dh_strip:", + "ifeq (,$(filter nostrip,$(DEB_BUILD_OPTIONS)))", + " dh_strip -p{package_name:s} --dbg-package={package_name:s}-dbg", + "endif", + "", + ".PHONY: override_dh_shlibdeps", + "override_dh_shlibdeps:", + "\tdh_shlibdeps -L{package_name:s} -l${{CURDIR}}/debian/tmp/usr/lib", + "", + ] + ) + + # Force the build system to setup.py here in case the package ships + # a Makefile or equivalent. + _RULES_TEMPLATE_SETUP_PY = "\n".join( + [ + "#!/usr/bin/make -f", + "", + "%:", + "\tdh $@ --buildsystem=pybuild --with=python3", + "", + ".PHONY: override_dh_auto_test", + "override_dh_auto_test:", + "", + "", + ] + ) + + _SOURCE_FORMAT_TEMPLATE = "\n".join(["3.0 (quilt)", ""]) + + _SOURCE_OPTIONS_TEMPLATE = "\n".join( + [ + ( + 'extend-diff-ignore = "(^|/)(\\.eggs|config\\.h|config\\.log|' + "config\\.status|.*\\.egg-info|.*\\.egg-info/.*|.*\\.pxd|.*\\.pyx|" + 'Makefile|CMakeCache.txt|CMakeFiles.*)$"' + ), + "", + ] + ) + + def __init__( + self, + project_definition, + project_version, + data_path, + dependency_definitions, + build_configuration=None, + ): + """Initializes a dpkg build files generator. + + Args: + project_definition (ProjectDefinition): project definition. + project_version (str): version of the project. + data_path (str): path to the data directory which contains the dpkg + templates sub directory. + dependency_definitions (dict[str, ProjectDefinition]): definitions of all + projects, which is used to determine the properties of dependencies. + build_configuration (Optional[DPKGBuildConfiguration]): the dpgk build + configuration. + """ + super().__init__() + self._build_configuration = build_configuration + self._data_path = data_path + self._dependency_definitions = dependency_definitions + self._project_definition = project_definition + self._project_version = project_version + + def _GenerateFile( + self, template_filename, template_data, template_values, output_filename + ): + """Generates a file based on a template. + + Args: + template_filename (str): template filename or None if not defined. + If not defined template_data is used. + template_data (str): template data. + template_values (dict[str, str]): template values or None if not defined. + output_filename (str): name of the resulting file. + """ + if template_filename: + template_file_path = os.path.join( + self._data_path, "dpkg_templates", template_filename + ) + with open(template_file_path, "rb") as file_object: + template_data = file_object.read() + + template_data = template_data.decode("utf-8") + + if template_values: + template_data = template_data.format(**template_values) + + template_data = template_data.encode("utf-8") + + with open(output_filename, "wb") as file_object: + file_object.write(template_data) + + def _GenerateChangelogFile(self, dpkg_path): + """Generates the dpkg build changelog file. + + Args: + dpkg_path (str): path to the dpkg files. + """ + source_package_name = self._GetSourcePackageName() + + timezone_minutes, _ = divmod(time.timezone, 60) + timezone_hours, timezone_minutes = divmod(timezone_minutes, 60) + + # If timezone_hours is -1 {0:02d} will format as -1 instead of -01 + # hence we detect the sign and force a leading zero. + if timezone_hours < 0: + timezone_hours = -timezone_hours + timezone_string = f"-{timezone_hours:02d}{timezone_minutes:02d}" + else: + timezone_string = f"+{timezone_hours:02d}{timezone_minutes:02d}" - # Python modules names contain "_" instead of "-" - package_name = package_name.replace('-', '_') + date_time_string = time.strftime("%a, %d %b %Y %H:%M:%S") - template_values = {'package_name': package_name} + template_values = { + "date_time": f"{date_time_string:s} {timezone_string:s}", + "maintainer_email_address": self._EMAIL_ADDRESS, + "project_version": self._project_version, + "source_package_name": source_package_name, + } - if self._project_definition.dpkg_template_install: - for template_file in self._project_definition.dpkg_template_install: - output_filename = os.path.join(dpkg_path, template_file) + output_filename = os.path.join(dpkg_path, "changelog") self._GenerateFile( - template_file, None, template_values, output_filename) - - # Generate install files if there is more than 1 package. - elif (self._build_configuration and - self._build_configuration.has_bin_directory): - self._GeneratePython3ModuleInstallFile(dpkg_path, template_values) - - install_package_name = self._GetPackageName(self._project_definition) - output_filename = os.path.join( - dpkg_path, f'{install_package_name:s}-tools.install') - self._GenerateFile( - None, self._INSTALL_TEMPLATE_PYTHON_TOOLS, template_values, - output_filename) - - # TODO: add support for data install files. - - def _GeneratePy3DistOverridesFile(self, dpkg_path): - """Generates the dpkg build py3dist-overrides file if required. - - Args: - dpkg_path (str): path to the dpkg files. - """ - if self._project_definition.dpkg_template_py3dist_overrides: - output_filename = os.path.join(dpkg_path, 'py3dist-overrides') - - self._GenerateFile( - self._project_definition.dpkg_template_py3dist_overrides, - None, None, output_filename) - - def _GeneratePython3ModuleInstallFile(self, dpkg_path, template_values): - """Generates the dpkg build Python 3 module .install file. - - Args: - dpkg_path (str): path to the dpkg files. - template_values (dict[str, str]): template values or None if not defined. - """ - python3_package_name = self._GetPython3PackageName() - - template_files = ( - self._project_definition.dpkg_template_install_python3 or [None]) - - for template_file in template_files: - if template_file: - output_filename = template_file - template_data = None - else: - output_filename = f'{python3_package_name:s}.install' - if not self._build_configuration: - template_data = self._INSTALL_TEMPLATE_PYTHON3 + None, self._CHANGELOG_TEMPLATE, template_values, output_filename + ) + + def _GenerateCleanFile(self, dpkg_path): + """Generates the dpkg build clean file. + + Args: + dpkg_path (str): path to the dpkg files. + """ + # TODO: add support for configure_make + + if self._project_definition.build_system in ( + "flit", + "hatchling", + "poetry", + "scikit", + "setup_py", + "setuptools", + ): + setup_name = self._GetPythonSetupName() + + template_values = {"setup_name": setup_name} + + output_filename = os.path.join(dpkg_path, "clean") + self._GenerateFile( + None, self._CLEAN_TEMPLATE_PYTHON, template_values, output_filename + ) + + def _GenerateCompatFile(self, dpkg_path): + """Generates the dpkg build compat file. + + Args: + dpkg_path (str): path to the dpkg files. + """ + output_filename = os.path.join(dpkg_path, "compat") + self._GenerateFile(None, self._COMPAT_TEMPLATE, None, output_filename) + + def _GenerateControlFile(self, dpkg_path): + """Generates the dpkg build control file. + + Args: + dpkg_path (str): path to the dpkg files. + """ + source_package_name = self._GetSourcePackageName() + + package_name = self._GetPackageName(self._project_definition) + python3_package_name = self._GetPython3PackageName() + + architecture = self._GetArchitecture() + + build_depends = [] + python3_build_depends = [] + + if self._project_definition.build_system == "configure_make": + build_depends.append("autotools-dev") + + elif self._project_definition.build_system in ( + "flit", + "hatchling", + "poetry", + "scikit", + "setup_py", + "setuptools", + ): + build_depends.append("dh-python") + + if self._project_definition.build_system == "pyproject": + build_depends.append("pybuild-plugin-pyproject") + + python3_build_depends.append("python3-all (>= 3.10~)") + python3_build_depends.append("python3-setuptools") + + if self._project_definition.architecture_dependent: + python3_build_depends.append("python3-all-dev") + + for dependency in self._project_definition.dpkg_build_dependencies: + if self._project_definition.build_system in ( + "flit", + "hatchling", + "poetry", + "scikit", + "setup_py", + "setuptools", + ): + if dependency.startswith("python-"): + python3_build_depends.append(f"python3-{dependency[7:]:s}") + continue + + if dependency.startswith("python2-") or dependency.startswith( + "python3-" + ): + python3_build_depends.append(f"python3-{dependency[8:]:s}") + continue + + build_depends.append(dependency) + + if self._project_definition.build_system in ( + "flit", + "hatchling", + "poetry", + "scikit", + "setup_py", + "setuptools", + ): + build_depends.extend(python3_build_depends) + + if build_depends: + build_depends = ", ".join(build_depends) + build_depends = f", {build_depends:s}" else: - template_data = [] - - if self._build_configuration.has_module_source_files: - template_data.append('usr/lib/python3*/dist-packages/*.py') - if self._build_configuration.has_module_shared_object: - template_data.append('usr/lib/python3*/dist-packages/*.so') - - module_directories = self._build_configuration.module_directories - template_data.extend([ - f'usr/lib/python3*/dist-packages/{module_directory:s}' - for module_directory in module_directories]) - - if self._build_configuration.has_dist_info_directory: - template_data.append( - 'usr/lib/python3*/dist-packages/*.dist-info/*') - - elif self._build_configuration.has_egg_info_directory: - template_data.append( - 'usr/lib/python3*/dist-packages/*.egg-info/*') - - elif self._build_configuration.has_egg_info_file: - template_data.append( - 'usr/lib/python3*/dist-packages/*.egg-info') - - template_data = '\n'.join(template_data) - - output_filename = os.path.join(dpkg_path, output_filename) - self._GenerateFile( - template_file, template_data, template_values, output_filename) - - def _GenerateRulesFile(self, dpkg_path): - """Generates the dpkg build rules file. - - Args: - dpkg_path (str): path to the dpkg files. - """ - if self._project_definition.build_system == 'configure_make': - self._GenerateConfigureMakeRulesFile(dpkg_path) - - else: - self._GeneratePyProjectRulesFile(dpkg_path) - - filename = os.path.join(dpkg_path, 'rules') - stat_info = os.stat(filename) - os.chmod(filename, stat_info.st_mode | stat.S_IEXEC) - - def _GenerateConfigureMakeRulesFile(self, dpkg_path): - """Generates the dpkg build rules file. - - Args: - dpkg_path (str): path to the dpkg files. - """ - package_name = self._GetPackageName(self._project_definition) - - build_system = '--buildsystem=autoconf' - - configure_options = '' - if self._project_definition.dpkg_configure_options: - configure_options = ' '.join( - self._project_definition.dpkg_configure_options) - - elif self._project_definition.configure_options: - configure_options = ' '.join( - self._project_definition.configure_options) - - template_values = { - 'build_system': build_system, - 'configure_options': configure_options, - 'package_name': package_name} - - output_filename = os.path.join(dpkg_path, 'rules') - self._GenerateFile( - self._project_definition.dpkg_template_rules, - self._RULES_TEMPLATE_CONFIGURE_MAKE, template_values, output_filename) - - def _GeneratePyProjectRulesFile(self, dpkg_path): - """Generates the dpkg build rules file. - - Args: - dpkg_path (str): path to the dpkg files. - """ - setup_name = self._GetPythonSetupName() - - template_values = { - 'setup_name': setup_name} - - rules_template = self._RULES_TEMPLATE_SETUP_PY - - # TODO: replace manual write of rules file by call to _GenerateFile. - template_filename = self._project_definition.dpkg_template_rules - if template_filename: - template_file_path = os.path.join( - self._data_path, 'dpkg_templates', template_filename) - with open(template_file_path, 'rb') as file_object: - rules_template = file_object.read() - - rules_template = rules_template.decode('utf-8') + build_depends = "" + + # description short needs to be a single line. + description_short = self._project_definition.description_short or "" + description_short = " ".join(description_short.split("\n")) + + # description long needs a space at the start of every line after + # the first. + description_long = self._project_definition.description_long or "" + description_long = "\n ".join(description_long.split("\n")) + + depends = [] + python3_depends = [] + + for dependency in self._project_definition.dpkg_dependencies: + if dependency.startswith("python-"): + python3_depends.append(f"python3-{dependency[7:]:s}") + elif dependency.startswith("python2-") or dependency.startswith("python3-"): + python3_depends.append(f"python3-{dependency[8:]:s}") + else: + depends.append(dependency) + + depends.append("${shlibs:Depends}") + depends.append("${misc:Depends}") + depends = ", ".join(depends) + + python3_depends.append("${python3:Depends}") + python3_depends.append("${misc:Depends}") + python3_depends = ", ".join(python3_depends) + + template_values = { + "architecture": architecture, + "build_depends": build_depends, + "depends": depends, + "description_long": description_long, + "description_name": self._project_definition.name, + "description_short": description_short, + "package_name": package_name, + "python3_depends": python3_depends, + "python3_package_name": python3_package_name, + "source_package_name": source_package_name, + "upstream_homepage": self._project_definition.homepage_url, + "upstream_maintainer": self._project_definition.maintainer, + } + + control_template = [] + if self._project_definition.build_system == "configure_make": + control_template.extend(self._CONTROL_TEMPLATE_CONFIGURE_MAKE) + + elif self._project_definition.build_system in ( + "flit", + "hatchling", + "poetry", + "scikit", + "setup_py", + "setuptools", + ): + control_template.extend(self._CONTROL_TEMPLATE_SETUP_PY_PYTHON3) + + # TODO: add configuration setting to indicate tools should be packaged. + if package_name != "psutil": + if ( + self._build_configuration + and self._build_configuration.has_bin_directory + ): + control_template.extend(self._CONTROL_TEMPLATE_SETUP_PY_TOOLS) + + control_template = "\n".join(control_template) + + output_filename = os.path.join(dpkg_path, "control") + self._GenerateFile( + self._project_definition.dpkg_template_control, + control_template, + template_values, + output_filename, + ) + + def _GenerateCopyrightFile(self, dpkg_path): + """Generates the dpkg build copyright file. + + Args: + dpkg_path (str): path to the dpkg files. + """ + license_file = os.path.dirname(__file__) + license_file = os.path.dirname(license_file) + license_file = os.path.join( + license_file, + "data", + "licenses", + f"LICENSE.{self._project_definition.name:s}", + ) + + filename = os.path.join(dpkg_path, "copyright") + + if os.path.exists(license_file): + shutil.copy(license_file, filename) - output_filename = os.path.join(dpkg_path, 'rules') - with open(output_filename, 'wb') as file_object: - data = rules_template.format(**template_values) - file_object.write(data.encode('utf-8')) + else: + logging.warning(f"Missing license file: {license_file:s}") + with open(filename, "wb") as file_object: + file_object.write(b"\n") + + def _GenerateInstallFiles(self, dpkg_path): + """Generates the dpkg build .install files. + + Args: + dpkg_path (str): path to the dpkg files. + """ + package_name = self._GetPackageName(self._project_definition) + + # Python modules names contain "_" instead of "-" + package_name = package_name.replace("-", "_") + + template_values = {"package_name": package_name} + + if self._project_definition.dpkg_template_install: + for template_file in self._project_definition.dpkg_template_install: + output_filename = os.path.join(dpkg_path, template_file) + self._GenerateFile( + template_file, None, template_values, output_filename + ) + + # Generate install files if there is more than 1 package. + elif self._build_configuration and self._build_configuration.has_bin_directory: + self._GeneratePython3ModuleInstallFile(dpkg_path, template_values) + + install_package_name = self._GetPackageName(self._project_definition) + output_filename = os.path.join( + dpkg_path, f"{install_package_name:s}-tools.install" + ) + self._GenerateFile( + None, + self._INSTALL_TEMPLATE_PYTHON_TOOLS, + template_values, + output_filename, + ) + + # TODO: add support for data install files. + + def _GeneratePy3DistOverridesFile(self, dpkg_path): + """Generates the dpkg build py3dist-overrides file if required. + + Args: + dpkg_path (str): path to the dpkg files. + """ + if self._project_definition.dpkg_template_py3dist_overrides: + output_filename = os.path.join(dpkg_path, "py3dist-overrides") + + self._GenerateFile( + self._project_definition.dpkg_template_py3dist_overrides, + None, + None, + output_filename, + ) + + def _GeneratePython3ModuleInstallFile(self, dpkg_path, template_values): + """Generates the dpkg build Python 3 module .install file. + + Args: + dpkg_path (str): path to the dpkg files. + template_values (dict[str, str]): template values or None if not defined. + """ + python3_package_name = self._GetPython3PackageName() + + template_files = self._project_definition.dpkg_template_install_python3 or [ + None + ] + + for template_file in template_files: + if template_file: + output_filename = template_file + template_data = None + else: + output_filename = f"{python3_package_name:s}.install" + if not self._build_configuration: + template_data = self._INSTALL_TEMPLATE_PYTHON3 + else: + template_data = [] + + if self._build_configuration.has_module_source_files: + template_data.append("usr/lib/python3*/dist-packages/*.py") + if self._build_configuration.has_module_shared_object: + template_data.append("usr/lib/python3*/dist-packages/*.so") + + module_directories = self._build_configuration.module_directories + template_data.extend( + [ + f"usr/lib/python3*/dist-packages/{module_directory:s}" + for module_directory in module_directories + ] + ) + + if self._build_configuration.has_dist_info_directory: + template_data.append( + "usr/lib/python3*/dist-packages/*.dist-info/*" + ) + + elif self._build_configuration.has_egg_info_directory: + template_data.append( + "usr/lib/python3*/dist-packages/*.egg-info/*" + ) + + elif self._build_configuration.has_egg_info_file: + template_data.append( + "usr/lib/python3*/dist-packages/*.egg-info" + ) + + template_data = "\n".join(template_data) + + output_filename = os.path.join(dpkg_path, output_filename) + self._GenerateFile( + template_file, template_data, template_values, output_filename + ) + + def _GenerateRulesFile(self, dpkg_path): + """Generates the dpkg build rules file. + + Args: + dpkg_path (str): path to the dpkg files. + """ + if self._project_definition.build_system == "configure_make": + self._GenerateConfigureMakeRulesFile(dpkg_path) - def _GenerateSourceFormatFile(self, dpkg_path): - """Generates the dpkg build source/format file. + else: + self._GeneratePyProjectRulesFile(dpkg_path) - Args: - dpkg_path (str): path to the dpkg files. - """ - template_file = self._SOURCE_FORMAT_TEMPLATE + filename = os.path.join(dpkg_path, "rules") + stat_info = os.stat(filename) + os.chmod(filename, stat_info.st_mode | stat.S_IEXEC) - output_filename = os.path.join(dpkg_path, 'source', 'format') + def _GenerateConfigureMakeRulesFile(self, dpkg_path): + """Generates the dpkg build rules file. - self._GenerateFile(None, template_file, None, output_filename) + Args: + dpkg_path (str): path to the dpkg files. + """ + package_name = self._GetPackageName(self._project_definition) - def _GenerateSourceOptionsFile(self, dpkg_path): - """Generates the dpkg build source/options file. + build_system = "--buildsystem=autoconf" - Args: - dpkg_path (str): path to the dpkg files. - """ - output_filename = os.path.join(dpkg_path, 'source', 'options') + configure_options = "" + if self._project_definition.dpkg_configure_options: + configure_options = " ".join( + self._project_definition.dpkg_configure_options + ) - self._GenerateFile( - self._project_definition.dpkg_template_source_options, - self._SOURCE_OPTIONS_TEMPLATE, None, output_filename) + elif self._project_definition.configure_options: + configure_options = " ".join(self._project_definition.configure_options) - def _GetArchitecture(self): - """Retrieves the architecture. + template_values = { + "build_system": build_system, + "configure_options": configure_options, + "package_name": package_name, + } - Returns: - str: architecture. - """ - if not self._project_definition.architecture_dependent: - return 'all' - - return 'any' + output_filename = os.path.join(dpkg_path, "rules") + self._GenerateFile( + self._project_definition.dpkg_template_rules, + self._RULES_TEMPLATE_CONFIGURE_MAKE, + template_values, + output_filename, + ) - def _GetPackageName(self, project_definition): - """Retrieves the package name. + def _GeneratePyProjectRulesFile(self, dpkg_path): + """Generates the dpkg build rules file. - Args: - project_definition (ProjectDefinition): project definition. + Args: + dpkg_path (str): path to the dpkg files. + """ + setup_name = self._GetPythonSetupName() - Returns: - str: package name. - """ - if project_definition.dpkg_name: - package_name = project_definition.dpkg_name - else: - package_name = project_definition.name + template_values = {"setup_name": setup_name} - if package_name.startswith('python-'): - package_name = package_name[7:] - elif (package_name.startswith('python2-') or - package_name.startswith('python3-')): - package_name = package_name[8:] + rules_template = self._RULES_TEMPLATE_SETUP_PY - return package_name + # TODO: replace manual write of rules file by call to _GenerateFile. + template_filename = self._project_definition.dpkg_template_rules + if template_filename: + template_file_path = os.path.join( + self._data_path, "dpkg_templates", template_filename + ) + with open(template_file_path, "rb") as file_object: + rules_template = file_object.read() - def _GetPython3PackageName(self): - """Retrieves the Python 3 package name. - - Returns: - str: Python 3 package name. - """ - package_name = self._GetPackageName(self._project_definition) - return f'python3-{package_name:s}' + rules_template = rules_template.decode("utf-8") - def _GetPythonSetupName(self): - """Retrieves the Python setup.py name. + output_filename = os.path.join(dpkg_path, "rules") + with open(output_filename, "wb") as file_object: + data = rules_template.format(**template_values) + file_object.write(data.encode("utf-8")) - Returns: - str: setup.py name. - """ - if self._project_definition.setup_name: - return self._project_definition.setup_name + def _GenerateSourceFormatFile(self, dpkg_path): + """Generates the dpkg build source/format file. - return self._project_definition.name + Args: + dpkg_path (str): path to the dpkg files. + """ + template_file = self._SOURCE_FORMAT_TEMPLATE - def _GetSourcePackageName(self): - """Retrieves the source package name. + output_filename = os.path.join(dpkg_path, "source", "format") - Returns: - str: source package name. - """ - if self._project_definition.dpkg_source_name: - return self._project_definition.dpkg_source_name + self._GenerateFile(None, template_file, None, output_filename) - return self._project_definition.name + def _GenerateSourceOptionsFile(self, dpkg_path): + """Generates the dpkg build source/options file. - def GenerateFiles(self, dpkg_path): - """Generates the dpkg build files. + Args: + dpkg_path (str): path to the dpkg files. + """ + output_filename = os.path.join(dpkg_path, "source", "options") - Args: - dpkg_path (str): path to the dpkg files. - """ - os.mkdir(dpkg_path) - self._GenerateChangelogFile(dpkg_path) - self._GenerateCleanFile(dpkg_path) - self._GenerateCompatFile(dpkg_path) - self._GenerateControlFile(dpkg_path) - self._GenerateCopyrightFile(dpkg_path) - self._GenerateInstallFiles(dpkg_path) - self._GenerateRulesFile(dpkg_path) - - for filename in self._project_definition.dpkg_template_additional: - output_filename = os.path.join(dpkg_path, filename) - self._GenerateFile(filename, '', None, output_filename) - - os.mkdir(os.path.join(dpkg_path, 'source')) - self._GeneratePy3DistOverridesFile(dpkg_path) - self._GenerateSourceFormatFile(dpkg_path) - self._GenerateSourceOptionsFile(dpkg_path) + self._GenerateFile( + self._project_definition.dpkg_template_source_options, + self._SOURCE_OPTIONS_TEMPLATE, + None, + output_filename, + ) + + def _GetArchitecture(self): + """Retrieves the architecture. + + Returns: + str: architecture. + """ + if not self._project_definition.architecture_dependent: + return "all" + + return "any" + + def _GetPackageName(self, project_definition): + """Retrieves the package name. + + Args: + project_definition (ProjectDefinition): project definition. + + Returns: + str: package name. + """ + if project_definition.dpkg_name: + package_name = project_definition.dpkg_name + else: + package_name = project_definition.name + + if package_name.startswith("python-"): + package_name = package_name[7:] + elif package_name.startswith("python2-") or package_name.startswith("python3-"): + package_name = package_name[8:] + + return package_name + + def _GetPython3PackageName(self): + """Retrieves the Python 3 package name. + + Returns: + str: Python 3 package name. + """ + package_name = self._GetPackageName(self._project_definition) + return f"python3-{package_name:s}" + + def _GetPythonSetupName(self): + """Retrieves the Python setup.py name. + + Returns: + str: setup.py name. + """ + if self._project_definition.setup_name: + return self._project_definition.setup_name + + return self._project_definition.name + + def _GetSourcePackageName(self): + """Retrieves the source package name. + + Returns: + str: source package name. + """ + if self._project_definition.dpkg_source_name: + return self._project_definition.dpkg_source_name + + return self._project_definition.name + + def GenerateFiles(self, dpkg_path): + """Generates the dpkg build files. + + Args: + dpkg_path (str): path to the dpkg files. + """ + os.mkdir(dpkg_path) + self._GenerateChangelogFile(dpkg_path) + self._GenerateCleanFile(dpkg_path) + self._GenerateCompatFile(dpkg_path) + self._GenerateControlFile(dpkg_path) + self._GenerateCopyrightFile(dpkg_path) + self._GenerateInstallFiles(dpkg_path) + self._GenerateRulesFile(dpkg_path) + + for filename in self._project_definition.dpkg_template_additional: + output_filename = os.path.join(dpkg_path, filename) + self._GenerateFile(filename, "", None, output_filename) + + os.mkdir(os.path.join(dpkg_path, "source")) + self._GeneratePy3DistOverridesFile(dpkg_path) + self._GenerateSourceFormatFile(dpkg_path) + self._GenerateSourceOptionsFile(dpkg_path) diff --git a/l2tdevtools/helpers/project.py b/l2tdevtools/helpers/project.py index 26feda96..91646c5f 100644 --- a/l2tdevtools/helpers/project.py +++ b/l2tdevtools/helpers/project.py @@ -8,131 +8,134 @@ class ProjectHelper(cli.CLIHelper): - """Helper for interacting with a project. - - Attributes: - project_name (str): name of the project. - """ - - _AUTHORS_FILE_HEADER = [ - '# Names should be added to this file with this pattern:', - '#', - '# For individuals:', - '# Name (email address)', - '#', - '# For organizations:', - '# Organization (fnmatch pattern)', - '#', - '# See python fnmatch module documentation for more information.', - '', - 'Google Inc. (*@google.com)'] - - SUPPORTED_PROJECTS = frozenset([ - 'acstore', - 'artifacts', - 'artifacts-kb', - 'clitooltester', - 'dfdatetime', - 'dfkinds', - 'dfimagetools', - 'dftimewolf', - 'dfvfs', - 'dfvfs-snippets', - 'dfwinreg', - 'dtfabric', - 'dtformats', - 'esedb-kb', - 'l2tdevtools', - 'l2tdocs', - 'l2tscaffolder', - 'olecf-kb', - 'plaso', - 'plist-kb', - 'sqlite-kb', - 'vstools', - 'winevt-kb', - 'winreg-kb', - 'winshl-kb', - 'winsps-kb']) - - def __init__(self, project_path): - """Initializes a project helper. - - Args: - project_path (str): path to the project. - - Raises: - ValueError: if the project name is not supported. - """ - super().__init__() - self._project_definition = None - self.project_name = self._GetProjectName(project_path) - - @property - def version_file_path(self): - """str: path of the version file.""" - return os.path.join(self.project_name, '__init__.py') - - def _GetProjectName(self, project_path): - """Retrieves the project name from the script path. - - Args: - project_path (str): path to the root of the project. - - Returns: - str: project name. - - Raises: - ValueError: if the project name is not supported. - """ - project_name = os.path.abspath(project_path) - project_name = os.path.basename(project_name) - - # The review.py check is needed for the l2tdevtools tests. - if (project_name != 'review.py' and - project_name not in self.SUPPORTED_PROJECTS): - raise ValueError(f'Unsupported project name: {project_name:s}.') - - return project_name - - def _ReadFileContents(self, path): - """Reads the contents of a file. - - Args: - path (str): path of the file. - - Returns: - bytes: file content or None if not available. - """ - if not os.path.exists(path): - logging.error(f'Missing file: {path:s}') - return None - - try: - with open(path, 'rb') as file_object: - file_contents = file_object.read() - - except IOError as exception: - logging.error(f'Unable to read file with error: {exception!s}') - return None - - try: - file_contents = file_contents.decode('utf-8') - except UnicodeDecodeError as exception: - logging.error(f'Unable to read file with error: {exception!s}') - return None - - return file_contents - - def ReadDefinitionFile(self): - """Reads the project definitions file (project_name.ini). + """Helper for interacting with a project. - Returns: - ProjectDefinition: project definition. + Attributes: + project_name (str): name of the project. """ - if self._project_definition is None: - project_reader = project_config.ProjectDefinitionReader() - with open(f'{self.project_name:s}.ini', encoding='utf-8') as file_object: - self._project_definition = project_reader.Read(file_object) - return self._project_definition + _AUTHORS_FILE_HEADER = [ + "# Names should be added to this file with this pattern:", + "#", + "# For individuals:", + "# Name (email address)", + "#", + "# For organizations:", + "# Organization (fnmatch pattern)", + "#", + "# See python fnmatch module documentation for more information.", + "", + "Google Inc. (*@google.com)", + ] + + SUPPORTED_PROJECTS = frozenset( + [ + "acstore", + "artifacts", + "artifacts-kb", + "clitooltester", + "dfdatetime", + "dfkinds", + "dfimagetools", + "dftimewolf", + "dfvfs", + "dfvfs-snippets", + "dfwinreg", + "dtfabric", + "dtformats", + "esedb-kb", + "l2tdevtools", + "l2tdocs", + "l2tscaffolder", + "olecf-kb", + "plaso", + "plist-kb", + "sqlite-kb", + "vstools", + "winevt-kb", + "winreg-kb", + "winshl-kb", + "winsps-kb", + ] + ) + + def __init__(self, project_path): + """Initializes a project helper. + + Args: + project_path (str): path to the project. + + Raises: + ValueError: if the project name is not supported. + """ + super().__init__() + self._project_definition = None + self.project_name = self._GetProjectName(project_path) + + @property + def version_file_path(self): + """str: path of the version file.""" + return os.path.join(self.project_name, "__init__.py") + + def _GetProjectName(self, project_path): + """Retrieves the project name from the script path. + + Args: + project_path (str): path to the root of the project. + + Returns: + str: project name. + + Raises: + ValueError: if the project name is not supported. + """ + project_name = os.path.abspath(project_path) + project_name = os.path.basename(project_name) + + # The review.py check is needed for the l2tdevtools tests. + if project_name != "review.py" and project_name not in self.SUPPORTED_PROJECTS: + raise ValueError(f"Unsupported project name: {project_name:s}.") + + return project_name + + def _ReadFileContents(self, path): + """Reads the contents of a file. + + Args: + path (str): path of the file. + + Returns: + bytes: file content or None if not available. + """ + if not os.path.exists(path): + logging.error(f"Missing file: {path:s}") + return None + + try: + with open(path, "rb") as file_object: + file_contents = file_object.read() + + except IOError as exception: + logging.error(f"Unable to read file with error: {exception!s}") + return None + + try: + file_contents = file_contents.decode("utf-8") + except UnicodeDecodeError as exception: + logging.error(f"Unable to read file with error: {exception!s}") + return None + + return file_contents + + def ReadDefinitionFile(self): + """Reads the project definitions file (project_name.ini). + + Returns: + ProjectDefinition: project definition. + """ + if self._project_definition is None: + project_reader = project_config.ProjectDefinitionReader() + with open(f"{self.project_name:s}.ini", encoding="utf-8") as file_object: + self._project_definition = project_reader.Read(file_object) + + return self._project_definition diff --git a/l2tdevtools/lib/definitions.py b/l2tdevtools/lib/definitions.py index 41e4e73a..db4322c4 100644 --- a/l2tdevtools/lib/definitions.py +++ b/l2tdevtools/lib/definitions.py @@ -1,8 +1,7 @@ """The l2tdevtools definitions.""" - # The default Fedora distribution -DEFAULT_FEDORA_DISTRIBUTION = '43' +DEFAULT_FEDORA_DISTRIBUTION = "43" # The default Ubuntu distribution -DEFAULT_UBUNTU_DISTRIBUTION = 'resolute' +DEFAULT_UBUNTU_DISTRIBUTION = "resolute" diff --git a/l2tdevtools/lib/errors.py b/l2tdevtools/lib/errors.py index fce5451c..3955642f 100644 --- a/l2tdevtools/lib/errors.py +++ b/l2tdevtools/lib/errors.py @@ -2,8 +2,8 @@ class Error(Exception): - """Generic error.""" + """Generic error.""" class ConnectivityError(Error): - """Connectivity error.""" + """Connectivity error.""" diff --git a/l2tdevtools/presets.py b/l2tdevtools/presets.py index 0ee6697d..dcd03293 100644 --- a/l2tdevtools/presets.py +++ b/l2tdevtools/presets.py @@ -4,79 +4,81 @@ class PresetDefinition: - """Class that implements a preset definition. + """Class that implements a preset definition. - Attributes: - name (str): name of the dependency. - preset_names (list[str]): project preset names. - project_names (list[str]): project names. - """ - - def __init__(self, name): - """Initializes the preset definition. - - Args: - name (str): name of the preset. + Attributes: + name (str): name of the dependency. + preset_names (list[str]): project preset names. + project_names (list[str]): project names. """ - super().__init__() - self.name = name - self.preset_names = None - self.project_names = None - - -class PresetDefinitionReader: - """Preset definition reader.""" - def _GetConfigValue(self, config_parser, section_name, value_name): - """Retrieves a value from the config parser. + def __init__(self, name): + """Initializes the preset definition. - Args: - config_parser (ConfigParser): configuration parser. - section_name (str): name of the section that contains the value. - value_name (str): name of the value. + Args: + name (str): name of the preset. + """ + super().__init__() + self.name = name + self.preset_names = None + self.project_names = None - Returns: - object: value or None if the value does not exists. - """ - try: - return config_parser.get(section_name, value_name) - except configparser.NoOptionError: - return None - - def Read(self, file_object): - """Reads preset definitions. - - Args: - file_object (file): file-like object to read from. - Yields: - PresetDefinition: preset definitions. - """ - config_parser = configparser.ConfigParser(interpolation=None) - config_parser.read_file(file_object) - - for section_name in config_parser.sections(): - preset_definition = PresetDefinition(section_name) - - preset_definition.preset_names = self._GetConfigValue( - config_parser, section_name, 'presets') - preset_definition.project_names = self._GetConfigValue( - config_parser, section_name, 'projects') - - if preset_definition.preset_names is None: - preset_definition.preset_names = [] - elif isinstance( - preset_definition.preset_names, str): - preset_definition.preset_names = ( - preset_definition.preset_names.split(',')) - - if preset_definition.project_names is None: - preset_definition.project_names = [] - elif isinstance( - preset_definition.project_names, str): - preset_definition.project_names = ( - preset_definition.project_names.split(',')) - - # Need at minimum a name. - if preset_definition.name: - yield preset_definition +class PresetDefinitionReader: + """Preset definition reader.""" + + def _GetConfigValue(self, config_parser, section_name, value_name): + """Retrieves a value from the config parser. + + Args: + config_parser (ConfigParser): configuration parser. + section_name (str): name of the section that contains the value. + value_name (str): name of the value. + + Returns: + object: value or None if the value does not exists. + """ + try: + return config_parser.get(section_name, value_name) + except configparser.NoOptionError: + return None + + def Read(self, file_object): + """Reads preset definitions. + + Args: + file_object (file): file-like object to read from. + + Yields: + PresetDefinition: preset definitions. + """ + config_parser = configparser.ConfigParser(interpolation=None) + config_parser.read_file(file_object) + + for section_name in config_parser.sections(): + preset_definition = PresetDefinition(section_name) + + preset_definition.preset_names = self._GetConfigValue( + config_parser, section_name, "presets" + ) + preset_definition.project_names = self._GetConfigValue( + config_parser, section_name, "projects" + ) + + if preset_definition.preset_names is None: + preset_definition.preset_names = [] + elif isinstance(preset_definition.preset_names, str): + preset_definition.preset_names = preset_definition.preset_names.split( + "," + ) + + if preset_definition.project_names is None: + preset_definition.project_names = [] + elif isinstance(preset_definition.project_names, str): + preset_definition.project_names = preset_definition.project_names.split( + "," + ) + + # Need at minimum a name. + if preset_definition.name: + yield preset_definition diff --git a/l2tdevtools/project_config.py b/l2tdevtools/project_config.py index 13727f6b..1ccc62b6 100644 --- a/l2tdevtools/project_config.py +++ b/l2tdevtools/project_config.py @@ -4,79 +4,82 @@ class ProjectDefinition: - """Project definition. - - Attributes: - description_long (str): long description. - description_short (str): short description. - git_url (str): URL of the git repository. - homepage_url (str): URL of the homepage. - maintainer (str): maintainer. - name (str): name of the project. - name_description (str): name of the project to use in descriptions. - pypi_token (str): AppVeyor encrypted PyPI token. - status (str): development status of the projects, such as "alpha" or "beta". - """ - - def __init__(self): - """Initializes a project configuration.""" - super().__init__() - self.description_long = None - self.description_short = None - self.git_url = None - self.homepage_url = None - self.maintainer = None - self.name = None - self.name_description = None - self.pypi_token = None - self.status = 'alpha' - - -class ProjectDefinitionReader: - """Project definition reader.""" - - _VALUE_NAMES = frozenset([ - 'description_long', - 'description_short', - 'git_url', - 'homepage_url', - 'maintainer', - 'name', - 'name_description', - 'pypi_token', - 'status']) - - def _GetConfigValue(self, config_parser, section_name, value_name): - """Retrieves a value from the config parser. - - Args: - config_parser (ConfigParser): configuration parser. - section_name (str): name of the section that contains the value. - value_name (str): name of the value. - - Returns: - object: value or None if the value does not exists. + """Project definition. + + Attributes: + description_long (str): long description. + description_short (str): short description. + git_url (str): URL of the git repository. + homepage_url (str): URL of the homepage. + maintainer (str): maintainer. + name (str): name of the project. + name_description (str): name of the project to use in descriptions. + pypi_token (str): AppVeyor encrypted PyPI token. + status (str): development status of the projects, such as "alpha" or "beta". """ - try: - return config_parser.get(section_name, value_name) - except configparser.NoOptionError: - return None - def Read(self, file_object): - """Reads project definitions. + def __init__(self): + """Initializes a project configuration.""" + super().__init__() + self.description_long = None + self.description_short = None + self.git_url = None + self.homepage_url = None + self.maintainer = None + self.name = None + self.name_description = None + self.pypi_token = None + self.status = "alpha" - Args: - file_object (file): file-like object to read from. - Returns: - ProjectDefinition: project definition. - """ - config_parser = configparser.ConfigParser(interpolation=None) - config_parser.read_file(file_object) - - project_definition = ProjectDefinition() - for value_name in self._VALUE_NAMES: - value = self._GetConfigValue(config_parser, 'project', value_name) - setattr(project_definition, value_name, value) - - return project_definition +class ProjectDefinitionReader: + """Project definition reader.""" + + _VALUE_NAMES = frozenset( + [ + "description_long", + "description_short", + "git_url", + "homepage_url", + "maintainer", + "name", + "name_description", + "pypi_token", + "status", + ] + ) + + def _GetConfigValue(self, config_parser, section_name, value_name): + """Retrieves a value from the config parser. + + Args: + config_parser (ConfigParser): configuration parser. + section_name (str): name of the section that contains the value. + value_name (str): name of the value. + + Returns: + object: value or None if the value does not exists. + """ + try: + return config_parser.get(section_name, value_name) + except configparser.NoOptionError: + return None + + def Read(self, file_object): + """Reads project definitions. + + Args: + file_object (file): file-like object to read from. + + Returns: + ProjectDefinition: project definition. + """ + config_parser = configparser.ConfigParser(interpolation=None) + config_parser.read_file(file_object) + + project_definition = ProjectDefinition() + for value_name in self._VALUE_NAMES: + value = self._GetConfigValue(config_parser, "project", value_name) + setattr(project_definition, value_name, value) + + return project_definition diff --git a/l2tdevtools/projects.py b/l2tdevtools/projects.py index c4568c80..7b82dcad 100644 --- a/l2tdevtools/projects.py +++ b/l2tdevtools/projects.py @@ -6,364 +6,418 @@ class ProjectDefinition: - """Project definition. - - Attributes: - architecture_dependent (bool): True if the project is architecture - dependent. - build_dependencies (list[str]): build dependencies. - build_system (str): build system. - configure_options (list[str]): configure options. - description_long (str): long description of the project. - description_short (str): short description of the project. - disabled (list[str]): names of the build targets that are disabled for - this project. - download_url (str): source package download URL. - dpkg_build_dependencies (list[str]): dpkg build dependencies. - dpkg_configure_options (list[str]): configure options when building a deb. - dpkg_dependencies (list[str]): dpkg dependencies. - dpkg_name (str): dpkg package name. - dpkg_source_name (str): dpkg source package name. - dpkg_template_additional (list[str]): names of additional dpkg template - files. - dpkg_template_control (str): name of the dpkg control template file. - dpkg_template_install (list[str]): names of the dpkg install template files. - dpkg_template_install_python3 (list[str]): names of the dpkg Python 3 - install template files. - dpkg_template_py3dist_overrides (str): name of the dpkg py3dist-overrides - template file. - dpkg_template_rules (str): name of the dpkg rules template file. - dpkg_template_source_options (str): name of the dpkg source options template - file. - git_url (str): git repository URL. - github_release_is_archive (bool): github release is the source archive. - github_release_prefix (str): github release prefix. - github_release_tag_prefix (str): github release tag prefix. - homepage_url (str): project homepage URL. - license (str): license. - maintainer (str): name and email address of the maintainer. - name (str): name of the project. - optional_build_dependencies (list[str]): optional build dependencies. - pkg_configure_options (list[str]): configure options when building a pkg. - pypi_name (str): name of the project on PyPI. - pypi_source_name (str): name used in the source package file on PyPI. - rpm_build_dependencies (list[str]): rpm build dependencies. - rpm_dependencies (list[str]): rpm dependencies. - rpm_name (str): RPM package name. - rpm_template_spec (str): name of the rpm spec file. - setup_name (str): project name used by setup.py or pyproject.toml. - srpm_name (str): source RPM package name. - version (ProjectVersionDefinition): version requirements. - wheel_name (str): Python wheel package name. - """ - - def __init__(self, name): - """Initializes a project definition. - - Args: + """Project definition. + + Attributes: + architecture_dependent (bool): True if the project is architecture + dependent. + build_dependencies (list[str]): build dependencies. + build_system (str): build system. + configure_options (list[str]): configure options. + description_long (str): long description of the project. + description_short (str): short description of the project. + disabled (list[str]): names of the build targets that are disabled for + this project. + download_url (str): source package download URL. + dpkg_build_dependencies (list[str]): dpkg build dependencies. + dpkg_configure_options (list[str]): configure options when building a deb. + dpkg_dependencies (list[str]): dpkg dependencies. + dpkg_name (str): dpkg package name. + dpkg_source_name (str): dpkg source package name. + dpkg_template_additional (list[str]): names of additional dpkg template + files. + dpkg_template_control (str): name of the dpkg control template file. + dpkg_template_install (list[str]): names of the dpkg install template files. + dpkg_template_install_python3 (list[str]): names of the dpkg Python 3 + install template files. + dpkg_template_py3dist_overrides (str): name of the dpkg py3dist-overrides + template file. + dpkg_template_rules (str): name of the dpkg rules template file. + dpkg_template_source_options (str): name of the dpkg source options template + file. + git_url (str): git repository URL. + github_release_is_archive (bool): github release is the source archive. + github_release_prefix (str): github release prefix. + github_release_tag_prefix (str): github release tag prefix. + homepage_url (str): project homepage URL. + license (str): license. + maintainer (str): name and email address of the maintainer. name (str): name of the project. + optional_build_dependencies (list[str]): optional build dependencies. + pkg_configure_options (list[str]): configure options when building a pkg. + pypi_name (str): name of the project on PyPI. + pypi_source_name (str): name used in the source package file on PyPI. + rpm_build_dependencies (list[str]): rpm build dependencies. + rpm_dependencies (list[str]): rpm dependencies. + rpm_name (str): RPM package name. + rpm_template_spec (str): name of the rpm spec file. + setup_name (str): project name used by setup.py or pyproject.toml. + srpm_name (str): source RPM package name. + version (ProjectVersionDefinition): version requirements. + wheel_name (str): Python wheel package name. """ - super().__init__() - self.architecture_dependent = False - self.build_dependencies = None - self.build_system = None - self.configure_options = None - self.description_long = None - self.description_short = None - self.disabled = None - self.download_url = None - self.dpkg_build_dependencies = None - self.dpkg_configure_options = None - self.dpkg_dependencies = None - self.dpkg_name = None - self.dpkg_source_name = None - self.dpkg_template_additional = None - self.dpkg_template_control = None - self.dpkg_template_install = None - self.dpkg_template_install_python3 = None - self.dpkg_template_py3dist_overrides = None - self.dpkg_template_rules = None - self.dpkg_template_source_options = None - self.git_url = None - self.github_release_is_archive = False - self.github_release_prefix = None - self.github_release_tag_prefix = None - self.homepage_url = None - self.license = None - self.maintainer = None - self.name = name - self.optional_build_dependencies = None - self.pkg_configure_options = None - self.pypi_name = None - self.pypi_source_name = None - self.rpm_build_dependencies = None - self.rpm_dependencies = None - self.rpm_name = None - self.rpm_template_spec = None - self.setup_name = None - self.srpm_name = None - self.version = None - self.wheel_name = None + + def __init__(self, name): + """Initializes a project definition. + + Args: + name (str): name of the project. + """ + super().__init__() + self.architecture_dependent = False + self.build_dependencies = None + self.build_system = None + self.configure_options = None + self.description_long = None + self.description_short = None + self.disabled = None + self.download_url = None + self.dpkg_build_dependencies = None + self.dpkg_configure_options = None + self.dpkg_dependencies = None + self.dpkg_name = None + self.dpkg_source_name = None + self.dpkg_template_additional = None + self.dpkg_template_control = None + self.dpkg_template_install = None + self.dpkg_template_install_python3 = None + self.dpkg_template_py3dist_overrides = None + self.dpkg_template_rules = None + self.dpkg_template_source_options = None + self.git_url = None + self.github_release_is_archive = False + self.github_release_prefix = None + self.github_release_tag_prefix = None + self.homepage_url = None + self.license = None + self.maintainer = None + self.name = name + self.optional_build_dependencies = None + self.pkg_configure_options = None + self.pypi_name = None + self.pypi_source_name = None + self.rpm_build_dependencies = None + self.rpm_dependencies = None + self.rpm_name = None + self.rpm_template_spec = None + self.setup_name = None + self.srpm_name = None + self.version = None + self.wheel_name = None class ProjectVersionDefinition: - """Project version definition.""" + """Project version definition.""" - _VERSION_STRING_PART_RE = re.compile( - r'^(<[=]?|>[=]?|==)([0-9]+)[.]?([0-9]+|)[.]?([0-9]+|)[.-]?([0-9]+|)$') + _VERSION_STRING_PART_RE = re.compile( + r"^(<[=]?|>[=]?|==)([0-9]+)[.]?([0-9]+|)[.]?([0-9]+|)[.-]?([0-9]+|)$" + ) - def __init__(self, version_string): - """Initializes a project version definition. + def __init__(self, version_string): + """Initializes a project version definition. - Args: - version_string (str): version string. - """ - super().__init__() - self._version_string_parts = [] - - if not version_string: - return - - version_string_parts = version_string.split(',') - number_of_version_string_parts = len(version_string_parts) - if number_of_version_string_parts > 2: - logging.warning(f'Unsupported version string: {version_string:s}') - return - - self._version_string_parts = [] - for index, version_string_part in enumerate(version_string_parts): - if index == 1 and not version_string_part.startswith('<'): - logging.warning( - f'Unsupported version string part: {version_string_part:s}') - return - - matches = self._VERSION_STRING_PART_RE.findall(version_string_part) - if not matches: - logging.warning( - f'Unsupported version string part: {version_string_part:s}') - return - - self._version_string_parts.append([ - match for match in matches[0] if match or match == 0]) - - self._version_string = version_string - - @property - def version_string(self): - """str: string representation of the object.""" - return self._version_string - - def GetEarliestVersion(self): - """Retrieves the earliest version. - - Returns: - str: earliest version or None if version string parts are missing. - """ - if not self._version_string_parts: - return None + Args: + version_string (str): version string. + """ + super().__init__() + self._version_string_parts = [] - return self._version_string_parts[0] + if not version_string: + return - def GetLatestVersion(self): - """Retrieves the latest version. + version_string_parts = version_string.split(",") + number_of_version_string_parts = len(version_string_parts) + if number_of_version_string_parts > 2: + logging.warning(f"Unsupported version string: {version_string:s}") + return - Returns: - str: latest version or None if version string parts are missing. - """ - if not self._version_string_parts or len(self._version_string_parts) == 1: - return None + self._version_string_parts = [] + for index, version_string_part in enumerate(version_string_parts): + if index == 1 and not version_string_part.startswith("<"): + logging.warning( + f"Unsupported version string part: {version_string_part:s}" + ) + return - return self._version_string_parts[1] + matches = self._VERSION_STRING_PART_RE.findall(version_string_part) + if not matches: + logging.warning( + f"Unsupported version string part: {version_string_part:s}" + ) + return + self._version_string_parts.append( + [match for match in matches[0] if match or match == 0] + ) -class ProjectDefinitionReader: - """Project definition reader.""" + self._version_string = version_string - def _GetConfigValue(self, config_parser, section_name, value_name): - """Retrieves a value from the config parser. + @property + def version_string(self): + """str: string representation of the object.""" + return self._version_string - Args: - config_parser (ConfigParser): configuration parser. - section_name (str): name of the section that contains the value. - value_name (str): name of the value. + def GetEarliestVersion(self): + """Retrieves the earliest version. - Returns: - object: value or None if the value does not exists. - """ - try: - return config_parser.get(section_name, value_name) - except configparser.NoOptionError: - return None + Returns: + str: earliest version or None if version string parts are missing. + """ + if not self._version_string_parts: + return None - def Read(self, file_object): - """Reads project definitions. + return self._version_string_parts[0] - Args: - file_object (file): file-like object to read from. + def GetLatestVersion(self): + """Retrieves the latest version. - Yields: - ProjectDefinition: project definition. - """ - config_parser = configparser.ConfigParser(interpolation=None) - config_parser.read_file(file_object) - - for section_name in config_parser.sections(): - project_definition = ProjectDefinition(section_name) - - project_definition.architecture_dependent = self._GetConfigValue( - config_parser, section_name, 'architecture_dependent') - project_definition.build_dependencies = self._GetConfigValue( - config_parser, section_name, 'build_dependencies') - project_definition.optional_build_dependencies = self._GetConfigValue( - config_parser, section_name, 'optional_build_dependencies') - project_definition.build_system = self._GetConfigValue( - config_parser, section_name, 'build_system') - project_definition.configure_options = self._GetConfigValue( - config_parser, section_name, 'configure_options') - project_definition.description_long = self._GetConfigValue( - config_parser, section_name, 'description_long') - project_definition.description_short = self._GetConfigValue( - config_parser, section_name, 'description_short') - project_definition.disabled = self._GetConfigValue( - config_parser, section_name, 'disabled') - project_definition.dpkg_build_dependencies = self._GetConfigValue( - config_parser, section_name, 'dpkg_build_dependencies') - project_definition.dpkg_configure_options = self._GetConfigValue( - config_parser, section_name, 'dpkg_configure_options') - project_definition.dpkg_dependencies = self._GetConfigValue( - config_parser, section_name, 'dpkg_dependencies') - project_definition.dpkg_name = self._GetConfigValue( - config_parser, section_name, 'dpkg_name') - project_definition.dpkg_source_name = self._GetConfigValue( - config_parser, section_name, 'dpkg_source_name') - project_definition.dpkg_template_additional = self._GetConfigValue( - config_parser, section_name, 'dpkg_template_additional') - project_definition.dpkg_template_control = self._GetConfigValue( - config_parser, section_name, 'dpkg_template_control') - project_definition.dpkg_template_install = self._GetConfigValue( - config_parser, section_name, 'dpkg_template_install') - project_definition.dpkg_template_install_python3 = self._GetConfigValue( - config_parser, section_name, 'dpkg_template_install_python3') - project_definition.dpkg_template_py3dist_overrides = self._GetConfigValue( - config_parser, section_name, 'dpkg_template_py3dist_overrides') - project_definition.dpkg_template_rules = self._GetConfigValue( - config_parser, section_name, 'dpkg_template_rules') - project_definition.dpkg_template_source_options = self._GetConfigValue( - config_parser, section_name, 'dpkg_template_source_options') - project_definition.download_url = self._GetConfigValue( - config_parser, section_name, 'download_url') - project_definition.git_url = self._GetConfigValue( - config_parser, section_name, 'git_url') - project_definition.github_release_is_archive = self._GetConfigValue( - config_parser, section_name, 'github_release_is_archive') - project_definition.github_release_prefix = self._GetConfigValue( - config_parser, section_name, 'github_release_prefix') - project_definition.github_release_tag_prefix = self._GetConfigValue( - config_parser, section_name, 'github_release_tag_prefix') - project_definition.homepage_url = self._GetConfigValue( - config_parser, section_name, 'homepage_url') - project_definition.license = self._GetConfigValue( - config_parser, section_name, 'license') - project_definition.maintainer = self._GetConfigValue( - config_parser, section_name, 'maintainer') - project_definition.rpm_build_dependencies = self._GetConfigValue( - config_parser, section_name, 'rpm_build_dependencies') - project_definition.rpm_dependencies = self._GetConfigValue( - config_parser, section_name, 'rpm_dependencies') - project_definition.rpm_name = self._GetConfigValue( - config_parser, section_name, 'rpm_name') - project_definition.rpm_template_spec = self._GetConfigValue( - config_parser, section_name, 'rpm_template_spec') - project_definition.pkg_configure_options = self._GetConfigValue( - config_parser, section_name, 'pkg_configure_options') - project_definition.pypi_name = self._GetConfigValue( - config_parser, section_name, 'pypi_name') - project_definition.pypi_source_name = self._GetConfigValue( - config_parser, section_name, 'pypi_source_name') - project_definition.setup_name = self._GetConfigValue( - config_parser, section_name, 'setup_name') - project_definition.srpm_name = self._GetConfigValue( - config_parser, section_name, 'srpm_name') - project_definition.version = self._GetConfigValue( - config_parser, section_name, 'version') - project_definition.wheel_name = self._GetConfigValue( - config_parser, section_name, 'wheel_name') - - if project_definition.build_dependencies is None: - project_definition.build_dependencies = [] - elif isinstance(project_definition.build_dependencies, str): - project_definition.build_dependencies = ( - project_definition.build_dependencies.split(',')) - - if project_definition.configure_options is None: - project_definition.configure_options = [] - elif isinstance(project_definition.configure_options, str): - project_definition.configure_options = ( - project_definition.configure_options.split(',')) - - if project_definition.disabled is None: - project_definition.disabled = [] - elif isinstance(project_definition.disabled, str): - project_definition.disabled = project_definition.disabled.split( - ',') - - if project_definition.dpkg_build_dependencies is None: - project_definition.dpkg_build_dependencies = [] - elif isinstance(project_definition.dpkg_build_dependencies, str): - project_definition.dpkg_build_dependencies = ( - project_definition.dpkg_build_dependencies.split(',')) - - if project_definition.dpkg_configure_options is None: - project_definition.dpkg_configure_options = [] - elif isinstance(project_definition.dpkg_configure_options, str): - project_definition.dpkg_configure_options = ( - project_definition.dpkg_configure_options.split(',')) - - if project_definition.dpkg_dependencies is None: - project_definition.dpkg_dependencies = [] - elif isinstance(project_definition.dpkg_dependencies, str): - project_definition.dpkg_dependencies = ( - project_definition.dpkg_dependencies.split(',')) - - if project_definition.dpkg_template_additional is None: - project_definition.dpkg_template_additional = [] - elif isinstance(project_definition.dpkg_template_additional, str): - project_definition.dpkg_template_additional = ( - project_definition.dpkg_template_additional.split(',')) - - if project_definition.dpkg_template_install is None: - project_definition.dpkg_template_install = [] - elif isinstance(project_definition.dpkg_template_install, str): - project_definition.dpkg_template_install = ( - project_definition.dpkg_template_install.split(',')) - - if project_definition.dpkg_template_install_python3 is None: - project_definition.dpkg_template_install_python3 = [] - elif isinstance(project_definition.dpkg_template_install_python3, str): - project_definition.dpkg_template_install_python3 = ( - project_definition.dpkg_template_install_python3.split(',')) - - if project_definition.rpm_build_dependencies is None: - project_definition.rpm_build_dependencies = [] - elif isinstance(project_definition.rpm_build_dependencies, str): - project_definition.rpm_build_dependencies = ( - project_definition.rpm_build_dependencies.split(',')) - - if project_definition.rpm_dependencies is None: - project_definition.rpm_dependencies = [] - elif isinstance(project_definition.rpm_dependencies, str): - project_definition.rpm_dependencies = ( - project_definition.rpm_dependencies.split(',')) - - if project_definition.pkg_configure_options is None: - project_definition.pkg_configure_options = [] - elif isinstance(project_definition.pkg_configure_options, str): - project_definition.pkg_configure_options = ( - project_definition.pkg_configure_options.split(',')) - - # Need at minimum a name and a download URL. - if project_definition.name and project_definition.download_url: - yield project_definition - - project_definition.version = ProjectVersionDefinition( - project_definition.version) + Returns: + str: latest version or None if version string parts are missing. + """ + if not self._version_string_parts or len(self._version_string_parts) == 1: + return None + + return self._version_string_parts[1] + + +class ProjectDefinitionReader: + """Project definition reader.""" + + def _GetConfigValue(self, config_parser, section_name, value_name): + """Retrieves a value from the config parser. + + Args: + config_parser (ConfigParser): configuration parser. + section_name (str): name of the section that contains the value. + value_name (str): name of the value. + + Returns: + object: value or None if the value does not exists. + """ + try: + return config_parser.get(section_name, value_name) + except configparser.NoOptionError: + return None + + def Read(self, file_object): + """Reads project definitions. + + Args: + file_object (file): file-like object to read from. + + Yields: + ProjectDefinition: project definition. + """ + config_parser = configparser.ConfigParser(interpolation=None) + config_parser.read_file(file_object) + + for section_name in config_parser.sections(): + project_definition = ProjectDefinition(section_name) + + project_definition.architecture_dependent = self._GetConfigValue( + config_parser, section_name, "architecture_dependent" + ) + project_definition.build_dependencies = self._GetConfigValue( + config_parser, section_name, "build_dependencies" + ) + project_definition.optional_build_dependencies = self._GetConfigValue( + config_parser, section_name, "optional_build_dependencies" + ) + project_definition.build_system = self._GetConfigValue( + config_parser, section_name, "build_system" + ) + project_definition.configure_options = self._GetConfigValue( + config_parser, section_name, "configure_options" + ) + project_definition.description_long = self._GetConfigValue( + config_parser, section_name, "description_long" + ) + project_definition.description_short = self._GetConfigValue( + config_parser, section_name, "description_short" + ) + project_definition.disabled = self._GetConfigValue( + config_parser, section_name, "disabled" + ) + project_definition.dpkg_build_dependencies = self._GetConfigValue( + config_parser, section_name, "dpkg_build_dependencies" + ) + project_definition.dpkg_configure_options = self._GetConfigValue( + config_parser, section_name, "dpkg_configure_options" + ) + project_definition.dpkg_dependencies = self._GetConfigValue( + config_parser, section_name, "dpkg_dependencies" + ) + project_definition.dpkg_name = self._GetConfigValue( + config_parser, section_name, "dpkg_name" + ) + project_definition.dpkg_source_name = self._GetConfigValue( + config_parser, section_name, "dpkg_source_name" + ) + project_definition.dpkg_template_additional = self._GetConfigValue( + config_parser, section_name, "dpkg_template_additional" + ) + project_definition.dpkg_template_control = self._GetConfigValue( + config_parser, section_name, "dpkg_template_control" + ) + project_definition.dpkg_template_install = self._GetConfigValue( + config_parser, section_name, "dpkg_template_install" + ) + project_definition.dpkg_template_install_python3 = self._GetConfigValue( + config_parser, section_name, "dpkg_template_install_python3" + ) + project_definition.dpkg_template_py3dist_overrides = self._GetConfigValue( + config_parser, section_name, "dpkg_template_py3dist_overrides" + ) + project_definition.dpkg_template_rules = self._GetConfigValue( + config_parser, section_name, "dpkg_template_rules" + ) + project_definition.dpkg_template_source_options = self._GetConfigValue( + config_parser, section_name, "dpkg_template_source_options" + ) + project_definition.download_url = self._GetConfigValue( + config_parser, section_name, "download_url" + ) + project_definition.git_url = self._GetConfigValue( + config_parser, section_name, "git_url" + ) + project_definition.github_release_is_archive = self._GetConfigValue( + config_parser, section_name, "github_release_is_archive" + ) + project_definition.github_release_prefix = self._GetConfigValue( + config_parser, section_name, "github_release_prefix" + ) + project_definition.github_release_tag_prefix = self._GetConfigValue( + config_parser, section_name, "github_release_tag_prefix" + ) + project_definition.homepage_url = self._GetConfigValue( + config_parser, section_name, "homepage_url" + ) + project_definition.license = self._GetConfigValue( + config_parser, section_name, "license" + ) + project_definition.maintainer = self._GetConfigValue( + config_parser, section_name, "maintainer" + ) + project_definition.rpm_build_dependencies = self._GetConfigValue( + config_parser, section_name, "rpm_build_dependencies" + ) + project_definition.rpm_dependencies = self._GetConfigValue( + config_parser, section_name, "rpm_dependencies" + ) + project_definition.rpm_name = self._GetConfigValue( + config_parser, section_name, "rpm_name" + ) + project_definition.rpm_template_spec = self._GetConfigValue( + config_parser, section_name, "rpm_template_spec" + ) + project_definition.pkg_configure_options = self._GetConfigValue( + config_parser, section_name, "pkg_configure_options" + ) + project_definition.pypi_name = self._GetConfigValue( + config_parser, section_name, "pypi_name" + ) + project_definition.pypi_source_name = self._GetConfigValue( + config_parser, section_name, "pypi_source_name" + ) + project_definition.setup_name = self._GetConfigValue( + config_parser, section_name, "setup_name" + ) + project_definition.srpm_name = self._GetConfigValue( + config_parser, section_name, "srpm_name" + ) + project_definition.version = self._GetConfigValue( + config_parser, section_name, "version" + ) + project_definition.wheel_name = self._GetConfigValue( + config_parser, section_name, "wheel_name" + ) + + if project_definition.build_dependencies is None: + project_definition.build_dependencies = [] + elif isinstance(project_definition.build_dependencies, str): + project_definition.build_dependencies = ( + project_definition.build_dependencies.split(",") + ) + + if project_definition.configure_options is None: + project_definition.configure_options = [] + elif isinstance(project_definition.configure_options, str): + project_definition.configure_options = ( + project_definition.configure_options.split(",") + ) + + if project_definition.disabled is None: + project_definition.disabled = [] + elif isinstance(project_definition.disabled, str): + project_definition.disabled = project_definition.disabled.split(",") + + if project_definition.dpkg_build_dependencies is None: + project_definition.dpkg_build_dependencies = [] + elif isinstance(project_definition.dpkg_build_dependencies, str): + project_definition.dpkg_build_dependencies = ( + project_definition.dpkg_build_dependencies.split(",") + ) + + if project_definition.dpkg_configure_options is None: + project_definition.dpkg_configure_options = [] + elif isinstance(project_definition.dpkg_configure_options, str): + project_definition.dpkg_configure_options = ( + project_definition.dpkg_configure_options.split(",") + ) + + if project_definition.dpkg_dependencies is None: + project_definition.dpkg_dependencies = [] + elif isinstance(project_definition.dpkg_dependencies, str): + project_definition.dpkg_dependencies = ( + project_definition.dpkg_dependencies.split(",") + ) + + if project_definition.dpkg_template_additional is None: + project_definition.dpkg_template_additional = [] + elif isinstance(project_definition.dpkg_template_additional, str): + project_definition.dpkg_template_additional = ( + project_definition.dpkg_template_additional.split(",") + ) + + if project_definition.dpkg_template_install is None: + project_definition.dpkg_template_install = [] + elif isinstance(project_definition.dpkg_template_install, str): + project_definition.dpkg_template_install = ( + project_definition.dpkg_template_install.split(",") + ) + + if project_definition.dpkg_template_install_python3 is None: + project_definition.dpkg_template_install_python3 = [] + elif isinstance(project_definition.dpkg_template_install_python3, str): + project_definition.dpkg_template_install_python3 = ( + project_definition.dpkg_template_install_python3.split(",") + ) + + if project_definition.rpm_build_dependencies is None: + project_definition.rpm_build_dependencies = [] + elif isinstance(project_definition.rpm_build_dependencies, str): + project_definition.rpm_build_dependencies = ( + project_definition.rpm_build_dependencies.split(",") + ) + + if project_definition.rpm_dependencies is None: + project_definition.rpm_dependencies = [] + elif isinstance(project_definition.rpm_dependencies, str): + project_definition.rpm_dependencies = ( + project_definition.rpm_dependencies.split(",") + ) + + if project_definition.pkg_configure_options is None: + project_definition.pkg_configure_options = [] + elif isinstance(project_definition.pkg_configure_options, str): + project_definition.pkg_configure_options = ( + project_definition.pkg_configure_options.split(",") + ) + + # Need at minimum a name and a download URL. + if project_definition.name and project_definition.download_url: + yield project_definition + + project_definition.version = ProjectVersionDefinition( + project_definition.version + ) diff --git a/l2tdevtools/review_helpers/cli.py b/l2tdevtools/review_helpers/cli.py index d337b0d0..0b80463b 100644 --- a/l2tdevtools/review_helpers/cli.py +++ b/l2tdevtools/review_helpers/cli.py @@ -8,60 +8,63 @@ class CLIHelper: - """Command line interface (CLI) helper. + """Command line interface (CLI) helper. - Attributes: - mock_responses (dict[str, str]): mappings of commands to responses. - preferred_encoding (str): preferred encoding of output. - """ + Attributes: + mock_responses (dict[str, str]): mappings of commands to responses. + preferred_encoding (str): preferred encoding of output. + """ - def __init__(self, mock_responses=None): - """Initializes a CLI helper. + def __init__(self, mock_responses=None): + """Initializes a CLI helper. - Args: - mock_responses (Optional[dict[str, str]]): mappings of commands to - responses, for testing. - """ - super().__init__() - self.mock_responses = mock_responses - self.preferred_encoding = locale.getpreferredencoding() + Args: + mock_responses (Optional[dict[str, str]]): mappings of commands to + responses, for testing. + """ + super().__init__() + self.mock_responses = mock_responses + self.preferred_encoding = locale.getpreferredencoding() - def RunCommand(self, command): - """Runs a command. + def RunCommand(self, command): + """Runs a command. - Args: - command (str): command to run. + Args: + command (str): command to run. - Returns: - tuple[int, str, str]: exit code, output that was written to stdout - and stderr. + Returns: + tuple[int, str, str]: exit code, output that was written to stdout + and stderr. - Raises: - AttributeError: if the command is not recognized. - """ - if self.mock_responses: - return_values = self.mock_responses.get(command, None) - if not return_values: - raise AttributeError('Unrecognized command.') - return return_values + Raises: + AttributeError: if the command is not recognized. + """ + if self.mock_responses: + return_values = self.mock_responses.get(command, None) + if not return_values: + raise AttributeError("Unrecognized command.") + return return_values - arguments = shlex.split(command) + arguments = shlex.split(command) - exit_code = 1 - output = None - error = None + exit_code = 1 + output = None + error = None - try: - with subprocess.Popen( - arguments, stderr=subprocess.PIPE, stdout=subprocess.PIPE) as process: - output, error = process.communicate() - output = codecs.decode(output, self.preferred_encoding) - error = codecs.decode(error, self.preferred_encoding) - exit_code = process.returncode - if exit_code != 0: - logging.error(f'Running: "{command:s}" failed with error: {error!s}.') + try: + with subprocess.Popen( + arguments, stderr=subprocess.PIPE, stdout=subprocess.PIPE + ) as process: + output, error = process.communicate() + output = codecs.decode(output, self.preferred_encoding) + error = codecs.decode(error, self.preferred_encoding) + exit_code = process.returncode + if exit_code != 0: + logging.error( + f'Running: "{command:s}" failed with error: {error!s}.' + ) - except OSError as exception: - logging.error(f'Running: "{command:s}" failed with error: {exception!s}') + except OSError as exception: + logging.error(f'Running: "{command:s}" failed with error: {exception!s}') - return exit_code, output, error + return exit_code, output, error diff --git a/l2tdevtools/review_helpers/git.py b/l2tdevtools/review_helpers/git.py index 667a5d8e..a9d062cc 100644 --- a/l2tdevtools/review_helpers/git.py +++ b/l2tdevtools/review_helpers/git.py @@ -6,310 +6,311 @@ class GitHelper(cli.CLIHelper): - """Git command helper.""" - - def __init__(self, git_repo_url): - """Initializes a git helper. - - Args: - git_repo_url (str): git repo URL. - """ - super().__init__() - self._git_repo_url = git_repo_url - self._remotes = [] - - def _GetRemotes(self): - """Retrieves the git repository remotes. - - Returns: - list[str]: git repository remotes or None. - """ - if not self._remotes: - exit_code, output, _ = self.RunCommand('git remote -v') - if exit_code == 0: - self._remotes = list(filter(None, output.split('\n'))) - - return self._remotes - - def AddPath(self, path): - """Adds a specific path to be managed by git. - - Args: - path (str): path. - - Returns: - bool: True if the path was added. - """ - exit_code, _, _ = self.RunCommand(f'git add -A {path:s}') - return exit_code == 0 - - def CheckHasBranch(self, branch): - """Checks if the git repo has a specific branch. - - Args: - branch (str): name of the feature branch. - - Returns: - bool: True if git repo has the specific branch. - """ - exit_code, output, _ = self.RunCommand('git branch') - if exit_code != 0: - return False - - # Check for remote entries starting with upstream. - for line in output.split('\n'): - # Ignore the first 2 characters of the line. - if line[2:] == branch: - return True - return False - - def CheckHasProjectOrigin(self): - """Checks if the git repo has the project remote origin defined. - - Returns: - bool: True if the git repo has the project origin defined. - """ - origin_git_repo_url = self.GetRemoteOrigin() - - is_match = origin_git_repo_url == self._git_repo_url - if not is_match: - is_match = origin_git_repo_url == self._git_repo_url[:-4] - - return is_match - - def CheckHasProjectUpstream(self): - """Checks if the git repo has the project remote upstream defined. - - Returns: - bool: True if the git repo has the project remote upstream defined. - """ - # Check for remote entries starting with upstream. - for remote in self._GetRemotes(): - if remote.startswith(f'upstream\t{self._git_repo_url:s}'): - return True - return False - - def CheckHasUncommittedChanges(self): - """Checks if the git repo has uncommitted changes. - - Returns: - bool: True if the git repo has uncommitted changes. - """ - exit_code, output, _ = self.RunCommand('git status -s') - if exit_code != 0: - return False - - # Check if 'git status -s' yielded any output. - for line in output.split('\n'): - if line: - return True - return False - - def CheckSynchronizedWithUpstream(self): - """Checks if the git repo is synchronized with upstream. - - Returns: - bool: True if the git repo is synchronized with upstream. - """ - # Fetch the entire upstream repo information not only that of - # the main branch. Otherwise the information about the current - # upstream HEAD is not updated. - exit_code, _, _ = self.RunCommand('git fetch upstream') - if exit_code != 0: - return False - - # The result of "git log HEAD..upstream/main --oneline" should be empty - # if the git repo is synchronized with upstream. - exit_code, output, _ = self.RunCommand( - 'git log HEAD..upstream/main --oneline') - return exit_code == 0 and not output - - def DropUncommittedChanges(self): - """Drops the uncommitted changes.""" - self.RunCommand('git stash') - self.RunCommand('git stash drop') - - def GetActiveBranch(self): - """Retrieves the active branch. - - Returns: - str: name of the active branch or None if not available. - """ - exit_code, output, _ = self.RunCommand('git branch') - if exit_code != 0: - return False - - # Check for remote entries starting with upstream. - for line in output.split('\n'): - if line.startswith('* '): - # Ignore the first 2 characters of the line. - return line[2:] - return None - - def GetChangedFiles(self, diffbase=None): - """Retrieves the changed files. - - Args: - diffbase (Optional[str]): git diffbase, for example "upstream/main". - - Returns: - list[str]: names of the changed files. - """ - if diffbase: - command = f'git diff --name-only {diffbase:s}' - else: - command = 'git ls-files' - - exit_code, output, _ = self.RunCommand(command) - if exit_code != 0: - return [] - - return output.split('\n') - - def GetChangedPythonFiles(self, diffbase=None): - """Retrieves the changed Python files. - - Note that several Python files are excluded: - * Python files generated by the protobuf compiler (*_pb2.py) - * Python files used as test data (test_data/*.py) - * l2tdevtools/lib/upload.py - - Args: - diffbase (Optional[str]): git diffbase, for example "upstream/main". - - Returns: - list[str]: names of the changed Python files. - """ - python_files = [] - for changed_file in self.GetChangedFiles(diffbase=diffbase): - if (not changed_file.endswith('.py') or - changed_file.endswith('_pb2.py') or - not os.path.exists(changed_file) or - changed_file.startswith('data') or - changed_file.startswith('docs') or - changed_file.startswith('test_data') or - changed_file == os.path.join('l2tdevtools', 'lib', 'upload.py')): - continue - - python_files.append(changed_file) - - return python_files - - def GetLastCommitMessage(self): - """Retrieves the last commit message. - - Returns: - str: last commit message or None if not available. - """ - exit_code, output, _ = self.RunCommand('git log -1') - if exit_code != 0: - return None - - # Expecting 6 lines of output where the 5th line contains - # the commit message. - output_lines = output.split('\n') - if len(output_lines) != 6: - return None - - return output_lines[4].strip() - - def GetRemoteOrigin(self): - """Retrieves the remote origin. - - Returns: - str: git repository URL or None if not available. - """ - # Check for remote entries starting with origin. - for remote in self._GetRemotes(): - if remote.startswith('origin\t'): - values = remote.split() - if len(values) == 3: - return values[1] - - return None - - def PullFromFork(self, git_repo_url, branch): - """Pulls changes from a feature branch on a fork. - - Args: - git_repo_url (str): git repository URL of the fork. - branch (str): name of the feature branch of the fork. - - Returns: - bool: True if the pull was successful. - """ - exit_code, _, _ = self.RunCommand( - f'git pull --squash {git_repo_url:s} {branch:s}') - return exit_code == 0 - - def PushToOrigin(self, branch, force=False): - """Forces a push of the active branch of the git repo to origin. - - Args: - branch (str): name of the feature branch. - force (Optional[bool]): True if the push should be forced. - - Returns: - bool: True if the push was successful. - """ - if force: - command = f'git push -f --set-upstream origin {branch:s}' - else: - command = f'git push --set-upstream origin {branch:s}' - - exit_code, _, _ = self.RunCommand(command) - return exit_code == 0 - - def RemoveFeatureBranch(self, branch): - """Removes the git feature branch both local and from origin. - - Args: - branch (str): name of the feature branch. - """ - if branch == 'main': - return - - self.RunCommand(f'git push origin --delete {branch:s}') - self.RunCommand(f'git branch -D {branch:s}') - - def SynchronizeWithOrigin(self): - """Synchronizes git with origin. - - Returns: - bool: True if the git repository has synchronized with origin. - """ - exit_code, _, _ = self.RunCommand('git fetch origin') - if exit_code != 0: - return False - - exit_code, _, _ = self.RunCommand('git pull --no-edit origin main') - - return exit_code == 0 - - def SynchronizeWithUpstream(self): - """Synchronizes git with upstream. - - Returns: - bool: True if the git repository has synchronized with upstream. - """ - exit_code, _, _ = self.RunCommand('git fetch upstream') - if exit_code != 0: - return False - - exit_code, _, _ = self.RunCommand( - 'git pull --no-edit --rebase upstream main') - if exit_code != 0: - return False - - exit_code, _, _ = self.RunCommand('git push') - - return exit_code == 0 - - def SwitchToMainBranch(self): - """Switches git to the main branch. - - Returns: - bool: True if the git repository has switched to the main branch. - """ - exit_code, _, _ = self.RunCommand('git checkout main') - return exit_code == 0 + """Git command helper.""" + + def __init__(self, git_repo_url): + """Initializes a git helper. + + Args: + git_repo_url (str): git repo URL. + """ + super().__init__() + self._git_repo_url = git_repo_url + self._remotes = [] + + def _GetRemotes(self): + """Retrieves the git repository remotes. + + Returns: + list[str]: git repository remotes or None. + """ + if not self._remotes: + exit_code, output, _ = self.RunCommand("git remote -v") + if exit_code == 0: + self._remotes = list(filter(None, output.split("\n"))) + + return self._remotes + + def AddPath(self, path): + """Adds a specific path to be managed by git. + + Args: + path (str): path. + + Returns: + bool: True if the path was added. + """ + exit_code, _, _ = self.RunCommand(f"git add -A {path:s}") + return exit_code == 0 + + def CheckHasBranch(self, branch): + """Checks if the git repo has a specific branch. + + Args: + branch (str): name of the feature branch. + + Returns: + bool: True if git repo has the specific branch. + """ + exit_code, output, _ = self.RunCommand("git branch") + if exit_code != 0: + return False + + # Check for remote entries starting with upstream. + for line in output.split("\n"): + # Ignore the first 2 characters of the line. + if line[2:] == branch: + return True + return False + + def CheckHasProjectOrigin(self): + """Checks if the git repo has the project remote origin defined. + + Returns: + bool: True if the git repo has the project origin defined. + """ + origin_git_repo_url = self.GetRemoteOrigin() + + is_match = origin_git_repo_url == self._git_repo_url + if not is_match: + is_match = origin_git_repo_url == self._git_repo_url[:-4] + + return is_match + + def CheckHasProjectUpstream(self): + """Checks if the git repo has the project remote upstream defined. + + Returns: + bool: True if the git repo has the project remote upstream defined. + """ + # Check for remote entries starting with upstream. + for remote in self._GetRemotes(): + if remote.startswith(f"upstream\t{self._git_repo_url:s}"): + return True + return False + + def CheckHasUncommittedChanges(self): + """Checks if the git repo has uncommitted changes. + + Returns: + bool: True if the git repo has uncommitted changes. + """ + exit_code, output, _ = self.RunCommand("git status -s") + if exit_code != 0: + return False + + # Check if 'git status -s' yielded any output. + for line in output.split("\n"): + if line: + return True + return False + + def CheckSynchronizedWithUpstream(self): + """Checks if the git repo is synchronized with upstream. + + Returns: + bool: True if the git repo is synchronized with upstream. + """ + # Fetch the entire upstream repo information not only that of + # the main branch. Otherwise the information about the current + # upstream HEAD is not updated. + exit_code, _, _ = self.RunCommand("git fetch upstream") + if exit_code != 0: + return False + + # The result of "git log HEAD..upstream/main --oneline" should be empty + # if the git repo is synchronized with upstream. + exit_code, output, _ = self.RunCommand("git log HEAD..upstream/main --oneline") + return exit_code == 0 and not output + + def DropUncommittedChanges(self): + """Drops the uncommitted changes.""" + self.RunCommand("git stash") + self.RunCommand("git stash drop") + + def GetActiveBranch(self): + """Retrieves the active branch. + + Returns: + str: name of the active branch or None if not available. + """ + exit_code, output, _ = self.RunCommand("git branch") + if exit_code != 0: + return False + + # Check for remote entries starting with upstream. + for line in output.split("\n"): + if line.startswith("* "): + # Ignore the first 2 characters of the line. + return line[2:] + return None + + def GetChangedFiles(self, diffbase=None): + """Retrieves the changed files. + + Args: + diffbase (Optional[str]): git diffbase, for example "upstream/main". + + Returns: + list[str]: names of the changed files. + """ + if diffbase: + command = f"git diff --name-only {diffbase:s}" + else: + command = "git ls-files" + + exit_code, output, _ = self.RunCommand(command) + if exit_code != 0: + return [] + + return output.split("\n") + + def GetChangedPythonFiles(self, diffbase=None): + """Retrieves the changed Python files. + + Note that several Python files are excluded: + * Python files generated by the protobuf compiler (*_pb2.py) + * Python files used as test data (test_data/*.py) + * l2tdevtools/lib/upload.py + + Args: + diffbase (Optional[str]): git diffbase, for example "upstream/main". + + Returns: + list[str]: names of the changed Python files. + """ + python_files = [] + for changed_file in self.GetChangedFiles(diffbase=diffbase): + if ( + not changed_file.endswith(".py") + or changed_file.endswith("_pb2.py") + or not os.path.exists(changed_file) + or changed_file.startswith("data") + or changed_file.startswith("docs") + or changed_file.startswith("test_data") + or changed_file == os.path.join("l2tdevtools", "lib", "upload.py") + ): + continue + + python_files.append(changed_file) + + return python_files + + def GetLastCommitMessage(self): + """Retrieves the last commit message. + + Returns: + str: last commit message or None if not available. + """ + exit_code, output, _ = self.RunCommand("git log -1") + if exit_code != 0: + return None + + # Expecting 6 lines of output where the 5th line contains + # the commit message. + output_lines = output.split("\n") + if len(output_lines) != 6: + return None + + return output_lines[4].strip() + + def GetRemoteOrigin(self): + """Retrieves the remote origin. + + Returns: + str: git repository URL or None if not available. + """ + # Check for remote entries starting with origin. + for remote in self._GetRemotes(): + if remote.startswith("origin\t"): + values = remote.split() + if len(values) == 3: + return values[1] + + return None + + def PullFromFork(self, git_repo_url, branch): + """Pulls changes from a feature branch on a fork. + + Args: + git_repo_url (str): git repository URL of the fork. + branch (str): name of the feature branch of the fork. + + Returns: + bool: True if the pull was successful. + """ + exit_code, _, _ = self.RunCommand( + f"git pull --squash {git_repo_url:s} {branch:s}" + ) + return exit_code == 0 + + def PushToOrigin(self, branch, force=False): + """Forces a push of the active branch of the git repo to origin. + + Args: + branch (str): name of the feature branch. + force (Optional[bool]): True if the push should be forced. + + Returns: + bool: True if the push was successful. + """ + if force: + command = f"git push -f --set-upstream origin {branch:s}" + else: + command = f"git push --set-upstream origin {branch:s}" + + exit_code, _, _ = self.RunCommand(command) + return exit_code == 0 + + def RemoveFeatureBranch(self, branch): + """Removes the git feature branch both local and from origin. + + Args: + branch (str): name of the feature branch. + """ + if branch == "main": + return + + self.RunCommand(f"git push origin --delete {branch:s}") + self.RunCommand(f"git branch -D {branch:s}") + + def SynchronizeWithOrigin(self): + """Synchronizes git with origin. + + Returns: + bool: True if the git repository has synchronized with origin. + """ + exit_code, _, _ = self.RunCommand("git fetch origin") + if exit_code != 0: + return False + + exit_code, _, _ = self.RunCommand("git pull --no-edit origin main") + + return exit_code == 0 + + def SynchronizeWithUpstream(self): + """Synchronizes git with upstream. + + Returns: + bool: True if the git repository has synchronized with upstream. + """ + exit_code, _, _ = self.RunCommand("git fetch upstream") + if exit_code != 0: + return False + + exit_code, _, _ = self.RunCommand("git pull --no-edit --rebase upstream main") + if exit_code != 0: + return False + + exit_code, _, _ = self.RunCommand("git push") + + return exit_code == 0 + + def SwitchToMainBranch(self): + """Switches git to the main branch. + + Returns: + bool: True if the git repository has switched to the main branch. + """ + exit_code, _, _ = self.RunCommand("git checkout main") + return exit_code == 0 diff --git a/l2tdevtools/review_helpers/github.py b/l2tdevtools/review_helpers/github.py index 8002016e..ea9096b5 100644 --- a/l2tdevtools/review_helpers/github.py +++ b/l2tdevtools/review_helpers/github.py @@ -8,187 +8,188 @@ class GitHubHelper: - """Github helper.""" - - def __init__(self, organization, project): - """Initializes a GitHub helper. - - Args: - organization (str): GitHub organization name. - project (str): GitHub project name. - """ - super().__init__() - - self._organization = organization - self._project = project - self._url_lib_helper = url_lib.URLLibHelper() - - def AssignPullRequest( - self, pull_request_number, access_token, assignees): - """Adds assignees to a GitHub pull request. - - Assignees are responsible that a pull request is closed or merged. - - Args: - pull_request_number (int): GitHub issue number of the pull request. - access_token (str): GitHub access token. - assignees (list[str]): GitHub usernames to assign. + """Github helper.""" + + def __init__(self, organization, project): + """Initializes a GitHub helper. + + Args: + organization (str): GitHub organization name. + project (str): GitHub project name. + """ + super().__init__() + + self._organization = organization + self._project = project + self._url_lib_helper = url_lib.URLLibHelper() + + def AssignPullRequest(self, pull_request_number, access_token, assignees): + """Adds assignees to a GitHub pull request. + + Assignees are responsible that a pull request is closed or merged. + + Args: + pull_request_number (int): GitHub issue number of the pull request. + access_token (str): GitHub access token. + assignees (list[str]): GitHub usernames to assign. + + Returns: + bool: True if the assignees were successfully added. + """ + post_data = json.dumps({"assignees": assignees}) - Returns: - bool: True if the assignees were successfully added. - """ - post_data = json.dumps({"assignees": assignees}) + github_url = ( + f"https://api.github.com/repos/" + f"{self._organization:s}/{self._project:s}/issues/" + f"{pull_request_number:d}/assignees?" + f"access_token={access_token:s}" + ) + + try: + self._url_lib_helper.Request(github_url, post_data=post_data) + + except errors.ConnectivityError: + return False - github_url = ( - f'https://api.github.com/repos/' - f'{self._organization:s}/{self._project:s}/issues/' - f'{pull_request_number:d}/assignees?' - f'access_token={access_token:s}') + return True + + def CreatePullRequest(self, access_token, origin, title, body, no_edit=False): + """Creates a pull request. + + Args: + access_token (str): GitHub access token. + origin (str): origin of the pull request, formatted as: + "username:feature". + title (str): title of the pull request. + body (str): body of the pull request. + no_edit (Optional[bool]): True if maintainers should not be allowed to + edit the pull request. - try: - self._url_lib_helper.Request(github_url, post_data=post_data) + Returns: + int: GitHub issue number of the pull request. - except errors.ConnectivityError: - return False + Raises: + ConnectivityError: if there's an error communicating with GitHub. + """ + if no_edit: + maintainer_can_modify = "false" + else: + maintainer_can_modify = "true" - return True + # Note that the maintainer_can_modify is a JSON boolean value. + post_data = ( + f"{{\n" + f' "title": "{title:s}",\n' + f' "body": "{body:s}",\n' + f' "head": "{origin:s}",\n' + f' "base": "main",\n' + f' "maintainer_can_modify": {maintainer_can_modify:s}\n' + f"}}\n" + ) - def CreatePullRequest(self, access_token, origin, title, body, no_edit=False): - """Creates a pull request. - - Args: - access_token (str): GitHub access token. - origin (str): origin of the pull request, formatted as: - "username:feature". - title (str): title of the pull request. - body (str): body of the pull request. - no_edit (Optional[bool]): True if maintainers should not be allowed to - edit the pull request. + github_url = ( + f"https://api.github.com/repos/" + f"{self._organization:s}/{self._project:s}/pulls?" + f"access_token={access_token:s}" + ) - Returns: - int: GitHub issue number of the pull request. + response_data = self._url_lib_helper.Request(github_url, post_data=post_data) - Raises: - ConnectivityError: if there's an error communicating with GitHub. - """ - if no_edit: - maintainer_can_modify = 'false' - else: - maintainer_can_modify = 'true' + if isinstance(response_data, bytes): + response_data = response_data.decode("utf-8") - # Note that the maintainer_can_modify is a JSON boolean value. - post_data = ( - f'{{\n' - f' "title": "{title:s}",\n' - f' "body": "{body:s}",\n' - f' "head": "{origin:s}",\n' - f' "base": "main",\n' - f' "maintainer_can_modify": {maintainer_can_modify:s}\n' - f'}}\n') + response_data = json.loads(response_data) - github_url = ( - f'https://api.github.com/repos/' - f'{self._organization:s}/{self._project:s}/pulls?' - f'access_token={access_token:s}') + pull_request_number = response_data.get("number") - response_data = self._url_lib_helper.Request( - github_url, post_data=post_data) + return pull_request_number - if isinstance(response_data, bytes): - response_data = response_data.decode('utf-8') + def CreatePullRequestReview(self, pull_request_number, access_token, reviewers): + """Requests a GitHub review of a pull request. - response_data = json.loads(response_data) + Args: + pull_request_number (int): GitHub issue number of the pull request. + access_token (str): GitHub access token. + reviewers (list[str]): GitHub usernames to assign as reviewers. - pull_request_number = response_data.get('number') + Returns: + bool: True if the review was created. + """ + post_data = json.dumps({"reviewers": reviewers}) - return pull_request_number + github_url = ( + f"https://api.github.com/repos/" + f"{self._organization:s}/{self._project:s}/pulls/" + f"{pull_request_number:d}/requested_reviewers?" + f"access_token={access_token:s}" + ) - def CreatePullRequestReview( - self, pull_request_number, access_token, reviewers): - """Requests a GitHub review of a pull request. + try: + self._url_lib_helper.Request(github_url, post_data=post_data) - Args: - pull_request_number (int): GitHub issue number of the pull request. - access_token (str): GitHub access token. - reviewers (list[str]): GitHub usernames to assign as reviewers. + except errors.ConnectivityError: + return False - Returns: - bool: True if the review was created. - """ - post_data = json.dumps({"reviewers": reviewers}) + return True - github_url = ( - f'https://api.github.com/repos/' - f'{self._organization:s}/{self._project:s}/pulls/' - f'{pull_request_number:d}/requested_reviewers?' - f'access_token={access_token:s}') + def GetForkGitRepoUrl(self, username): + """Retrieves the git repository URL of a fork. - try: - self._url_lib_helper.Request(github_url, post_data=post_data) + Args: + username (str): GitHub username of the fork. - except errors.ConnectivityError: - return False + Returns: + str: git repository URL or None. + """ + return f"https://github.com/{username:s}/{self._project:s}.git" - return True + def GetUsername(self, access_token): + """Retrieves a GitHub user. - def GetForkGitRepoUrl(self, username): - """Retrieves the git repository URL of a fork. + Args: + access_token (str): GitHub access token. - Args: - username (str): GitHub username of the fork. + Returns: + str: GitHub user name or None if not available. + """ + github_url = f"https://api.github.com/user?access_token={access_token:s}" - Returns: - str: git repository URL or None. - """ - return f'https://github.com/{username:s}/{self._project:s}.git' + try: + response_data = self._url_lib_helper.Request(github_url) + except errors.ConnectivityError: + return None - def GetUsername(self, access_token): - """Retrieves a GitHub user. + if not response_data: + return None - Args: - access_token (str): GitHub access token. + if isinstance(response_data, bytes): + response_data = response_data.decode("utf-8") - Returns: - str: GitHub user name or None if not available. - """ - github_url = f'https://api.github.com/user?access_token={access_token:s}' + response_data = json.loads(response_data) + return response_data.get("login", None) - try: - response_data = self._url_lib_helper.Request(github_url) - except errors.ConnectivityError: - return None + def QueryUser(self, username): + """Queries a GitHub user. - if not response_data: - return None + Args: + username (str): GitHub user name. - if isinstance(response_data, bytes): - response_data = response_data.decode('utf-8') + Returns: + dict[str,object]: JSON response or None if not available. + """ + github_url = f"https://api.github.com/users/{username:s}" - response_data = json.loads(response_data) - return response_data.get('login', None) + try: + response_data = self._url_lib_helper.Request(github_url) - def QueryUser(self, username): - """Queries a GitHub user. + except errors.ConnectivityError as exception: + logging.warning(f"{exception!s}") + return None - Args: - username (str): GitHub user name. + if isinstance(response_data, bytes): + response_data = response_data.decode("utf-8") - Returns: - dict[str,object]: JSON response or None if not available. - """ - github_url = f'https://api.github.com/users/{username:s}' + if response_data: + return json.loads(response_data) - try: - response_data = self._url_lib_helper.Request(github_url) - - except errors.ConnectivityError as exception: - logging.warning(f'{exception!s}') - return None - - if isinstance(response_data, bytes): - response_data = response_data.decode('utf-8') - - if response_data: - return json.loads(response_data) - - return None + return None diff --git a/l2tdevtools/review_helpers/pylint.py b/l2tdevtools/review_helpers/pylint.py index 20146a6c..a0b24a7b 100644 --- a/l2tdevtools/review_helpers/pylint.py +++ b/l2tdevtools/review_helpers/pylint.py @@ -7,97 +7,101 @@ class PylintHelper(cli.CLIHelper): - """Pylint helper.""" - - MINIMUM_VERSION = '1.7.0' - - # pylint: disable=consider-using-generator - _MINIMUM_VERSION_TUPLE = tuple( - [int(digit, 10) for digit in MINIMUM_VERSION.split('.')]) - - _RCFILE_NAME = '.pylintrc' - - def _GetVersion(self): - """Retrieves the pylint version. - - Returns: - tuple[int]: pylint version as a tuple of integers or (0, 0, 0) if - not available. - """ - version_tuple = (0, 0, 0) - - exit_code, output, _ = self.RunCommand('pylint --version') - if exit_code == 0: - for line in output.split('\n'): - if line.startswith('pylint '): - _, _, version = line.partition(' ') - # Remove a trailing comma. - version, _, _ = version.partition(',') - - # pylint: disable=consider-using-generator - version_tuple = tuple([ - int(digit, 10) for digit in version.split('.')]) - - return version_tuple - - def CheckFiles(self, filenames, rcfile): - """Checks if the linting of the files is correct using pylint. - - Args: - filenames (list[str]): names of the files to lint. - rcfile (str): path to the pylint configuration file to use. - - Returns: - bool: True if the files were linted without errors. - """ - version_tuple = self._GetVersion() - - print('Running linter on changed files.') - failed_filenames = [] - for filename in filenames: - print(f'Checking: {filename:s}') - - command = f'pylint --rcfile="{rcfile:s}" {filename:s}' - # For now disable pylint 2.1.1 and later specific checks. - if version_tuple >= (2, 1, 1): - additional_checks = [ - 'assignment-from-none', 'chained-comparison', - 'useless-object-inheritance'] - disable_checks = ','.join(additional_checks) - command = f'{command:s} --disable={disable_checks:s}' - - exit_code = subprocess.call(command, shell=True) - if exit_code != 0: - failed_filenames.append(filename) - - if failed_filenames: - filenames_string = '\n'.join(failed_filenames) - print(f'\nFiles with linter errors:\n{filenames_string:s}\n') - return False - - return True - - def CheckUpToDateVersion(self): - """Checks if the pylint version is up to date. - - Returns: - bool: True if the pylint version is up to date. - """ - version_tuple = self._GetVersion() - return version_tuple >= self._MINIMUM_VERSION_TUPLE - - def GetRCFile(self, project_path): - """Gets the path to the pylint configuration file for a project. - - Args: - project_path (str): path to the root of the project. - - Returns: - str: absolute path to the pylint configuration file for the project. - """ - project_file_path = os.path.join(project_path, self._RCFILE_NAME) - if os.path.exists(project_file_path): - return os.path.abspath(project_file_path) - - default_path = os.path.join(__file__, '..', '..', '..', self._RCFILE_NAME) - return os.path.abspath(default_path) + """Pylint helper.""" + + MINIMUM_VERSION = "1.7.0" + + # pylint: disable=consider-using-generator + _MINIMUM_VERSION_TUPLE = tuple( + [int(digit, 10) for digit in MINIMUM_VERSION.split(".")] + ) + + _RCFILE_NAME = ".pylintrc" + + def _GetVersion(self): + """Retrieves the pylint version. + + Returns: + tuple[int]: pylint version as a tuple of integers or (0, 0, 0) if + not available. + """ + version_tuple = (0, 0, 0) + + exit_code, output, _ = self.RunCommand("pylint --version") + if exit_code == 0: + for line in output.split("\n"): + if line.startswith("pylint "): + _, _, version = line.partition(" ") + # Remove a trailing comma. + version, _, _ = version.partition(",") + + # pylint: disable=consider-using-generator + version_tuple = tuple( + [int(digit, 10) for digit in version.split(".")] + ) + + return version_tuple + + def CheckFiles(self, filenames, rcfile): + """Checks if the linting of the files is correct using pylint. + + Args: + filenames (list[str]): names of the files to lint. + rcfile (str): path to the pylint configuration file to use. + + Returns: + bool: True if the files were linted without errors. + """ + version_tuple = self._GetVersion() + + print("Running linter on changed files.") + failed_filenames = [] + for filename in filenames: + print(f"Checking: {filename:s}") + + command = f'pylint --rcfile="{rcfile:s}" {filename:s}' + # For now disable pylint 2.1.1 and later specific checks. + if version_tuple >= (2, 1, 1): + additional_checks = [ + "assignment-from-none", + "chained-comparison", + "useless-object-inheritance", + ] + disable_checks = ",".join(additional_checks) + command = f"{command:s} --disable={disable_checks:s}" + + exit_code = subprocess.call(command, shell=True) + if exit_code != 0: + failed_filenames.append(filename) + + if failed_filenames: + filenames_string = "\n".join(failed_filenames) + print(f"\nFiles with linter errors:\n{filenames_string:s}\n") + return False + + return True + + def CheckUpToDateVersion(self): + """Checks if the pylint version is up to date. + + Returns: + bool: True if the pylint version is up to date. + """ + version_tuple = self._GetVersion() + return version_tuple >= self._MINIMUM_VERSION_TUPLE + + def GetRCFile(self, project_path): + """Gets the path to the pylint configuration file for a project. + + Args: + project_path (str): path to the root of the project. + + Returns: + str: absolute path to the pylint configuration file for the project. + """ + project_file_path = os.path.join(project_path, self._RCFILE_NAME) + if os.path.exists(project_file_path): + return os.path.abspath(project_file_path) + + default_path = os.path.join(__file__, "..", "..", "..", self._RCFILE_NAME) + return os.path.abspath(default_path) diff --git a/l2tdevtools/review_helpers/review.py b/l2tdevtools/review_helpers/review.py index af87a54e..4f6258de 100644 --- a/l2tdevtools/review_helpers/review.py +++ b/l2tdevtools/review_helpers/review.py @@ -1,6 +1,5 @@ """Helper for conducting code reviews.""" - import os import re import subprocess @@ -13,215 +12,236 @@ class ReviewHelper: - """Helper for conducting code reviews.""" - - _SUPPORTED_PROJECTS_PATTERN = '|'.join( - project.ProjectHelper.SUPPORTED_PROJECTS) - _PROJECT_NAME_PREFIX_REGEX = re.compile( - rf'\[({_SUPPORTED_PROJECTS_PATTERN})\] ') - - _CODE_INSPECTION_COMMANDS = frozenset([ - 'create-pr', 'create_pr', 'lint', 'lint-test', 'lint_test']) - - def __init__( - self, command, project_path, github_origin, feature_branch, - all_files=False): - """Initializes a review helper. - - Args: - command (str): user provided command, for example "create", "lint". - project_path (str): path to the project being reviewed. - github_origin (str): GitHub origin. - feature_branch (str): feature branch. - all_files (Optional[bool]): True if the command should apply to all - files. Currently this only affects the lint command. - """ - super().__init__() - self._active_branch = None - self._all_files = all_files - self._command = command - self._feature_branch = feature_branch - self._git_helper = None - self._git_repo_url = None - self._github_helper = None - self._github_organization = None - self._github_origin = github_origin - self._fork_feature_branch = None - self._fork_username = None - self._maintainer = None - self._project_helper = None - self._project_name = None - self._project_path = project_path - - if self._github_origin: - self._fork_username, _, self._fork_feature_branch = ( - self._github_origin.partition(':')) - - def CheckLocalGitState(self): - """Checks the state of the local git repository. - - Returns: - bool: True if the state of the local git repository is sane. - """ - if self._github_organization in ('ForensicArtifacts', 'log2timeline'): - if self._command in ( - 'close', 'create-pr', 'create_pr', 'lint', 'lint-test', 'lint_test'): - if not self._git_helper.CheckHasProjectUpstream(): - command_title = self._command.title() - print(f'{command_title:s} aborted - missing project upstream.') - print(f'Run: git remote add upstream {self._git_repo_url:s}') - return False - - if self._command not in ( - 'lint', 'lint-test', 'lint_test', 'test', 'update-version', - 'update_version'): - if self._git_helper.CheckHasUncommittedChanges(): - command_title = self._command.title() - print(f'{command_title:s} aborted - detected uncommitted changes.') - print('Run: git commit') - return False - - if self._github_organization in ('ForensicArtifacts', 'log2timeline'): - self._active_branch = self._git_helper.GetActiveBranch() - if self._command in ('create-pr', 'create_pr'): - if self._active_branch == 'main': - command_title = self._command.title() - print(f'{command_title:s} aborted - active branch is main.') - return False - - elif self._command == 'close': - if self._feature_branch == 'main': - command_title = self._command.title() - print(f'{command_title:s} aborted - feature branch cannot be main.') - return False - - if self._active_branch != 'main': - self._git_helper.SwitchToMainBranch() - self._active_branch = 'main' - - return True - - def Close(self): - """Closes a review. - - Returns: - bool: True if the close was successful. - """ - if not self._git_helper.CheckHasBranch(self._feature_branch): - print(f'No such feature branch: {self._feature_branch:s}') - else: - self._git_helper.RemoveFeatureBranch(self._feature_branch) - - if not self._git_helper.SynchronizeWithUpstream(): - command_title = self._command.title() - print( - f'{command_title:s} aborted - unable to synchronize with ' - f'upstream/main.') - - return True - - def InitializeHelpers(self): - """Initializes the helpers. - - Returns: - bool: True if the helper initialization was successful. - """ - project_path = os.path.abspath(self._project_path) - - self._project_helper = project.ProjectHelper(project_path) - - self._project_name = self._project_helper.project_name - if not self._project_name: - command_title = self._command.title() - print(f'{command_title:s} aborted - unable to determine project name.') - return False - - project_definition = self._project_helper.ReadDefinitionFile() - - self._github_organization = None - if project_definition.homepage_url.startswith('https://github.com/'): - self._github_organization = project_definition.homepage_url[ - len('https://github.com/'):] - - self._github_organization, _, _ = self._github_organization.partition('/') - - if not self._github_organization: - command_title = self._command.title() - print( - f'{command_title:s} aborted - unable to determine GitHub ' - f'organization.') - return False - - self._git_repo_url = ( - f'https://github.com/' - f'{self._github_organization:s}/{self._project_name:s}.git') - - self._git_helper = git.GitHelper(self._git_repo_url) - - self._github_helper = github.GitHubHelper( - self._github_organization, self._project_name) - - return True - - def Lint(self): - """Lints a review. - - Returns: - bool: True if linting was successful. - """ - if self._project_name == 'l2tdocs': - return True - - if self._command not in self._CODE_INSPECTION_COMMANDS: - return True - - pylint_helper = pylint.PylintHelper() - if not pylint_helper.CheckUpToDateVersion(): - command_title = self._command.title() - min_version = pylint.PylintHelper.MINIMUM_VERSION - print( - f'{command_title:s} aborted - pylint version {min_version:s} ' - f'or later required.') - return False - - if self._all_files: - diffbase = None - else: - diffbase = 'main' - - changed_python_files = self._git_helper.GetChangedPythonFiles( - diffbase=diffbase) - - pylint_configuration = pylint_helper.GetRCFile(self._project_path) - if not pylint_helper.CheckFiles(changed_python_files, pylint_configuration): - command_title = self._command.title() - print(f'{command_title:s} aborted - unable to pass linter.') - - return False - - return True - - def Test(self): - """Tests a review. - - Returns: - bool: True if the review were successful. - """ - if self._project_name == 'l2tdocs': - return True - - if self._command not in ( - 'create-pr', 'create_pr', 'lint-test', 'lint_test', 'test'): - return True - - # TODO: determine why this alters the behavior of argparse. - # Currently affects this script being used in plaso. - command = f'{sys.executable:s} run_tests.py' - exit_code = subprocess.call(command, shell=True) - if exit_code != 0: - command_title = self._command.title() - print(f'{command_title:s} aborted - unable to pass review.') - - return False - - return True + """Helper for conducting code reviews.""" + + _SUPPORTED_PROJECTS_PATTERN = "|".join(project.ProjectHelper.SUPPORTED_PROJECTS) + _PROJECT_NAME_PREFIX_REGEX = re.compile(rf"\[({_SUPPORTED_PROJECTS_PATTERN})\] ") + + _CODE_INSPECTION_COMMANDS = frozenset( + ["create-pr", "create_pr", "lint", "lint-test", "lint_test"] + ) + + def __init__( + self, command, project_path, github_origin, feature_branch, all_files=False + ): + """Initializes a review helper. + + Args: + command (str): user provided command, for example "create", "lint". + project_path (str): path to the project being reviewed. + github_origin (str): GitHub origin. + feature_branch (str): feature branch. + all_files (Optional[bool]): True if the command should apply to all + files. Currently this only affects the lint command. + """ + super().__init__() + self._active_branch = None + self._all_files = all_files + self._command = command + self._feature_branch = feature_branch + self._git_helper = None + self._git_repo_url = None + self._github_helper = None + self._github_organization = None + self._github_origin = github_origin + self._fork_feature_branch = None + self._fork_username = None + self._maintainer = None + self._project_helper = None + self._project_name = None + self._project_path = project_path + + if self._github_origin: + self._fork_username, _, self._fork_feature_branch = ( + self._github_origin.partition(":") + ) + + def CheckLocalGitState(self): + """Checks the state of the local git repository. + + Returns: + bool: True if the state of the local git repository is sane. + """ + if self._github_organization in ("ForensicArtifacts", "log2timeline"): + if self._command in ( + "close", + "create-pr", + "create_pr", + "lint", + "lint-test", + "lint_test", + ): + if not self._git_helper.CheckHasProjectUpstream(): + command_title = self._command.title() + print(f"{command_title:s} aborted - missing project upstream.") + print(f"Run: git remote add upstream {self._git_repo_url:s}") + return False + + if self._command not in ( + "lint", + "lint-test", + "lint_test", + "test", + "update-version", + "update_version", + ): + if self._git_helper.CheckHasUncommittedChanges(): + command_title = self._command.title() + print(f"{command_title:s} aborted - detected uncommitted changes.") + print("Run: git commit") + return False + + if self._github_organization in ("ForensicArtifacts", "log2timeline"): + self._active_branch = self._git_helper.GetActiveBranch() + if self._command in ("create-pr", "create_pr"): + if self._active_branch == "main": + command_title = self._command.title() + print(f"{command_title:s} aborted - active branch is main.") + return False + + elif self._command == "close": + if self._feature_branch == "main": + command_title = self._command.title() + print(f"{command_title:s} aborted - feature branch cannot be main.") + return False + + if self._active_branch != "main": + self._git_helper.SwitchToMainBranch() + self._active_branch = "main" + + return True + + def Close(self): + """Closes a review. + + Returns: + bool: True if the close was successful. + """ + if not self._git_helper.CheckHasBranch(self._feature_branch): + print(f"No such feature branch: {self._feature_branch:s}") + else: + self._git_helper.RemoveFeatureBranch(self._feature_branch) + + if not self._git_helper.SynchronizeWithUpstream(): + command_title = self._command.title() + print( + f"{command_title:s} aborted - unable to synchronize with " + f"upstream/main." + ) + + return True + + def InitializeHelpers(self): + """Initializes the helpers. + + Returns: + bool: True if the helper initialization was successful. + """ + project_path = os.path.abspath(self._project_path) + + self._project_helper = project.ProjectHelper(project_path) + + self._project_name = self._project_helper.project_name + if not self._project_name: + command_title = self._command.title() + print(f"{command_title:s} aborted - unable to determine project name.") + return False + + project_definition = self._project_helper.ReadDefinitionFile() + + self._github_organization = None + if project_definition.homepage_url.startswith("https://github.com/"): + self._github_organization = project_definition.homepage_url[ + len("https://github.com/") : + ] + + self._github_organization, _, _ = self._github_organization.partition("/") + + if not self._github_organization: + command_title = self._command.title() + print( + f"{command_title:s} aborted - unable to determine GitHub " + f"organization." + ) + return False + + self._git_repo_url = ( + f"https://github.com/" + f"{self._github_organization:s}/{self._project_name:s}.git" + ) + + self._git_helper = git.GitHelper(self._git_repo_url) + + self._github_helper = github.GitHubHelper( + self._github_organization, self._project_name + ) + + return True + + def Lint(self): + """Lints a review. + + Returns: + bool: True if linting was successful. + """ + if self._project_name == "l2tdocs": + return True + + if self._command not in self._CODE_INSPECTION_COMMANDS: + return True + + pylint_helper = pylint.PylintHelper() + if not pylint_helper.CheckUpToDateVersion(): + command_title = self._command.title() + min_version = pylint.PylintHelper.MINIMUM_VERSION + print( + f"{command_title:s} aborted - pylint version {min_version:s} " + f"or later required." + ) + return False + + if self._all_files: + diffbase = None + else: + diffbase = "main" + + changed_python_files = self._git_helper.GetChangedPythonFiles(diffbase=diffbase) + + pylint_configuration = pylint_helper.GetRCFile(self._project_path) + if not pylint_helper.CheckFiles(changed_python_files, pylint_configuration): + command_title = self._command.title() + print(f"{command_title:s} aborted - unable to pass linter.") + + return False + + return True + + def Test(self): + """Tests a review. + + Returns: + bool: True if the review were successful. + """ + if self._project_name == "l2tdocs": + return True + + if self._command not in ( + "create-pr", + "create_pr", + "lint-test", + "lint_test", + "test", + ): + return True + + # TODO: determine why this alters the behavior of argparse. + # Currently affects this script being used in plaso. + command = f"{sys.executable:s} run_tests.py" + exit_code = subprocess.call(command, shell=True) + if exit_code != 0: + command_title = self._command.title() + print(f"{command_title:s} aborted - unable to pass review.") + + return False + + return True diff --git a/l2tdevtools/review_helpers/url_lib.py b/l2tdevtools/review_helpers/url_lib.py index 405b4421..90d90061 100644 --- a/l2tdevtools/review_helpers/url_lib.py +++ b/l2tdevtools/review_helpers/url_lib.py @@ -7,42 +7,44 @@ class URLLibHelper: - """URL library (urllib) helper.""" + """URL library (urllib) helper.""" - def Request(self, url, post_data=None): - """Sends a request to an URL. + def Request(self, url, post_data=None): + """Sends a request to an URL. - Args: - url (str): URL to send the request. - post_data (Optional[bytes]): data to send. + Args: + url (str): URL to send the request. + post_data (Optional[bytes]): data to send. - Returns: - bytes: response data. + Returns: + bytes: response data. - Raises: - ConnectivityError: if the request failed. - """ - request = urllib_request.Request(url) + Raises: + ConnectivityError: if the request failed. + """ + request = urllib_request.Request(url) - if post_data is not None: - # This will change the request into a POST. - request.data = post_data.encode('utf-8') + if post_data is not None: + # This will change the request into a POST. + request.data = post_data.encode("utf-8") - response_code = None - page_content = None + response_code = None + page_content = None - try: - with urllib_request.urlopen(request) as url_object: - response_code = url_object.code - if response_code in (200, 201): - page_content = url_object.read() + try: + with urllib_request.urlopen(request) as url_object: + response_code = url_object.code + if response_code in (200, 201): + page_content = url_object.read() - except urllib_error.HTTPError as exception: - raise errors.ConnectivityError( - f'Failed requesting URL {url:s} with error: {exception!s}') + except urllib_error.HTTPError as exception: + raise errors.ConnectivityError( + f"Failed requesting URL {url:s} with error: {exception!s}" + ) - if response_code not in (200, 201): - raise errors.ConnectivityError( - f'Failed requesting URL {url:s} with status code: {response_code:d}') + if response_code not in (200, 201): + raise errors.ConnectivityError( + f"Failed requesting URL {url:s} with status code: {response_code:d}" + ) - return page_content + return page_content diff --git a/l2tdevtools/source_helper.py b/l2tdevtools/source_helper.py index 4d7c3c65..b78f5ef7 100644 --- a/l2tdevtools/source_helper.py +++ b/l2tdevtools/source_helper.py @@ -12,422 +12,440 @@ class SourceHelper: - """Helper to manage project source code.""" + """Helper to manage project source code.""" - def __init__(self, project_name, project_definition): - """Initializes a source helper. + def __init__(self, project_name, project_definition): + """Initializes a source helper. - Args: - project_name (str): name of the project. - project_definition (ProjectDefinition): project definition. - """ - super().__init__() - self._project_definition = project_definition - self.project_name = project_name + Args: + project_name (str): name of the project. + project_definition (ProjectDefinition): project definition. + """ + super().__init__() + self._project_definition = project_definition + self.project_name = project_name - @abc.abstractmethod - def Create(self): - """Creates the source directory. + @abc.abstractmethod + def Create(self): + """Creates the source directory. - Returns: - str: name of the source directory or None on error. - """ + Returns: + str: name of the source directory or None on error. + """ - @abc.abstractmethod - def GetProjectIdentifier(self): - """Retrieves the project identifier for a given project name. + @abc.abstractmethod + def GetProjectIdentifier(self): + """Retrieves the project identifier for a given project name. - Returns: - str: project identifier or None on error. - """ + Returns: + str: project identifier or None on error. + """ - # TODO: add GetProjectVersion as interface function. + # TODO: add GetProjectVersion as interface function. - # TODO: add GetSourceDirectoryPath as interface function. + # TODO: add GetSourceDirectoryPath as interface function. - # TODO: add GetSourcePackageFilename as interface function, - # or move into GetSourceDirectoryPath of SourcePackageHelper. + # TODO: add GetSourcePackageFilename as interface function, + # or move into GetSourceDirectoryPath of SourcePackageHelper. class GitRepositorySourceHelper(SourceHelper): - """Class that manages the source code from a git repository.""" - - def __init__(self, project_name, project_definition): - """Initializes a source helper. - - Args: - project_name (str): name of the project. - project_definition (ProjectDefinition): project definition. - """ - super().__init__( - project_name, project_definition) - self._git_url = project_definition.git_url - - def Clean(self): - """Removes a previous version of the source directory.""" - if os.path.exists(self.project_name): - logging.info(f'Removing: {self.project_name:s}') - shutil.rmtree(self.project_name) - - def Create(self): - """Creates the source directory from the git repository. - - Returns: - str: name of the source directory or None on error. - """ - if not self.project_name or not self._git_url: - return None + """Class that manages the source code from a git repository.""" + + def __init__(self, project_name, project_definition): + """Initializes a source helper. + + Args: + project_name (str): name of the project. + project_definition (ProjectDefinition): project definition. + """ + super().__init__(project_name, project_definition) + self._git_url = project_definition.git_url + + def Clean(self): + """Removes a previous version of the source directory.""" + if os.path.exists(self.project_name): + logging.info(f"Removing: {self.project_name:s}") + shutil.rmtree(self.project_name) + + def Create(self): + """Creates the source directory from the git repository. + + Returns: + str: name of the source directory or None on error. + """ + if not self.project_name or not self._git_url: + return None - command = f'git clone {self._git_url:s}' - exit_code = subprocess.call(f'{command:s}', shell=True) - if exit_code != 0: - logging.error(f'Running: "{command:s}" failed.') - return None + command = f"git clone {self._git_url:s}" + exit_code = subprocess.call(f"{command:s}", shell=True) + if exit_code != 0: + logging.error(f'Running: "{command:s}" failed.') + return None - return self.project_name + return self.project_name - # pylint: disable=redundant-returns-doc - def GetProjectIdentifier(self): - """Retrieves the project identifier for a given project name. + # pylint: disable=redundant-returns-doc + def GetProjectIdentifier(self): + """Retrieves the project identifier for a given project name. - Returns: - str: project identifier or None on error. - """ - # TODO: determine project identifier based on git url. - return None + Returns: + str: project identifier or None on error. + """ + # TODO: determine project identifier based on git url. + return None class LibyalGitRepositorySourceHelper(GitRepositorySourceHelper): - """Class that manages the source code from a libyal git repository.""" - - def Create(self): - """Creates the source directory from the git repository. + """Class that manages the source code from a libyal git repository.""" - Returns: - str: name of the source directory or None on error. - """ - if not self.project_name or not self._git_url: - return None + def Create(self): + """Creates the source directory from the git repository. - command = f'git clone {self._git_url:s}' - exit_code = subprocess.call(f'{command:s}', shell=True) - if exit_code != 0: - logging.error(f'Running: "{command:s}" failed.') - return None - - source_directory = self.project_name - - command = './synclibs.sh' - exit_code = subprocess.call( - f'(cd {source_directory:s} && {command:s})', shell=True) - if exit_code != 0: - logging.error(f'Running: "{command:s}" failed.') - return None - - command = './autogen.sh' - exit_code = subprocess.call( - f'(cd {source_directory:s} && {command:s})', shell=True) - if exit_code != 0: - logging.error(f'Running: "{command:s}" failed.') - return None - - command = './configure' - exit_code = subprocess.call( - f'(cd {source_directory:s} && {command:s})', shell=True) - if exit_code != 0: - logging.error(f'Running: "{command:s}" failed.') - return None - - return source_directory - - -class SourcePackageHelper(SourceHelper): - """Class that manages the source code from a source package.""" - - ENCODING = 'utf-8' - - def __init__( - self, project_name, project_definition, downloads_directory, - download_helper_object): - """Initializes a source package helper. - - Args: - project_name (str): name of the project. - project_definition (ProjectDefinition): project definition. - downloads_directory (str): path to the directory where source package - is downloaded. - download_helper_object (DownloadHelper): download helper. - """ - super().__init__(project_name, project_definition) - self._download_helper = download_helper_object - self._downloads_directory = os.path.abspath(downloads_directory) - self._project_version = None - self._source_directory_path = None - self._source_package_filename = None - self._source_package_path = None - - def _CleanDownloads(self, project_name, project_version): - """Removes previous versions of downloaded source packages. - - Args: - project_name (str): name of the project. - project_version (str): current version of the project. - """ - filenames_to_ignore = re.compile(f'^{project_name:s}-.*{project_version!s}') - - # Remove previous versions of source packages in the format: - # -*[0-9]*.tar.gz - filenames = glob.glob(f'{project_name:s}-*[0-9]*.tar.gz') - for filename in filenames: - if not filenames_to_ignore.match(filename): - logging.info(f'Removing: {filename:s}') - os.remove(filename) - - # Remove previous versions of source packages in the format: - # -*[0-9]*.tgz - filenames = glob.glob(f'{project_name:s}-*[0-9]*.tgz') - for filename in filenames: - if not filenames_to_ignore.match(filename): - logging.info(f'Removing: {filename:s}') - os.remove(filename) - - # Remove previous versions of source packages in the format: - # -*[0-9]*.zip - filenames = glob.glob(f'{project_name:s}-*[0-9]*.zip') - for filename in filenames: - if not filenames_to_ignore.match(filename): - logging.info(f'Removing: {filename:s}') - os.remove(filename) - - def _CreateFromTar(self, source_package_filename): - """Creates the source directory from a .tar source package. - - Args: - source_package_filename (str): filename of the source package. - - Returns: - str: name of the source directory or None if no files can be extracted - from the .tar.gz source package. - """ - with tarfile.open( - source_package_filename, 'r:*', encoding='utf-8') as archive: - directory_name = '' - - for tar_info in archive.getmembers(): - filename = getattr(tar_info, 'name', None) - - if isinstance(filename, bytes): - try: - filename = filename.decode(self.ENCODING) - except UnicodeDecodeError: - logging.warning( - f'Unable to decode filename in tar file: ' - f'{source_package_filename:s}') - continue - - if filename is None: - logging.warning( - f'Missing filename in tar file: {source_package_filename:s}') - continue - - if not directory_name: - # Note that this will set directory name to an empty string - # if filename start with a /. - directory_name, _, _ = filename.partition('/') - if not directory_name or directory_name.startswith('..'): - logging.error( - f'Unsupported directory name in tar file: ' - f'{source_package_filename:s}') + Returns: + str: name of the source directory or None on error. + """ + if not self.project_name or not self._git_url: return None - if os.path.exists(directory_name): - break - - logging.info(f'Extracting: {source_package_filename:s}') - - elif not filename.startswith(directory_name): - logging.warning( - f'Skipping: {filename:s} in tar file: ' - f'{source_package_filename:s}') - continue - - archive.extract(tar_info) - - return directory_name - - def _CreateFromZip(self, source_package_filename): - """Creates the source directory from a .zip source package. - - Args: - source_package_filename (str): filename of the source package. - - Returns: - str: name of the source directory or None if no files can be extracted - from the .zip source package. - """ - with zipfile.ZipFile(source_package_filename, 'r') as archive: - directory_name = '' - - for zip_info in archive.infolist(): - filename = getattr(zip_info, 'filename', None) - if filename is None: - logging.warning( - f'Missing filename in zip file: {source_package_filename:s}') - continue - - if not directory_name: - # Note that this will set directory name to an empty string - # if filename start with a /. - directory_name, _, _ = filename.partition('/') - if not directory_name or directory_name.startswith('..'): - logging.error( - f'Unsupported directory name in zip file: ' - f'{source_package_filename:s}') + command = f"git clone {self._git_url:s}" + exit_code = subprocess.call(f"{command:s}", shell=True) + if exit_code != 0: + logging.error(f'Running: "{command:s}" failed.') return None - if os.path.exists(directory_name): - break - - logging.info(f'Extracting: {source_package_filename:s}') - - elif not filename.startswith(directory_name): - logging.warning( - f'Skipping: {filename:s} in zip file: ' - f'{source_package_filename:s}') - continue - - archive.extract(zip_info) - - return directory_name - - def Clean(self): - """Removes previous versions of source packages and directories.""" - project_version = self.GetProjectVersion() - if not project_version: - return - - current_working_directory = os.getcwd() - os.chdir(self._downloads_directory) - - try: - self._CleanDownloads(self.project_name, project_version) - finally: - os.chdir(current_working_directory) - - filenames_to_ignore = re.compile( - f'^{self.project_name:s}-.*{project_version!s}') - - # Remove previous versions of source directories in the format: - # -[0-9]* - filenames = glob.glob(f'{self.project_name:s}-[0-9]*') - for filename in filenames: - if os.path.isdir(filename) and not filenames_to_ignore.match(filename): - logging.info(f'Removing: {filename:s}') - shutil.rmtree(filename) - - def Create(self): - """Creates the source directory from the source package. - - Returns: - bool: True if the source directory was created successfully. - """ - if (not self._source_package_path or - not os.path.exists(self._source_package_path)): - logging.info( - f'Missing source package of: {self.project_name:s}') - return False - - directory_name = None - if (self._source_package_path.endswith('.tar.bz2') or - self._source_package_path.endswith('.tar.gz') or - self._source_package_path.endswith('.tgz')): - directory_name = self._CreateFromTar(self._source_package_path) - - elif self._source_package_path.endswith('.zip'): - directory_name = self._CreateFromZip(self._source_package_path) - - self._source_directory_path = directory_name - - return bool(directory_name) - - def Download(self): - """Downloads the source package. - - Returns: - str: path of the source package if the download was successful or - if the file was already downloaded or None on error. - """ - if not self._source_package_path: - project_version = self.GetProjectVersion() - if not project_version: - return None - - current_working_directory = os.getcwd() - os.chdir(self._downloads_directory) - - try: - self._source_package_filename = self._download_helper.Download( - self.project_name, project_version) - finally: - os.chdir(current_working_directory) - - if self._source_package_filename: - self._source_package_path = os.path.join( - self._downloads_directory, self._source_package_filename) - - return self._source_package_path - - def GetProjectIdentifier(self): - """Retrieves the project identifier for a given project name. - - Returns: - str: project identifier or None on error. - """ - return self._download_helper.GetProjectIdentifier() - - def GetProjectVersion(self): - """Retrieves the version number for a given project name. - - Returns: - str: version number or None on error. - """ - if not self._project_version: - version_definition = getattr(self._project_definition, 'version', None) - self._project_version = self._download_helper.GetLatestVersion( - self.project_name, version_definition) + source_directory = self.project_name - return self._project_version - - def GetSourceDirectoryPath(self): - """Retrieves the path of the source directory. - - Returns: - str: path of the source directory or None if not available. - """ - return self._source_directory_path - - def GetSourcePackageFilename(self): - """Retrieves the filename of the source package. - - This function downloads the source package if not done so previously. + command = "./synclibs.sh" + exit_code = subprocess.call( + f"(cd {source_directory:s} && {command:s})", shell=True + ) + if exit_code != 0: + logging.error(f'Running: "{command:s}" failed.') + return None - Returns: - str: filename of the source package or None if not available. - """ - if not self._source_package_filename: - self.Download() + command = "./autogen.sh" + exit_code = subprocess.call( + f"(cd {source_directory:s} && {command:s})", shell=True + ) + if exit_code != 0: + logging.error(f'Running: "{command:s}" failed.') + return None - return self._source_package_filename + command = "./configure" + exit_code = subprocess.call( + f"(cd {source_directory:s} && {command:s})", shell=True + ) + if exit_code != 0: + logging.error(f'Running: "{command:s}" failed.') + return None - def GetSourcePackagePath(self): - """Retrieves the filename of the source package. + return source_directory - This function downloads the source package if not done so previously. - Returns: - str: path of the source package or None if not available. - """ - if not self._source_package_path: - self.Download() +class SourcePackageHelper(SourceHelper): + """Class that manages the source code from a source package.""" + + ENCODING = "utf-8" + + def __init__( + self, + project_name, + project_definition, + downloads_directory, + download_helper_object, + ): + """Initializes a source package helper. + + Args: + project_name (str): name of the project. + project_definition (ProjectDefinition): project definition. + downloads_directory (str): path to the directory where source package + is downloaded. + download_helper_object (DownloadHelper): download helper. + """ + super().__init__(project_name, project_definition) + self._download_helper = download_helper_object + self._downloads_directory = os.path.abspath(downloads_directory) + self._project_version = None + self._source_directory_path = None + self._source_package_filename = None + self._source_package_path = None + + def _CleanDownloads(self, project_name, project_version): + """Removes previous versions of downloaded source packages. + + Args: + project_name (str): name of the project. + project_version (str): current version of the project. + """ + filenames_to_ignore = re.compile(f"^{project_name:s}-.*{project_version!s}") + + # Remove previous versions of source packages in the format: + # -*[0-9]*.tar.gz + filenames = glob.glob(f"{project_name:s}-*[0-9]*.tar.gz") + for filename in filenames: + if not filenames_to_ignore.match(filename): + logging.info(f"Removing: {filename:s}") + os.remove(filename) + + # Remove previous versions of source packages in the format: + # -*[0-9]*.tgz + filenames = glob.glob(f"{project_name:s}-*[0-9]*.tgz") + for filename in filenames: + if not filenames_to_ignore.match(filename): + logging.info(f"Removing: {filename:s}") + os.remove(filename) + + # Remove previous versions of source packages in the format: + # -*[0-9]*.zip + filenames = glob.glob(f"{project_name:s}-*[0-9]*.zip") + for filename in filenames: + if not filenames_to_ignore.match(filename): + logging.info(f"Removing: {filename:s}") + os.remove(filename) + + def _CreateFromTar(self, source_package_filename): + """Creates the source directory from a .tar source package. + + Args: + source_package_filename (str): filename of the source package. + + Returns: + str: name of the source directory or None if no files can be extracted + from the .tar.gz source package. + """ + with tarfile.open(source_package_filename, "r:*", encoding="utf-8") as archive: + directory_name = "" + + for tar_info in archive.getmembers(): + filename = getattr(tar_info, "name", None) + + if isinstance(filename, bytes): + try: + filename = filename.decode(self.ENCODING) + except UnicodeDecodeError: + logging.warning( + f"Unable to decode filename in tar file: " + f"{source_package_filename:s}" + ) + continue + + if filename is None: + logging.warning( + f"Missing filename in tar file: {source_package_filename:s}" + ) + continue + + if not directory_name: + # Note that this will set directory name to an empty string + # if filename start with a /. + directory_name, _, _ = filename.partition("/") + if not directory_name or directory_name.startswith(".."): + logging.error( + f"Unsupported directory name in tar file: " + f"{source_package_filename:s}" + ) + return None + + if os.path.exists(directory_name): + break + + logging.info(f"Extracting: {source_package_filename:s}") + + elif not filename.startswith(directory_name): + logging.warning( + f"Skipping: {filename:s} in tar file: " + f"{source_package_filename:s}" + ) + continue + + archive.extract(tar_info) + + return directory_name + + def _CreateFromZip(self, source_package_filename): + """Creates the source directory from a .zip source package. + + Args: + source_package_filename (str): filename of the source package. + + Returns: + str: name of the source directory or None if no files can be extracted + from the .zip source package. + """ + with zipfile.ZipFile(source_package_filename, "r") as archive: + directory_name = "" + + for zip_info in archive.infolist(): + filename = getattr(zip_info, "filename", None) + if filename is None: + logging.warning( + f"Missing filename in zip file: {source_package_filename:s}" + ) + continue + + if not directory_name: + # Note that this will set directory name to an empty string + # if filename start with a /. + directory_name, _, _ = filename.partition("/") + if not directory_name or directory_name.startswith(".."): + logging.error( + f"Unsupported directory name in zip file: " + f"{source_package_filename:s}" + ) + return None + + if os.path.exists(directory_name): + break + + logging.info(f"Extracting: {source_package_filename:s}") + + elif not filename.startswith(directory_name): + logging.warning( + f"Skipping: {filename:s} in zip file: " + f"{source_package_filename:s}" + ) + continue + + archive.extract(zip_info) + + return directory_name + + def Clean(self): + """Removes previous versions of source packages and directories.""" + project_version = self.GetProjectVersion() + if not project_version: + return + + current_working_directory = os.getcwd() + os.chdir(self._downloads_directory) + + try: + self._CleanDownloads(self.project_name, project_version) + finally: + os.chdir(current_working_directory) + + filenames_to_ignore = re.compile( + f"^{self.project_name:s}-.*{project_version!s}" + ) + + # Remove previous versions of source directories in the format: + # -[0-9]* + filenames = glob.glob(f"{self.project_name:s}-[0-9]*") + for filename in filenames: + if os.path.isdir(filename) and not filenames_to_ignore.match(filename): + logging.info(f"Removing: {filename:s}") + shutil.rmtree(filename) + + def Create(self): + """Creates the source directory from the source package. + + Returns: + bool: True if the source directory was created successfully. + """ + if not self._source_package_path or not os.path.exists( + self._source_package_path + ): + logging.info(f"Missing source package of: {self.project_name:s}") + return False + + directory_name = None + if ( + self._source_package_path.endswith(".tar.bz2") + or self._source_package_path.endswith(".tar.gz") + or self._source_package_path.endswith(".tgz") + ): + directory_name = self._CreateFromTar(self._source_package_path) + + elif self._source_package_path.endswith(".zip"): + directory_name = self._CreateFromZip(self._source_package_path) + + self._source_directory_path = directory_name + + return bool(directory_name) + + def Download(self): + """Downloads the source package. + + Returns: + str: path of the source package if the download was successful or + if the file was already downloaded or None on error. + """ + if not self._source_package_path: + project_version = self.GetProjectVersion() + if not project_version: + return None + + current_working_directory = os.getcwd() + os.chdir(self._downloads_directory) + + try: + self._source_package_filename = self._download_helper.Download( + self.project_name, project_version + ) + finally: + os.chdir(current_working_directory) + + if self._source_package_filename: + self._source_package_path = os.path.join( + self._downloads_directory, self._source_package_filename + ) + + return self._source_package_path + + def GetProjectIdentifier(self): + """Retrieves the project identifier for a given project name. + + Returns: + str: project identifier or None on error. + """ + return self._download_helper.GetProjectIdentifier() + + def GetProjectVersion(self): + """Retrieves the version number for a given project name. + + Returns: + str: version number or None on error. + """ + if not self._project_version: + version_definition = getattr(self._project_definition, "version", None) + self._project_version = self._download_helper.GetLatestVersion( + self.project_name, version_definition + ) + + return self._project_version + + def GetSourceDirectoryPath(self): + """Retrieves the path of the source directory. + + Returns: + str: path of the source directory or None if not available. + """ + return self._source_directory_path + + def GetSourcePackageFilename(self): + """Retrieves the filename of the source package. + + This function downloads the source package if not done so previously. + + Returns: + str: filename of the source package or None if not available. + """ + if not self._source_package_filename: + self.Download() + + return self._source_package_filename + + def GetSourcePackagePath(self): + """Retrieves the filename of the source package. + + This function downloads the source package if not done so previously. + + Returns: + str: path of the source package or None if not available. + """ + if not self._source_package_path: + self.Download() - return self._source_package_path + return self._source_package_path diff --git a/l2tdevtools/spec_file.py b/l2tdevtools/spec_file.py index 2de7710b..0cdd2e04 100644 --- a/l2tdevtools/spec_file.py +++ b/l2tdevtools/spec_file.py @@ -11,716 +11,808 @@ class RPMSpecFileGenerator: - """Class that helps in generating RPM spec files.""" - - _EMAIL_ADDRESS = ( - 'log2timeline development team ') - - _DOC_FILENAMES = [ - 'CHANGES', 'CHANGES.txt', 'CHANGES.TXT', - 'README', 'README.md', 'README.txt', 'README.TXT'] - - _LICENSE_FILENAMES = [ - 'LICENSE', 'LICENSE.txt', 'LICENSE.TXT'] - - _SPEC_TEMPLATE_DATA_PACKAGE_DEFINITION = [ - '%package -n %{{name}}-data', - 'Summary: Data files for {summary:s}', - '', - '%description -n %{{name}}-data', - '{description:s}', - '', - ''] - - _SPEC_TEMPLATE_PYTHON3_BODY = [ - '%prep', - '%autosetup -p1 -n {setup_name:s}-%{{version}}', - '', - '%build', - '%pyproject_wheel', - '', - '%install', - '%pyproject_install', - '', - ''] - - _SPEC_TEMPLATE_TOOLS_PACKAGE_DEFINITION = [ - '%package -n %{{name}}-tools', - 'Requires: python3-{project_name:s} >= %{{version}}', - 'Summary: Tools for {summary:s}', - '', - '%description -n %{{name}}-tools', - '{description:s}', - '', - ''] - - LOG_FILENAME = 'build.log' - - def __init__(self, data_path): - """Initializes the RPM spec file generator. - - Args: - data_path (str): path to the data directory which contains the RPM - templates sub directory. - """ - super().__init__() - self._data_path = data_path - - def _GenerateSpecFile( - self, project_definition, source_directory, source_package_filename, - project_name, rpm_build_dependencies, output_file_object): - """Generates a RPM spec file. - - Args: - project_definition (ProjectDefinition): project definition. - source_directory (str): path of the source directory. - source_package_filename (str): name of the source package. - project_name (str): name of the project. - rpm_build_dependencies (list[str]): RPM build dependencies. - output_file_object (file): output file-like object to write to. - - Returns: - bool: True if successful, False otherwise. - - Raises: - ValueError: if required configuration values are missing. - """ - configuration = { - 'description': None, - 'name': None, - 'license': None, - 'summary': None, - 'url': None, - 'vendor': None, - 'version': None} - - has_data_package = False - - python_module_name = project_name - - tools_directory = os.path.join(source_directory, 'scripts') - if not os.path.isdir(tools_directory): - tools_directory = os.path.join(source_directory, 'tools') - if not os.path.isdir(tools_directory): - tools_directory = os.path.join( - source_directory, python_module_name, 'scripts') - - has_tools_package = bool(os.path.isdir( - tools_directory) and glob.glob(os.path.join(tools_directory, '*.py'))) - - if project_definition.srpm_name: - package_name = project_definition.srpm_name - elif project_definition.rpm_name: - package_name = project_definition.rpm_name - else: - package_name = project_name - - if package_name.startswith('python-'): - package_name = package_name[7:] - - if project_definition.description_long: - configuration['description'] = ( - f'{project_definition.description_long:s}\n\n') - - if project_definition.description_short: - configuration['summary'] = project_definition.description_short.replace( - '\n', ' ') - - if project_definition.homepage_url: - configuration['url'] = project_definition.homepage_url - - if project_definition.license: - configuration['license'] = project_definition.license - - if project_definition.maintainer: - configuration['vendor'] = project_definition.maintainer - - pyproject_toml_file = os.path.join(source_directory, 'pyproject.toml') - if os.path.isfile(pyproject_toml_file): - with open(pyproject_toml_file, 'rb') as file_object: - pyproject_toml = tomllib.load(file_object) - - pyproject_toml_project = pyproject_toml.get('project', {}) - - if not configuration['description']: - configuration['description'] = pyproject_toml_project.get( - 'description', None) - - if not configuration['license']: - configuration['license'] = pyproject_toml_project.get('license', None) - - if not configuration['summary']: - configuration['summary'] = pyproject_toml_project.get( - 'description', None) - - if not configuration['vendor']: - pyproject_toml_maintainers = pyproject_toml_project.get( - 'maintainers', []) - if pyproject_toml_maintainers: - maintainer = pyproject_toml_maintainers[0] - configuration['vendor'] = ( - f'{maintainer["name"]:s} <{maintainer["email"]:s}>') - - if not configuration['version']: - configuration['version'] = pyproject_toml_project.get('version', None) - - pyproject_toml_urls = pyproject_toml_project.get('urls', {}) - - if not configuration['url']: - configuration['url'] = pyproject_toml_urls.get('Homepage', None) - - if pyproject_toml_project.get('scripts', {}): - has_tools_package = True - - setup_cfg_file = os.path.join(source_directory, 'setup.cfg') - if os.path.isfile(setup_cfg_file): - setup_cfg = setupcfg.read_configuration(setup_cfg_file) - setup_cfg_metadata = setup_cfg.get('metadata', {}) - - if not configuration['version']: - configuration['version'] = setup_cfg_metadata.get('version', None) - - if not configuration['license']: - configuration['license'] = setup_cfg_metadata.get('license', None) - - if not configuration['description']: - configuration['description'] = setup_cfg_metadata.get( - 'long_description', None) - - if not configuration['summary']: - configuration['summary'] = setup_cfg_metadata.get('description', None) - - if not configuration['url']: - configuration['url'] = setup_cfg_metadata.get('url', None) - - if not configuration['vendor']: - configuration['vendor'] = setup_cfg_metadata.get('maintainer', None) - - if not configuration['version']: - for version_file in glob.glob(os.path.join( - source_directory, '**', '__init__.py')): - with open(version_file, 'r', encoding='utf8') as file_object: - for line in file_object: - line = line.strip() - if line.startswith('__version__') and '=' in line: - version = line.rsplit('=', maxsplit=1)[-1] - version = version.strip().strip('\'').strip('"') - # pytz sets __version__ to VERSION - if version != 'VERSION': - configuration['version'] = version - break - - if not configuration['version']: - for version_file in glob.glob(os.path.join(source_directory, 'PKG-INFO')): - with open(version_file, 'r', encoding='utf8') as file_object: - for line in file_object: - if line.startswith('Version: '): - version = line.strip().rsplit(':', maxsplit=1)[-1] - configuration['version'] = version.strip() - break - - if rpm_build_dependencies: - build_requires = rpm_build_dependencies - else: - build_requires = self._SplitRequires(build_requires) - - configuration['name'] = project_name - - for key, value in configuration.items(): - if value is None: - raise ValueError(f'Missing configuration value: {key:s}') - - self._WriteSourcePackageDefinition( - output_file_object, source_package_filename, project_definition, - build_requires, configuration) - - if project_name != package_name: - python_package_name = f'python3-{package_name:s}' - else: - python_package_name = 'python3-%{name}' - - if has_data_package: - self._WriteDataPackageDefinition(output_file_object, configuration) - - self._WritePython3PackageDefinition( - project_definition, source_directory, python_package_name, - configuration, output_file_object) - - if has_tools_package: - self._WriteToolsPackageDefinition(output_file_object, configuration) - - license_line = self._GetLicenseFileDefinition(source_directory) - - doc_line = self._GetDocumentationFilesDefinition(source_directory) - - self._WritePython3Body(output_file_object, project_definition) - - if has_data_package: - self._WriteDataPackageFiles(output_file_object) - - self._WritePython3PackageFiles( - output_file_object, project_definition, project_name, - python_package_name, license_line, doc_line) - - if has_tools_package: - self._WriteToolsPackageFiles(output_file_object) - - # TODO make this more generic. - if project_name in ('chardet', 'dtfabric', 'pbr'): - output_file_object.write(( - '%exclude %{_bindir}/*\n' - '\n')) - - self._WriteChangeLog(output_file_object, configuration['version']) - - return True - - def _GetDocumentationFilesDefinition(self, source_directory): - """Retrieves the documentation files definition. - - Args: - source_directory (str): path of the source directory. - - Returns: - str: documentation files definition. - """ - doc_files = [] - for doc_file in self._DOC_FILENAMES: - doc_file_path = os.path.join(source_directory, doc_file) - if os.path.exists(doc_file_path): - doc_files.append(doc_file) - - if not doc_files: - doc_file_definition = '' - else: - doc_files = ' '.join(doc_files) - doc_file_definition = f'%doc {doc_files:s}\n' - - return doc_file_definition - - def _GetInstallDefinition(self, project_name): - """Retrieves the install definition. - - Args: - project_name (str): name of the project. - - Returns: - str: install definition. - """ - lines = [ - '%py3_install', - ('rm -rf %{buildroot}/usr/lib/python*/site-packages/*.dist-info/' - 'requires.txt'), - 'rm -rf %{buildroot}/usr/share/doc/%{name}/'] - - if project_name == 'astroid': - lines.extend([ - 'rm -rf %{buildroot}%{python3_sitelib}/astroid/tests']) - - elif project_name == 'pylint': - lines.extend([ - 'rm -rf %{buildroot}%{python3_sitelib}/pylint/test']) - - lines.append('') - return '\n'.join(lines) - - def _GetLicenseFileDefinition(self, source_directory): - """Retrieves the license file definition. - - Args: - source_directory (str): path of the source directory. - - Returns: - str: license file definition. - """ - license_file_definition = '' - for license_file in self._LICENSE_FILENAMES: - license_file_path = os.path.join(source_directory, license_file) - if os.path.exists(license_file_path): - license_file_definition = f'%license {license_file:s}\n' - break - - return license_file_definition - - def _SplitRequires(self, requires): - """Splits a spec file requires statement. - - The requires statement starts with "Requires: " and is either space or - comma separated. - - Args: - requires (str): requires statement. - - Returns: - list[str]: individual required dependencies, such as "libbde" or - "liblnk >= 20190520", sorted by name. - - Raises: - ValueError: if the requires statement does not start with "Requires: ". - """ - if not requires: - return [] - - if (not requires.startswith('BuildRequires: ') and - not requires.startswith('Requires: ')): - raise ValueError(f'Unsupported requires statement: "{requires:s}"') - - _, _, requires = requires.strip().partition(' ') - - # The requires statement can be space or comma separated. If it is space - # separated we want to keep the name of the requirement and its version - # grouped together. - if ',' in requires: - return sorted([require.strip() for require in requires.split(',')]) - - requires_list = [] - requires_segments = [require.strip() for require in requires.split(' ')] - number_of_segments = len(requires_segments) - - group_start_index = 0 - while group_start_index < number_of_segments: - group_end_index = group_start_index + 1 - if (group_end_index < number_of_segments and - requires_segments[group_end_index] in ('>=', '==')): - group_end_index += 2 - - group = ' '.join(requires_segments[group_start_index:group_end_index]) - requires_list.append(group) - - group_start_index = group_end_index - - return sorted(requires_list) - - def _WriteChangeLog(self, output_file_object, version): - """Writes the change log. - - Args: - output_file_object (file): output file-like object to write to. - version (str): version. - """ - date_time = datetime.datetime.now() - date_time_string = date_time.strftime('%a %b %e %Y') - - output_file_object.write(( - '%changelog\n' - f'* {date_time_string:s} {self._EMAIL_ADDRESS:s} {version:s}-1\n' - '- Auto-generated\n')) - - def _WriteDataPackageDefinition(self, output_file_object, configuration): - """Writes the data package definition. - - Args: - output_file_object (file): output file-like object to write to. - configuration (dict[str, str]): package configuration. - """ - template_mappings = { - 'description': configuration['description'], - 'summary': configuration['summary']} - - output_string = '\n'.join(self._SPEC_TEMPLATE_DATA_PACKAGE_DEFINITION) - output_string = output_string.format(**template_mappings) - output_file_object.write(output_string) - - def _WriteDataPackageFiles(self, output_file_object): - """Writes the data package files. - - Args: - output_file_object (file): output file-like object to write to. - """ - template = [ - '%files -n %{name}-data', - '%defattr(644,root,root,755)', - '%license LICENSE', - '%doc ACKNOWLEDGEMENTS AUTHORS README.md', - '%{_datadir}/%{name}/*', - '', - ''] - - output_string = '\n'.join(template) - output_file_object.write(output_string) - - def _WritePython3Body(self, output_file_object, project_definition): - """Writes the Python 3 body. - - Args: - output_file_object (file): output file-like object to write to. - project_definition (ProjectDefinition): project definition. - """ - # TODO: handle GetInstallDefinition - - if project_definition.setup_name: - setup_name = project_definition.setup_name - else: - setup_name = '%{name}' - - template_mappings = { - 'setup_name': setup_name} - - output_string = '\n'.join(self._SPEC_TEMPLATE_PYTHON3_BODY) - output_string = output_string.format(**template_mappings) - output_file_object.write(output_string) - - def _WritePython3PackageDefinition( - self, project_definition, source_directory, python_package_name, - configuration, output_file_object): - """Writes the Python 3 package definition. - - Args: - project_definition (ProjectDefinition): project definition. - source_directory (str): path of the source directory. - python_package_name (str): Python package name. - configuration (dict[str, str]): package configuration. - output_file_object (file): output file-like object to write to. - """ - requires = project_definition.rpm_dependencies - - if not requires: - dependencies_file = os.path.join(source_directory, 'dependencies.ini') - if os.path.isfile(dependencies_file): - dependency_helper = dependencies.DependencyHelper( - dependencies_file=dependencies_file) - requires = dependency_helper.GetRPMRequires() - - template_mappings = { - 'description': configuration['description'], - 'name': python_package_name, - 'requires': ', '.join(requires), - 'summary': configuration['summary']} - - template = ['%package -n {name:s}'] - - if requires: - template.append('Requires: {requires:s}') - - template.extend([ - 'Summary: Python 3 module of {summary:s}', - '', - '%description -n {name:s}', - '{description:s}', - '', - '']) - - output_string = '\n'.join(template) - output_string = output_string.format(**template_mappings) - output_file_object.write(output_string) - - def _WritePython3PackageFiles( - self, output_file_object, project_definition, project_name, name, - license_line, doc_line): - """Writes the Python 3 package files. - - Args: - output_file_object (file): output file-like object to write to. - project_definition (ProjectDefinition): project definition. - project_name (str): name of the project. - name (str): package name. - license_line (str): line containing the license file definition. - doc_line (str): line containing the document files definition. - """ - # Note that copr currently fails if %{python3_sitelib} is used. - - if project_definition.setup_name: - setup_name = project_definition.setup_name - else: - setup_name = project_name - - # Python modules names contain "_" instead of "-" - setup_name = setup_name.replace('-', '_') - - template_mappings = { - 'doc': doc_line.rstrip(), - 'license': license_line.rstrip(), - 'name': name, - 'setup_name': setup_name} - - template = [ - '%files -n {name:s}', - '{license:s}', - '{doc:s}'] - - if project_definition.architecture_dependent: - template.extend([ - '%{{_libdir}}/python3*/site-packages/{setup_name:s}', - '%{{_libdir}}/python3*/site-packages/{setup_name:s}*.dist-info']) - - else: - template.extend([ - '%{{python3_sitelib}}/{setup_name:s}', - '%{{python3_sitelib}}/{setup_name:s}*.dist-info']) - - template.extend(['', '']) - - output_string = '\n'.join(template) - output_string = output_string.format(**template_mappings) - output_file_object.write(output_string) - - def _WriteSpecFileFromTempate( - self, project_definition, source_directory, project_version, - output_file_object): - """Writes the RPM spec file from a template. - - Args: - project_definition (ProjectDefinition): project definition. - source_directory (str): path of the source directory. - project_version (str): project version. - output_file_object (file): output file-like object to write to. - - Returns: - bool: True if successful, False otherwise. - """ - date_time = datetime.datetime.now() - date_time_string = date_time.strftime('%a %b %e %Y') - - rpm_requires = [] - - dependencies_file = os.path.join(source_directory, 'dependencies.ini') - if os.path.isfile(dependencies_file): - dependency_helper = dependencies.DependencyHelper( - dependencies_file=dependencies_file) - rpm_requires = dependency_helper.GetRPMRequires() - - template_values = { - 'date_time': date_time_string, - 'rpm_requires': ', '.join(rpm_requires), - 'version': project_version} - - template_file_path = os.path.join( - self._data_path, 'rpm_templates', project_definition.rpm_template_spec) - with open(template_file_path, 'r', encoding='utf8') as file_object: - rules_template = file_object.read() - - data = rules_template.format(**template_values) - - output_file_object.write(data) - - return True - - def _WriteSourcePackageDefinition( - self, output_file_object, source_package_filename, project_definition, - build_requires, configuration): - """Writes the source package definition. - - Args: - output_file_object (file): output file-like object to write to. - source_package_filename (str): name of the package source file. - project_definition (ProjectDefinition): project definition. - build_requires (list[str]): package build requires definition. - configuration (dict[str, str]): package configuration. - """ - if source_package_filename.endswith('.zip'): - source_extension = 'zip' - else: - source_extension = 'tar.gz' - - if project_definition.setup_name: - setup_name = project_definition.setup_name - else: - setup_name = '%{name}' - - template_mappings = { - 'build_requires': ', '.join(build_requires), - 'description': configuration['description'], - 'group': 'Development/Libraries', - 'license': configuration['license'], - 'name': configuration['name'], - 'source': f'{setup_name:s}-%{{version}}.{source_extension:s}', - 'summary': configuration['summary'], - 'url': configuration['url'], - 'vendor': configuration['vendor'], - 'version': configuration['version']} - - template = [ - 'Name: {name:s}', - 'Version: {version:s}', - 'Release: 1', - 'Group: {group:s}', - 'License: {license:s}', - 'Summary: {summary:s}', - 'Url: {url:s}', - 'Vendor: {vendor:s}'] - - template.append('Source0: {source:s}') - - if not project_definition.architecture_dependent: - template.append('BuildArch: noarch') - - if build_requires: - template.append('BuildRequires: {build_requires:s}') - - template.extend([ - '', - '%{{?python_disable_dependency_generator}}', - '', - '%description', - '{description:s}', - '', - '']) - - output_string = '\n'.join(template) - output_string = output_string.format(**template_mappings) - output_file_object.write(output_string) - - def _WriteToolsPackageDefinition(self, output_file_object, configuration): - """Writes the tools package definition. - - Args: - output_file_object (file): output file-like object to write to. - configuration (dict[str, str]): package configuration. - """ - template_mappings = { - 'description': configuration['description'], - 'project_name': configuration['name'], - 'summary': configuration['summary']} - - output_string = '\n'.join(self._SPEC_TEMPLATE_TOOLS_PACKAGE_DEFINITION) - output_string = output_string.format(**template_mappings) - output_file_object.write(output_string) - - def _WriteToolsPackageFiles(self, output_file_object): - """Writes the tools package files. - - Args: - output_file_object (file): output file-like object to write to. - """ - output_file_object.write( - '%files -n %{name}-tools\n' - '%{_bindir}/*\n' - '\n') - - def Generate( - self, project_definition, source_directory, source_package_filename, - project_name, project_version, output_file): - """Generated a RPM spec file. - - Args: - project_definition (ProjectDefinition): project definition. - source_directory (str): path of the source directory. - source_package_filename (str): name of the source package. - project_name (str): name of the project. - project_version (str): version of the project. - output_file (str): path of the output RPM spec file. - - Returns: - bool: True if successful, False otherwise. - """ - rpm_build_dependencies = [ - 'python3-devel', 'pyproject-rpm-macros', 'python3-pip', - 'python3-setuptools', 'python3-wheel'] - - if project_definition.architecture_dependent: - rpm_build_dependencies.append('gcc') - - if project_definition.rpm_build_dependencies: - rpm_build_dependencies.extend(project_definition.rpm_build_dependencies) - - with open(output_file, 'w', encoding='utf8') as output_file_object: - if project_definition.rpm_template_spec: - result = self._WriteSpecFileFromTempate( - project_definition, source_directory, project_version, - output_file_object) - else: - result = self._GenerateSpecFile( - project_definition, source_directory, source_package_filename, - project_name, rpm_build_dependencies, output_file_object) - - return result + """Class that helps in generating RPM spec files.""" + + _EMAIL_ADDRESS = "log2timeline development team " + + _DOC_FILENAMES = [ + "CHANGES", + "CHANGES.txt", + "CHANGES.TXT", + "README", + "README.md", + "README.txt", + "README.TXT", + ] + + _LICENSE_FILENAMES = ["LICENSE", "LICENSE.txt", "LICENSE.TXT"] + + _SPEC_TEMPLATE_DATA_PACKAGE_DEFINITION = [ + "%package -n %{{name}}-data", + "Summary: Data files for {summary:s}", + "", + "%description -n %{{name}}-data", + "{description:s}", + "", + "", + ] + + _SPEC_TEMPLATE_PYTHON3_BODY = [ + "%prep", + "%autosetup -p1 -n {setup_name:s}-%{{version}}", + "", + "%build", + "%pyproject_wheel", + "", + "%install", + "%pyproject_install", + "", + "", + ] + + _SPEC_TEMPLATE_TOOLS_PACKAGE_DEFINITION = [ + "%package -n %{{name}}-tools", + "Requires: python3-{project_name:s} >= %{{version}}", + "Summary: Tools for {summary:s}", + "", + "%description -n %{{name}}-tools", + "{description:s}", + "", + "", + ] + + LOG_FILENAME = "build.log" + + def __init__(self, data_path): + """Initializes the RPM spec file generator. + + Args: + data_path (str): path to the data directory which contains the RPM + templates sub directory. + """ + super().__init__() + self._data_path = data_path + + def _GenerateSpecFile( + self, + project_definition, + source_directory, + source_package_filename, + project_name, + rpm_build_dependencies, + output_file_object, + ): + """Generates a RPM spec file. + + Args: + project_definition (ProjectDefinition): project definition. + source_directory (str): path of the source directory. + source_package_filename (str): name of the source package. + project_name (str): name of the project. + rpm_build_dependencies (list[str]): RPM build dependencies. + output_file_object (file): output file-like object to write to. + + Returns: + bool: True if successful, False otherwise. + + Raises: + ValueError: if required configuration values are missing. + """ + configuration = { + "description": None, + "name": None, + "license": None, + "summary": None, + "url": None, + "vendor": None, + "version": None, + } + + has_data_package = False + + python_module_name = project_name + + tools_directory = os.path.join(source_directory, "scripts") + if not os.path.isdir(tools_directory): + tools_directory = os.path.join(source_directory, "tools") + if not os.path.isdir(tools_directory): + tools_directory = os.path.join( + source_directory, python_module_name, "scripts" + ) + + has_tools_package = bool( + os.path.isdir(tools_directory) + and glob.glob(os.path.join(tools_directory, "*.py")) + ) + + if project_definition.srpm_name: + package_name = project_definition.srpm_name + elif project_definition.rpm_name: + package_name = project_definition.rpm_name + else: + package_name = project_name + + if package_name.startswith("python-"): + package_name = package_name[7:] + + if project_definition.description_long: + configuration["description"] = ( + f"{project_definition.description_long:s}\n\n" + ) + + if project_definition.description_short: + configuration["summary"] = project_definition.description_short.replace( + "\n", " " + ) + + if project_definition.homepage_url: + configuration["url"] = project_definition.homepage_url + + if project_definition.license: + configuration["license"] = project_definition.license + + if project_definition.maintainer: + configuration["vendor"] = project_definition.maintainer + + pyproject_toml_file = os.path.join(source_directory, "pyproject.toml") + if os.path.isfile(pyproject_toml_file): + with open(pyproject_toml_file, "rb") as file_object: + pyproject_toml = tomllib.load(file_object) + + pyproject_toml_project = pyproject_toml.get("project", {}) + + if not configuration["description"]: + configuration["description"] = pyproject_toml_project.get( + "description", None + ) + + if not configuration["license"]: + configuration["license"] = pyproject_toml_project.get("license", None) + + if not configuration["summary"]: + configuration["summary"] = pyproject_toml_project.get( + "description", None + ) + + if not configuration["vendor"]: + pyproject_toml_maintainers = pyproject_toml_project.get( + "maintainers", [] + ) + if pyproject_toml_maintainers: + maintainer = pyproject_toml_maintainers[0] + configuration["vendor"] = ( + f'{maintainer["name"]:s} <{maintainer["email"]:s}>' + ) + + if not configuration["version"]: + configuration["version"] = pyproject_toml_project.get("version", None) + + pyproject_toml_urls = pyproject_toml_project.get("urls", {}) + + if not configuration["url"]: + configuration["url"] = pyproject_toml_urls.get("Homepage", None) + + if pyproject_toml_project.get("scripts", {}): + has_tools_package = True + + setup_cfg_file = os.path.join(source_directory, "setup.cfg") + if os.path.isfile(setup_cfg_file): + setup_cfg = setupcfg.read_configuration(setup_cfg_file) + setup_cfg_metadata = setup_cfg.get("metadata", {}) + + if not configuration["version"]: + configuration["version"] = setup_cfg_metadata.get("version", None) + + if not configuration["license"]: + configuration["license"] = setup_cfg_metadata.get("license", None) + + if not configuration["description"]: + configuration["description"] = setup_cfg_metadata.get( + "long_description", None + ) + + if not configuration["summary"]: + configuration["summary"] = setup_cfg_metadata.get("description", None) + + if not configuration["url"]: + configuration["url"] = setup_cfg_metadata.get("url", None) + + if not configuration["vendor"]: + configuration["vendor"] = setup_cfg_metadata.get("maintainer", None) + + if not configuration["version"]: + for version_file in glob.glob( + os.path.join(source_directory, "**", "__init__.py") + ): + with open(version_file, "r", encoding="utf8") as file_object: + for line in file_object: + line = line.strip() + if line.startswith("__version__") and "=" in line: + version = line.rsplit("=", maxsplit=1)[-1] + version = version.strip().strip("'").strip('"') + # pytz sets __version__ to VERSION + if version != "VERSION": + configuration["version"] = version + break + + if not configuration["version"]: + for version_file in glob.glob(os.path.join(source_directory, "PKG-INFO")): + with open(version_file, "r", encoding="utf8") as file_object: + for line in file_object: + if line.startswith("Version: "): + version = line.strip().rsplit(":", maxsplit=1)[-1] + configuration["version"] = version.strip() + break + + if rpm_build_dependencies: + build_requires = rpm_build_dependencies + else: + build_requires = self._SplitRequires(build_requires) + + configuration["name"] = project_name + + for key, value in configuration.items(): + if value is None: + raise ValueError(f"Missing configuration value: {key:s}") + + self._WriteSourcePackageDefinition( + output_file_object, + source_package_filename, + project_definition, + build_requires, + configuration, + ) + + if project_name != package_name: + python_package_name = f"python3-{package_name:s}" + else: + python_package_name = "python3-%{name}" + + if has_data_package: + self._WriteDataPackageDefinition(output_file_object, configuration) + + self._WritePython3PackageDefinition( + project_definition, + source_directory, + python_package_name, + configuration, + output_file_object, + ) + + if has_tools_package: + self._WriteToolsPackageDefinition(output_file_object, configuration) + + license_line = self._GetLicenseFileDefinition(source_directory) + + doc_line = self._GetDocumentationFilesDefinition(source_directory) + + self._WritePython3Body(output_file_object, project_definition) + + if has_data_package: + self._WriteDataPackageFiles(output_file_object) + + self._WritePython3PackageFiles( + output_file_object, + project_definition, + project_name, + python_package_name, + license_line, + doc_line, + ) + + if has_tools_package: + self._WriteToolsPackageFiles(output_file_object) + + # TODO make this more generic. + if project_name in ("chardet", "dtfabric", "pbr"): + output_file_object.write(("%exclude %{_bindir}/*\n\n")) + + self._WriteChangeLog(output_file_object, configuration["version"]) + + return True + + def _GetDocumentationFilesDefinition(self, source_directory): + """Retrieves the documentation files definition. + + Args: + source_directory (str): path of the source directory. + + Returns: + str: documentation files definition. + """ + doc_files = [] + for doc_file in self._DOC_FILENAMES: + doc_file_path = os.path.join(source_directory, doc_file) + if os.path.exists(doc_file_path): + doc_files.append(doc_file) + + if not doc_files: + doc_file_definition = "" + else: + doc_files = " ".join(doc_files) + doc_file_definition = f"%doc {doc_files:s}\n" + + return doc_file_definition + + def _GetInstallDefinition(self, project_name): + """Retrieves the install definition. + + Args: + project_name (str): name of the project. + + Returns: + str: install definition. + """ + lines = [ + "%py3_install", + ( + "rm -rf %{buildroot}/usr/lib/python*/site-packages/*.dist-info/" + "requires.txt" + ), + "rm -rf %{buildroot}/usr/share/doc/%{name}/", + ] + + if project_name == "astroid": + lines.extend(["rm -rf %{buildroot}%{python3_sitelib}/astroid/tests"]) + + elif project_name == "pylint": + lines.extend(["rm -rf %{buildroot}%{python3_sitelib}/pylint/test"]) + + lines.append("") + return "\n".join(lines) + + def _GetLicenseFileDefinition(self, source_directory): + """Retrieves the license file definition. + + Args: + source_directory (str): path of the source directory. + + Returns: + str: license file definition. + """ + license_file_definition = "" + for license_file in self._LICENSE_FILENAMES: + license_file_path = os.path.join(source_directory, license_file) + if os.path.exists(license_file_path): + license_file_definition = f"%license {license_file:s}\n" + break + + return license_file_definition + + def _SplitRequires(self, requires): + """Splits a spec file requires statement. + + The requires statement starts with "Requires: " and is either space or + comma separated. + + Args: + requires (str): requires statement. + + Returns: + list[str]: individual required dependencies, such as "libbde" or + "liblnk >= 20190520", sorted by name. + + Raises: + ValueError: if the requires statement does not start with "Requires: ". + """ + if not requires: + return [] + + if not requires.startswith("BuildRequires: ") and not requires.startswith( + "Requires: " + ): + raise ValueError(f'Unsupported requires statement: "{requires:s}"') + + _, _, requires = requires.strip().partition(" ") + + # The requires statement can be space or comma separated. If it is space + # separated we want to keep the name of the requirement and its version + # grouped together. + if "," in requires: + return sorted([require.strip() for require in requires.split(",")]) + + requires_list = [] + requires_segments = [require.strip() for require in requires.split(" ")] + number_of_segments = len(requires_segments) + + group_start_index = 0 + while group_start_index < number_of_segments: + group_end_index = group_start_index + 1 + if group_end_index < number_of_segments and requires_segments[ + group_end_index + ] in (">=", "=="): + group_end_index += 2 + + group = " ".join(requires_segments[group_start_index:group_end_index]) + requires_list.append(group) + + group_start_index = group_end_index + + return sorted(requires_list) + + def _WriteChangeLog(self, output_file_object, version): + """Writes the change log. + + Args: + output_file_object (file): output file-like object to write to. + version (str): version. + """ + date_time = datetime.datetime.now() + date_time_string = date_time.strftime("%a %b %e %Y") + + output_file_object.write( + ( + "%changelog\n" + f"* {date_time_string:s} {self._EMAIL_ADDRESS:s} {version:s}-1\n" + "- Auto-generated\n" + ) + ) + + def _WriteDataPackageDefinition(self, output_file_object, configuration): + """Writes the data package definition. + + Args: + output_file_object (file): output file-like object to write to. + configuration (dict[str, str]): package configuration. + """ + template_mappings = { + "description": configuration["description"], + "summary": configuration["summary"], + } + + output_string = "\n".join(self._SPEC_TEMPLATE_DATA_PACKAGE_DEFINITION) + output_string = output_string.format(**template_mappings) + output_file_object.write(output_string) + + def _WriteDataPackageFiles(self, output_file_object): + """Writes the data package files. + + Args: + output_file_object (file): output file-like object to write to. + """ + template = [ + "%files -n %{name}-data", + "%defattr(644,root,root,755)", + "%license LICENSE", + "%doc ACKNOWLEDGEMENTS AUTHORS README.md", + "%{_datadir}/%{name}/*", + "", + "", + ] + + output_string = "\n".join(template) + output_file_object.write(output_string) + + def _WritePython3Body(self, output_file_object, project_definition): + """Writes the Python 3 body. + + Args: + output_file_object (file): output file-like object to write to. + project_definition (ProjectDefinition): project definition. + """ + # TODO: handle GetInstallDefinition + + if project_definition.setup_name: + setup_name = project_definition.setup_name + else: + setup_name = "%{name}" + + template_mappings = {"setup_name": setup_name} + + output_string = "\n".join(self._SPEC_TEMPLATE_PYTHON3_BODY) + output_string = output_string.format(**template_mappings) + output_file_object.write(output_string) + + def _WritePython3PackageDefinition( + self, + project_definition, + source_directory, + python_package_name, + configuration, + output_file_object, + ): + """Writes the Python 3 package definition. + + Args: + project_definition (ProjectDefinition): project definition. + source_directory (str): path of the source directory. + python_package_name (str): Python package name. + configuration (dict[str, str]): package configuration. + output_file_object (file): output file-like object to write to. + """ + requires = project_definition.rpm_dependencies + + if not requires: + dependencies_file = os.path.join(source_directory, "dependencies.ini") + if os.path.isfile(dependencies_file): + dependency_helper = dependencies.DependencyHelper( + dependencies_file=dependencies_file + ) + requires = dependency_helper.GetRPMRequires() + + template_mappings = { + "description": configuration["description"], + "name": python_package_name, + "requires": ", ".join(requires), + "summary": configuration["summary"], + } + + template = ["%package -n {name:s}"] + + if requires: + template.append("Requires: {requires:s}") + + template.extend( + [ + "Summary: Python 3 module of {summary:s}", + "", + "%description -n {name:s}", + "{description:s}", + "", + "", + ] + ) + + output_string = "\n".join(template) + output_string = output_string.format(**template_mappings) + output_file_object.write(output_string) + + def _WritePython3PackageFiles( + self, + output_file_object, + project_definition, + project_name, + name, + license_line, + doc_line, + ): + """Writes the Python 3 package files. + + Args: + output_file_object (file): output file-like object to write to. + project_definition (ProjectDefinition): project definition. + project_name (str): name of the project. + name (str): package name. + license_line (str): line containing the license file definition. + doc_line (str): line containing the document files definition. + """ + # Note that copr currently fails if %{python3_sitelib} is used. + + if project_definition.setup_name: + setup_name = project_definition.setup_name + else: + setup_name = project_name + + # Python modules names contain "_" instead of "-" + setup_name = setup_name.replace("-", "_") + + template_mappings = { + "doc": doc_line.rstrip(), + "license": license_line.rstrip(), + "name": name, + "setup_name": setup_name, + } + + template = ["%files -n {name:s}", "{license:s}", "{doc:s}"] + + if project_definition.architecture_dependent: + template.extend( + [ + "%{{_libdir}}/python3*/site-packages/{setup_name:s}", + "%{{_libdir}}/python3*/site-packages/{setup_name:s}*.dist-info", + ] + ) + + else: + template.extend( + [ + "%{{python3_sitelib}}/{setup_name:s}", + "%{{python3_sitelib}}/{setup_name:s}*.dist-info", + ] + ) + + template.extend(["", ""]) + + output_string = "\n".join(template) + output_string = output_string.format(**template_mappings) + output_file_object.write(output_string) + + def _WriteSpecFileFromTempate( + self, project_definition, source_directory, project_version, output_file_object + ): + """Writes the RPM spec file from a template. + + Args: + project_definition (ProjectDefinition): project definition. + source_directory (str): path of the source directory. + project_version (str): project version. + output_file_object (file): output file-like object to write to. + + Returns: + bool: True if successful, False otherwise. + """ + date_time = datetime.datetime.now() + date_time_string = date_time.strftime("%a %b %e %Y") + + rpm_requires = [] + + dependencies_file = os.path.join(source_directory, "dependencies.ini") + if os.path.isfile(dependencies_file): + dependency_helper = dependencies.DependencyHelper( + dependencies_file=dependencies_file + ) + rpm_requires = dependency_helper.GetRPMRequires() + + template_values = { + "date_time": date_time_string, + "rpm_requires": ", ".join(rpm_requires), + "version": project_version, + } + + template_file_path = os.path.join( + self._data_path, "rpm_templates", project_definition.rpm_template_spec + ) + with open(template_file_path, "r", encoding="utf8") as file_object: + rules_template = file_object.read() + + data = rules_template.format(**template_values) + + output_file_object.write(data) + + return True + + def _WriteSourcePackageDefinition( + self, + output_file_object, + source_package_filename, + project_definition, + build_requires, + configuration, + ): + """Writes the source package definition. + + Args: + output_file_object (file): output file-like object to write to. + source_package_filename (str): name of the package source file. + project_definition (ProjectDefinition): project definition. + build_requires (list[str]): package build requires definition. + configuration (dict[str, str]): package configuration. + """ + if source_package_filename.endswith(".zip"): + source_extension = "zip" + else: + source_extension = "tar.gz" + + if project_definition.setup_name: + setup_name = project_definition.setup_name + else: + setup_name = "%{name}" + + template_mappings = { + "build_requires": ", ".join(build_requires), + "description": configuration["description"], + "group": "Development/Libraries", + "license": configuration["license"], + "name": configuration["name"], + "source": f"{setup_name:s}-%{{version}}.{source_extension:s}", + "summary": configuration["summary"], + "url": configuration["url"], + "vendor": configuration["vendor"], + "version": configuration["version"], + } + + template = [ + "Name: {name:s}", + "Version: {version:s}", + "Release: 1", + "Group: {group:s}", + "License: {license:s}", + "Summary: {summary:s}", + "Url: {url:s}", + "Vendor: {vendor:s}", + ] + + template.append("Source0: {source:s}") + + if not project_definition.architecture_dependent: + template.append("BuildArch: noarch") + + if build_requires: + template.append("BuildRequires: {build_requires:s}") + + template.extend( + [ + "", + "%{{?python_disable_dependency_generator}}", + "", + "%description", + "{description:s}", + "", + "", + ] + ) + + output_string = "\n".join(template) + output_string = output_string.format(**template_mappings) + output_file_object.write(output_string) + + def _WriteToolsPackageDefinition(self, output_file_object, configuration): + """Writes the tools package definition. + + Args: + output_file_object (file): output file-like object to write to. + configuration (dict[str, str]): package configuration. + """ + template_mappings = { + "description": configuration["description"], + "project_name": configuration["name"], + "summary": configuration["summary"], + } + + output_string = "\n".join(self._SPEC_TEMPLATE_TOOLS_PACKAGE_DEFINITION) + output_string = output_string.format(**template_mappings) + output_file_object.write(output_string) + + def _WriteToolsPackageFiles(self, output_file_object): + """Writes the tools package files. + + Args: + output_file_object (file): output file-like object to write to. + """ + output_file_object.write("%files -n %{name}-tools\n%{_bindir}/*\n\n") + + def Generate( + self, + project_definition, + source_directory, + source_package_filename, + project_name, + project_version, + output_file, + ): + """Generated a RPM spec file. + + Args: + project_definition (ProjectDefinition): project definition. + source_directory (str): path of the source directory. + source_package_filename (str): name of the source package. + project_name (str): name of the project. + project_version (str): version of the project. + output_file (str): path of the output RPM spec file. + + Returns: + bool: True if successful, False otherwise. + """ + rpm_build_dependencies = [ + "python3-devel", + "pyproject-rpm-macros", + "python3-pip", + "python3-setuptools", + "python3-wheel", + ] + + if project_definition.architecture_dependent: + rpm_build_dependencies.append("gcc") + + if project_definition.rpm_build_dependencies: + rpm_build_dependencies.extend(project_definition.rpm_build_dependencies) + + with open(output_file, "w", encoding="utf8") as output_file_object: + if project_definition.rpm_template_spec: + result = self._WriteSpecFileFromTempate( + project_definition, + source_directory, + project_version, + output_file_object, + ) + else: + result = self._GenerateSpecFile( + project_definition, + source_directory, + source_package_filename, + project_name, + rpm_build_dependencies, + output_file_object, + ) + + return result diff --git a/l2tdevtools/versions.py b/l2tdevtools/versions.py index 6bc4a8eb..42e6e5a4 100644 --- a/l2tdevtools/versions.py +++ b/l2tdevtools/versions.py @@ -2,38 +2,38 @@ def CompareVersions(first_version_list, second_version_list): - """Compares two lists containing version parts. + """Compares two lists containing version parts. - Note that the version parts can contain alpha numeric characters. + Note that the version parts can contain alpha numeric characters. - Args: - first_version_list (list[str]): first version parts. - second_version_list (list[str]): second version parts. + Args: + first_version_list (list[str]): first version parts. + second_version_list (list[str]): second version parts. - Returns: - int: 1 if the first is larger than the second, -1 if the first is smaller - than the second, or 0 if the first and second are equal. - """ - first_version_list_length = len(first_version_list) - second_version_list_length = len(second_version_list) + Returns: + int: 1 if the first is larger than the second, -1 if the first is smaller + than the second, or 0 if the first and second are equal. + """ + first_version_list_length = len(first_version_list) + second_version_list_length = len(second_version_list) - for index in range(0, first_version_list_length): - if index >= second_version_list_length: - return 1 + for index in range(0, first_version_list_length): + if index >= second_version_list_length: + return 1 - try: - first_version_part = int(first_version_list[index], 10) - second_version_part = int(second_version_list[index], 10) - except ValueError: - first_version_part = first_version_list[index] - second_version_part = second_version_list[index] + try: + first_version_part = int(first_version_list[index], 10) + second_version_part = int(second_version_list[index], 10) + except ValueError: + first_version_part = first_version_list[index] + second_version_part = second_version_list[index] - if first_version_part > second_version_part: - return 1 - if first_version_part < second_version_part: - return -1 + if first_version_part > second_version_part: + return 1 + if first_version_part < second_version_part: + return -1 - if first_version_list_length < second_version_list_length: - return -1 + if first_version_list_length < second_version_list_length: + return -1 - return 0 + return 0 diff --git a/pyproject.toml b/pyproject.toml index ab70e87f..7e57acd6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,12 @@ Repository = "https://github.com/log2timeline/l2tdevtools" line-length = 88 target-version = ["py310"] include = "\\.pyi?$" +force-exclude = """ +( + data/ + | test_data/linter_fail.py +) +""" [tool.setuptools] package-dir = {"l2tdevtools" = "l2tdevtools"} diff --git a/run_tests.py b/run_tests.py index aa76e2ef..351d3fae 100755 --- a/run_tests.py +++ b/run_tests.py @@ -5,8 +5,8 @@ import sys -if __name__ == '__main__': - test_suite = unittest.TestLoader().discover('tests', pattern='*.py') - test_results = unittest.TextTestRunner(verbosity=2).run(test_suite) - if not test_results.wasSuccessful(): - sys.exit(1) +if __name__ == "__main__": + test_suite = unittest.TestLoader().discover("tests", pattern="*.py") + test_results = unittest.TextTestRunner(verbosity=2).run(test_suite) + if not test_results.wasSuccessful(): + sys.exit(1) diff --git a/test_data/linter_fail.py b/test_data/linter_fail.py index 00883629..c3f7af9e 100644 --- a/test_data/linter_fail.py +++ b/test_data/linter_fail.py @@ -2,10 +2,10 @@ class LinterFailTestClass: - """Linter fail test class.""" + """Linter fail test class.""" - def __init__(self): - """Initializes a linter fail test object.""" - super().__init__() - # Note that the indentation here is deliberately broken. - self._attribute = None + def __init__(self): + """Initializes a linter fail test object.""" + super().__init__() + # Note that the indentation here is deliberately incorrect. + self._attribute = None diff --git a/test_data/linter_pass.py b/test_data/linter_pass.py index 71a37677..22650fb6 100644 --- a/test_data/linter_pass.py +++ b/test_data/linter_pass.py @@ -2,9 +2,9 @@ class LinterPassTestClass: - """Linter pass test class.""" + """Linter pass test class.""" - def __init__(self): - """Initializes a linter pass test object.""" - super().__init__() - self._attribute = None + def __init__(self): + """Initializes a linter pass test object.""" + super().__init__() + self._attribute = None diff --git a/tests/build_helpers/dpkg.py b/tests/build_helpers/dpkg.py index 992a0150..4166a2fb 100644 --- a/tests/build_helpers/dpkg.py +++ b/tests/build_helpers/dpkg.py @@ -11,97 +11,105 @@ class DPKGBuildHelperTest(test_lib.BaseTestCase): - """Tests for the helper to build dpkg packages (.deb).""" + """Tests for the helper to build dpkg packages (.deb).""" - # pylint: disable=protected-access + # pylint: disable=protected-access - # TODO: add tests for _BuildPrepare - # TODO: add tests for _BuildFinalize - # TODO: add tests for _CheckIsInstalled - # TODO: add tests for _CreateOriginalSourcePackage - # TODO: add tests for _CreateOriginalSourcePackageFromZip - # TODO: add tests for _CreatePackagingFiles - # TODO: add tests for _GetBuildHostDistribution + # TODO: add tests for _BuildPrepare + # TODO: add tests for _BuildFinalize + # TODO: add tests for _CheckIsInstalled + # TODO: add tests for _CreateOriginalSourcePackage + # TODO: add tests for _CreateOriginalSourcePackageFromZip + # TODO: add tests for _CreatePackagingFiles + # TODO: add tests for _GetBuildHostDistribution - def testReadLSBReleaseConfigurationFile(self): - """Tests the _ReadLSBReleaseConfigurationFile function.""" - test_path = self._GetTestFilePath(['lsb-release']) - self._SkipIfPathNotExists(test_path) + def testReadLSBReleaseConfigurationFile(self): + """Tests the _ReadLSBReleaseConfigurationFile function.""" + test_path = self._GetTestFilePath(["lsb-release"]) + self._SkipIfPathNotExists(test_path) - project_definition = projects.ProjectDefinition('test') + project_definition = projects.ProjectDefinition("test") - l2tdevtools_path = os.path.dirname(os.path.dirname(os.path.dirname( - os.path.abspath(__file__)))) + l2tdevtools_path = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + ) - test_build_helper = dpkg.DPKGBuildHelper( - project_definition, l2tdevtools_path, {}) - lsb_release_values = test_build_helper._ReadLSBReleaseConfigurationFile( - test_path) + test_build_helper = dpkg.DPKGBuildHelper( + project_definition, l2tdevtools_path, {} + ) + lsb_release_values = test_build_helper._ReadLSBReleaseConfigurationFile( + test_path + ) - self.assertEqual(len(lsb_release_values), 4) + self.assertEqual(len(lsb_release_values), 4) - expected_keys = [ - 'distrib_codename', 'distrib_description', 'distrib_id', - 'distrib_release'] + expected_keys = [ + "distrib_codename", + "distrib_description", + "distrib_id", + "distrib_release", + ] - self.assertEqual(sorted(lsb_release_values.keys()), expected_keys) + self.assertEqual(sorted(lsb_release_values.keys()), expected_keys) - # TODO: add tests for _RemoveOlderDPKGPackages - # TODO: add tests for _RemoveOlderOriginalSourcePackage - # TODO: add tests for _RemoveOlderSourceDPKGPackages + # TODO: add tests for _RemoveOlderDPKGPackages + # TODO: add tests for _RemoveOlderOriginalSourcePackage + # TODO: add tests for _RemoveOlderSourceDPKGPackages - def testRunLSBReleaseCommand(self): - """Tests the _RunLSBReleaseCommand function.""" - project_definition = projects.ProjectDefinition('test') + def testRunLSBReleaseCommand(self): + """Tests the _RunLSBReleaseCommand function.""" + project_definition = projects.ProjectDefinition("test") - l2tdevtools_path = os.path.dirname(os.path.dirname(os.path.dirname( - os.path.abspath(__file__)))) + l2tdevtools_path = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + ) - test_build_helper = dpkg.DPKGBuildHelper( - project_definition, l2tdevtools_path, {}) - output = test_build_helper._RunLSBReleaseCommand() + test_build_helper = dpkg.DPKGBuildHelper( + project_definition, l2tdevtools_path, {} + ) + output = test_build_helper._RunLSBReleaseCommand() - if os.path.exists('/usr/bin/lsb_release'): - self.assertIsNotNone(output) - else: - self.assertIsNone(output) + if os.path.exists("/usr/bin/lsb_release"): + self.assertIsNotNone(output) + else: + self.assertIsNone(output) - # TODO: add tests for CheckBuildDependencies + # TODO: add tests for CheckBuildDependencies class ConfigureMakeDPKGBuildHelperTest(test_lib.BaseTestCase): - """Tests for the helper to build dpkg packages (.deb).""" + """Tests for the helper to build dpkg packages (.deb).""" - # TODO: add tests for Build - # TODO: add tests for CheckBuildRequired - # TODO: add tests for Clean + # TODO: add tests for Build + # TODO: add tests for CheckBuildRequired + # TODO: add tests for Clean class ConfigureMakeSourceDPKGBuildHelperTest(test_lib.BaseTestCase): - """Tests for the helper to build source dpkg packages (.deb).""" + """Tests for the helper to build source dpkg packages (.deb).""" - # TODO: add tests for Build - # TODO: add tests for CheckBuildRequired - # TODO: add tests for Clean + # TODO: add tests for Build + # TODO: add tests for CheckBuildRequired + # TODO: add tests for Clean class PybuildDPKGBuildHelperTest(test_lib.BaseTestCase): - """Tests for the helper to build dpkg packages (.deb).""" + """Tests for the helper to build dpkg packages (.deb).""" - # TODO: add tests for _GetFilenameSafeProjectInformation - # TODO: add tests for Build - # TODO: add tests for CheckBuildRequired - # TODO: add tests for Clean + # TODO: add tests for _GetFilenameSafeProjectInformation + # TODO: add tests for Build + # TODO: add tests for CheckBuildRequired + # TODO: add tests for Clean class PybuildSourceDPKGBuildHelperTest(test_lib.BaseTestCase): - """Tests for the helper to build source dpkg packages (.deb).""" + """Tests for the helper to build source dpkg packages (.deb).""" - # TODO: add tests for _GetFilenameSafeProjectInformation - # TODO: add tests for Build - # TODO: add tests for CheckBuildRequired - # TODO: add tests for Clean + # TODO: add tests for _GetFilenameSafeProjectInformation + # TODO: add tests for Build + # TODO: add tests for CheckBuildRequired + # TODO: add tests for Clean -if __name__ == '__main__': - unittest.main() +if __name__ == "__main__": + unittest.main() diff --git a/tests/build_helpers/factory.py b/tests/build_helpers/factory.py index 6daa1650..794ebed1 100644 --- a/tests/build_helpers/factory.py +++ b/tests/build_helpers/factory.py @@ -11,28 +11,32 @@ class BuildHelperFactoryTest(test_lib.BaseTestCase): - """Tests the factory class for build helpers.""" + """Tests the factory class for build helpers.""" - def testNewBuildHelper(self): - """Tests the NewBuildHelper function.""" - project_definition = projects.ProjectDefinition('test') + def testNewBuildHelper(self): + """Tests the NewBuildHelper function.""" + project_definition = projects.ProjectDefinition("test") - l2tdevtools_path = os.path.dirname(os.path.dirname(os.path.dirname( - os.path.abspath(__file__)))) + l2tdevtools_path = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + ) - build_helper = factory.BuildHelperFactory.NewBuildHelper( - project_definition, 'source', l2tdevtools_path, {}) - self.assertIsNone(build_helper) + build_helper = factory.BuildHelperFactory.NewBuildHelper( + project_definition, "source", l2tdevtools_path, {} + ) + self.assertIsNone(build_helper) - project_definition.build_system = 'setup_py' - build_helper = factory.BuildHelperFactory.NewBuildHelper( - project_definition, 'source', l2tdevtools_path, {}) - self.assertIsNotNone(build_helper) + project_definition.build_system = "setup_py" + build_helper = factory.BuildHelperFactory.NewBuildHelper( + project_definition, "source", l2tdevtools_path, {} + ) + self.assertIsNotNone(build_helper) - build_helper = factory.BuildHelperFactory.NewBuildHelper( - project_definition, 'bogus', l2tdevtools_path, {}) - self.assertIsNone(build_helper) + build_helper = factory.BuildHelperFactory.NewBuildHelper( + project_definition, "bogus", l2tdevtools_path, {} + ) + self.assertIsNone(build_helper) -if __name__ == '__main__': - unittest.main() +if __name__ == "__main__": + unittest.main() diff --git a/tests/build_helpers/interface.py b/tests/build_helpers/interface.py index d0e4b088..bd93a99d 100644 --- a/tests/build_helpers/interface.py +++ b/tests/build_helpers/interface.py @@ -10,24 +10,24 @@ class BuildHelperTest(test_lib.BaseTestCase): - """Tests for the helper to build projects from source.""" + """Tests for the helper to build projects from source.""" - def testCheckBuildDependencies(self): - """Tests the CheckBuildDependencies function.""" - project_definition = projects.ProjectDefinition('test') - build_helper = interface.BuildHelper(project_definition, '', {}) + def testCheckBuildDependencies(self): + """Tests the CheckBuildDependencies function.""" + project_definition = projects.ProjectDefinition("test") + build_helper = interface.BuildHelper(project_definition, "", {}) - build_dependencies = build_helper.CheckBuildDependencies() - self.assertEqual(build_dependencies, []) + build_dependencies = build_helper.CheckBuildDependencies() + self.assertEqual(build_dependencies, []) - def testCheckBuildRequired(self): - """Tests the CheckBuildRequired function.""" - project_definition = projects.ProjectDefinition('test') - build_helper = interface.BuildHelper(project_definition, '', {}) + def testCheckBuildRequired(self): + """Tests the CheckBuildRequired function.""" + project_definition = projects.ProjectDefinition("test") + build_helper = interface.BuildHelper(project_definition, "", {}) - result = build_helper.CheckBuildRequired(None) - self.assertTrue(result) + result = build_helper.CheckBuildRequired(None) + self.assertTrue(result) -if __name__ == '__main__': - unittest.main() +if __name__ == "__main__": + unittest.main() diff --git a/tests/build_helpers/rpm.py b/tests/build_helpers/rpm.py index 1db7e629..f6af4bfb 100644 --- a/tests/build_helpers/rpm.py +++ b/tests/build_helpers/rpm.py @@ -14,5 +14,5 @@ # TODO: add PyprojectSRPMBuildHelper tests. -if __name__ == '__main__': - unittest.main() +if __name__ == "__main__": + unittest.main() diff --git a/tests/build_helpers/source.py b/tests/build_helpers/source.py index d3336230..10c5fb81 100644 --- a/tests/build_helpers/source.py +++ b/tests/build_helpers/source.py @@ -9,5 +9,5 @@ # TODO: add SetupPySourceBuildHelper tests. -if __name__ == '__main__': - unittest.main() +if __name__ == "__main__": + unittest.main() diff --git a/tests/build_helpers/test_lib.py b/tests/build_helpers/test_lib.py index 8957ec3d..35f3fca7 100644 --- a/tests/build_helpers/test_lib.py +++ b/tests/build_helpers/test_lib.py @@ -8,126 +8,132 @@ class TestSourceHelper(source_helper.SourceHelper): - """Test helper to manage project source code.""" - - def __init__(self, project_name, project_definition, project_version): - """Initializes a source helper. - - Args: - project_name (str): name of the project. - project_definition (ProjectDefinition): project definition. - project_version (str): version of the project source code. - """ - super().__init__(project_name, project_definition) - self._project_version = project_version - self._source_directory_path = f'{project_name:s}-{project_version!s}' - self._source_package_filename = ( - f'{project_name:s}-{project_version!s}.tar.gz') - - # pylint: disable=redundant-returns-doc + """Test helper to manage project source code.""" + + def __init__(self, project_name, project_definition, project_version): + """Initializes a source helper. + + Args: + project_name (str): name of the project. + project_definition (ProjectDefinition): project definition. + project_version (str): version of the project source code. + """ + super().__init__(project_name, project_definition) + self._project_version = project_version + self._source_directory_path = f"{project_name:s}-{project_version!s}" + self._source_package_filename = f"{project_name:s}-{project_version!s}.tar.gz" + + # pylint: disable=redundant-returns-doc + + def _CreateFromTar(self, source_filename): + """Creates the source directory from a .tar source package. + + Args: + source_filename (str): filename of the source package. + + Returns: + str: name of the source directory or None if no files can be extracted + from the .tar.gz source package. + """ + with tarfile.open(source_filename, "r:*", encoding="utf-8") as archive: + directory_name = "" + + for tar_info in archive.getmembers(): + filename = getattr(tar_info, "name", None) + + if isinstance(filename, bytes): + try: + filename = filename.decode("utf8") + except UnicodeDecodeError: + logging.warning( + f"Unable to decode filename in tar file: " + f"{source_filename:s}" + ) + continue + + if filename is None: + logging.warning( + f"Missing filename in tar file: {source_filename:s}" + ) + continue + + if not directory_name: + # Note that this will set directory name to an empty string + # if filename start with a /. + directory_name, _, _ = filename.partition("/") + if not directory_name or directory_name.startswith(".."): + logging.error( + f"Unsupported directory name in tar file: " + f"{source_filename:s}" + ) + return None + + if os.path.exists(directory_name): + break + + logging.info(f"Extracting: {source_filename:s}") + + elif not filename.startswith(directory_name): + logging.warning( + f"Skipping: {filename:s} in tar file: {source_filename:s}" + ) + continue + + archive.extract(tar_info) + + return directory_name + + def Create(self): + """Creates the source directory. + + Returns: + str: name of the source directory or None on error. + """ + # TODO: use shutil.unpack_archive(test_path, temp_directory) when Python 2 + # support has been removed. + + return self._CreateFromTar(self._source_package_filename) + + def GetProjectIdentifier(self): + """Retrieves the project identifier for a given project name. + + Returns: + str: project identifier or None on error. + """ + return f"com.github.log2timeline.{self.project_name:s}" + + def GetProjectVersion(self): + """Retrieves the version number for a given project name. + + Returns: + str: version number or None on error. + """ + return self._project_version + + def GetSourceDirectoryPath(self): + """Retrieves the path of the source directory. + + Returns: + str: path of the source directory or None if not available. + """ + return self._source_directory_path + + def GetSourcePackageFilename(self): + """Retrieves the filename of the source package. + + This function downloads the source package if not done so previously. - def _CreateFromTar(self, source_filename): - """Creates the source directory from a .tar source package. - - Args: - source_filename (str): filename of the source package. - - Returns: - str: name of the source directory or None if no files can be extracted - from the .tar.gz source package. - """ - with tarfile.open(source_filename, 'r:*', encoding='utf-8') as archive: - directory_name = '' - - for tar_info in archive.getmembers(): - filename = getattr(tar_info, 'name', None) - - if isinstance(filename, bytes): - try: - filename = filename.decode('utf8') - except UnicodeDecodeError: - logging.warning( - f'Unable to decode filename in tar file: {source_filename:s}') - continue - - if filename is None: - logging.warning(f'Missing filename in tar file: {source_filename:s}') - continue - - if not directory_name: - # Note that this will set directory name to an empty string - # if filename start with a /. - directory_name, _, _ = filename.partition('/') - if not directory_name or directory_name.startswith('..'): - logging.error( - f'Unsupported directory name in tar file: {source_filename:s}') - return None - - if os.path.exists(directory_name): - break - - logging.info(f'Extracting: {source_filename:s}') - - elif not filename.startswith(directory_name): - logging.warning( - f'Skipping: {filename:s} in tar file: {source_filename:s}') - continue - - archive.extract(tar_info) - - return directory_name - - def Create(self): - """Creates the source directory. - - Returns: - str: name of the source directory or None on error. - """ - # TODO: use shutil.unpack_archive(test_path, temp_directory) when Python 2 - # support has been removed. - - return self._CreateFromTar(self._source_package_filename) - - def GetProjectIdentifier(self): - """Retrieves the project identifier for a given project name. - - Returns: - str: project identifier or None on error. - """ - return f'com.github.log2timeline.{self.project_name:s}' - - def GetProjectVersion(self): - """Retrieves the version number for a given project name. - - Returns: - str: version number or None on error. - """ - return self._project_version - - def GetSourceDirectoryPath(self): - """Retrieves the path of the source directory. - - Returns: - str: path of the source directory or None if not available. - """ - return self._source_directory_path - - def GetSourcePackageFilename(self): - """Retrieves the filename of the source package. - - This function downloads the source package if not done so previously. - - Returns: - str: filename of the source package or None if not available. - """ - return self._source_package_filename - - def GetSourcePackagePath(self): - """Retrieves the path of the source package. - - This function downloads the source package if not done so previously. - - Returns: - str: path of the source package or None if not available. - """ - return self._source_package_filename + Returns: + str: filename of the source package or None if not available. + """ + return self._source_package_filename + + def GetSourcePackagePath(self): + """Retrieves the path of the source package. + + This function downloads the source package if not done so previously. + + Returns: + str: path of the source package or None if not available. + """ + return self._source_package_filename diff --git a/tests/build_helpers/wheel.py b/tests/build_helpers/wheel.py index 7209f3f8..0bec76f4 100644 --- a/tests/build_helpers/wheel.py +++ b/tests/build_helpers/wheel.py @@ -13,171 +13,189 @@ class WheelBuildHelperTest(shared_test_lib.BaseTestCase): - """Tests for the helper to build Python wheel packages (.whl).""" + """Tests for the helper to build Python wheel packages (.whl).""" - # pylint: disable=protected-access + # pylint: disable=protected-access - _TEST_PROJECT_NAME = 'dfdatetime' - _TEST_PROJECT_VERSION = '20190517' + _TEST_PROJECT_NAME = "dfdatetime" + _TEST_PROJECT_VERSION = "20190517" - def testInitialize(self): - """Tests the __init__ function.""" - project_definition = projects.ProjectDefinition('test') + def testInitialize(self): + """Tests the __init__ function.""" + project_definition = projects.ProjectDefinition("test") - l2tdevtools_path = os.path.dirname(os.path.dirname(os.path.dirname( - os.path.abspath(__file__)))) + l2tdevtools_path = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + ) - test_build_helper = wheel.WheelBuildHelper( - project_definition, l2tdevtools_path, {}) - self.assertIsNotNone(test_build_helper) + test_build_helper = wheel.WheelBuildHelper( + project_definition, l2tdevtools_path, {} + ) + self.assertIsNotNone(test_build_helper) - def testGetWheelFilenameProjectInformation(self): - """Tests the _GetWheelFilenameProjectInformation function.""" - project_definition = projects.ProjectDefinition(self._TEST_PROJECT_NAME) + def testGetWheelFilenameProjectInformation(self): + """Tests the _GetWheelFilenameProjectInformation function.""" + project_definition = projects.ProjectDefinition(self._TEST_PROJECT_NAME) - l2tdevtools_path = os.path.dirname(os.path.dirname(os.path.dirname( - os.path.abspath(__file__)))) + l2tdevtools_path = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + ) - test_build_helper = wheel.WheelBuildHelper( - project_definition, l2tdevtools_path, {}) + test_build_helper = wheel.WheelBuildHelper( + project_definition, l2tdevtools_path, {} + ) - source_helper_object = test_lib.TestSourceHelper( - self._TEST_PROJECT_NAME, project_definition, self._TEST_PROJECT_VERSION) + source_helper_object = test_lib.TestSourceHelper( + self._TEST_PROJECT_NAME, project_definition, self._TEST_PROJECT_VERSION + ) - project_name, project_version = ( - test_build_helper._GetWheelFilenameProjectInformation( - source_helper_object)) - self.assertEqual(project_name, self._TEST_PROJECT_NAME) - self.assertEqual(project_version, self._TEST_PROJECT_VERSION) + project_name, project_version = ( + test_build_helper._GetWheelFilenameProjectInformation(source_helper_object) + ) + self.assertEqual(project_name, self._TEST_PROJECT_NAME) + self.assertEqual(project_version, self._TEST_PROJECT_VERSION) - # TODO: add tests for _MoveWheel + # TODO: add tests for _MoveWheel - def testCheckBuildDependencies(self): - """Tests the CheckBuildDependencies function.""" - project_definition = projects.ProjectDefinition(self._TEST_PROJECT_NAME) - project_definition.build_dependencies = [] + def testCheckBuildDependencies(self): + """Tests the CheckBuildDependencies function.""" + project_definition = projects.ProjectDefinition(self._TEST_PROJECT_NAME) + project_definition.build_dependencies = [] - l2tdevtools_path = os.path.dirname(os.path.dirname(os.path.dirname( - os.path.abspath(__file__)))) + l2tdevtools_path = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + ) - test_build_helper = wheel.WheelBuildHelper( - project_definition, l2tdevtools_path, {}) + test_build_helper = wheel.WheelBuildHelper( + project_definition, l2tdevtools_path, {} + ) - missing_packages = test_build_helper.CheckBuildDependencies() - self.assertEqual(missing_packages, []) + missing_packages = test_build_helper.CheckBuildDependencies() + self.assertEqual(missing_packages, []) - def testCheckBuildRequired(self): - """Tests the CheckBuildRequired function.""" - project_definition = projects.ProjectDefinition(self._TEST_PROJECT_NAME) - project_definition.build_dependencies = [] + def testCheckBuildRequired(self): + """Tests the CheckBuildRequired function.""" + project_definition = projects.ProjectDefinition(self._TEST_PROJECT_NAME) + project_definition.build_dependencies = [] - l2tdevtools_path = os.path.dirname(os.path.dirname(os.path.dirname( - os.path.abspath(__file__)))) + l2tdevtools_path = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + ) - test_build_helper = wheel.WheelBuildHelper( - project_definition, l2tdevtools_path, {}) + test_build_helper = wheel.WheelBuildHelper( + project_definition, l2tdevtools_path, {} + ) - source_helper_object = test_lib.TestSourceHelper( - self._TEST_PROJECT_NAME, project_definition, self._TEST_PROJECT_VERSION) + source_helper_object = test_lib.TestSourceHelper( + self._TEST_PROJECT_NAME, project_definition, self._TEST_PROJECT_VERSION + ) - result = test_build_helper.CheckBuildRequired(source_helper_object) - self.assertTrue(result) + result = test_build_helper.CheckBuildRequired(source_helper_object) + self.assertTrue(result) - def testClean(self): - """Tests the Clean function.""" - project_definition = projects.ProjectDefinition(self._TEST_PROJECT_NAME) - project_definition.build_dependencies = [] + def testClean(self): + """Tests the Clean function.""" + project_definition = projects.ProjectDefinition(self._TEST_PROJECT_NAME) + project_definition.build_dependencies = [] - l2tdevtools_path = os.path.dirname(os.path.dirname(os.path.dirname( - os.path.abspath(__file__)))) + l2tdevtools_path = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + ) - test_build_helper = wheel.WheelBuildHelper( - project_definition, l2tdevtools_path, {}) + test_build_helper = wheel.WheelBuildHelper( + project_definition, l2tdevtools_path, {} + ) - source_helper_object = test_lib.TestSourceHelper( - self._TEST_PROJECT_NAME, project_definition, self._TEST_PROJECT_VERSION) + source_helper_object = test_lib.TestSourceHelper( + self._TEST_PROJECT_NAME, project_definition, self._TEST_PROJECT_VERSION + ) - with shared_test_lib.TempDirectory() as temp_directory: - test_filename = f'{self._TEST_PROJECT_NAME:s}-20180101-py3-none-any.whl' - test_path = os.path.join(temp_directory, test_filename) + with shared_test_lib.TempDirectory() as temp_directory: + test_filename = f"{self._TEST_PROJECT_NAME:s}-20180101-py3-none-any.whl" + test_path = os.path.join(temp_directory, test_filename) - with open(test_path, 'a', encoding='utf-8'): - pass + with open(test_path, "a", encoding="utf-8"): + pass - test_wheel_filename = ( - f'{self._TEST_PROJECT_NAME:s}-{self._TEST_PROJECT_VERSION:s}-py3-' - f'none-any.whl') + test_wheel_filename = ( + f"{self._TEST_PROJECT_NAME:s}-{self._TEST_PROJECT_VERSION:s}-py3-" + f"none-any.whl" + ) - test_path = os.path.join(temp_directory, test_wheel_filename) - with open(test_path, 'a', encoding='utf-8'): - pass + test_path = os.path.join(temp_directory, test_wheel_filename) + with open(test_path, "a", encoding="utf-8"): + pass - directory_entries = os.listdir(temp_directory) - self.assertEqual(len(directory_entries), 2) + directory_entries = os.listdir(temp_directory) + self.assertEqual(len(directory_entries), 2) - current_working_directory = os.getcwd() - os.chdir(temp_directory) + current_working_directory = os.getcwd() + os.chdir(temp_directory) - try: - test_build_helper.Clean(source_helper_object) - finally: - os.chdir(current_working_directory) + try: + test_build_helper.Clean(source_helper_object) + finally: + os.chdir(current_working_directory) - directory_entries = os.listdir(temp_directory) - self.assertEqual(len(directory_entries), 1) - self.assertIn(test_wheel_filename, directory_entries) + directory_entries = os.listdir(temp_directory) + self.assertEqual(len(directory_entries), 1) + self.assertIn(test_wheel_filename, directory_entries) class ConfigureMakeWheelBuildHelperTest(shared_test_lib.BaseTestCase): - """Tests for the helper to build Python wheel packages (.whl).""" + """Tests for the helper to build Python wheel packages (.whl).""" - _TEST_PROJECT_NAME = 'libsigscan' - _TEST_PROJECT_VERSION = '20231201' + _TEST_PROJECT_NAME = "libsigscan" + _TEST_PROJECT_VERSION = "20231201" - def testBuild(self): - """Tests the Build function.""" - project_definition = projects.ProjectDefinition(self._TEST_PROJECT_NAME) - project_definition.build_dependencies = [] - project_definition.wheel_name = 'libsigscan_python' + def testBuild(self): + """Tests the Build function.""" + project_definition = projects.ProjectDefinition(self._TEST_PROJECT_NAME) + project_definition.build_dependencies = [] + project_definition.wheel_name = "libsigscan_python" - l2tdevtools_path = os.path.dirname(os.path.dirname(os.path.dirname( - os.path.abspath(__file__)))) + l2tdevtools_path = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + ) - test_build_helper = wheel.ConfigureMakeWheelBuildHelper( - project_definition, l2tdevtools_path, {}) + test_build_helper = wheel.ConfigureMakeWheelBuildHelper( + project_definition, l2tdevtools_path, {} + ) - source_helper_object = test_lib.TestSourceHelper( - self._TEST_PROJECT_NAME, project_definition, self._TEST_PROJECT_VERSION) + source_helper_object = test_lib.TestSourceHelper( + self._TEST_PROJECT_NAME, project_definition, self._TEST_PROJECT_VERSION + ) - with shared_test_lib.TempDirectory() as temp_directory: - test_filename = ( - f'{self._TEST_PROJECT_NAME:s}-{self._TEST_PROJECT_VERSION:s}.tar.gz') - test_path = os.path.join(l2tdevtools_path, 'test_data', test_filename) + with shared_test_lib.TempDirectory() as temp_directory: + test_filename = ( + f"{self._TEST_PROJECT_NAME:s}-{self._TEST_PROJECT_VERSION:s}.tar.gz" + ) + test_path = os.path.join(l2tdevtools_path, "test_data", test_filename) - shutil.copy(test_path, temp_directory) + shutil.copy(test_path, temp_directory) - directory_entries = os.listdir(temp_directory) - self.assertEqual(len(directory_entries), 1) + directory_entries = os.listdir(temp_directory) + self.assertEqual(len(directory_entries), 1) - current_working_directory = os.getcwd() - os.chdir(temp_directory) + current_working_directory = os.getcwd() + os.chdir(temp_directory) - try: - source_helper_object.Create() - test_build_helper.Build(source_helper_object) - finally: - os.chdir(current_working_directory) + try: + source_helper_object.Create() + test_build_helper.Build(source_helper_object) + finally: + os.chdir(current_working_directory) - directory_entries = os.listdir(temp_directory) - self.assertIn('build.log', directory_entries) + directory_entries = os.listdir(temp_directory) + self.assertIn("build.log", directory_entries) - if len(directory_entries) < 4: - build_log_path = os.path.join(temp_directory, 'build.log') - with open(build_log_path, 'r', encoding='utf-8') as file_object: - print(''.join(file_object.readlines())) + if len(directory_entries) < 4: + build_log_path = os.path.join(temp_directory, "build.log") + with open(build_log_path, "r", encoding="utf-8") as file_object: + print("".join(file_object.readlines())) - self.assertEqual(len(directory_entries), 4) + self.assertEqual(len(directory_entries), 4) -if __name__ == '__main__': - unittest.main() +if __name__ == "__main__": + unittest.main() diff --git a/tests/dependencies.py b/tests/dependencies.py index 3dc3c2da..3fc0cb5a 100644 --- a/tests/dependencies.py +++ b/tests/dependencies.py @@ -11,193 +11,210 @@ class DependencyDefinitionTest(test_lib.BaseTestCase): - """Tests for the dependency definition.""" + """Tests for the dependency definition.""" - def testInitialize(self): - """Tests the __init__ function.""" - dependency_definition = dependencies.DependencyDefinition('test') - self.assertIsNotNone(dependency_definition) + def testInitialize(self): + """Tests the __init__ function.""" + dependency_definition = dependencies.DependencyDefinition("test") + self.assertIsNotNone(dependency_definition) class DependencyDefinitionReaderTest(test_lib.BaseTestCase): - """Tests for the dependency definition reader.""" + """Tests for the dependency definition reader.""" - # pylint: disable=protected-access + # pylint: disable=protected-access - _TEST_CONFIGURATION_DATA = '\n'.join([ - '[dfdatetime]', - 'dpkg_name: python3-dfdatetime', - 'minimum_version: 20160814', - 'rpm_name: python3-dfdatetime', - 'version_property: __version__']) + _TEST_CONFIGURATION_DATA = "\n".join( + [ + "[dfdatetime]", + "dpkg_name: python3-dfdatetime", + "minimum_version: 20160814", + "rpm_name: python3-dfdatetime", + "version_property: __version__", + ] + ) - def testGetConfigValue(self): - """Tests the _GetConfigValue function.""" - test_reader = dependencies.DependencyDefinitionReader() + def testGetConfigValue(self): + """Tests the _GetConfigValue function.""" + test_reader = dependencies.DependencyDefinitionReader() - file_object = io.StringIO(self._TEST_CONFIGURATION_DATA) - config_parser = configparser.ConfigParser(interpolation=None) - config_parser.read_file(file_object) + file_object = io.StringIO(self._TEST_CONFIGURATION_DATA) + config_parser = configparser.ConfigParser(interpolation=None) + config_parser.read_file(file_object) - configuration_value = test_reader._GetConfigValue( - config_parser, 'dfdatetime', 'dpkg_name') - self.assertEqual(configuration_value, 'python3-dfdatetime') + configuration_value = test_reader._GetConfigValue( + config_parser, "dfdatetime", "dpkg_name" + ) + self.assertEqual(configuration_value, "python3-dfdatetime") - with self.assertRaises(configparser.NoSectionError): - test_reader._GetConfigValue(config_parser, 'bogus', 'dpkg_name') + with self.assertRaises(configparser.NoSectionError): + test_reader._GetConfigValue(config_parser, "bogus", "dpkg_name") - configuration_value = test_reader._GetConfigValue( - config_parser, 'dfdatetime', 'bogus') - self.assertIsNone(configuration_value) + configuration_value = test_reader._GetConfigValue( + config_parser, "dfdatetime", "bogus" + ) + self.assertIsNone(configuration_value) - def testRead(self): - """Tests the Read function.""" - test_reader = dependencies.DependencyDefinitionReader() + def testRead(self): + """Tests the Read function.""" + test_reader = dependencies.DependencyDefinitionReader() - file_object = io.StringIO(self._TEST_CONFIGURATION_DATA) - definitions = list(test_reader.Read(file_object)) + file_object = io.StringIO(self._TEST_CONFIGURATION_DATA) + definitions = list(test_reader.Read(file_object)) - self.assertEqual(len(definitions), 1) + self.assertEqual(len(definitions), 1) class DependencyHelperTest(test_lib.BaseTestCase): - """Tests for the dependency helper.""" + """Tests for the dependency helper.""" - # pylint: disable=protected-access + # pylint: disable=protected-access - def testInitialize(self): - """Tests the __init__ function.""" - dependencies_file = self._GetTestFilePath(['dependencies.ini']) - self._SkipIfPathNotExists(dependencies_file) + def testInitialize(self): + """Tests the __init__ function.""" + dependencies_file = self._GetTestFilePath(["dependencies.ini"]) + self._SkipIfPathNotExists(dependencies_file) - dependency_helper = dependencies.DependencyHelper( - dependencies_file=dependencies_file) - self.assertIsNotNone(dependency_helper) + dependency_helper = dependencies.DependencyHelper( + dependencies_file=dependencies_file + ) + self.assertIsNotNone(dependency_helper) - dependencies_file = self._GetTestFilePath(['bogus.ini']) - with self.assertRaises(IOError): - dependencies.DependencyHelper(dependencies_file=dependencies_file) + dependencies_file = self._GetTestFilePath(["bogus.ini"]) + with self.assertRaises(IOError): + dependencies.DependencyHelper(dependencies_file=dependencies_file) - def testCheckPythonModule(self): - """Tests the _CheckPythonModule function.""" - dependencies_file = self._GetTestFilePath(['dependencies.ini']) - self._SkipIfPathNotExists(dependencies_file) + def testCheckPythonModule(self): + """Tests the _CheckPythonModule function.""" + dependencies_file = self._GetTestFilePath(["dependencies.ini"]) + self._SkipIfPathNotExists(dependencies_file) - dependency_helper = dependencies.DependencyHelper( - dependencies_file=dependencies_file) + dependency_helper = dependencies.DependencyHelper( + dependencies_file=dependencies_file + ) - dependency = dependencies.DependencyDefinition('os') - result, _ = dependency_helper._CheckPythonModule(dependency) - self.assertTrue(result) + dependency = dependencies.DependencyDefinition("os") + result, _ = dependency_helper._CheckPythonModule(dependency) + self.assertTrue(result) - dependency = dependencies.DependencyDefinition('bogus') - result, _ = dependency_helper._CheckPythonModule(dependency) - self.assertFalse(result) + dependency = dependencies.DependencyDefinition("bogus") + result, _ = dependency_helper._CheckPythonModule(dependency) + self.assertFalse(result) - def testCheckPythonModuleVersion(self): - """Tests the _CheckPythonModuleVersion function.""" - dependencies_file = self._GetTestFilePath(['dependencies.ini']) - self._SkipIfPathNotExists(dependencies_file) + def testCheckPythonModuleVersion(self): + """Tests the _CheckPythonModuleVersion function.""" + dependencies_file = self._GetTestFilePath(["dependencies.ini"]) + self._SkipIfPathNotExists(dependencies_file) - dependency_helper = dependencies.DependencyHelper( - dependencies_file=dependencies_file) + dependency_helper = dependencies.DependencyHelper( + dependencies_file=dependencies_file + ) - module_object = dependency_helper._ImportPythonModule('os') + module_object = dependency_helper._ImportPythonModule("os") - result, _ = dependency_helper._CheckPythonModuleVersion( - 'os', module_object, '__version__', '1.0', '2.0') - self.assertFalse(result) + result, _ = dependency_helper._CheckPythonModuleVersion( + "os", module_object, "__version__", "1.0", "2.0" + ) + self.assertFalse(result) - # TODO: add test with version with suffix 17.0.0b1 + # TODO: add test with version with suffix 17.0.0b1 - def testImportPythonModule(self): - """Tests the _ImportPythonModule function.""" - dependencies_file = self._GetTestFilePath(['dependencies.ini']) - self._SkipIfPathNotExists(dependencies_file) + def testImportPythonModule(self): + """Tests the _ImportPythonModule function.""" + dependencies_file = self._GetTestFilePath(["dependencies.ini"]) + self._SkipIfPathNotExists(dependencies_file) - dependency_helper = dependencies.DependencyHelper( - dependencies_file=dependencies_file) + dependency_helper = dependencies.DependencyHelper( + dependencies_file=dependencies_file + ) - module_object = dependency_helper._ImportPythonModule('os') - self.assertIsNotNone(module_object) + module_object = dependency_helper._ImportPythonModule("os") + self.assertIsNotNone(module_object) - # TODO: add test with submodule. + # TODO: add test with submodule. - # TODO: add tests for _PrintCheckDependencyStatus + # TODO: add tests for _PrintCheckDependencyStatus - def testCheckDependencies(self): - """Tests the CheckDependencies function.""" - dependencies_file = self._GetTestFilePath(['dependencies.ini']) - self._SkipIfPathNotExists(dependencies_file) + def testCheckDependencies(self): + """Tests the CheckDependencies function.""" + dependencies_file = self._GetTestFilePath(["dependencies.ini"]) + self._SkipIfPathNotExists(dependencies_file) - dependency_helper = dependencies.DependencyHelper( - dependencies_file=dependencies_file) + dependency_helper = dependencies.DependencyHelper( + dependencies_file=dependencies_file + ) - dependency_helper.CheckDependencies(verbose_output=False) + dependency_helper.CheckDependencies(verbose_output=False) - def testCheckTestDependencies(self): - """Tests the CheckTestDependencies function.""" - dependencies_file = self._GetTestFilePath(['dependencies.ini']) - self._SkipIfPathNotExists(dependencies_file) + def testCheckTestDependencies(self): + """Tests the CheckTestDependencies function.""" + dependencies_file = self._GetTestFilePath(["dependencies.ini"]) + self._SkipIfPathNotExists(dependencies_file) - dependency_helper = dependencies.DependencyHelper( - dependencies_file=dependencies_file) + dependency_helper = dependencies.DependencyHelper( + dependencies_file=dependencies_file + ) - dependency_helper.CheckTestDependencies(verbose_output=False) + dependency_helper.CheckTestDependencies(verbose_output=False) - def testGetDPKGDepends(self): - """Tests the GetDPKGDepends function.""" - dependencies_file = self._GetTestFilePath(['dependencies.ini']) - self._SkipIfPathNotExists(dependencies_file) + def testGetDPKGDepends(self): + """Tests the GetDPKGDepends function.""" + dependencies_file = self._GetTestFilePath(["dependencies.ini"]) + self._SkipIfPathNotExists(dependencies_file) - dependency_helper = dependencies.DependencyHelper( - dependencies_file=dependencies_file) + dependency_helper = dependencies.DependencyHelper( + dependencies_file=dependencies_file + ) - dpkg_depends = dependency_helper.GetDPKGDepends() - self.assertEqual(len(dpkg_depends), 1) + dpkg_depends = dependency_helper.GetDPKGDepends() + self.assertEqual(len(dpkg_depends), 1) - def testGetL2TBinaries(self): - """Tests the GetL2TBinaries function.""" - dependencies_file = self._GetTestFilePath(['dependencies.ini']) - self._SkipIfPathNotExists(dependencies_file) + def testGetL2TBinaries(self): + """Tests the GetL2TBinaries function.""" + dependencies_file = self._GetTestFilePath(["dependencies.ini"]) + self._SkipIfPathNotExists(dependencies_file) - dependency_helper = dependencies.DependencyHelper( - dependencies_file=dependencies_file) + dependency_helper = dependencies.DependencyHelper( + dependencies_file=dependencies_file + ) - l2tbinaries = dependency_helper.GetL2TBinaries() - self.assertEqual(len(l2tbinaries), 1) + l2tbinaries = dependency_helper.GetL2TBinaries() + self.assertEqual(len(l2tbinaries), 1) - def testGetInstallRequires(self): - """Tests the GetInstallRequires function.""" - dependencies_file = self._GetTestFilePath(['dependencies.ini']) - self._SkipIfPathNotExists(dependencies_file) + def testGetInstallRequires(self): + """Tests the GetInstallRequires function.""" + dependencies_file = self._GetTestFilePath(["dependencies.ini"]) + self._SkipIfPathNotExists(dependencies_file) - dependency_helper = dependencies.DependencyHelper( - dependencies_file=dependencies_file) + dependency_helper = dependencies.DependencyHelper( + dependencies_file=dependencies_file + ) - install_requires = dependency_helper.GetInstallRequires() - self.assertEqual(len(install_requires), 1) + install_requires = dependency_helper.GetInstallRequires() + self.assertEqual(len(install_requires), 1) - def testGetPylintRcExtensionPkgs(self): - """Tests the GetPylintRcExtensionPkgs function.""" - dependencies_file = self._GetTestFilePath(['dependencies.ini']) - dependency_helper = dependencies.DependencyHelper( - dependencies_file=dependencies_file) + def testGetPylintRcExtensionPkgs(self): + """Tests the GetPylintRcExtensionPkgs function.""" + dependencies_file = self._GetTestFilePath(["dependencies.ini"]) + dependency_helper = dependencies.DependencyHelper( + dependencies_file=dependencies_file + ) - extension_packages = dependency_helper.GetPylintRcExtensionPkgs() - self.assertEqual(len(extension_packages), 0) + extension_packages = dependency_helper.GetPylintRcExtensionPkgs() + self.assertEqual(len(extension_packages), 0) - def testGetRPMRequires(self): - """Tests the GetRPMRequires function.""" - dependencies_file = self._GetTestFilePath(['dependencies.ini']) - self._SkipIfPathNotExists(dependencies_file) + def testGetRPMRequires(self): + """Tests the GetRPMRequires function.""" + dependencies_file = self._GetTestFilePath(["dependencies.ini"]) + self._SkipIfPathNotExists(dependencies_file) - dependency_helper = dependencies.DependencyHelper( - dependencies_file=dependencies_file) + dependency_helper = dependencies.DependencyHelper( + dependencies_file=dependencies_file + ) - rpm_requires = dependency_helper.GetRPMRequires() - self.assertEqual(len(rpm_requires), 1) + rpm_requires = dependency_helper.GetRPMRequires() + self.assertEqual(len(rpm_requires), 1) -if __name__ == '__main__': - unittest.main() +if __name__ == "__main__": + unittest.main() diff --git a/tests/dependency_writers/appveyor_yml.py b/tests/dependency_writers/appveyor_yml.py index 1fd1f9d0..39c3e677 100644 --- a/tests/dependency_writers/appveyor_yml.py +++ b/tests/dependency_writers/appveyor_yml.py @@ -10,25 +10,27 @@ class AppveyorYMLTest(test_lib.BaseTestCase): - """Tests the appveyor.yml writer.""" + """Tests the appveyor.yml writer.""" - def testInitialize(self): - """Tests that the writer can be initialized.""" + def testInitialize(self): + """Tests that the writer can be initialized.""" - l2tdevtools_path = '/fake/l2tdevtools/' - project_definition = project.ProjectHelper(l2tdevtools_path) - dependencies_file = self._GetTestFilePath(['dependencies.ini']) - test_dependencies_file = self._GetTestFilePath(['test_dependencies.ini']) - dependency_helper = dependencies.DependencyHelper( - dependencies_file=dependencies_file, - test_dependencies_file=test_dependencies_file) + l2tdevtools_path = "/fake/l2tdevtools/" + project_definition = project.ProjectHelper(l2tdevtools_path) + dependencies_file = self._GetTestFilePath(["dependencies.ini"]) + test_dependencies_file = self._GetTestFilePath(["test_dependencies.ini"]) + dependency_helper = dependencies.DependencyHelper( + dependencies_file=dependencies_file, + test_dependencies_file=test_dependencies_file, + ) - writer = appveyor_yml.AppveyorYmlWriter( - l2tdevtools_path, project_definition, dependency_helper) - self.assertIsNotNone(writer) + writer = appveyor_yml.AppveyorYmlWriter( + l2tdevtools_path, project_definition, dependency_helper + ) + self.assertIsNotNone(writer) - # TODO: Add test for the Write method. + # TODO: Add test for the Write method. -if __name__ == '__main__': - unittest.main() +if __name__ == "__main__": + unittest.main() diff --git a/tests/dependency_writers/dependencies_py.py b/tests/dependency_writers/dependencies_py.py index 2a0306fe..85c5b460 100644 --- a/tests/dependency_writers/dependencies_py.py +++ b/tests/dependency_writers/dependencies_py.py @@ -10,25 +10,27 @@ class DependenciesPyWriterTest(test_lib.BaseTestCase): - """Tests the dependencies.py writer.""" + """Tests the dependencies.py writer.""" - def testInitialize(self): - """Tests that the writer can be initialized.""" + def testInitialize(self): + """Tests that the writer can be initialized.""" - l2tdevtools_path = '/fake/l2tdevtools/' - project_definition = project.ProjectHelper(l2tdevtools_path) - dependencies_file = self._GetTestFilePath(['dependencies.ini']) - test_dependencies_file = self._GetTestFilePath(['test_dependencies.ini']) - dependency_helper = dependencies.DependencyHelper( - dependencies_file=dependencies_file, - test_dependencies_file=test_dependencies_file) + l2tdevtools_path = "/fake/l2tdevtools/" + project_definition = project.ProjectHelper(l2tdevtools_path) + dependencies_file = self._GetTestFilePath(["dependencies.ini"]) + test_dependencies_file = self._GetTestFilePath(["test_dependencies.ini"]) + dependency_helper = dependencies.DependencyHelper( + dependencies_file=dependencies_file, + test_dependencies_file=test_dependencies_file, + ) - writer = dependencies_py.DependenciesPyWriter( - l2tdevtools_path, project_definition, dependency_helper) - self.assertIsNotNone(writer) + writer = dependencies_py.DependenciesPyWriter( + l2tdevtools_path, project_definition, dependency_helper + ) + self.assertIsNotNone(writer) - # TODO: Add test for the Write method. + # TODO: Add test for the Write method. -if __name__ == '__main__': - unittest.main() +if __name__ == "__main__": + unittest.main() diff --git a/tests/dependency_writers/dpkg.py b/tests/dependency_writers/dpkg.py index 0d17fd50..dc20bfb2 100644 --- a/tests/dependency_writers/dpkg.py +++ b/tests/dependency_writers/dpkg.py @@ -11,46 +11,50 @@ class DPKGCompatWriterTest(test_lib.BaseTestCase): - """Tests the dpkg compat writer.""" + """Tests the dpkg compat writer.""" - def testInitialize(self): - """Tests that the writer can be initialized.""" + def testInitialize(self): + """Tests that the writer can be initialized.""" - l2tdevtools_path = '/fake/l2tdevtools/' - project_definition = project.ProjectHelper(l2tdevtools_path) - dependencies_file = self._GetTestFilePath(['dependencies.ini']) - test_dependencies_file = self._GetTestFilePath(['test_dependencies.ini']) - dependency_helper = dependencies.DependencyHelper( - dependencies_file=dependencies_file, - test_dependencies_file=test_dependencies_file) + l2tdevtools_path = "/fake/l2tdevtools/" + project_definition = project.ProjectHelper(l2tdevtools_path) + dependencies_file = self._GetTestFilePath(["dependencies.ini"]) + test_dependencies_file = self._GetTestFilePath(["test_dependencies.ini"]) + dependency_helper = dependencies.DependencyHelper( + dependencies_file=dependencies_file, + test_dependencies_file=test_dependencies_file, + ) - writer = dpkg.DPKGCompatWriter( - l2tdevtools_path, project_definition, dependency_helper) - self.assertIsNotNone(writer) + writer = dpkg.DPKGCompatWriter( + l2tdevtools_path, project_definition, dependency_helper + ) + self.assertIsNotNone(writer) - # TODO: Add test for the Write method. + # TODO: Add test for the Write method. class DPKGControlWriterTest(test_lib.BaseTestCase): - """Tests the dpkg control writer.""" + """Tests the dpkg control writer.""" - def testInitialize(self): - """Tests that the writer can be initialized.""" + def testInitialize(self): + """Tests that the writer can be initialized.""" - l2tdevtools_path = '/fake/l2tdevtools/' - project_definition = project.ProjectHelper(l2tdevtools_path) - dependencies_file = self._GetTestFilePath(['dependencies.ini']) - test_dependencies_file = self._GetTestFilePath(['test_dependencies.ini']) - dependency_helper = dependencies.DependencyHelper( - dependencies_file=dependencies_file, - test_dependencies_file=test_dependencies_file) + l2tdevtools_path = "/fake/l2tdevtools/" + project_definition = project.ProjectHelper(l2tdevtools_path) + dependencies_file = self._GetTestFilePath(["dependencies.ini"]) + test_dependencies_file = self._GetTestFilePath(["test_dependencies.ini"]) + dependency_helper = dependencies.DependencyHelper( + dependencies_file=dependencies_file, + test_dependencies_file=test_dependencies_file, + ) - writer = dpkg.DPKGControlWriter( - l2tdevtools_path, project_definition, dependency_helper) - self.assertIsNotNone(writer) + writer = dpkg.DPKGControlWriter( + l2tdevtools_path, project_definition, dependency_helper + ) + self.assertIsNotNone(writer) - # TODO: Add test for the Write method. + # TODO: Add test for the Write method. -if __name__ == '__main__': - unittest.main() +if __name__ == "__main__": + unittest.main() diff --git a/tests/dependency_writers/gift_copr.py b/tests/dependency_writers/gift_copr.py index 02e217f3..09efa990 100644 --- a/tests/dependency_writers/gift_copr.py +++ b/tests/dependency_writers/gift_copr.py @@ -11,96 +11,107 @@ class GIFTCOPRInstallTest(test_lib.BaseTestCase): - """Tests the gift_copr_install.py writer.""" - - # pylint: disable=protected-access - - def _CreateTestWriter(self): - """Creates a dependency file writer for testing. - - Returns: - GIFTCOPRInstallScriptWriter: dependency file writer for testing. - """ - project_definition = projects.ProjectDefinition('test') - - dependencies_file = self._GetTestFilePath(['dependencies.ini']) - test_dependencies_file = self._GetTestFilePath(['test_dependencies.ini']) - dependency_helper = dependencies.DependencyHelper( - dependencies_file=dependencies_file, - test_dependencies_file=test_dependencies_file) - - return gift_copr.GIFTCOPRInstallScriptWriter( - '/fake/l2tdevtools/', project_definition, dependency_helper) - - def testFormatRPMDebugDependencies(self): - """Tests the _FormatRPMDebugDependencies function.""" - test_writer = self._CreateTestWriter() - - expected_formatted_debug_dependencies = '' - - python_dependencies = test_writer._GetRPMPythonDependencies() - debug_dependencies = test_writer._GetRPMDebugDependencies( - python_dependencies) - formatted_debug_dependencies = test_writer._FormatRPMDebugDependencies( - debug_dependencies) - self.assertEqual( - formatted_debug_dependencies, expected_formatted_debug_dependencies) - - def testFormatRPMDevelopmentDependencies(self): - """Tests the _FormatRPMDevelopmentDependencies function.""" - test_writer = self._CreateTestWriter() - - expected_formatted_development_dependencies = ( - 'DEVELOPMENT_DEPENDENCIES="pylint";') - - development_dependencies = ['pylint'] - formatted_development_dependencies = ( - test_writer._FormatRPMDevelopmentDependencies(development_dependencies)) - self.assertEqual( - formatted_development_dependencies, - expected_formatted_development_dependencies) - - def testFormatRPMPythonDependencies(self): - """Tests the _FormatRPMPythonDependencies function.""" - test_writer = self._CreateTestWriter() - - expected_formatted_python_dependencies = ( - 'PYTHON3_DEPENDENCIES="python3-pyyaml";') - - python_dependencies = test_writer._GetRPMPythonDependencies() - formatted_python_dependencies = test_writer._FormatRPMPythonDependencies( - python_dependencies) - self.assertEqual( - formatted_python_dependencies, expected_formatted_python_dependencies) - - def testFormatRPMTestDependencies(self): - """Tests the _FormatRPMTestDependencies function.""" - test_writer = self._CreateTestWriter() - - expected_formatted_test_dependencies = ( - 'TEST_DEPENDENCIES="python3-pbr\n' - ' python3-setuptools";') - - python_dependencies = test_writer._GetRPMPythonDependencies() - test_dependencies = test_writer._GetRPMTestDependencies(python_dependencies) - formatted_test_dependencies = test_writer._FormatRPMTestDependencies( - test_dependencies) - self.assertEqual( - formatted_test_dependencies, expected_formatted_test_dependencies) - - def testGetRPMDebugDependencies(self): - """Tests the _GetRPMDebugDependencies function.""" - test_writer = self._CreateTestWriter() - - expected_debug_dependencies = [] - - python_dependencies = test_writer._GetRPMPythonDependencies() - debug_dependencies = test_writer._GetRPMDebugDependencies( - python_dependencies) - self.assertEqual(debug_dependencies, expected_debug_dependencies) - - # TODO: Add tests for the Write method. - - -if __name__ == '__main__': - unittest.main() + """Tests the gift_copr_install.py writer.""" + + # pylint: disable=protected-access + + def _CreateTestWriter(self): + """Creates a dependency file writer for testing. + + Returns: + GIFTCOPRInstallScriptWriter: dependency file writer for testing. + """ + project_definition = projects.ProjectDefinition("test") + + dependencies_file = self._GetTestFilePath(["dependencies.ini"]) + test_dependencies_file = self._GetTestFilePath(["test_dependencies.ini"]) + dependency_helper = dependencies.DependencyHelper( + dependencies_file=dependencies_file, + test_dependencies_file=test_dependencies_file, + ) + + return gift_copr.GIFTCOPRInstallScriptWriter( + "/fake/l2tdevtools/", project_definition, dependency_helper + ) + + def testFormatRPMDebugDependencies(self): + """Tests the _FormatRPMDebugDependencies function.""" + test_writer = self._CreateTestWriter() + + expected_formatted_debug_dependencies = "" + + python_dependencies = test_writer._GetRPMPythonDependencies() + debug_dependencies = test_writer._GetRPMDebugDependencies(python_dependencies) + formatted_debug_dependencies = test_writer._FormatRPMDebugDependencies( + debug_dependencies + ) + self.assertEqual( + formatted_debug_dependencies, expected_formatted_debug_dependencies + ) + + def testFormatRPMDevelopmentDependencies(self): + """Tests the _FormatRPMDevelopmentDependencies function.""" + test_writer = self._CreateTestWriter() + + expected_formatted_development_dependencies = ( + 'DEVELOPMENT_DEPENDENCIES="pylint";' + ) + + development_dependencies = ["pylint"] + formatted_development_dependencies = ( + test_writer._FormatRPMDevelopmentDependencies(development_dependencies) + ) + self.assertEqual( + formatted_development_dependencies, + expected_formatted_development_dependencies, + ) + + def testFormatRPMPythonDependencies(self): + """Tests the _FormatRPMPythonDependencies function.""" + test_writer = self._CreateTestWriter() + + expected_formatted_python_dependencies = ( + 'PYTHON3_DEPENDENCIES="python3-pyyaml";' + ) + + python_dependencies = test_writer._GetRPMPythonDependencies() + formatted_python_dependencies = test_writer._FormatRPMPythonDependencies( + python_dependencies + ) + self.assertEqual( + formatted_python_dependencies, expected_formatted_python_dependencies + ) + + def testFormatRPMTestDependencies(self): + """Tests the _FormatRPMTestDependencies function.""" + test_writer = self._CreateTestWriter() + + expected_formatted_test_dependencies = ( + "TEST_DEPENDENCIES=\"python3-pbr\n" + " python3-setuptools\";" + ) + + python_dependencies = test_writer._GetRPMPythonDependencies() + test_dependencies = test_writer._GetRPMTestDependencies(python_dependencies) + formatted_test_dependencies = test_writer._FormatRPMTestDependencies( + test_dependencies + ) + self.assertEqual( + formatted_test_dependencies, expected_formatted_test_dependencies + ) + + def testGetRPMDebugDependencies(self): + """Tests the _GetRPMDebugDependencies function.""" + test_writer = self._CreateTestWriter() + + expected_debug_dependencies = [] + + python_dependencies = test_writer._GetRPMPythonDependencies() + debug_dependencies = test_writer._GetRPMDebugDependencies(python_dependencies) + self.assertEqual(debug_dependencies, expected_debug_dependencies) + + # TODO: Add tests for the Write method. + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/dependency_writers/gift_ppa.py b/tests/dependency_writers/gift_ppa.py index 5d2275d3..d19ddc07 100644 --- a/tests/dependency_writers/gift_ppa.py +++ b/tests/dependency_writers/gift_ppa.py @@ -11,99 +11,106 @@ class GIFTPPAInstallScriptWriterTest(test_lib.BaseTestCase): - """Tests the hared functionality for GIFT PPA installation script writer.""" - - # pylint: disable=protected-access - - def _CreateTestWriter(self): - """Creates a dependency file writer for testing. - - Returns: - GIFTPPAInstallScriptWriter: dependency file writer for testing. - """ - project_definition = projects.ProjectDefinition('test') - dependencies_file = self._GetTestFilePath(['dependencies.ini']) - test_dependencies_file = self._GetTestFilePath(['test_dependencies.ini']) - dependency_helper = dependencies.DependencyHelper( - dependencies_file=dependencies_file, - test_dependencies_file=test_dependencies_file) - - return gift_ppa.GIFTPPAInstallScriptWriter( - '/fake/l2tdevtools/', project_definition, dependency_helper) - - def testFormatDPKGDebugDependencies(self): - """Tests the _FormatDPKGDebugDependencies function.""" - test_writer = self._CreateTestWriter() - - expected_formatted_debug_dependencies = '' - - python_dependencies = test_writer._GetDPKGPythonDependencies() - debug_dependencies = test_writer._GetDPKGDebugDependencies( - python_dependencies) - formatted_debug_dependencies = test_writer._FormatDPKGDebugDependencies( - debug_dependencies) - self.assertEqual( - formatted_debug_dependencies, expected_formatted_debug_dependencies) - - def testFormatDPKGDevelopmentDependencies(self): - """Tests the _FormatDPKGDevelopmentDependencies function.""" - test_writer = self._CreateTestWriter() - - expected_formatted_development_dependencies = ( - 'DEVELOPMENT_DEPENDENCIES="pylint";') - - development_dependencies = ['pylint'] - formatted_development_dependencies = ( - test_writer._FormatDPKGDevelopmentDependencies( - development_dependencies)) - self.assertEqual( - formatted_development_dependencies, - expected_formatted_development_dependencies) - - def testFormatDPKGPythonDependencies(self): - """Tests the _FormatDPKGPythonDependencies function.""" - test_writer = self._CreateTestWriter() - - expected_formatted_python_dependencies = ( - 'PYTHON_DEPENDENCIES="python3-yaml";') - - python_dependencies = test_writer._GetDPKGPythonDependencies() - formatted_python_dependencies = test_writer._FormatDPKGPythonDependencies( - python_dependencies) - self.assertEqual( - formatted_python_dependencies, expected_formatted_python_dependencies) - - def testFormatDPKGTestDependencies(self): - """Tests the _FormatDPKGTestDependencies function.""" - test_writer = self._CreateTestWriter() - - expected_formatted_test_dependencies = ( - 'TEST_DEPENDENCIES="python3-pbr\n' - ' python3-setuptools";') - - python_dependencies = test_writer._GetDPKGPythonDependencies() - test_dependencies = test_writer._GetDPKGTestDependencies( - python_dependencies) - test_dependencies.extend(['python3-setuptools']) - - formatted_test_dependencies = test_writer._FormatDPKGTestDependencies( - test_dependencies) - self.assertEqual( - formatted_test_dependencies, expected_formatted_test_dependencies) - - def testGetDPKGDebugDependencies(self): - """Tests the _GetDPKGDebugDependencies function.""" - test_writer = self._CreateTestWriter() - - expected_debug_dependencies = [] - - python_dependencies = test_writer._GetDPKGPythonDependencies() - debug_dependencies = test_writer._GetDPKGDebugDependencies( - python_dependencies) - self.assertEqual(debug_dependencies, expected_debug_dependencies) - - # TODO: Add test for the Write method. - - -if __name__ == '__main__': - unittest.main() + """Tests the hared functionality for GIFT PPA installation script writer.""" + + # pylint: disable=protected-access + + def _CreateTestWriter(self): + """Creates a dependency file writer for testing. + + Returns: + GIFTPPAInstallScriptWriter: dependency file writer for testing. + """ + project_definition = projects.ProjectDefinition("test") + dependencies_file = self._GetTestFilePath(["dependencies.ini"]) + test_dependencies_file = self._GetTestFilePath(["test_dependencies.ini"]) + dependency_helper = dependencies.DependencyHelper( + dependencies_file=dependencies_file, + test_dependencies_file=test_dependencies_file, + ) + + return gift_ppa.GIFTPPAInstallScriptWriter( + "/fake/l2tdevtools/", project_definition, dependency_helper + ) + + def testFormatDPKGDebugDependencies(self): + """Tests the _FormatDPKGDebugDependencies function.""" + test_writer = self._CreateTestWriter() + + expected_formatted_debug_dependencies = "" + + python_dependencies = test_writer._GetDPKGPythonDependencies() + debug_dependencies = test_writer._GetDPKGDebugDependencies(python_dependencies) + formatted_debug_dependencies = test_writer._FormatDPKGDebugDependencies( + debug_dependencies + ) + self.assertEqual( + formatted_debug_dependencies, expected_formatted_debug_dependencies + ) + + def testFormatDPKGDevelopmentDependencies(self): + """Tests the _FormatDPKGDevelopmentDependencies function.""" + test_writer = self._CreateTestWriter() + + expected_formatted_development_dependencies = ( + 'DEVELOPMENT_DEPENDENCIES="pylint";' + ) + + development_dependencies = ["pylint"] + formatted_development_dependencies = ( + test_writer._FormatDPKGDevelopmentDependencies(development_dependencies) + ) + self.assertEqual( + formatted_development_dependencies, + expected_formatted_development_dependencies, + ) + + def testFormatDPKGPythonDependencies(self): + """Tests the _FormatDPKGPythonDependencies function.""" + test_writer = self._CreateTestWriter() + + expected_formatted_python_dependencies = 'PYTHON_DEPENDENCIES="python3-yaml";' + + python_dependencies = test_writer._GetDPKGPythonDependencies() + formatted_python_dependencies = test_writer._FormatDPKGPythonDependencies( + python_dependencies + ) + self.assertEqual( + formatted_python_dependencies, expected_formatted_python_dependencies + ) + + def testFormatDPKGTestDependencies(self): + """Tests the _FormatDPKGTestDependencies function.""" + test_writer = self._CreateTestWriter() + + expected_formatted_test_dependencies = ( + "TEST_DEPENDENCIES=\"python3-pbr\n" + " python3-setuptools\";" + ) + + python_dependencies = test_writer._GetDPKGPythonDependencies() + test_dependencies = test_writer._GetDPKGTestDependencies(python_dependencies) + test_dependencies.extend(["python3-setuptools"]) + + formatted_test_dependencies = test_writer._FormatDPKGTestDependencies( + test_dependencies + ) + self.assertEqual( + formatted_test_dependencies, expected_formatted_test_dependencies + ) + + def testGetDPKGDebugDependencies(self): + """Tests the _GetDPKGDebugDependencies function.""" + test_writer = self._CreateTestWriter() + + expected_debug_dependencies = [] + + python_dependencies = test_writer._GetDPKGPythonDependencies() + debug_dependencies = test_writer._GetDPKGDebugDependencies(python_dependencies) + self.assertEqual(debug_dependencies, expected_debug_dependencies) + + # TODO: Add test for the Write method. + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/dependency_writers/interface.py b/tests/dependency_writers/interface.py index e6db9045..34882ed2 100644 --- a/tests/dependency_writers/interface.py +++ b/tests/dependency_writers/interface.py @@ -10,91 +10,95 @@ class DependencyFileWriterTest(test_lib.BaseTestCase): - """Tests the base class for dependency file writers.""" + """Tests the base class for dependency file writers.""" - # pylint: disable=protected-access + # pylint: disable=protected-access - def _CreateTestWriter(self): - """Creates a dependency file writer for testing. + def _CreateTestWriter(self): + """Creates a dependency file writer for testing. - Returns: - DependencyFileWriter: dependency file writer for testing. - """ - project_definition = projects.ProjectDefinition('test') - dependencies_file = self._GetTestFilePath(['dependencies.ini']) - test_dependencies_file = self._GetTestFilePath(['test_dependencies.ini']) - dependency_helper = dependencies.DependencyHelper( - dependencies_file=dependencies_file, - test_dependencies_file=test_dependencies_file) + Returns: + DependencyFileWriter: dependency file writer for testing. + """ + project_definition = projects.ProjectDefinition("test") + dependencies_file = self._GetTestFilePath(["dependencies.ini"]) + test_dependencies_file = self._GetTestFilePath(["test_dependencies.ini"]) + dependency_helper = dependencies.DependencyHelper( + dependencies_file=dependencies_file, + test_dependencies_file=test_dependencies_file, + ) - return interface.DependencyFileWriter( - '/fake/l2tdevtools/', project_definition, dependency_helper) + return interface.DependencyFileWriter( + "/fake/l2tdevtools/", project_definition, dependency_helper + ) - # TODO: add tests for _GenerateFromTemplate. + # TODO: add tests for _GenerateFromTemplate. - def testGetDPKGPythonDependencies(self): - """Tests the _GetDPKGPythonDependencies function.""" - test_writer = self._CreateTestWriter() + def testGetDPKGPythonDependencies(self): + """Tests the _GetDPKGPythonDependencies function.""" + test_writer = self._CreateTestWriter() - expected_python_dependencies = ['python3-yaml'] + expected_python_dependencies = ["python3-yaml"] - python_dependencies = test_writer._GetDPKGPythonDependencies() - self.assertEqual(python_dependencies, expected_python_dependencies) + python_dependencies = test_writer._GetDPKGPythonDependencies() + self.assertEqual(python_dependencies, expected_python_dependencies) - def testGetDPKGTestDependencies(self): - """Tests the _GetDPKGTestDependencies function.""" - test_writer = self._CreateTestWriter() + def testGetDPKGTestDependencies(self): + """Tests the _GetDPKGTestDependencies function.""" + test_writer = self._CreateTestWriter() - expected_test_dependencies = ['python3-pbr'] + expected_test_dependencies = ["python3-pbr"] - python_dependencies = test_writer._GetDPKGPythonDependencies() - test_dependencies = test_writer._GetDPKGTestDependencies( - python_dependencies) - self.assertEqual(test_dependencies, expected_test_dependencies) + python_dependencies = test_writer._GetDPKGPythonDependencies() + test_dependencies = test_writer._GetDPKGTestDependencies(python_dependencies) + self.assertEqual(test_dependencies, expected_test_dependencies) - def testGetPyPIPythonDependencies(self): - """Tests the _GetPyPIPythonDependencies function.""" - test_writer = self._CreateTestWriter() + def testGetPyPIPythonDependencies(self): + """Tests the _GetPyPIPythonDependencies function.""" + test_writer = self._CreateTestWriter() - expected_python_dependencies = ['PyYAML'] + expected_python_dependencies = ["PyYAML"] - python_dependencies = test_writer._GetPyPIPythonDependencies( - exclude_version=True) - self.assertEqual(python_dependencies, expected_python_dependencies) + python_dependencies = test_writer._GetPyPIPythonDependencies( + exclude_version=True + ) + self.assertEqual(python_dependencies, expected_python_dependencies) - def testGetPyPITestDependencies(self): - """Tests the _GetPyPITestDependencies function.""" - test_writer = self._CreateTestWriter() + def testGetPyPITestDependencies(self): + """Tests the _GetPyPITestDependencies function.""" + test_writer = self._CreateTestWriter() - expected_test_dependencies = ['pbr'] + expected_test_dependencies = ["pbr"] - python_dependencies = test_writer._GetPyPIPythonDependencies( - exclude_version=True) - test_dependencies = test_writer._GetPyPITestDependencies( - python_dependencies, exclude_version=True) - self.assertEqual(test_dependencies, expected_test_dependencies) + python_dependencies = test_writer._GetPyPIPythonDependencies( + exclude_version=True + ) + test_dependencies = test_writer._GetPyPITestDependencies( + python_dependencies, exclude_version=True + ) + self.assertEqual(test_dependencies, expected_test_dependencies) - def testGetRPMPythonDependencies(self): - """Tests the _GetRPMPythonDependencies function.""" - test_writer = self._CreateTestWriter() + def testGetRPMPythonDependencies(self): + """Tests the _GetRPMPythonDependencies function.""" + test_writer = self._CreateTestWriter() - expected_python_dependencies = ['python3-pyyaml'] + expected_python_dependencies = ["python3-pyyaml"] - python_dependencies = test_writer._GetRPMPythonDependencies() - self.assertEqual(python_dependencies, expected_python_dependencies) + python_dependencies = test_writer._GetRPMPythonDependencies() + self.assertEqual(python_dependencies, expected_python_dependencies) - def testGetRPMTestDependencies(self): - """Tests the _GetRPMTestDependencies function.""" - test_writer = self._CreateTestWriter() + def testGetRPMTestDependencies(self): + """Tests the _GetRPMTestDependencies function.""" + test_writer = self._CreateTestWriter() - expected_test_dependencies = ['python3-pbr', 'python3-setuptools'] + expected_test_dependencies = ["python3-pbr", "python3-setuptools"] - python_dependencies = test_writer._GetRPMPythonDependencies() - test_dependencies = test_writer._GetRPMTestDependencies(python_dependencies) - self.assertEqual(test_dependencies, expected_test_dependencies) + python_dependencies = test_writer._GetRPMPythonDependencies() + test_dependencies = test_writer._GetRPMTestDependencies(python_dependencies) + self.assertEqual(test_dependencies, expected_test_dependencies) - # TODO: add tests for _ReadTemplateFile. + # TODO: add tests for _ReadTemplateFile. -if __name__ == '__main__': - unittest.main() +if __name__ == "__main__": + unittest.main() diff --git a/tests/dependency_writers/jenkins_scripts.py b/tests/dependency_writers/jenkins_scripts.py index ae64a2b3..7f40daa0 100644 --- a/tests/dependency_writers/jenkins_scripts.py +++ b/tests/dependency_writers/jenkins_scripts.py @@ -10,44 +10,48 @@ class LinuxRunEndToEndTestsScriptWriterTest(test_lib.BaseTestCase): - """Tests the Linux run end-to-end test script file writer.""" + """Tests the Linux run end-to-end test script file writer.""" - def testInitialize(self): - """Tests the __init__ function.""" - l2tdevtools_path = '/fake/l2tdevtools/' - project_definition = project.ProjectHelper(l2tdevtools_path) - dependencies_file = self._GetTestFilePath(['dependencies.ini']) - test_dependencies_file = self._GetTestFilePath(['test_dependencies.ini']) - dependency_helper = dependencies.DependencyHelper( - dependencies_file=dependencies_file, - test_dependencies_file=test_dependencies_file) + def testInitialize(self): + """Tests the __init__ function.""" + l2tdevtools_path = "/fake/l2tdevtools/" + project_definition = project.ProjectHelper(l2tdevtools_path) + dependencies_file = self._GetTestFilePath(["dependencies.ini"]) + test_dependencies_file = self._GetTestFilePath(["test_dependencies.ini"]) + dependency_helper = dependencies.DependencyHelper( + dependencies_file=dependencies_file, + test_dependencies_file=test_dependencies_file, + ) - writer = jenkins_scripts.LinuxRunEndToEndTestsScriptWriter( - l2tdevtools_path, project_definition, dependency_helper) - self.assertIsNotNone(writer) + writer = jenkins_scripts.LinuxRunEndToEndTestsScriptWriter( + l2tdevtools_path, project_definition, dependency_helper + ) + self.assertIsNotNone(writer) - # TODO: Add test for the Write method. + # TODO: Add test for the Write method. class RunPython3EndToEndTestsScriptWriterTest(test_lib.BaseTestCase): - """Tests the run end-to-end test script file writer.""" + """Tests the run end-to-end test script file writer.""" - def testInitialize(self): - """Tests the __init__ function.""" - l2tdevtools_path = '/fake/l2tdevtools/' - project_definition = project.ProjectHelper(l2tdevtools_path) - dependencies_file = self._GetTestFilePath(['dependencies.ini']) - test_dependencies_file = self._GetTestFilePath(['test_dependencies.ini']) - dependency_helper = dependencies.DependencyHelper( - dependencies_file=dependencies_file, - test_dependencies_file=test_dependencies_file) + def testInitialize(self): + """Tests the __init__ function.""" + l2tdevtools_path = "/fake/l2tdevtools/" + project_definition = project.ProjectHelper(l2tdevtools_path) + dependencies_file = self._GetTestFilePath(["dependencies.ini"]) + test_dependencies_file = self._GetTestFilePath(["test_dependencies.ini"]) + dependency_helper = dependencies.DependencyHelper( + dependencies_file=dependencies_file, + test_dependencies_file=test_dependencies_file, + ) - writer = jenkins_scripts.RunPython3EndToEndTestsScriptWriter( - l2tdevtools_path, project_definition, dependency_helper) - self.assertIsNotNone(writer) + writer = jenkins_scripts.RunPython3EndToEndTestsScriptWriter( + l2tdevtools_path, project_definition, dependency_helper + ) + self.assertIsNotNone(writer) - # TODO: Add test for the Write method. + # TODO: Add test for the Write method. -if __name__ == '__main__': - unittest.main() +if __name__ == "__main__": + unittest.main() diff --git a/tests/dependency_writers/linux_scripts.py b/tests/dependency_writers/linux_scripts.py index d51e845c..d6a47dca 100644 --- a/tests/dependency_writers/linux_scripts.py +++ b/tests/dependency_writers/linux_scripts.py @@ -11,99 +11,106 @@ class UbuntuInstallationScriptWriterTest(test_lib.BaseTestCase): - """Tests the hared functionality for GIFT PPA installation script writer.""" - - # pylint: disable=protected-access - - def _CreateTestWriter(self): - """Creates a dependency file writer for testing. - - Returns: - UbuntuInstallationScriptWriter: dependency file writer for testing. - """ - project_definition = projects.ProjectDefinition('test') - dependencies_file = self._GetTestFilePath(['dependencies.ini']) - test_dependencies_file = self._GetTestFilePath(['test_dependencies.ini']) - dependency_helper = dependencies.DependencyHelper( - dependencies_file=dependencies_file, - test_dependencies_file=test_dependencies_file) - - return linux_scripts.UbuntuInstallationScriptWriter( - '/fake/l2tdevtools/', project_definition, dependency_helper) - - def testFormatDPKGDebugDependencies(self): - """Tests the _FormatDPKGDebugDependencies function.""" - test_writer = self._CreateTestWriter() - - expected_formatted_debug_dependencies = '' - - python_dependencies = test_writer._GetDPKGPythonDependencies() - debug_dependencies = test_writer._GetDPKGDebugDependencies( - python_dependencies) - formatted_debug_dependencies = test_writer._FormatDPKGDebugDependencies( - debug_dependencies) - self.assertEqual( - formatted_debug_dependencies, expected_formatted_debug_dependencies) - - def testFormatDPKGDevelopmentDependencies(self): - """Tests the _FormatDPKGDevelopmentDependencies function.""" - test_writer = self._CreateTestWriter() - - expected_formatted_development_dependencies = ( - 'DEVELOPMENT_DEPENDENCIES="pylint";') - - development_dependencies = ['pylint'] - formatted_development_dependencies = ( - test_writer._FormatDPKGDevelopmentDependencies( - development_dependencies)) - self.assertEqual( - formatted_development_dependencies, - expected_formatted_development_dependencies) - - def testFormatDPKGPythonDependencies(self): - """Tests the _FormatDPKGPythonDependencies function.""" - test_writer = self._CreateTestWriter() - - expected_formatted_python_dependencies = ( - 'PYTHON_DEPENDENCIES="python3-yaml";') - - python_dependencies = test_writer._GetDPKGPythonDependencies() - formatted_python_dependencies = test_writer._FormatDPKGPythonDependencies( - python_dependencies) - self.assertEqual( - formatted_python_dependencies, expected_formatted_python_dependencies) - - def testFormatDPKGTestDependencies(self): - """Tests the _FormatDPKGTestDependencies function.""" - test_writer = self._CreateTestWriter() - - expected_formatted_test_dependencies = ( - 'TEST_DEPENDENCIES="python3-pbr\n' - ' python3-setuptools";') - - python_dependencies = test_writer._GetDPKGPythonDependencies() - test_dependencies = test_writer._GetDPKGTestDependencies( - python_dependencies) - test_dependencies.extend(['python3-setuptools']) - - formatted_test_dependencies = test_writer._FormatDPKGTestDependencies( - test_dependencies) - self.assertEqual( - formatted_test_dependencies, expected_formatted_test_dependencies) - - def testGetDPKGDebugDependencies(self): - """Tests the _GetDPKGDebugDependencies function.""" - test_writer = self._CreateTestWriter() - - expected_debug_dependencies = [] - - python_dependencies = test_writer._GetDPKGPythonDependencies() - debug_dependencies = test_writer._GetDPKGDebugDependencies( - python_dependencies) - self.assertEqual(debug_dependencies, expected_debug_dependencies) - - # TODO: Add test for the Write method. - - -if __name__ == '__main__': - unittest.main() + """Tests the hared functionality for GIFT PPA installation script writer.""" + + # pylint: disable=protected-access + + def _CreateTestWriter(self): + """Creates a dependency file writer for testing. + + Returns: + UbuntuInstallationScriptWriter: dependency file writer for testing. + """ + project_definition = projects.ProjectDefinition("test") + dependencies_file = self._GetTestFilePath(["dependencies.ini"]) + test_dependencies_file = self._GetTestFilePath(["test_dependencies.ini"]) + dependency_helper = dependencies.DependencyHelper( + dependencies_file=dependencies_file, + test_dependencies_file=test_dependencies_file, + ) + + return linux_scripts.UbuntuInstallationScriptWriter( + "/fake/l2tdevtools/", project_definition, dependency_helper + ) + + def testFormatDPKGDebugDependencies(self): + """Tests the _FormatDPKGDebugDependencies function.""" + test_writer = self._CreateTestWriter() + + expected_formatted_debug_dependencies = "" + + python_dependencies = test_writer._GetDPKGPythonDependencies() + debug_dependencies = test_writer._GetDPKGDebugDependencies(python_dependencies) + formatted_debug_dependencies = test_writer._FormatDPKGDebugDependencies( + debug_dependencies + ) + self.assertEqual( + formatted_debug_dependencies, expected_formatted_debug_dependencies + ) + + def testFormatDPKGDevelopmentDependencies(self): + """Tests the _FormatDPKGDevelopmentDependencies function.""" + test_writer = self._CreateTestWriter() + + expected_formatted_development_dependencies = ( + 'DEVELOPMENT_DEPENDENCIES="pylint";' + ) + + development_dependencies = ["pylint"] + formatted_development_dependencies = ( + test_writer._FormatDPKGDevelopmentDependencies(development_dependencies) + ) + self.assertEqual( + formatted_development_dependencies, + expected_formatted_development_dependencies, + ) + + def testFormatDPKGPythonDependencies(self): + """Tests the _FormatDPKGPythonDependencies function.""" + test_writer = self._CreateTestWriter() + + expected_formatted_python_dependencies = 'PYTHON_DEPENDENCIES="python3-yaml";' + + python_dependencies = test_writer._GetDPKGPythonDependencies() + formatted_python_dependencies = test_writer._FormatDPKGPythonDependencies( + python_dependencies + ) + self.assertEqual( + formatted_python_dependencies, expected_formatted_python_dependencies + ) + + def testFormatDPKGTestDependencies(self): + """Tests the _FormatDPKGTestDependencies function.""" + test_writer = self._CreateTestWriter() + + expected_formatted_test_dependencies = ( + "TEST_DEPENDENCIES=\"python3-pbr\n" + " python3-setuptools\";" + ) + + python_dependencies = test_writer._GetDPKGPythonDependencies() + test_dependencies = test_writer._GetDPKGTestDependencies(python_dependencies) + test_dependencies.extend(["python3-setuptools"]) + + formatted_test_dependencies = test_writer._FormatDPKGTestDependencies( + test_dependencies + ) + self.assertEqual( + formatted_test_dependencies, expected_formatted_test_dependencies + ) + + def testGetDPKGDebugDependencies(self): + """Tests the _GetDPKGDebugDependencies function.""" + test_writer = self._CreateTestWriter() + + expected_debug_dependencies = [] + + python_dependencies = test_writer._GetDPKGPythonDependencies() + debug_dependencies = test_writer._GetDPKGDebugDependencies(python_dependencies) + self.assertEqual(debug_dependencies, expected_debug_dependencies) + + # TODO: Add test for the Write method. + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/dependency_writers/pylint_rc.py b/tests/dependency_writers/pylint_rc.py index d872b4b6..025046be 100644 --- a/tests/dependency_writers/pylint_rc.py +++ b/tests/dependency_writers/pylint_rc.py @@ -10,24 +10,26 @@ class PylintRc(test_lib.BaseTestCase): - """Tests the pylint.rc writer.""" + """Tests the pylint.rc writer.""" - def testInitialize(self): - """Tests that the writer can be initialized.""" - l2tdevtools_path = '/fake/l2tdevtools/' - project_definition = project.ProjectHelper(l2tdevtools_path) - dependencies_file = self._GetTestFilePath(['dependencies.ini']) - test_dependencies_file = self._GetTestFilePath(['test_dependencies.ini']) - dependency_helper = dependencies.DependencyHelper( - dependencies_file=dependencies_file, - test_dependencies_file=test_dependencies_file) + def testInitialize(self): + """Tests that the writer can be initialized.""" + l2tdevtools_path = "/fake/l2tdevtools/" + project_definition = project.ProjectHelper(l2tdevtools_path) + dependencies_file = self._GetTestFilePath(["dependencies.ini"]) + test_dependencies_file = self._GetTestFilePath(["test_dependencies.ini"]) + dependency_helper = dependencies.DependencyHelper( + dependencies_file=dependencies_file, + test_dependencies_file=test_dependencies_file, + ) - writer = pylint_rc.PylintRcWriter( - l2tdevtools_path, project_definition, dependency_helper) - self.assertIsNotNone(writer) + writer = pylint_rc.PylintRcWriter( + l2tdevtools_path, project_definition, dependency_helper + ) + self.assertIsNotNone(writer) - # TODO: Add test for the Write method. + # TODO: Add test for the Write method. -if __name__ == '__main__': - unittest.main() +if __name__ == "__main__": + unittest.main() diff --git a/tests/dependency_writers/setup.py b/tests/dependency_writers/setup.py index 114a07cf..498d9473 100644 --- a/tests/dependency_writers/setup.py +++ b/tests/dependency_writers/setup.py @@ -10,24 +10,26 @@ class PyprojectTomlWriterTest(test_lib.BaseTestCase): - """Tests the pyproject.toml configuration file writer.""" + """Tests the pyproject.toml configuration file writer.""" - def testInitialize(self): - """Tests that the writer can be initialized.""" - l2tdevtools_path = '/fake/l2tdevtools/' - project_definition = project.ProjectHelper(l2tdevtools_path) - dependencies_file = self._GetTestFilePath(['dependencies.ini']) - test_dependencies_file = self._GetTestFilePath(['test_dependencies.ini']) - dependency_helper = dependencies.DependencyHelper( - dependencies_file=dependencies_file, - test_dependencies_file=test_dependencies_file) + def testInitialize(self): + """Tests that the writer can be initialized.""" + l2tdevtools_path = "/fake/l2tdevtools/" + project_definition = project.ProjectHelper(l2tdevtools_path) + dependencies_file = self._GetTestFilePath(["dependencies.ini"]) + test_dependencies_file = self._GetTestFilePath(["test_dependencies.ini"]) + dependency_helper = dependencies.DependencyHelper( + dependencies_file=dependencies_file, + test_dependencies_file=test_dependencies_file, + ) - writer = setup.PyprojectTomlWriter( - l2tdevtools_path, project_definition, dependency_helper) - self.assertIsNotNone(writer) + writer = setup.PyprojectTomlWriter( + l2tdevtools_path, project_definition, dependency_helper + ) + self.assertIsNotNone(writer) - # TODO: Add test for the Write method. + # TODO: Add test for the Write method. -if __name__ == '__main__': - unittest.main() +if __name__ == "__main__": + unittest.main() diff --git a/tests/dependency_writers/sphinx_docs.py b/tests/dependency_writers/sphinx_docs.py index 49671b45..326b0cd4 100644 --- a/tests/dependency_writers/sphinx_docs.py +++ b/tests/dependency_writers/sphinx_docs.py @@ -11,64 +11,70 @@ class ReadthedocsConfigurationWriterTest(test_lib.BaseTestCase): - """Tests the Readthedocs configuration file writer.""" + """Tests the Readthedocs configuration file writer.""" - def testInitialize(self): - """Tests that the writer can be initialized.""" - l2tdevtools_path = '/fake/l2tdevtools/' - project_definition = project.ProjectHelper(l2tdevtools_path) - dependencies_file = self._GetTestFilePath(['dependencies.ini']) - test_dependencies_file = self._GetTestFilePath(['test_dependencies.ini']) - dependency_helper = dependencies.DependencyHelper( - dependencies_file=dependencies_file, - test_dependencies_file=test_dependencies_file) + def testInitialize(self): + """Tests that the writer can be initialized.""" + l2tdevtools_path = "/fake/l2tdevtools/" + project_definition = project.ProjectHelper(l2tdevtools_path) + dependencies_file = self._GetTestFilePath(["dependencies.ini"]) + test_dependencies_file = self._GetTestFilePath(["test_dependencies.ini"]) + dependency_helper = dependencies.DependencyHelper( + dependencies_file=dependencies_file, + test_dependencies_file=test_dependencies_file, + ) - writer = sphinx_docs.ReadthedocsConfigurationWriter( - l2tdevtools_path, project_definition, dependency_helper) - self.assertIsNotNone(writer) + writer = sphinx_docs.ReadthedocsConfigurationWriter( + l2tdevtools_path, project_definition, dependency_helper + ) + self.assertIsNotNone(writer) - # TODO: Add test for the Write method. + # TODO: Add test for the Write method. class SphinxBuildConfigurationWriterTest(test_lib.BaseTestCase): - """Tests the Sphinx build configuration file writer.""" + """Tests the Sphinx build configuration file writer.""" - def testInitialize(self): - """Tests that the writer can be initialized.""" - l2tdevtools_path = '/fake/l2tdevtools/' - project_definition = project.ProjectHelper(l2tdevtools_path) - dependencies_file = self._GetTestFilePath(['dependencies.ini']) - test_dependencies_file = self._GetTestFilePath(['test_dependencies.ini']) - dependency_helper = dependencies.DependencyHelper( - dependencies_file=dependencies_file, - test_dependencies_file=test_dependencies_file) + def testInitialize(self): + """Tests that the writer can be initialized.""" + l2tdevtools_path = "/fake/l2tdevtools/" + project_definition = project.ProjectHelper(l2tdevtools_path) + dependencies_file = self._GetTestFilePath(["dependencies.ini"]) + test_dependencies_file = self._GetTestFilePath(["test_dependencies.ini"]) + dependency_helper = dependencies.DependencyHelper( + dependencies_file=dependencies_file, + test_dependencies_file=test_dependencies_file, + ) - writer = sphinx_docs.SphinxBuildConfigurationWriter( - l2tdevtools_path, project_definition, dependency_helper) - self.assertIsNotNone(writer) + writer = sphinx_docs.SphinxBuildConfigurationWriter( + l2tdevtools_path, project_definition, dependency_helper + ) + self.assertIsNotNone(writer) - # TODO: Add test for the Write method. + # TODO: Add test for the Write method. class SphinxBuildRequirementsWriterTest(test_lib.BaseTestCase): - """Tests the Sphinx build requirements file writer.""" + """Tests the Sphinx build requirements file writer.""" - def testInitialize(self): - """Tests that the writer can be initialized.""" - l2tdevtools_path = '/fake/l2tdevtools/' - project_definition = project.ProjectHelper(l2tdevtools_path) - dependencies_file = self._GetTestFilePath(['dependencies.ini']) - test_dependencies_file = self._GetTestFilePath(['test_dependencies.ini']) - dependency_helper = dependencies.DependencyHelper( - dependencies_file=dependencies_file, - test_dependencies_file=test_dependencies_file) + def testInitialize(self): + """Tests that the writer can be initialized.""" + l2tdevtools_path = "/fake/l2tdevtools/" + project_definition = project.ProjectHelper(l2tdevtools_path) + dependencies_file = self._GetTestFilePath(["dependencies.ini"]) + test_dependencies_file = self._GetTestFilePath(["test_dependencies.ini"]) + dependency_helper = dependencies.DependencyHelper( + dependencies_file=dependencies_file, + test_dependencies_file=test_dependencies_file, + ) - writer = sphinx_docs.SphinxBuildRequirementsWriter( - l2tdevtools_path, project_definition, dependency_helper) - self.assertIsNotNone(writer) + writer = sphinx_docs.SphinxBuildRequirementsWriter( + l2tdevtools_path, project_definition, dependency_helper + ) + self.assertIsNotNone(writer) - # TODO: Add test for the Write method. + # TODO: Add test for the Write method. -if __name__ == '__main__': - unittest.main() +if __name__ == "__main__": + unittest.main() diff --git a/tests/dependency_writers/tox_ini.py b/tests/dependency_writers/tox_ini.py index 5238de1a..51df37a7 100644 --- a/tests/dependency_writers/tox_ini.py +++ b/tests/dependency_writers/tox_ini.py @@ -10,24 +10,26 @@ class ToxIniWriterTest(test_lib.BaseTestCase): - """Tests the tox.ini writer.""" + """Tests the tox.ini writer.""" - def testInitialize(self): - """Tests that the writer can be initialized.""" - l2tdevtools_path = '/fake/l2tdevtools/' - project_definition = project.ProjectHelper(l2tdevtools_path) - dependencies_file = self._GetTestFilePath(['dependencies.ini']) - test_dependencies_file = self._GetTestFilePath(['test_dependencies.ini']) - dependency_helper = dependencies.DependencyHelper( - dependencies_file=dependencies_file, - test_dependencies_file=test_dependencies_file) + def testInitialize(self): + """Tests that the writer can be initialized.""" + l2tdevtools_path = "/fake/l2tdevtools/" + project_definition = project.ProjectHelper(l2tdevtools_path) + dependencies_file = self._GetTestFilePath(["dependencies.ini"]) + test_dependencies_file = self._GetTestFilePath(["test_dependencies.ini"]) + dependency_helper = dependencies.DependencyHelper( + dependencies_file=dependencies_file, + test_dependencies_file=test_dependencies_file, + ) - writer = tox_ini.ToxIniWriter( - l2tdevtools_path, project_definition, dependency_helper) - self.assertIsNotNone(writer) + writer = tox_ini.ToxIniWriter( + l2tdevtools_path, project_definition, dependency_helper + ) + self.assertIsNotNone(writer) - # TODO: Add test for the Write method. + # TODO: Add test for the Write method. -if __name__ == '__main__': - unittest.main() +if __name__ == "__main__": + unittest.main() diff --git a/tests/download_helper.py b/tests/download_helper.py index 8d089747..0043421a 100644 --- a/tests/download_helper.py +++ b/tests/download_helper.py @@ -7,5 +7,5 @@ # TODO: add tests for DownloadHelperFactory -if __name__ == '__main__': - unittest.main() +if __name__ == "__main__": + unittest.main() diff --git a/tests/download_helpers/github.py b/tests/download_helpers/github.py index 065f805c..f7d4ba07 100644 --- a/tests/download_helpers/github.py +++ b/tests/download_helpers/github.py @@ -11,224 +11,239 @@ class PefileGitHubReleasesDownloadHelperTest(test_lib.BaseTestCase): - """Tests for the pefile GitHub releases download helper.""" + """Tests for the pefile GitHub releases download helper.""" - _DOWNLOAD_URL = 'https://github.com/erocarrera/pefile/releases' - _GIT_URL = 'https://github.com/erocarrera/pefile.git' + _DOWNLOAD_URL = "https://github.com/erocarrera/pefile/releases" + _GIT_URL = "https://github.com/erocarrera/pefile.git" - _PROJECT_ORGANIZATION = 'erocarrera' - _PROJECT_NAME = 'pefile' - _PROJECT_VERSION = '2024.8.26' + _PROJECT_ORGANIZATION = "erocarrera" + _PROJECT_NAME = "pefile" + _PROJECT_VERSION = "2024.8.26" - @classmethod - def setUpClass(cls): - """Determines the project version from the latest git tag.""" - arguments = shlex.split(f'git ls-remote --tags {cls._GIT_URL:s}') + @classmethod + def setUpClass(cls): + """Determines the project version from the latest git tag.""" + arguments = shlex.split(f"git ls-remote --tags {cls._GIT_URL:s}") - try: - with subprocess.Popen( - arguments, stderr=subprocess.PIPE, stdout=subprocess.PIPE) as process: - output, _ = process.communicate() - if process.returncode != 0: - return + try: + with subprocess.Popen( + arguments, stderr=subprocess.PIPE, stdout=subprocess.PIPE + ) as process: + output, _ = process.communicate() + if process.returncode != 0: + return - except OSError: - return + except OSError: + return - output = output.decode('ascii') + output = output.decode("ascii") - latest_version = ('0', '0', '0') - for line in output.split('\n'): - line = line.strip() - if 'refs/tags/' in line and not line.endswith('^{}'): - _, _, version = line.rpartition('refs/tags/') - if version.startswith('pefile-'): - version = version[7:] - elif version.startswith('v'): - version = version[1:] - version = tuple(version.split('.')) - latest_version = max(latest_version, version) + latest_version = ("0", "0", "0") + for line in output.split("\n"): + line = line.strip() + if "refs/tags/" in line and not line.endswith("^{}"): + _, _, version = line.rpartition("refs/tags/") + if version.startswith("pefile-"): + version = version[7:] + elif version.startswith("v"): + version = version[1:] + version = tuple(version.split(".")) + latest_version = max(latest_version, version) - cls._PROJECT_VERSION = '.'.join(latest_version) + cls._PROJECT_VERSION = ".".join(latest_version) - def testGetLatestVersion(self): - """Tests the GetLatestVersion functions.""" - download_helper = github.GitHubReleasesDownloadHelper( - self._DOWNLOAD_URL, release_prefix='pefile-', release_tag_prefix='v') + def testGetLatestVersion(self): + """Tests the GetLatestVersion functions.""" + download_helper = github.GitHubReleasesDownloadHelper( + self._DOWNLOAD_URL, release_prefix="pefile-", release_tag_prefix="v" + ) - latest_version = download_helper.GetLatestVersion(self._PROJECT_NAME, None) + latest_version = download_helper.GetLatestVersion(self._PROJECT_NAME, None) - self.assertEqual(latest_version, self._PROJECT_VERSION) + self.assertEqual(latest_version, self._PROJECT_VERSION) - def testGetDownloadURL(self): - """Tests the GetDownloadURL functions.""" - download_helper = github.GitHubReleasesDownloadHelper( - self._DOWNLOAD_URL, release_prefix='pefile-', release_tag_prefix='v') + def testGetDownloadURL(self): + """Tests the GetDownloadURL functions.""" + download_helper = github.GitHubReleasesDownloadHelper( + self._DOWNLOAD_URL, release_prefix="pefile-", release_tag_prefix="v" + ) - download_url = download_helper.GetDownloadURL( - self._PROJECT_NAME, self._PROJECT_VERSION) + download_url = download_helper.GetDownloadURL( + self._PROJECT_NAME, self._PROJECT_VERSION + ) - expected_download_url = ( - f'https://github.com/{self._PROJECT_ORGANIZATION:s}/' - f'{self._PROJECT_NAME:s}/releases/download/' - f'v{self._PROJECT_VERSION:s}/pefile-{self._PROJECT_VERSION:s}.tar.gz') + expected_download_url = ( + f"https://github.com/{self._PROJECT_ORGANIZATION:s}/" + f"{self._PROJECT_NAME:s}/releases/download/" + f"v{self._PROJECT_VERSION:s}/pefile-{self._PROJECT_VERSION:s}.tar.gz" + ) - self.assertEqual(download_url, expected_download_url) + self.assertEqual(download_url, expected_download_url) - def testGetProjectIdentifier(self): - """Tests the GetProjectIdentifier functions.""" - download_helper = github.GitHubReleasesDownloadHelper( - self._DOWNLOAD_URL, release_prefix='pefile-', release_tag_prefix='v') + def testGetProjectIdentifier(self): + """Tests the GetProjectIdentifier functions.""" + download_helper = github.GitHubReleasesDownloadHelper( + self._DOWNLOAD_URL, release_prefix="pefile-", release_tag_prefix="v" + ) - project_identifier = download_helper.GetProjectIdentifier() + project_identifier = download_helper.GetProjectIdentifier() - expected_project_identifier = ( - f'com.github.{self._PROJECT_ORGANIZATION:s}.{self._PROJECT_NAME:s}') + expected_project_identifier = ( + f"com.github.{self._PROJECT_ORGANIZATION:s}.{self._PROJECT_NAME:s}" + ) - self.assertEqual(project_identifier, expected_project_identifier) + self.assertEqual(project_identifier, expected_project_identifier) class LibyalGitHubReleasesDownloadHelperTest(test_lib.BaseTestCase): - """Tests for the libyal GitHub releases download helper.""" + """Tests for the libyal GitHub releases download helper.""" - _DOWNLOAD_URL = 'https://github.com/libyal/libevt/releases' - _GIT_URL = 'https://github.com/libyal/libevt.git' + _DOWNLOAD_URL = "https://github.com/libyal/libevt/releases" + _GIT_URL = "https://github.com/libyal/libevt.git" - _PROJECT_ORGANIZATION = 'libyal' - _PROJECT_NAME = 'libevt' - _PROJECT_STATUS = 'alpha' - _PROJECT_VERSION = '20240421' + _PROJECT_ORGANIZATION = "libyal" + _PROJECT_NAME = "libevt" + _PROJECT_STATUS = "alpha" + _PROJECT_VERSION = "20240421" - @classmethod - def setUpClass(cls): - """Determines the project version from the latest git tag.""" - arguments = shlex.split('git ls-remote --tags {cls._GIT_URL:s}') + @classmethod + def setUpClass(cls): + """Determines the project version from the latest git tag.""" + arguments = shlex.split("git ls-remote --tags {cls._GIT_URL:s}") - try: - with subprocess.Popen( - arguments, stderr=subprocess.PIPE, stdout=subprocess.PIPE) as process: - output, _ = process.communicate() - if process.returncode != 0: - return + try: + with subprocess.Popen( + arguments, stderr=subprocess.PIPE, stdout=subprocess.PIPE + ) as process: + output, _ = process.communicate() + if process.returncode != 0: + return - except OSError: - return + except OSError: + return - output = output.decode('ascii') + output = output.decode("ascii") - latest_version = '0' - for line in output.split('\n'): - line = line.strip() - if 'refs/tags/' in line and not line.endswith('^{}'): - _, _, version = line.rpartition('refs/tags/') - latest_version = max(latest_version, version) + latest_version = "0" + for line in output.split("\n"): + line = line.strip() + if "refs/tags/" in line and not line.endswith("^{}"): + _, _, version = line.rpartition("refs/tags/") + latest_version = max(latest_version, version) - cls._PROJECT_VERSION = latest_version + cls._PROJECT_VERSION = latest_version - def testGetLatestVersion(self): - """Tests the GetLatestVersion functions.""" - download_helper = github.GitHubReleasesDownloadHelper(self._DOWNLOAD_URL) + def testGetLatestVersion(self): + """Tests the GetLatestVersion functions.""" + download_helper = github.GitHubReleasesDownloadHelper(self._DOWNLOAD_URL) - latest_version = download_helper.GetLatestVersion(self._PROJECT_NAME, None) + latest_version = download_helper.GetLatestVersion(self._PROJECT_NAME, None) - self.assertEqual(latest_version, self._PROJECT_VERSION) + self.assertEqual(latest_version, self._PROJECT_VERSION) - def testGetDownloadURL(self): - """Tests the GetDownloadURL functions.""" - download_helper = github.GitHubReleasesDownloadHelper(self._DOWNLOAD_URL) + def testGetDownloadURL(self): + """Tests the GetDownloadURL functions.""" + download_helper = github.GitHubReleasesDownloadHelper(self._DOWNLOAD_URL) - download_url = download_helper.GetDownloadURL( - self._PROJECT_NAME, self._PROJECT_VERSION) + download_url = download_helper.GetDownloadURL( + self._PROJECT_NAME, self._PROJECT_VERSION + ) - expected_download_url = ( - f'https://github.com/{self._PROJECT_ORGANIZATION:s}/' - f'{self._PROJECT_NAME:s}/releases/download/{self._PROJECT_VERSION:s}/' - f'{self._PROJECT_NAME:s}-{self._PROJECT_STATUS:s}-' - f'{self._PROJECT_VERSION:s}.tar.gz') + expected_download_url = ( + f"https://github.com/{self._PROJECT_ORGANIZATION:s}/" + f"{self._PROJECT_NAME:s}/releases/download/{self._PROJECT_VERSION:s}/" + f"{self._PROJECT_NAME:s}-{self._PROJECT_STATUS:s}-" + f"{self._PROJECT_VERSION:s}.tar.gz" + ) - self.assertEqual(download_url, expected_download_url) + self.assertEqual(download_url, expected_download_url) - def testGetProjectIdentifier(self): - """Tests the GetProjectIdentifier functions.""" - download_helper = github.GitHubReleasesDownloadHelper(self._DOWNLOAD_URL) + def testGetProjectIdentifier(self): + """Tests the GetProjectIdentifier functions.""" + download_helper = github.GitHubReleasesDownloadHelper(self._DOWNLOAD_URL) - project_identifier = download_helper.GetProjectIdentifier() + project_identifier = download_helper.GetProjectIdentifier() - expected_project_identifier = ( - f'com.github.{self._PROJECT_ORGANIZATION:s}.{self._PROJECT_NAME:s}') + expected_project_identifier = ( + f"com.github.{self._PROJECT_ORGANIZATION:s}.{self._PROJECT_NAME:s}" + ) - self.assertEqual(project_identifier, expected_project_identifier) + self.assertEqual(project_identifier, expected_project_identifier) class Log2TimelineGitHubReleasesDownloadHelperTest(test_lib.BaseTestCase): - """Tests for the log2timeline GitHub releases download helper.""" + """Tests for the log2timeline GitHub releases download helper.""" - _DOWNLOAD_URL = 'https://github.com/log2timeline/dfvfs/releases' - _GIT_URL = 'https://github.com/log2timeline/dfvfs.git' + _DOWNLOAD_URL = "https://github.com/log2timeline/dfvfs/releases" + _GIT_URL = "https://github.com/log2timeline/dfvfs.git" - _PROJECT_ORGANIZATION = 'log2timeline' - _PROJECT_NAME = 'dfvfs' - _PROJECT_VERSION = '20260411' + _PROJECT_ORGANIZATION = "log2timeline" + _PROJECT_NAME = "dfvfs" + _PROJECT_VERSION = "20260411" - @classmethod - def setUpClass(cls): - """Determines the project version from the latest git tag.""" - arguments = shlex.split(f'git ls-remote --tags {cls._GIT_URL:s}') + @classmethod + def setUpClass(cls): + """Determines the project version from the latest git tag.""" + arguments = shlex.split(f"git ls-remote --tags {cls._GIT_URL:s}") - try: - with subprocess.Popen( - arguments, stderr=subprocess.PIPE, stdout=subprocess.PIPE) as process: - output, _ = process.communicate() - if process.returncode != 0: - return + try: + with subprocess.Popen( + arguments, stderr=subprocess.PIPE, stdout=subprocess.PIPE + ) as process: + output, _ = process.communicate() + if process.returncode != 0: + return - except OSError: - return + except OSError: + return - output = output.decode('ascii') + output = output.decode("ascii") - latest_version = '0' - for line in output.split('\n'): - line = line.strip() - if 'refs/tags/' in line and not line.endswith('^{}'): - _, _, version = line.rpartition('refs/tags/') - latest_version = max(latest_version, version) + latest_version = "0" + for line in output.split("\n"): + line = line.strip() + if "refs/tags/" in line and not line.endswith("^{}"): + _, _, version = line.rpartition("refs/tags/") + latest_version = max(latest_version, version) - cls._PROJECT_VERSION = latest_version + cls._PROJECT_VERSION = latest_version - def testGetLatestVersion(self): - """Tests the GetLatestVersion functions.""" - download_helper = github.GitHubReleasesDownloadHelper(self._DOWNLOAD_URL) + def testGetLatestVersion(self): + """Tests the GetLatestVersion functions.""" + download_helper = github.GitHubReleasesDownloadHelper(self._DOWNLOAD_URL) - latest_version = download_helper.GetLatestVersion(self._PROJECT_NAME, None) + latest_version = download_helper.GetLatestVersion(self._PROJECT_NAME, None) - self.assertEqual(latest_version, self._PROJECT_VERSION) + self.assertEqual(latest_version, self._PROJECT_VERSION) - def testGetDownloadURL(self): - """Tests the GetDownloadURL functions.""" - download_helper = github.GitHubReleasesDownloadHelper(self._DOWNLOAD_URL) + def testGetDownloadURL(self): + """Tests the GetDownloadURL functions.""" + download_helper = github.GitHubReleasesDownloadHelper(self._DOWNLOAD_URL) - download_url = download_helper.GetDownloadURL( - self._PROJECT_NAME, self._PROJECT_VERSION) + download_url = download_helper.GetDownloadURL( + self._PROJECT_NAME, self._PROJECT_VERSION + ) - expected_download_url = ( - f'https://github.com/{self._PROJECT_ORGANIZATION:s}/' - f'{self._PROJECT_NAME:s}/releases/download/{self._PROJECT_VERSION:s}/' - f'{self._PROJECT_NAME:s}-{self._PROJECT_VERSION:s}.tar.gz') + expected_download_url = ( + f"https://github.com/{self._PROJECT_ORGANIZATION:s}/" + f"{self._PROJECT_NAME:s}/releases/download/{self._PROJECT_VERSION:s}/" + f"{self._PROJECT_NAME:s}-{self._PROJECT_VERSION:s}.tar.gz" + ) - self.assertEqual(download_url, expected_download_url) + self.assertEqual(download_url, expected_download_url) - def testGetProjectIdentifier(self): - """Tests the GetProjectIdentifier functions.""" - download_helper = github.GitHubReleasesDownloadHelper(self._DOWNLOAD_URL) + def testGetProjectIdentifier(self): + """Tests the GetProjectIdentifier functions.""" + download_helper = github.GitHubReleasesDownloadHelper(self._DOWNLOAD_URL) - project_identifier = download_helper.GetProjectIdentifier() + project_identifier = download_helper.GetProjectIdentifier() - expected_project_identifier = ( - f'com.github.{self._PROJECT_ORGANIZATION:s}.{self._PROJECT_NAME:s}') + expected_project_identifier = ( + f"com.github.{self._PROJECT_ORGANIZATION:s}.{self._PROJECT_NAME:s}" + ) - self.assertEqual(project_identifier, expected_project_identifier) + self.assertEqual(project_identifier, expected_project_identifier) -if __name__ == '__main__': - unittest.main() +if __name__ == "__main__": + unittest.main() diff --git a/tests/download_helpers/interface.py b/tests/download_helpers/interface.py index 87bdf7e6..b888ddef 100644 --- a/tests/download_helpers/interface.py +++ b/tests/download_helpers/interface.py @@ -10,51 +10,52 @@ class DownloadHelperTest(test_lib.BaseTestCase): - """Tests for the download helper.""" + """Tests for the download helper.""" - _FILENAME = 'LICENSE' + _FILENAME = "LICENSE" - def setUp(self): - """Sets up a test case.""" - self._download_url = ( - f'https://raw.githubusercontent.com/log2timeline/l2tdevtools/main/' - f'{self._FILENAME:s}') + def setUp(self): + """Sets up a test case.""" + self._download_url = ( + f"https://raw.githubusercontent.com/log2timeline/l2tdevtools/main/" + f"{self._FILENAME:s}" + ) - def testDownloadPageContent(self): - """Tests the DownloadPageContent functions.""" - download_helper = interface.DownloadHelper('') + def testDownloadPageContent(self): + """Tests the DownloadPageContent functions.""" + download_helper = interface.DownloadHelper("") - page_content = download_helper.DownloadPageContent(self._download_url) + page_content = download_helper.DownloadPageContent(self._download_url) - expected_page_content = b'' - with open(self._FILENAME, 'rb') as file_object: - expected_page_content = file_object.read() - expected_page_content = expected_page_content.decode('utf-8') + expected_page_content = b"" + with open(self._FILENAME, "rb") as file_object: + expected_page_content = file_object.read() + expected_page_content = expected_page_content.decode("utf-8") - self.assertEqual(page_content, expected_page_content) + self.assertEqual(page_content, expected_page_content) - def testDownloadFile(self): - """Tests the DownloadFile functions.""" - download_helper = interface.DownloadHelper(self._download_url) + def testDownloadFile(self): + """Tests the DownloadFile functions.""" + download_helper = interface.DownloadHelper(self._download_url) - current_working_directory = os.getcwd() + current_working_directory = os.getcwd() - page_content = b'' - with test_lib.TempDirectory() as temporary_directory: - os.chdir(temporary_directory) - filename = download_helper.DownloadFile(self._download_url) + page_content = b"" + with test_lib.TempDirectory() as temporary_directory: + os.chdir(temporary_directory) + filename = download_helper.DownloadFile(self._download_url) - with open(filename, 'rb') as file_object: - page_content = file_object.read() + with open(filename, "rb") as file_object: + page_content = file_object.read() - os.chdir(current_working_directory) + os.chdir(current_working_directory) - expected_page_content = b'' - with open(self._FILENAME, 'rb') as file_object: - expected_page_content = file_object.read() + expected_page_content = b"" + with open(self._FILENAME, "rb") as file_object: + expected_page_content = file_object.read() - self.assertEqual(page_content, expected_page_content) + self.assertEqual(page_content, expected_page_content) -if __name__ == '__main__': - unittest.main() +if __name__ == "__main__": + unittest.main() diff --git a/tests/download_helpers/pypi.py b/tests/download_helpers/pypi.py index 4ff04500..d9a8d0d8 100644 --- a/tests/download_helpers/pypi.py +++ b/tests/download_helpers/pypi.py @@ -12,70 +12,73 @@ class PyPIDownloadHelperTest(test_lib.BaseTestCase): - """Tests for the PyPi download helper.""" + """Tests for the PyPi download helper.""" - _DOWNLOAD_URL = 'https://pypi.org/project/dfvfs' - _GIT_URL = 'https://github.com/log2timeline/dfvfs.git' + _DOWNLOAD_URL = "https://pypi.org/project/dfvfs" + _GIT_URL = "https://github.com/log2timeline/dfvfs.git" - _PROJECT_NAME = 'dfvfs' - _PROJECT_VERSION = '20260411' - _PYPI_VERSION = '20260411' + _PROJECT_NAME = "dfvfs" + _PROJECT_VERSION = "20260411" + _PYPI_VERSION = "20260411" - @classmethod - def setUpClass(cls): - """Determines the project version from the latest git tag.""" - arguments = shlex.split(f'git ls-remote --tags {cls._GIT_URL:s}') + @classmethod + def setUpClass(cls): + """Determines the project version from the latest git tag.""" + arguments = shlex.split(f"git ls-remote --tags {cls._GIT_URL:s}") - try: - with subprocess.Popen( - arguments, stderr=subprocess.PIPE, stdout=subprocess.PIPE) as process: - output, _ = process.communicate() - if process.returncode != 0: - return + try: + with subprocess.Popen( + arguments, stderr=subprocess.PIPE, stdout=subprocess.PIPE + ) as process: + output, _ = process.communicate() + if process.returncode != 0: + return - except OSError: - return + except OSError: + return - output = output.decode('ascii') + output = output.decode("ascii") - latest_version = '0' - for line in output.split('\n'): - line = line.strip() - if 'refs/tags/' in line and not line.endswith('^{}'): - _, _, version = line.rpartition('refs/tags/') - latest_version = max(latest_version, version) + latest_version = "0" + for line in output.split("\n"): + line = line.strip() + if "refs/tags/" in line and not line.endswith("^{}"): + _, _, version = line.rpartition("refs/tags/") + latest_version = max(latest_version, version) - cls._PROJECT_VERSION = latest_version + cls._PROJECT_VERSION = latest_version - def testGetLatestVersion(self): - """Tests the GetLatestVersion functions.""" - download_helper = pypi.PyPIDownloadHelper(self._DOWNLOAD_URL) + def testGetLatestVersion(self): + """Tests the GetLatestVersion functions.""" + download_helper = pypi.PyPIDownloadHelper(self._DOWNLOAD_URL) - latest_version = download_helper.GetLatestVersion(self._PROJECT_NAME, None) + latest_version = download_helper.GetLatestVersion(self._PROJECT_NAME, None) - self.assertEqual(latest_version, self._PYPI_VERSION) + self.assertEqual(latest_version, self._PYPI_VERSION) - def testGetDownloadURL(self): - """Tests the GetDownloadURL functions.""" - download_helper = pypi.PyPIDownloadHelper(self._DOWNLOAD_URL) + def testGetDownloadURL(self): + """Tests the GetDownloadURL functions.""" + download_helper = pypi.PyPIDownloadHelper(self._DOWNLOAD_URL) - expected_download_url_regexp = re.compile( - f'https://files.pythonhosted.org/packages/' - f'[\\da-f/]+{self._PROJECT_NAME:s}-\\d{{8}}.tar.gz') + expected_download_url_regexp = re.compile( + f"https://files.pythonhosted.org/packages/" + f"[\\da-f/]+{self._PROJECT_NAME:s}-\\d{{8}}.tar.gz" + ) - download_url = download_helper.GetDownloadURL( - self._PROJECT_NAME, self._PYPI_VERSION) + download_url = download_helper.GetDownloadURL( + self._PROJECT_NAME, self._PYPI_VERSION + ) - self.assertRegex(download_url, expected_download_url_regexp) + self.assertRegex(download_url, expected_download_url_regexp) - def testGetProjectIdentifier(self): - """Tests the GetProjectIdentifier functions.""" - download_helper = pypi.PyPIDownloadHelper(self._DOWNLOAD_URL) + def testGetProjectIdentifier(self): + """Tests the GetProjectIdentifier functions.""" + download_helper = pypi.PyPIDownloadHelper(self._DOWNLOAD_URL) - project_identifier = download_helper.GetProjectIdentifier() + project_identifier = download_helper.GetProjectIdentifier() - self.assertEqual(project_identifier, f'org.pypi.{self._PROJECT_NAME:s}') + self.assertEqual(project_identifier, f"org.pypi.{self._PROJECT_NAME:s}") -if __name__ == '__main__': - unittest.main() +if __name__ == "__main__": + unittest.main() diff --git a/tests/dpkg_files.py b/tests/dpkg_files.py index b7d02242..85f3257b 100644 --- a/tests/dpkg_files.py +++ b/tests/dpkg_files.py @@ -10,28 +10,29 @@ class DPKGBuildFilesGeneratorTest(test_lib.BaseTestCase): - """Tests for the dpkg build files generator.""" + """Tests for the dpkg build files generator.""" - def testGenerateChangelogFile(self): - """Tests the _GenerateChangelogFile function.""" - project_definition = projects.ProjectDefinition('test') + def testGenerateChangelogFile(self): + """Tests the _GenerateChangelogFile function.""" + project_definition = projects.ProjectDefinition("test") - dpkg_files_generator = dpkg_files.DPKGBuildFilesGenerator( - project_definition, '1.0', 'data_path', {}) + dpkg_files_generator = dpkg_files.DPKGBuildFilesGenerator( + project_definition, "1.0", "data_path", {} + ) - _ = dpkg_files_generator - # TODO: implement tests. + _ = dpkg_files_generator + # TODO: implement tests. - # TODO: test _GenerateCompatFile function. - # TODO: test _GenerateControlFile function. - # TODO: test _GenerateCopyrightFile function. - # TODO: test _GenerateDocsFiles function. - # TODO: test _GenerateRulesFile function. - # TODO: test _GenerateConfigureMakeRulesFile function. - # TODO: test _GenerateSetupPyRulesFile function. - # TODO: test _GenerateSourceFormatFile function. - # TODO: test GenerateFiles function. + # TODO: test _GenerateCompatFile function. + # TODO: test _GenerateControlFile function. + # TODO: test _GenerateCopyrightFile function. + # TODO: test _GenerateDocsFiles function. + # TODO: test _GenerateRulesFile function. + # TODO: test _GenerateConfigureMakeRulesFile function. + # TODO: test _GenerateSetupPyRulesFile function. + # TODO: test _GenerateSourceFormatFile function. + # TODO: test GenerateFiles function. -if __name__ == '__main__': - unittest.main() +if __name__ == "__main__": + unittest.main() diff --git a/tests/helpers/project.py b/tests/helpers/project.py index 8d620111..a7645545 100644 --- a/tests/helpers/project.py +++ b/tests/helpers/project.py @@ -9,19 +9,19 @@ class ProjectHelperTest(test_lib.BaseTestCase): - """Tests the project helper.""" + """Tests the project helper.""" - # pylint: disable=protected-access + # pylint: disable=protected-access - def testInitialize(self): - """Tests that the helper can be initialized.""" - helper = project.ProjectHelper('/home/plaso/l2tdevtools/review.py') - self.assertIsNotNone(helper) + def testInitialize(self): + """Tests that the helper can be initialized.""" + helper = project.ProjectHelper("/home/plaso/l2tdevtools/review.py") + self.assertIsNotNone(helper) - # TODO: add test for version_file_path property - # TODO: add test for _GetProjectName - # TODO: add test for _ReadFileContents + # TODO: add test for version_file_path property + # TODO: add test for _GetProjectName + # TODO: add test for _ReadFileContents -if __name__ == '__main__': - unittest.main() +if __name__ == "__main__": + unittest.main() diff --git a/tests/presets.py b/tests/presets.py index 88b2963d..1388cb36 100644 --- a/tests/presets.py +++ b/tests/presets.py @@ -9,16 +9,16 @@ class PresetDefinitionTest(test_lib.BaseTestCase): - """Tests for the preset definition.""" + """Tests for the preset definition.""" - def testInitialize(self): - """Tests the __init__ function.""" - preset_definition = presets.PresetDefinition('test') - self.assertIsNotNone(preset_definition) + def testInitialize(self): + """Tests the __init__ function.""" + preset_definition = presets.PresetDefinition("test") + self.assertIsNotNone(preset_definition) # TODO: add tests. -if __name__ == '__main__': - unittest.main() +if __name__ == "__main__": + unittest.main() diff --git a/tests/projects.py b/tests/projects.py index ae8114a2..a4d91bc8 100644 --- a/tests/projects.py +++ b/tests/projects.py @@ -11,90 +11,87 @@ class ProjectDefinitionTest(test_lib.BaseTestCase): - """Tests for the project definition.""" + """Tests for the project definition.""" - def testInitialize(self): - """Tests the __init__ function.""" - project_definition = projects.ProjectDefinition('name') - self.assertIsNotNone(project_definition) + def testInitialize(self): + """Tests the __init__ function.""" + project_definition = projects.ProjectDefinition("name") + self.assertIsNotNone(project_definition) class ProjectVersionDefinitionTest(test_lib.BaseTestCase): - """Tests for the project version definition.""" + """Tests for the project version definition.""" - def testInitialize(self): - """Tests the __init__ function.""" - project_version_definition = projects.ProjectVersionDefinition('') - self.assertIsNotNone(project_version_definition) + def testInitialize(self): + """Tests the __init__ function.""" + project_version_definition = projects.ProjectVersionDefinition("") + self.assertIsNotNone(project_version_definition) - project_version_definition = projects.ProjectVersionDefinition('>1.0') - self.assertIsNotNone(project_version_definition) + project_version_definition = projects.ProjectVersionDefinition(">1.0") + self.assertIsNotNone(project_version_definition) - project_version_definition = projects.ProjectVersionDefinition('<=1.0') - self.assertIsNotNone(project_version_definition) + project_version_definition = projects.ProjectVersionDefinition("<=1.0") + self.assertIsNotNone(project_version_definition) - project_version_definition = projects.ProjectVersionDefinition( - '>=1.0,<2.0') - self.assertIsNotNone(project_version_definition) + project_version_definition = projects.ProjectVersionDefinition(">=1.0,<2.0") + self.assertIsNotNone(project_version_definition) - project_version_definition = projects.ProjectVersionDefinition( - '>=1.0,==2.0') - self.assertIsNotNone(project_version_definition) + project_version_definition = projects.ProjectVersionDefinition(">=1.0,==2.0") + self.assertIsNotNone(project_version_definition) - project_version_definition = projects.ProjectVersionDefinition( - '>=1.0,<2.0,>3.0') - self.assertIsNotNone(project_version_definition) + project_version_definition = projects.ProjectVersionDefinition( + ">=1.0,<2.0,>3.0" + ) + self.assertIsNotNone(project_version_definition) - project_version_definition = projects.ProjectVersionDefinition('bogus') - self.assertIsNotNone(project_version_definition) + project_version_definition = projects.ProjectVersionDefinition("bogus") + self.assertIsNotNone(project_version_definition) - def testVersionStringAttribute(self): - """Tests the version_string attribute.""" - project_version_definition = projects.ProjectVersionDefinition('>1.0') - self.assertIsNotNone(project_version_definition) + def testVersionStringAttribute(self): + """Tests the version_string attribute.""" + project_version_definition = projects.ProjectVersionDefinition(">1.0") + self.assertIsNotNone(project_version_definition) - self.assertEqual(project_version_definition.version_string, '>1.0') + self.assertEqual(project_version_definition.version_string, ">1.0") - def testGetEarliestVersion(self): - """Tests the GetEarliestVersion function.""" - project_version_definition = projects.ProjectVersionDefinition('>1.0') - self.assertIsNotNone(project_version_definition) + def testGetEarliestVersion(self): + """Tests the GetEarliestVersion function.""" + project_version_definition = projects.ProjectVersionDefinition(">1.0") + self.assertIsNotNone(project_version_definition) - earliest_version = project_version_definition.GetEarliestVersion() - self.assertEqual(earliest_version, ['>', '1', '0']) + earliest_version = project_version_definition.GetEarliestVersion() + self.assertEqual(earliest_version, [">", "1", "0"]) class ProjectDefinitionReaderTest(test_lib.BaseTestCase): - """Tests for the project definition reader.""" + """Tests for the project definition reader.""" - # TODO: test _GetConfigValue function. + # TODO: test _GetConfigValue function. - def testRead(self): - """Tests the Read function.""" - config_file = os.path.join('data', 'projects.ini') + def testRead(self): + """Tests the Read function.""" + config_file = os.path.join("data", "projects.ini") - project_definitions = {} - with io.open(config_file, 'r', encoding='utf-8') as file_object: - project_definition_reader = projects.ProjectDefinitionReader() - for project_definition in project_definition_reader.Read( - file_object): - project_definitions[project_definition.name] = ( - project_definition) + project_definitions = {} + with io.open(config_file, "r", encoding="utf-8") as file_object: + project_definition_reader = projects.ProjectDefinitionReader() + for project_definition in project_definition_reader.Read(file_object): + project_definitions[project_definition.name] = project_definition - self.assertGreaterEqual(len(project_definitions), 1) + self.assertGreaterEqual(len(project_definitions), 1) - project_definition = project_definitions['artifacts'] + project_definition = project_definitions["artifacts"] - self.assertEqual(project_definition.name, 'artifacts') - self.assertIsNotNone(project_definition.version) + self.assertEqual(project_definition.name, "artifacts") + self.assertIsNotNone(project_definition.version) - self.assertEqual( - project_definition.version.version_string, '>=20150409') + self.assertEqual(project_definition.version.version_string, ">=20150409") - expected_download_url = ( - 'https://github.com/ForensicArtifacts/artifacts/releases') - self.assertEqual(project_definition.download_url, expected_download_url) + expected_download_url = ( + "https://github.com/ForensicArtifacts/artifacts/releases" + ) + self.assertEqual(project_definition.download_url, expected_download_url) -if __name__ == '__main__': - unittest.main() +if __name__ == "__main__": + unittest.main() diff --git a/tests/review_helpers/cli.py b/tests/review_helpers/cli.py index 4ce50295..f3357167 100644 --- a/tests/review_helpers/cli.py +++ b/tests/review_helpers/cli.py @@ -9,25 +9,25 @@ class CLIHelperTest(test_lib.BaseTestCase): - """Tests the command line helper.""" - - def testRunCommand(self): - """Tests that the helper can be initialized.""" - mock_responses = {'echo hi': [0, 'hi\n', '']} - test_helper = cli.CLIHelper(mock_responses=mock_responses) - with self.assertRaises(AttributeError): - test_helper.RunCommand('echo hello') - exit_code, stdout, stderr = test_helper.RunCommand('echo hi') - self.assertEqual(exit_code, 0) - self.assertEqual(stdout, 'hi\n') - self.assertEqual(stderr, '') - - real_helper = cli.CLIHelper() - exit_code, stdout, stderr = real_helper.RunCommand('echo hello') - self.assertEqual(exit_code, 0) - self.assertEqual(stdout, 'hello\n') - self.assertEqual(stderr, '') - - -if __name__ == '__main__': - unittest.main() + """Tests the command line helper.""" + + def testRunCommand(self): + """Tests that the helper can be initialized.""" + mock_responses = {"echo hi": [0, "hi\n", ""]} + test_helper = cli.CLIHelper(mock_responses=mock_responses) + with self.assertRaises(AttributeError): + test_helper.RunCommand("echo hello") + exit_code, stdout, stderr = test_helper.RunCommand("echo hi") + self.assertEqual(exit_code, 0) + self.assertEqual(stdout, "hi\n") + self.assertEqual(stderr, "") + + real_helper = cli.CLIHelper() + exit_code, stdout, stderr = real_helper.RunCommand("echo hello") + self.assertEqual(exit_code, 0) + self.assertEqual(stdout, "hello\n") + self.assertEqual(stderr, "") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/review_helpers/git.py b/tests/review_helpers/git.py index 27860e8b..23dd3579 100644 --- a/tests/review_helpers/git.py +++ b/tests/review_helpers/git.py @@ -9,13 +9,13 @@ class GitHelperTest(test_lib.BaseTestCase): - """Tests the git helper.""" + """Tests the git helper.""" - def testInitialize(self): - """Tests that the helper can be initialized.""" - helper = git.GitHelper('https://github.com/log2timeline/l2tdevtools.git') - self.assertIsNotNone(helper) + def testInitialize(self): + """Tests that the helper can be initialized.""" + helper = git.GitHelper("https://github.com/log2timeline/l2tdevtools.git") + self.assertIsNotNone(helper) -if __name__ == '__main__': - unittest.main() +if __name__ == "__main__": + unittest.main() diff --git a/tests/review_helpers/github.py b/tests/review_helpers/github.py index 452c86b3..a6d4d5fc 100644 --- a/tests/review_helpers/github.py +++ b/tests/review_helpers/github.py @@ -11,62 +11,57 @@ class GitHubHelperTest(shared_test_lib.BaseTestCase): - """Tests the command line helper.""" + """Tests the command line helper.""" - # pylint: disable=protected-access + # pylint: disable=protected-access - def testAssignPullRequest(self): - """Tests the AssignPullReview function.""" - helper = github.GitHubHelper( - organization='test', project='test_project') - helper._url_lib_helper = test_lib.TestURLLibHelper() + def testAssignPullRequest(self): + """Tests the AssignPullReview function.""" + helper = github.GitHubHelper(organization="test", project="test_project") + helper._url_lib_helper = test_lib.TestURLLibHelper() - result = helper.AssignPullRequest(4, 'TOKEN', ['Onager']) + result = helper.AssignPullRequest(4, "TOKEN", ["Onager"]) - self.assertTrue(result) + self.assertTrue(result) - def testCreatePullRequest(self): - """Tests the CreatePullRequest function.""" - create_result = json.dumps({"number": 1}) + def testCreatePullRequest(self): + """Tests the CreatePullRequest function.""" + create_result = json.dumps({"number": 1}) - helper = github.GitHubHelper( - organization='test', project='test_project') - helper._url_lib_helper = test_lib.TestURLLibHelper(result=create_result) + helper = github.GitHubHelper(organization="test", project="test_project") + helper._url_lib_helper = test_lib.TestURLLibHelper(result=create_result) - result = helper.CreatePullRequest('TOKEN', 'origin', 'title', 'body') - self.assertEqual(result, 1) + result = helper.CreatePullRequest("TOKEN", "origin", "title", "body") + self.assertEqual(result, 1) - def testCreatePullRequestReview(self): - """Tests the CreatePullRequestReview function.""" - helper = github.GitHubHelper( - organization='test', project='test_project') - helper._url_lib_helper = test_lib.TestURLLibHelper() + def testCreatePullRequestReview(self): + """Tests the CreatePullRequestReview function.""" + helper = github.GitHubHelper(organization="test", project="test_project") + helper._url_lib_helper = test_lib.TestURLLibHelper() - result = helper.CreatePullRequestReview(4, 'TOKEN', ['Onager']) + result = helper.CreatePullRequestReview(4, "TOKEN", ["Onager"]) - self.assertTrue(result) + self.assertTrue(result) - def testGetForkGitRepoUrl(self): - """Tests the GetForkGitRepoUrl function.""" - helper = github.GitHubHelper( - organization='test', project='test_project') - helper._url_lib_helper = test_lib.TestURLLibHelper() + def testGetForkGitRepoUrl(self): + """Tests the GetForkGitRepoUrl function.""" + helper = github.GitHubHelper(organization="test", project="test_project") + helper._url_lib_helper = test_lib.TestURLLibHelper() - expected_url = 'https://github.com/test_user/test_project.git' - url = helper.GetForkGitRepoUrl('test_user') - self.assertEqual(url, expected_url) + expected_url = "https://github.com/test_user/test_project.git" + url = helper.GetForkGitRepoUrl("test_user") + self.assertEqual(url, expected_url) - # TODO: add tests for GetUsername. + # TODO: add tests for GetUsername. - def testQueryUser(self): - """Tests the QueryUser function.""" - helper = github.GitHubHelper( - organization='test', project='test_project') - helper._url_lib_helper = test_lib.TestURLLibHelper() + def testQueryUser(self): + """Tests the QueryUser function.""" + helper = github.GitHubHelper(organization="test", project="test_project") + helper._url_lib_helper = test_lib.TestURLLibHelper() - result = helper.QueryUser('test_user') - self.assertIsNone(result) + result = helper.QueryUser("test_user") + self.assertIsNone(result) -if __name__ == '__main__': - unittest.main() +if __name__ == "__main__": + unittest.main() diff --git a/tests/review_helpers/pylint.py b/tests/review_helpers/pylint.py index da1ee292..f64dc88a 100644 --- a/tests/review_helpers/pylint.py +++ b/tests/review_helpers/pylint.py @@ -9,25 +9,25 @@ class PylintHelperTest(test_lib.BaseTestCase): - """Tests the pylint helper.""" + """Tests the pylint helper.""" - # pylint: disable=protected-access + # pylint: disable=protected-access - def testInitialize(self): - """Tests the __init__ function.""" - helper = pylint.PylintHelper() - self.assertIsNotNone(helper) + def testInitialize(self): + """Tests the __init__ function.""" + helper = pylint.PylintHelper() + self.assertIsNotNone(helper) - def testGetVersion(self): - """Tests the _GetVersion function.""" - helper = pylint.PylintHelper() + def testGetVersion(self): + """Tests the _GetVersion function.""" + helper = pylint.PylintHelper() - helper._GetVersion() + helper._GetVersion() - # TODO: add tests for CheckFiles - # TODO: add tests for CheckUpToDateVersion - # TODO: add tests for GetRCFile + # TODO: add tests for CheckFiles + # TODO: add tests for CheckUpToDateVersion + # TODO: add tests for GetRCFile -if __name__ == '__main__': - unittest.main() +if __name__ == "__main__": + unittest.main() diff --git a/tests/review_helpers/review.py b/tests/review_helpers/review.py index 22426ef9..ae3c5f85 100644 --- a/tests/review_helpers/review.py +++ b/tests/review_helpers/review.py @@ -9,28 +9,32 @@ class ReviewHelperTest(test_lib.BaseTestCase): - """Tests the review helper.""" - - def testInitialize(self): - """Tests that the helper can be initialized.""" - helper = review.ReviewHelper( - 'test', '.', 'https://github.com/log2timeline/l2tdevtools.git', - 'import', 'upstream/main') - self.assertIsNotNone(helper) - - # TODO: test CheckLocalGitState. - # TODO: test CheckRemoteGitState. - # TODO: test Close. - # TODO: test CreatePullRequest. - # TODO: test InitializeHelpers. - # TODO: test Lint. - # TODO: test Merge. - # TODO: test PrepareUpdate. - # TODO: test PullChangesFromFork. - # TODO: test Test. - # TODO: test UpdateAuthors. - # TODO: test UpdateVersion. - - -if __name__ == '__main__': - unittest.main() + """Tests the review helper.""" + + def testInitialize(self): + """Tests that the helper can be initialized.""" + helper = review.ReviewHelper( + "test", + ".", + "https://github.com/log2timeline/l2tdevtools.git", + "import", + "upstream/main", + ) + self.assertIsNotNone(helper) + + # TODO: test CheckLocalGitState. + # TODO: test CheckRemoteGitState. + # TODO: test Close. + # TODO: test CreatePullRequest. + # TODO: test InitializeHelpers. + # TODO: test Lint. + # TODO: test Merge. + # TODO: test PrepareUpdate. + # TODO: test PullChangesFromFork. + # TODO: test Test. + # TODO: test UpdateAuthors. + # TODO: test UpdateVersion. + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/review_helpers/test_lib.py b/tests/review_helpers/test_lib.py index fee8d886..b1c24f78 100644 --- a/tests/review_helpers/test_lib.py +++ b/tests/review_helpers/test_lib.py @@ -2,28 +2,28 @@ class TestURLLibHelper: - """URL library (urllib) helper for testing.""" + """URL library (urllib) helper for testing.""" - def __init__(self, result=b''): - """Initializes an URL library (urllib) helper. + def __init__(self, result=b""): + """Initializes an URL library (urllib) helper. - Args: - result (bytes): result that should be returned. - """ - super().__init__() - self._result = result + Args: + result (bytes): result that should be returned. + """ + super().__init__() + self._result = result - # pylint: disable=unused-argument - def Request(self, url, **unused_kwargs): - """Sends a request to an URL. + # pylint: disable=unused-argument + def Request(self, url, **unused_kwargs): + """Sends a request to an URL. - Args: - url (str): URL to send the request. + Args: + url (str): URL to send the request. - Returns: - bytes: response data. + Returns: + bytes: response data. - Raises: - ConnectivityError: if the request failed. - """ - return self._result + Raises: + ConnectivityError: if the request failed. + """ + return self._result diff --git a/tests/schema_extractor.py b/tests/schema_extractor.py index 9bb9a718..613da886 100644 --- a/tests/schema_extractor.py +++ b/tests/schema_extractor.py @@ -9,18 +9,20 @@ class SQLiteSchemaExtractorTest(test_lib.BaseTestCase): - """Tests for the SQLite database file schema extractor.""" - - _TEST_SCHEMA = { - 'moz_downloads': ( - 'CREATE TABLE moz_downloads (id INTEGER PRIMARY KEY, name TEXT, ' - 'source TEXT, target TEXT, tempPath TEXT, startTime INTEGER, endTime ' - 'INTEGER, state INTEGER, referrer TEXT, entityID TEXT, currBytes ' - 'INTEGER NOT NULL DEFAULT 0, maxBytes INTEGER NOT NULL DEFAULT -1, ' - 'mimeType TEXT, preferredApplication TEXT, preferredAction INTEGER ' - 'NOT NULL DEFAULT 0, autoResume INTEGER NOT NULL DEFAULT 0)')} - - _TEST_FORMATTED_SCHEMA = """\ + """Tests for the SQLite database file schema extractor.""" + + _TEST_SCHEMA = { + "moz_downloads": ( + "CREATE TABLE moz_downloads (id INTEGER PRIMARY KEY, name TEXT, " + "source TEXT, target TEXT, tempPath TEXT, startTime INTEGER, endTime " + "INTEGER, state INTEGER, referrer TEXT, entityID TEXT, currBytes " + "INTEGER NOT NULL DEFAULT 0, maxBytes INTEGER NOT NULL DEFAULT -1, " + "mimeType TEXT, preferredApplication TEXT, preferredAction INTEGER " + "NOT NULL DEFAULT 0, autoResume INTEGER NOT NULL DEFAULT 0)" + ) + } + + _TEST_FORMATTED_SCHEMA = """\ 'moz_downloads': ( 'CREATE TABLE moz_downloads (id INTEGER PRIMARY KEY, name TEXT, ' 'source TEXT, target TEXT, tempPath TEXT, startTime INTEGER, ' @@ -30,23 +32,23 @@ class SQLiteSchemaExtractorTest(test_lib.BaseTestCase): 'preferredAction INTEGER NOT NULL DEFAULT 0, autoResume INTEGER NOT ' 'NULL DEFAULT 0)')}]""" - def testFormatSchema(self): - """Tests the FormatSchema function.""" - test_extractor = schema_extractor.SQLiteSchemaExtractor() - schema = test_extractor.FormatSchema(self._TEST_SCHEMA) + def testFormatSchema(self): + """Tests the FormatSchema function.""" + test_extractor = schema_extractor.SQLiteSchemaExtractor() + schema = test_extractor.FormatSchema(self._TEST_SCHEMA) - self.assertEqual(schema, self._TEST_FORMATTED_SCHEMA) + self.assertEqual(schema, self._TEST_FORMATTED_SCHEMA) - def testGetDatabaseSchema(self): - """Tests the GetDatabaseSchema function.""" - test_path = self._GetTestFilePath(['downloads.sqlite']) - self._SkipIfPathNotExists(test_path) + def testGetDatabaseSchema(self): + """Tests the GetDatabaseSchema function.""" + test_path = self._GetTestFilePath(["downloads.sqlite"]) + self._SkipIfPathNotExists(test_path) - test_extractor = schema_extractor.SQLiteSchemaExtractor() - schema = test_extractor.GetDatabaseSchema(test_path) + test_extractor = schema_extractor.SQLiteSchemaExtractor() + schema = test_extractor.GetDatabaseSchema(test_path) - self.assertEqual(schema, self._TEST_SCHEMA) + self.assertEqual(schema, self._TEST_SCHEMA) -if __name__ == '__main__': - unittest.main() +if __name__ == "__main__": + unittest.main() diff --git a/tests/source_helper.py b/tests/source_helper.py index 731ff445..6f96d9de 100644 --- a/tests/source_helper.py +++ b/tests/source_helper.py @@ -9,15 +9,15 @@ class SourceHelperTest(test_lib.BaseTestCase): - """Tests the helper to manager project source code.""" + """Tests the helper to manager project source code.""" - def testInitialize(self): - """Tests the __init__ function.""" - source_helper_object = source_helper.SourceHelper('test', None) - self.assertIsNotNone(source_helper_object) + def testInitialize(self): + """Tests the __init__ function.""" + source_helper_object = source_helper.SourceHelper("test", None) + self.assertIsNotNone(source_helper_object) - # TODO: more add tests. + # TODO: more add tests. -if __name__ == '__main__': - unittest.main() +if __name__ == "__main__": + unittest.main() diff --git a/tests/spec_file.py b/tests/spec_file.py index f7124889..d510a8e3 100644 --- a/tests/spec_file.py +++ b/tests/spec_file.py @@ -9,55 +9,58 @@ class RPMSpecFileGeneratorTest(test_lib.BaseTestCase): - """Tests for the RPM spec file generator.""" + """Tests for the RPM spec file generator.""" - # pylint: disable=protected-access + # pylint: disable=protected-access - def testGetBuildDefinition(self): - """Tests the _GetBuildDefinition function.""" - spec_file_generator = spec_file.RPMSpecFileGenerator('') + def testGetBuildDefinition(self): + """Tests the _GetBuildDefinition function.""" + spec_file_generator = spec_file.RPMSpecFileGenerator("") - _ = spec_file_generator - # TODO: implement tests. + _ = spec_file_generator + # TODO: implement tests. - # TODO: test _GetDocumentationFilesDefinition function. - # TODO: test _GetInstallDefinition function. - # TODO: test _GetLicenseFileDefinition function. - # TODO: test _WriteChangeLog function. - # TODO: test _WritePython2PackageDefinition function. - # TODO: test _WritePython2PackageFiles function. - # TODO: test _WritePython3PackageDefinition function. - # TODO: test _WritePython3PackageFiles function. - # TODO: test GenerateWithSetupPy function. - # TODO: test _RewriteSetupPyGeneratedFile function. + # TODO: test _GetDocumentationFilesDefinition function. + # TODO: test _GetInstallDefinition function. + # TODO: test _GetLicenseFileDefinition function. + # TODO: test _WriteChangeLog function. + # TODO: test _WritePython2PackageDefinition function. + # TODO: test _WritePython2PackageFiles function. + # TODO: test _WritePython3PackageDefinition function. + # TODO: test _WritePython3PackageFiles function. + # TODO: test GenerateWithSetupPy function. + # TODO: test _RewriteSetupPyGeneratedFile function. - def testSplitRequires(self): - """Tests the _SplitRequires function.""" - spec_file_generator = spec_file.RPMSpecFileGenerator('') + def testSplitRequires(self): + """Tests the _SplitRequires function.""" + spec_file_generator = spec_file.RPMSpecFileGenerator("") - requires_list = spec_file_generator._SplitRequires( - 'Requires: libbde, liblnk >= 20190520') + requires_list = spec_file_generator._SplitRequires( + "Requires: libbde, liblnk >= 20190520" + ) - self.assertEqual(requires_list, ['libbde', 'liblnk >= 20190520']) + self.assertEqual(requires_list, ["libbde", "liblnk >= 20190520"]) - requires_list = spec_file_generator._SplitRequires( - 'Requires: libbde liblnk >= 20190520') + requires_list = spec_file_generator._SplitRequires( + "Requires: libbde liblnk >= 20190520" + ) - self.assertEqual(requires_list, ['libbde', 'liblnk >= 20190520']) + self.assertEqual(requires_list, ["libbde", "liblnk >= 20190520"]) - requires_list = spec_file_generator._SplitRequires( - 'Requires: liblnk >= 20190520 libbde') + requires_list = spec_file_generator._SplitRequires( + "Requires: liblnk >= 20190520 libbde" + ) - self.assertEqual(requires_list, ['libbde', 'liblnk >= 20190520']) + self.assertEqual(requires_list, ["libbde", "liblnk >= 20190520"]) - requires_list = spec_file_generator._SplitRequires(None) - self.assertEqual(requires_list, []) + requires_list = spec_file_generator._SplitRequires(None) + self.assertEqual(requires_list, []) - with self.assertRaises(ValueError): - spec_file_generator._SplitRequires('Bogus') + with self.assertRaises(ValueError): + spec_file_generator._SplitRequires("Bogus") - # TODO: test RewriteSetupPyGeneratedFile function. + # TODO: test RewriteSetupPyGeneratedFile function. -if __name__ == '__main__': - unittest.main() +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_lib.py b/tests/test_lib.py index 3087dea0..4758a97e 100644 --- a/tests/test_lib.py +++ b/tests/test_lib.py @@ -7,54 +7,54 @@ class BaseTestCase(unittest.TestCase): - """The base test case.""" + """The base test case.""" - _TEST_DATA_PATH = os.path.join(os.getcwd(), 'test_data') + _TEST_DATA_PATH = os.path.join(os.getcwd(), "test_data") - # Show full diff results, part of TestCase so does not follow our naming - # conventions. - maxDiff = None + # Show full diff results, part of TestCase so does not follow our naming + # conventions. + maxDiff = None - def _GetTestFilePath(self, path_segments): - """Retrieves the path of a test file relative to the test data directory. + def _GetTestFilePath(self, path_segments): + """Retrieves the path of a test file relative to the test data directory. - Args: - path_segments (list[str]): path segments inside the test data directory. + Args: + path_segments (list[str]): path segments inside the test data directory. - Returns: - str: a path of the test file. - """ - # Note that we need to pass the individual path segments to os.path.join - # and not a list. - return os.path.join(self._TEST_DATA_PATH, *path_segments) + Returns: + str: a path of the test file. + """ + # Note that we need to pass the individual path segments to os.path.join + # and not a list. + return os.path.join(self._TEST_DATA_PATH, *path_segments) - def _SkipIfPathNotExists(self, path): - """Skips the test if the path does not exist. + def _SkipIfPathNotExists(self, path): + """Skips the test if the path does not exist. - Args: - path (str): path of a test file. + Args: + path (str): path of a test file. - Raises: - SkipTest: if the path path does not exist and the test should be skipped. - """ - if not os.path.exists(path): - filename = os.path.basename(path) - raise unittest.SkipTest(f'missing test file: {filename:s}') + Raises: + SkipTest: if the path path does not exist and the test should be skipped. + """ + if not os.path.exists(path): + filename = os.path.basename(path) + raise unittest.SkipTest(f"missing test file: {filename:s}") class TempDirectory: - """A self cleaning temporary directory.""" + """A self cleaning temporary directory.""" - def __init__(self): - """Initializes the temporary directory.""" - super().__init__() - self.name = '' + def __init__(self): + """Initializes the temporary directory.""" + super().__init__() + self.name = "" - def __enter__(self): - """Make this work with the 'with' statement.""" - self.name = tempfile.mkdtemp() - return self.name + def __enter__(self): + """Make this work with the 'with' statement.""" + self.name = tempfile.mkdtemp() + return self.name - def __exit__(self, unused_type, unused_value, unused_traceback): - """Make this work with the 'with' statement.""" - shutil.rmtree(self.name, True) + def __exit__(self, unused_type, unused_value, unused_traceback): + """Make this work with the 'with' statement.""" + shutil.rmtree(self.name, True) diff --git a/tests/update.py b/tests/update.py index 50ece938..9a116c5c 100644 --- a/tests/update.py +++ b/tests/update.py @@ -12,93 +12,108 @@ class GithubRepoDownloadHelperTest(test_lib.BaseTestCase): - """Tests for the GitHub repo download helper class.""" + """Tests for the GitHub repo download helper class.""" - # pylint: disable=protected-access + # pylint: disable=protected-access - _DOWNLOAD_URL = 'https://github.com/ForensicArtifacts/artifacts/releases' + _DOWNLOAD_URL = "https://github.com/ForensicArtifacts/artifacts/releases" - _PROJECT_NAME = 'artifacts' - _PROJECT_VERSION = '20260421' + _PROJECT_NAME = "artifacts" + _PROJECT_VERSION = "20260421" - def testGetPackageDownloadURLs(self): - """Tests the GetPackageDownloadURLs function.""" - download_helper = update.GithubRepoDownloadHelper( - self._DOWNLOAD_URL, branch='dev') + def testGetPackageDownloadURLs(self): + """Tests the GetPackageDownloadURLs function.""" + download_helper = update.GithubRepoDownloadHelper( + self._DOWNLOAD_URL, branch="dev" + ) - package_download_urls = download_helper.GetPackageDownloadURLs( - preferred_machine_type='x86', preferred_operating_system='Windows') + package_download_urls = download_helper.GetPackageDownloadURLs( + preferred_machine_type="x86", preferred_operating_system="Windows" + ) - if (sys.version_info[0], sys.version_info[1]) not in ( - download_helper._SUPPORTED_PYTHON_VERSIONS): - self.assertIsNone(package_download_urls) + if (sys.version_info[0], sys.version_info[1]) not in ( + download_helper._SUPPORTED_PYTHON_VERSIONS + ): + self.assertIsNone(package_download_urls) - else: - self.assertIsNotNone(package_download_urls) + else: + self.assertIsNotNone(package_download_urls) - expected_url = ( - f'https://github.com/log2timeline/l2tbinaries/raw/dev/win32/' - f'{self._PROJECT_NAME:s}-{self._PROJECT_VERSION:s}-py3-none-any.whl') - self.assertIn(expected_url, package_download_urls) + expected_url = ( + f"https://github.com/log2timeline/l2tbinaries/raw/dev/win32/" + f"{self._PROJECT_NAME:s}-{self._PROJECT_VERSION:s}-py3-none-any.whl" + ) + self.assertIn(expected_url, package_download_urls) class DependencyUpdaterTest(test_lib.BaseTestCase): - """Tests for the dependency updater class.""" + """Tests for the dependency updater class.""" - # pylint: disable=protected-access + # pylint: disable=protected-access - _PROJECT_NAME = 'dfvfs' - _PROJECT_VERSION = '20260411' + _PROJECT_NAME = "dfvfs" + _PROJECT_VERSION = "20260411" - def testGetAvailableWheelPackages(self): - """Tests the _GetAvailableWheelPackages function.""" - dependency_updater = update.DependencyUpdater( - download_track='dev', preferred_machine_type='x86', - preferred_operating_system='Windows') - - available_packages = dependency_updater._GetAvailableWheelPackages() - - if (sys.version_info[0], sys.version_info[1]) not in ( - update.GithubRepoDownloadHelper._SUPPORTED_PYTHON_VERSIONS): - self.assertEqual(available_packages, []) - - else: - self.assertNotEqual(available_packages, []) - - for package_download in available_packages: - if package_download.name == self._PROJECT_NAME: - expected_package_filename = ( - f'{self._PROJECT_NAME:s}-{self._PROJECT_VERSION:s}-' - f'py3-none-any.whl') - self.assertEqual(package_download.filename, expected_package_filename) - - expected_package_version = [self._PROJECT_VERSION] - self.assertEqual(package_download.version, expected_package_version) - - def testUpdatePackages(self): - """Tests the UpdatePackages function.""" - projects_file = os.path.join('data', 'projects.ini') - - if (sys.version_info[0], sys.version_info[1]) in ( - update.GithubRepoDownloadHelper._SUPPORTED_PYTHON_VERSIONS): - with test_lib.TempDirectory() as temp_directory: + def testGetAvailableWheelPackages(self): + """Tests the _GetAvailableWheelPackages function.""" dependency_updater = update.DependencyUpdater( - download_directory=temp_directory, download_only=True, - download_track='dev', preferred_machine_type='x86', - preferred_operating_system='Windows') - - dependency_updater.UpdatePackages(projects_file, [self._PROJECT_NAME]) - - glob_results = sorted(glob.glob(os.path.join(temp_directory, '*.whl'))) - - self.assertEqual(len(glob_results), 1) - - expected_filename = ( - f'{self._PROJECT_NAME:s}-{self._PROJECT_VERSION:s}-' - f'py3-none-any.whl') - expected_path = os.path.join(temp_directory, expected_filename) - self.assertEqual(glob_results[0], expected_path) - - -if __name__ == '__main__': - unittest.main() + download_track="dev", + preferred_machine_type="x86", + preferred_operating_system="Windows", + ) + + available_packages = dependency_updater._GetAvailableWheelPackages() + + if (sys.version_info[0], sys.version_info[1]) not in ( + update.GithubRepoDownloadHelper._SUPPORTED_PYTHON_VERSIONS + ): + self.assertEqual(available_packages, []) + + else: + self.assertNotEqual(available_packages, []) + + for package_download in available_packages: + if package_download.name == self._PROJECT_NAME: + expected_package_filename = ( + f"{self._PROJECT_NAME:s}-{self._PROJECT_VERSION:s}-" + f"py3-none-any.whl" + ) + self.assertEqual( + package_download.filename, expected_package_filename + ) + + expected_package_version = [self._PROJECT_VERSION] + self.assertEqual(package_download.version, expected_package_version) + + def testUpdatePackages(self): + """Tests the UpdatePackages function.""" + projects_file = os.path.join("data", "projects.ini") + + if (sys.version_info[0], sys.version_info[1]) in ( + update.GithubRepoDownloadHelper._SUPPORTED_PYTHON_VERSIONS + ): + with test_lib.TempDirectory() as temp_directory: + dependency_updater = update.DependencyUpdater( + download_directory=temp_directory, + download_only=True, + download_track="dev", + preferred_machine_type="x86", + preferred_operating_system="Windows", + ) + + dependency_updater.UpdatePackages(projects_file, [self._PROJECT_NAME]) + + glob_results = sorted(glob.glob(os.path.join(temp_directory, "*.whl"))) + + self.assertEqual(len(glob_results), 1) + + expected_filename = ( + f"{self._PROJECT_NAME:s}-{self._PROJECT_VERSION:s}-" + f"py3-none-any.whl" + ) + expected_path = os.path.join(temp_directory, expected_filename) + self.assertEqual(glob_results[0], expected_path) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/build.py b/tools/build.py index c54efe6a..28f56d3a 100755 --- a/tools/build.py +++ b/tools/build.py @@ -24,536 +24,596 @@ # TODO: look into merging functionality with update script. -class ProjectBuilder: - """Class that helps in building projects. - - Attributes: - project_definitions (dict[str, ProjectDefinition]): project definitions. - """ - - # The distributions to build dpkg-source packages for. - _DPKG_SOURCE_DISTRIBUTIONS = frozenset(['resolute']) - - def __init__(self, build_target, l2tdevtools_path, downloads_directory): - """Initializes the project builder. - - Args: - build_target (str): build target. - l2tdevtools_path (str): path to l2tdevtools. - downloads_directory (str): path to the directory where projects are - downloaded. - """ - super().__init__() - self._build_helpers = {} - self._build_target = build_target - self._downloads_directory = downloads_directory - self._l2tdevtools_path = l2tdevtools_path - self._source_helpers = {} - - self.project_definitions = {} - - def _BuildProject( - self, build_helper_object, source_helper_object, distribution): - """Builds a project. - - Args: - build_helper_object (BuildHelper): build helper. - source_helper_object (SourceHelper): source helper. - distribution (str): name of the distribution or None if not available. - - Returns: - bool: True if the build is successful or False on error. - """ - if distribution: - build_helper_object.distribution = distribution - - build_required = build_helper_object.CheckBuildRequired( - source_helper_object) - - build_helper_object.Clean(source_helper_object) - - if not build_required or build_helper_object.Build(source_helper_object): - return True - if not os.path.exists(build_helper_object.LOG_FILENAME): - logging.warning( - f'Build of: {source_helper_object.project_name:s} failed.') - else: - log_filename = '_'.join([ - source_helper_object.project_name, build_helper_object.LOG_FILENAME]) - - # Remove older logfiles if they exists otherwise the rename - # fails on Windows. - if os.path.exists(log_filename): - os.remove(log_filename) - - os.rename(build_helper_object.LOG_FILENAME, log_filename) - logging.warning(( - f'Build of: {source_helper_object.project_name:s} failed, for more ' - f'information check {log_filename:s}')) - - return False - - def _ExpandPresets(self, preset_definitions, preset_names): - """Expands preset names to project names. - - Args: - preset_definitions (dict[str, PresetDefinition]): preset definitions. - preset_names (str): names of the presets to expand. +class ProjectBuilder: + """Class that helps in building projects. - Returns: - set[str]: project names. + Attributes: + project_definitions (dict[str, ProjectDefinition]): project definitions. """ - project_names = set() - for preset_name in preset_names: - preset_definition = preset_definitions.get(preset_name, None) - if not preset_definition: - continue - - if preset_definition.preset_names: - sub_project_names = self._ExpandPresets( - preset_definitions, preset_definition.preset_names) - project_names = project_names.union(sub_project_names) - - project_names = project_names.union( - set(preset_definition.project_names)) - - return project_names - def Build(self, project_definition, distributions=None): - """Builds a project. + # The distributions to build dpkg-source packages for. + _DPKG_SOURCE_DISTRIBUTIONS = frozenset(["resolute"]) + + def __init__(self, build_target, l2tdevtools_path, downloads_directory): + """Initializes the project builder. + + Args: + build_target (str): build target. + l2tdevtools_path (str): path to l2tdevtools. + downloads_directory (str): path to the directory where projects are + downloaded. + """ + super().__init__() + self._build_helpers = {} + self._build_target = build_target + self._downloads_directory = downloads_directory + self._l2tdevtools_path = l2tdevtools_path + self._source_helpers = {} + + self.project_definitions = {} + + def _BuildProject(self, build_helper_object, source_helper_object, distribution): + """Builds a project. + + Args: + build_helper_object (BuildHelper): build helper. + source_helper_object (SourceHelper): source helper. + distribution (str): name of the distribution or None if not available. + + Returns: + bool: True if the build is successful or False on error. + """ + if distribution: + build_helper_object.distribution = distribution + + build_required = build_helper_object.CheckBuildRequired(source_helper_object) + + build_helper_object.Clean(source_helper_object) + + if not build_required or build_helper_object.Build(source_helper_object): + return True + + if not os.path.exists(build_helper_object.LOG_FILENAME): + logging.warning(f"Build of: {source_helper_object.project_name:s} failed.") + else: + log_filename = "_".join( + [source_helper_object.project_name, build_helper_object.LOG_FILENAME] + ) + + # Remove older logfiles if they exists otherwise the rename + # fails on Windows. + if os.path.exists(log_filename): + os.remove(log_filename) + + os.rename(build_helper_object.LOG_FILENAME, log_filename) + logging.warning( + ( + f"Build of: {source_helper_object.project_name:s} failed, for more " + f"information check {log_filename:s}" + ) + ) - Args: - project_definition (ProjectDefinition): project definition. - distributions (Optional[list[str]]): distributions to build. - - Returns: - bool: True if the build is successful or False on error. - """ - build_helper_object = self._build_helpers.get( - project_definition.name, None) - if not build_helper_object: - logging.warning('Missing build helper.') - return False - - source_helper_object = self._source_helpers.get( - project_definition.name, None) - if not source_helper_object: - logging.warning('Missing source helper.') - return False - - if not distributions: - if self._build_target == 'dpkg-source': - distributions = self._DPKG_SOURCE_DISTRIBUTIONS - else: - distributions = [None] - - for distribution in distributions: - if not self._BuildProject( - build_helper_object, source_helper_object, distribution): return False - if os.path.exists(build_helper_object.LOG_FILENAME): - logging.info(f'Removing: {build_helper_object.LOG_FILENAME:s}') - os.remove(build_helper_object.LOG_FILENAME) + def _ExpandPresets(self, preset_definitions, preset_names): + """Expands preset names to project names. + + Args: + preset_definitions (dict[str, PresetDefinition]): preset definitions. + preset_names (str): names of the presets to expand. + + Returns: + set[str]: project names. + """ + project_names = set() + for preset_name in preset_names: + preset_definition = preset_definitions.get(preset_name, None) + if not preset_definition: + continue + + if preset_definition.preset_names: + sub_project_names = self._ExpandPresets( + preset_definitions, preset_definition.preset_names + ) + project_names = project_names.union(sub_project_names) + + project_names = project_names.union(set(preset_definition.project_names)) + + return project_names + + def Build(self, project_definition, distributions=None): + """Builds a project. + + Args: + project_definition (ProjectDefinition): project definition. + distributions (Optional[list[str]]): distributions to build. + + Returns: + bool: True if the build is successful or False on error. + """ + build_helper_object = self._build_helpers.get(project_definition.name, None) + if not build_helper_object: + logging.warning("Missing build helper.") + return False + + source_helper_object = self._source_helpers.get(project_definition.name, None) + if not source_helper_object: + logging.warning("Missing source helper.") + return False + + if not distributions: + if self._build_target == "dpkg-source": + distributions = self._DPKG_SOURCE_DISTRIBUTIONS + else: + distributions = [None] + + for distribution in distributions: + if not self._BuildProject( + build_helper_object, source_helper_object, distribution + ): + return False + + if os.path.exists(build_helper_object.LOG_FILENAME): + logging.info(f"Removing: {build_helper_object.LOG_FILENAME:s}") + os.remove(build_helper_object.LOG_FILENAME) + + return True + + def CheckBuildDependencies(self, project_definition): + """Checks if the build dependencies of a project are met. + + Args: + project_definition (ProjectDefinition): project definition. + + Returns: + list[str]: build dependency names that are not met or an empty list. + """ + source_helper_object = self._source_helpers.get(project_definition.name, None) + if not source_helper_object: + logging.warning("Missing source helper.") + return [] + + source_package_path = source_helper_object.GetSourcePackagePath() + if not source_package_path: + logging.info( + f"Missing source package of: {source_helper_object.project_name:s}" + ) + return [] + + if not source_helper_object.Create(): + source_filename = source_helper_object.GetSourcePackageFilename() + logging.error(f"Extraction of source package: {source_filename:s} failed") + return [] + + source_directory = source_helper_object.GetSourceDirectoryPath() + if not source_directory: + logging.info( + f"Missing source directory of: {source_helper_object.project_name:s}" + ) + return [] + + if not project_definition.build_system: + if os.path.exists(os.path.join(source_directory, "configure")): + project_definition.build_system = "configure_make" + elif os.path.exists(os.path.join(source_directory, "setup.py")): + project_definition.build_system = "setup_py" + elif os.path.exists(os.path.join(source_directory, "pyproject.toml")): + project_definition.build_system = "pyproject" + else: + logging.warning( + f"Unable to determine build system of: {project_definition.name:s}" + ) + return [] + + build_helper_object = build_helper.BuildHelperFactory.NewBuildHelper( + project_definition, + self._build_target, + self._l2tdevtools_path, + self.project_definitions, + ) + if not build_helper_object: + logging.warning( + f"Unable to determine how to build: {project_definition.name:s}" + ) + return [] + + self._build_helpers[project_definition.name] = build_helper_object + + return build_helper_object.CheckBuildDependencies() + + def CheckProjectConfiguration(self, project_definition): + """Checks if the project configuration is correct. + + Args: + project_definition (ProjectDefinition): project definition. + + Returns: + bool: True if the project configuration is correct, False otherwise. + """ + build_helper_object = self._build_helpers.get(project_definition.name, None) + if not build_helper_object: + logging.warning("Missing build helper.") + return False + + return build_helper_object.CheckProjectConfiguration() + + def Download(self, project_definition): + """Downloads the source package of a project. + + Args: + project_definition (ProjectDefinition): project definition. + + Returns: + bool: True if the download is successful or False on error. + + Raises: + ValueError: if the project download URL is not supported. + """ + download_helper_object = ( + download_helper.DownloadHelperFactory.NewDownloadHelper(project_definition) + ) + + source_helper_object = source_helper.SourcePackageHelper( + project_definition.name, + project_definition, + self._downloads_directory, + download_helper_object, + ) + + source_helper_object.Clean() + + # TODO: add a step to make sure build environment is sane + # e.g. _CheckStatusIsClean() + + source_package_path = source_helper_object.Download() + + if self._build_target == "download": + # If available run the script post-download.sh after download. + if os.path.exists("post-download.sh"): + command = f"sh ./post-download.sh {source_package_path:s}" + exit_code = subprocess.call(command, shell=True) + if exit_code != 0: + logging.error(f'Running: "{command:s}" failed.') + return False + + self._source_helpers[project_definition.name] = source_helper_object + + return True + + def ReadProjectDefinitions(self, path): + """Reads project definitions. + + Args: + path (str): path of the project definitions file. + """ + with io.open(path, "r", encoding="utf-8") as file_object: + project_definition_reader = projects.ProjectDefinitionReader() + self.project_definitions = { + definition.name: definition + for definition in project_definition_reader.Read(file_object) + } + + def ReadProjectsPreset(self, path, preset_name): + """Reads a projects preset from the preset file. + + Args: + path (str): path of the projects preset file. + preset_name (str): name of the preset. + + Returns: + list[str]: names of the projects defined by the preset or an empty list + if the preset was not defined. + """ + preset_definitions = {} + + with io.open(path, "r", encoding="utf-8") as file_object: + definition_reader = presets.PresetDefinitionReader() + preset_definitions = { + preset_definition.name: preset_definition + for preset_definition in definition_reader.Read(file_object) + } - return True + return list(self._ExpandPresets(preset_definitions, [preset_name])) - def CheckBuildDependencies(self, project_definition): - """Checks if the build dependencies of a project are met. - Args: - project_definition (ProjectDefinition): project definition. - - Returns: - list[str]: build dependency names that are not met or an empty list. - """ - source_helper_object = self._source_helpers.get( - project_definition.name, None) - if not source_helper_object: - logging.warning('Missing source helper.') - return [] - - source_package_path = source_helper_object.GetSourcePackagePath() - if not source_package_path: - logging.info( - f'Missing source package of: {source_helper_object.project_name:s}') - return [] - - if not source_helper_object.Create(): - source_filename = source_helper_object.GetSourcePackageFilename() - logging.error(f'Extraction of source package: {source_filename:s} failed') - return [] - - source_directory = source_helper_object.GetSourceDirectoryPath() - if not source_directory: - logging.info( - f'Missing source directory of: {source_helper_object.project_name:s}') - return [] - - if not project_definition.build_system: - if os.path.exists(os.path.join(source_directory, 'configure')): - project_definition.build_system = 'configure_make' - elif os.path.exists(os.path.join(source_directory, 'setup.py')): - project_definition.build_system = 'setup_py' - elif os.path.exists(os.path.join(source_directory, 'pyproject.toml')): - project_definition.build_system = 'pyproject' - else: - logging.warning( - f'Unable to determine build system of: {project_definition.name:s}') - return [] - - build_helper_object = build_helper.BuildHelperFactory.NewBuildHelper( - project_definition, self._build_target, self._l2tdevtools_path, - self.project_definitions) - if not build_helper_object: - logging.warning( - f'Unable to determine how to build: {project_definition.name:s}') - return [] - - self._build_helpers[project_definition.name] = build_helper_object - - return build_helper_object.CheckBuildDependencies() - - def CheckProjectConfiguration(self, project_definition): - """Checks if the project configuration is correct. - - Args: - project_definition (ProjectDefinition): project definition. - - Returns: - bool: True if the project configuration is correct, False otherwise. - """ - build_helper_object = self._build_helpers.get( - project_definition.name, None) - if not build_helper_object: - logging.warning('Missing build helper.') - return False - - return build_helper_object.CheckProjectConfiguration() - - def Download(self, project_definition): - """Downloads the source package of a project. - - Args: - project_definition (ProjectDefinition): project definition. +def Main(): + """The main program function. Returns: - bool: True if the download is successful or False on error. - - Raises: - ValueError: if the project download URL is not supported. + bool: True if successful or False if not. """ - download_helper_object = ( - download_helper.DownloadHelperFactory.NewDownloadHelper( - project_definition)) - - source_helper_object = source_helper.SourcePackageHelper( - project_definition.name, project_definition, self._downloads_directory, - download_helper_object) - - source_helper_object.Clean() - - # TODO: add a step to make sure build environment is sane - # e.g. _CheckStatusIsClean() - - source_package_path = source_helper_object.Download() - - if self._build_target == 'download': - # If available run the script post-download.sh after download. - if os.path.exists('post-download.sh'): - command = f'sh ./post-download.sh {source_package_path:s}' - exit_code = subprocess.call(command, shell=True) - if exit_code != 0: - logging.error(f'Running: "{command:s}" failed.') - return False - - self._source_helpers[project_definition.name] = source_helper_object - - return True - - def ReadProjectDefinitions(self, path): - """Reads project definitions. + build_targets = frozenset( + ["download", "dpkg", "dpkg-source", "rpm", "source", "srpm", "wheel"] + ) + + argument_parser = argparse.ArgumentParser( + description=("Downloads and builds the latest versions of projects.") + ) + + argument_parser.add_argument( + "build_target", + choices=sorted(build_targets), + action="store", + metavar="BUILD_TARGET", + default=None, + help="The build target.", + ) + + default_builds_directory = os.path.join("..", "l2tbuilds") + argument_parser.add_argument( + "--build-directory", + "--builds-directory", + "--build_directory", + "--builds_directory", + action="store", + metavar="DIRECTORY", + dest="builds_directory", + type=str, + default=default_builds_directory, + help="The location of the build directory.", + ) + + argument_parser.add_argument( + "-c", + "--config", + dest="config_path", + action="store", + metavar="CONFIG_PATH", + default=None, + help=( + "path of the directory containing the build configuration " + "files e.g. projects.ini." + ), + ) + + argument_parser.add_argument( + "--distributions", + dest="distributions", + action="store", + metavar="NAME(S)", + default="", + help=("comma separated list of specific distribution names to build."), + ) + + argument_parser.add_argument( + "--download-directory", + "--downloads-directory", + "--download_directory", + "--downloads_directory", + action="store", + metavar="DIRECTORY", + dest="downloads_directory", + type=str, + default=None, + help="The location of the downloads directory.", + ) + + argument_parser.add_argument( + "--preset", + dest="preset", + action="store", + metavar="PRESET_NAME", + default=None, + help=( + "name of the preset of project names to build. The default is to " + "build all project defined in the projects.ini configuration file. " + "The presets are defined in the preset.ini configuration file." + ), + ) + + argument_parser.add_argument( + "--projects", + dest="projects", + action="store", + metavar="PROJECT_NAME(S)", + default=None, + help=( + "comma separated list of specific project names to build. The " + "default is to build all project defined in the projects.ini " + "configuration file." + ), + ) + + options = argument_parser.parse_args() + + if not options.build_target: + print("Build target missing.") + print("") + argument_parser.print_help() + print("") + return False - Args: - path (str): path of the project definitions file. - """ - with io.open(path, 'r', encoding='utf-8') as file_object: - project_definition_reader = projects.ProjectDefinitionReader() - self.project_definitions = { - definition.name: definition - for definition in project_definition_reader.Read(file_object)} + if options.build_target not in build_targets: + print(f"Unsupported build target: {options.build_target:s}") + print("") + argument_parser.print_help() + print("") + return False - def ReadProjectsPreset(self, path, preset_name): - """Reads a projects preset from the preset file. + config_path = options.config_path + if not config_path: + l2tdevtools_path = os.path.dirname(__file__) + l2tdevtools_path = os.path.dirname(l2tdevtools_path) + config_path = os.path.join(l2tdevtools_path, "data") - Args: - path (str): path of the projects preset file. - preset_name (str): name of the preset. + if not options.preset and not options.projects: + print("Please define a preset or projects to build.") + print("") + return False - Returns: - list[str]: names of the projects defined by the preset or an empty list - if the preset was not defined. - """ - preset_definitions = {} + presets_file = os.path.join(config_path, "presets.ini") + if options.preset and not os.path.exists(presets_file): + print(f"No such config file: {presets_file:s}") + print("") + return False - with io.open(path, 'r', encoding='utf-8') as file_object: - definition_reader = presets.PresetDefinitionReader() - preset_definitions = { - preset_definition.name: preset_definition - for preset_definition in definition_reader.Read(file_object)} + projects_file = os.path.join(config_path, "projects.ini") + if not os.path.exists(projects_file): + print(f"No such config file: {projects_file:s}") + print("") + return False - return list(self._ExpandPresets(preset_definitions, [preset_name])) + if os.path.abspath(options.builds_directory).startswith(l2tdevtools_path): + print( + "Builds directory cannot be within l2tdevtools directory due to " + "usage of pbr" + ) + print("") + return False + logging.basicConfig(level=logging.INFO, format="[%(levelname)s] %(message)s") -def Main(): - """The main program function. - - Returns: - bool: True if successful or False if not. - """ - build_targets = frozenset([ - 'download', 'dpkg', 'dpkg-source', 'rpm', 'source', 'srpm', 'wheel']) - - argument_parser = argparse.ArgumentParser(description=( - 'Downloads and builds the latest versions of projects.')) - - argument_parser.add_argument( - 'build_target', choices=sorted(build_targets), action='store', - metavar='BUILD_TARGET', default=None, help='The build target.') - - default_builds_directory = os.path.join('..', 'l2tbuilds') - argument_parser.add_argument( - '--build-directory', '--builds-directory', '--build_directory', - '--builds_directory', action='store', metavar='DIRECTORY', - dest='builds_directory', type=str, default=default_builds_directory, - help='The location of the build directory.') - - argument_parser.add_argument( - '-c', '--config', dest='config_path', action='store', - metavar='CONFIG_PATH', default=None, help=( - 'path of the directory containing the build configuration ' - 'files e.g. projects.ini.')) - - argument_parser.add_argument( - '--distributions', dest='distributions', action='store', - metavar='NAME(S)', default='', help=( - 'comma separated list of specific distribution names to build.')) - - argument_parser.add_argument( - '--download-directory', '--downloads-directory', '--download_directory', - '--downloads_directory', action='store', metavar='DIRECTORY', - dest='downloads_directory', type=str, - default=None, help='The location of the downloads directory.') - - argument_parser.add_argument( - '--preset', dest='preset', action='store', - metavar='PRESET_NAME', default=None, help=( - 'name of the preset of project names to build. The default is to ' - 'build all project defined in the projects.ini configuration file. ' - 'The presets are defined in the preset.ini configuration file.')) - - argument_parser.add_argument( - '--projects', dest='projects', action='store', - metavar='PROJECT_NAME(S)', default=None, help=( - 'comma separated list of specific project names to build. The ' - 'default is to build all project defined in the projects.ini ' - 'configuration file.')) - - options = argument_parser.parse_args() - - if not options.build_target: - print('Build target missing.') - print('') - argument_parser.print_help() - print('') - return False - - if options.build_target not in build_targets: - print(f'Unsupported build target: {options.build_target:s}') - print('') - argument_parser.print_help() - print('') - return False - - config_path = options.config_path - if not config_path: - l2tdevtools_path = os.path.dirname(__file__) - l2tdevtools_path = os.path.dirname(l2tdevtools_path) - config_path = os.path.join(l2tdevtools_path, 'data') - - if not options.preset and not options.projects: - print('Please define a preset or projects to build.') - print('') - return False - - presets_file = os.path.join(config_path, 'presets.ini') - if options.preset and not os.path.exists(presets_file): - print(f'No such config file: {presets_file:s}') - print('') - return False - - projects_file = os.path.join(config_path, 'projects.ini') - if not os.path.exists(projects_file): - print(f'No such config file: {projects_file:s}') - print('') - return False - - if os.path.abspath(options.builds_directory).startswith(l2tdevtools_path): - print(('Builds directory cannot be within l2tdevtools directory due to ' - 'usage of pbr')) - print('') - return False - - logging.basicConfig( - level=logging.INFO, format='[%(levelname)s] %(message)s') - - distributions = options.distributions.split(',') or None - - if not options.downloads_directory: - options.downloads_directory = options.builds_directory - - project_builder = ProjectBuilder( - options.build_target, l2tdevtools_path, options.downloads_directory) - - project_names = [] - if options.preset: - project_names = project_builder.ReadProjectsPreset( - presets_file, options.preset) - if not project_names: - print(f'Undefined preset: {options.preset:s}') - print('') - return False - - elif options.projects: - project_names = options.projects.split(',') - - project_builder.ReadProjectDefinitions(projects_file) - - operating_system = platform.system().lower() - - builds = [] - disabled_projects = [] - for name, definition in project_builder.project_definitions.items(): - if name not in project_names: - continue - - is_disabled = False - if (options.build_target in definition.disabled or - operating_system in definition.disabled or - 'all' in definition.disabled): - if options.preset: - is_disabled = True - else: - # If a project is manually specified ignore the disabled status. - logging.info(f'Ignoring disabled status for: {name:s}') - - if is_disabled: - disabled_projects.append(name) + distributions = options.distributions.split(",") or None + + if not options.downloads_directory: + options.downloads_directory = options.builds_directory + + project_builder = ProjectBuilder( + options.build_target, l2tdevtools_path, options.downloads_directory + ) + + project_names = [] + if options.preset: + project_names = project_builder.ReadProjectsPreset(presets_file, options.preset) + if not project_names: + print(f"Undefined preset: {options.preset:s}") + print("") + return False + + elif options.projects: + project_names = options.projects.split(",") + + project_builder.ReadProjectDefinitions(projects_file) + + operating_system = platform.system().lower() + + builds = [] + disabled_projects = [] + for name, definition in project_builder.project_definitions.items(): + if name not in project_names: + continue + + is_disabled = False + if ( + options.build_target in definition.disabled + or operating_system in definition.disabled + or "all" in definition.disabled + ): + if options.preset: + is_disabled = True + else: + # If a project is manually specified ignore the disabled status. + logging.info(f"Ignoring disabled status for: {name:s}") + + if is_disabled: + disabled_projects.append(name) + else: + builds.append(definition) + + if not os.path.exists(options.builds_directory): + os.mkdir(options.builds_directory) + + if not os.path.exists(options.downloads_directory): + os.mkdir(options.downloads_directory) + + undefined_projects = set(project_names) + for disabled_package in disabled_projects: + undefined_projects.remove(disabled_package) + + configuration_errors = set() + failed_builds = set() + failed_downloads = set() + missing_build_dependencies = set() + + for project_definition in list(builds): + if project_names and project_definition.name not in project_names: + builds.remove(project_definition) + continue + + undefined_projects.remove(project_definition.name) + + if not project_builder.Download(project_definition): + builds.remove(project_definition) + + print(f"Failed downloading: {project_definition.name:s}") + failed_downloads.add(project_definition.name) + + if options.build_target != "download": + current_working_directory = os.getcwd() + os.chdir(options.builds_directory) + + try: + for project_definition in list(builds): + project_name = project_definition.name + dependencies = project_builder.CheckBuildDependencies( + project_definition + ) + + if dependencies: + builds.remove(project_definition) + build_dependencies = ", ".join(dependencies) + + print( + f"Unable to build: {project_name:s} missing build " + f"dependencies: {build_dependencies:s}" + ) + missing_build_dependencies.update(dependencies) + + if not project_builder.CheckProjectConfiguration(project_definition): + print(f"Detected error in configuration of: {project_name:s}") + configuration_errors.add(project_name) + + for project_definition in list(builds): + project_name = project_definition.name + logging.info(f"Building: {project_name:s}") + + # TODO: add support for dokan, bzip2 + # TODO: setup sqlite in build directory. + if not project_builder.Build( + project_definition, distributions=distributions + ): + print(f"Failed building: {project_name:s}") + failed_builds.add(project_name) + + finally: + os.chdir(current_working_directory) + + if undefined_projects: + print("") + print("Undefined projects:") + for name in sorted(undefined_projects): + print(f"\t{name:s}") + + if configuration_errors: + print("") + print("Projects with configuration errors:") + for name in sorted(configuration_errors): + print(f"\t{name:s}") + + if failed_downloads: + print("") + print("Failed downloading:") + for name in sorted(failed_downloads): + print(f"\t{name:s}") + + if missing_build_dependencies: + print("") + print("Missing build dependencies:") + for dependency in sorted(missing_build_dependencies): + print(f"\t{dependency:s}") + + if failed_builds: + print("") + print("Failed building:") + for name in sorted(failed_builds): + print(f"\t{name:s}") + + return not failed_downloads and not missing_build_dependencies and not failed_builds + + +if __name__ == "__main__": + if not Main(): + sys.exit(1) else: - builds.append(definition) - - if not os.path.exists(options.builds_directory): - os.mkdir(options.builds_directory) - - if not os.path.exists(options.downloads_directory): - os.mkdir(options.downloads_directory) - - undefined_projects = set(project_names) - for disabled_package in disabled_projects: - undefined_projects.remove(disabled_package) - - configuration_errors = set() - failed_builds = set() - failed_downloads = set() - missing_build_dependencies = set() - - for project_definition in list(builds): - if project_names and project_definition.name not in project_names: - builds.remove(project_definition) - continue - - undefined_projects.remove(project_definition.name) - - if not project_builder.Download(project_definition): - builds.remove(project_definition) - - print(f'Failed downloading: {project_definition.name:s}') - failed_downloads.add(project_definition.name) - - if options.build_target != 'download': - current_working_directory = os.getcwd() - os.chdir(options.builds_directory) - - try: - for project_definition in list(builds): - project_name = project_definition.name - dependencies = project_builder.CheckBuildDependencies( - project_definition) - - if dependencies: - builds.remove(project_definition) - build_dependencies = ', '.join(dependencies) - - print(( - f'Unable to build: {project_name:s} missing build dependencies: ' - f'{build_dependencies:s}')) - missing_build_dependencies.update(dependencies) - - if not project_builder.CheckProjectConfiguration(project_definition): - print(f'Detected error in configuration of: {project_name:s}') - configuration_errors.add(project_name) - - for project_definition in list(builds): - project_name = project_definition.name - logging.info(f'Building: {project_name:s}') - - # TODO: add support for dokan, bzip2 - # TODO: setup sqlite in build directory. - if not project_builder.Build( - project_definition, distributions=distributions): - print(f'Failed building: {project_name:s}') - failed_builds.add(project_name) - - finally: - os.chdir(current_working_directory) - - if undefined_projects: - print('') - print('Undefined projects:') - for name in sorted(undefined_projects): - print(f'\t{name:s}') - - if configuration_errors: - print('') - print('Projects with configuration errors:') - for name in sorted(configuration_errors): - print(f'\t{name:s}') - - if failed_downloads: - print('') - print('Failed downloading:') - for name in sorted(failed_downloads): - print(f'\t{name:s}') - - if missing_build_dependencies: - print('') - print('Missing build dependencies:') - for dependency in sorted(missing_build_dependencies): - print(f'\t{dependency:s}') - - if failed_builds: - print('') - print('Failed building:') - for name in sorted(failed_builds): - print(f'\t{name:s}') - - return (not failed_downloads and not missing_build_dependencies and - not failed_builds) - - -if __name__ == '__main__': - if not Main(): - sys.exit(1) - else: - sys.exit(0) + sys.exit(0) diff --git a/tools/dpkg-generate.py b/tools/dpkg-generate.py index ca4c763f..ea5911ac 100755 --- a/tools/dpkg-generate.py +++ b/tools/dpkg-generate.py @@ -13,114 +13,135 @@ def Main(): - """The main program function. - - Returns: - bool: True if successful or False if not. - """ - argument_parser = argparse.ArgumentParser(description=( - 'Generates dpkg packaging files for a project.')) - - argument_parser.add_argument( - 'project_name', action='store', metavar='NAME', type=str, help=( - 'Project name for which the dpkg packaging files should be ' - 'generated.')) - - argument_parser.add_argument( - '-c', '--config', dest='config_file', action='store', - metavar='CONFIG_FILE', default=None, - help='path of the build configuration file.') - - argument_parser.add_argument( - '--source-directory', '--source_directory', action='store', - metavar='DIRECTORY', dest='source_directory', type=str, - default=None, help='The location of the the source directory.') - - options = argument_parser.parse_args() - - logging.basicConfig( - level=logging.INFO, format='[%(levelname)s] %(message)s') - - if not options.config_file: - options.config_file = os.path.dirname(__file__) - options.config_file = os.path.dirname(options.config_file) - options.config_file = os.path.join( - options.config_file, 'data', 'projects.ini') - - if not os.path.exists(options.config_file): - print(f'No such config file: {options.config_file:s}') - print('') - return False - - project_definition_match = None - with open(options.config_file, 'r', encoding='utf-8') as file_object: - project_definition_reader = projects.ProjectDefinitionReader() - for project_definition in project_definition_reader.Read(file_object): - if options.project_name == project_definition.name: - project_definition_match = project_definition - - if not project_definition_match: - print(f'No such package name: {options.project_name:s}') - print('') - return False - - source_path = options.source_directory - if not source_path: - globbed_paths = [] - for path in glob.glob(f'{options.project_name:s}*'): - if not os.path.isdir(path): - continue - globbed_paths.append(path) - - if len(globbed_paths) != 1: - print('Unable to determine source directory.') - print('') - return False - - source_path = globbed_paths[0] - - if not os.path.exists(source_path): - print(f'No such source directory: {source_path:s}') - print('') - return False - - source_path = os.path.abspath(source_path) - project_version = os.path.basename(source_path) - if not project_version.startswith(f'{options.project_name:s}-'): - print(( - f'Unable to determine project version based on source ' - f'directory: {source_path:s}')) - print('') - return False - - _, _, project_version = project_version.partition('-') - - dpkg_path = os.path.join(source_path, 'dpkg') - if os.path.exists(dpkg_path): - print(f'Destination dpkg directory: {dpkg_path:s} already exists.') - print('') - return False - - tools_path = os.path.dirname(__file__) - data_path = os.path.join(os.path.dirname(tools_path), 'data') - - build_files_generator = dpkg_files.DPKGBuildFilesGenerator( - options.project_name, project_version, - project_definition_match, data_path) - - print(( - f'Generating dpkg files for: {options.project_name:s} ' - f'{project_version!s} in: {dpkg_path:s}')) - - build_files_generator.GenerateFiles(dpkg_path) - - print('') - - return True - - -if __name__ == '__main__': - if not Main(): - sys.exit(1) - else: - sys.exit(0) + """The main program function. + + Returns: + bool: True if successful or False if not. + """ + argument_parser = argparse.ArgumentParser( + description=("Generates dpkg packaging files for a project.") + ) + + argument_parser.add_argument( + "project_name", + action="store", + metavar="NAME", + type=str, + help=( + "Project name for which the dpkg packaging files should be " "generated." + ), + ) + + argument_parser.add_argument( + "-c", + "--config", + dest="config_file", + action="store", + metavar="CONFIG_FILE", + default=None, + help="path of the build configuration file.", + ) + + argument_parser.add_argument( + "--source-directory", + "--source_directory", + action="store", + metavar="DIRECTORY", + dest="source_directory", + type=str, + default=None, + help="The location of the the source directory.", + ) + + options = argument_parser.parse_args() + + logging.basicConfig(level=logging.INFO, format="[%(levelname)s] %(message)s") + + if not options.config_file: + options.config_file = os.path.dirname(__file__) + options.config_file = os.path.dirname(options.config_file) + options.config_file = os.path.join(options.config_file, "data", "projects.ini") + + if not os.path.exists(options.config_file): + print(f"No such config file: {options.config_file:s}") + print("") + return False + + project_definition_match = None + with open(options.config_file, "r", encoding="utf-8") as file_object: + project_definition_reader = projects.ProjectDefinitionReader() + for project_definition in project_definition_reader.Read(file_object): + if options.project_name == project_definition.name: + project_definition_match = project_definition + + if not project_definition_match: + print(f"No such package name: {options.project_name:s}") + print("") + return False + + source_path = options.source_directory + if not source_path: + globbed_paths = [] + for path in glob.glob(f"{options.project_name:s}*"): + if not os.path.isdir(path): + continue + globbed_paths.append(path) + + if len(globbed_paths) != 1: + print("Unable to determine source directory.") + print("") + return False + + source_path = globbed_paths[0] + + if not os.path.exists(source_path): + print(f"No such source directory: {source_path:s}") + print("") + return False + + source_path = os.path.abspath(source_path) + project_version = os.path.basename(source_path) + if not project_version.startswith(f"{options.project_name:s}-"): + print( + ( + f"Unable to determine project version based on source " + f"directory: {source_path:s}" + ) + ) + print("") + return False + + _, _, project_version = project_version.partition("-") + + dpkg_path = os.path.join(source_path, "dpkg") + if os.path.exists(dpkg_path): + print(f"Destination dpkg directory: {dpkg_path:s} already exists.") + print("") + return False + + tools_path = os.path.dirname(__file__) + data_path = os.path.join(os.path.dirname(tools_path), "data") + + build_files_generator = dpkg_files.DPKGBuildFilesGenerator( + options.project_name, project_version, project_definition_match, data_path + ) + + print( + ( + f"Generating dpkg files for: {options.project_name:s} " + f"{project_version!s} in: {dpkg_path:s}" + ) + ) + + build_files_generator.GenerateFiles(dpkg_path) + + print("") + + return True + + +if __name__ == "__main__": + if not Main(): + sys.exit(1) + else: + sys.exit(0) diff --git a/tools/manage.py b/tools/manage.py index 7434f4b9..47b10230 100755 --- a/tools/manage.py +++ b/tools/manage.py @@ -22,990 +22,1081 @@ class COPRProjectManager: - """Defines a COPR project manager.""" - - _COPR_BASE_URL = 'https://copr.fedorainfracloud.org{0:s}' - - _COPR_URL = ( - 'https://copr.fedorainfracloud.org/api_2/projects?group={name:s}&' - 'name={project:s}') - - _COPR_REPO_URL = ( - 'https://copr-be.cloud.fedoraproject.org/results/%40{name:s}/' - '{project:s}/fedora-{fedora_version:s}-x86_64') - - _PRIMARY_XML_XPATH = ( - './{http://linux.duke.edu/metadata/repo}data[@type="primary"]/' - '{http://linux.duke.edu/metadata/repo}location') - - def __init__(self, name, distribution=None): - """Initializes a COPR manager. - - Args: - name (str): name of the group. - distribution (Optional[str]): name of the distribution. - """ - super().__init__() - self._distribution = distribution or definitions.DEFAULT_FEDORA_DISTRIBUTION - self._download_helper = interface.DownloadHelper('') - self._name = name - - def GetPackages(self, project): - """Retrieves a list of packages of a specific project. - - Args: - project (str): project name. - - Returns: - dict[str, str]: package names and versions as values or None if - the packages cannot be determined. - """ - # TODO: do not use builds information, it is incomplete - # instead use https://copr-be.cloud.fedoraproject.org/results/%40gift/ - # testing/fedora-26-x86_64/repodata/repomd.xml - # to find primary.xml.gz or primary.sqlite.bz2 - - kwargs = { - 'fedora_version': self._distribution, - 'name': self._name, - 'project': project} - copr_repo_url = self._COPR_REPO_URL.format(**kwargs) - - download_url = '/'.join([copr_repo_url, 'repodata', 'repomd.xml']) - page_content = self._download_helper.DownloadPageContent(download_url) - if not page_content: - logging.error('Unable to retrieve repomd.xml.') - return None - - repomd_xml = ElementTree.fromstring(page_content) - xml_elements = repomd_xml.findall(self._PRIMARY_XML_XPATH) - if not xml_elements or not xml_elements[0].items(): - logging.error('Primary data type missing from repomd.xml.') - return None - - href_value_tuple = xml_elements[0].items()[0] - if not href_value_tuple[1]: - logging.error('Primary data type missing from repomd.xml.') - return None - - download_url = '/'.join([copr_repo_url, href_value_tuple[1]]) - page_content = self._download_helper.DownloadPageContent( - download_url, encoding=None) - if not page_content: - _, _, download_url = download_url.rpartition('/') - logging.error('Unable to retrieve primary.xml.gz.') - return None - - with gzip.GzipFile(fileobj=io.BytesIO(page_content)) as file_object: - page_content = file_object.read() - - primary_xml = ElementTree.fromstring(page_content) - # Note explicitly checking xml.Element against None because of deprecation - # warning. - if primary_xml is None: - logging.error('Packages missing from primary.xml.') - return None - - packages = {} - for project_xml in primary_xml: - arch_xml = project_xml.find('{http://linux.duke.edu/metadata/common}arch') - if arch_xml is None or arch_xml.text != 'src': - continue - - package_name_xml = project_xml.find( - '{http://linux.duke.edu/metadata/common}name') - package_version_xml = project_xml.find( - '{http://linux.duke.edu/metadata/common}version') - if package_name_xml is None or package_version_xml is None: - continue - - package_name = package_name_xml.text - package_version = package_version_xml.attrib['ver'] - - if not package_name or not package_version: - continue - - if package_name in packages: - package_version_tuple = package_version.split('.') - version_tuple = packages[package_name].split('.') - compare_result = versions.CompareVersions( - package_version_tuple, version_tuple) - if compare_result < 0: - continue - - packages[package_name] = package_version - - return packages + """Defines a COPR project manager.""" + + _COPR_BASE_URL = "https://copr.fedorainfracloud.org{0:s}" + + _COPR_URL = ( + "https://copr.fedorainfracloud.org/api_2/projects?group={name:s}&" + "name={project:s}" + ) + + _COPR_REPO_URL = ( + "https://copr-be.cloud.fedoraproject.org/results/%40{name:s}/" + "{project:s}/fedora-{fedora_version:s}-x86_64" + ) + + _PRIMARY_XML_XPATH = ( + './{http://linux.duke.edu/metadata/repo}data[@type="primary"]/' + "{http://linux.duke.edu/metadata/repo}location" + ) + + def __init__(self, name, distribution=None): + """Initializes a COPR manager. + + Args: + name (str): name of the group. + distribution (Optional[str]): name of the distribution. + """ + super().__init__() + self._distribution = distribution or definitions.DEFAULT_FEDORA_DISTRIBUTION + self._download_helper = interface.DownloadHelper("") + self._name = name + + def GetPackages(self, project): + """Retrieves a list of packages of a specific project. + + Args: + project (str): project name. + + Returns: + dict[str, str]: package names and versions as values or None if + the packages cannot be determined. + """ + # TODO: do not use builds information, it is incomplete + # instead use https://copr-be.cloud.fedoraproject.org/results/%40gift/ + # testing/fedora-26-x86_64/repodata/repomd.xml + # to find primary.xml.gz or primary.sqlite.bz2 + + kwargs = { + "fedora_version": self._distribution, + "name": self._name, + "project": project, + } + copr_repo_url = self._COPR_REPO_URL.format(**kwargs) + + download_url = "/".join([copr_repo_url, "repodata", "repomd.xml"]) + page_content = self._download_helper.DownloadPageContent(download_url) + if not page_content: + logging.error("Unable to retrieve repomd.xml.") + return None + + repomd_xml = ElementTree.fromstring(page_content) + xml_elements = repomd_xml.findall(self._PRIMARY_XML_XPATH) + if not xml_elements or not xml_elements[0].items(): + logging.error("Primary data type missing from repomd.xml.") + return None + + href_value_tuple = xml_elements[0].items()[0] + if not href_value_tuple[1]: + logging.error("Primary data type missing from repomd.xml.") + return None + + download_url = "/".join([copr_repo_url, href_value_tuple[1]]) + page_content = self._download_helper.DownloadPageContent( + download_url, encoding=None + ) + if not page_content: + _, _, download_url = download_url.rpartition("/") + logging.error("Unable to retrieve primary.xml.gz.") + return None + + with gzip.GzipFile(fileobj=io.BytesIO(page_content)) as file_object: + page_content = file_object.read() + + primary_xml = ElementTree.fromstring(page_content) + # Note explicitly checking xml.Element against None because of deprecation + # warning. + if primary_xml is None: + logging.error("Packages missing from primary.xml.") + return None + + packages = {} + for project_xml in primary_xml: + arch_xml = project_xml.find("{http://linux.duke.edu/metadata/common}arch") + if arch_xml is None or arch_xml.text != "src": + continue + + package_name_xml = project_xml.find( + "{http://linux.duke.edu/metadata/common}name" + ) + package_version_xml = project_xml.find( + "{http://linux.duke.edu/metadata/common}version" + ) + if package_name_xml is None or package_version_xml is None: + continue + + package_name = package_name_xml.text + package_version = package_version_xml.attrib["ver"] + + if not package_name or not package_version: + continue + + if package_name in packages: + package_version_tuple = package_version.split(".") + version_tuple = packages[package_name].split(".") + compare_result = versions.CompareVersions( + package_version_tuple, version_tuple + ) + if compare_result < 0: + continue + + packages[package_name] = package_version + + return packages class GithubRepoManager: - """Defines a GitHub repository manager.""" - - _GITHUB_REPO_API_URL = ( - 'https://api.github.com/repos/log2timeline/l2tbinaries') - - _GITHUB_REPO_URL = ( - 'https://github.com/log2timeline/l2tbinaries') - - def __init__(self): - """Initializes a GitHub repository manager.""" - super().__init__() - self._download_helper = interface.DownloadHelper('') - - def _GetDownloadURL(self, sub_directory, track, use_api=False): - """Retrieves the download URL. - - Args: - sub_directory (str): machine type sub directory. - track (str): track name. - use_api (Optional[bool]): True if the API should be used. - - Returns: - str: download URL or None if sub directory is missing. - """ - if not sub_directory: - return None - - if track == 'stable': - branch = 'main' - else: - branch = track - - if use_api: - return ( - f'{self._GITHUB_REPO_API_URL:s}/contents/{sub_directory:s}?' - f'ref={branch:s}') - - return f'{self._GITHUB_REPO_URL:s}/tree/{branch:s}/{sub_directory:s}' - - def GetPackages(self, sub_directory, track, use_api=False): - """Retrieves a list of packages of a specific sub directory. - - Args: - sub_directory (str): machine type sub directory. - track (str): track name. - use_api (Optional[bool]): True if the API should be used. - - Returns: - dict[str, str]: package names and versions as values or None if - the packages cannot be determined. - """ - if not sub_directory: - logging.info('Missing machine type sub directory.') - return None - - download_url = self._GetDownloadURL(sub_directory, track, use_api=use_api) - if not download_url: - logging.info('Missing download URL.') - return None - - page_content = self._download_helper.DownloadPageContent(download_url) - if not page_content: - return None - - filenames = [] - if use_api: - # The page content consist of JSON data that contains a list of dicts. - # Each dict consists of: - # { - # "name":"PyYAML-3.11.win-amd64-py2.7.msi", - # "path":"win64/PyYAML-3.11.win-amd64-py2.7.msi", - # "sha":"8fca8c1e2549cf54bf993c55930365d01658f418", - # "size":196608, - # "url":"https://api.github.com/...", - # "html_url":"https://github.com/...", - # "git_url":"https://api.github.com/...", - # "download_url":"https://raw.githubusercontent.com/...", - # "type":"file", - # "_links":{ - # "self":"https://api.github.com/...", - # "git":"https://api.github.com/...", - # "html":"https://github.com/..." - # } - # } - - for directory_entry in json.loads(page_content): - filename = directory_entry.get('name', None) - if filename: - filenames.append(filename) - - else: - # The format of the download URL is: - # {pypi_package_name:s} ([^ ]*) : Python Package ' - f'Index') - matches = re.findall(expression_string, page_content) - if not matches or len(matches) != 1: - logging.warning( - f'Unable to determine PyPI package: {pypi_package_name:s} ' - f'information.') - continue - - packages[package_name] = matches - - return packages + """Defines a PyPI manager.""" + + _PYPI_URL = "https://pypi.python.org/pypi/{package_name:s}" + + def __init__(self, projects_file): + """Initializes a PyPI manager. + + Args: + projects_file (str): path to the projects.ini file. + """ + super().__init__() + self._download_helper = interface.DownloadHelper("") + self._package_names = [] + self._pypi_package_names = {} + + if projects_file: + with open(projects_file, "r", encoding="utf-8") as file_object: + project_definition_reader = projects.ProjectDefinitionReader() + for project_definition in project_definition_reader.Read(file_object): + self._package_names.append(project_definition.name) + + if project_definition.pypi_name: + self._pypi_package_names[project_definition.pypi_name] = ( + project_definition.name + ) + + def CopyPackages(self): + """Copies packages.""" + # TODO: implement: + # send post to https://launchpad.net/~gift/+archive/ubuntu/testing + # /+copy-packages + return + + def GetPackages(self): + """Retrieves a list of packages. + + Returns: + dict[str, str]: package names and versions as values or None if + the packages cannot be determined. + """ + packages = {} + for package_name in self._package_names: + pypi_package_name = self._pypi_package_names.get(package_name, package_name) + + kwargs = {"package_name": pypi_package_name} + download_url = self._PYPI_URL.format(**kwargs) + + page_content = self._download_helper.DownloadPageContent(download_url) + if not page_content: + logging.error( + f"Unable to retrieve PyPI package: {pypi_package_name:s} page." + ) + continue + + try: + page_content = page_content.decode("utf-8") + except UnicodeDecodeError as exception: + logging.error( + f"Unable to decode PyPI package: {pypi_package_name:s} page with " + f"error: {exception!s}" + ) + continue + + expression_string = ( + f"{pypi_package_name:s} ([^ ]*) : Python Package " + f"Index" + ) + matches = re.findall(expression_string, page_content) + if not matches or len(matches) != 1: + logging.warning( + f"Unable to determine PyPI package: {pypi_package_name:s} " + f"information." + ) + continue + + packages[package_name] = matches + + return packages class PackagesManager: - """Manages packages across various repositories.""" + """Manages packages across various repositories.""" + + def __init__(self, projects_file, distribution=None): + """Initializes a packages manager. + + Args: + projects_file (str): path to the projects.ini file. + distribution (Optional[str]): name of the distribution. + """ + fedora_distribution = distribution or definitions.DEFAULT_FEDORA_DISTRIBUTION + ubuntu_distribution = distribution or definitions.DEFAULT_UBUNTU_DISTRIBUTION + + super().__init__() + self._copr_project_manager = COPRProjectManager( + "gift", distribution=fedora_distribution + ) + self._github_repo_manager = GithubRepoManager() + self._launchpad_ppa_manager = LaunchpadPPAManager( + "gift", distribution=ubuntu_distribution + ) + self._pypi_manager = PyPIManager(projects_file) + self._ubuntu_distribution = ubuntu_distribution + + def _ComparePackages(self, reference_packages, packages): + """Compares the packages. + + Args: + reference_packages (dict[str, str]): reference package names and versions. + packages (dict[str, str]): package names and versions. + + Returns: + tuple: containing: + + dict[str, str]: new package names and versions. New packages are those + that are present in the reference packages but not in the packages. + dict[str, str]: newer existing package names and versions. Newer + existing packages are those that have a newer version in the + reference packages. + """ + new_packages = {} + new_versions = {} + for name, version in iter(reference_packages.items()): + if not packages or name not in packages: + new_packages[name] = version + continue + + version_tuple = packages[name].split(".") + new_version_tuple = version.split(".") + + compare_result = versions.CompareVersions(version_tuple, new_version_tuple) + if compare_result < 0: + new_versions[name] = version + + return new_packages, new_versions + + def CompareDirectoryWithCOPRProject(self, reference_directory, project): + """Compares a directory containing source rpm packages with a COPR project. + + Args: + reference_directory (str): path of the reference directory that contains + dpkg source packages. + project (str): name of the COPR project. + + Returns: + tuple: containing: + + dict[str, str]: new package names and versions. New packages are those + that are present in the reference directory but not in the project. + dict[str, str]: newer existing package names and versions. Newer + existing packages are those that have a newer version in the + reference directory. + """ + reference_packages = {} + for directory_entry in os.listdir(reference_directory): + # The directory contains various files and we are only interested + # in the source RPM packages that use the naming convention: + # package-version-#.src.rpm + if not directory_entry.endswith(".src.rpm"): + continue + + package_name, _, _ = directory_entry.rpartition("-") + package_name, _, package_version = package_name.rpartition("-") + + if package_name in reference_packages: + package_version_tuple = package_version.split(".") + version_tuple = reference_packages[package_name].split(".") + compare_result = versions.CompareVersions( + package_version_tuple, version_tuple + ) + if compare_result < 0: + continue + + reference_packages[package_name] = package_version + + packages = self._copr_project_manager.GetPackages(project) + return self._ComparePackages(reference_packages, packages) + + def CompareDirectoryWithCSV(self, reference_directory, csv_file): + """Compares a directory containing source packages with a CSV file. + + Args: + reference_directory (str): path of the reference directory that contains + dpkg source packages. + csv_file (str): path of the CSV file. + + Returns: + tuple: containing: + + dict[str, str]: new package names and versions. New packages are those + that are present in the reference directory but not in the project. + dict[str, str]: newer existing package names and versions. Newer + existing packages are those that have a newer version in the + reference directory. + """ + reference_packages = {} + for directory_entry in os.listdir(reference_directory): + # The directory contains various files and we are only interested + # in the source packages that use the naming convention: + # package-version-#.tar.gz + # package-version-#.zip + if not directory_entry.endswith(".tar.gz") and not directory_entry.endswith( + ".zip" + ): + continue + + if ( + directory_entry.endswith(".debian.tar.gz") + or directory_entry.endswith(".orig.tar.gz") + or directory_entry.endswith("-1.tar.gz") + ): + continue + + package_name, _, _ = directory_entry.rpartition(".") + if package_name.endswith(".tar"): + package_name, _, _ = package_name.rpartition(".") + package_name, _, package_version = package_name.rpartition("-") + + if ( + package_name.endswith("-alpha") + or package_name.endswith("-beta") + or package_name.endswith("-experimental") + ): + package_name, _, _ = package_name.rpartition("-") + + if package_name in reference_packages: + package_version_tuple = package_version.split(".") + version_tuple = reference_packages[package_name].split(".") + compare_result = versions.CompareVersions( + package_version_tuple, version_tuple + ) + if compare_result < 0: + continue + + reference_packages[package_name] = package_version + + packages = {} + with open(csv_file, "r", encoding="utf-8") as file_object: + for row in csv.DictReader(file_object): + packages[row["project"]] = row["version"] + + return self._ComparePackages(reference_packages, packages) + + def CompareDirectoryWithGithubRepo(self, reference_directory, sub_directory, track): + """Compares a directory containing wheel packages with a GitHub repo. + + Args: + reference_directory (str): path of the reference directory that contains + wheel packages. + sub_directory (str): name of the machine type sub directory. + track (str): name of the track. + + Returns: + tuple: containing: + + dict[str, str]: new package names and versions. New packages are those + that are present in the reference directory but not in the sub + directory. + dict[str, str]: newer existing package names and versions. Newer + existing packages are those that have a newer version in the + reference directory. + """ + reference_packages = {} + for directory_entry in os.listdir(reference_directory): + if not directory_entry or not directory_entry.endswith(".whl"): + continue + + directory_entry, _, _ = directory_entry.rpartition("-") + package_name, _, package_version = directory_entry.rpartition("-") + + if package_name in reference_packages: + package_version_tuple = package_version.split(".") + version_tuple = reference_packages[package_name].split(".") + compare_result = versions.CompareVersions( + package_version_tuple, version_tuple + ) + if compare_result < 0: + continue + + reference_packages[package_name] = package_version + + packages = self._github_repo_manager.GetPackages(sub_directory, track) + return self._ComparePackages(reference_packages, packages) + + def CompareDirectoryWithLaunchpadPPATrack(self, reference_directory, track): + """Compares a directory containing dpkg packages with a Launchpad PPA track. + + Args: + reference_directory (str): path of the reference directory that contains + dpkg source packages. + track (str): name of the track. + + Returns: + tuple: containing: + + dict[str, str]: new package names and versions. New packages are those + that are present in the reference directory but not in the track. + dict[str, str]: newer existing package names and versions. Newer + existing packages are those that have a newer version in the + reference directory. + """ + reference_packages = {} + for directory_entry in os.listdir(reference_directory): + # The directory contains various files and we are only interested + # in the source dpkg packages that use the naming convention: + # package_version-#ppa1~distribution_source.changes + if not directory_entry.endswith( + f"ppa1~{self._ubuntu_distribution:s}_source.changes" + ): + continue + + package_name, _, _ = directory_entry.rpartition("-") + package_name, _, package_version = package_name.rpartition("_") + + if package_name in reference_packages: + package_version_tuple = package_version.split(".") + version_tuple = reference_packages[package_name].split(".") + compare_result = versions.CompareVersions( + package_version_tuple, version_tuple + ) + if compare_result < 0: + continue + + reference_packages[package_name] = package_version + + packages = self._launchpad_ppa_manager.GetPackages(track) + return self._ComparePackages(reference_packages, packages) + + def CompareCOPRProjects(self, reference_project, project): + """Compares two COPR projects. + + Args: + reference_project (str): name of the reference project. + project (str): name of the project. + + Returns: + tuple: containing: + + dict[str, str]: new package names and versions. New packages are those + that are present in the reference project but not in the project. + dict[str, str]: newer existing package names and versions. Newer + existing packages are those that have a newer version in the + reference project. + """ + reference_packages = self._copr_project_manager.GetPackages(reference_project) + packages = self._copr_project_manager.GetPackages(project) + + return self._ComparePackages(reference_packages, packages) + + def CompareGithubRepos(self, sub_directory, reference_track, track): + """Compares two GitHub repos PPA tracks. + + Args: + sub_directory (str): name of the machine type sub directory. + reference_track (str): name of the reference track. + track (str): name of the track. + + Returns: + tuple: containing: + + dict[str, str]: new package names and versions. New packages are those + that are present in the reference track but not in the track. + dict[str, str]: newer existing package names and versions. Newer + existing packages are those that have a newer version in the + reference track. + """ + reference_packages = self._github_repo_manager.GetPackages( + sub_directory, reference_track + ) + packages = self._github_repo_manager.GetPackages(sub_directory, track) + + return self._ComparePackages(reference_packages, packages) + + def CompareLaunchpadPPATracks(self, reference_track, track): + """Compares two Launchpad PPA tracks. + + Args: + reference_track (str): name of the reference track. + track (str): name of the track. + + Returns: + tuple: containing: + + dict[str, str]: new package names and versions. New packages are those + that are present in the reference track but not in the track. + dict[str, str]: newer existing package names and versions. Newer + existing packages are those that have a newer version in the + reference track. + """ + reference_packages = self._launchpad_ppa_manager.GetPackages(reference_track) + packages = self._launchpad_ppa_manager.GetPackages(track) + + return self._ComparePackages(reference_packages, packages) + + def CompareDirectoryWithPyPI(self, reference_directory): + """Compares a directory containing .tar.gz packages with PyPI. + + Args: + reference_directory (str): path of the reference directory that + contains wheel packages. + + Returns: + tuple: containing: + + dict[str, str]: new package names and versions. New packages are those + that are present in the reference directory but not on PyPI. + dict[str, str]: newer existing package names and versions. Newer + existing packages are those that have a newer version in the + reference directory. + """ + reference_packages = {} + for directory_entry in os.listdir(reference_directory): + if not directory_entry.endswith(".tar.gz"): + continue + + directory_entry, _, _ = directory_entry.rpartition(".tar.gz") + name, _, version = directory_entry.rpartition("-") + + if ( + name.endswith("-alpha") + or name.endswith("-beta") + or name.endswith("-experimental") + ): + name, _, _ = name.rpartition("-") + + reference_packages[name] = version + + packages = self._pypi_manager.GetPackages() + return self._ComparePackages(reference_packages, packages) + + def GetMachineTypeSubDirectory( + self, preferred_machine_type=None, preferred_operating_system=None + ): + """Retrieves the machine type sub directory. + + Args: + preferred_machine_type (Optional[str]): preferred machine type, where + None, which will auto-detect the current machine type. + preferred_operating_system (Optional[str]): preferred operating system, + where None, which will auto-detect the current operating system. + + Returns: + str: machine type sub directory or None if system configuration is not + supported. + """ + if preferred_operating_system: + operating_system = preferred_operating_system + else: + operating_system = platform.system() + + if preferred_machine_type: + cpu_architecture = preferred_machine_type + else: + cpu_architecture = platform.machine().lower() + + sub_directory = None + + if operating_system == "Windows": + if cpu_architecture == "x86": + sub_directory = "win32" + + elif cpu_architecture == "amd64": + sub_directory = "win64" + + else: + logging.error(f"CPU architecture: {cpu_architecture:s} not supported.") + return None + + else: + logging.error(f"Operating system: {operating_system:s} not supported.") + return None + + return sub_directory - def __init__(self, projects_file, distribution=None): - """Initializes a packages manager. - Args: - projects_file (str): path to the projects.ini file. - distribution (Optional[str]): name of the distribution. - """ - fedora_distribution = ( - distribution or definitions.DEFAULT_FEDORA_DISTRIBUTION) - ubuntu_distribution = ( - distribution or definitions.DEFAULT_UBUNTU_DISTRIBUTION) - - super().__init__() - self._copr_project_manager = COPRProjectManager( - 'gift', distribution=fedora_distribution) - self._github_repo_manager = GithubRepoManager() - self._launchpad_ppa_manager = LaunchpadPPAManager( - 'gift', distribution=ubuntu_distribution) - self._pypi_manager = PyPIManager(projects_file) - self._ubuntu_distribution = ubuntu_distribution - - def _ComparePackages(self, reference_packages, packages): - """Compares the packages. - - Args: - reference_packages (dict[str, str]): reference package names and versions. - packages (dict[str, str]): package names and versions. +def Main(): + """The main program function. Returns: - tuple: containing: - - dict[str, str]: new package names and versions. New packages are those - that are present in the reference packages but not in the packages. - dict[str, str]: newer existing package names and versions. Newer - existing packages are those that have a newer version in the - reference packages. + bool: True if successful or False if not. """ + actions = frozenset( + [ + "copr-diff-dev", + "copr-diff-stable", + "copr-diff-staging", + "copr-diff-testing", + "csv-diff", + "l2tbinaries-diff-dev", + "l2tbinaries-diff-stable", + "l2tbinaries-diff-staging", + "l2tbinaries-diff-testing", + "launchpad-diff-dev", + "launchpad-diff-stable", + "launchpad-diff-staging", + "launchpad-diff-testing", + "pypi-diff", + ] + ) + + argument_parser = argparse.ArgumentParser( + description=("Manages the GIFT copr, launchpad PPA and l2tbinaries.") + ) + + argument_parser.add_argument( + "action", + choices=sorted(actions), + action="store", + metavar="ACTION", + default=None, + help="The action.", + ) + + argument_parser.add_argument( + "--build-directory", + "--build_directory", + action="store", + metavar="DIRECTORY", + dest="build_directory", + type=str, + default=os.path.join("..", "l2tbuilds"), + help=("The location of the build directory."), + ) + + argument_parser.add_argument( + "-c", + "--config", + dest="config_path", + action="store", + metavar="CONFIG_PATH", + default=None, + help=( + "path of the directory containing the build configuration " + "files e.g. projects.ini." + ), + ) + + argument_parser.add_argument( + "--csv-file", + "--csv_file", + action="store", + metavar="FILE", + dest="csv_file", + type=str, + default="", + help=("The location of the CSV file."), + ) + + argument_parser.add_argument( + "--distribution", + action="store", + metavar="NAME", + dest="distribution", + type=str, + default=None, + help="The name or version of the distribution.", + ) + + argument_parser.add_argument( + "--machine-type", + "--machine_type", + action="store", + metavar="TYPE", + dest="machine_type", + type=str, + default=None, + help=( + "Manually sets the machine type instead of using the value returned " + "by platform.machine(). Usage of this argument is not recommended " + "unless want to force the installation of one machine type e.g. " + "'x86' onto another 'amd64'." + ), + ) + + options = argument_parser.parse_args() + + if not options.action: + print("Missing action.") + print("") + argument_parser.print_help() + print("") + return False + + config_path = options.config_path + if not config_path: + config_path = os.path.dirname(__file__) + config_path = os.path.dirname(config_path) + config_path = os.path.join(config_path, "data") + + projects_file = os.path.join(config_path, "projects.ini") + if not os.path.exists(projects_file): + print(f"No such config file: {projects_file:s}") + print("") + return False + + logging.basicConfig(level=logging.INFO, format="[%(levelname)s] %(message)s") + + # TODO: add action to upload files to PPA. + # TODO: add action to copy files between PPA tracks. + # TODO: add pypi support. + + packages_manager = PackagesManager(projects_file, distribution=options.distribution) + + action_tuple = options.action.split("-") + diff_header = None new_packages = {} new_versions = {} - for name, version in iter(reference_packages.items()): - if not packages or name not in packages: - new_packages[name] = version - continue - - version_tuple = packages[name].split('.') - new_version_tuple = version.split('.') - - compare_result = versions.CompareVersions( - version_tuple, new_version_tuple) - if compare_result < 0: - new_versions[name] = version - - return new_packages, new_versions - - def CompareDirectoryWithCOPRProject(self, reference_directory, project): - """Compares a directory containing source rpm packages with a COPR project. - - Args: - reference_directory (str): path of the reference directory that contains - dpkg source packages. - project (str): name of the COPR project. - - Returns: - tuple: containing: - - dict[str, str]: new package names and versions. New packages are those - that are present in the reference directory but not in the project. - dict[str, str]: newer existing package names and versions. Newer - existing packages are those that have a newer version in the - reference directory. - """ - reference_packages = {} - for directory_entry in os.listdir(reference_directory): - # The directory contains various files and we are only interested - # in the source RPM packages that use the naming convention: - # package-version-#.src.rpm - if not directory_entry.endswith('.src.rpm'): - continue - - package_name, _, _ = directory_entry.rpartition('-') - package_name, _, package_version = package_name.rpartition('-') - - if package_name in reference_packages: - package_version_tuple = package_version.split('.') - version_tuple = reference_packages[package_name].split('.') - compare_result = versions.CompareVersions( - package_version_tuple, version_tuple) - if compare_result < 0: - continue - - reference_packages[package_name] = package_version - - packages = self._copr_project_manager.GetPackages(project) - return self._ComparePackages(reference_packages, packages) - - def CompareDirectoryWithCSV(self, reference_directory, csv_file): - """Compares a directory containing source packages with a CSV file. - - Args: - reference_directory (str): path of the reference directory that contains - dpkg source packages. - csv_file (str): path of the CSV file. - Returns: - tuple: containing: - - dict[str, str]: new package names and versions. New packages are those - that are present in the reference directory but not in the project. - dict[str, str]: newer existing package names and versions. Newer - existing packages are those that have a newer version in the - reference directory. - """ - reference_packages = {} - for directory_entry in os.listdir(reference_directory): - # The directory contains various files and we are only interested - # in the source packages that use the naming convention: - # package-version-#.tar.gz - # package-version-#.zip - if (not directory_entry.endswith('.tar.gz') and - not directory_entry.endswith('.zip')): - continue - - if (directory_entry.endswith('.debian.tar.gz') or - directory_entry.endswith('.orig.tar.gz') or - directory_entry.endswith('-1.tar.gz')): - continue - - package_name, _, _ = directory_entry.rpartition('.') - if package_name.endswith('.tar'): - package_name, _, _ = package_name.rpartition('.') - package_name, _, package_version = package_name.rpartition('-') - - if (package_name.endswith('-alpha') or - package_name.endswith('-beta') or - package_name.endswith('-experimental')): - package_name, _, _ = package_name.rpartition('-') - - if package_name in reference_packages: - package_version_tuple = package_version.split('.') - version_tuple = reference_packages[package_name].split('.') - compare_result = versions.CompareVersions( - package_version_tuple, version_tuple) - if compare_result < 0: - continue - - reference_packages[package_name] = package_version - - packages = {} - with open(csv_file, 'r', encoding='utf-8') as file_object: - for row in csv.DictReader(file_object): - packages[row['project']] = row['version'] - - return self._ComparePackages(reference_packages, packages) - - def CompareDirectoryWithGithubRepo( - self, reference_directory, sub_directory, track): - """Compares a directory containing wheel packages with a GitHub repo. - - Args: - reference_directory (str): path of the reference directory that contains - wheel packages. - sub_directory (str): name of the machine type sub directory. - track (str): name of the track. - - Returns: - tuple: containing: - - dict[str, str]: new package names and versions. New packages are those - that are present in the reference directory but not in the sub - directory. - dict[str, str]: newer existing package names and versions. Newer - existing packages are those that have a newer version in the - reference directory. - """ - reference_packages = {} - for directory_entry in os.listdir(reference_directory): - if not directory_entry or not directory_entry.endswith('.whl'): - continue - - directory_entry, _, _ = directory_entry.rpartition('-') - package_name, _, package_version = directory_entry.rpartition('-') - - if package_name in reference_packages: - package_version_tuple = package_version.split('.') - version_tuple = reference_packages[package_name].split('.') - compare_result = versions.CompareVersions( - package_version_tuple, version_tuple) - if compare_result < 0: - continue + if action_tuple[0] == "copr" and action_tuple[1] == "diff": + track = action_tuple[2] + + if track == "testing": + reference_directory = options.build_directory + + new_packages, new_versions = ( + packages_manager.CompareDirectoryWithCOPRProject( + reference_directory, track + ) + ) + + diff_header = ( + f"Difference between: {reference_directory:s} and COPR project: " + f"{track:s}" + ) + + else: + if track == "dev": + reference_track = "testing" + elif track == "staging": + reference_track = "dev" + else: + reference_track = "staging" + + new_packages, new_versions = packages_manager.CompareCOPRProjects( + reference_track, track + ) + + diff_header = ( + f"Difference between COPR project: {reference_track:s} and {track:s}" + ) + + elif action_tuple[0] == "csv" and action_tuple[1] == "diff": + reference_directory = options.build_directory + + new_packages, new_versions = packages_manager.CompareDirectoryWithCSV( + reference_directory, options.csv_file + ) + + diff_header = f"Difference between: {reference_directory:s} and CSV" - reference_packages[package_name] = package_version + elif action_tuple[0] == "l2tbinaries" and action_tuple[1] == "diff": + track = action_tuple[2] + + sub_directory = packages_manager.GetMachineTypeSubDirectory( + preferred_machine_type=options.machine_type + ) + + if track == "testing": + reference_directory = options.build_directory + + new_packages, new_versions = ( + packages_manager.CompareDirectoryWithGithubRepo( + reference_directory, sub_directory, track + ) + ) + + diff_header = ( + f"Difference between: {reference_directory:s} and testing for: " + f"{sub_directory:s}" + ) + + else: + if track == "dev": + reference_track = "testing" + elif track == "staging": + reference_track = "dev" + else: + reference_track = "staging" + + new_packages, new_versions = packages_manager.CompareGithubRepos( + sub_directory, reference_track, track + ) + + diff_header = ( + f"Difference between l2tbinaries tracks: {reference_track:s} and " + f"{track:s} for: {sub_directory:s}" + ) + + elif action_tuple[0] == "launchpad" and action_tuple[1] == "diff": + track = action_tuple[2] + + if track == "testing": + reference_directory = options.build_directory + + new_packages, new_versions = ( + packages_manager.CompareDirectoryWithLaunchpadPPATrack( + reference_directory, track + ) + ) + + diff_header = ( + f"Difference between: {reference_directory:s} and Launchpad track: " + f"{track:s}" + ) + + else: + if track == "dev": + reference_track = "testing" + elif track == "staging": + reference_track = "dev" + else: + reference_track = "staging" + + new_packages, new_versions = packages_manager.CompareLaunchpadPPATracks( + reference_track, track + ) + + diff_header = ( + f"Difference between Launchpad tracks: {reference_track:s} and " + f"{track:s}" + ) + + # elif action_tuple[0] == 'osb' and action_tuple[1] == 'diff': + + elif action_tuple[0] == "pypi" and action_tuple[1] == "diff": + reference_directory = options.build_directory + + new_packages, new_versions = packages_manager.CompareDirectoryWithPyPI( + reference_directory + ) - packages = self._github_repo_manager.GetPackages(sub_directory, track) - return self._ComparePackages(reference_packages, packages) + diff_header = f"Difference between: {reference_directory:s} and release" - def CompareDirectoryWithLaunchpadPPATrack( - self, reference_directory, track): - """Compares a directory containing dpkg packages with a Launchpad PPA track. + if action_tuple[1] == "diff": + print(diff_header) + print("") - Args: - reference_directory (str): path of the reference directory that contains - dpkg source packages. - track (str): name of the track. - - Returns: - tuple: containing: + print("New packages:") + for package in sorted(new_packages.keys()): + print(f" {package:s}") + print("") + + print("New versions:") + for package in sorted(new_versions.keys()): + print(f" {package:s}") + print("") - dict[str, str]: new package names and versions. New packages are those - that are present in the reference directory but not in the track. - dict[str, str]: newer existing package names and versions. Newer - existing packages are those that have a newer version in the - reference directory. - """ - reference_packages = {} - for directory_entry in os.listdir(reference_directory): - # The directory contains various files and we are only interested - # in the source dpkg packages that use the naming convention: - # package_version-#ppa1~distribution_source.changes - if not directory_entry.endswith( - f'ppa1~{self._ubuntu_distribution:s}_source.changes'): - continue - - package_name, _, _ = directory_entry.rpartition('-') - package_name, _, package_version = package_name.rpartition('_') - - if package_name in reference_packages: - package_version_tuple = package_version.split('.') - version_tuple = reference_packages[package_name].split('.') - compare_result = versions.CompareVersions( - package_version_tuple, version_tuple) - if compare_result < 0: - continue - - reference_packages[package_name] = package_version - - packages = self._launchpad_ppa_manager.GetPackages(track) - return self._ComparePackages(reference_packages, packages) - - def CompareCOPRProjects(self, reference_project, project): - """Compares two COPR projects. - - Args: - reference_project (str): name of the reference project. - project (str): name of the project. - - Returns: - tuple: containing: - - dict[str, str]: new package names and versions. New packages are those - that are present in the reference project but not in the project. - dict[str, str]: newer existing package names and versions. Newer - existing packages are those that have a newer version in the - reference project. - """ - reference_packages = self._copr_project_manager.GetPackages( - reference_project) - packages = self._copr_project_manager.GetPackages(project) - - return self._ComparePackages(reference_packages, packages) - - def CompareGithubRepos(self, sub_directory, reference_track, track): - """Compares two GitHub repos PPA tracks. - - Args: - sub_directory (str): name of the machine type sub directory. - reference_track (str): name of the reference track. - track (str): name of the track. - - Returns: - tuple: containing: - - dict[str, str]: new package names and versions. New packages are those - that are present in the reference track but not in the track. - dict[str, str]: newer existing package names and versions. Newer - existing packages are those that have a newer version in the - reference track. - """ - reference_packages = self._github_repo_manager.GetPackages( - sub_directory, reference_track) - packages = self._github_repo_manager.GetPackages(sub_directory, track) - - return self._ComparePackages(reference_packages, packages) - - def CompareLaunchpadPPATracks(self, reference_track, track): - """Compares two Launchpad PPA tracks. - - Args: - reference_track (str): name of the reference track. - track (str): name of the track. - - Returns: - tuple: containing: - - dict[str, str]: new package names and versions. New packages are those - that are present in the reference track but not in the track. - dict[str, str]: newer existing package names and versions. Newer - existing packages are those that have a newer version in the - reference track. - """ - reference_packages = self._launchpad_ppa_manager.GetPackages( - reference_track) - packages = self._launchpad_ppa_manager.GetPackages(track) - - return self._ComparePackages(reference_packages, packages) - - def CompareDirectoryWithPyPI(self, reference_directory): - """Compares a directory containing .tar.gz packages with PyPI. - - Args: - reference_directory (str): path of the reference directory that - contains wheel packages. - - Returns: - tuple: containing: - - dict[str, str]: new package names and versions. New packages are those - that are present in the reference directory but not on PyPI. - dict[str, str]: newer existing package names and versions. Newer - existing packages are those that have a newer version in the - reference directory. - """ - reference_packages = {} - for directory_entry in os.listdir(reference_directory): - if not directory_entry.endswith('.tar.gz'): - continue - - directory_entry, _, _ = directory_entry.rpartition('.tar.gz') - name, _, version = directory_entry.rpartition('-') - - if (name.endswith('-alpha') or name.endswith('-beta') or - name.endswith('-experimental')): - name, _, _ = name.rpartition('-') - - reference_packages[name] = version - - packages = self._pypi_manager.GetPackages() - return self._ComparePackages(reference_packages, packages) - - def GetMachineTypeSubDirectory( - self, preferred_machine_type=None, preferred_operating_system=None): - """Retrieves the machine type sub directory. - - Args: - preferred_machine_type (Optional[str]): preferred machine type, where - None, which will auto-detect the current machine type. - preferred_operating_system (Optional[str]): preferred operating system, - where None, which will auto-detect the current operating system. - - Returns: - str: machine type sub directory or None if system configuration is not - supported. - """ - if preferred_operating_system: - operating_system = preferred_operating_system - else: - operating_system = platform.system() - - if preferred_machine_type: - cpu_architecture = preferred_machine_type + return True + + +if __name__ == "__main__": + if not Main(): + sys.exit(1) else: - cpu_architecture = platform.machine().lower() - - sub_directory = None - - if operating_system == 'Windows': - if cpu_architecture == 'x86': - sub_directory = 'win32' - - elif cpu_architecture == 'amd64': - sub_directory = 'win64' - - else: - logging.error(f'CPU architecture: {cpu_architecture:s} not supported.') - return None - - else: - logging.error(f'Operating system: {operating_system:s} not supported.') - return None - - return sub_directory - - -def Main(): - """The main program function. - - Returns: - bool: True if successful or False if not. - """ - actions = frozenset([ - 'copr-diff-dev', 'copr-diff-stable', 'copr-diff-staging', - 'copr-diff-testing', 'csv-diff', 'l2tbinaries-diff-dev', - 'l2tbinaries-diff-stable', 'l2tbinaries-diff-staging', - 'l2tbinaries-diff-testing', 'launchpad-diff-dev', 'launchpad-diff-stable', - 'launchpad-diff-staging', 'launchpad-diff-testing', 'pypi-diff']) - - argument_parser = argparse.ArgumentParser(description=( - 'Manages the GIFT copr, launchpad PPA and l2tbinaries.')) - - argument_parser.add_argument( - 'action', choices=sorted(actions), action='store', - metavar='ACTION', default=None, help='The action.') - - argument_parser.add_argument( - '--build-directory', '--build_directory', action='store', - metavar='DIRECTORY', dest='build_directory', type=str, - default=os.path.join('..', 'l2tbuilds'), help=( - 'The location of the build directory.')) - - argument_parser.add_argument( - '-c', '--config', dest='config_path', action='store', - metavar='CONFIG_PATH', default=None, help=( - 'path of the directory containing the build configuration ' - 'files e.g. projects.ini.')) - - argument_parser.add_argument( - '--csv-file', '--csv_file', action='store', metavar='FILE', - dest='csv_file', type=str, default='', help=( - 'The location of the CSV file.')) - - argument_parser.add_argument( - '--distribution', action='store', metavar='NAME', dest='distribution', - type=str, default=None, help='The name or version of the distribution.') - - argument_parser.add_argument( - '--machine-type', '--machine_type', action='store', metavar='TYPE', - dest='machine_type', type=str, default=None, help=( - 'Manually sets the machine type instead of using the value returned ' - 'by platform.machine(). Usage of this argument is not recommended ' - 'unless want to force the installation of one machine type e.g. ' - '\'x86\' onto another \'amd64\'.')) - - options = argument_parser.parse_args() - - if not options.action: - print('Missing action.') - print('') - argument_parser.print_help() - print('') - return False - - config_path = options.config_path - if not config_path: - config_path = os.path.dirname(__file__) - config_path = os.path.dirname(config_path) - config_path = os.path.join(config_path, 'data') - - projects_file = os.path.join(config_path, 'projects.ini') - if not os.path.exists(projects_file): - print(f'No such config file: {projects_file:s}') - print('') - return False - - logging.basicConfig( - level=logging.INFO, format='[%(levelname)s] %(message)s') - - # TODO: add action to upload files to PPA. - # TODO: add action to copy files between PPA tracks. - # TODO: add pypi support. - - packages_manager = PackagesManager( - projects_file, distribution=options.distribution) - - action_tuple = options.action.split('-') - diff_header = None - new_packages = {} - new_versions = {} - - if action_tuple[0] == 'copr' and action_tuple[1] == 'diff': - track = action_tuple[2] - - if track == 'testing': - reference_directory = options.build_directory - - new_packages, new_versions = ( - packages_manager.CompareDirectoryWithCOPRProject( - reference_directory, track)) - - diff_header = ( - f'Difference between: {reference_directory:s} and COPR project: ' - f'{track:s}') - - else: - if track == 'dev': - reference_track = 'testing' - elif track == 'staging': - reference_track = 'dev' - else: - reference_track = 'staging' - - new_packages, new_versions = packages_manager.CompareCOPRProjects( - reference_track, track) - - diff_header = ( - f'Difference between COPR project: {reference_track:s} and {track:s}') - - elif action_tuple[0] == 'csv' and action_tuple[1] == 'diff': - reference_directory = options.build_directory - - new_packages, new_versions = ( - packages_manager.CompareDirectoryWithCSV( - reference_directory, options.csv_file)) - - diff_header = f'Difference between: {reference_directory:s} and CSV' - - elif action_tuple[0] == 'l2tbinaries' and action_tuple[1] == 'diff': - track = action_tuple[2] - - sub_directory = packages_manager.GetMachineTypeSubDirectory( - preferred_machine_type=options.machine_type) - - if track == 'testing': - reference_directory = options.build_directory - - new_packages, new_versions = ( - packages_manager.CompareDirectoryWithGithubRepo( - reference_directory, sub_directory, track)) - - diff_header = ( - f'Difference between: {reference_directory:s} and testing for: ' - f'{sub_directory:s}') - - else: - if track == 'dev': - reference_track = 'testing' - elif track == 'staging': - reference_track = 'dev' - else: - reference_track = 'staging' - - new_packages, new_versions = packages_manager.CompareGithubRepos( - sub_directory, reference_track, track) - - diff_header = ( - f'Difference between l2tbinaries tracks: {reference_track:s} and ' - f'{track:s} for: {sub_directory:s}') - - elif action_tuple[0] == 'launchpad' and action_tuple[1] == 'diff': - track = action_tuple[2] - - if track == 'testing': - reference_directory = options.build_directory - - new_packages, new_versions = ( - packages_manager.CompareDirectoryWithLaunchpadPPATrack( - reference_directory, track)) - - diff_header = ( - f'Difference between: {reference_directory:s} and Launchpad track: ' - f'{track:s}') - - else: - if track == 'dev': - reference_track = 'testing' - elif track == 'staging': - reference_track = 'dev' - else: - reference_track = 'staging' - - new_packages, new_versions = packages_manager.CompareLaunchpadPPATracks( - reference_track, track) - - diff_header = ( - f'Difference between Launchpad tracks: {reference_track:s} and ' - f'{track:s}') - - # elif action_tuple[0] == 'osb' and action_tuple[1] == 'diff': - - elif action_tuple[0] == 'pypi' and action_tuple[1] == 'diff': - reference_directory = options.build_directory - - new_packages, new_versions = ( - packages_manager.CompareDirectoryWithPyPI(reference_directory)) - - diff_header = f'Difference between: {reference_directory:s} and release' - - if action_tuple[1] == 'diff': - print(diff_header) - print('') - - print('New packages:') - for package in sorted(new_packages.keys()): - print(f' {package:s}') - print('') - - print('New versions:') - for package in sorted(new_versions.keys()): - print(f' {package:s}') - print('') - - return True - - -if __name__ == '__main__': - if not Main(): - sys.exit(1) - else: - sys.exit(0) + sys.exit(0) diff --git a/tools/review.py b/tools/review.py index f82cfa90..6a1d1fcf 100755 --- a/tools/review.py +++ b/tools/review.py @@ -9,101 +9,118 @@ def Main(): - """The main program function. - - Returns: - bool: True if successful or False if not. - """ - argument_parser = argparse.ArgumentParser( - description='Script to manage code reviews.') - - argument_parser.add_argument( - '--project-path', '--project_path', '-p', dest='project_path', - action='store', default=os.getcwd(), help=( - 'Path to the project being reviewed.')) - - argument_parser.add_argument( - '--allfiles', '--all-files', '--all_files', dest='all_files', - action='store_true', default=False, help=( - 'Apply command to all files, currently only affects the lint ' - 'command.')) - - commands_parser = argument_parser.add_subparsers(dest='command') - - close_command_parser = commands_parser.add_parser('close') - - # TODO: add this to help output. - close_command_parser.add_argument( - 'branch', action='store', metavar='BRANCH', default=None, - help='name of the corresponding feature branch.') - - commands_parser.add_parser('lint') - - commands_parser.add_parser('lint-test') - commands_parser.add_parser('lint_test') - - commands_parser.add_parser('test') - - options = argument_parser.parse_args() - - feature_branch = None - github_origin = None - - print_help_on_error = False - if options.command == 'close': - feature_branch = getattr(options, 'branch', None) - if not feature_branch: - print('Feature branch value is missing.') - print_help_on_error = True - - # Support "username:branch" notation. - elif ':' in str(feature_branch): - _, _, feature_branch = feature_branch.rpartition(':') - - if print_help_on_error: - print('') - argument_parser.print_help() - print('') - return False - - home_path = os.path.expanduser('~') - netrc_path = os.path.join(home_path, '.netrc') - if not os.path.exists(netrc_path): - command = options.command.title() - print(f'{command:s} aborted - unable to find .netrc') - return False - - review_helper = review.ReviewHelper( - options.command, - options.project_path, - github_origin, - feature_branch, - all_files=options.all_files) - - if not review_helper.InitializeHelpers(): - return False - - if not review_helper.CheckLocalGitState(): - return False - - if not review_helper.Lint(): - return False - - if not review_helper.Test(): - return False - - result = False - if options.command == 'close': - result = review_helper.Close() - - elif options.command in ('lint', 'lint-test', 'lint_test', 'test'): - result = True - - return result - - -if __name__ == '__main__': - if not Main(): - sys.exit(1) - else: - sys.exit(0) + """The main program function. + + Returns: + bool: True if successful or False if not. + """ + argument_parser = argparse.ArgumentParser( + description="Script to manage code reviews." + ) + + argument_parser.add_argument( + "--project-path", + "--project_path", + "-p", + dest="project_path", + action="store", + default=os.getcwd(), + help=("Path to the project being reviewed."), + ) + + argument_parser.add_argument( + "--allfiles", + "--all-files", + "--all_files", + dest="all_files", + action="store_true", + default=False, + help=( + "Apply command to all files, currently only affects the lint " "command." + ), + ) + + commands_parser = argument_parser.add_subparsers(dest="command") + + close_command_parser = commands_parser.add_parser("close") + + # TODO: add this to help output. + close_command_parser.add_argument( + "branch", + action="store", + metavar="BRANCH", + default=None, + help="name of the corresponding feature branch.", + ) + + commands_parser.add_parser("lint") + + commands_parser.add_parser("lint-test") + commands_parser.add_parser("lint_test") + + commands_parser.add_parser("test") + + options = argument_parser.parse_args() + + feature_branch = None + github_origin = None + + print_help_on_error = False + if options.command == "close": + feature_branch = getattr(options, "branch", None) + if not feature_branch: + print("Feature branch value is missing.") + print_help_on_error = True + + # Support "username:branch" notation. + elif ":" in str(feature_branch): + _, _, feature_branch = feature_branch.rpartition(":") + + if print_help_on_error: + print("") + argument_parser.print_help() + print("") + return False + + home_path = os.path.expanduser("~") + netrc_path = os.path.join(home_path, ".netrc") + if not os.path.exists(netrc_path): + command = options.command.title() + print(f"{command:s} aborted - unable to find .netrc") + return False + + review_helper = review.ReviewHelper( + options.command, + options.project_path, + github_origin, + feature_branch, + all_files=options.all_files, + ) + + if not review_helper.InitializeHelpers(): + return False + + if not review_helper.CheckLocalGitState(): + return False + + if not review_helper.Lint(): + return False + + if not review_helper.Test(): + return False + + result = False + if options.command == "close": + result = review_helper.Close() + + elif options.command in ("lint", "lint-test", "lint_test", "test"): + result = True + + return result + + +if __name__ == "__main__": + if not Main(): + sys.exit(1) + else: + sys.exit(0) diff --git a/tools/schema_extractor.py b/tools/schema_extractor.py index d8309230..5f6c1299 100755 --- a/tools/schema_extractor.py +++ b/tools/schema_extractor.py @@ -9,135 +9,145 @@ import textwrap try: - import pyperclip + import pyperclip except ImportError: - pyperclip = None + pyperclip = None class SQLiteSchemaExtractor: - """SQLite database file schema extractor.""" + """SQLite database file schema extractor.""" - _SCHEMA_QUERY = ( - 'SELECT tbl_name, sql ' - 'FROM sqlite_master ' - 'WHERE type = "table" AND tbl_name != "xp_proc" ' - 'AND tbl_name != "sqlite_sequence"') + _SCHEMA_QUERY = ( + "SELECT tbl_name, sql " + "FROM sqlite_master " + 'WHERE type = "table" AND tbl_name != "xp_proc" ' + 'AND tbl_name != "sqlite_sequence"' + ) - def FormatSchema(self, schema): - """Formats a schema into a word-wrapped string. + def FormatSchema(self, schema): + """Formats a schema into a word-wrapped string. - Args: - schema (dict[str, str]): schema as an SQL query per table name. + Args: + schema (dict[str, str]): schema as an SQL query per table name. - Returns: - str: schema formatted as word-wrapped string. - """ - textwrapper = textwrap.TextWrapper() - textwrapper.break_long_words = False - textwrapper.drop_whitespace = True - textwrapper.width = 80 - (10 + 4) + Returns: + str: schema formatted as word-wrapped string. + """ + textwrapper = textwrap.TextWrapper() + textwrapper.break_long_words = False + textwrapper.drop_whitespace = True + textwrapper.width = 80 - (10 + 4) - lines = [] - table_index = 1 - number_of_tables = len(schema) - for table_name, query in sorted(schema.items()): - lines.append(f' \'{table_name:s}\': (') + lines = [] + table_index = 1 + number_of_tables = len(schema) + for table_name, query in sorted(schema.items()): + lines.append(f" '{table_name:s}': (") - query = query.replace('\'', '\\\'') - query = textwrapper.wrap(query) - indentation = ' ' * 10 - query = [f'{indentation:s}\'{line:s} \'' for line in query] + query = query.replace("'", "\\'") + query = textwrapper.wrap(query) + indentation = " " * 10 + query = [f"{indentation:s}'{line:s} '" for line in query] - name = query[-1][:-2] - if table_index == number_of_tables: - query[-1] = f'{name:s}\')}}]' - else: - query[-1] = f'{name:s}\'),' + name = query[-1][:-2] + if table_index == number_of_tables: + query[-1] = f"{name:s}')}}]" + else: + query[-1] = f"{name:s}')," - lines.extend(query) - table_index += 1 + lines.extend(query) + table_index += 1 - return '\n'.join(lines) + return "\n".join(lines) - def GetDatabaseSchema(self, database_path): - """Retrieves schema from given database. + def GetDatabaseSchema(self, database_path): + """Retrieves schema from given database. - Args: - database_path (str): file path to database. + Args: + database_path (str): file path to database. - Returns: - dict[str, str]: schema as an SQL query per table name or None if - the schema could not be retrieved. - """ - schema = None + Returns: + dict[str, str]: schema as an SQL query per table name or None if + the schema could not be retrieved. + """ + schema = None - database = sqlite3.connect(database_path) - database.row_factory = sqlite3.Row + database = sqlite3.connect(database_path) + database.row_factory = sqlite3.Row - try: - cursor = database.cursor() + try: + cursor = database.cursor() - rows = cursor.execute(self._SCHEMA_QUERY) + rows = cursor.execute(self._SCHEMA_QUERY) - schema = { - table_name: ' '.join(query.split()) for table_name, query in rows} + schema = {table_name: " ".join(query.split()) for table_name, query in rows} - except sqlite3.DatabaseError as exception: - logging.error(f'Unable to query schema with error: {exception!s}') + except sqlite3.DatabaseError as exception: + logging.error(f"Unable to query schema with error: {exception!s}") - finally: - database.close() + finally: + database.close() - return schema + return schema def Main(): - """The main program function. + """The main program function. - Returns: - bool: True if successful or False if not. - """ - argument_parser = argparse.ArgumentParser(description=( - 'Extract the database schema from a SQLite database file.')) + Returns: + bool: True if successful or False if not. + """ + argument_parser = argparse.ArgumentParser( + description=("Extract the database schema from a SQLite database file.") + ) + + if pyperclip: + argument_parser.add_argument( + "--to-clipboard", + "--to_clipboard", + dest="to_clipboard", + action="store_true", + default=False, + help=( + "copy the database schema to the clipboard instead of writing " + "to stdout." + ), + ) - if pyperclip: argument_parser.add_argument( - '--to-clipboard', '--to_clipboard', dest='to_clipboard', - action='store_true', default=False, help=( - 'copy the database schema to the clipboard instead of writing ' - 'to stdout.')) - - argument_parser.add_argument( - 'database_path', type=str, - help='path to the database file to extract schema from.') + "database_path", + type=str, + help="path to the database file to extract schema from.", + ) - options = argument_parser.parse_args() + options = argument_parser.parse_args() - if not os.path.exists(options.database_path): - print(f'No such database file: {options.database_path:s}') - return False + if not os.path.exists(options.database_path): + print(f"No such database file: {options.database_path:s}") + return False - extractor = SQLiteSchemaExtractor() + extractor = SQLiteSchemaExtractor() - database_schema = extractor.GetDatabaseSchema(options.database_path) - if not database_schema: - print( - f'Unable to determine schema from database file: ' - f'{options.database_path:s}') - return False + database_schema = extractor.GetDatabaseSchema(options.database_path) + if not database_schema: + print( + f"Unable to determine schema from database file: " + f"{options.database_path:s}" + ) + return False - database_schema = extractor.FormatSchema(database_schema) + database_schema = extractor.FormatSchema(database_schema) - if pyperclip and options.to_clipboard: - pyperclip.copy(database_schema) - else: - print(database_schema) + if pyperclip and options.to_clipboard: + pyperclip.copy(database_schema) + else: + print(database_schema) - return True + return True -if __name__ == '__main__': - if not Main(): - sys.exit(1) - else: - sys.exit(0) +if __name__ == "__main__": + if not Main(): + sys.exit(1) + else: + sys.exit(0) diff --git a/tools/stats.py b/tools/stats.py index 46b64568..423e4f6d 100755 --- a/tools/stats.py +++ b/tools/stats.py @@ -15,483 +15,535 @@ class StatsDefinitionReader: - """Class that implements a stats definition reader.""" + """Class that implements a stats definition reader.""" - def _GetConfigValue(self, config_parser, section_name, value_name): - """Retrieves a value from the config parser. + def _GetConfigValue(self, config_parser, section_name, value_name): + """Retrieves a value from the config parser. - Args: - config_parser (ConfigParser): configuration parser. - section_name (str): name of the section that contains the value. - value_name (str): name of the value. + Args: + config_parser (ConfigParser): configuration parser. + section_name (str): name of the section that contains the value. + value_name (str): name of the value. - Returns: - object: value or None if the value does not exists. - """ - try: - return config_parser.get(section_name, value_name) - except configparser.NoOptionError: - return None + Returns: + object: value or None if the value does not exists. + """ + try: + return config_parser.get(section_name, value_name) + except configparser.NoOptionError: + return None - def ReadProjectsPerOrganization(self, file_object): - """Reads the projects per organization. + def ReadProjectsPerOrganization(self, file_object): + """Reads the projects per organization. - Args: - file_object (file): file-like object to read from. + Args: + file_object (file): file-like object to read from. - Returns: - dict[str, list[str]]: organization names with corresponding project names. - """ - config_parser = configparser.ConfigParser(interpolation=None) - config_parser.read_file(file_object) + Returns: + dict[str, list[str]]: organization names with corresponding project names. + """ + config_parser = configparser.ConfigParser(interpolation=None) + config_parser.read_file(file_object) - projects_per_organization = {} - for option_name in config_parser.options('organizations'): - project_names = self._GetConfigValue( - config_parser, 'organizations', option_name) + projects_per_organization = {} + for option_name in config_parser.options("organizations"): + project_names = self._GetConfigValue( + config_parser, "organizations", option_name + ) - if project_names is None: - project_names = [] - elif isinstance(project_names, str): - project_names = project_names.split(',') + if project_names is None: + project_names = [] + elif isinstance(project_names, str): + project_names = project_names.split(",") - projects_per_organization[option_name] = project_names + projects_per_organization[option_name] = project_names - return projects_per_organization + return projects_per_organization - def ReadUserMappings(self, file_object): - """Reads the username mappings. + def ReadUserMappings(self, file_object): + """Reads the username mappings. - Args: - file_object (file): file-like object to read from. + Args: + file_object (file): file-like object to read from. - Returns: - dict[str, str]: user names with corresponding email address. - """ - config_parser = configparser.ConfigParser(interpolation=None) - config_parser.read_file(file_object) + Returns: + dict[str, str]: user names with corresponding email address. + """ + config_parser = configparser.ConfigParser(interpolation=None) + config_parser.read_file(file_object) - user_mappings = {} - for option_name in config_parser.options('user_mappings'): - user_mapping = self._GetConfigValue( - config_parser, 'user_mappings', option_name) + user_mappings = {} + for option_name in config_parser.options("user_mappings"): + user_mapping = self._GetConfigValue( + config_parser, "user_mappings", option_name + ) - option_name = option_name.lower() - user_mappings[option_name] = user_mapping.lower() + option_name = option_name.lower() + user_mappings[option_name] = user_mapping.lower() - return user_mappings + return user_mappings - def ReadUsernames(self, file_object): - """Reads the usernames. + def ReadUsernames(self, file_object): + """Reads the usernames. - Args: - file_object (file): file-like object to read from. + Args: + file_object (file): file-like object to read from. - Returns: - dict[str, str]: user names with corresponding email address. - """ - config_parser = configparser.ConfigParser(interpolation=None) - config_parser.read_file(file_object) + Returns: + dict[str, str]: user names with corresponding email address. + """ + config_parser = configparser.ConfigParser(interpolation=None) + config_parser.read_file(file_object) - usernames = {} - for option_name in config_parser.options('usernames'): - email_address = self._GetConfigValue( - config_parser, 'usernames', option_name) + usernames = {} + for option_name in config_parser.options("usernames"): + email_address = self._GetConfigValue( + config_parser, "usernames", option_name + ) - usernames[option_name] = email_address + usernames[option_name] = email_address - return usernames + return usernames class DownloadHelper: - """Class that defines a download helper.""" + """Class that defines a download helper.""" - def _DownloadPageContent(self, download_url): - """Downloads the page content from the URL. + def _DownloadPageContent(self, download_url): + """Downloads the page content from the URL. - Args: - download_url (str): URL where to download the page content. + Args: + download_url (str): URL where to download the page content. - Returns: - tuple[bytes, TODO]: page content and response headers if successful or - None otherwise. - """ - if not download_url: - return None, None + Returns: + tuple[bytes, TODO]: page content and response headers if successful or + None otherwise. + """ + if not download_url: + return None, None - page_content = None - response_headers = None + page_content = None + response_headers = None - try: - with urlopen(download_url) as url_object: - if url_object.code == 200: - page_content = url_object.read() - response_headers = url_object.info() + try: + with urlopen(download_url) as url_object: + if url_object.code == 200: + page_content = url_object.read() + response_headers = url_object.info() - except urllib_error.URLError as exception: - logging.warning( - f'Unable to download URL: {download_url:s} with error: {exception!s}') + except urllib_error.URLError as exception: + logging.warning( + f"Unable to download URL: {download_url:s} with error: {exception!s}" + ) - return page_content, response_headers + return page_content, response_headers class GithubContributionsHelper(DownloadHelper): - """Class that defines a GitHub contributions helper.""" - - def _ListContributionsForProject( - self, organization, project_name, output_writer): - """Lists the contributions of a specific project. - - Args: - organization (str): name of the organization. - project_name (str): name of the project. - output_writer (OutputWriter): output writer. - """ - download_url = ( - f'https://api.github.com/repos/{organization:s}/{project_name:s}/' - f'stats/contributors') - - contributors_data, response = self._DownloadPageContent(download_url) - if not contributors_data: - return - - # TODO: check if response is not None - _ = response - - contributors_json = json.loads(contributors_data) - self._WriteContributions(project_name, contributors_json, output_writer) - - def _ListPullRequestsForProject( - self, organization, project_name, output_writer): - """Lists the pull requests of a specific project. - - Args: - organization (str): name of the organization. - project_name (str): name of the project. - output_writer (OutputWriter): output writer. - """ - download_url = ( - f'https://api.github.com/repos/{organization:s}/{project_name:s}/' - f'pulls?state=all') - - pulls_data, response = self._DownloadPageContent(download_url) - if not pulls_data: - return - - # TODO: check if response is not None - _ = response - - pulls_json = json.loads(pulls_data) - self._WritePullRequests(project_name, pulls_json, output_writer) - - def _WriteContributions(self, project_name, contributors_json, output_writer): - """Writes the contributions to the output writer. - - Args: - project_name (str): name of the project. - contributors_json (list[object]): JSON formatted contributors objects. - output_writer (OutputWriter): output writer. - """ - # https://developer.github.com/v3/repos/statistics/ - # [{ - # "author": { - # "login": string containing the login name, - # ... - # } - # "weeks": [{ - # "a": integer containing the number of lines added, - # "c": integer containing the number of contributions, - # "d": integer containing the number of lines deleted, - # "w": integer containing a POSIX timestamp of the start of the week, - # }, ...], - # }, ...] - - contributions_per_week = {} - for contributions_per_author_json in contributors_json: - author_json = contributions_per_author_json.get("author", None) - if not author_json: - logging.error('Missing author JSON dictionary.') - continue - - weeks_json = contributions_per_author_json.get("weeks", None) - if not weeks_json: - logging.error('Missing weeks JSON list.') - continue - - login_name = author_json.get("login", None) - if not login_name: - logging.error('Missing login name JSON value.') - continue - - # TODO: map login name to username - - for week_json in weeks_json: - number_of_lines_added = week_json.get('a', 0) - number_of_contributions = week_json.get('c', 0) - number_of_lines_deleted = week_json.get('d', 0) - - if (not number_of_lines_added and not number_of_contributions and - not number_of_lines_deleted): - continue - - week_timestamp = week_json.get('w', None) - if not week_timestamp: - logging.error('Missing week timestamp JSON value.') - continue - - time_elements = time.gmtime(week_timestamp) - year = time.strftime('%Y', time_elements) - week_number = time.strftime('%U', time_elements) - - if week_number not in contributions_per_week: - contributions_per_week[week_number] = {} - - contributions_per_week[week_number][login_name] = ( - number_of_contributions, number_of_lines_added, - number_of_lines_deleted) - - output_writer.WriteContribution( - year, week_number, login_name, project_name, - number_of_contributions, number_of_lines_added, - number_of_lines_deleted) - - def _WritePullRequests(self, project_name, pulls_json, output_writer): - """Writes the pull requests to the output writer. - - Args: - project_name (str): name of the project. - pulls_json (list[object]): JSON formatted pull objects. - output_writer (OutputWriter): output writer. - """ - # https://developer.github.com/v3/pulls/#list-pull-requests - # [{ - # "created_at": creation date and time of the CL. - # "state": state of the CL. - # "title": string containing the CL description. - # "user": { - # "login": github username. - # }, ...] - # ... - # }, ...] - - def ListContributions(self, projects_per_organization, output_writer): - """Lists the contributions of projects. - - Args: - projects_per_organization (dict[str, list[str]]): organization names - with corresponding projects names. - output_writer (OutputWriter): output writer. - """ - for organization, projects in iter(projects_per_organization.items()): - for project_name in projects: - self._ListContributionsForProject( - organization, project_name, output_writer) - - def ListPullRequests(self, projects_per_organization, output_writer): - """Lists the pull requests of projects. - - Args: - projects_per_organization (dict[str, list[str]]): organization names - with corresponding projects names. - output_writer (OutputWriter): output writer. - """ - for organization, projects in iter(projects_per_organization.items()): - for project_name in projects: - self._ListPullRequestsForProject( - organization, project_name, output_writer) + """Class that defines a GitHub contributions helper.""" + + def _ListContributionsForProject(self, organization, project_name, output_writer): + """Lists the contributions of a specific project. + + Args: + organization (str): name of the organization. + project_name (str): name of the project. + output_writer (OutputWriter): output writer. + """ + download_url = ( + f"https://api.github.com/repos/{organization:s}/{project_name:s}/" + f"stats/contributors" + ) + + contributors_data, response = self._DownloadPageContent(download_url) + if not contributors_data: + return + + # TODO: check if response is not None + _ = response + + contributors_json = json.loads(contributors_data) + self._WriteContributions(project_name, contributors_json, output_writer) + + def _ListPullRequestsForProject(self, organization, project_name, output_writer): + """Lists the pull requests of a specific project. + + Args: + organization (str): name of the organization. + project_name (str): name of the project. + output_writer (OutputWriter): output writer. + """ + download_url = ( + f"https://api.github.com/repos/{organization:s}/{project_name:s}/" + f"pulls?state=all" + ) + + pulls_data, response = self._DownloadPageContent(download_url) + if not pulls_data: + return + + # TODO: check if response is not None + _ = response + + pulls_json = json.loads(pulls_data) + self._WritePullRequests(project_name, pulls_json, output_writer) + + def _WriteContributions(self, project_name, contributors_json, output_writer): + """Writes the contributions to the output writer. + + Args: + project_name (str): name of the project. + contributors_json (list[object]): JSON formatted contributors objects. + output_writer (OutputWriter): output writer. + """ + # https://developer.github.com/v3/repos/statistics/ + # [{ + # "author": { + # "login": string containing the login name, + # ... + # } + # "weeks": [{ + # "a": integer containing the number of lines added, + # "c": integer containing the number of contributions, + # "d": integer containing the number of lines deleted, + # "w": integer containing a POSIX timestamp of the start of the week, + # }, ...], + # }, ...] + + contributions_per_week = {} + for contributions_per_author_json in contributors_json: + author_json = contributions_per_author_json.get("author", None) + if not author_json: + logging.error("Missing author JSON dictionary.") + continue + + weeks_json = contributions_per_author_json.get("weeks", None) + if not weeks_json: + logging.error("Missing weeks JSON list.") + continue + + login_name = author_json.get("login", None) + if not login_name: + logging.error("Missing login name JSON value.") + continue + + # TODO: map login name to username + + for week_json in weeks_json: + number_of_lines_added = week_json.get("a", 0) + number_of_contributions = week_json.get("c", 0) + number_of_lines_deleted = week_json.get("d", 0) + + if ( + not number_of_lines_added + and not number_of_contributions + and not number_of_lines_deleted + ): + continue + + week_timestamp = week_json.get("w", None) + if not week_timestamp: + logging.error("Missing week timestamp JSON value.") + continue + + time_elements = time.gmtime(week_timestamp) + year = time.strftime("%Y", time_elements) + week_number = time.strftime("%U", time_elements) + + if week_number not in contributions_per_week: + contributions_per_week[week_number] = {} + + contributions_per_week[week_number][login_name] = ( + number_of_contributions, + number_of_lines_added, + number_of_lines_deleted, + ) + + output_writer.WriteContribution( + year, + week_number, + login_name, + project_name, + number_of_contributions, + number_of_lines_added, + number_of_lines_deleted, + ) + + def _WritePullRequests(self, project_name, pulls_json, output_writer): + """Writes the pull requests to the output writer. + + Args: + project_name (str): name of the project. + pulls_json (list[object]): JSON formatted pull objects. + output_writer (OutputWriter): output writer. + """ + # https://developer.github.com/v3/pulls/#list-pull-requests + # [{ + # "created_at": creation date and time of the CL. + # "state": state of the CL. + # "title": string containing the CL description. + # "user": { + # "login": github username. + # }, ...] + # ... + # }, ...] + + def ListContributions(self, projects_per_organization, output_writer): + """Lists the contributions of projects. + + Args: + projects_per_organization (dict[str, list[str]]): organization names + with corresponding projects names. + output_writer (OutputWriter): output writer. + """ + for organization, projects in iter(projects_per_organization.items()): + for project_name in projects: + self._ListContributionsForProject( + organization, project_name, output_writer + ) + + def ListPullRequests(self, projects_per_organization, output_writer): + """Lists the pull requests of projects. + + Args: + projects_per_organization (dict[str, list[str]]): organization names + with corresponding projects names. + output_writer (OutputWriter): output writer. + """ + for organization, projects in iter(projects_per_organization.items()): + for project_name in projects: + self._ListPullRequestsForProject( + organization, project_name, output_writer + ) class StdoutWriter: - """Class that defines a stdout output writer.""" + """Class that defines a stdout output writer.""" + + def __init__(self, user_mappings, output_format="csv"): + """Initializes a stdout output writer. + + Args: + user_mappings (dict[str, str]): mapping between GitHub username and + another username. + output_format (Optional[str]): output format. + """ + super().__init__() + self._header_written = False + self._output_format = output_format + self._user_mappings = user_mappings + + def Open(self): + """Opens the output writer object. + + Returns: + bool: True if successful or False if not. + """ + return True + + def Close(self): + """Closes the output writer object.""" + return - def __init__(self, user_mappings, output_format='csv'): - """Initializes a stdout output writer. + def Write(self, data): + """Writes the data to stdout (without the default trailing newline). + + Args: + data (str): data to write. + """ + print(data, end="") + + def WriteContribution( + self, + year, + week_number, + login_name, + project_name, + number_of_contributions, + number_of_lines_added, + number_of_lines_deleted, + ): + """Writes a contribution to stdout. + + Args: + year (int): year of the contribution. + week_number (int): week number of the contribution. + login_name (str): log-in name. + project_name (str): project name. + number_of_contributions (int): number of contributed CLs. + number_of_lines_added (int): total number of lines added. + number_of_lines_deleted (int): total number of lines deleted. + """ + username = login_name + + if self._user_mappings: + username = username.lower() + username = self._user_mappings.get(username, None) + # TODO: add flag to control this behavior. + if not username: + # Skip login names without a username mapping. + return + + if self._output_format == "csv": + if not self._header_written: + self.Write( + "year\tweek number\tlogin name\tproject\tnumber of contributions\t" + "number lines added\tnumber lines deleted\n" + ) + + self._header_written = True + + self.Write( + ( + f"{year:s}\t{week_number:s}\t{username:s}\t{project_name:s}\t" + f"{number_of_contributions:d}\t{number_of_lines_added:d}\t" + f"{number_of_lines_deleted:d}\n" + ) + ) + + elif self._output_format == "tilde": + date_time = datetime.datetime.strptime( + f"{year:s}-W{week_number:s}-0", "%Y-W%W-%w" + ) + date_time_string = date_time.isoformat() + + # TODO: add description. + self.Write( + ( + f"{date_time_string:s} [github] ~ author:{username:s} ~ " + f"project:{project_name:s} ~ " + f"number_of_cls:{number_of_contributions:d} ~ " + f"delta_added:{number_of_lines_added:d} ~ " + f"delta_deleted:{number_of_lines_deleted:d} ~ " + f"py:{number_of_lines_added:d} ~ file_type:py ~ op_type:ADD ~\n" + ) + ) + + def WriteReview( + self, creation_time, created_by, issue_number, description, reviewers, status + ): + """Writes a review to stdout. + + Args: + creation_time (str): creation date and time. + created_by (str): created by. + issue_number (int): code review issue number. + description (str): description. + reviewers (str): reviewers. + status (str): status. + """ + if self._output_format == "csv": + if not self._header_written: + self.Write( + "creation time\tcreated by\tissue number\tdescription\treviewers\t" + "status\n" + ) + + self._header_written = True + + self.Write( + ( + f"{creation_time:s}\t{created_by:s}\t{issue_number:d}\t" + f"{description:s}\t{reviewers:s}\t{status:s}\n" + ) + ) - Args: - user_mappings (dict[str, str]): mapping between GitHub username and - another username. - output_format (Optional[str]): output format. - """ - super().__init__() - self._header_written = False - self._output_format = output_format - self._user_mappings = user_mappings - def Open(self): - """Opens the output writer object. +def Main(): + """The main program function. Returns: bool: True if successful or False if not. """ - return True + statistics_types = frozenset(["contributions"]) + + argument_parser = argparse.ArgumentParser( + description=("Generates an overview of project statistics of github projects.") + ) + + argument_parser.add_argument( + "-c", + "--config", + dest="config_path", + action="store", + metavar="PATH", + default=None, + help=( + "path of the directory containing the statistics configuration " + "files e.g. stats.ini." + ), + ) + + argument_parser.add_argument( + "-f", + "--format", + dest="output_format", + action="store", + metavar="FORMAT", + choices=["csv", "tilde"], + default="csv", + help="output format.", + ) + + argument_parser.add_argument( + "statistics_type", + action="store", + metavar="TYPE", + choices=sorted(statistics_types), + default=None, + help="The statistics type.", + ) + + options = argument_parser.parse_args() + + if not options.statistics_type: + print("Statistics type missing.") + print("") + argument_parser.print_help() + print("") + return False + + config_path = options.config_path + if not config_path: + config_path = os.path.dirname(__file__) + config_path = os.path.dirname(config_path) + config_path = os.path.join(config_path, "data") + + stats_file = os.path.join(config_path, "stats.ini") + if not os.path.exists(stats_file): + print(f"No such config file: {stats_file:s}") + print("") + return False + + stats_definition_reader = StatsDefinitionReader() - def Close(self): - """Closes the output writer object.""" - return + user_mappings = {} + with open(stats_file, "r", encoding="utf-8") as file_object: + user_mappings = stats_definition_reader.ReadUserMappings(file_object) - def Write(self, data): - """Writes the data to stdout (without the default trailing newline). + output_writer = StdoutWriter(user_mappings, output_format=options.output_format) - Args: - data (str): data to write. - """ - print(data, end='') - - def WriteContribution( - self, year, week_number, login_name, project_name, - number_of_contributions, number_of_lines_added, number_of_lines_deleted): - """Writes a contribution to stdout. - - Args: - year (int): year of the contribution. - week_number (int): week number of the contribution. - login_name (str): log-in name. - project_name (str): project name. - number_of_contributions (int): number of contributed CLs. - number_of_lines_added (int): total number of lines added. - number_of_lines_deleted (int): total number of lines deleted. - """ - username = login_name - - if self._user_mappings: - username = username.lower() - username = self._user_mappings.get(username, None) - # TODO: add flag to control this behavior. - if not username: - # Skip login names without a username mapping. - return + if not output_writer.Open(): + print("Unable to open output writer.") + print("") + return False - if self._output_format == 'csv': - if not self._header_written: - self.Write( - 'year\tweek number\tlogin name\tproject\tnumber of contributions\t' - 'number lines added\tnumber lines deleted\n') - - self._header_written = True - - self.Write(( - f'{year:s}\t{week_number:s}\t{username:s}\t{project_name:s}\t' - f'{number_of_contributions:d}\t{number_of_lines_added:d}\t' - f'{number_of_lines_deleted:d}\n')) - - elif self._output_format == 'tilde': - date_time = datetime.datetime.strptime( - f'{year:s}-W{week_number:s}-0', '%Y-W%W-%w') - date_time_string = date_time.isoformat() - - # TODO: add description. - self.Write(( - f'{date_time_string:s} [github] ~ author:{username:s} ~ ' - f'project:{project_name:s} ~ ' - f'number_of_cls:{number_of_contributions:d} ~ ' - f'delta_added:{number_of_lines_added:d} ~ ' - f'delta_deleted:{number_of_lines_deleted:d} ~ ' - f'py:{number_of_lines_added:d} ~ file_type:py ~ op_type:ADD ~\n')) - - def WriteReview( - self, creation_time, created_by, issue_number, description, reviewers, - status): - """Writes a review to stdout. - - Args: - creation_time (str): creation date and time. - created_by (str): created by. - issue_number (int): code review issue number. - description (str): description. - reviewers (str): reviewers. - status (str): status. - """ - if self._output_format == 'csv': - if not self._header_written: - self.Write( - 'creation time\tcreated by\tissue number\tdescription\treviewers\t' - 'status\n') + if options.statistics_type == "contributions": + projects_per_organization = {} + with open(stats_file, "r", encoding="utf-8") as file_object: + stats_definition_reader = StatsDefinitionReader() + projects_per_organization = ( + stats_definition_reader.ReadProjectsPerOrganization(file_object) + ) - self._header_written = True + contributions_helper = GithubContributionsHelper() + contributions_helper.ListContributions(projects_per_organization, output_writer) - self.Write(( - f'{creation_time:s}\t{created_by:s}\t{issue_number:d}\t' - f'{description:s}\t{reviewers:s}\t{status:s}\n')) + return True -def Main(): - """The main program function. - - Returns: - bool: True if successful or False if not. - """ - statistics_types = frozenset(['contributions']) - - argument_parser = argparse.ArgumentParser(description=( - 'Generates an overview of project statistics of github projects.')) - - argument_parser.add_argument( - '-c', '--config', dest='config_path', action='store', - metavar='PATH', default=None, help=( - 'path of the directory containing the statistics configuration ' - 'files e.g. stats.ini.')) - - argument_parser.add_argument( - '-f', '--format', dest='output_format', action='store', - metavar='FORMAT', choices=['csv', 'tilde'], default='csv', - help='output format.') - - argument_parser.add_argument( - 'statistics_type', action='store', metavar='TYPE', - choices=sorted(statistics_types), default=None, - help='The statistics type.') - - options = argument_parser.parse_args() - - if not options.statistics_type: - print('Statistics type missing.') - print('') - argument_parser.print_help() - print('') - return False - - config_path = options.config_path - if not config_path: - config_path = os.path.dirname(__file__) - config_path = os.path.dirname(config_path) - config_path = os.path.join(config_path, 'data') - - stats_file = os.path.join(config_path, 'stats.ini') - if not os.path.exists(stats_file): - print(f'No such config file: {stats_file:s}') - print('') - return False - - stats_definition_reader = StatsDefinitionReader() - - user_mappings = {} - with open(stats_file, 'r', encoding='utf-8') as file_object: - user_mappings = stats_definition_reader.ReadUserMappings(file_object) - - output_writer = StdoutWriter( - user_mappings, output_format=options.output_format) - - if not output_writer.Open(): - print('Unable to open output writer.') - print('') - return False - - if options.statistics_type == 'contributions': - projects_per_organization = {} - with open(stats_file, 'r', encoding='utf-8') as file_object: - stats_definition_reader = StatsDefinitionReader() - projects_per_organization = ( - stats_definition_reader.ReadProjectsPerOrganization(file_object)) - - contributions_helper = GithubContributionsHelper() - contributions_helper.ListContributions( - projects_per_organization, output_writer) - - return True - - -if __name__ == '__main__': - if not Main(): - sys.exit(1) - else: - sys.exit(0) +if __name__ == "__main__": + if not Main(): + sys.exit(1) + else: + sys.exit(0) diff --git a/tools/update-dependencies.py b/tools/update-dependencies.py index 26b57e6f..bbddb682 100755 --- a/tools/update-dependencies.py +++ b/tools/update-dependencies.py @@ -27,93 +27,93 @@ def Main(): - """The main program function. - - Returns: - bool: True if successful or False if not. - """ - l2tdevtools_path = os.path.abspath(__file__) - l2tdevtools_path = os.path.dirname(l2tdevtools_path) - l2tdevtools_path = os.path.dirname(l2tdevtools_path) - - project_path = os.getcwd() - projects_helper = project.ProjectHelper(project_path) - project_definition = projects_helper.ReadDefinitionFile() - - dependencies_helper = dependencies.DependencyHelper() - - for writer_class in ( - pylint_rc.PylintRcWriter, setup.PyprojectTomlWriter): - writer = writer_class( - l2tdevtools_path, project_definition, dependencies_helper) - writer.Write() - - for writer_class in ( - github_actions.GitHubActionsLintYmlWriter, - github_actions.GitHubActionsTestDockerYmlWriter, - github_actions.GitHubActionsTestDocsYmlWriter, - github_actions.GitHubActionsTestMacOSYmlWriter, - github_actions.GitHubActionsTestToxYmlWriter, - appveyor_scripts.AppVeyorInstallPS1ScriptWriter, - appveyor_yml.AppveyorYmlWriter, - check_dependencies.CheckDependenciesWriter, - dependencies_py.DependenciesPyWriter, dpkg.DPKGCompatWriter, - dpkg.DPKGControlWriter, dpkg.DPKGRulesWriter, - gift_copr.GIFTCOPRInstallScriptWriter, - gift_ppa.GIFTPPAInstallScriptWriter, - jenkins_scripts.LinuxRunEndToEndTestsScriptWriter, - jenkins_scripts.RunPython3EndToEndTestsScriptWriter, - linux_scripts.UbuntuInstallationScriptWriter, - sphinx_docs.ReadthedocsConfigurationWriter, - sphinx_docs.SphinxBuildConfigurationWriter, - sphinx_docs.SphinxBuildRequirementsWriter, tox_ini.ToxIniWriter): - if not os.path.exists(writer_class.PATH): - continue - - writer = writer_class( - l2tdevtools_path, project_definition, dependencies_helper) - writer.Write() - - output_path = os.path.join('utils', 'dependencies.py') - if os.path.exists(output_path): - input_path = os.path.join( - l2tdevtools_path, 'l2tdevtools', 'dependencies.py') - file_data = [] - with io.open(input_path, 'r', encoding='utf-8') as file_object: - for line in file_object.readlines(): - if '# The following functions should not be included in ' in line: - break - - file_data.append(line) - - file_data.pop() - file_data = ''.join(file_data) - - with io.open(output_path, 'w', encoding='utf-8') as file_object: - file_object.write(file_data) - - # Remove old configurations and scripts. - script_path = os.path.join('config', 'linux', 'gift_ppa_install.sh') - if os.path.isfile(script_path): - os.remove(script_path) - - script_path = os.path.join('config', 'macos') - if os.path.isfile(script_path): - shutil.rmtree(script_path) - - script_path = os.path.join('.travis.yml') - if os.path.isfile(script_path): - os.remove(script_path) - - script_path = os.path.join('config', 'travis') - if os.path.isfile(script_path): - shutil.rmtree(script_path) - - return True - - -if __name__ == '__main__': - if not Main(): - sys.exit(1) - else: - sys.exit(0) + """The main program function. + + Returns: + bool: True if successful or False if not. + """ + l2tdevtools_path = os.path.abspath(__file__) + l2tdevtools_path = os.path.dirname(l2tdevtools_path) + l2tdevtools_path = os.path.dirname(l2tdevtools_path) + + project_path = os.getcwd() + projects_helper = project.ProjectHelper(project_path) + project_definition = projects_helper.ReadDefinitionFile() + + dependencies_helper = dependencies.DependencyHelper() + + for writer_class in (pylint_rc.PylintRcWriter, setup.PyprojectTomlWriter): + writer = writer_class(l2tdevtools_path, project_definition, dependencies_helper) + writer.Write() + + for writer_class in ( + github_actions.GitHubActionsLintYmlWriter, + github_actions.GitHubActionsTestDockerYmlWriter, + github_actions.GitHubActionsTestDocsYmlWriter, + github_actions.GitHubActionsTestMacOSYmlWriter, + github_actions.GitHubActionsTestToxYmlWriter, + appveyor_scripts.AppVeyorInstallPS1ScriptWriter, + appveyor_yml.AppveyorYmlWriter, + check_dependencies.CheckDependenciesWriter, + dependencies_py.DependenciesPyWriter, + dpkg.DPKGCompatWriter, + dpkg.DPKGControlWriter, + dpkg.DPKGRulesWriter, + gift_copr.GIFTCOPRInstallScriptWriter, + gift_ppa.GIFTPPAInstallScriptWriter, + jenkins_scripts.LinuxRunEndToEndTestsScriptWriter, + jenkins_scripts.RunPython3EndToEndTestsScriptWriter, + linux_scripts.UbuntuInstallationScriptWriter, + sphinx_docs.ReadthedocsConfigurationWriter, + sphinx_docs.SphinxBuildConfigurationWriter, + sphinx_docs.SphinxBuildRequirementsWriter, + tox_ini.ToxIniWriter, + ): + if not os.path.exists(writer_class.PATH): + continue + + writer = writer_class(l2tdevtools_path, project_definition, dependencies_helper) + writer.Write() + + output_path = os.path.join("utils", "dependencies.py") + if os.path.exists(output_path): + input_path = os.path.join(l2tdevtools_path, "l2tdevtools", "dependencies.py") + file_data = [] + with io.open(input_path, "r", encoding="utf-8") as file_object: + for line in file_object.readlines(): + if "# The following functions should not be included in " in line: + break + + file_data.append(line) + + file_data.pop() + file_data = "".join(file_data) + + with io.open(output_path, "w", encoding="utf-8") as file_object: + file_object.write(file_data) + + # Remove old configurations and scripts. + script_path = os.path.join("config", "linux", "gift_ppa_install.sh") + if os.path.isfile(script_path): + os.remove(script_path) + + script_path = os.path.join("config", "macos") + if os.path.isfile(script_path): + shutil.rmtree(script_path) + + script_path = os.path.join(".travis.yml") + if os.path.isfile(script_path): + os.remove(script_path) + + script_path = os.path.join("config", "travis") + if os.path.isfile(script_path): + shutil.rmtree(script_path) + + return True + + +if __name__ == "__main__": + if not Main(): + sys.exit(1) + else: + sys.exit(0) diff --git a/tools/update.py b/tools/update.py index b0e05d3d..5a69f2ec 100755 --- a/tools/update.py +++ b/tools/update.py @@ -18,720 +18,826 @@ class PackageDownload: - """Information about a package download. + """Information about a package download. - Attributes: - filename (str): name of the package file. - name (str): name of the package. - url (str): download URL of the package file. - version (str): version of the package. - """ - - def __init__(self, name, version, filename, url): - """Initializes a package download. - - Args: - name (str): name of the package. - version (str): version of the package. + Attributes: filename (str): name of the package file. + name (str): name of the package. url (str): download URL of the package file. + version (str): version of the package. """ - super().__init__() - self.filename = filename - self.name = name - self.url = url - self.version = version - - -class GithubRepoDownloadHelper(interface.DownloadHelper): - """Helps in downloading from a GitHub repository.""" - - _GITHUB_REPO_API_URL = ( - 'https://api.github.com/repos/log2timeline/l2tbinaries') - - _GITHUB_REPO_URL = ( - 'https://github.com/log2timeline/l2tbinaries') - - _SUPPORTED_PYTHON_VERSIONS = frozenset([(3, 12), (3, 14)]) - - def __init__(self, download_url, branch='main'): - """Initializes a download helper. - - Args: - download_url (str): download URL. - branch (Optional[str]): git branch to download from. - """ - super().__init__(download_url) - self._branch = branch - - def _GetMachineTypeSubDirectory( - self, preferred_machine_type=None, preferred_operating_system=None): - """Retrieves the machine type sub directory. - - Args: - preferred_machine_type (Optional[str]): preferred machine type, where - None, which will auto-detect the current machine type. - preferred_operating_system (Optional[str]): preferred operating system, - where None, which will auto-detect the current operating system. - - Returns: - str: machine type sub directory or None. - """ - if preferred_operating_system: - operating_system = preferred_operating_system - else: - operating_system = platform.system() - - if preferred_machine_type: - cpu_architecture = preferred_machine_type - else: - cpu_architecture = platform.machine().lower() - - sub_directory = None - - if operating_system != 'Windows': - logging.error(f'Operating system: {operating_system:s} not supported.') - return None - - if (sys.version_info[0], sys.version_info[1]) not in ( - self._SUPPORTED_PYTHON_VERSIONS): - major_version = sys.version_info[0] - minor_version = sys.version_info[1] - logging.error( - f'Python version: {major_version:d}.{minor_version:d} not supported.') - return None - - if cpu_architecture == 'x86': - sub_directory = 'win32' - - elif cpu_architecture == 'amd64': - sub_directory = 'win64' - - if not sub_directory: - logging.error(f'CPU architecture: {cpu_architecture:s} not supported.') - return None - - return sub_directory - def _GetDownloadURL( - self, preferred_machine_type=None, preferred_operating_system=None, - use_api=False): - """Retrieves the download URL. + def __init__(self, name, version, filename, url): + """Initializes a package download. - Args: - preferred_machine_type (Optional[str]): preferred machine type, where - None, which will auto-detect the current machine type. - preferred_operating_system (Optional[str]): preferred operating system, - where None, which will auto-detect the current operating system. - use_api (Optional[bool]): True if the GitHub API should be used to - determine the download URL. + Args: + name (str): name of the package. + version (str): version of the package. + filename (str): name of the package file. + url (str): download URL of the package file. + """ + super().__init__() + self.filename = filename + self.name = name + self.url = url + self.version = version - Returns: - str: download URL or None. - """ - sub_directory = self._GetMachineTypeSubDirectory( - preferred_machine_type=preferred_machine_type, - preferred_operating_system=preferred_operating_system) - if not sub_directory: - return None - - if use_api: - return ( - f'{self._GITHUB_REPO_API_URL:s}/contents/{sub_directory:s}?' - f'ref={self._branch:s}') - - return f'{self._GITHUB_REPO_URL:s}/tree/{self._branch:s}/{sub_directory:s}' - - def GetPackageDownloadURLs( - self, preferred_machine_type=None, preferred_operating_system=None, - use_api=False): - """Retrieves the package download URLs for a given system configuration. - - Args: - preferred_machine_type (Optional[str]): preferred machine type, where - None, which will auto-detect the current machine type. - preferred_operating_system (Optional[str]): preferred operating system, - where None, which will auto-detect the current operating system. - use_api (Optional[bool]): True if the GitHub API should be used to - determine the download URL. - - Returns: - list[str]: list of package download URLs or None if no package download - URLs could be determined. - """ - download_url = self._GetDownloadURL( - preferred_machine_type=preferred_machine_type, - preferred_operating_system=preferred_operating_system, use_api=use_api) - if not download_url: - logging.info('Missing download URL.') - return None - - page_content = self.DownloadPageContent(download_url) - if not page_content: - return None - - # TODO: skip SHA256SUMS - - download_urls = [] - if use_api: - # The page content consist of JSON data that contains a list of dicts. - # Each dict consists of: - # { - # "name":"PyYAML-3.11.win-amd64-py2.7.msi", - # "path":"win64/PyYAML-3.11.win-amd64-py2.7.msi", - # "sha":"8fca8c1e2549cf54bf993c55930365d01658f418", - # "size":196608, - # "url":"https://api.github.com/...", - # "html_url":"https://github.com/...", - # "git_url":"https://api.github.com/...", - # "download_url":"https://raw.githubusercontent.com/...", - # "type":"file", - # "_links":{ - # "self":"https://api.github.com/...", - # "git":"https://api.github.com/...", - # "html":"https://github.com/..." - # } - # } - - for directory_entry in json.loads(page_content): - download_url = directory_entry.get('download_url', None) - if download_url: - download_urls.append(download_url) - else: - sub_directory = self._GetMachineTypeSubDirectory( - preferred_machine_type=preferred_machine_type, - preferred_operating_system=preferred_operating_system) - if not sub_directory: - return None - - # The format of the download URL is: - # - expression_string = ( - '') - matches = re.findall(expression_string, page_content) - - if len(matches) == 1: - json_dict = json.loads(matches[0]) - payload = json_dict.get('payload', {}) - code_view_tree_route = payload.get('codeViewTreeRoute', {}) - tree = code_view_tree_route.get('tree', {}) - for item in tree.get('items', []): - item_path = item.get('path', None) - download_url = ( - f'https://github.com/log2timeline/l2tbinaries/raw/' - f'{self._branch:s}/{item_path:s}') - download_urls.append(download_url) - - return download_urls +class GithubRepoDownloadHelper(interface.DownloadHelper): + """Helps in downloading from a GitHub repository.""" + + _GITHUB_REPO_API_URL = "https://api.github.com/repos/log2timeline/l2tbinaries" + + _GITHUB_REPO_URL = "https://github.com/log2timeline/l2tbinaries" + + _SUPPORTED_PYTHON_VERSIONS = frozenset([(3, 12), (3, 14)]) + + def __init__(self, download_url, branch="main"): + """Initializes a download helper. + + Args: + download_url (str): download URL. + branch (Optional[str]): git branch to download from. + """ + super().__init__(download_url) + self._branch = branch + + def _GetMachineTypeSubDirectory( + self, preferred_machine_type=None, preferred_operating_system=None + ): + """Retrieves the machine type sub directory. + + Args: + preferred_machine_type (Optional[str]): preferred machine type, where + None, which will auto-detect the current machine type. + preferred_operating_system (Optional[str]): preferred operating system, + where None, which will auto-detect the current operating system. + + Returns: + str: machine type sub directory or None. + """ + if preferred_operating_system: + operating_system = preferred_operating_system + else: + operating_system = platform.system() + + if preferred_machine_type: + cpu_architecture = preferred_machine_type + else: + cpu_architecture = platform.machine().lower() + + sub_directory = None + + if operating_system != "Windows": + logging.error(f"Operating system: {operating_system:s} not supported.") + return None + + if (sys.version_info[0], sys.version_info[1]) not in ( + self._SUPPORTED_PYTHON_VERSIONS + ): + major_version = sys.version_info[0] + minor_version = sys.version_info[1] + logging.error( + f"Python version: {major_version:d}.{minor_version:d} not supported." + ) + return None + + if cpu_architecture == "x86": + sub_directory = "win32" + + elif cpu_architecture == "amd64": + sub_directory = "win64" + + if not sub_directory: + logging.error(f"CPU architecture: {cpu_architecture:s} not supported.") + return None + + return sub_directory + + def _GetDownloadURL( + self, + preferred_machine_type=None, + preferred_operating_system=None, + use_api=False, + ): + """Retrieves the download URL. + + Args: + preferred_machine_type (Optional[str]): preferred machine type, where + None, which will auto-detect the current machine type. + preferred_operating_system (Optional[str]): preferred operating system, + where None, which will auto-detect the current operating system. + use_api (Optional[bool]): True if the GitHub API should be used to + determine the download URL. + + Returns: + str: download URL or None. + """ + sub_directory = self._GetMachineTypeSubDirectory( + preferred_machine_type=preferred_machine_type, + preferred_operating_system=preferred_operating_system, + ) + if not sub_directory: + return None + + if use_api: + return ( + f"{self._GITHUB_REPO_API_URL:s}/contents/{sub_directory:s}?" + f"ref={self._branch:s}" + ) + + return f"{self._GITHUB_REPO_URL:s}/tree/{self._branch:s}/{sub_directory:s}" + + def GetPackageDownloadURLs( + self, + preferred_machine_type=None, + preferred_operating_system=None, + use_api=False, + ): + """Retrieves the package download URLs for a given system configuration. + + Args: + preferred_machine_type (Optional[str]): preferred machine type, where + None, which will auto-detect the current machine type. + preferred_operating_system (Optional[str]): preferred operating system, + where None, which will auto-detect the current operating system. + use_api (Optional[bool]): True if the GitHub API should be used to + determine the download URL. + + Returns: + list[str]: list of package download URLs or None if no package download + URLs could be determined. + """ + download_url = self._GetDownloadURL( + preferred_machine_type=preferred_machine_type, + preferred_operating_system=preferred_operating_system, + use_api=use_api, + ) + if not download_url: + logging.info("Missing download URL.") + return None + + page_content = self.DownloadPageContent(download_url) + if not page_content: + return None + + # TODO: skip SHA256SUMS + + download_urls = [] + if use_api: + # The page content consist of JSON data that contains a list of dicts. + # Each dict consists of: + # { + # "name":"PyYAML-3.11.win-amd64-py2.7.msi", + # "path":"win64/PyYAML-3.11.win-amd64-py2.7.msi", + # "sha":"8fca8c1e2549cf54bf993c55930365d01658f418", + # "size":196608, + # "url":"https://api.github.com/...", + # "html_url":"https://github.com/...", + # "git_url":"https://api.github.com/...", + # "download_url":"https://raw.githubusercontent.com/...", + # "type":"file", + # "_links":{ + # "self":"https://api.github.com/...", + # "git":"https://api.github.com/...", + # "html":"https://github.com/..." + # } + # } + + for directory_entry in json.loads(page_content): + download_url = directory_entry.get("download_url", None) + if download_url: + download_urls.append(download_url) + + else: + sub_directory = self._GetMachineTypeSubDirectory( + preferred_machine_type=preferred_machine_type, + preferred_operating_system=preferred_operating_system, + ) + if not sub_directory: + return None + + # The format of the download URL is: + # + expression_string = ( + '' + ) + matches = re.findall(expression_string, page_content) + + if len(matches) == 1: + json_dict = json.loads(matches[0]) + payload = json_dict.get("payload", {}) + code_view_tree_route = payload.get("codeViewTreeRoute", {}) + tree = code_view_tree_route.get("tree", {}) + for item in tree.get("items", []): + item_path = item.get("path", None) + download_url = ( + f"https://github.com/log2timeline/l2tbinaries/raw/" + f"{self._branch:s}/{item_path:s}" + ) + download_urls.append(download_url) + + return download_urls class DependencyUpdater: - """Helps in updating dependencies. - - Attributes: - operating_system (str): the operating system on which to update - dependencies and remove previous versions. - """ - - _DOWNLOAD_URL = 'https://github.com/log2timeline/l2tbinaries/releases' - - _GIT_BRANCH_PER_TRACK = { - 'dev': 'dev', - 'stable': 'main', - 'staging': 'staging', - 'testing': 'testing'} - - # Some projects have different PyPI names than their project names. - _PYPI_ALIASES = { - 'flor': 'Flor', - 'lz4': 'python-lz4', - 'redis': 'redis-py', - 'snappy': 'python-snappy', - 'zstd': 'python-zstd'} - - # Some projects have different wheel names in l2tbinaries than defined - # in their project definitions. - # TODO: remove after l2tbinaries Python 3.14 upgrade. - _WHEEL_ALIASES = { - 'bencode.py': 'bencode_py', - 'PyYAML': 'pyyaml', - 'XlsxWriter': 'xlsxwriter'} - - # Some projects have different wheel names than their project names. - _PROJECT_ALIASES = { - 'bencode.py': 'bencode', - 'bencode_py': 'bencode', - 'lz4': 'python-lz4', - 'zstd': 'python-zstd'} - - def __init__( - self, download_directory='build', download_only=False, - download_track='stable', exclude_packages=False, force_install=False, - preferred_machine_type=None, preferred_operating_system=None, - verbose_output=False): - """Initializes the dependency updater. - - Args: - download_directory (Optional[str]): path of the download directory. - download_only (Optional[bool]): True if the dependency packages should - only be downloaded. - download_track (Optional[str]): track to download from. - exclude_packages (Optional[bool]): True if packages should be excluded - instead of included. - force_install (Optional[bool]): True if the installation (update) should - be forced. - preferred_machine_type (Optional[str]): preferred machine type, where - None, which will auto-detect the current machine type. - preferred_operating_system (Optional[str]): preferred operating system, - where None, which will auto-detect the current operating system. - verbose_output (Optional[bool]): True more verbose output should be - provided. - """ - branch = self._GIT_BRANCH_PER_TRACK.get(download_track, 'main') - - super().__init__() - self._download_directory = download_directory - self._download_helper = GithubRepoDownloadHelper( - self._DOWNLOAD_URL, branch=branch) - self._download_only = download_only - self._download_track = download_track - self._exclude_packages = exclude_packages - self._force_install = force_install - self._verbose_output = verbose_output - - if preferred_operating_system: - self.operating_system = preferred_operating_system - else: - self.operating_system = platform.system() - - if preferred_machine_type: - self._preferred_machine_type = preferred_machine_type.lower() - else: - self._preferred_machine_type = None - - def _GetAvailableWheelPackages(self): - """Determines the wheel packages available for download. - - Returns: - list[PackageDownload]: packages available for download. - """ - major_version = sys.version_info[0] - minor_version = sys.version_info[1] - python_version_indicator = f'cp{major_version:d}{minor_version:d}' - - # The API is rate limited, so we scrape the web page instead. - package_urls = self._download_helper.GetPackageDownloadURLs( - preferred_machine_type=self._preferred_machine_type, - preferred_operating_system=self.operating_system) - if not package_urls: - logging.error('Unable to determine package download URLs.') - return [] - - # Use a dictionary so we can more efficiently set a newer version of - # a package that was set previously. - available_packages = {} - - package_versions = {} - for package_url in package_urls: - _, _, package_filename = package_url.rpartition('/') - package_filename = package_filename.lower() - - if not package_filename.endswith('.whl'): - # Ignore all other file extensions. - continue - - (package_name, package_version, python_tag1, python_tag2, - _) = package_filename[:-4].split('-') - - if (python_tag1 not in (python_version_indicator, 'py2.py3', 'py3') and - python_tag2 != 'abi3'): - # Ignore packages that are for different versions of Python. - continue - - version = package_version.split('.') - - if package_name not in package_versions: - compare_result = 1 - else: - compare_result = versions.CompareVersions( - version, package_versions[package_name]) - - if compare_result > 0: - package_versions[package_name] = version - - package_download = PackageDownload( - package_name, version, package_filename, package_url) - available_packages[package_name] = package_download - - return available_packages.values() - - def _GetWheelPackageFilenamesAndVersions( - self, project_definitions, available_packages, - user_defined_wheel_package_names): - """Determines the wheel package filenames and versions. - - Args: - project_definitions (dist[str, ProjectDefinition]): project definitions - per name. - available_packages (list[PackageDownload]): packages available for - download. - user_defined_wheel_package_names (list[str]): names of the wheels of - packages that should be updated if an update is available. These - package names are derived from the user specified names of projects. - An empty list represents all available packages. - - Returns: - tuple: containing: - dict[str, str]: filenames per package. - dict[str, str]: versions per package. - """ - project_definition_per_package_name = {} - for project_name, project_definition in project_definitions.items(): - package_name = getattr( - project_definition, 'wheel_name', None) or project_name - package_name = package_name.lower() - project_definition_per_package_name[package_name] = project_definition - - package_filenames = {} - package_versions = {} - - for package_download in available_packages: - package_name = package_download.name - package_filename = package_download.filename - package_download_path = os.path.join( - self._download_directory, package_filename) - - # Ignore package names if user defined. - if user_defined_wheel_package_names: - in_package_names = package_name in user_defined_wheel_package_names - - alternate_name = self._WHEEL_ALIASES.get(package_name, None) - if alternate_name: - if ((self._exclude_packages and in_package_names) or - (not self._exclude_packages and not in_package_names)): - in_package_names = ( - alternate_name in user_defined_wheel_package_names) - - if ((self._exclude_packages and in_package_names) or - (not self._exclude_packages and not in_package_names)): - logging.info(f'Skipping: {package_name:s} because it was excluded') - continue - - # Remove previous versions of a package. - package_name_suffix = package_filename[:-4] - filenames_glob = f'{package_name:s}*{package_name_suffix:s}' - filenames = glob.glob(os.path.join( - self._download_directory, filenames_glob)) - for filename in filenames: - if filename != package_download_path and os.path.isfile(filename): - logging.info(f'Removing: {filename:s}') - os.remove(filename) - - project_definition = project_definition_per_package_name.get( - package_name, None) - if not project_definition: - alternate_name = self._PROJECT_ALIASES.get(package_name, None) - if alternate_name: - project_definition = project_definitions.get(alternate_name, None) - - if not project_definition: - logging.error( - f'Missing project definition for package: {package_name:s}') - continue - - if not os.path.exists(package_download_path): - logging.info(f'Downloading: {package_filename:s}') - os.chdir(self._download_directory) - try: - self._download_helper.DownloadFile(package_download.url) - finally: - os.chdir('..') - - package_filenames[package_name] = package_filename - package_versions[package_name] = package_download.version - - return package_filenames, package_versions - - def _GetProjectDefinitions(self, projects_file): - """Retrieves the project definitions from the projects file. - - Args: - projects_file (str): path to the projects.ini configuration file. - - Returns: - dist[str, ProjectDefinition]: project definitions per name. - """ - project_definitions = {} - - with open(projects_file, 'r', encoding='utf-8') as file_object: - project_definition_reader = projects.ProjectDefinitionReader() - for project_definition in project_definition_reader.Read(file_object): - project_definitions[project_definition.name] = project_definition - - return project_definitions - - def _GetUserDefinedWheelPackageNames( - self, project_definitions, user_defined_project_names): - """Determines names of wheel packages that should be updated. - - Args: - project_definitions (dist[str, ProjectDefinition]): project definitions - per name. - user_defined_project_names (list[str]): user specified names of projects, - that should be updated if an update is available. An empty list - represents all available projects. + """Helps in updating dependencies. - Returns: - list[str]: names of packages that should be updated if an update is - available. These package names are derived from the user specified - names of projects. An empty list represents all available packages. + Attributes: + operating_system (str): the operating system on which to update + dependencies and remove previous versions. """ - user_defined_wheel_package_names = [] - for project_name in user_defined_project_names: - project_definition = project_definitions.get(project_name, None) - if not project_definition: - alternate_name = self._PYPI_ALIASES.get(project_name, None) - if alternate_name: - project_definition = project_definitions.get(alternate_name, None) - if not project_definition: - logging.error( - f'Missing project definition for package: {project_name:s}') - continue + _DOWNLOAD_URL = "https://github.com/log2timeline/l2tbinaries/releases" + + _GIT_BRANCH_PER_TRACK = { + "dev": "dev", + "stable": "main", + "staging": "staging", + "testing": "testing", + } + + # Some projects have different PyPI names than their project names. + _PYPI_ALIASES = { + "flor": "Flor", + "lz4": "python-lz4", + "redis": "redis-py", + "snappy": "python-snappy", + "zstd": "python-zstd", + } + + # Some projects have different wheel names in l2tbinaries than defined + # in their project definitions. + # TODO: remove after l2tbinaries Python 3.14 upgrade. + _WHEEL_ALIASES = { + "bencode.py": "bencode_py", + "PyYAML": "pyyaml", + "XlsxWriter": "xlsxwriter", + } + + # Some projects have different wheel names than their project names. + _PROJECT_ALIASES = { + "bencode.py": "bencode", + "bencode_py": "bencode", + "lz4": "python-lz4", + "zstd": "python-zstd", + } + + def __init__( + self, + download_directory="build", + download_only=False, + download_track="stable", + exclude_packages=False, + force_install=False, + preferred_machine_type=None, + preferred_operating_system=None, + verbose_output=False, + ): + """Initializes the dependency updater. + + Args: + download_directory (Optional[str]): path of the download directory. + download_only (Optional[bool]): True if the dependency packages should + only be downloaded. + download_track (Optional[str]): track to download from. + exclude_packages (Optional[bool]): True if packages should be excluded + instead of included. + force_install (Optional[bool]): True if the installation (update) should + be forced. + preferred_machine_type (Optional[str]): preferred machine type, where + None, which will auto-detect the current machine type. + preferred_operating_system (Optional[str]): preferred operating system, + where None, which will auto-detect the current operating system. + verbose_output (Optional[bool]): True more verbose output should be + provided. + """ + branch = self._GIT_BRANCH_PER_TRACK.get(download_track, "main") + + super().__init__() + self._download_directory = download_directory + self._download_helper = GithubRepoDownloadHelper( + self._DOWNLOAD_URL, branch=branch + ) + self._download_only = download_only + self._download_track = download_track + self._exclude_packages = exclude_packages + self._force_install = force_install + self._verbose_output = verbose_output + + if preferred_operating_system: + self.operating_system = preferred_operating_system + else: + self.operating_system = platform.system() + + if preferred_machine_type: + self._preferred_machine_type = preferred_machine_type.lower() + else: + self._preferred_machine_type = None + + def _GetAvailableWheelPackages(self): + """Determines the wheel packages available for download. + + Returns: + list[PackageDownload]: packages available for download. + """ + major_version = sys.version_info[0] + minor_version = sys.version_info[1] + python_version_indicator = f"cp{major_version:d}{minor_version:d}" + + # The API is rate limited, so we scrape the web page instead. + package_urls = self._download_helper.GetPackageDownloadURLs( + preferred_machine_type=self._preferred_machine_type, + preferred_operating_system=self.operating_system, + ) + if not package_urls: + logging.error("Unable to determine package download URLs.") + return [] + + # Use a dictionary so we can more efficiently set a newer version of + # a package that was set previously. + available_packages = {} + + package_versions = {} + for package_url in package_urls: + _, _, package_filename = package_url.rpartition("/") + package_filename = package_filename.lower() + + if not package_filename.endswith(".whl"): + # Ignore all other file extensions. + continue + + (package_name, package_version, python_tag1, python_tag2, _) = ( + package_filename[:-4].split("-") + ) + + if ( + python_tag1 not in (python_version_indicator, "py2.py3", "py3") + and python_tag2 != "abi3" + ): + # Ignore packages that are for different versions of Python. + continue + + version = package_version.split(".") + + if package_name not in package_versions: + compare_result = 1 + else: + compare_result = versions.CompareVersions( + version, package_versions[package_name] + ) + + if compare_result > 0: + package_versions[package_name] = version + + package_download = PackageDownload( + package_name, version, package_filename, package_url + ) + available_packages[package_name] = package_download + + return available_packages.values() + + def _GetWheelPackageFilenamesAndVersions( + self, project_definitions, available_packages, user_defined_wheel_package_names + ): + """Determines the wheel package filenames and versions. + + Args: + project_definitions (dist[str, ProjectDefinition]): project definitions + per name. + available_packages (list[PackageDownload]): packages available for + download. + user_defined_wheel_package_names (list[str]): names of the wheels of + packages that should be updated if an update is available. These + package names are derived from the user specified names of projects. + An empty list represents all available packages. + + Returns: + tuple: containing: + dict[str, str]: filenames per package. + dict[str, str]: versions per package. + """ + project_definition_per_package_name = {} + for project_name, project_definition in project_definitions.items(): + package_name = ( + getattr(project_definition, "wheel_name", None) or project_name + ) + package_name = package_name.lower() + project_definition_per_package_name[package_name] = project_definition + + package_filenames = {} + package_versions = {} + + for package_download in available_packages: + package_name = package_download.name + package_filename = package_download.filename + package_download_path = os.path.join( + self._download_directory, package_filename + ) + + # Ignore package names if user defined. + if user_defined_wheel_package_names: + in_package_names = package_name in user_defined_wheel_package_names + + alternate_name = self._WHEEL_ALIASES.get(package_name, None) + if alternate_name: + if (self._exclude_packages and in_package_names) or ( + not self._exclude_packages and not in_package_names + ): + in_package_names = ( + alternate_name in user_defined_wheel_package_names + ) + + if (self._exclude_packages and in_package_names) or ( + not self._exclude_packages and not in_package_names + ): + logging.info(f"Skipping: {package_name:s} because it was excluded") + continue + + # Remove previous versions of a package. + package_name_suffix = package_filename[:-4] + filenames_glob = f"{package_name:s}*{package_name_suffix:s}" + filenames = glob.glob( + os.path.join(self._download_directory, filenames_glob) + ) + for filename in filenames: + if filename != package_download_path and os.path.isfile(filename): + logging.info(f"Removing: {filename:s}") + os.remove(filename) + + project_definition = project_definition_per_package_name.get( + package_name, None + ) + if not project_definition: + alternate_name = self._PROJECT_ALIASES.get(package_name, None) + if alternate_name: + project_definition = project_definitions.get(alternate_name, None) + + if not project_definition: + logging.error( + f"Missing project definition for package: {package_name:s}" + ) + continue + + if not os.path.exists(package_download_path): + logging.info(f"Downloading: {package_filename:s}") + os.chdir(self._download_directory) + try: + self._download_helper.DownloadFile(package_download.url) + finally: + os.chdir("..") + + package_filenames[package_name] = package_filename + package_versions[package_name] = package_download.version + + return package_filenames, package_versions + + def _GetProjectDefinitions(self, projects_file): + """Retrieves the project definitions from the projects file. + + Args: + projects_file (str): path to the projects.ini configuration file. + + Returns: + dist[str, ProjectDefinition]: project definitions per name. + """ + project_definitions = {} + + with open(projects_file, "r", encoding="utf-8") as file_object: + project_definition_reader = projects.ProjectDefinitionReader() + for project_definition in project_definition_reader.Read(file_object): + project_definitions[project_definition.name] = project_definition + + return project_definitions + + def _GetUserDefinedWheelPackageNames( + self, project_definitions, user_defined_project_names + ): + """Determines names of wheel packages that should be updated. + + Args: + project_definitions (dist[str, ProjectDefinition]): project definitions + per name. + user_defined_project_names (list[str]): user specified names of projects, + that should be updated if an update is available. An empty list + represents all available projects. + + Returns: + list[str]: names of packages that should be updated if an update is + available. These package names are derived from the user specified + names of projects. An empty list represents all available packages. + """ + user_defined_wheel_package_names = [] + for project_name in user_defined_project_names: + project_definition = project_definitions.get(project_name, None) + if not project_definition: + alternate_name = self._PYPI_ALIASES.get(project_name, None) + if alternate_name: + project_definition = project_definitions.get(alternate_name, None) + + if not project_definition: + logging.error( + f"Missing project definition for package: {project_name:s}" + ) + continue + + package_name = ( + getattr(project_definition, "wheel_name", None) or project_name + ) + + package_name = package_name.lower() + user_defined_wheel_package_names.append(package_name) + + return user_defined_wheel_package_names + + def _InstallWheelPackagesWindows(self, package_filenames, package_versions): + """Installs wheel packages on Windows. + + Args: + package_filenames (dict[str, str]): filenames per package. + package_versions (dict[str, str]): versions per package. + + Returns: + bool: True if the installation was successful. + """ + package_paths = [ + os.path.join(self._download_directory, package_filenames[name]) + for name in package_versions + ] + + result = True + if package_paths: + package_paths = " ".join(package_paths) + logging.info(f"Installing: {package_paths:s}") + + command = f"{sys.executable:s} -m pip install {package_paths:s}" + exit_code = subprocess.call(command, shell=False) + if exit_code != 0: + logging.error(f'Running: "{command:s}" failed.') + result = False + + return result + + def ExpandPresets(self, preset_definitions, preset_names): + """Expands preset names to project names. + + Args: + preset_definitions (dict[str, PresetDefinition]): preset definitions. + preset_names (str): names of the presets to expand. + + Returns: + set[str]: project names. + """ + project_names = set() + for preset_name in preset_names: + preset_definition = preset_definitions.get(preset_name, None) + if not preset_definition: + continue + + if preset_definition.preset_names: + sub_project_names = self.ExpandPresets( + preset_definitions, preset_definition.preset_names + ) + project_names = project_names.union(sub_project_names) + + project_names = project_names.union(set(preset_definition.project_names)) + + return project_names + + def UpdatePackages(self, projects_file, user_defined_project_names): + """Updates packages. + + Args: + projects_file (str): path to the projects.ini configuration file. + user_defined_project_names (list[str]): user specified names of projects, + that should be updated if an update is available. An empty list + represents all available projects. + + Returns: + bool: True if the update was successful. + """ + project_definitions = self._GetProjectDefinitions(projects_file) + + user_defined_wheel_package_names = self._GetUserDefinedWheelPackageNames( + project_definitions, user_defined_project_names + ) + + available_packages = self._GetAvailableWheelPackages() + if not available_packages: + logging.error("No packages found.") + return False + + if not os.path.exists(self._download_directory): + os.mkdir(self._download_directory) - package_name = getattr( - project_definition, 'wheel_name', None) or project_name + package_filenames, package_versions = self._GetWheelPackageFilenamesAndVersions( + project_definitions, available_packages, user_defined_wheel_package_names + ) - package_name = package_name.lower() - user_defined_wheel_package_names.append(package_name) + if self._download_only: + return True - return user_defined_wheel_package_names + return self._InstallWheelPackagesWindows(package_filenames, package_versions) - def _InstallWheelPackagesWindows(self, package_filenames, package_versions): - """Installs wheel packages on Windows. - Args: - package_filenames (dict[str, str]): filenames per package. - package_versions (dict[str, str]): versions per package. +def Main(): + """The main program function. Returns: - bool: True if the installation was successful. + bool: True if successful or False if not. """ - package_paths = [ - os.path.join(self._download_directory, package_filenames[name]) - for name in package_versions] - - result = True - if package_paths: - package_paths = ' '.join(package_paths) - logging.info(f'Installing: {package_paths:s}') - - command = f'{sys.executable:s} -m pip install {package_paths:s}' - exit_code = subprocess.call(command, shell=False) - if exit_code != 0: - logging.error(f'Running: "{command:s}" failed.') - result = False + tracks = ["dev", "stable", "staging", "testing"] + + argument_parser = argparse.ArgumentParser( + description=("Installs the latest versions of project dependencies.") + ) + + argument_parser.add_argument( + "-c", + "--config", + dest="config_path", + action="store", + metavar="CONFIG_PATH", + default=None, + help=( + "path of the directory containing the build configuration " + "files e.g. projects.ini." + ), + ) + + argument_parser.add_argument( + "--download-directory", + "--download_directory", + action="store", + metavar="DIRECTORY", + dest="download_directory", + type=str, + default="build", + help="The location of the download directory.", + ) + + argument_parser.add_argument( + "--download-only", + "--download_only", + action="store_true", + dest="download_only", + default=False, + help=( + "Only download the dependencies. The default behavior is to " + "download and update the dependencies." + ), + ) + + argument_parser.add_argument( + "-e", + "--exclude", + action="store_true", + dest="exclude_packages", + default=False, + help=("Excludes the package names instead of including them."), + ) + + argument_parser.add_argument( + "-f", + "--force", + action="store_true", + dest="force_install", + default=False, + help=( + "Force installation. This option removes existing versions " + "of installed dependencies. The default behavior is to only" + "install a dependency if not or an older version is installed." + ), + ) + + argument_parser.add_argument( + "--machine-type", + "--machine_type", + action="store", + metavar="TYPE", + dest="machine_type", + type=str, + default=None, + help=( + "Manually sets the machine type instead of using the value returned " + "by platform.machine(). Usage of this argument is not recommended " + "unless want to force the installation of one machine type e.g. " + "'x86' onto another 'amd64'." + ), + ) + + argument_parser.add_argument( + "--preset", + dest="preset", + action="store", + metavar="PRESET_NAME", + default=None, + help=( + "name of the preset of project names to update. The default is to " + "update all project defined in the projects.ini configuration file. " + "The presets are defined in the preset.ini configuration file." + ), + ) + + argument_parser.add_argument( + "-t", + "--track", + dest="track", + action="store", + metavar="TRACK", + default="stable", + choices=sorted(tracks), + help=("the l2tbinaries track to download from. The default is stable."), + ) + + argument_parser.add_argument( + "-v", + "--verbose", + dest="verbose", + action="store_true", + default=False, + help="have more verbose output.", + ) + + argument_parser.add_argument( + "project_names", + nargs="*", + action="store", + metavar="NAME", + type=str, + help=( + "Optional project names which should be updated if an update is " + "available. The corresponding package names are derived from " + "the projects.ini configuration file. If no value is provided " + "all available packages are updated." + ), + ) + + options = argument_parser.parse_args() + + config_path = options.config_path + if not config_path: + config_path = os.path.dirname(__file__) + config_path = os.path.dirname(config_path) + config_path = os.path.join(config_path, "data") + + presets_file = os.path.join(config_path, "presets.ini") + if options.preset and not os.path.exists(presets_file): + print(f"No such config file: {presets_file:s}") + print("") + return False + + projects_file = os.path.join(config_path, "projects.ini") + if not os.path.exists(projects_file): + print(f"No such config file: {projects_file:s}") + print("") + return False + + logging.basicConfig(level=logging.INFO, format="[%(levelname)s] %(message)s") + + dependency_updater = DependencyUpdater( + download_directory=options.download_directory, + download_only=options.download_only, + download_track=options.track, + exclude_packages=options.exclude_packages, + force_install=options.force_install, + preferred_machine_type=options.machine_type, + verbose_output=options.verbose, + ) + + user_defined_project_names = [] + if options.preset: + preset_definitions = {} + + with open(presets_file, "r", encoding="utf-8") as file_object: + definition_reader = presets.PresetDefinitionReader() + preset_definitions = { + preset_definition.name: preset_definition + for preset_definition in definition_reader.Read(file_object) + } + + user_defined_project_names = dependency_updater.ExpandPresets( + preset_definitions, options.preset + ) + if not user_defined_project_names: + print(f"Undefined preset: {options.preset:s}") + print("") + return False + + elif options.project_names: + user_defined_project_names = options.project_names + + result = dependency_updater.UpdatePackages( + projects_file, user_defined_project_names + ) return result - def ExpandPresets(self, preset_definitions, preset_names): - """Expands preset names to project names. - - Args: - preset_definitions (dict[str, PresetDefinition]): preset definitions. - preset_names (str): names of the presets to expand. - Returns: - set[str]: project names. - """ - project_names = set() - for preset_name in preset_names: - preset_definition = preset_definitions.get(preset_name, None) - if not preset_definition: - continue - - if preset_definition.preset_names: - sub_project_names = self.ExpandPresets( - preset_definitions, preset_definition.preset_names) - project_names = project_names.union(sub_project_names) - - project_names = project_names.union( - set(preset_definition.project_names)) - - return project_names - - def UpdatePackages(self, projects_file, user_defined_project_names): - """Updates packages. - - Args: - projects_file (str): path to the projects.ini configuration file. - user_defined_project_names (list[str]): user specified names of projects, - that should be updated if an update is available. An empty list - represents all available projects. - - Returns: - bool: True if the update was successful. - """ - project_definitions = self._GetProjectDefinitions(projects_file) - - user_defined_wheel_package_names = self._GetUserDefinedWheelPackageNames( - project_definitions, user_defined_project_names) - - available_packages = self._GetAvailableWheelPackages() - if not available_packages: - logging.error('No packages found.') - return False - - if not os.path.exists(self._download_directory): - os.mkdir(self._download_directory) - - package_filenames, package_versions = ( - self._GetWheelPackageFilenamesAndVersions( - project_definitions, available_packages, - user_defined_wheel_package_names)) - - if self._download_only: - return True - - return self._InstallWheelPackagesWindows( - package_filenames, package_versions) - - -def Main(): - """The main program function. - - Returns: - bool: True if successful or False if not. - """ - tracks = ['dev', 'stable', 'staging', 'testing'] - - argument_parser = argparse.ArgumentParser(description=( - 'Installs the latest versions of project dependencies.')) - - argument_parser.add_argument( - '-c', '--config', dest='config_path', action='store', - metavar='CONFIG_PATH', default=None, help=( - 'path of the directory containing the build configuration ' - 'files e.g. projects.ini.')) - - argument_parser.add_argument( - '--download-directory', '--download_directory', action='store', - metavar='DIRECTORY', dest='download_directory', type=str, - default='build', help='The location of the download directory.') - - argument_parser.add_argument( - '--download-only', '--download_only', action='store_true', - dest='download_only', default=False, help=( - 'Only download the dependencies. The default behavior is to ' - 'download and update the dependencies.')) - - argument_parser.add_argument( - '-e', '--exclude', action='store_true', dest='exclude_packages', - default=False, help=( - 'Excludes the package names instead of including them.')) - - argument_parser.add_argument( - '-f', '--force', action='store_true', dest='force_install', - default=False, help=( - 'Force installation. This option removes existing versions ' - 'of installed dependencies. The default behavior is to only' - 'install a dependency if not or an older version is installed.')) - - argument_parser.add_argument( - '--machine-type', '--machine_type', action='store', metavar='TYPE', - dest='machine_type', type=str, default=None, help=( - 'Manually sets the machine type instead of using the value returned ' - 'by platform.machine(). Usage of this argument is not recommended ' - 'unless want to force the installation of one machine type e.g. ' - '\'x86\' onto another \'amd64\'.')) - - argument_parser.add_argument( - '--preset', dest='preset', action='store', - metavar='PRESET_NAME', default=None, help=( - 'name of the preset of project names to update. The default is to ' - 'update all project defined in the projects.ini configuration file. ' - 'The presets are defined in the preset.ini configuration file.')) - - argument_parser.add_argument( - '-t', '--track', dest='track', action='store', metavar='TRACK', - default='stable', choices=sorted(tracks), help=( - 'the l2tbinaries track to download from. The default is stable.')) - - argument_parser.add_argument( - '-v', '--verbose', dest='verbose', action='store_true', default=False, - help='have more verbose output.') - - argument_parser.add_argument( - 'project_names', nargs='*', action='store', metavar='NAME', - type=str, help=( - 'Optional project names which should be updated if an update is ' - 'available. The corresponding package names are derived from ' - 'the projects.ini configuration file. If no value is provided ' - 'all available packages are updated.')) - - options = argument_parser.parse_args() - - config_path = options.config_path - if not config_path: - config_path = os.path.dirname(__file__) - config_path = os.path.dirname(config_path) - config_path = os.path.join(config_path, 'data') - - presets_file = os.path.join(config_path, 'presets.ini') - if options.preset and not os.path.exists(presets_file): - print(f'No such config file: {presets_file:s}') - print('') - return False - - projects_file = os.path.join(config_path, 'projects.ini') - if not os.path.exists(projects_file): - print(f'No such config file: {projects_file:s}') - print('') - return False - - logging.basicConfig( - level=logging.INFO, format='[%(levelname)s] %(message)s') - - dependency_updater = DependencyUpdater( - download_directory=options.download_directory, - download_only=options.download_only, - download_track=options.track, - exclude_packages=options.exclude_packages, - force_install=options.force_install, - preferred_machine_type=options.machine_type, - verbose_output=options.verbose) - - user_defined_project_names = [] - if options.preset: - preset_definitions = {} - - with open(presets_file, 'r', encoding='utf-8') as file_object: - definition_reader = presets.PresetDefinitionReader() - preset_definitions = { - preset_definition.name: preset_definition - for preset_definition in definition_reader.Read(file_object)} - - user_defined_project_names = dependency_updater.ExpandPresets( - preset_definitions, options.preset) - if not user_defined_project_names: - print(f'Undefined preset: {options.preset:s}') - print('') - return False - - elif options.project_names: - user_defined_project_names = options.project_names - - result = dependency_updater.UpdatePackages( - projects_file, user_defined_project_names) - - return result - - -if __name__ == '__main__': - if not Main(): - sys.exit(1) - else: - sys.exit(0) +if __name__ == "__main__": + if not Main(): + sys.exit(1) + else: + sys.exit(0) From dedfebdcca35fa34b9598d5da1314feb446bf81d Mon Sep 17 00:00:00 2001 From: Joachim Metz Date: Fri, 15 May 2026 19:24:00 +0200 Subject: [PATCH 4/7] Changes to use black Python formatter --- .github/workflows/lint.yml | 17 ----------------- run_tests.py | 1 - tests/build_helpers/rpm.py | 1 - tests/build_helpers/source.py | 1 - tests/dependency_writers/gift_copr.py | 3 +-- tests/dependency_writers/gift_ppa.py | 3 +-- tests/dependency_writers/linux_scripts.py | 3 +-- tests/download_helper.py | 1 - tools/build.py | 1 - tools/update.py | 2 +- tox.ini | 18 +----------------- 11 files changed, 5 insertions(+), 46 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 9558379b..d80d2d48 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -49,20 +49,3 @@ jobs: LANG: en_US.UTF-8 run: | tox -e pylint - yamllint: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ['3.12'] - steps: - - uses: actions/checkout@v6 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 - with: - python-version: ${{ matrix.python-version }} - - name: Install tox - run: | - python -m pip install tox - - name: Run linter - run: | - tox -e yamllint diff --git a/run_tests.py b/run_tests.py index 351d3fae..b40c806b 100755 --- a/run_tests.py +++ b/run_tests.py @@ -4,7 +4,6 @@ import unittest import sys - if __name__ == "__main__": test_suite = unittest.TestLoader().discover("tests", pattern="*.py") test_results = unittest.TextTestRunner(verbosity=2).run(test_suite) diff --git a/tests/build_helpers/rpm.py b/tests/build_helpers/rpm.py index f6af4bfb..274a69fb 100644 --- a/tests/build_helpers/rpm.py +++ b/tests/build_helpers/rpm.py @@ -3,7 +3,6 @@ import unittest - # TODO: add BaseRPMBuildHelper tests. # TODO: add RPMBuildHelper tests. # TODO: add ConfigureMakeRPMBuildHelper tests. diff --git a/tests/build_helpers/source.py b/tests/build_helpers/source.py index 10c5fb81..0cb4d7a9 100644 --- a/tests/build_helpers/source.py +++ b/tests/build_helpers/source.py @@ -3,7 +3,6 @@ import unittest - # TODO: add SourceBuildHelper tests. # TODO: add ConfigureMakeSourceBuildHelper tests. # TODO: add SetupPySourceBuildHelper tests. diff --git a/tests/dependency_writers/gift_copr.py b/tests/dependency_writers/gift_copr.py index 09efa990..3775d252 100644 --- a/tests/dependency_writers/gift_copr.py +++ b/tests/dependency_writers/gift_copr.py @@ -87,8 +87,7 @@ def testFormatRPMTestDependencies(self): test_writer = self._CreateTestWriter() expected_formatted_test_dependencies = ( - "TEST_DEPENDENCIES=\"python3-pbr\n" - " python3-setuptools\";" + 'TEST_DEPENDENCIES="python3-pbr\n' ' python3-setuptools";' ) python_dependencies = test_writer._GetRPMPythonDependencies() diff --git a/tests/dependency_writers/gift_ppa.py b/tests/dependency_writers/gift_ppa.py index d19ddc07..77205b1b 100644 --- a/tests/dependency_writers/gift_ppa.py +++ b/tests/dependency_writers/gift_ppa.py @@ -84,8 +84,7 @@ def testFormatDPKGTestDependencies(self): test_writer = self._CreateTestWriter() expected_formatted_test_dependencies = ( - "TEST_DEPENDENCIES=\"python3-pbr\n" - " python3-setuptools\";" + 'TEST_DEPENDENCIES="python3-pbr\n' ' python3-setuptools";' ) python_dependencies = test_writer._GetDPKGPythonDependencies() diff --git a/tests/dependency_writers/linux_scripts.py b/tests/dependency_writers/linux_scripts.py index d6a47dca..4d8a9641 100644 --- a/tests/dependency_writers/linux_scripts.py +++ b/tests/dependency_writers/linux_scripts.py @@ -84,8 +84,7 @@ def testFormatDPKGTestDependencies(self): test_writer = self._CreateTestWriter() expected_formatted_test_dependencies = ( - "TEST_DEPENDENCIES=\"python3-pbr\n" - " python3-setuptools\";" + 'TEST_DEPENDENCIES="python3-pbr\n' ' python3-setuptools";' ) python_dependencies = test_writer._GetDPKGPythonDependencies() diff --git a/tests/download_helper.py b/tests/download_helper.py index 0043421a..fe5f75ec 100644 --- a/tests/download_helper.py +++ b/tests/download_helper.py @@ -3,7 +3,6 @@ import unittest - # TODO: add tests for DownloadHelperFactory diff --git a/tools/build.py b/tools/build.py index 28f56d3a..8afe375f 100755 --- a/tools/build.py +++ b/tools/build.py @@ -15,7 +15,6 @@ from l2tdevtools import source_helper from l2tdevtools.build_helpers import factory as build_helper - # Since os.path.abspath() uses the current working directory (cwd) # os.path.abspath(__file__) will point to a different location if # cwd has been changed. Hence we preserve the absolute location of __file__. diff --git a/tools/update.py b/tools/update.py index 5a69f2ec..a7a22cd5 100755 --- a/tools/update.py +++ b/tools/update.py @@ -369,7 +369,7 @@ def _GetAvailableWheelPackages(self): # Ignore all other file extensions. continue - (package_name, package_version, python_tag1, python_tag2, _) = ( + package_name, package_version, python_tag1, python_tag2, _ = ( package_filename[:-4].split("-") ) diff --git a/tox.ini b/tox.ini index e36fc0b6..49217a05 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py3{10,11,12,13,14},black,coverage,pylint,wheel,yamllint +envlist = py3{10,11,12,13,14},black,coverage,pylint,wheel [testenv] allowlist_externals = ./run_tests.py @@ -54,19 +54,3 @@ deps = commands = pylint --version pylint --rcfile=.pylintrc l2tdevtools tests tools - -[testenv:yamllint] -skipsdist = True -pip_pre = True -passenv = - CFLAGS - CPPFLAGS - LDFLAGS -setenv = - PYTHONPATH = {toxinidir} -deps = - setuptools >= 65 - yamllint >= 1.26.0 -commands = - yamllint -v - yamllint -c .yamllint.yaml data From ac1b81093e5836c134279e5e2d1ebcf5be7f2ff6 Mon Sep 17 00:00:00 2001 From: Joachim Metz Date: Fri, 15 May 2026 19:27:17 +0200 Subject: [PATCH 5/7] Changes to use black Python formatter --- l2tdevtools/dependency_writers/github_actions.py | 2 +- l2tdevtools/dependency_writers/tox_ini.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/l2tdevtools/dependency_writers/github_actions.py b/l2tdevtools/dependency_writers/github_actions.py index 9ff19b38..5878ce33 100644 --- a/l2tdevtools/dependency_writers/github_actions.py +++ b/l2tdevtools/dependency_writers/github_actions.py @@ -78,7 +78,7 @@ def Write(self): template_data = self._GenerateFromTemplate("header.yml", template_mappings) file_content.append(template_data) - if paths_to_lint_yaml: + if paths_to_lint_yaml and self._project_definition.name != "l2tdevtools": template_data = self._GenerateFromTemplate( "yamllint.yml", template_mappings ) diff --git a/l2tdevtools/dependency_writers/tox_ini.py b/l2tdevtools/dependency_writers/tox_ini.py index f3eac98e..e1c60c3f 100644 --- a/l2tdevtools/dependency_writers/tox_ini.py +++ b/l2tdevtools/dependency_writers/tox_ini.py @@ -80,7 +80,7 @@ def Write(self): envlist.extend(["pylint", "wheel"]) - if paths_to_lint_yaml: + if paths_to_lint_yaml and self._project_definition.name != "l2tdevtools": envlist.append("yamllint") template_mappings = { @@ -108,7 +108,7 @@ def Write(self): template_data = self._GenerateFromTemplate("testenv_pylint", template_mappings) file_content.append(template_data) - if paths_to_lint_yaml: + if paths_to_lint_yaml and self._project_definition.name != "l2tdevtools": template_data = self._GenerateFromTemplate( "testenv_yamllint", template_mappings ) From 347948564dd493568a21e48597aae165dc65efa3 Mon Sep 17 00:00:00 2001 From: Joachim Metz Date: Fri, 15 May 2026 19:34:04 +0200 Subject: [PATCH 6/7] Changes to use black Python formatter --- tests/dependency_writers/gift_copr.py | 3 ++- tests/dependency_writers/gift_ppa.py | 3 ++- tests/dependency_writers/linux_scripts.py | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/dependency_writers/gift_copr.py b/tests/dependency_writers/gift_copr.py index 3775d252..09efa990 100644 --- a/tests/dependency_writers/gift_copr.py +++ b/tests/dependency_writers/gift_copr.py @@ -87,7 +87,8 @@ def testFormatRPMTestDependencies(self): test_writer = self._CreateTestWriter() expected_formatted_test_dependencies = ( - 'TEST_DEPENDENCIES="python3-pbr\n' ' python3-setuptools";' + "TEST_DEPENDENCIES=\"python3-pbr\n" + " python3-setuptools\";" ) python_dependencies = test_writer._GetRPMPythonDependencies() diff --git a/tests/dependency_writers/gift_ppa.py b/tests/dependency_writers/gift_ppa.py index 77205b1b..d19ddc07 100644 --- a/tests/dependency_writers/gift_ppa.py +++ b/tests/dependency_writers/gift_ppa.py @@ -84,7 +84,8 @@ def testFormatDPKGTestDependencies(self): test_writer = self._CreateTestWriter() expected_formatted_test_dependencies = ( - 'TEST_DEPENDENCIES="python3-pbr\n' ' python3-setuptools";' + "TEST_DEPENDENCIES=\"python3-pbr\n" + " python3-setuptools\";" ) python_dependencies = test_writer._GetDPKGPythonDependencies() diff --git a/tests/dependency_writers/linux_scripts.py b/tests/dependency_writers/linux_scripts.py index 4d8a9641..d6a47dca 100644 --- a/tests/dependency_writers/linux_scripts.py +++ b/tests/dependency_writers/linux_scripts.py @@ -84,7 +84,8 @@ def testFormatDPKGTestDependencies(self): test_writer = self._CreateTestWriter() expected_formatted_test_dependencies = ( - 'TEST_DEPENDENCIES="python3-pbr\n' ' python3-setuptools";' + "TEST_DEPENDENCIES=\"python3-pbr\n" + " python3-setuptools\";" ) python_dependencies = test_writer._GetDPKGPythonDependencies() From cdf2c624bd2dccb2d414f81d4c718ee94d379d73 Mon Sep 17 00:00:00 2001 From: Joachim Metz Date: Fri, 15 May 2026 19:38:22 +0200 Subject: [PATCH 7/7] Changes to use black Python formatter --- tests/dependency_writers/gift_copr.py | 8 +++++--- tests/dependency_writers/gift_ppa.py | 8 +++++--- tests/dependency_writers/linux_scripts.py | 8 +++++--- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/tests/dependency_writers/gift_copr.py b/tests/dependency_writers/gift_copr.py index 09efa990..57fdbc56 100644 --- a/tests/dependency_writers/gift_copr.py +++ b/tests/dependency_writers/gift_copr.py @@ -86,9 +86,11 @@ def testFormatRPMTestDependencies(self): """Tests the _FormatRPMTestDependencies function.""" test_writer = self._CreateTestWriter() - expected_formatted_test_dependencies = ( - "TEST_DEPENDENCIES=\"python3-pbr\n" - " python3-setuptools\";" + expected_formatted_test_dependencies = "".join( + [ + 'TEST_DEPENDENCIES="python3-pbr\n', + ' python3-setuptools";', + ] ) python_dependencies = test_writer._GetRPMPythonDependencies() diff --git a/tests/dependency_writers/gift_ppa.py b/tests/dependency_writers/gift_ppa.py index d19ddc07..749426ba 100644 --- a/tests/dependency_writers/gift_ppa.py +++ b/tests/dependency_writers/gift_ppa.py @@ -83,9 +83,11 @@ def testFormatDPKGTestDependencies(self): """Tests the _FormatDPKGTestDependencies function.""" test_writer = self._CreateTestWriter() - expected_formatted_test_dependencies = ( - "TEST_DEPENDENCIES=\"python3-pbr\n" - " python3-setuptools\";" + expected_formatted_test_dependencies = "".join( + [ + 'TEST_DEPENDENCIES="python3-pbr\n', + ' python3-setuptools";', + ] ) python_dependencies = test_writer._GetDPKGPythonDependencies() diff --git a/tests/dependency_writers/linux_scripts.py b/tests/dependency_writers/linux_scripts.py index d6a47dca..3567749c 100644 --- a/tests/dependency_writers/linux_scripts.py +++ b/tests/dependency_writers/linux_scripts.py @@ -83,9 +83,11 @@ def testFormatDPKGTestDependencies(self): """Tests the _FormatDPKGTestDependencies function.""" test_writer = self._CreateTestWriter() - expected_formatted_test_dependencies = ( - "TEST_DEPENDENCIES=\"python3-pbr\n" - " python3-setuptools\";" + expected_formatted_test_dependencies = "".join( + [ + 'TEST_DEPENDENCIES="python3-pbr\n', + ' python3-setuptools";', + ] ) python_dependencies = test_writer._GetDPKGPythonDependencies()