From 89ece7e902191b507013753d41895c63ea29ec74 Mon Sep 17 00:00:00 2001 From: codejedi365 Date: Sat, 11 Jan 2025 20:18:34 -0700 Subject: [PATCH 1/5] test(fixtures): remove import checking/loading of custom parser in `use_custom_parser` --- tests/fixtures/example_project.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/tests/fixtures/example_project.py b/tests/fixtures/example_project.py index 83d0b6171..2d9783cdf 100644 --- a/tests/fixtures/example_project.py +++ b/tests/fixtures/example_project.py @@ -1,7 +1,6 @@ from __future__ import annotations import os -from importlib import import_module from pathlib import Path from textwrap import dedent from typing import TYPE_CHECKING, Generator @@ -60,7 +59,7 @@ class UpdatePyprojectTomlFn(Protocol): def __call__(self, setting: str, value: Any) -> None: ... class UseCustomParserFn(Protocol): - def __call__(self, module_import_str: str) -> type[CommitParser]: ... + def __call__(self, module_import_str: str) -> None: ... class UseHvcsFn(Protocol): def __call__(self, domain: str | None = None) -> type[HvcsBase]: ... @@ -497,17 +496,8 @@ def use_custom_parser( ) -> UseCustomParserFn: """Modify the configuration file to use a user defined string parser.""" - def _use_custom_parser(module_import_str: str) -> type[CommitParser]: - # validate this is importable before writing to parser - module_name, attr = module_import_str.split(":", maxsplit=1) - try: - module = import_module(module_name) - custom_class = getattr(module, attr) - except (ModuleNotFoundError, AttributeError) as err: - raise ValueError("Custom parser object not found!") from err - + def _use_custom_parser(module_import_str: str) -> None: update_pyproject_toml(pyproject_toml_config_option_parser, module_import_str) - return custom_class return _use_custom_parser From fb5163fafdc75994d3df50bbcd3ee8da0746c2a6 Mon Sep 17 00:00:00 2001 From: codejedi365 Date: Sat, 11 Jan 2025 20:19:47 -0700 Subject: [PATCH 2/5] test(config): extend import parser unit tests to evaluate file paths to modules --- .../unit/semantic_release/cli/test_config.py | 39 ++++++++++++++++--- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/tests/unit/semantic_release/cli/test_config.py b/tests/unit/semantic_release/cli/test_config.py index d76456e7f..d083d01d1 100644 --- a/tests/unit/semantic_release/cli/test_config.py +++ b/tests/unit/semantic_release/cli/test_config.py @@ -1,6 +1,9 @@ from __future__ import annotations import os +import shutil +import sys +from pathlib import Path from re import compile as regexp from typing import TYPE_CHECKING from unittest import mock @@ -37,7 +40,6 @@ ) if TYPE_CHECKING: - from pathlib import Path from typing import Any from tests.fixtures.example_project import ExProjectDir, UpdatePyprojectTomlFn @@ -226,8 +228,12 @@ def test_load_valid_runtime_config( @pytest.mark.parametrize( "commit_parser", [ + # Module:Class string f"{CustomParserWithNoOpts.__module__}:{CustomParserWithNoOpts.__name__}", f"{CustomParserWithOpts.__module__}:{CustomParserWithOpts.__name__}", + # File path module:Class string + f"{CustomParserWithNoOpts.__module__.replace('.', '/')}.py:{CustomParserWithNoOpts.__name__}", + f"{CustomParserWithOpts.__module__.replace('.', '/')}.py:{CustomParserWithOpts.__name__}", ], ) def test_load_valid_runtime_config_w_custom_parser( @@ -236,17 +242,32 @@ def test_load_valid_runtime_config_w_custom_parser( example_project_dir: ExProjectDir, example_pyproject_toml: Path, change_to_ex_proj_dir: None, + request: pytest.FixtureRequest, ): + fake_sys_modules = {**sys.modules} + + if ".py" in commit_parser: + module_filepath = Path(commit_parser.split(":")[0]) + module_filepath.parent.mkdir(parents=True, exist_ok=True) + module_filepath.parent.joinpath("__init__.py").touch() + shutil.copy( + src=str(request.config.rootpath / module_filepath), + dst=str(module_filepath), + ) + fake_sys_modules.pop( + str(Path(module_filepath).with_suffix("")).replace(os.sep, ".") + ) + build_configured_base_repo( example_project_dir, commit_type=commit_parser, ) - runtime_ctx = RuntimeContext.from_raw_config( - RawConfig.model_validate(load_raw_config_file(example_pyproject_toml)), - global_cli_options=GlobalCommandLineOptions(), - ) - assert runtime_ctx + with mock.patch.dict(sys.modules, fake_sys_modules, clear=True): + assert RuntimeContext.from_raw_config( + RawConfig.model_validate(load_raw_config_file(example_pyproject_toml)), + global_cli_options=GlobalCommandLineOptions(), + ) @pytest.mark.parametrize( @@ -258,6 +279,12 @@ def test_load_valid_runtime_config_w_custom_parser( f"{CustomParserWithOpts.__module__}:MissingCustomParser", # Incomplete class implementation f"{IncompleteCustomParser.__module__}:{IncompleteCustomParser.__name__}", + # Non-existant module file + "tests/missing_module.py:CustomParser", + # Non-existant class in module file + f"{CustomParserWithOpts.__module__.replace('.', '/')}.py:MissingCustomParser", + # Incomplete class implementation in module file + f"{IncompleteCustomParser.__module__.replace('.', '/')}.py:{IncompleteCustomParser.__name__}", ], ) def test_load_invalid_custom_parser( From e322ddcff9db5e2bb52c13ec9f1231da8842560e Mon Sep 17 00:00:00 2001 From: codejedi365 Date: Thu, 9 Jan 2025 14:11:12 -0700 Subject: [PATCH 3/5] feat(config): expand dynamic parser import to handle a filepath to module --- src/semantic_release/helpers.py | 45 ++++++++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/src/semantic_release/helpers.py b/src/semantic_release/helpers.py index 3725ddfa5..db82a97eb 100644 --- a/src/semantic_release/helpers.py +++ b/src/semantic_release/helpers.py @@ -1,9 +1,11 @@ -import importlib +import importlib.util import logging +import os import re import string +import sys from functools import lru_cache, wraps -from pathlib import PurePosixPath +from pathlib import Path, PurePosixPath from typing import Any, Callable, NamedTuple, TypeVar from urllib.parse import urlsplit @@ -69,8 +71,43 @@ def dynamic_import(import_path: str) -> Any: """ log.debug("Trying to import %s", import_path) module_name, attr = import_path.split(":", maxsplit=1) - module = importlib.import_module(module_name) - return getattr(module, attr) + + # Check if the module is a file path, if it can be resolved and exists on disk then import as a file + module_filepath = Path(module_name).resolve() + if module_filepath.exists(): + module_path = ( + module_filepath.stem + if Path(module_name).is_absolute() + else str(Path(module_name).with_suffix("")).replace(os.sep, ".") + ) + + if module_path not in sys.modules: + spec = importlib.util.spec_from_file_location( + module_path, str(module_filepath) + ) + if spec is None: + raise ImportError(f"Could not import {module_filepath}") + + module = importlib.util.module_from_spec(spec) # type: ignore[arg-type] + sys.modules.update({spec.name: module}) + spec.loader.exec_module(module) # type: ignore[union-attr] + + return getattr(sys.modules[module_path], attr) + + # Otherwise, import as a module + try: + module = importlib.import_module(module_name) + return getattr(module, attr) + except TypeError as err: + raise ImportError( + str.join( + "\n", + [ + str(err.args[0]), + "Verify the import format matches 'module:attribute' or 'path/to/module:attribute'", + ], + ) + ) from err class ParsedGitUrl(NamedTuple): From b9ad8b8624aef86f171ee668d861a588537c6e29 Mon Sep 17 00:00:00 2001 From: codejedi365 Date: Thu, 9 Jan 2025 22:35:54 -0700 Subject: [PATCH 4/5] docs(commit-parsing): add the new custom parser import spec description for direct path imports Resolves: #687 --- docs/commit_parsing.rst | 47 +++++++++++++++++++++++++++++------------ 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/docs/commit_parsing.rst b/docs/commit_parsing.rst index 7c804e695..fe1d3f376 100644 --- a/docs/commit_parsing.rst +++ b/docs/commit_parsing.rst @@ -334,19 +334,40 @@ where appropriate to assist with static type-checking. The :ref:`commit_parser ` option, if set to a string which does not match one of Python Semantic Release's built-in commit parsers, will be -used to attempt to dynamically import a custom commit parser class. As such you will -need to ensure that your custom commit parser is import-able from the environment in -which you are running Python Semantic Release. The string should be structured in the -standard ``module:attr`` format; for example, to import the class ``MyCommitParser`` -from the file ``custom_parser.py`` at the root of your repository, you should specify -``"commit_parser=custom_parser:MyCommitParser"`` in your configuration, and run the -``semantic-release`` command line interface from the root of your repository. Equally -you can ensure that the module containing your parser class is installed in the same -virtual environment as semantic-release. If you can run -``python -c "from $MODULE import $CLASS"`` successfully, specifying -``commit_parser="$MODULE:$CLASS"`` is sufficient. You may need to set the -``PYTHONPATH`` environment variable to the directory containing the module with -your commit parser. +used to attempt to dynamically import a custom commit parser class. + +In order to use your custom parser, you must provide how to import the module and class +via the configuration option. There are two ways to provide the import string: + +1. **File Path & Class**: The format is ``"path/to/module_file.py:ClassName"``. This + is the easiest way to provide a custom parser. This method allows you to store your + custom parser directly in the repository with no additional installation steps. PSR + will locate the file, load the module, and instantiate the class. Relative paths are + recommended and it should be provided relative to the current working directory. This + import variant is available in v9.16.0 and later. + +2. **Module Path & Class**: The format is ``"package.module_name:ClassName"``. This + method allows you to store your custom parser in a package that is installed in the + same environment as PSR. This method is useful if you want to share your custom parser + across multiple repositories. To share it across multiple repositories generally you will + need to publish the parser as its own separate package and then ``pip install`` it into + the current virtual environment. You can also keep it in the same repository as your + project as long as it is in the current directory of the virtual environment and is + locatable by the Python import system. You may need to set the ``PYTHONPATH`` environment + variable if you have a more complex directory structure. This import variant is available + in v8.0.0 and later. + + To test that your custom parser is importable, you can run the following command in the + directory where PSR will be executed: + + .. code-block:: bash + + python -c "from package.module_name import ClassName" + + .. note:: + Remember this is basic python import rules so the package name is optional and generally + packages are defined by a directory with ``__init__.py`` files. + .. _commit_parser-tokens: From b89229cce6d6d6be2d7038f561dd101042104af4 Mon Sep 17 00:00:00 2001 From: codejedi365 Date: Thu, 9 Jan 2025 22:37:05 -0700 Subject: [PATCH 5/5] docs(configuration): adjust `commit_parser` option definition for direct path imports --- docs/configuration.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index a3a587045..fac883479 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -796,7 +796,7 @@ Built-in parsers: * ``tag`` - :ref:`TagCommitParser ` *(deprecated in v9.12.0)* You can set any of the built-in parsers by their keyword but you can also specify -your own commit parser in ``module:attr`` form. +your own commit parser in ``path/to/module_file.py:Class`` or ``module:Class`` form. For more information see :ref:`commit-parsing`.