diff --git a/docs/changelog_templates.rst b/docs/changelog_templates.rst index 1ba78dba9..1200c0159 100644 --- a/docs/changelog_templates.rst +++ b/docs/changelog_templates.rst @@ -831,6 +831,31 @@ The filters provided vary based on the VCS configured and available features: {% set prev_changelog_contents = prev_changelog_file | read_file | safe %} +* ``sort_numerically (Callable[[Iterable[str], bool], list[str]])``: given a + sequence of strings with possibly some non-number characters as a prefix or suffix, + sort the strings as if they were just numbers from lowest to highest. This filter + is useful when you want to sort issue numbers or other strings that have a numeric + component in them but cannot be cast to a number directly to sort them. If you want + to sort the strings in reverse order, you can pass a boolean value of ``True`` as the + second argument. + + *Introduced in v9.16.0.* + + **Example Usage:** + + .. code:: jinja + + {{ ["#222", "#1023", "#444"] | sort_numerically }} + {{ ["#222", "#1023", "#444"] | sort_numerically(True) }} + + **Markdown Output:** + + .. code:: markdown + + ['#222', '#444', '#1023'] + ['#1023', '#444', '#222'] + + Availability of the documented filters can be found in the table below: ====================== ========= ===== ====== ====== @@ -846,6 +871,7 @@ issue_url ❌ ✅ ✅ ✅ merge_request_url ❌ ❌ ❌ ✅ pull_request_url ✅ ✅ ✅ ✅ read_file ✅ ✅ ✅ ✅ +sort_numerically ✅ ✅ ✅ ✅ ====================== ========= ===== ====== ====== .. seealso:: diff --git a/src/semantic_release/changelog/context.py b/src/semantic_release/changelog/context.py index 76f499163..9b8b102fe 100644 --- a/src/semantic_release/changelog/context.py +++ b/src/semantic_release/changelog/context.py @@ -8,6 +8,8 @@ from re import compile as regexp from typing import TYPE_CHECKING, Any, Callable, Literal +from semantic_release.helpers import sort_numerically + if TYPE_CHECKING: # pragma: no cover from jinja2 import Environment @@ -87,6 +89,7 @@ def make_changelog_context( read_file, convert_md_to_rst, autofit_text_width, + sort_numerically, ), ) diff --git a/src/semantic_release/cli/changelog_writer.py b/src/semantic_release/cli/changelog_writer.py index 2a9accab8..5c6ab9f55 100644 --- a/src/semantic_release/cli/changelog_writer.py +++ b/src/semantic_release/cli/changelog_writer.py @@ -24,6 +24,7 @@ ) from semantic_release.cli.util import noop_report from semantic_release.errors import InternalError +from semantic_release.helpers import sort_numerically if TYPE_CHECKING: # pragma: no cover from jinja2 import Environment @@ -254,7 +255,11 @@ def generate_release_notes( version=release["version"], release=release, mask_initial_release=mask_initial_release, - filters=(*hvcs_client.get_changelog_context_filters(), autofit_text_width), + filters=( + *hvcs_client.get_changelog_context_filters(), + autofit_text_width, + sort_numerically, + ), ).bind_to_environment( # Use a new, non-configurable environment for release notes - # not user-configurable at the moment diff --git a/src/semantic_release/commit_parser/angular.py b/src/semantic_release/commit_parser/angular.py index 533c235e2..c22d80f06 100644 --- a/src/semantic_release/commit_parser/angular.py +++ b/src/semantic_release/commit_parser/angular.py @@ -21,13 +21,10 @@ ParseError, ParseResult, ) -from semantic_release.commit_parser.util import ( - breaking_re, - parse_paragraphs, - sort_numerically, -) +from semantic_release.commit_parser.util import breaking_re, parse_paragraphs from semantic_release.enums import LevelBump from semantic_release.errors import InvalidParserOptions +from semantic_release.helpers import sort_numerically if TYPE_CHECKING: # pragma: no cover from git.objects.commit import Commit diff --git a/src/semantic_release/commit_parser/emoji.py b/src/semantic_release/commit_parser/emoji.py index 0cefdbeee..df8aeba38 100644 --- a/src/semantic_release/commit_parser/emoji.py +++ b/src/semantic_release/commit_parser/emoji.py @@ -18,9 +18,10 @@ ParsedMessageResult, ParseResult, ) -from semantic_release.commit_parser.util import parse_paragraphs, sort_numerically +from semantic_release.commit_parser.util import parse_paragraphs from semantic_release.enums import LevelBump from semantic_release.errors import InvalidParserOptions +from semantic_release.helpers import sort_numerically logger = logging.getLogger(__name__) diff --git a/src/semantic_release/commit_parser/util.py b/src/semantic_release/commit_parser/util.py index 996077d28..60ed63b2f 100644 --- a/src/semantic_release/commit_parser/util.py +++ b/src/semantic_release/commit_parser/util.py @@ -4,17 +4,20 @@ from re import MULTILINE, compile as regexp from typing import TYPE_CHECKING +# TODO: remove in v10 +from semantic_release.helpers import ( + sort_numerically, # noqa: F401 # TODO: maintained for compatibility +) + if TYPE_CHECKING: # pragma: no cover from re import Pattern - from typing import Sequence, TypedDict + from typing import TypedDict class RegexReplaceDef(TypedDict): pattern: Pattern repl: str -number_pattern = regexp(r"(\d+)") - breaking_re = regexp(r"BREAKING[ -]CHANGE:\s?(.*)") un_word_wrap: RegexReplaceDef = { @@ -71,7 +74,3 @@ def parse_paragraphs(text: str) -> list[str]: ], ) ) - - -def sort_numerically(iterable: Sequence[str] | set[str]) -> list[str]: - return sorted(iterable, key=lambda x: int((number_pattern.search(x) or [-1])[0])) diff --git a/src/semantic_release/helpers.py b/src/semantic_release/helpers.py index 83d700bfe..0840169ed 100644 --- a/src/semantic_release/helpers.py +++ b/src/semantic_release/helpers.py @@ -1,16 +1,87 @@ +from __future__ import annotations + import importlib.util import logging import os import re import string import sys -from functools import lru_cache, wraps +from functools import lru_cache, reduce, wraps from pathlib import Path, PurePosixPath -from typing import Any, Callable, NamedTuple, TypeVar +from re import IGNORECASE, compile as regexp +from typing import TYPE_CHECKING, Any, Callable, NamedTuple, TypeVar from urllib.parse import urlsplit +if TYPE_CHECKING: # pragma: no cover + from typing import Iterable + + log = logging.getLogger(__name__) +number_pattern = regexp(r"(?P\S*?)(?P\d[\d,]*)\b") +hex_number_pattern = regexp( + r"(?P\S*?)(?:0x)?(?P[0-9a-f]+)\b", IGNORECASE +) + + +def get_number_from_str( + string: str, default: int = -1, interpret_hex: bool = False +) -> int: + if interpret_hex and (match := hex_number_pattern.search(string)): + return abs(int(match.group("number"), 16)) + + if match := number_pattern.search(string): + return int(match.group("number")) + + return default + + +def sort_numerically( + iterable: Iterable[str], reverse: bool = False, allow_hex: bool = False +) -> list[str]: + # Alphabetically sort prefixes first, then sort by number + alphabetized_list = sorted(iterable) + + # Extract prefixes in order to group items by prefix + unmatched_items = [] + prefixes: dict[str, list[str]] = {} + for item in alphabetized_list: + if not ( + pattern_match := ( + (hex_number_pattern.search(item) if allow_hex else None) + or number_pattern.search(item) + ) + ): + unmatched_items.append(item) + continue + + prefix = prefix if (prefix := pattern_match.group("prefix")) else "" + + if prefix not in prefixes: + prefixes[prefix] = [] + + prefixes[prefix].append(item) + + # Sort prefixes and items by number mixing in unmatched items as alphabetized with other prefixes + return reduce( + lambda acc, next_item: acc + next_item, + [ + ( + sorted( + prefixes[prefix], + key=lambda x: get_number_from_str( + x, default=-1, interpret_hex=allow_hex + ), + reverse=reverse, + ) + if prefix in prefixes + else [prefix] + ) + for prefix in sorted([*prefixes.keys(), *unmatched_items]) + ], + [], + ) + def format_arg(value: Any) -> str: """Helper to format an argument an argument for logging""" diff --git a/tests/unit/semantic_release/changelog/test_changelog_context.py b/tests/unit/semantic_release/changelog/test_changelog_context.py index 7dedd0015..c80344fa1 100644 --- a/tests/unit/semantic_release/changelog/test_changelog_context.py +++ b/tests/unit/semantic_release/changelog/test_changelog_context.py @@ -497,3 +497,91 @@ def test_changelog_context_autofit_text_width_w_indent( # Evaluate assert expected_changelog == actual_changelog + + +def test_changelog_context_sort_numerically( + example_git_https_url: str, + artificial_release_history: ReleaseHistory, + changelog_md_file: Path, +): + changelog_tpl = dedent( + """\ + {{ [ + ".. _#5: link", + ".. _PR#3: link", + ".. _PR#10: link", + ".. _#100: link" + ] | sort_numerically | join("\\n") + }} + """ + ) + + expected_changelog = dedent( + """\ + .. _#5: link + .. _#100: link + .. _PR#3: link + .. _PR#10: link + """ + ) + + env = environment(trim_blocks=True, lstrip_blocks=True, keep_trailing_newline=True) + context = make_changelog_context( + hvcs_client=Gitlab(example_git_https_url), + release_history=artificial_release_history, + mode=ChangelogMode.UPDATE, + prev_changelog_file=changelog_md_file, + insertion_flag="", + mask_initial_release=False, + ) + context.bind_to_environment(env) + + # Create changelog from template with environment + actual_changelog = env.from_string(changelog_tpl).render() + + # Evaluate + assert expected_changelog == actual_changelog + + +def test_changelog_context_sort_numerically_reverse( + example_git_https_url: str, + artificial_release_history: ReleaseHistory, + changelog_md_file: Path, +): + changelog_tpl = dedent( + """\ + {{ [ + ".. _#5: link", + ".. _PR#3: link", + ".. _PR#10: link", + ".. _#100: link" + ] | sort_numerically(reverse=True) | join("\\n") + }} + """ + ) + + expected_changelog = dedent( + """\ + .. _#100: link + .. _#5: link + .. _PR#10: link + .. _PR#3: link + """ + ) + + env = environment(trim_blocks=True, lstrip_blocks=True, keep_trailing_newline=True) + context = make_changelog_context( + hvcs_client=Gitlab(example_git_https_url), + release_history=artificial_release_history, + mode=ChangelogMode.UPDATE, + prev_changelog_file=changelog_md_file, + insertion_flag="", + mask_initial_release=False, + ) + context.bind_to_environment(env) + + # Create changelog from template with environment + actual_changelog = env.from_string(changelog_tpl).render() + + # Evaluate + assert expected_changelog == actual_changelog diff --git a/tests/unit/semantic_release/changelog/test_release_notes.py b/tests/unit/semantic_release/changelog/test_release_notes.py index 6021ee7bb..7158dd670 100644 --- a/tests/unit/semantic_release/changelog/test_release_notes.py +++ b/tests/unit/semantic_release/changelog/test_release_notes.py @@ -2,6 +2,7 @@ import os from pathlib import Path +from textwrap import dedent from typing import TYPE_CHECKING import pytest @@ -17,6 +18,8 @@ if TYPE_CHECKING: from semantic_release.changelog.release_history import ReleaseHistory + from tests.fixtures.example_project import ExProjectDir + @pytest.fixture(scope="module") def release_notes_template() -> str: @@ -450,3 +453,97 @@ def test_default_release_notes_template_first_release_unmasked( ) assert expected_content == actual_content + + +def test_release_notes_context_sort_numerically_filter( + example_git_https_url: str, + single_release_history: ReleaseHistory, + example_project_dir: ExProjectDir, + change_to_ex_proj_dir: None, +): + version = list(single_release_history.released.keys())[-1] + release = single_release_history.released[version] + + example_project_dir.joinpath(".release_notes.md.j2").write_text( + dedent( + """\ + {{ [ + ".. _#5: link", + ".. _PR#3: link", + ".. _PR#10: link", + ".. _#100: link" + ] | sort_numerically | join("\\n") + }} + """ + ) + ) + + expected_content = str.join( + os.linesep, + dedent( + """\ + .. _#5: link + .. _#100: link + .. _PR#3: link + .. _PR#10: link + """ + ).split("\n"), + ) + + actual_content = generate_release_notes( + hvcs_client=Github(remote_url=example_git_https_url), + release=release, + template_dir=example_project_dir, + history=single_release_history, + style="angular", + mask_initial_release=False, + ) + + assert expected_content == actual_content + + +def test_release_notes_context_sort_numerically_filter_reversed( + example_git_https_url: str, + single_release_history: ReleaseHistory, + example_project_dir: ExProjectDir, + change_to_ex_proj_dir: None, +): + version = list(single_release_history.released.keys())[-1] + release = single_release_history.released[version] + + example_project_dir.joinpath(".release_notes.md.j2").write_text( + dedent( + """\ + {{ [ + ".. _#5: link", + ".. _PR#3: link", + ".. _PR#10: link", + ".. _#100: link" + ] | sort_numerically(reverse=True) | join("\\n") + }} + """ + ) + ) + + expected_content = str.join( + os.linesep, + dedent( + """\ + .. _#100: link + .. _#5: link + .. _PR#10: link + .. _PR#3: link + """ + ).split("\n"), + ) + + actual_content = generate_release_notes( + hvcs_client=Github(remote_url=example_git_https_url), + release=release, + template_dir=example_project_dir, + history=single_release_history, + style="angular", + mask_initial_release=False, + ) + + assert expected_content == actual_content diff --git a/tests/unit/semantic_release/test_helpers.py b/tests/unit/semantic_release/test_helpers.py index e7db7adfd..4877d3892 100644 --- a/tests/unit/semantic_release/test_helpers.py +++ b/tests/unit/semantic_release/test_helpers.py @@ -1,6 +1,8 @@ +from typing import Iterable + import pytest -from semantic_release.helpers import ParsedGitUrl, parse_git_url +from semantic_release.helpers import ParsedGitUrl, parse_git_url, sort_numerically @pytest.mark.parametrize( @@ -131,3 +133,165 @@ def test_parse_invalid_git_urls(url: str): """Test that an invalid git remote url throws a ValueError.""" with pytest.raises(ValueError): parse_git_url(url) + + +@pytest.mark.parametrize( + "unsorted_list, sorted_list, reverse, allow_hex", + [ + pytest.param( + unsorted_list, + sorted_list, + reverse, + allow_hex, + id=f"({i}) {test_id}", + ) + for i, (test_id, unsorted_list, sorted_list, reverse, allow_hex) in enumerate( + [ + ( + "Only numbers (with mixed digits, ASC)", + ["5", "3", "10"], + ["3", "5", "10"], + False, + False, + ), + ( + "Only numbers (with mixed digits, DESC)", + ["5", "3", "10"], + ["10", "5", "3"], + True, + False, + ), + ( + "Only PR numbers (ASC)", + ["#5", "#3", "#10"], + ["#3", "#5", "#10"], + False, + False, + ), + ( + "Only PR numbers (DESC)", + ["#5", "#3", "#10"], + ["#10", "#5", "#3"], + True, + False, + ), + ( + "Multiple prefixes (ASC)", + ["#5", "PR#3", "PR#10", "#100"], + ["#5", "#100", "PR#3", "PR#10"], + False, + False, + ), + ( + "Multiple prefixes (DESC)", + ["#5", "PR#3", "PR#10", "#100"], + ["#100", "#5", "PR#10", "PR#3"], + True, + False, + ), + ( + "No numbers mixed with mulitple prefixes (ASC)", + ["word", "#100", "#1000", "PR#45"], + ["#100", "#1000", "PR#45", "word"], + False, + False, + ), + ( + "No numbers mixed with mulitple prefixes (DESC)", + ["word", "#100", "#1000", "PR#45"], + ["#1000", "#100", "PR#45", "word"], + True, + False, + ), + ( + "Commit hash links in RST link format (ASC)", + [".. _8ab43ed:", ".. _7ffed34:", ".. _a3b4c54:"], + [".. _7ffed34:", ".. _8ab43ed:", ".. _a3b4c54:"], + False, + True, + ), + ( + "Commit hash links in RST link format (DESC)", + [".. _8ab43ed:", ".. _7ffed34:", ".. _a3b4c54:"], + [".. _a3b4c54:", ".. _8ab43ed:", ".. _7ffed34:"], + True, + True, + ), + ( + "Mixed numbers, PR numbers, and commit hash links in RST link format (ASC)", + [ + ".. _#5:", + ".. _8ab43ed:", + ".. _PR#3:", + ".. _#20:", + ".. _7ffed34:", + ".. _#100:", + ".. _a3b4c54:", + ], + [ + ".. _7ffed34:", + ".. _8ab43ed:", + ".. _a3b4c54:", + ".. _#5:", + ".. _#20:", + ".. _#100:", + ".. _PR#3:", + ], + False, + True, + ), + ( + "Mixed numbers, PR numbers, and commit hash links in RST link format (DESC)", + [ + ".. _#5:", + ".. _8ab43ed:", + ".. _PR#3:", + ".. _#20:", + ".. _7ffed34:", + ".. _#100:", + ".. _a3b4c54:", + ], + [ + ".. _a3b4c54:", + ".. _8ab43ed:", + ".. _7ffed34:", + ".. _#100:", + ".. _#20:", + ".. _#5:", + ".. _PR#3:", + ], + True, + True, + ), + ( + # No change since the prefixes are always alphabetical, asc/desc only is b/w numbers + "Same numbers with different prefixes (ASC)", + ["PR#5", "#5"], + ["#5", "PR#5"], + False, + False, + ), + ( + "Same numbers with different prefixes (DESC)", + ["#5", "PR#5"], + ["#5", "PR#5"], + True, + False, + ), + ], + start=1, + ) + ], +) +def test_sort_numerically( + unsorted_list: Iterable[str], + sorted_list: Iterable[str], + reverse: bool, + allow_hex: bool, +): + actual_list = sort_numerically( + iterable=unsorted_list, + reverse=reverse, + allow_hex=allow_hex, + ) + assert sorted_list == actual_list