From 67c5cf70b79207df2b54c2ff4706744f12da30a1 Mon Sep 17 00:00:00 2001 From: Tomas Roun Date: Mon, 24 Feb 2025 15:22:06 +0100 Subject: [PATCH 01/12] Allow passing multiple keywords with the same funcname. --- .../i18n_data/multiple_keywords.pot | 38 ++++++++++ .../test_tools/i18n_data/multiple_keywords.py | 7 ++ Lib/test/test_tools/test_i18n.py | 53 ++++++++++--- Tools/i18n/pygettext.py | 74 ++++++++++++------- 4 files changed, 135 insertions(+), 37 deletions(-) create mode 100644 Lib/test/test_tools/i18n_data/multiple_keywords.pot create mode 100644 Lib/test/test_tools/i18n_data/multiple_keywords.py diff --git a/Lib/test/test_tools/i18n_data/multiple_keywords.pot b/Lib/test/test_tools/i18n_data/multiple_keywords.pot new file mode 100644 index 00000000000000..35d5c6ef5ce4ec --- /dev/null +++ b/Lib/test/test_tools/i18n_data/multiple_keywords.pot @@ -0,0 +1,38 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR ORGANIZATION +# FIRST AUTHOR , YEAR. +# +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"POT-Creation-Date: 2000-01-01 00:00+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: pygettext.py 1.5\n" + + +#: multiple_keywords.py:3 +msgid "bar" +msgstr "" + +#: multiple_keywords.py:4 +msgctxt "baz" +msgid "qux" +msgstr "" + +#: multiple_keywords.py:6 +msgctxt "corge" +msgid "grault" +msgstr "" + +#: multiple_keywords.py:7 +msgctxt "xyzzy" +msgid "foo" +msgid_plural "foos" +msgstr[0] "" +msgstr[1] "" + diff --git a/Lib/test/test_tools/i18n_data/multiple_keywords.py b/Lib/test/test_tools/i18n_data/multiple_keywords.py new file mode 100644 index 00000000000000..fac1ac3a278cc8 --- /dev/null +++ b/Lib/test/test_tools/i18n_data/multiple_keywords.py @@ -0,0 +1,7 @@ +from gettext import gettext as foo + +foo('bar') +foo('baz', 'qux') +# The 't' specifier is not supported, so this is extracted as pgettext +foo('corge', 'grault', 1) +foo('xyzzy', 'foo', 'foos', 1) diff --git a/Lib/test/test_tools/test_i18n.py b/Lib/test/test_tools/test_i18n.py index 2ba086109d30ab..5c6ea848b4e9c0 100644 --- a/Lib/test/test_tools/test_i18n.py +++ b/Lib/test/test_tools/test_i18n.py @@ -18,7 +18,7 @@ with imports_under_tool("i18n"): - from pygettext import parse_spec + from pygettext import parse_spec, process_keywords, DEFAULTKEYWORDS def normalize_POT_file(pot): @@ -483,16 +483,16 @@ def test_comments_not_extracted_without_tags(self): def test_parse_keyword_spec(self): valid = ( - ('foo', ('foo', {0: 'msgid'})), - ('foo:1', ('foo', {0: 'msgid'})), - ('foo:1,2', ('foo', {0: 'msgid', 1: 'msgid_plural'})), - ('foo:1, 2', ('foo', {0: 'msgid', 1: 'msgid_plural'})), - ('foo:1,2c', ('foo', {0: 'msgid', 1: 'msgctxt'})), - ('foo:2c,1', ('foo', {0: 'msgid', 1: 'msgctxt'})), - ('foo:2c ,1', ('foo', {0: 'msgid', 1: 'msgctxt'})), - ('foo:1,2,3c', ('foo', {0: 'msgid', 1: 'msgid_plural', 2: 'msgctxt'})), - ('foo:1, 2, 3c', ('foo', {0: 'msgid', 1: 'msgid_plural', 2: 'msgctxt'})), - ('foo:3c,1,2', ('foo', {0: 'msgid', 1: 'msgid_plural', 2: 'msgctxt'})), + ('foo', ('foo', {'msgid': 0})), + ('foo:1', ('foo', {'msgid': 0})), + ('foo:1,2', ('foo', {'msgid': 0, 'msgid_plural': 1})), + ('foo:1, 2', ('foo', {'msgid': 0, 'msgid_plural': 1})), + ('foo:1,2c', ('foo', {'msgid': 0, 'msgctxt': 1})), + ('foo:2c,1', ('foo', {'msgid': 0, 'msgctxt': 1})), + ('foo:2c ,1', ('foo', {'msgid': 0, 'msgctxt': 1})), + ('foo:1,2,3c', ('foo', {'msgid': 0, 'msgid_plural': 1, 'msgctxt': 2})), + ('foo:1, 2, 3c', ('foo', {'msgid': 0, 'msgid_plural': 1, 'msgctxt': 2})), + ('foo:3c,1,2', ('foo', {'msgid': 0, 'msgid_plural': 1, 'msgctxt': 2})), ) for spec, expected in valid: with self.subTest(spec=spec): @@ -516,6 +516,33 @@ def test_parse_keyword_spec(self): parse_spec(spec) self.assertEqual(str(cm.exception), message) + def test_process_keywords(self): + default_keywords = {name: [spec] for name, spec + in DEFAULTKEYWORDS.items()} + inputs = ( + (['foo'], True), + (['_:1,2'], True), + (['foo', 'foo:1,2'], True), + (['foo'], False), + (['_:1,2', '_:1c,2,3', 'pgettext'], False), + ) + expected = ( + {'foo': [{'msgid': 0}]}, + {'_': [{'msgid': 0, 'msgid_plural': 1}]}, + {'foo': [{'msgid': 0}, {'msgid': 0, 'msgid_plural': 1}]}, + default_keywords | {'foo': [{'msgid': 0}]}, + default_keywords | {'_': [{'msgid': 0, 'msgid_plural': 1}, + {'msgctxt': 0, 'msgid': 1, 'msgid_plural': 2}], + 'pgettext': [{'msgid': 0}]}, + ) + for (keywords, no_default_keywords), expected in zip(inputs, expected): + with self.subTest(keywords=keywords, + no_default_keywords=no_default_keywords): + processed = process_keywords( + keywords, + no_default_keywords=no_default_keywords) + self.assertEqual(processed, expected) + def extract_from_snapshots(): snapshots = { @@ -526,6 +553,10 @@ def extract_from_snapshots(): 'custom_keywords.py': ('--keyword=foo', '--keyword=nfoo:1,2', '--keyword=pfoo:1c,2', '--keyword=npfoo:1c,2,3', '--keyword=_:1,2'), + 'multiple_keywords.py': ('--keyword=foo:1c,2,3', '--keyword=foo:1c,2', + '--keyword=foo:1,2', + # repeat a keyword to make sure it is extracted only once + '--keyword=foo', '--keyword=foo'), } for filename, args in snapshots.items(): diff --git a/Tools/i18n/pygettext.py b/Tools/i18n/pygettext.py index 0ec257020918b1..65a5df1f5ac90d 100755 --- a/Tools/i18n/pygettext.py +++ b/Tools/i18n/pygettext.py @@ -282,15 +282,15 @@ def getFilesForName(name): # Key is the function name, value is a dictionary mapping argument positions to the # type of the argument. The type is one of 'msgid', 'msgid_plural', or 'msgctxt'. DEFAULTKEYWORDS = { - '_': {0: 'msgid'}, - 'gettext': {0: 'msgid'}, - 'ngettext': {0: 'msgid', 1: 'msgid_plural'}, - 'pgettext': {0: 'msgctxt', 1: 'msgid'}, - 'npgettext': {0: 'msgctxt', 1: 'msgid', 2: 'msgid_plural'}, - 'dgettext': {1: 'msgid'}, - 'dngettext': {1: 'msgid', 2: 'msgid_plural'}, - 'dpgettext': {1: 'msgctxt', 2: 'msgid'}, - 'dnpgettext': {1: 'msgctxt', 2: 'msgid', 3: 'msgid_plural'}, + '_': {'msgid': 0}, + 'gettext': {'msgid': 0}, + 'ngettext': {'msgid': 0, 'msgid_plural': 1}, + 'pgettext': {'msgctxt': 0, 'msgid': 1}, + 'npgettext': {'msgctxt': 0, 'msgid': 1, 'msgid_plural': 2}, + 'dgettext': {'msgid': 1}, + 'dngettext': {'msgid': 1, 'msgid_plural': 2}, + 'dpgettext': {'msgctxt': 1, 'msgid': 2}, + 'dnpgettext': {'msgctxt': 1, 'msgid': 2, 'msgid_plural': 3}, } @@ -327,7 +327,7 @@ def parse_spec(spec): parts = spec.strip().split(':', 1) if len(parts) == 1: name = parts[0] - return name, {0: 'msgid'} + return name, {'msgid': 0} name, args = parts if not args: @@ -373,7 +373,23 @@ def parse_spec(spec): raise ValueError(f'Invalid keyword spec {spec!r}: ' 'msgctxt cannot appear without msgid') - return name, {v: k for k, v in result.items()} + return name, result + + +def process_keywords(keywords, *, no_default_keywords): + custom_keywords_list = [parse_spec(spec) for spec in keywords] + custom_keywords = {} + for name, spec in custom_keywords_list: + if name not in custom_keywords: + custom_keywords[name] = [] + custom_keywords[name].append(spec) + + if no_default_keywords: + return custom_keywords + + default_keywords = {name: [spec] for name, spec in DEFAULTKEYWORDS.items()} + # custom keywords override default keywords + return default_keywords | custom_keywords @dataclass(frozen=True) @@ -459,37 +475,46 @@ def _extract_docstring(self, node): def _extract_message(self, node): func_name = self._get_func_name(node) - spec = self.options.keywords.get(func_name) - if spec is None: - return + specs = self.options.keywords.get(func_name, []) + for spec in specs: + extracted = self._extract_message_with_spec(node, spec) + if extracted: + break + + def _extract_message_with_spec(self, node, spec): + """Extract a gettext call with the given spec. - max_index = max(spec) + Return True if the gettext call was successfully extracted, False + otherwise. + """ + max_index = max(spec.values()) has_var_positional = any(isinstance(arg, ast.Starred) for arg in node.args[:max_index+1]) if has_var_positional: print(f'*** {self.filename}:{node.lineno}: Variable positional ' f'arguments are not allowed in gettext calls', file=sys.stderr) - return + return False if max_index >= len(node.args): print(f'*** {self.filename}:{node.lineno}: Expected at least ' - f'{max(spec) + 1} positional argument(s) in gettext call, ' + f'{max_index + 1} positional argument(s) in gettext call, ' f'got {len(node.args)}', file=sys.stderr) - return + return False msg_data = {} - for position, arg_type in spec.items(): + for arg_type, position in spec.items(): arg = node.args[position] if not self._is_string_const(arg): print(f'*** {self.filename}:{arg.lineno}: Expected a string ' f'constant for argument {position + 1}, ' f'got {ast.unparse(arg)}', file=sys.stderr) - return + return False msg_data[arg_type] = arg.value lineno = node.lineno comments = self._extract_comments(node) self._add_message(lineno, **msg_data, comments=comments) + return True def _extract_comments(self, node): """Extract translator comments. @@ -729,15 +754,12 @@ class Options: # calculate all keywords try: - custom_keywords = dict(parse_spec(spec) for spec in options.keywords) + options.keywords = process_keywords( + options.keywords, + no_default_keywords=no_default_keywords) except ValueError as e: print(e, file=sys.stderr) sys.exit(1) - options.keywords = {} - if not no_default_keywords: - options.keywords |= DEFAULTKEYWORDS - # custom keywords override default keywords - options.keywords |= custom_keywords # initialize list of strings to exclude if options.excludefilename: From 63516d12f0f39adbdb210222918722c22c3ba905 Mon Sep 17 00:00:00 2001 From: Tomas Roun Date: Mon, 10 Mar 2025 08:20:12 +0100 Subject: [PATCH 02/12] Add news entry --- Lib/test/test_tools/i18n_data/multiple_keywords.pot | 6 +++--- Lib/test/test_tools/i18n_data/multiple_keywords.py | 6 +++++- .../2025-03-10-08-19-22.gh-issue-130453.9B0x8k.rst | 2 ++ 3 files changed, 10 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2025-03-10-08-19-22.gh-issue-130453.9B0x8k.rst diff --git a/Lib/test/test_tools/i18n_data/multiple_keywords.pot b/Lib/test/test_tools/i18n_data/multiple_keywords.pot index 35d5c6ef5ce4ec..954cb8e994838a 100644 --- a/Lib/test/test_tools/i18n_data/multiple_keywords.pot +++ b/Lib/test/test_tools/i18n_data/multiple_keywords.pot @@ -19,17 +19,17 @@ msgstr "" msgid "bar" msgstr "" -#: multiple_keywords.py:4 +#: multiple_keywords.py:5 msgctxt "baz" msgid "qux" msgstr "" -#: multiple_keywords.py:6 +#: multiple_keywords.py:9 msgctxt "corge" msgid "grault" msgstr "" -#: multiple_keywords.py:7 +#: multiple_keywords.py:11 msgctxt "xyzzy" msgid "foo" msgid_plural "foos" diff --git a/Lib/test/test_tools/i18n_data/multiple_keywords.py b/Lib/test/test_tools/i18n_data/multiple_keywords.py index fac1ac3a278cc8..7bde349505b839 100644 --- a/Lib/test/test_tools/i18n_data/multiple_keywords.py +++ b/Lib/test/test_tools/i18n_data/multiple_keywords.py @@ -1,7 +1,11 @@ from gettext import gettext as foo foo('bar') + foo('baz', 'qux') -# The 't' specifier is not supported, so this is extracted as pgettext + +# The 't' specifier is not supported, so the following +# call is extracted as pgettext instead of ngettext. foo('corge', 'grault', 1) + foo('xyzzy', 'foo', 'foos', 1) diff --git a/Misc/NEWS.d/next/Tools-Demos/2025-03-10-08-19-22.gh-issue-130453.9B0x8k.rst b/Misc/NEWS.d/next/Tools-Demos/2025-03-10-08-19-22.gh-issue-130453.9B0x8k.rst new file mode 100644 index 00000000000000..fdab48a2f7b25c --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2025-03-10-08-19-22.gh-issue-130453.9B0x8k.rst @@ -0,0 +1,2 @@ +Allow passing multiple keyword arguments with the same function name in +:program:`pygettext`. From 4426daf62d94818c9f00ec61dedff60d89e3f17c Mon Sep 17 00:00:00 2001 From: Tomas Roun Date: Tue, 18 Mar 2025 23:34:39 +0100 Subject: [PATCH 03/12] Simplify code --- Tools/i18n/pygettext.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Tools/i18n/pygettext.py b/Tools/i18n/pygettext.py index 65a5df1f5ac90d..5daf93b6675362 100755 --- a/Tools/i18n/pygettext.py +++ b/Tools/i18n/pygettext.py @@ -475,10 +475,8 @@ def _extract_docstring(self, node): def _extract_message(self, node): func_name = self._get_func_name(node) - specs = self.options.keywords.get(func_name, []) - for spec in specs: - extracted = self._extract_message_with_spec(node, spec) - if extracted: + for spec in self.options.keywords.get(func_name, []): + if self._extract_message_with_spec(node, spec): break def _extract_message_with_spec(self, node, spec): From 9f8e9b16a6cf44eb348d0c4ea2d6cdf09d64d7ca Mon Sep 17 00:00:00 2001 From: Tomas Roun Date: Wed, 19 Mar 2025 00:24:02 +0100 Subject: [PATCH 04/12] Do not print error messages if atleast 1 spec matches --- Lib/test/test_tools/test_i18n.py | 26 ++++++++++++++++++++++++++ Tools/i18n/pygettext.py | 32 +++++++++++++++++--------------- 2 files changed, 43 insertions(+), 15 deletions(-) diff --git a/Lib/test/test_tools/test_i18n.py b/Lib/test/test_tools/test_i18n.py index 5c6ea848b4e9c0..194e04ad98a0b4 100644 --- a/Lib/test/test_tools/test_i18n.py +++ b/Lib/test/test_tools/test_i18n.py @@ -543,6 +543,32 @@ def test_process_keywords(self): no_default_keywords=no_default_keywords) self.assertEqual(processed, expected) + def test_multiple_keywords_same_funcname_errors(self): + # If at least one keyword spec for a given funcname matches, + # no error should be printed. + msgids, stderr = self.extract_from_str(dedent('''\ + _("foo", 42) + _(42, "bar") + '''), args=('--keyword=_:1', '--keyword=_:2'), with_stderr=True) + self.assertIn('foo', msgids) + self.assertIn('bar', msgids) + self.assertEqual(stderr, b'') + + # If no keyword spec for a given funcname matches, + # all errors are printed. + msgids, stderr = self.extract_from_str(dedent('''\ + _(x, 42) + _(42, y) + '''), args=('--keyword=_:1', '--keyword=_:2'), with_stderr=True, + strict=False) + self.assertEqual(msgids, ['']) + self.assertEqual( + stderr, + b'*** test.py:1: Expected a string constant for argument 1, got x\n' + b'*** test.py:1: Expected a string constant for argument 2, got 42\n' + b'*** test.py:2: Expected a string constant for argument 1, got 42\n' + b'*** test.py:2: Expected a string constant for argument 2, got y\n') + def extract_from_snapshots(): snapshots = { diff --git a/Tools/i18n/pygettext.py b/Tools/i18n/pygettext.py index 5daf93b6675362..304eef192ebe52 100755 --- a/Tools/i18n/pygettext.py +++ b/Tools/i18n/pygettext.py @@ -475,44 +475,46 @@ def _extract_docstring(self, node): def _extract_message(self, node): func_name = self._get_func_name(node) + errors = [] for spec in self.options.keywords.get(func_name, []): - if self._extract_message_with_spec(node, spec): + err = self._extract_message_with_spec(node, spec) + if err is None: break + errors.append(err) + else: + for err in errors: + print(err, file=sys.stderr) def _extract_message_with_spec(self, node, spec): """Extract a gettext call with the given spec. - Return True if the gettext call was successfully extracted, False - otherwise. + Return `None` if the gettext call was successfully extracted, + otherwise return an error message. """ max_index = max(spec.values()) has_var_positional = any(isinstance(arg, ast.Starred) for arg in node.args[:max_index+1]) if has_var_positional: - print(f'*** {self.filename}:{node.lineno}: Variable positional ' - f'arguments are not allowed in gettext calls', file=sys.stderr) - return False + return (f'*** {self.filename}:{node.lineno}: Variable positional ' + f'arguments are not allowed in gettext calls') if max_index >= len(node.args): - print(f'*** {self.filename}:{node.lineno}: Expected at least ' - f'{max_index + 1} positional argument(s) in gettext call, ' - f'got {len(node.args)}', file=sys.stderr) - return False + return (f'*** {self.filename}:{node.lineno}: Expected at least ' + f'{max_index + 1} positional argument(s) in gettext call, ' + f'got {len(node.args)}') msg_data = {} for arg_type, position in spec.items(): arg = node.args[position] if not self._is_string_const(arg): - print(f'*** {self.filename}:{arg.lineno}: Expected a string ' - f'constant for argument {position + 1}, ' - f'got {ast.unparse(arg)}', file=sys.stderr) - return False + return (f'*** {self.filename}:{arg.lineno}: Expected a string ' + f'constant for argument {position + 1}, ' + f'got {ast.unparse(arg)}') msg_data[arg_type] = arg.value lineno = node.lineno comments = self._extract_comments(node) self._add_message(lineno, **msg_data, comments=comments) - return True def _extract_comments(self, node): """Extract translator comments. From ec02b0755daf986a80a46dec18eddabdd15b3fe9 Mon Sep 17 00:00:00 2001 From: Tomas Roun Date: Thu, 20 Mar 2025 20:43:31 +0100 Subject: [PATCH 05/12] Add unparse_spec --- Lib/test/test_tools/test_i18n.py | 5 ++++- Tools/i18n/pygettext.py | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_tools/test_i18n.py b/Lib/test/test_tools/test_i18n.py index 194e04ad98a0b4..2429af39edb8cc 100644 --- a/Lib/test/test_tools/test_i18n.py +++ b/Lib/test/test_tools/test_i18n.py @@ -18,7 +18,8 @@ with imports_under_tool("i18n"): - from pygettext import parse_spec, process_keywords, DEFAULTKEYWORDS + from pygettext import (parse_spec, process_keywords, DEFAULTKEYWORDS, + unparse_spec) def normalize_POT_file(pot): @@ -497,6 +498,8 @@ def test_parse_keyword_spec(self): for spec, expected in valid: with self.subTest(spec=spec): self.assertEqual(parse_spec(spec), expected) + # test unparse-parse round-trip + self.assertEqual(parse_spec(unparse_spec(*expected)), expected) invalid = ( ('foo:', "Invalid keyword spec 'foo:': missing argument positions"), diff --git a/Tools/i18n/pygettext.py b/Tools/i18n/pygettext.py index 304eef192ebe52..cae9cb716937d4 100755 --- a/Tools/i18n/pygettext.py +++ b/Tools/i18n/pygettext.py @@ -376,6 +376,20 @@ def parse_spec(spec): return name, result +def unparse_spec(name, spec): + """Unparse a keyword spec dictionary into a string.""" + if spec == {'msgid': 0}: + return name + + parts = [] + for arg, pos in sorted(spec.items(), key=lambda x: x[1]): + if arg == 'msgctxt': + parts.append(f'{pos + 1}c') + else: + parts.append(str(pos + 1)) + return f'{name}:{','.join(parts)}' + + def process_keywords(keywords, *, no_default_keywords): custom_keywords_list = [parse_spec(spec) for spec in keywords] custom_keywords = {} From 4aa5151ff4a8a9a4b895b193100eb98f7688aae6 Mon Sep 17 00:00:00 2001 From: Tomas Roun Date: Thu, 20 Mar 2025 21:09:28 +0100 Subject: [PATCH 06/12] Print errors in a more structured way --- Lib/test/test_tools/test_i18n.py | 10 +++++---- Tools/i18n/pygettext.py | 36 +++++++++++++++++++++----------- 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/Lib/test/test_tools/test_i18n.py b/Lib/test/test_tools/test_i18n.py index 2429af39edb8cc..03b151dac5a6d0 100644 --- a/Lib/test/test_tools/test_i18n.py +++ b/Lib/test/test_tools/test_i18n.py @@ -567,10 +567,12 @@ def test_multiple_keywords_same_funcname_errors(self): self.assertEqual(msgids, ['']) self.assertEqual( stderr, - b'*** test.py:1: Expected a string constant for argument 1, got x\n' - b'*** test.py:1: Expected a string constant for argument 2, got 42\n' - b'*** test.py:2: Expected a string constant for argument 1, got 42\n' - b'*** test.py:2: Expected a string constant for argument 2, got y\n') + b'*** test.py:1: No keywords matched gettext call "_":\n' + b'\tkeyword="_": Expected a string constant for argument 1, got x\n' + b'\tkeyword="_:2": Expected a string constant for argument 2, got 42\n' + b'*** test.py:2: No keywords matched gettext call "_":\n' + b'\tkeyword="_": Expected a string constant for argument 1, got 42\n' + b'\tkeyword="_:2": Expected a string constant for argument 2, got y\n') def extract_from_snapshots(): diff --git a/Tools/i18n/pygettext.py b/Tools/i18n/pygettext.py index cae9cb716937d4..422e2c92066213 100755 --- a/Tools/i18n/pygettext.py +++ b/Tools/i18n/pygettext.py @@ -490,14 +490,28 @@ def _extract_docstring(self, node): def _extract_message(self, node): func_name = self._get_func_name(node) errors = [] - for spec in self.options.keywords.get(func_name, []): + specs = self.options.keywords.get(func_name, []) + for spec in specs: err = self._extract_message_with_spec(node, spec) if err is None: - break + return errors.append(err) + + if not errors: + return + if len(errors) == 1: + print(f'*** {self.filename}:{node.lineno}: {errors[0]}', + file=sys.stderr) else: - for err in errors: - print(err, file=sys.stderr) + # There are multiple keyword specs for the function name and + # none of them could be exxtracted. Print a general error + # message and list the errors for each keyword spec. + print(f'*** {self.filename}:{node.lineno}: ' + f'No keywords matched gettext call "{func_name}":', + file=sys.stderr) + for spec, err in zip(specs, errors, strict=True): + unparsed = unparse_spec(func_name, spec) + print(f'\tkeyword="{unparsed}": {err}', file=sys.stderr) def _extract_message_with_spec(self, node, spec): """Extract a gettext call with the given spec. @@ -509,21 +523,19 @@ def _extract_message_with_spec(self, node, spec): has_var_positional = any(isinstance(arg, ast.Starred) for arg in node.args[:max_index+1]) if has_var_positional: - return (f'*** {self.filename}:{node.lineno}: Variable positional ' - f'arguments are not allowed in gettext calls') + return ('Variable positional arguments are not ' + 'allowed in gettext calls') if max_index >= len(node.args): - return (f'*** {self.filename}:{node.lineno}: Expected at least ' - f'{max_index + 1} positional argument(s) in gettext call, ' - f'got {len(node.args)}') + return (f'Expected at least {max_index + 1} positional ' + f'argument(s) in gettext call, got {len(node.args)}') msg_data = {} for arg_type, position in spec.items(): arg = node.args[position] if not self._is_string_const(arg): - return (f'*** {self.filename}:{arg.lineno}: Expected a string ' - f'constant for argument {position + 1}, ' - f'got {ast.unparse(arg)}') + return (f'Expected a string constant for argument ' + f'{position + 1}, got {ast.unparse(arg)}') msg_data[arg_type] = arg.value lineno = node.lineno From 2dcab79f793aac43b197118a9fdf015a36a85eee Mon Sep 17 00:00:00 2001 From: Tomas Roun Date: Thu, 20 Mar 2025 22:37:56 +0100 Subject: [PATCH 07/12] Improve keyword merging --- Lib/test/test_tools/test_i18n.py | 11 +++++++++-- Tools/i18n/pygettext.py | 11 ++++++++--- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_tools/test_i18n.py b/Lib/test/test_tools/test_i18n.py index 03b151dac5a6d0..3ab9114e8162ba 100644 --- a/Lib/test/test_tools/test_i18n.py +++ b/Lib/test/test_tools/test_i18n.py @@ -528,6 +528,9 @@ def test_process_keywords(self): (['foo', 'foo:1,2'], True), (['foo'], False), (['_:1,2', '_:1c,2,3', 'pgettext'], False), + # Duplicate entries + (['foo', 'foo'], True), + (['_'], False) ) expected = ( {'foo': [{'msgid': 0}]}, @@ -535,8 +538,12 @@ def test_process_keywords(self): {'foo': [{'msgid': 0}, {'msgid': 0, 'msgid_plural': 1}]}, default_keywords | {'foo': [{'msgid': 0}]}, default_keywords | {'_': [{'msgid': 0, 'msgid_plural': 1}, - {'msgctxt': 0, 'msgid': 1, 'msgid_plural': 2}], - 'pgettext': [{'msgid': 0}]}, + {'msgctxt': 0, 'msgid': 1, 'msgid_plural': 2}, + {'msgid': 0}], + 'pgettext': [{'msgid': 0}, + {'msgctxt': 0, 'msgid': 1}]}, + {'foo': [{'msgid': 0}]}, + default_keywords, ) for (keywords, no_default_keywords), expected in zip(inputs, expected): with self.subTest(keywords=keywords, diff --git a/Tools/i18n/pygettext.py b/Tools/i18n/pygettext.py index 422e2c92066213..34c1251d1895de 100755 --- a/Tools/i18n/pygettext.py +++ b/Tools/i18n/pygettext.py @@ -391,7 +391,8 @@ def unparse_spec(name, spec): def process_keywords(keywords, *, no_default_keywords): - custom_keywords_list = [parse_spec(spec) for spec in keywords] + custom_keywords_list = [parse_spec(spec) for + spec in dict.fromkeys(keywords)] custom_keywords = {} for name, spec in custom_keywords_list: if name not in custom_keywords: @@ -401,9 +402,13 @@ def process_keywords(keywords, *, no_default_keywords): if no_default_keywords: return custom_keywords - default_keywords = {name: [spec] for name, spec in DEFAULTKEYWORDS.items()} # custom keywords override default keywords - return default_keywords | custom_keywords + for name, spec in DEFAULTKEYWORDS.items(): + if name not in custom_keywords: + custom_keywords[name] = [] + if spec not in custom_keywords[name]: + custom_keywords[name].append(spec) + return custom_keywords @dataclass(frozen=True) From a354d1908e917ea0b9aa38330892c306fc25f7af Mon Sep 17 00:00:00 2001 From: "Tomas R." Date: Thu, 20 Mar 2025 23:03:26 +0100 Subject: [PATCH 08/12] Update Tools/i18n/pygettext.py Co-authored-by: Serhiy Storchaka --- Tools/i18n/pygettext.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/i18n/pygettext.py b/Tools/i18n/pygettext.py index 34c1251d1895de..dacac818d652c3 100755 --- a/Tools/i18n/pygettext.py +++ b/Tools/i18n/pygettext.py @@ -521,7 +521,7 @@ def _extract_message(self, node): def _extract_message_with_spec(self, node, spec): """Extract a gettext call with the given spec. - Return `None` if the gettext call was successfully extracted, + Return None if the gettext call was successfully extracted, otherwise return an error message. """ max_index = max(spec.values()) From ab02d21b9d04a9fe8c6c4b67df05c58ff70782ba Mon Sep 17 00:00:00 2001 From: Tomas Roun Date: Thu, 20 Mar 2025 23:27:04 +0100 Subject: [PATCH 09/12] Windows fix --- Lib/test/test_tools/test_i18n.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_tools/test_i18n.py b/Lib/test/test_tools/test_i18n.py index 3ab9114e8162ba..397f4a79a10827 100644 --- a/Lib/test/test_tools/test_i18n.py +++ b/Lib/test/test_tools/test_i18n.py @@ -572,14 +572,16 @@ def test_multiple_keywords_same_funcname_errors(self): '''), args=('--keyword=_:1', '--keyword=_:2'), with_stderr=True, strict=False) self.assertEqual(msgids, ['']) + # Normalize line endings on Windows + stderr = stderr.decode('utf-8').replace('\r', '') self.assertEqual( stderr, - b'*** test.py:1: No keywords matched gettext call "_":\n' - b'\tkeyword="_": Expected a string constant for argument 1, got x\n' - b'\tkeyword="_:2": Expected a string constant for argument 2, got 42\n' - b'*** test.py:2: No keywords matched gettext call "_":\n' - b'\tkeyword="_": Expected a string constant for argument 1, got 42\n' - b'\tkeyword="_:2": Expected a string constant for argument 2, got y\n') + '*** test.py:1: No keywords matched gettext call "_":\n' + '\tkeyword="_": Expected a string constant for argument 1, got x\n' + '\tkeyword="_:2": Expected a string constant for argument 2, got 42\n' + '*** test.py:2: No keywords matched gettext call "_":\n' + '\tkeyword="_": Expected a string constant for argument 1, got 42\n' + '\tkeyword="_:2": Expected a string constant for argument 2, got y\n') def extract_from_snapshots(): From bac8b8d98645bc24acbcd0e57eb96923e16569b4 Mon Sep 17 00:00:00 2001 From: Tomas Roun Date: Sun, 23 Mar 2025 21:13:43 +0100 Subject: [PATCH 10/12] PEP8 --- Tools/i18n/pygettext.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Tools/i18n/pygettext.py b/Tools/i18n/pygettext.py index dacac818d652c3..f1b8a4bba688dd 100755 --- a/Tools/i18n/pygettext.py +++ b/Tools/i18n/pygettext.py @@ -506,14 +506,14 @@ def _extract_message(self, node): return if len(errors) == 1: print(f'*** {self.filename}:{node.lineno}: {errors[0]}', - file=sys.stderr) + file=sys.stderr) else: # There are multiple keyword specs for the function name and # none of them could be exxtracted. Print a general error # message and list the errors for each keyword spec. print(f'*** {self.filename}:{node.lineno}: ' - f'No keywords matched gettext call "{func_name}":', - file=sys.stderr) + f'No keywords matched gettext call "{func_name}":', + file=sys.stderr) for spec, err in zip(specs, errors, strict=True): unparsed = unparse_spec(func_name, spec) print(f'\tkeyword="{unparsed}": {err}', file=sys.stderr) From 51b48f07153aa59d6436decc1953dd681d51f168 Mon Sep 17 00:00:00 2001 From: "Tomas R." Date: Thu, 10 Apr 2025 09:34:48 +0200 Subject: [PATCH 11/12] Fix typo Co-authored-by: Serhiy Storchaka --- Tools/i18n/pygettext.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/i18n/pygettext.py b/Tools/i18n/pygettext.py index fe97604c6de4af..1f16bf73a0abba 100755 --- a/Tools/i18n/pygettext.py +++ b/Tools/i18n/pygettext.py @@ -509,7 +509,7 @@ def _extract_message(self, node): file=sys.stderr) else: # There are multiple keyword specs for the function name and - # none of them could be exxtracted. Print a general error + # none of them could be extracted. Print a general error # message and list the errors for each keyword spec. print(f'*** {self.filename}:{node.lineno}: ' f'No keywords matched gettext call "{func_name}":', From 4daad1bfacf1db545d31307df54ac8365acaf356 Mon Sep 17 00:00:00 2001 From: "Tomas R." Date: Thu, 10 Apr 2025 09:35:48 +0200 Subject: [PATCH 12/12] Simplify keyword parsing Co-authored-by: Serhiy Storchaka --- Tools/i18n/pygettext.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Tools/i18n/pygettext.py b/Tools/i18n/pygettext.py index 1f16bf73a0abba..351b47a160e999 100755 --- a/Tools/i18n/pygettext.py +++ b/Tools/i18n/pygettext.py @@ -391,10 +391,9 @@ def unparse_spec(name, spec): def process_keywords(keywords, *, no_default_keywords): - custom_keywords_list = [parse_spec(spec) for - spec in dict.fromkeys(keywords)] custom_keywords = {} - for name, spec in custom_keywords_list: + for spec in dict.fromkeys(keywords): + name, spec = parse_spec(spec) if name not in custom_keywords: custom_keywords[name] = [] custom_keywords[name].append(spec)