Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit b6760b7

Browse filesBrowse files
tomasr8tomasr8
authored
gh-130453: pygettext: Allow specifying multiple keywords with the same function name (GH-131380)
1 parent 619edb8 commit b6760b7
Copy full SHA for b6760b7

File tree

Expand file treeCollapse file tree

5 files changed

+217
-43
lines changed
Filter options
Expand file treeCollapse file tree

5 files changed

+217
-43
lines changed

‎Lib/test/test_tools/i18n_data/multiple_keywords.pot

Copy file name to clipboard
+38Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
# SOME DESCRIPTIVE TITLE.
2+
# Copyright (C) YEAR ORGANIZATION
3+
# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
4+
#
5+
msgid ""
6+
msgstr ""
7+
"Project-Id-Version: PACKAGE VERSION\n"
8+
"POT-Creation-Date: 2000-01-01 00:00+0000\n"
9+
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
10+
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
11+
"Language-Team: LANGUAGE <LL@li.org>\n"
12+
"MIME-Version: 1.0\n"
13+
"Content-Type: text/plain; charset=UTF-8\n"
14+
"Content-Transfer-Encoding: 8bit\n"
15+
"Generated-By: pygettext.py 1.5\n"
16+
17+
18+
#: multiple_keywords.py:3
19+
msgid "bar"
20+
msgstr ""
21+
22+
#: multiple_keywords.py:5
23+
msgctxt "baz"
24+
msgid "qux"
25+
msgstr ""
26+
27+
#: multiple_keywords.py:9
28+
msgctxt "corge"
29+
msgid "grault"
30+
msgstr ""
31+
32+
#: multiple_keywords.py:11
33+
msgctxt "xyzzy"
34+
msgid "foo"
35+
msgid_plural "foos"
36+
msgstr[0] ""
37+
msgstr[1] ""
38+

‎Lib/test/test_tools/i18n_data/multiple_keywords.py

Copy file name to clipboard
+11Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
from gettext import gettext as foo
2+
3+
foo('bar')
4+
5+
foo('baz', 'qux')
6+
7+
# The 't' specifier is not supported, so the following
8+
# call is extracted as pgettext instead of ngettext.
9+
foo('corge', 'grault', 1)
10+
11+
foo('xyzzy', 'foo', 'foos', 1)

‎Lib/test/test_tools/test_i18n.py

Copy file name to clipboardExpand all lines: Lib/test/test_tools/test_i18n.py
+82-11Lines changed: 82 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,8 @@
1818

1919

2020
with imports_under_tool("i18n"):
21-
from pygettext import parse_spec
21+
from pygettext import (parse_spec, process_keywords, DEFAULTKEYWORDS,
22+
unparse_spec)
2223

2324

2425
def normalize_POT_file(pot):
@@ -483,20 +484,22 @@ def test_comments_not_extracted_without_tags(self):
483484

484485
def test_parse_keyword_spec(self):
485486
valid = (
486-
('foo', ('foo', {0: 'msgid'})),
487-
('foo:1', ('foo', {0: 'msgid'})),
488-
('foo:1,2', ('foo', {0: 'msgid', 1: 'msgid_plural'})),
489-
('foo:1, 2', ('foo', {0: 'msgid', 1: 'msgid_plural'})),
490-
('foo:1,2c', ('foo', {0: 'msgid', 1: 'msgctxt'})),
491-
('foo:2c,1', ('foo', {0: 'msgid', 1: 'msgctxt'})),
492-
('foo:2c ,1', ('foo', {0: 'msgid', 1: 'msgctxt'})),
493-
('foo:1,2,3c', ('foo', {0: 'msgid', 1: 'msgid_plural', 2: 'msgctxt'})),
494-
('foo:1, 2, 3c', ('foo', {0: 'msgid', 1: 'msgid_plural', 2: 'msgctxt'})),
495-
('foo:3c,1,2', ('foo', {0: 'msgid', 1: 'msgid_plural', 2: 'msgctxt'})),
487+
('foo', ('foo', {'msgid': 0})),
488+
('foo:1', ('foo', {'msgid': 0})),
489+
('foo:1,2', ('foo', {'msgid': 0, 'msgid_plural': 1})),
490+
('foo:1, 2', ('foo', {'msgid': 0, 'msgid_plural': 1})),
491+
('foo:1,2c', ('foo', {'msgid': 0, 'msgctxt': 1})),
492+
('foo:2c,1', ('foo', {'msgid': 0, 'msgctxt': 1})),
493+
('foo:2c ,1', ('foo', {'msgid': 0, 'msgctxt': 1})),
494+
('foo:1,2,3c', ('foo', {'msgid': 0, 'msgid_plural': 1, 'msgctxt': 2})),
495+
('foo:1, 2, 3c', ('foo', {'msgid': 0, 'msgid_plural': 1, 'msgctxt': 2})),
496+
('foo:3c,1,2', ('foo', {'msgid': 0, 'msgid_plural': 1, 'msgctxt': 2})),
496497
)
497498
for spec, expected in valid:
498499
with self.subTest(spec=spec):
499500
self.assertEqual(parse_spec(spec), expected)
501+
# test unparse-parse round-trip
502+
self.assertEqual(parse_spec(unparse_spec(*expected)), expected)
500503

501504
invalid = (
502505
('foo:', "Invalid keyword spec 'foo:': missing argument positions"),
@@ -516,6 +519,70 @@ def test_parse_keyword_spec(self):
516519
parse_spec(spec)
517520
self.assertEqual(str(cm.exception), message)
518521

522+
def test_process_keywords(self):
523+
default_keywords = {name: [spec] for name, spec
524+
in DEFAULTKEYWORDS.items()}
525+
inputs = (
526+
(['foo'], True),
527+
(['_:1,2'], True),
528+
(['foo', 'foo:1,2'], True),
529+
(['foo'], False),
530+
(['_:1,2', '_:1c,2,3', 'pgettext'], False),
531+
# Duplicate entries
532+
(['foo', 'foo'], True),
533+
(['_'], False)
534+
)
535+
expected = (
536+
{'foo': [{'msgid': 0}]},
537+
{'_': [{'msgid': 0, 'msgid_plural': 1}]},
538+
{'foo': [{'msgid': 0}, {'msgid': 0, 'msgid_plural': 1}]},
539+
default_keywords | {'foo': [{'msgid': 0}]},
540+
default_keywords | {'_': [{'msgid': 0, 'msgid_plural': 1},
541+
{'msgctxt': 0, 'msgid': 1, 'msgid_plural': 2},
542+
{'msgid': 0}],
543+
'pgettext': [{'msgid': 0},
544+
{'msgctxt': 0, 'msgid': 1}]},
545+
{'foo': [{'msgid': 0}]},
546+
default_keywords,
547+
)
548+
for (keywords, no_default_keywords), expected in zip(inputs, expected):
549+
with self.subTest(keywords=keywords,
550+
no_default_keywords=no_default_keywords):
551+
processed = process_keywords(
552+
keywords,
553+
no_default_keywords=no_default_keywords)
554+
self.assertEqual(processed, expected)
555+
556+
def test_multiple_keywords_same_funcname_errors(self):
557+
# If at least one keyword spec for a given funcname matches,
558+
# no error should be printed.
559+
msgids, stderr = self.extract_from_str(dedent('''\
560+
_("foo", 42)
561+
_(42, "bar")
562+
'''), args=('--keyword=_:1', '--keyword=_:2'), with_stderr=True)
563+
self.assertIn('foo', msgids)
564+
self.assertIn('bar', msgids)
565+
self.assertEqual(stderr, b'')
566+
567+
# If no keyword spec for a given funcname matches,
568+
# all errors are printed.
569+
msgids, stderr = self.extract_from_str(dedent('''\
570+
_(x, 42)
571+
_(42, y)
572+
'''), args=('--keyword=_:1', '--keyword=_:2'), with_stderr=True,
573+
strict=False)
574+
self.assertEqual(msgids, [''])
575+
# Normalize line endings on Windows
576+
stderr = stderr.decode('utf-8').replace('\r', '')
577+
self.assertEqual(
578+
stderr,
579+
'*** test.py:1: No keywords matched gettext call "_":\n'
580+
'\tkeyword="_": Expected a string constant for argument 1, got x\n'
581+
'\tkeyword="_:2": Expected a string constant for argument 2, got 42\n'
582+
'*** test.py:2: No keywords matched gettext call "_":\n'
583+
'\tkeyword="_": Expected a string constant for argument 1, got 42\n'
584+
'\tkeyword="_:2": Expected a string constant for argument 2, got y\n')
585+
519586

520587
def extract_from_snapshots():
521588
snapshots = {
@@ -526,6 +593,10 @@ def extract_from_snapshots():
526593
'custom_keywords.py': ('--keyword=foo', '--keyword=nfoo:1,2',
527594
'--keyword=pfoo:1c,2',
528595
'--keyword=npfoo:1c,2,3', '--keyword=_:1,2'),
596+
'multiple_keywords.py': ('--keyword=foo:1c,2,3', '--keyword=foo:1c,2',
597+
'--keyword=foo:1,2',
598+
# repeat a keyword to make sure it is extracted only once
599+
'--keyword=foo', '--keyword=foo'),
529600
# == Test character escaping
530601
# Escape ascii and unicode:
531602
'escapes.py': ('--escape', '--add-comments='),

‎Misc/NEWS.d/next/Tools-Demos/2025-03-10-08-19-22.gh-issue-130453.9B0x8k.rst

Copy file name to clipboard
+2Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Allow passing multiple keyword arguments with the same function name in
2+
:program:`pygettext`.

‎Tools/i18n/pygettext.py

Copy file name to clipboardExpand all lines: Tools/i18n/pygettext.py
+84-32Lines changed: 84 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -282,15 +282,15 @@ def getFilesForName(name):
282282
# Key is the function name, value is a dictionary mapping argument positions to the
283283
# type of the argument. The type is one of 'msgid', 'msgid_plural', or 'msgctxt'.
284284
DEFAULTKEYWORDS = {
285-
'_': {0: 'msgid'},
286-
'gettext': {0: 'msgid'},
287-
'ngettext': {0: 'msgid', 1: 'msgid_plural'},
288-
'pgettext': {0: 'msgctxt', 1: 'msgid'},
289-
'npgettext': {0: 'msgctxt', 1: 'msgid', 2: 'msgid_plural'},
290-
'dgettext': {1: 'msgid'},
291-
'dngettext': {1: 'msgid', 2: 'msgid_plural'},
292-
'dpgettext': {1: 'msgctxt', 2: 'msgid'},
293-
'dnpgettext': {1: 'msgctxt', 2: 'msgid', 3: 'msgid_plural'},
285+
'_': {'msgid': 0},
286+
'gettext': {'msgid': 0},
287+
'ngettext': {'msgid': 0, 'msgid_plural': 1},
288+
'pgettext': {'msgctxt': 0, 'msgid': 1},
289+
'npgettext': {'msgctxt': 0, 'msgid': 1, 'msgid_plural': 2},
290+
'dgettext': {'msgid': 1},
291+
'dngettext': {'msgid': 1, 'msgid_plural': 2},
292+
'dpgettext': {'msgctxt': 1, 'msgid': 2},
293+
'dnpgettext': {'msgctxt': 1, 'msgid': 2, 'msgid_plural': 3},
294294
}
295295

296296

@@ -327,7 +327,7 @@ def parse_spec(spec):
327327
parts = spec.strip().split(':', 1)
328328
if len(parts) == 1:
329329
name = parts[0]
330-
return name, {0: 'msgid'}
330+
return name, {'msgid': 0}
331331

332332
name, args = parts
333333
if not args:
@@ -373,7 +373,41 @@ def parse_spec(spec):
373373
raise ValueError(f'Invalid keyword spec {spec!r}: '
374374
'msgctxt cannot appear without msgid')
375375

376-
return name, {v: k for k, v in result.items()}
376+
return name, result
377+
378+
379+
def unparse_spec(name, spec):
380+
"""Unparse a keyword spec dictionary into a string."""
381+
if spec == {'msgid': 0}:
382+
return name
383+
384+
parts = []
385+
for arg, pos in sorted(spec.items(), key=lambda x: x[1]):
386+
if arg == 'msgctxt':
387+
parts.append(f'{pos + 1}c')
388+
else:
389+
parts.append(str(pos + 1))
390+
return f'{name}:{','.join(parts)}'
391+
392+
393+
def process_keywords(keywords, *, no_default_keywords):
394+
custom_keywords = {}
395+
for spec in dict.fromkeys(keywords):
396+
name, spec = parse_spec(spec)
397+
if name not in custom_keywords:
398+
custom_keywords[name] = []
399+
custom_keywords[name].append(spec)
400+
401+
if no_default_keywords:
402+
return custom_keywords
403+
404+
# custom keywords override default keywords
405+
for name, spec in DEFAULTKEYWORDS.items():
406+
if name not in custom_keywords:
407+
custom_keywords[name] = []
408+
if spec not in custom_keywords[name]:
409+
custom_keywords[name].append(spec)
410+
return custom_keywords
377411

378412

379413
@dataclass(frozen=True)
@@ -459,32 +493,53 @@ def _extract_docstring(self, node):
459493

460494
def _extract_message(self, node):
461495
func_name = self._get_func_name(node)
462-
spec = self.options.keywords.get(func_name)
463-
if spec is None:
496+
errors = []
497+
specs = self.options.keywords.get(func_name, [])
498+
for spec in specs:
499+
err = self._extract_message_with_spec(node, spec)
500+
if err is None:
501+
return
502+
errors.append(err)
503+
504+
if not errors:
464505
return
506+
if len(errors) == 1:
507+
print(f'*** {self.filename}:{node.lineno}: {errors[0]}',
508+
file=sys.stderr)
509+
else:
510+
# There are multiple keyword specs for the function name and
511+
# none of them could be extracted. Print a general error
512+
# message and list the errors for each keyword spec.
513+
print(f'*** {self.filename}:{node.lineno}: '
514+
f'No keywords matched gettext call "{func_name}":',
515+
file=sys.stderr)
516+
for spec, err in zip(specs, errors, strict=True):
517+
unparsed = unparse_spec(func_name, spec)
518+
print(f'\tkeyword="{unparsed}": {err}', file=sys.stderr)
519+
520+
def _extract_message_with_spec(self, node, spec):
521+
"""Extract a gettext call with the given spec.
465522
466-
max_index = max(spec)
523+
Return None if the gettext call was successfully extracted,
524+
otherwise return an error message.
525+
"""
526+
max_index = max(spec.values())
467527
has_var_positional = any(isinstance(arg, ast.Starred) for
468528
arg in node.args[:max_index+1])
469529
if has_var_positional:
470-
print(f'*** {self.filename}:{node.lineno}: Variable positional '
471-
f'arguments are not allowed in gettext calls', file=sys.stderr)
472-
return
530+
return ('Variable positional arguments are not '
531+
'allowed in gettext calls')
473532

474533
if max_index >= len(node.args):
475-
print(f'*** {self.filename}:{node.lineno}: Expected at least '
476-
f'{max(spec) + 1} positional argument(s) in gettext call, '
477-
f'got {len(node.args)}', file=sys.stderr)
478-
return
534+
return (f'Expected at least {max_index + 1} positional '
535+
f'argument(s) in gettext call, got {len(node.args)}')
479536

480537
msg_data = {}
481-
for position, arg_type in spec.items():
538+
for arg_type, position in spec.items():
482539
arg = node.args[position]
483540
if not self._is_string_const(arg):
484-
print(f'*** {self.filename}:{arg.lineno}: Expected a string '
485-
f'constant for argument {position + 1}, '
486-
f'got {ast.unparse(arg)}', file=sys.stderr)
487-
return
541+
return (f'Expected a string constant for argument '
542+
f'{position + 1}, got {ast.unparse(arg)}')
488543
msg_data[arg_type] = arg.value
489544

490545
lineno = node.lineno
@@ -729,15 +784,12 @@ class Options:
729784

730785
# calculate all keywords
731786
try:
732-
custom_keywords = dict(parse_spec(spec) for spec in options.keywords)
787+
options.keywords = process_keywords(
788+
options.keywords,
789+
no_default_keywords=no_default_keywords)
733790
except ValueError as e:
734791
print(e, file=sys.stderr)
735792
sys.exit(1)
736-
options.keywords = {}
737-
if not no_default_keywords:
738-
options.keywords |= DEFAULTKEYWORDS
739-
# custom keywords override default keywords
740-
options.keywords |= custom_keywords
741793

742794
# initialize list of strings to exclude
743795
if options.excludefilename:

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.