Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

Appearance settings

Commit edeff24

Browse filesBrowse files
committed
gh-99891: Fix infinite recursion in the tokenizer when showing warnings
1 parent 05dfc53 commit edeff24
Copy full SHA for edeff24

File tree

Expand file treeCollapse file tree

4 files changed

+25
-0
lines changed
Filter options
Expand file treeCollapse file tree

4 files changed

+25
-0
lines changed

‎Lib/test/test_source_encoding.py

Copy file name to clipboardExpand all lines: Lib/test/test_source_encoding.py
+13Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,19 @@ def test_file_parse_error_multiline(self):
160160
finally:
161161
os.unlink(TESTFN)
162162

163+
def test_tokenizer_fstring_warning_in_first_line(self):
164+
source = "0b1and 2"
165+
with open(TESTFN, "w") as fd:
166+
fd.write("{}".format(source))
167+
try:
168+
retcode, stdout, stderr = script_helper.assert_python_failure(TESTFN, PYTHONWARNINGS="error")
169+
self.assertGreater(retcode, 0)
170+
self.assertIn(b"SyntaxError: invalid binary litera", stderr)
171+
self.assertEqual(stderr.count(source.encode()), 1)
172+
finally:
173+
os.unlink(TESTFN)
174+
175+
163176
class AbstractSourceEncodingTest:
164177

165178
def test_default_coding(self):
+3Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Fix a bug in the tokenizer that could cause infinite recursion when showing
2+
syntax warnings that happen in the first line of the source. Patch by Pablo
3+
Galindo

‎Parser/tokenizer.c

Copy file name to clipboardExpand all lines: Parser/tokenizer.c
+8Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,7 @@ tok_new(void)
9797
tok->async_def_nl = 0;
9898
tok->interactive_underflow = IUNDERFLOW_NORMAL;
9999
tok->str = NULL;
100+
tok->report_warnings = 1;
100101
#ifdef Py_DEBUG
101102
tok->debug = _Py_GetConfig()->parser_debug;
102103
#endif
@@ -1201,6 +1202,10 @@ indenterror(struct tok_state *tok)
12011202
static int
12021203
parser_warn(struct tok_state *tok, PyObject *category, const char *format, ...)
12031204
{
1205+
if (!tok->report_warnings) {
1206+
return 0;
1207+
}
1208+
12041209
PyObject *errmsg;
12051210
va_list vargs;
12061211
va_start(vargs, format);
@@ -2239,6 +2244,9 @@ _PyTokenizer_FindEncodingFilename(int fd, PyObject *filename)
22392244
}
22402245
}
22412246
struct token token;
2247+
// We don't want to report warnings here because it could cause infinite recursion
2248+
// if fetching the encoding shows a warning.
2249+
tok->report_warnings = 0;
22422250
while (tok->lineno < 2 && tok->done == E_OK) {
22432251
_PyTokenizer_Get(tok, &token);
22442252
}

‎Parser/tokenizer.h

Copy file name to clipboardExpand all lines: Parser/tokenizer.h
+1Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@ struct tok_state {
9292
NEWLINE token after it. */
9393
/* How to proceed when asked for a new token in interactive mode */
9494
enum interactive_underflow_t interactive_underflow;
95+
int report_warnings;
9596
#ifdef Py_DEBUG
9697
int debug;
9798
#endif

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.