Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit c8cf9b4

Browse filesBrowse files
pablogsalpablogsal
authored
pythongh-104825: Remove implicit newline in the line attribute in tokens emitted in the tokenize module (python#104846)
1 parent c45701e commit c8cf9b4
Copy full SHA for c8cf9b4

File tree

Expand file treeCollapse file tree

5 files changed

+14
-8
lines changed
Filter options
Expand file treeCollapse file tree

5 files changed

+14
-8
lines changed

‎Lib/idlelib/idle_test/test_editor.py

Copy file name to clipboardExpand all lines: Lib/idlelib/idle_test/test_editor.py
+2-2Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -201,8 +201,8 @@ def test_searcher(self):
201201
test_info = (# text, (block, indent))
202202
("", (None, None)),
203203
("[1,", (None, None)), # TokenError
204-
("if 1:\n", ('if 1:\n', None)),
205-
("if 1:\n 2\n 3\n", ('if 1:\n', ' 2\n')),
204+
("if 1:\n", ('if 1:', None)),
205+
("if 1:\n 2\n 3\n", ('if 1:', ' 2')),
206206
)
207207
for code, expected_pair in test_info:
208208
with self.subTest(code=code):

‎Lib/test/test_tabnanny.py

Copy file name to clipboardExpand all lines: Lib/test/test_tabnanny.py
+4-4Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -222,7 +222,7 @@ def test_when_nannynag_error_verbose(self):
222222
"""
223223
with TemporaryPyFile(SOURCE_CODES["nannynag_errored"]) as file_path:
224224
out = f"{file_path!r}: *** Line 3: trouble in tab city! ***\n"
225-
out += "offending line: '\\tprint(\"world\")\\n'\n"
225+
out += "offending line: '\\tprint(\"world\")'\n"
226226
out += "inconsistent use of tabs and spaces in indentation\n"
227227

228228
tabnanny.verbose = 1
@@ -231,7 +231,7 @@ def test_when_nannynag_error_verbose(self):
231231
def test_when_nannynag_error(self):
232232
"""A python source code file eligible for raising `tabnanny.NannyNag`."""
233233
with TemporaryPyFile(SOURCE_CODES["nannynag_errored"]) as file_path:
234-
out = f"{file_path} 3 '\\tprint(\"world\")\\n'\n"
234+
out = f"{file_path} 3 '\\tprint(\"world\")'\n"
235235
self.verify_tabnanny_check(file_path, out=out)
236236

237237
def test_when_no_file(self):
@@ -341,14 +341,14 @@ def test_verbose_mode(self):
341341
"""Should display more error information if verbose mode is on."""
342342
with TemporaryPyFile(SOURCE_CODES["nannynag_errored"]) as path:
343343
stdout = textwrap.dedent(
344-
"offending line: '\\tprint(\"world\")\\n'"
344+
"offending line: '\\tprint(\"world\")'"
345345
).strip()
346346
self.validate_cmd("-v", path, stdout=stdout, partial=True)
347347

348348
def test_double_verbose_mode(self):
349349
"""Should display detailed error information if double verbose is on."""
350350
with TemporaryPyFile(SOURCE_CODES["nannynag_errored"]) as path:
351351
stdout = textwrap.dedent(
352-
"offending line: '\\tprint(\"world\")\\n'"
352+
"offending line: '\\tprint(\"world\")'"
353353
).strip()
354354
self.validate_cmd("-vv", path, stdout=stdout, partial=True)

‎Lib/test/test_tokenize.py

Copy file name to clipboardExpand all lines: Lib/test/test_tokenize.py
+2-2Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ def k(x):
103103
e.exception.msg,
104104
'unindent does not match any outer indentation level')
105105
self.assertEqual(e.exception.offset, 9)
106-
self.assertEqual(e.exception.text, ' x += 5\n')
106+
self.assertEqual(e.exception.text, ' x += 5')
107107

108108
def test_int(self):
109109
# Ordinary integers and binary operators
@@ -1157,7 +1157,7 @@ def readline():
11571157

11581158
# skip the initial encoding token and the end tokens
11591159
tokens = list(_tokenize(readline(), encoding='utf-8'))[:-2]
1160-
expected_tokens = [TokenInfo(3, '"ЉЊЈЁЂ"', (1, 0), (1, 7), '"ЉЊЈЁЂ"\n')]
1160+
expected_tokens = [TokenInfo(3, '"ЉЊЈЁЂ"', (1, 0), (1, 7), '"ЉЊЈЁЂ"')]
11611161
self.assertEqual(tokens, expected_tokens,
11621162
"bytes not decoded with encoding")
11631163

‎Misc/NEWS.d/next/Core and Builtins/2023-05-24-09-59-56.gh-issue-104825.mQesie.rst

Copy file name to clipboard
+2Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Tokens emitted by the :mod:`tokenize` module do not include an implicit
2+
``\n`` character in the ``line`` attribute anymore. Patch by Pablo Galindo

‎Python/Python-tokenize.c

Copy file name to clipboardExpand all lines: Python/Python-tokenize.c
+4Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,8 @@ _tokenizer_error(struct tok_state *tok)
123123
int result = 0;
124124

125125
Py_ssize_t size = tok->inp - tok->buf;
126+
assert(tok->buf[size-1] == '\n');
127+
size -= 1; // Remove the newline character from the end of the line
126128
error_line = PyUnicode_DecodeUTF8(tok->buf, size, "replace");
127129
if (!error_line) {
128130
result = -1;
@@ -193,6 +195,8 @@ tokenizeriter_next(tokenizeriterobject *it)
193195
}
194196

195197
Py_ssize_t size = it->tok->inp - it->tok->buf;
198+
assert(it->tok->buf[size-1] == '\n');
199+
size -= 1; // Remove the newline character from the end of the line
196200
PyObject *line = PyUnicode_DecodeUTF8(it->tok->buf, size, "replace");
197201
if (line == NULL) {
198202
Py_DECREF(str);

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.