From 057a41eec704112c7f6380c3682d069f4da598f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Simon?= Date: Sun, 25 May 2025 18:23:38 +0200 Subject: [PATCH] gh-134582: Fix t-strings untokenize() roundtrip removing space between braces (GH-134603) (cherry picked from commit 52509cc94b1a18cb325dbfa7e5f830b32759a903) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Loïc Simon --- Lib/test/test_tokenize.py | 4 ++++ Lib/tokenize.py | 14 +++++++------- .../2025-05-23-23-43-39.gh-issue-134582.9POq3l.rst | 1 + 3 files changed, 12 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-23-23-43-39.gh-issue-134582.9POq3l.rst diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 2d41a5e5ac0697..e6b19fe1812d44 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -1975,6 +1975,10 @@ def test_roundtrip(self): for case in cases: self.check_roundtrip(case) + self.check_roundtrip(r"t'{ {}}'") + self.check_roundtrip(r"t'{f'{ {}}'}{ {}}'") + self.check_roundtrip(r"f'{t'{ {}}'}{ {}}'") + def test_continuation(self): # Balancing continuation diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 8d01fd7bce41b0..559a7aecbde2d1 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -274,7 +274,7 @@ def compat(self, token, iterable): toks_append = self.tokens.append startline = token[0] in (NEWLINE, NL) prevstring = False - in_fstring = 0 + in_fstring_or_tstring = 0 for tok in _itertools.chain([token], iterable): toknum, tokval = tok[:2] @@ -293,10 +293,10 @@ def compat(self, token, iterable): else: prevstring = False - if toknum == FSTRING_START: - in_fstring += 1 - elif toknum == FSTRING_END: - in_fstring -= 1 + if toknum in {FSTRING_START, TSTRING_START}: + in_fstring_or_tstring += 1 + elif toknum in {FSTRING_END, TSTRING_END}: + in_fstring_or_tstring -= 1 if toknum == INDENT: indents.append(tokval) continue @@ -311,8 +311,8 @@ def compat(self, token, iterable): elif toknum in {FSTRING_MIDDLE, TSTRING_MIDDLE}: tokval = self.escape_brackets(tokval) - # Insert a space between two consecutive brackets if we are in an f-string - if tokval in {"{", "}"} and self.tokens and self.tokens[-1] == tokval and in_fstring: + # Insert a space between two consecutive brackets if we are in an f-string or t-string + if tokval in {"{", "}"} and self.tokens and self.tokens[-1] == tokval and in_fstring_or_tstring: tokval = ' ' + tokval # Insert a space between two consecutive f-strings diff --git a/Misc/NEWS.d/next/Library/2025-05-23-23-43-39.gh-issue-134582.9POq3l.rst b/Misc/NEWS.d/next/Library/2025-05-23-23-43-39.gh-issue-134582.9POq3l.rst new file mode 100644 index 00000000000000..23e1d5891b685f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-23-23-43-39.gh-issue-134582.9POq3l.rst @@ -0,0 +1 @@ +Fix tokenize.untokenize() round-trip errors related to t-strings braces escaping