Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit 0c4aca5

Browse filesBrowse files
ambvambv
authored
Make rb'' strings work in lib2to3 (#1724)
This partially solves bpo-23894.
1 parent 7bac69d commit 0c4aca5
Copy full SHA for 0c4aca5

File tree

2 files changed

+37
-7
lines changed
Filter options

2 files changed

+37
-7
lines changed

‎Lib/lib2to3/pgen2/tokenize.py

Copy file name to clipboardExpand all lines: Lib/lib2to3/pgen2/tokenize.py
+16-7Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -74,10 +74,11 @@ def maybe(*choices): return group(*choices) + '?'
7474
Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''"
7575
# Tail end of """ string.
7676
Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""'
77-
Triple = group("[ubUB]?[rR]?'''", '[ubUB]?[rR]?"""')
77+
_litprefix = r"(?:[uUrRbB]|[rR][bB]|[bBuU][rR])?"
78+
Triple = group(_litprefix + "'''", _litprefix + '"""')
7879
# Single-line ' or " string.
79-
String = group(r"[uU]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*'",
80-
r'[uU]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*"')
80+
String = group(_litprefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*'",
81+
_litprefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*"')
8182

8283
# Because of leftmost-then-longest match semantics, be sure to put the
8384
# longest operators first (e.g., if = came before ==, == would get
@@ -95,9 +96,9 @@ def maybe(*choices): return group(*choices) + '?'
9596
Token = Ignore + PlainToken
9697

9798
# First (or only) line of ' or " string.
98-
ContStr = group(r"[uUbB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" +
99+
ContStr = group(_litprefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" +
99100
group("'", r'\\\r?\n'),
100-
r'[uUbB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
101+
_litprefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
101102
group('"', r'\\\r?\n'))
102103
PseudoExtras = group(r'\\\r?\n', Comment, Triple)
103104
PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
@@ -111,6 +112,7 @@ def maybe(*choices): return group(*choices) + '?'
111112
"b'''": single3prog, 'b"""': double3prog,
112113
"ur'''": single3prog, 'ur"""': double3prog,
113114
"br'''": single3prog, 'br"""': double3prog,
115+
"rb'''": single3prog, 'rb"""': double3prog,
114116
"R'''": single3prog, 'R"""': double3prog,
115117
"U'''": single3prog, 'U"""': double3prog,
116118
"B'''": single3prog, 'B"""': double3prog,
@@ -120,6 +122,9 @@ def maybe(*choices): return group(*choices) + '?'
120122
"bR'''": single3prog, 'bR"""': double3prog,
121123
"Br'''": single3prog, 'Br"""': double3prog,
122124
"BR'''": single3prog, 'BR"""': double3prog,
125+
"rB'''": single3prog, 'rB"""': double3prog,
126+
"Rb'''": single3prog, 'Rb"""': double3prog,
127+
"RB'''": single3prog, 'RB"""': double3prog,
123128
'r': None, 'R': None,
124129
'u': None, 'U': None,
125130
'b': None, 'B': None}
@@ -132,7 +137,9 @@ def maybe(*choices): return group(*choices) + '?'
132137
"ur'''", 'ur"""', "Ur'''", 'Ur"""',
133138
"uR'''", 'uR"""', "UR'''", 'UR"""',
134139
"br'''", 'br"""', "Br'''", 'Br"""',
135-
"bR'''", 'bR"""', "BR'''", 'BR"""',):
140+
"bR'''", 'bR"""', "BR'''", 'BR"""',
141+
"rb'''", 'rb"""', "Rb'''", 'Rb"""',
142+
"rB'''", 'rB"""', "RB'''", 'RB"""',):
136143
triple_quoted[t] = t
137144
single_quoted = {}
138145
for t in ("'", '"',
@@ -142,7 +149,9 @@ def maybe(*choices): return group(*choices) + '?'
142149
"ur'", 'ur"', "Ur'", 'Ur"',
143150
"uR'", 'uR"', "UR'", 'UR"',
144151
"br'", 'br"', "Br'", 'Br"',
145-
"bR'", 'bR"', "BR'", 'BR"', ):
152+
"bR'", 'bR"', "BR'", 'BR"',
153+
"rb'", 'rb"', "Rb'", 'Rb"',
154+
"rB'", 'rB"', "RB'", 'RB"',):
146155
single_quoted[t] = t
147156

148157
tabsize = 8

‎Lib/lib2to3/tests/test_parser.py

Copy file name to clipboardExpand all lines: Lib/lib2to3/tests/test_parser.py
+21Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -320,6 +320,7 @@ def test_5(self):
320320
def test_6(self):
321321
self.validate("lst: List[int] = []")
322322

323+
323324
class TestExcept(GrammarTest):
324325
def test_new(self):
325326
s = """
@@ -338,6 +339,26 @@ def test_old(self):
338339
self.validate(s)
339340

340341

342+
class TestStringLiterals(GrammarTest):
343+
prefixes = ("'", '"',
344+
"r'", 'r"', "R'", 'R"',
345+
"u'", 'u"', "U'", 'U"',
346+
"b'", 'b"', "B'", 'B"',
347+
"ur'", 'ur"', "Ur'", 'Ur"',
348+
"uR'", 'uR"', "UR'", 'UR"',
349+
"br'", 'br"', "Br'", 'Br"',
350+
"bR'", 'bR"', "BR'", 'BR"',
351+
"rb'", 'rb"', "Rb'", 'Rb"',
352+
"rB'", 'rB"', "RB'", 'RB"',)
353+
354+
def test_lit(self):
355+
for pre in self.prefixes:
356+
single = "{p}spamspamspam{s}".format(p=pre, s=pre[-1])
357+
self.validate(single)
358+
triple = "{p}{s}{s}eggs{s}{s}{s}".format(p=pre, s=pre[-1])
359+
self.validate(triple)
360+
361+
341362
# Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.testAtoms
342363
class TestSetLiteral(GrammarTest):
343364
def test_1(self):

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.