diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py index d14db60f7da89ea..213a72624d85065 100644 --- a/Lib/lib2to3/pgen2/tokenize.py +++ b/Lib/lib2to3/pgen2/tokenize.py @@ -95,9 +95,10 @@ def maybe(*choices): return group(*choices) + '?' Token = Ignore + PlainToken # First (or only) line of ' or " string. -ContStr = group(r"[uUbB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" + +_litprefix = r"(?:[uUrRbBfF]|[rR][bB]|[bBuU][rR])?" +ContStr = group(_litprefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" + group("'", r'\\\r?\n'), - r'[uUbB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' + + _litprefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*' + group('"', r'\\\r?\n')) PseudoExtras = group(r'\\\r?\n', Comment, Triple) PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name) @@ -108,11 +109,14 @@ def maybe(*choices): return group(*choices) + '?' "'''": single3prog, '"""': double3prog, "r'''": single3prog, 'r"""': double3prog, "u'''": single3prog, 'u"""': double3prog, + "f'''": single3prog, 'f"""': double3prog, "b'''": single3prog, 'b"""': double3prog, "ur'''": single3prog, 'ur"""': double3prog, "br'''": single3prog, 'br"""': double3prog, + "rb'''": single3prog, 'rb"""': double3prog, "R'''": single3prog, 'R"""': double3prog, "U'''": single3prog, 'U"""': double3prog, + "F'''": single3prog, 'F"""': double3prog, "B'''": single3prog, 'B"""': double3prog, "uR'''": single3prog, 'uR"""': double3prog, "Ur'''": single3prog, 'Ur"""': double3prog, @@ -120,29 +124,39 @@ def maybe(*choices): return group(*choices) + '?' "bR'''": single3prog, 'bR"""': double3prog, "Br'''": single3prog, 'Br"""': double3prog, "BR'''": single3prog, 'BR"""': double3prog, + "rB'''": single3prog, 'rB"""': double3prog, + "Rb'''": single3prog, 'Rb"""': double3prog, + "RB'''": single3prog, 'RB"""': double3prog, 'r': None, 'R': None, 'u': None, 'U': None, + 'f': None, 'F': None, 'b': None, 'B': None} triple_quoted = {} for t in ("'''", '"""', "r'''", 'r"""', "R'''", 'R"""', "u'''", 'u"""', "U'''", 'U"""', + "f'''", 'f"""', "F'''", 'F"""', "b'''", 'b"""', "B'''", 'B"""', "ur'''", 'ur"""', "Ur'''", 'Ur"""', "uR'''", 'uR"""', "UR'''", 'UR"""', "br'''", 'br"""', "Br'''", 'Br"""', - "bR'''", 'bR"""', "BR'''", 'BR"""',): + "bR'''", 'bR"""', "BR'''", 'BR"""', + "rb'''", 'rb"""', "Rb'''", 'Rb"""', + "rB'''", 'rB"""', "RB'''", 'RB"""',): triple_quoted[t] = t single_quoted = {} for t in ("'", '"', "r'", 'r"', "R'", 'R"', "u'", 'u"', "U'", 'U"', + "f'", 'f"', "F'", 'F"', "b'", 'b"', "B'", 'B"', "ur'", 'ur"', "Ur'", 'Ur"', "uR'", 'uR"', "UR'", 'UR"', "br'", 'br"', "Br'", 'Br"', - "bR'", 'bR"', "BR'", 'BR"', ): + "bR'", 'bR"', "BR'", 'BR"', + "rb'", 'rb"', "Rb'", 'Rb"', + "rB'", 'rB"', "RB'", 'RB"',): single_quoted[t] = t tabsize = 8 diff --git a/Misc/NEWS b/Misc/NEWS index 299a8388ff6e39d..d3a99f90fe07735 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -29,6 +29,8 @@ Library exception) to exception(s) raised in the dispatched methods. Patch by Petr Motejlek. +- bpo-23894: lib2to3 now parses rb'...' and f'...' strings. + What's New in Python 3.6.1 release candidate 1 ==============================================