From 11e98982461524e296b4a8bb3a60f4b7ef90161d Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Sat, 21 Jul 2018 16:58:15 -0600 Subject: [PATCH 01/67] Add tokenization of := - Add token to Include/token.h. Add token to documentation in Doc/library/token.rst. - Run `./python Lib/token.py` to regenerate Lib/token.py. - Update Parser/tokenizer.c: add case to handle `:=`. --- Include/token.h | 5 +- Lib/token.py | 119 +++++++++++++++++++++++++----------------------- 2 files changed, 66 insertions(+), 58 deletions(-) diff --git a/Include/token.h b/Include/token.h index 2d491e6927d1a9..f7a44126feb4a2 100644 --- a/Include/token.h +++ b/Include/token.h @@ -63,8 +63,9 @@ extern "C" { #define ATEQUAL 50 #define RARROW 51 #define ELLIPSIS 52 -#define OP 53 -#define ERRORTOKEN 54 +#define COLONEQUAL 53 +#define OP 54 +#define ERRORTOKEN 55 #define N_TOKENS 58 #define NT_OFFSET 256 diff --git a/Lib/token.py b/Lib/token.py index 5af7e6b91eaca3..c78c631162a93f 100644 --- a/Lib/token.py +++ b/Lib/token.py @@ -1,7 +1,7 @@ """Token constants.""" # Auto-generated by Tools/scripts/generate_token.py -__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF'] +__all__ = ["tok_name", "ISTERMINAL", "ISNONTERMINAL", "ISEOF"] ENDMARKER = 0 NAME = 1 @@ -56,75 +56,82 @@ ATEQUAL = 50 RARROW = 51 ELLIPSIS = 52 -OP = 53 +COLONEQUAL = 53 +# Don't forget to update the table _PyParser_TokenNames in tokenizer.c! +OP = 54 +ERRORTOKEN = 55 # These aren't used by the C tokenizer but are needed for tokenize.py -ERRORTOKEN = 54 -COMMENT = 55 -NL = 56 -ENCODING = 57 -N_TOKENS = 58 +COMMENT = 56 +NL = 57 +ENCODING = 58 +N_TOKENS = 59 # Special definitions for cooperation with parser NT_OFFSET = 256 -tok_name = {value: name - for name, value in globals().items() - if isinstance(value, int) and not name.startswith('_')} +tok_name = { + value: name + for name, value in globals().items() + if isinstance(value, int) and not name.startswith("_") +} __all__.extend(tok_name.values()) EXACT_TOKEN_TYPES = { - '!=': NOTEQUAL, - '%': PERCENT, - '%=': PERCENTEQUAL, - '&': AMPER, - '&=': AMPEREQUAL, - '(': LPAR, - ')': RPAR, - '*': STAR, - '**': DOUBLESTAR, - '**=': DOUBLESTAREQUAL, - '*=': STAREQUAL, - '+': PLUS, - '+=': PLUSEQUAL, - ',': COMMA, - '-': MINUS, - '-=': MINEQUAL, - '->': RARROW, - '.': DOT, - '...': ELLIPSIS, - '/': SLASH, - '//': DOUBLESLASH, - '//=': DOUBLESLASHEQUAL, - '/=': SLASHEQUAL, - ':': COLON, - ';': SEMI, - '<': LESS, - '<<': LEFTSHIFT, - '<<=': LEFTSHIFTEQUAL, - '<=': LESSEQUAL, - '=': EQUAL, - '==': EQEQUAL, - '>': GREATER, - '>=': GREATEREQUAL, - '>>': RIGHTSHIFT, - '>>=': RIGHTSHIFTEQUAL, - '@': AT, - '@=': ATEQUAL, - '[': LSQB, - ']': RSQB, - '^': CIRCUMFLEX, - '^=': CIRCUMFLEXEQUAL, - '{': LBRACE, - '|': VBAR, - '|=': VBAREQUAL, - '}': RBRACE, - '~': TILDE, + "!=": NOTEQUAL, + "%": PERCENT, + "%=": PERCENTEQUAL, + "&": AMPER, + "&=": AMPEREQUAL, + "(": LPAR, + ")": RPAR, + "*": STAR, + "**": DOUBLESTAR, + "**=": DOUBLESTAREQUAL, + "*=": STAREQUAL, + "+": PLUS, + "+=": PLUSEQUAL, + ",": COMMA, + "-": MINUS, + "-=": MINEQUAL, + "->": RARROW, + ".": DOT, + "...": ELLIPSIS, + "/": SLASH, + "//": DOUBLESLASH, + "//=": DOUBLESLASHEQUAL, + "/=": SLASHEQUAL, + ":": COLON, + ";": SEMI, + "<": LESS, + "<<": LEFTSHIFT, + "<<=": LEFTSHIFTEQUAL, + "<=": LESSEQUAL, + "=": EQUAL, + "==": EQEQUAL, + ">": GREATER, + ">=": GREATEREQUAL, + ">>": RIGHTSHIFT, + ">>=": RIGHTSHIFTEQUAL, + "@": AT, + "@=": ATEQUAL, + "[": LSQB, + "]": RSQB, + "^": CIRCUMFLEX, + "^=": CIRCUMFLEXEQUAL, + "{": LBRACE, + "|": VBAR, + "|=": VBAREQUAL, + "}": RBRACE, + "~": TILDE, } + def ISTERMINAL(x): return x < NT_OFFSET + def ISNONTERMINAL(x): return x >= NT_OFFSET + def ISEOF(x): return x == ENDMARKER From 9e9156c2256885466a6d3ed2a9f773d3ad38e7c7 Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Sat, 21 Jul 2018 18:06:08 -0600 Subject: [PATCH 02/67] Add initial usage of := in grammar. --- Grammar/Grammar | 3 +- Include/graminit.h | 77 ++-- Python/graminit.c | 1011 ++++++++++++++++++++++---------------------- 3 files changed, 558 insertions(+), 533 deletions(-) diff --git a/Grammar/Grammar b/Grammar/Grammar index e232df979e2dae..3831342e06f2d9 100644 --- a/Grammar/Grammar +++ b/Grammar/Grammar @@ -83,6 +83,7 @@ with_item: test ['as' expr] except_clause: 'except' [test ['as' NAME]] suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT +namedexpr_test: test [':=' test] test: or_test ['if' or_test 'else' test] | lambdef test_nocond: or_test | lambdef_nocond lambdef: 'lambda' [varargslist] ':' test @@ -108,7 +109,7 @@ atom: ('(' [yield_expr|testlist_comp] ')' | '[' [testlist_comp] ']' | '{' [dictorsetmaker] '}' | NAME | NUMBER | STRING+ | '...' | 'None' | 'True' | 'False') -testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] ) +testlist_comp: (namedexpr_test|star_expr) ( comp_for | (',' (namedexpr_test|star_expr))* [','] ) trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME subscriptlist: subscript (',' subscript)* [','] subscript: test | [test] ':' [test] [sliceop] diff --git a/Include/graminit.h b/Include/graminit.h index bdfe821ad71686..e3acff8a1e8351 100644 --- a/Include/graminit.h +++ b/Include/graminit.h @@ -49,41 +49,42 @@ #define with_item 302 #define except_clause 303 #define suite 304 -#define test 305 -#define test_nocond 306 -#define lambdef 307 -#define lambdef_nocond 308 -#define or_test 309 -#define and_test 310 -#define not_test 311 -#define comparison 312 -#define comp_op 313 -#define star_expr 314 -#define expr 315 -#define xor_expr 316 -#define and_expr 317 -#define shift_expr 318 -#define arith_expr 319 -#define term 320 -#define factor 321 -#define power 322 -#define atom_expr 323 -#define atom 324 -#define testlist_comp 325 -#define trailer 326 -#define subscriptlist 327 -#define subscript 328 -#define sliceop 329 -#define exprlist 330 -#define testlist 331 -#define dictorsetmaker 332 -#define classdef 333 -#define arglist 334 -#define argument 335 -#define comp_iter 336 -#define sync_comp_for 337 -#define comp_for 338 -#define comp_if 339 -#define encoding_decl 340 -#define yield_expr 341 -#define yield_arg 342 +#define namedexpr_test 305 +#define test 306 +#define test_nocond 307 +#define lambdef 308 +#define lambdef_nocond 309 +#define or_test 310 +#define and_test 311 +#define not_test 312 +#define comparison 313 +#define comp_op 314 +#define star_expr 315 +#define expr 316 +#define xor_expr 317 +#define and_expr 318 +#define shift_expr 319 +#define arith_expr 320 +#define term 321 +#define factor 322 +#define power 323 +#define atom_expr 324 +#define atom 325 +#define testlist_comp 326 +#define trailer 327 +#define subscriptlist 328 +#define subscript 329 +#define sliceop 330 +#define exprlist 331 +#define testlist 332 +#define dictorsetmaker 333 +#define classdef 334 +#define arglist 335 +#define argument 336 +#define comp_iter 337 +#define sync_comp_for 338 +#define comp_for 339 +#define comp_if 340 +#define encoding_decl 341 +#define yield_expr 342 +#define yield_arg 343 diff --git a/Python/graminit.c b/Python/graminit.c index 0a681f7b797f48..a6c6c749962310 100644 --- a/Python/graminit.c +++ b/Python/graminit.c @@ -1150,70 +1150,66 @@ static state states_48[5] = { {1, arcs_48_3}, {2, arcs_48_4}, }; -static arc arcs_49_0[2] = { - {112, 1}, - {113, 2}, +static arc arcs_49_0[1] = { + {26, 1}, }; static arc arcs_49_1[2] = { - {97, 3}, + {113, 2}, {0, 1}, }; static arc arcs_49_2[1] = { - {0, 2}, + {26, 3}, }; static arc arcs_49_3[1] = { - {112, 4}, -}; -static arc arcs_49_4[1] = { - {99, 5}, -}; -static arc arcs_49_5[1] = { - {26, 2}, + {0, 3}, }; -static state states_49[6] = { - {2, arcs_49_0}, +static state states_49[4] = { + {1, arcs_49_0}, {2, arcs_49_1}, {1, arcs_49_2}, {1, arcs_49_3}, - {1, arcs_49_4}, - {1, arcs_49_5}, }; static arc arcs_50_0[2] = { - {112, 1}, - {115, 1}, + {114, 1}, + {115, 2}, }; -static arc arcs_50_1[1] = { +static arc arcs_50_1[2] = { + {97, 3}, {0, 1}, }; -static state states_50[2] = { - {2, arcs_50_0}, - {1, arcs_50_1}, +static arc arcs_50_2[1] = { + {0, 2}, }; -static arc arcs_51_0[1] = { - {116, 1}, +static arc arcs_50_3[1] = { + {114, 4}, }; -static arc arcs_51_1[2] = { - {35, 2}, - {27, 3}, +static arc arcs_50_4[1] = { + {99, 5}, }; -static arc arcs_51_2[1] = { - {27, 3}, +static arc arcs_50_5[1] = { + {26, 2}, }; -static arc arcs_51_3[1] = { - {26, 4}, +static state states_50[6] = { + {2, arcs_50_0}, + {2, arcs_50_1}, + {1, arcs_50_2}, + {1, arcs_50_3}, + {1, arcs_50_4}, + {1, arcs_50_5}, +}; +static arc arcs_51_0[2] = { + {114, 1}, + {117, 1}, }; -static arc arcs_51_4[1] = { - {0, 4}, +static arc arcs_51_1[1] = { + {0, 1}, }; -static state states_51[5] = { - {1, arcs_51_0}, - {2, arcs_51_1}, - {1, arcs_51_2}, - {1, arcs_51_3}, - {1, arcs_51_4}, +static state states_51[2] = { + {2, arcs_51_0}, + {1, arcs_51_1}, }; static arc arcs_52_0[1] = { - {116, 1}, + {118, 1}, }; static arc arcs_52_1[2] = { {35, 2}, @@ -1223,7 +1219,7 @@ static arc arcs_52_2[1] = { {27, 3}, }; static arc arcs_52_3[1] = { - {114, 4}, + {26, 4}, }; static arc arcs_52_4[1] = { {0, 4}, @@ -1236,15 +1232,27 @@ static state states_52[5] = { {1, arcs_52_4}, }; static arc arcs_53_0[1] = { - {117, 1}, + {118, 1}, }; static arc arcs_53_1[2] = { - {118, 0}, - {0, 1}, + {35, 2}, + {27, 3}, +}; +static arc arcs_53_2[1] = { + {27, 3}, +}; +static arc arcs_53_3[1] = { + {116, 4}, +}; +static arc arcs_53_4[1] = { + {0, 4}, }; -static state states_53[2] = { +static state states_53[5] = { {1, arcs_53_0}, {2, arcs_53_1}, + {1, arcs_53_2}, + {1, arcs_53_3}, + {1, arcs_53_4}, }; static arc arcs_54_0[1] = { {119, 1}, @@ -1257,84 +1265,84 @@ static state states_54[2] = { {1, arcs_54_0}, {2, arcs_54_1}, }; -static arc arcs_55_0[2] = { +static arc arcs_55_0[1] = { {121, 1}, - {122, 2}, }; -static arc arcs_55_1[1] = { - {119, 2}, +static arc arcs_55_1[2] = { + {122, 0}, + {0, 1}, +}; +static state states_55[2] = { + {1, arcs_55_0}, + {2, arcs_55_1}, +}; +static arc arcs_56_0[2] = { + {123, 1}, + {124, 2}, +}; +static arc arcs_56_1[1] = { + {121, 2}, }; -static arc arcs_55_2[1] = { +static arc arcs_56_2[1] = { {0, 2}, }; -static state states_55[3] = { - {2, arcs_55_0}, - {1, arcs_55_1}, - {1, arcs_55_2}, +static state states_56[3] = { + {2, arcs_56_0}, + {1, arcs_56_1}, + {1, arcs_56_2}, }; -static arc arcs_56_0[1] = { +static arc arcs_57_0[1] = { {108, 1}, }; -static arc arcs_56_1[2] = { - {123, 0}, +static arc arcs_57_1[2] = { + {125, 0}, {0, 1}, }; -static state states_56[2] = { - {1, arcs_56_0}, - {2, arcs_56_1}, +static state states_57[2] = { + {1, arcs_57_0}, + {2, arcs_57_1}, }; -static arc arcs_57_0[10] = { - {124, 1}, - {125, 1}, +static arc arcs_58_0[10] = { {126, 1}, {127, 1}, {128, 1}, {129, 1}, {130, 1}, + {131, 1}, + {132, 1}, {102, 1}, - {121, 2}, - {131, 3}, + {123, 2}, + {133, 3}, }; -static arc arcs_57_1[1] = { +static arc arcs_58_1[1] = { {0, 1}, }; -static arc arcs_57_2[1] = { +static arc arcs_58_2[1] = { {102, 1}, }; -static arc arcs_57_3[2] = { - {121, 1}, +static arc arcs_58_3[2] = { + {123, 1}, {0, 3}, }; -static state states_57[4] = { - {10, arcs_57_0}, - {1, arcs_57_1}, - {1, arcs_57_2}, - {2, arcs_57_3}, +static state states_58[4] = { + {10, arcs_58_0}, + {1, arcs_58_1}, + {1, arcs_58_2}, + {2, arcs_58_3}, }; -static arc arcs_58_0[1] = { +static arc arcs_59_0[1] = { {33, 1}, }; -static arc arcs_58_1[1] = { +static arc arcs_59_1[1] = { {108, 2}, }; -static arc arcs_58_2[1] = { +static arc arcs_59_2[1] = { {0, 2}, }; -static state states_58[3] = { - {1, arcs_58_0}, - {1, arcs_58_1}, - {1, arcs_58_2}, -}; -static arc arcs_59_0[1] = { - {132, 1}, -}; -static arc arcs_59_1[2] = { - {133, 0}, - {0, 1}, -}; -static state states_59[2] = { +static state states_59[3] = { {1, arcs_59_0}, - {2, arcs_59_1}, + {1, arcs_59_1}, + {1, arcs_59_2}, }; static arc arcs_60_0[1] = { {134, 1}, @@ -1361,21 +1369,20 @@ static state states_61[2] = { static arc arcs_62_0[1] = { {138, 1}, }; -static arc arcs_62_1[3] = { +static arc arcs_62_1[2] = { {139, 0}, - {140, 0}, {0, 1}, }; static state states_62[2] = { {1, arcs_62_0}, - {3, arcs_62_1}, + {2, arcs_62_1}, }; static arc arcs_63_0[1] = { - {141, 1}, + {140, 1}, }; static arc arcs_63_1[3] = { + {141, 0}, {142, 0}, - {143, 0}, {0, 1}, }; static state states_63[2] = { @@ -1383,543 +1390,555 @@ static state states_63[2] = { {3, arcs_63_1}, }; static arc arcs_64_0[1] = { - {144, 1}, + {143, 1}, }; -static arc arcs_64_1[6] = { - {33, 0}, - {11, 0}, +static arc arcs_64_1[3] = { + {144, 0}, {145, 0}, - {146, 0}, - {147, 0}, {0, 1}, }; static state states_64[2] = { {1, arcs_64_0}, - {6, arcs_64_1}, + {3, arcs_64_1}, }; -static arc arcs_65_0[4] = { - {142, 1}, - {143, 1}, - {148, 1}, - {149, 2}, +static arc arcs_65_0[1] = { + {146, 1}, +}; +static arc arcs_65_1[6] = { + {33, 0}, + {11, 0}, + {147, 0}, + {148, 0}, + {149, 0}, + {0, 1}, +}; +static state states_65[2] = { + {1, arcs_65_0}, + {6, arcs_65_1}, }; -static arc arcs_65_1[1] = { - {144, 2}, +static arc arcs_66_0[4] = { + {144, 1}, + {145, 1}, + {150, 1}, + {151, 2}, }; -static arc arcs_65_2[1] = { +static arc arcs_66_1[1] = { + {146, 2}, +}; +static arc arcs_66_2[1] = { {0, 2}, }; -static state states_65[3] = { - {4, arcs_65_0}, - {1, arcs_65_1}, - {1, arcs_65_2}, +static state states_66[3] = { + {4, arcs_66_0}, + {1, arcs_66_1}, + {1, arcs_66_2}, }; -static arc arcs_66_0[1] = { - {150, 1}, +static arc arcs_67_0[1] = { + {152, 1}, }; -static arc arcs_66_1[2] = { +static arc arcs_67_1[2] = { {34, 2}, {0, 1}, }; -static arc arcs_66_2[1] = { - {144, 3}, +static arc arcs_67_2[1] = { + {146, 3}, }; -static arc arcs_66_3[1] = { +static arc arcs_67_3[1] = { {0, 3}, }; -static state states_66[4] = { - {1, arcs_66_0}, - {2, arcs_66_1}, - {1, arcs_66_2}, - {1, arcs_66_3}, +static state states_67[4] = { + {1, arcs_67_0}, + {2, arcs_67_1}, + {1, arcs_67_2}, + {1, arcs_67_3}, }; -static arc arcs_67_0[2] = { - {151, 1}, - {152, 2}, +static arc arcs_68_0[2] = { + {153, 1}, + {154, 2}, }; -static arc arcs_67_1[1] = { - {152, 2}, +static arc arcs_68_1[1] = { + {154, 2}, }; -static arc arcs_67_2[2] = { - {153, 2}, +static arc arcs_68_2[2] = { + {155, 2}, {0, 2}, }; -static state states_67[3] = { - {2, arcs_67_0}, - {1, arcs_67_1}, - {2, arcs_67_2}, +static state states_68[3] = { + {2, arcs_68_0}, + {1, arcs_68_1}, + {2, arcs_68_2}, }; -static arc arcs_68_0[10] = { +static arc arcs_69_0[10] = { {13, 1}, - {155, 2}, - {157, 3}, + {157, 2}, + {159, 3}, {23, 4}, - {160, 4}, - {161, 5}, - {83, 4}, {162, 4}, - {163, 4}, + {163, 5}, + {83, 4}, {164, 4}, + {165, 4}, + {166, 4}, }; -static arc arcs_68_1[3] = { +static arc arcs_69_1[3] = { {50, 6}, - {154, 6}, + {156, 6}, {15, 4}, }; -static arc arcs_68_2[2] = { - {154, 7}, - {156, 4}, +static arc arcs_69_2[2] = { + {156, 7}, + {158, 4}, }; -static arc arcs_68_3[2] = { - {158, 8}, - {159, 4}, +static arc arcs_69_3[2] = { + {160, 8}, + {161, 4}, }; -static arc arcs_68_4[1] = { +static arc arcs_69_4[1] = { {0, 4}, }; -static arc arcs_68_5[2] = { - {161, 5}, +static arc arcs_69_5[2] = { + {163, 5}, {0, 5}, }; -static arc arcs_68_6[1] = { +static arc arcs_69_6[1] = { {15, 4}, }; -static arc arcs_68_7[1] = { - {156, 4}, +static arc arcs_69_7[1] = { + {158, 4}, }; -static arc arcs_68_8[1] = { - {159, 4}, +static arc arcs_69_8[1] = { + {161, 4}, }; -static state states_68[9] = { - {10, arcs_68_0}, - {3, arcs_68_1}, - {2, arcs_68_2}, - {2, arcs_68_3}, - {1, arcs_68_4}, - {2, arcs_68_5}, - {1, arcs_68_6}, - {1, arcs_68_7}, - {1, arcs_68_8}, -}; -static arc arcs_69_0[2] = { - {26, 1}, +static state states_69[9] = { + {10, arcs_69_0}, + {3, arcs_69_1}, + {2, arcs_69_2}, + {2, arcs_69_3}, + {1, arcs_69_4}, + {2, arcs_69_5}, + {1, arcs_69_6}, + {1, arcs_69_7}, + {1, arcs_69_8}, +}; +static arc arcs_70_0[2] = { + {112, 1}, {51, 1}, }; -static arc arcs_69_1[3] = { - {165, 2}, +static arc arcs_70_1[3] = { + {167, 2}, {32, 3}, {0, 1}, }; -static arc arcs_69_2[1] = { +static arc arcs_70_2[1] = { {0, 2}, }; -static arc arcs_69_3[3] = { - {26, 4}, +static arc arcs_70_3[3] = { + {112, 4}, {51, 4}, {0, 3}, }; -static arc arcs_69_4[2] = { +static arc arcs_70_4[2] = { {32, 3}, {0, 4}, }; -static state states_69[5] = { - {2, arcs_69_0}, - {3, arcs_69_1}, - {1, arcs_69_2}, - {3, arcs_69_3}, - {2, arcs_69_4}, +static state states_70[5] = { + {2, arcs_70_0}, + {3, arcs_70_1}, + {1, arcs_70_2}, + {3, arcs_70_3}, + {2, arcs_70_4}, }; -static arc arcs_70_0[3] = { +static arc arcs_71_0[3] = { {13, 1}, - {155, 2}, + {157, 2}, {82, 3}, }; -static arc arcs_70_1[2] = { +static arc arcs_71_1[2] = { {14, 4}, {15, 5}, }; -static arc arcs_70_2[1] = { - {166, 6}, +static arc arcs_71_2[1] = { + {168, 6}, }; -static arc arcs_70_3[1] = { +static arc arcs_71_3[1] = { {23, 5}, }; -static arc arcs_70_4[1] = { +static arc arcs_71_4[1] = { {15, 5}, }; -static arc arcs_70_5[1] = { +static arc arcs_71_5[1] = { {0, 5}, }; -static arc arcs_70_6[1] = { - {156, 5}, +static arc arcs_71_6[1] = { + {158, 5}, }; -static state states_70[7] = { - {3, arcs_70_0}, - {2, arcs_70_1}, - {1, arcs_70_2}, - {1, arcs_70_3}, - {1, arcs_70_4}, - {1, arcs_70_5}, - {1, arcs_70_6}, +static state states_71[7] = { + {3, arcs_71_0}, + {2, arcs_71_1}, + {1, arcs_71_2}, + {1, arcs_71_3}, + {1, arcs_71_4}, + {1, arcs_71_5}, + {1, arcs_71_6}, }; -static arc arcs_71_0[1] = { - {167, 1}, +static arc arcs_72_0[1] = { + {169, 1}, }; -static arc arcs_71_1[2] = { +static arc arcs_72_1[2] = { {32, 2}, {0, 1}, }; -static arc arcs_71_2[2] = { - {167, 1}, +static arc arcs_72_2[2] = { + {169, 1}, {0, 2}, }; -static state states_71[3] = { - {1, arcs_71_0}, - {2, arcs_71_1}, - {2, arcs_71_2}, +static state states_72[3] = { + {1, arcs_72_0}, + {2, arcs_72_1}, + {2, arcs_72_2}, }; -static arc arcs_72_0[2] = { +static arc arcs_73_0[2] = { {26, 1}, {27, 2}, }; -static arc arcs_72_1[2] = { +static arc arcs_73_1[2] = { {27, 2}, {0, 1}, }; -static arc arcs_72_2[3] = { +static arc arcs_73_2[3] = { {26, 3}, - {168, 4}, + {170, 4}, {0, 2}, }; -static arc arcs_72_3[2] = { - {168, 4}, +static arc arcs_73_3[2] = { + {170, 4}, {0, 3}, }; -static arc arcs_72_4[1] = { +static arc arcs_73_4[1] = { {0, 4}, }; -static state states_72[5] = { - {2, arcs_72_0}, - {2, arcs_72_1}, - {3, arcs_72_2}, - {2, arcs_72_3}, - {1, arcs_72_4}, +static state states_73[5] = { + {2, arcs_73_0}, + {2, arcs_73_1}, + {3, arcs_73_2}, + {2, arcs_73_3}, + {1, arcs_73_4}, }; -static arc arcs_73_0[1] = { +static arc arcs_74_0[1] = { {27, 1}, }; -static arc arcs_73_1[2] = { +static arc arcs_74_1[2] = { {26, 2}, {0, 1}, }; -static arc arcs_73_2[1] = { +static arc arcs_74_2[1] = { {0, 2}, }; -static state states_73[3] = { - {1, arcs_73_0}, - {2, arcs_73_1}, - {1, arcs_73_2}, +static state states_74[3] = { + {1, arcs_74_0}, + {2, arcs_74_1}, + {1, arcs_74_2}, }; -static arc arcs_74_0[2] = { +static arc arcs_75_0[2] = { {108, 1}, {51, 1}, }; -static arc arcs_74_1[2] = { +static arc arcs_75_1[2] = { {32, 2}, {0, 1}, }; -static arc arcs_74_2[3] = { +static arc arcs_75_2[3] = { {108, 1}, {51, 1}, {0, 2}, }; -static state states_74[3] = { - {2, arcs_74_0}, - {2, arcs_74_1}, - {3, arcs_74_2}, +static state states_75[3] = { + {2, arcs_75_0}, + {2, arcs_75_1}, + {3, arcs_75_2}, }; -static arc arcs_75_0[1] = { +static arc arcs_76_0[1] = { {26, 1}, }; -static arc arcs_75_1[2] = { +static arc arcs_76_1[2] = { {32, 2}, {0, 1}, }; -static arc arcs_75_2[2] = { +static arc arcs_76_2[2] = { {26, 1}, {0, 2}, }; -static state states_75[3] = { - {1, arcs_75_0}, - {2, arcs_75_1}, - {2, arcs_75_2}, +static state states_76[3] = { + {1, arcs_76_0}, + {2, arcs_76_1}, + {2, arcs_76_2}, }; -static arc arcs_76_0[3] = { +static arc arcs_77_0[3] = { {26, 1}, {34, 2}, {51, 3}, }; -static arc arcs_76_1[4] = { +static arc arcs_77_1[4] = { {27, 4}, - {165, 5}, + {167, 5}, {32, 6}, {0, 1}, }; -static arc arcs_76_2[1] = { +static arc arcs_77_2[1] = { {108, 7}, }; -static arc arcs_76_3[3] = { - {165, 5}, +static arc arcs_77_3[3] = { + {167, 5}, {32, 6}, {0, 3}, }; -static arc arcs_76_4[1] = { +static arc arcs_77_4[1] = { {26, 7}, }; -static arc arcs_76_5[1] = { +static arc arcs_77_5[1] = { {0, 5}, }; -static arc arcs_76_6[3] = { +static arc arcs_77_6[3] = { {26, 8}, {51, 8}, {0, 6}, }; -static arc arcs_76_7[3] = { - {165, 5}, +static arc arcs_77_7[3] = { + {167, 5}, {32, 9}, {0, 7}, }; -static arc arcs_76_8[2] = { +static arc arcs_77_8[2] = { {32, 6}, {0, 8}, }; -static arc arcs_76_9[3] = { +static arc arcs_77_9[3] = { {26, 10}, {34, 11}, {0, 9}, }; -static arc arcs_76_10[1] = { +static arc arcs_77_10[1] = { {27, 12}, }; -static arc arcs_76_11[1] = { +static arc arcs_77_11[1] = { {108, 13}, }; -static arc arcs_76_12[1] = { +static arc arcs_77_12[1] = { {26, 13}, }; -static arc arcs_76_13[2] = { +static arc arcs_77_13[2] = { {32, 9}, {0, 13}, }; -static state states_76[14] = { - {3, arcs_76_0}, - {4, arcs_76_1}, - {1, arcs_76_2}, - {3, arcs_76_3}, - {1, arcs_76_4}, - {1, arcs_76_5}, - {3, arcs_76_6}, - {3, arcs_76_7}, - {2, arcs_76_8}, - {3, arcs_76_9}, - {1, arcs_76_10}, - {1, arcs_76_11}, - {1, arcs_76_12}, - {2, arcs_76_13}, -}; -static arc arcs_77_0[1] = { - {169, 1}, +static state states_77[14] = { + {3, arcs_77_0}, + {4, arcs_77_1}, + {1, arcs_77_2}, + {3, arcs_77_3}, + {1, arcs_77_4}, + {1, arcs_77_5}, + {3, arcs_77_6}, + {3, arcs_77_7}, + {2, arcs_77_8}, + {3, arcs_77_9}, + {1, arcs_77_10}, + {1, arcs_77_11}, + {1, arcs_77_12}, + {2, arcs_77_13}, }; -static arc arcs_77_1[1] = { +static arc arcs_78_0[1] = { + {171, 1}, +}; +static arc arcs_78_1[1] = { {23, 2}, }; -static arc arcs_77_2[2] = { +static arc arcs_78_2[2] = { {13, 3}, {27, 4}, }; -static arc arcs_77_3[2] = { +static arc arcs_78_3[2] = { {14, 5}, {15, 6}, }; -static arc arcs_77_4[1] = { +static arc arcs_78_4[1] = { {28, 7}, }; -static arc arcs_77_5[1] = { +static arc arcs_78_5[1] = { {15, 6}, }; -static arc arcs_77_6[1] = { +static arc arcs_78_6[1] = { {27, 4}, }; -static arc arcs_77_7[1] = { +static arc arcs_78_7[1] = { {0, 7}, }; -static state states_77[8] = { - {1, arcs_77_0}, - {1, arcs_77_1}, - {2, arcs_77_2}, - {2, arcs_77_3}, - {1, arcs_77_4}, - {1, arcs_77_5}, - {1, arcs_77_6}, - {1, arcs_77_7}, +static state states_78[8] = { + {1, arcs_78_0}, + {1, arcs_78_1}, + {2, arcs_78_2}, + {2, arcs_78_3}, + {1, arcs_78_4}, + {1, arcs_78_5}, + {1, arcs_78_6}, + {1, arcs_78_7}, }; -static arc arcs_78_0[1] = { - {170, 1}, +static arc arcs_79_0[1] = { + {172, 1}, }; -static arc arcs_78_1[2] = { +static arc arcs_79_1[2] = { {32, 2}, {0, 1}, }; -static arc arcs_78_2[2] = { - {170, 1}, +static arc arcs_79_2[2] = { + {172, 1}, {0, 2}, }; -static state states_78[3] = { - {1, arcs_78_0}, - {2, arcs_78_1}, - {2, arcs_78_2}, +static state states_79[3] = { + {1, arcs_79_0}, + {2, arcs_79_1}, + {2, arcs_79_2}, }; -static arc arcs_79_0[3] = { +static arc arcs_80_0[3] = { {26, 1}, {34, 2}, {33, 2}, }; -static arc arcs_79_1[3] = { - {165, 3}, +static arc arcs_80_1[3] = { + {167, 3}, {31, 2}, {0, 1}, }; -static arc arcs_79_2[1] = { +static arc arcs_80_2[1] = { {26, 3}, }; -static arc arcs_79_3[1] = { +static arc arcs_80_3[1] = { {0, 3}, }; -static state states_79[4] = { - {3, arcs_79_0}, - {3, arcs_79_1}, - {1, arcs_79_2}, - {1, arcs_79_3}, +static state states_80[4] = { + {3, arcs_80_0}, + {3, arcs_80_1}, + {1, arcs_80_2}, + {1, arcs_80_3}, }; -static arc arcs_80_0[2] = { - {165, 1}, - {172, 1}, +static arc arcs_81_0[2] = { + {167, 1}, + {174, 1}, }; -static arc arcs_80_1[1] = { +static arc arcs_81_1[1] = { {0, 1}, }; -static state states_80[2] = { - {2, arcs_80_0}, - {1, arcs_80_1}, +static state states_81[2] = { + {2, arcs_81_0}, + {1, arcs_81_1}, }; -static arc arcs_81_0[1] = { +static arc arcs_82_0[1] = { {101, 1}, }; -static arc arcs_81_1[1] = { +static arc arcs_82_1[1] = { {66, 2}, }; -static arc arcs_81_2[1] = { +static arc arcs_82_2[1] = { {102, 3}, }; -static arc arcs_81_3[1] = { - {112, 4}, +static arc arcs_82_3[1] = { + {114, 4}, }; -static arc arcs_81_4[2] = { - {171, 5}, +static arc arcs_82_4[2] = { + {173, 5}, {0, 4}, }; -static arc arcs_81_5[1] = { +static arc arcs_82_5[1] = { {0, 5}, }; -static state states_81[6] = { - {1, arcs_81_0}, - {1, arcs_81_1}, - {1, arcs_81_2}, - {1, arcs_81_3}, - {2, arcs_81_4}, - {1, arcs_81_5}, +static state states_82[6] = { + {1, arcs_82_0}, + {1, arcs_82_1}, + {1, arcs_82_2}, + {1, arcs_82_3}, + {2, arcs_82_4}, + {1, arcs_82_5}, }; -static arc arcs_82_0[2] = { +static arc arcs_83_0[2] = { {21, 1}, - {173, 2}, + {175, 2}, }; -static arc arcs_82_1[1] = { - {173, 2}, +static arc arcs_83_1[1] = { + {175, 2}, }; -static arc arcs_82_2[1] = { +static arc arcs_83_2[1] = { {0, 2}, }; -static state states_82[3] = { - {2, arcs_82_0}, - {1, arcs_82_1}, - {1, arcs_82_2}, +static state states_83[3] = { + {2, arcs_83_0}, + {1, arcs_83_1}, + {1, arcs_83_2}, }; -static arc arcs_83_0[1] = { +static arc arcs_84_0[1] = { {97, 1}, }; -static arc arcs_83_1[1] = { - {114, 2}, +static arc arcs_84_1[1] = { + {116, 2}, }; -static arc arcs_83_2[2] = { - {171, 3}, +static arc arcs_84_2[2] = { + {173, 3}, {0, 2}, }; -static arc arcs_83_3[1] = { +static arc arcs_84_3[1] = { {0, 3}, }; -static state states_83[4] = { - {1, arcs_83_0}, - {1, arcs_83_1}, - {2, arcs_83_2}, - {1, arcs_83_3}, +static state states_84[4] = { + {1, arcs_84_0}, + {1, arcs_84_1}, + {2, arcs_84_2}, + {1, arcs_84_3}, }; -static arc arcs_84_0[1] = { +static arc arcs_85_0[1] = { {23, 1}, }; -static arc arcs_84_1[1] = { +static arc arcs_85_1[1] = { {0, 1}, }; -static state states_84[2] = { - {1, arcs_84_0}, - {1, arcs_84_1}, +static state states_85[2] = { + {1, arcs_85_0}, + {1, arcs_85_1}, }; -static arc arcs_85_0[1] = { - {175, 1}, +static arc arcs_86_0[1] = { + {177, 1}, }; -static arc arcs_85_1[2] = { - {176, 2}, +static arc arcs_86_1[2] = { + {178, 2}, {0, 1}, }; -static arc arcs_85_2[1] = { +static arc arcs_86_2[1] = { {0, 2}, }; -static state states_85[3] = { - {1, arcs_85_0}, - {2, arcs_85_1}, - {1, arcs_85_2}, +static state states_86[3] = { + {1, arcs_86_0}, + {2, arcs_86_1}, + {1, arcs_86_2}, }; -static arc arcs_86_0[2] = { +static arc arcs_87_0[2] = { {77, 1}, {47, 2}, }; -static arc arcs_86_1[1] = { +static arc arcs_87_1[1] = { {26, 2}, }; -static arc arcs_86_2[1] = { +static arc arcs_87_2[1] = { {0, 2}, }; -static state states_86[3] = { - {2, arcs_86_0}, - {1, arcs_86_1}, - {1, arcs_86_2}, +static state states_87[3] = { + {2, arcs_87_0}, + {1, arcs_87_1}, + {1, arcs_87_2}, }; -static dfa dfas[87] = { +static dfa dfas[88] = { {256, "single_input", 0, 3, states_0, - "\004\050\340\000\002\000\000\000\012\076\011\007\262\004\020\002\000\300\220\050\037\202\000"}, + "\004\050\340\000\002\000\000\000\012\076\011\007\262\004\100\010\000\000\103\242\174\010\002"}, {257, "file_input", 0, 2, states_1, - "\204\050\340\000\002\000\000\000\012\076\011\007\262\004\020\002\000\300\220\050\037\202\000"}, + "\204\050\340\000\002\000\000\000\012\076\011\007\262\004\100\010\000\000\103\242\174\010\002"}, {258, "eval_input", 0, 3, states_2, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\020\002\000\300\220\050\037\000\000"}, + "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\100\010\000\000\103\242\174\000\000"}, {259, "decorator", 0, 7, states_3, "\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {260, "decorators", 0, 2, states_4, @@ -1941,17 +1960,17 @@ static dfa dfas[87] = { {268, "vfpdef", 0, 2, states_12, "\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {269, "stmt", 0, 2, states_13, - "\000\050\340\000\002\000\000\000\012\076\011\007\262\004\020\002\000\300\220\050\037\202\000"}, + "\000\050\340\000\002\000\000\000\012\076\011\007\262\004\100\010\000\000\103\242\174\010\002"}, {270, "simple_stmt", 0, 4, states_14, - "\000\040\200\000\002\000\000\000\012\076\011\007\000\000\020\002\000\300\220\050\037\200\000"}, + "\000\040\200\000\002\000\000\000\012\076\011\007\000\000\100\010\000\000\103\242\174\000\002"}, {271, "small_stmt", 0, 2, states_15, - "\000\040\200\000\002\000\000\000\012\076\011\007\000\000\020\002\000\300\220\050\037\200\000"}, + "\000\040\200\000\002\000\000\000\012\076\011\007\000\000\100\010\000\000\103\242\174\000\002"}, {272, "expr_stmt", 0, 6, states_16, - "\000\040\200\000\002\000\000\000\000\000\010\000\000\000\020\002\000\300\220\050\037\000\000"}, + "\000\040\200\000\002\000\000\000\000\000\010\000\000\000\100\010\000\000\103\242\174\000\000"}, {273, "annassign", 0, 5, states_17, "\000\000\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {274, "testlist_star_expr", 0, 3, states_18, - "\000\040\200\000\002\000\000\000\000\000\010\000\000\000\020\002\000\300\220\050\037\000\000"}, + "\000\040\200\000\002\000\000\000\000\000\010\000\000\000\100\010\000\000\103\242\174\000\000"}, {275, "augassign", 0, 2, states_19, "\000\000\000\000\000\000\360\377\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {276, "del_stmt", 0, 3, states_20, @@ -1959,7 +1978,7 @@ static dfa dfas[87] = { {277, "pass_stmt", 0, 2, states_21, "\000\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {278, "flow_stmt", 0, 2, states_22, - "\000\000\000\000\000\000\000\000\000\036\000\000\000\000\000\000\000\000\000\000\000\200\000"}, + "\000\000\000\000\000\000\000\000\000\036\000\000\000\000\000\000\000\000\000\000\000\000\002"}, {279, "break_stmt", 0, 2, states_23, "\000\000\000\000\000\000\000\000\000\002\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {280, "continue_stmt", 0, 2, states_24, @@ -1967,7 +1986,7 @@ static dfa dfas[87] = { {281, "return_stmt", 0, 3, states_25, "\000\000\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {282, "yield_stmt", 0, 2, states_26, - "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\200\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\002"}, {283, "raise_stmt", 0, 5, states_27, "\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {284, "import_stmt", 0, 2, states_28, @@ -1993,7 +2012,7 @@ static dfa dfas[87] = { {294, "assert_stmt", 0, 5, states_38, "\000\000\000\000\000\000\000\000\000\000\000\004\000\000\000\000\000\000\000\000\000\000\000"}, {295, "compound_stmt", 0, 2, states_39, - "\000\010\140\000\000\000\000\000\000\000\000\000\262\004\000\000\000\000\000\000\000\002\000"}, + "\000\010\140\000\000\000\000\000\000\000\000\000\262\004\000\000\000\000\000\000\000\010\000"}, {296, "async_stmt", 0, 3, states_40, "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {297, "if_stmt", 0, 8, states_41, @@ -2007,89 +2026,91 @@ static dfa dfas[87] = { {301, "with_stmt", 0, 5, states_45, "\000\000\000\000\000\000\000\000\000\000\000\000\000\004\000\000\000\000\000\000\000\000\000"}, {302, "with_item", 0, 4, states_46, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\020\002\000\300\220\050\037\000\000"}, + "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\100\010\000\000\103\242\174\000\000"}, {303, "except_clause", 0, 5, states_47, "\000\000\000\000\000\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\000"}, {304, "suite", 0, 5, states_48, - "\004\040\200\000\002\000\000\000\012\076\011\007\000\000\020\002\000\300\220\050\037\200\000"}, - {305, "test", 0, 6, states_49, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\020\002\000\300\220\050\037\000\000"}, - {306, "test_nocond", 0, 2, states_50, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\020\002\000\300\220\050\037\000\000"}, - {307, "lambdef", 0, 5, states_51, - "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000"}, - {308, "lambdef_nocond", 0, 5, states_52, - "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000"}, - {309, "or_test", 0, 2, states_53, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\002\000\300\220\050\037\000\000"}, - {310, "and_test", 0, 2, states_54, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\002\000\300\220\050\037\000\000"}, - {311, "not_test", 0, 3, states_55, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\002\000\300\220\050\037\000\000"}, - {312, "comparison", 0, 2, states_56, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\300\220\050\037\000\000"}, - {313, "comp_op", 0, 4, states_57, - "\000\000\000\000\000\000\000\000\000\000\000\000\100\000\000\362\017\000\000\000\000\000\000"}, - {314, "star_expr", 0, 3, states_58, + "\004\040\200\000\002\000\000\000\012\076\011\007\000\000\100\010\000\000\103\242\174\000\002"}, + {305, "namedexpr_test", 0, 4, states_49, + "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\100\010\000\000\103\242\174\000\000"}, + {306, "test", 0, 6, states_50, + "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\100\010\000\000\103\242\174\000\000"}, + {307, "test_nocond", 0, 2, states_51, + "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\100\010\000\000\103\242\174\000\000"}, + {308, "lambdef", 0, 5, states_52, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000"}, + {309, "lambdef_nocond", 0, 5, states_53, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000"}, + {310, "or_test", 0, 2, states_54, + "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\010\000\000\103\242\174\000\000"}, + {311, "and_test", 0, 2, states_55, + "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\010\000\000\103\242\174\000\000"}, + {312, "not_test", 0, 3, states_56, + "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\010\000\000\103\242\174\000\000"}, + {313, "comparison", 0, 2, states_57, + "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\103\242\174\000\000"}, + {314, "comp_op", 0, 4, states_58, + "\000\000\000\000\000\000\000\000\000\000\000\000\100\000\000\310\077\000\000\000\000\000\000"}, + {315, "star_expr", 0, 3, states_59, "\000\000\000\000\002\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, - {315, "expr", 0, 2, states_59, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\300\220\050\037\000\000"}, - {316, "xor_expr", 0, 2, states_60, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\300\220\050\037\000\000"}, - {317, "and_expr", 0, 2, states_61, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\300\220\050\037\000\000"}, - {318, "shift_expr", 0, 2, states_62, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\300\220\050\037\000\000"}, - {319, "arith_expr", 0, 2, states_63, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\300\220\050\037\000\000"}, - {320, "term", 0, 2, states_64, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\300\220\050\037\000\000"}, - {321, "factor", 0, 3, states_65, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\300\220\050\037\000\000"}, - {322, "power", 0, 4, states_66, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\200\050\037\000\000"}, - {323, "atom_expr", 0, 3, states_67, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\200\050\037\000\000"}, - {324, "atom", 0, 9, states_68, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\000\050\037\000\000"}, - {325, "testlist_comp", 0, 5, states_69, - "\000\040\200\000\002\000\000\000\000\000\010\000\000\000\020\002\000\300\220\050\037\000\000"}, - {326, "trailer", 0, 7, states_70, - "\000\040\000\000\000\000\000\000\000\000\004\000\000\000\000\000\000\000\000\010\000\000\000"}, - {327, "subscriptlist", 0, 3, states_71, - "\000\040\200\010\000\000\000\000\000\000\010\000\000\000\020\002\000\300\220\050\037\000\000"}, - {328, "subscript", 0, 5, states_72, - "\000\040\200\010\000\000\000\000\000\000\010\000\000\000\020\002\000\300\220\050\037\000\000"}, - {329, "sliceop", 0, 3, states_73, + {316, "expr", 0, 2, states_60, + "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\103\242\174\000\000"}, + {317, "xor_expr", 0, 2, states_61, + "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\103\242\174\000\000"}, + {318, "and_expr", 0, 2, states_62, + "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\103\242\174\000\000"}, + {319, "shift_expr", 0, 2, states_63, + "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\103\242\174\000\000"}, + {320, "arith_expr", 0, 2, states_64, + "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\103\242\174\000\000"}, + {321, "term", 0, 2, states_65, + "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\103\242\174\000\000"}, + {322, "factor", 0, 3, states_66, + "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\103\242\174\000\000"}, + {323, "power", 0, 4, states_67, + "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\000\242\174\000\000"}, + {324, "atom_expr", 0, 3, states_68, + "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\000\242\174\000\000"}, + {325, "atom", 0, 9, states_69, + "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\000\240\174\000\000"}, + {326, "testlist_comp", 0, 5, states_70, + "\000\040\200\000\002\000\000\000\000\000\010\000\000\000\100\010\000\000\103\242\174\000\000"}, + {327, "trailer", 0, 7, states_71, + "\000\040\000\000\000\000\000\000\000\000\004\000\000\000\000\000\000\000\000\040\000\000\000"}, + {328, "subscriptlist", 0, 3, states_72, + "\000\040\200\010\000\000\000\000\000\000\010\000\000\000\100\010\000\000\103\242\174\000\000"}, + {329, "subscript", 0, 5, states_73, + "\000\040\200\010\000\000\000\000\000\000\010\000\000\000\100\010\000\000\103\242\174\000\000"}, + {330, "sliceop", 0, 3, states_74, "\000\000\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, - {330, "exprlist", 0, 3, states_74, - "\000\040\200\000\002\000\000\000\000\000\010\000\000\000\000\000\000\300\220\050\037\000\000"}, - {331, "testlist", 0, 3, states_75, - "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\020\002\000\300\220\050\037\000\000"}, - {332, "dictorsetmaker", 0, 14, states_76, - "\000\040\200\000\006\000\000\000\000\000\010\000\000\000\020\002\000\300\220\050\037\000\000"}, - {333, "classdef", 0, 8, states_77, - "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\002\000"}, - {334, "arglist", 0, 3, states_78, - "\000\040\200\000\006\000\000\000\000\000\010\000\000\000\020\002\000\300\220\050\037\000\000"}, - {335, "argument", 0, 4, states_79, - "\000\040\200\000\006\000\000\000\000\000\010\000\000\000\020\002\000\300\220\050\037\000\000"}, - {336, "comp_iter", 0, 2, states_80, + {331, "exprlist", 0, 3, states_75, + "\000\040\200\000\002\000\000\000\000\000\010\000\000\000\000\000\000\000\103\242\174\000\000"}, + {332, "testlist", 0, 3, states_76, + "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\100\010\000\000\103\242\174\000\000"}, + {333, "dictorsetmaker", 0, 14, states_77, + "\000\040\200\000\006\000\000\000\000\000\010\000\000\000\100\010\000\000\103\242\174\000\000"}, + {334, "classdef", 0, 8, states_78, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\010\000"}, + {335, "arglist", 0, 3, states_79, + "\000\040\200\000\006\000\000\000\000\000\010\000\000\000\100\010\000\000\103\242\174\000\000"}, + {336, "argument", 0, 4, states_80, + "\000\040\200\000\006\000\000\000\000\000\010\000\000\000\100\010\000\000\103\242\174\000\000"}, + {337, "comp_iter", 0, 2, states_81, "\000\000\040\000\000\000\000\000\000\000\000\000\042\000\000\000\000\000\000\000\000\000\000"}, - {337, "sync_comp_for", 0, 6, states_81, + {338, "sync_comp_for", 0, 6, states_82, "\000\000\000\000\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\000\000"}, - {338, "comp_for", 0, 3, states_82, + {339, "comp_for", 0, 3, states_83, "\000\000\040\000\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\000\000"}, - {339, "comp_if", 0, 4, states_83, + {340, "comp_if", 0, 4, states_84, "\000\000\000\000\000\000\000\000\000\000\000\000\002\000\000\000\000\000\000\000\000\000\000"}, - {340, "encoding_decl", 0, 2, states_84, + {341, "encoding_decl", 0, 2, states_85, "\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, - {341, "yield_expr", 0, 3, states_85, - "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\200\000"}, - {342, "yield_arg", 0, 3, states_86, - "\000\040\200\000\002\000\000\000\000\040\010\000\000\000\020\002\000\300\220\050\037\000\000"}, + {342, "yield_expr", 0, 3, states_86, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\002"}, + {343, "yield_arg", 0, 3, states_87, + "\000\040\200\000\002\000\000\000\000\040\010\000\000\000\100\010\000\000\103\242\174\000\000"}, }; -static label labels[177] = { +static label labels[179] = { {0, "EMPTY"}, {256, 0}, {4, 0}, @@ -2099,16 +2120,16 @@ static label labels[177] = { {269, 0}, {0, 0}, {258, 0}, - {331, 0}, + {332, 0}, {259, 0}, {49, 0}, {291, 0}, {7, 0}, - {334, 0}, + {335, 0}, {8, 0}, {260, 0}, {261, 0}, - {333, 0}, + {334, 0}, {263, 0}, {262, 0}, {1, "async"}, @@ -2116,7 +2137,7 @@ static label labels[177] = { {1, 0}, {264, 0}, {51, 0}, - {305, 0}, + {306, 0}, {11, 0}, {304, 0}, {265, 0}, @@ -2140,8 +2161,8 @@ static label labels[177] = { {274, 0}, {273, 0}, {275, 0}, - {341, 0}, - {314, 0}, + {342, 0}, + {315, 0}, {36, 0}, {37, 0}, {38, 0}, @@ -2156,7 +2177,7 @@ static label labels[177] = { {46, 0}, {48, 0}, {1, "del"}, - {330, 0}, + {331, 0}, {1, "pass"}, {279, 0}, {280, 0}, @@ -2198,22 +2219,24 @@ static label labels[177] = { {1, "finally"}, {1, "with"}, {302, 0}, - {315, 0}, + {316, 0}, {1, "except"}, {5, 0}, {6, 0}, - {309, 0}, - {307, 0}, - {306, 0}, + {305, 0}, + {53, 0}, + {310, 0}, {308, 0}, + {307, 0}, + {309, 0}, {1, "lambda"}, - {310, 0}, - {1, "or"}, {311, 0}, + {1, "or"}, + {312, 0}, {1, "and"}, {1, "not"}, - {312, 0}, {313, 0}, + {314, 0}, {20, 0}, {21, 0}, {27, 0}, @@ -2222,55 +2245,55 @@ static label labels[177] = { {28, 0}, {28, 0}, {1, "is"}, - {316, 0}, - {18, 0}, {317, 0}, - {32, 0}, + {18, 0}, {318, 0}, - {19, 0}, + {32, 0}, {319, 0}, + {19, 0}, + {320, 0}, {33, 0}, {34, 0}, - {320, 0}, + {321, 0}, {14, 0}, {15, 0}, - {321, 0}, + {322, 0}, {17, 0}, {24, 0}, {47, 0}, {31, 0}, - {322, 0}, {323, 0}, - {1, "await"}, {324, 0}, - {326, 0}, + {1, "await"}, {325, 0}, + {327, 0}, + {326, 0}, {9, 0}, {10, 0}, {25, 0}, - {332, 0}, + {333, 0}, {26, 0}, {2, 0}, {3, 0}, {1, "None"}, {1, "True"}, {1, "False"}, - {338, 0}, - {327, 0}, + {339, 0}, {328, 0}, {329, 0}, + {330, 0}, {1, "class"}, - {335, 0}, {336, 0}, - {339, 0}, {337, 0}, {340, 0}, + {338, 0}, + {341, 0}, {1, "yield"}, - {342, 0}, + {343, 0}, }; grammar _PyParser_Grammar = { - 87, + 88, dfas, - {177, labels}, + {179, labels}, 256 }; From 6eb766d7a908543d9e9a36fb4e5fe6155c8b2f06 Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Sat, 21 Jul 2018 19:18:15 -0600 Subject: [PATCH 03/67] Update Python.asdl to match the grammar updates. Regenerated Include/Python-ast.h and Python/Python-ast.c --- Include/Python-ast.h | 28 +++++++------- Parser/Python.asdl | 1 + Python/Python-ast.c | 87 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 102 insertions(+), 14 deletions(-) diff --git a/Include/Python-ast.h b/Include/Python-ast.h index f8394e6c26ad83..5cf814f70fe269 100644 --- a/Include/Python-ast.h +++ b/Include/Python-ast.h @@ -3,7 +3,8 @@ #ifndef Py_PYTHON_AST_H #define Py_PYTHON_AST_H #ifdef __cplusplus -extern "C" { +extern "C" +{ #endif #include "asdl.h" @@ -440,11 +441,11 @@ struct _withitem { // Note: these macros affect function definitions, not only call sites. #define Module(a0, a1) _Py_Module(a0, a1) -mod_ty _Py_Module(asdl_seq * body, PyArena *arena); + mod_ty _Py_Module(asdl_seq *body, PyArena *arena); #define Interactive(a0, a1) _Py_Interactive(a0, a1) -mod_ty _Py_Interactive(asdl_seq * body, PyArena *arena); + mod_ty _Py_Interactive(asdl_seq *body, PyArena *arena); #define Expression(a0, a1) _Py_Expression(a0, a1) -mod_ty _Py_Expression(expr_ty body, PyArena *arena); + mod_ty _Py_Expression(expr_ty body, PyArena *arena); #define Suite(a0, a1) _Py_Suite(a0, a1) mod_ty _Py_Suite(asdl_seq * body, PyArena *arena); #define FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) _Py_FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) @@ -630,11 +631,11 @@ expr_ty _Py_Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); #define Slice(a0, a1, a2, a3) _Py_Slice(a0, a1, a2, a3) -slice_ty _Py_Slice(expr_ty lower, expr_ty upper, expr_ty step, PyArena *arena); + slice_ty _Py_Slice(expr_ty lower, expr_ty upper, expr_ty step, PyArena *arena); #define ExtSlice(a0, a1) _Py_ExtSlice(a0, a1) -slice_ty _Py_ExtSlice(asdl_seq * dims, PyArena *arena); + slice_ty _Py_ExtSlice(asdl_seq *dims, PyArena *arena); #define Index(a0, a1) _Py_Index(a0, a1) -slice_ty _Py_Index(expr_ty value, PyArena *arena); + slice_ty _Py_Index(expr_ty value, PyArena *arena); #define comprehension(a0, a1, a2, a3, a4) _Py_comprehension(a0, a1, a2, a3, a4) comprehension_ty _Py_comprehension(expr_ty target, expr_ty iter, asdl_seq * ifs, int is_async, PyArena *arena); @@ -651,16 +652,15 @@ arguments_ty _Py_arguments(asdl_seq * args, arg_ty vararg, asdl_seq * arg_ty _Py_arg(identifier arg, expr_ty annotation, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); #define keyword(a0, a1, a2) _Py_keyword(a0, a1, a2) -keyword_ty _Py_keyword(identifier arg, expr_ty value, PyArena *arena); + keyword_ty _Py_keyword(identifier arg, expr_ty value, PyArena *arena); #define alias(a0, a1, a2) _Py_alias(a0, a1, a2) -alias_ty _Py_alias(identifier name, identifier asname, PyArena *arena); + alias_ty _Py_alias(identifier name, identifier asname, PyArena *arena); #define withitem(a0, a1, a2) _Py_withitem(a0, a1, a2) -withitem_ty _Py_withitem(expr_ty context_expr, expr_ty optional_vars, PyArena - *arena); + withitem_ty _Py_withitem(expr_ty context_expr, expr_ty optional_vars, PyArena *arena); -PyObject* PyAST_mod2obj(mod_ty t); -mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode); -int PyAST_Check(PyObject* obj); + PyObject *PyAST_mod2obj(mod_ty t); + mod_ty PyAST_obj2mod(PyObject *ast, PyArena *arena, int mode); + int PyAST_Check(PyObject *obj); #ifdef __cplusplus } diff --git a/Parser/Python.asdl b/Parser/Python.asdl index cedf37a2d9f9ee..b15899a7a0f3c1 100644 --- a/Parser/Python.asdl +++ b/Parser/Python.asdl @@ -54,6 +54,7 @@ module Python -- BoolOp() can use left & right? expr = BoolOp(boolop op, expr* values) + | NamedExpr(expr target, expr value) | BinOp(expr left, operator op, expr right) | UnaryOp(unaryop op, expr operand) | Lambda(arguments args, expr body) diff --git a/Python/Python-ast.c b/Python/Python-ast.c index e6c5bfe9b29e00..854ffe2117f646 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -203,6 +203,11 @@ static char *BoolOp_fields[]={ "op", "values", }; +static PyTypeObject *NamedExpr_type; +static char *NamedExpr_fields[]={ + "target", + "value", +}; static PyTypeObject *BinOp_type; _Py_IDENTIFIER(left); _Py_IDENTIFIER(right); @@ -872,6 +877,8 @@ static int init_types(void) if (!add_attributes(expr_type, expr_attributes, 4)) return 0; BoolOp_type = make_type("BoolOp", expr_type, BoolOp_fields, 2); if (!BoolOp_type) return 0; + NamedExpr_type = make_type("NamedExpr", expr_type, NamedExpr_fields, 2); + if (!NamedExpr_type) return 0; BinOp_type = make_type("BinOp", expr_type, BinOp_fields, 3); if (!BinOp_type) return 0; UnaryOp_type = make_type("UnaryOp", expr_type, UnaryOp_fields, 2); @@ -1772,6 +1779,32 @@ BoolOp(boolop_ty op, asdl_seq * values, int lineno, int col_offset, int return p; } +expr_ty +NamedExpr(expr_ty target, expr_ty value, int lineno, int col_offset, PyArena + *arena) +{ + expr_ty p; + if (!target) { + PyErr_SetString(PyExc_ValueError, + "field target is required for NamedExpr"); + return NULL; + } + if (!value) { + PyErr_SetString(PyExc_ValueError, + "field value is required for NamedExpr"); + return NULL; + } + p = (expr_ty)PyArena_Malloc(arena, sizeof(*p)); + if (!p) + return NULL; + p->kind = NamedExpr_kind; + p->v.NamedExpr.target = target; + p->v.NamedExpr.value = value; + p->lineno = lineno; + p->col_offset = col_offset; + return p; +} + expr_ty BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena) @@ -3062,6 +3095,20 @@ ast2obj_expr(void* _o) goto failed; Py_DECREF(value); break; + case NamedExpr_kind: + result = PyType_GenericNew(NamedExpr_type, NULL, NULL); + if (!result) goto failed; + value = ast2obj_expr(o->v.NamedExpr.target); + if (!value) goto failed; + if (_PyObject_SetAttrId(result, &PyId_target, value) == -1) + goto failed; + Py_DECREF(value); + value = ast2obj_expr(o->v.NamedExpr.value); + if (!value) goto failed; + if (_PyObject_SetAttrId(result, &PyId_value, value) == -1) + goto failed; + Py_DECREF(value); + break; case BinOp_kind: result = PyType_GenericNew(BinOp_type, NULL, NULL); if (!result) goto failed; @@ -5895,6 +5942,44 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) if (*out == NULL) goto failed; return 0; } + isinstance = PyObject_IsInstance(obj, (PyObject*)NamedExpr_type); + if (isinstance == -1) { + return 1; + } + if (isinstance) { + expr_ty target; + expr_ty value; + + if (_PyObject_LookupAttrId(obj, &PyId_target, &tmp) < 0) { + return 1; + } + if (tmp == NULL) { + PyErr_SetString(PyExc_TypeError, "required field \"target\" missing from NamedExpr"); + return 1; + } + else { + int res; + res = obj2ast_expr(tmp, &target, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + if (_PyObject_LookupAttrId(obj, &PyId_value, &tmp) < 0) { + return 1; + } + if (tmp == NULL) { + PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from NamedExpr"); + return 1; + } + else { + int res; + res = obj2ast_expr(tmp, &value, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + *out = NamedExpr(target, value, lineno, col_offset, arena); + if (*out == NULL) goto failed; + return 0; + } isinstance = PyObject_IsInstance(obj, (PyObject*)BinOp_type); if (isinstance == -1) { return 1; @@ -8251,6 +8336,8 @@ PyInit__ast(void) if (PyDict_SetItemString(d, "expr", (PyObject*)expr_type) < 0) return NULL; if (PyDict_SetItemString(d, "BoolOp", (PyObject*)BoolOp_type) < 0) return NULL; + if (PyDict_SetItemString(d, "NamedExpr", (PyObject*)NamedExpr_type) < 0) + return NULL; if (PyDict_SetItemString(d, "BinOp", (PyObject*)BinOp_type) < 0) return NULL; if (PyDict_SetItemString(d, "UnaryOp", (PyObject*)UnaryOp_type) < 0) return From 4f42586b696cdfda954a619b7370cfd7d4065736 Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Sun, 22 Jul 2018 16:05:01 -0600 Subject: [PATCH 04/67] Update AST and compiler files in Python/ast.c and Python/compile.c. Basic functionality, this isn't scoped properly --- Python/ast.c | 26 ++++++++++++++++++++++++++ Python/compile.c | 5 +++++ 2 files changed, 31 insertions(+) diff --git a/Python/ast.c b/Python/ast.c index 855acca29e7c3c..4ac44a6ee03efe 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -1691,6 +1691,26 @@ ast_for_decorated(struct compiling *c, const node *n) return thing; } +static expr_ty +ast_for_namedexpr(struct compiling *c, const node *n) +{ + /* namedexpr_test: test [':=' test] */ + expr_ty target, value; + + target = ast_for_expr(c, CHILD(n, 0)); + if (!target) + return NULL; + + value = ast_for_expr(c, CHILD(n, 2)); + if (!value) + return NULL; + + if(!set_context(c, target, Store, n)) + return NULL; + + return NamedExpr(target, value, LINENO(n), n->n_col_offset, c->c_arena); +} + static expr_ty ast_for_lambdef(struct compiling *c, const node *n) { @@ -2568,6 +2588,7 @@ static expr_ty ast_for_expr(struct compiling *c, const node *n) { /* handle the full range of simple expressions + namedexpr_test: test [':=' test] test: or_test ['if' or_test 'else' test] | lambdef test_nocond: or_test | lambdef_nocond or_test: and_test ('or' and_test)* @@ -2591,6 +2612,11 @@ ast_for_expr(struct compiling *c, const node *n) loop: switch (TYPE(n)) { + case namedexpr_test: + // If there are 3 children, continue with named expression + // XXX write better desc of why + if (NCH(n) == 3) + return ast_for_namedexpr(c, n); case test: case test_nocond: if (TYPE(CHILD(n, 0)) == lambdef || diff --git a/Python/compile.c b/Python/compile.c index 9713bfc9e9b737..a102fd15be792a 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -4569,6 +4569,11 @@ static int compiler_visit_expr1(struct compiler *c, expr_ty e) { switch (e->kind) { + case NamedExpr_kind: + VISIT(c, expr, e->v.NamedExpr.value); + ADDOP(c, DUP_TOP); + VISIT(c, expr, e->v.NamedExpr.target); + break; case BoolOp_kind: return compiler_boolop(c, e); case BinOp_kind: From e049ffaf806e647b2f2d2173aa456071cbf51218 Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Sun, 22 Jul 2018 16:53:39 -0600 Subject: [PATCH 05/67] Regenerate Lib/symbol.py using `./python Lib/symbol.py` --- Lib/symbol.py | 77 ++++++++++++++++++++++++++------------------------- 1 file changed, 39 insertions(+), 38 deletions(-) diff --git a/Lib/symbol.py b/Lib/symbol.py index 40d0ed1e035507..b3fa08984d87b1 100644 --- a/Lib/symbol.py +++ b/Lib/symbol.py @@ -61,44 +61,45 @@ with_item = 302 except_clause = 303 suite = 304 -test = 305 -test_nocond = 306 -lambdef = 307 -lambdef_nocond = 308 -or_test = 309 -and_test = 310 -not_test = 311 -comparison = 312 -comp_op = 313 -star_expr = 314 -expr = 315 -xor_expr = 316 -and_expr = 317 -shift_expr = 318 -arith_expr = 319 -term = 320 -factor = 321 -power = 322 -atom_expr = 323 -atom = 324 -testlist_comp = 325 -trailer = 326 -subscriptlist = 327 -subscript = 328 -sliceop = 329 -exprlist = 330 -testlist = 331 -dictorsetmaker = 332 -classdef = 333 -arglist = 334 -argument = 335 -comp_iter = 336 -sync_comp_for = 337 -comp_for = 338 -comp_if = 339 -encoding_decl = 340 -yield_expr = 341 -yield_arg = 342 +namedexpr_test = 305 +test = 306 +test_nocond = 307 +lambdef = 308 +lambdef_nocond = 309 +or_test = 310 +and_test = 311 +not_test = 312 +comparison = 313 +comp_op = 314 +star_expr = 315 +expr = 316 +xor_expr = 317 +and_expr = 318 +shift_expr = 319 +arith_expr = 320 +term = 321 +factor = 322 +power = 323 +atom_expr = 324 +atom = 325 +testlist_comp = 326 +trailer = 327 +subscriptlist = 328 +subscript = 329 +sliceop = 330 +exprlist = 331 +testlist = 332 +dictorsetmaker = 333 +classdef = 334 +arglist = 335 +argument = 336 +comp_iter = 337 +sync_comp_for = 338 +comp_for = 339 +comp_if = 340 +encoding_decl = 341 +yield_expr = 342 +yield_arg = 343 #--end constants-- sym_name = {} From 0b6bd95811cca350ddd5c197868672625fc2f3ee Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Sun, 22 Jul 2018 16:58:12 -0600 Subject: [PATCH 06/67] Tests - Fix failing tests in test_parser.py due to changes in token numbers for internal representation --- Lib/test/test_parser.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_parser.py b/Lib/test/test_parser.py index 9b58bb93c81c9d..e5d43a015699c0 100644 --- a/Lib/test/test_parser.py +++ b/Lib/test/test_parser.py @@ -694,16 +694,16 @@ def test_missing_import_source(self): def test_illegal_encoding(self): # Illegal encoding declaration tree = \ - (340, + (341, (257, (0, ''))) self.check_bad_tree(tree, "missed encoding") tree = \ - (340, + (341, (257, (0, '')), b'iso-8859-1') self.check_bad_tree(tree, "non-string encoding") tree = \ - (340, + (341, (257, (0, '')), '\udcff') with self.assertRaises(UnicodeEncodeError): From 24e95fd3cb78ac633aaf549cc14489117fcb2332 Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Sun, 22 Jul 2018 19:06:01 -0600 Subject: [PATCH 07/67] Tests - Add simple test for := token --- Lib/test/test_tokenize.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 04a12542c6ae25..4c90092893a224 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -1429,6 +1429,7 @@ def test_exact_type(self): self.assertExactTypeEqual('**=', token.DOUBLESTAREQUAL) self.assertExactTypeEqual('//', token.DOUBLESLASH) self.assertExactTypeEqual('//=', token.DOUBLESLASHEQUAL) + self.assertExactTypeEqual(':=', token.COLONEQUAL) self.assertExactTypeEqual('...', token.ELLIPSIS) self.assertExactTypeEqual('->', token.RARROW) self.assertExactTypeEqual('@', token.AT) From d31135f78c47ef67e38b004d118cb1306e315150 Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Sun, 22 Jul 2018 19:07:21 -0600 Subject: [PATCH 08/67] Tests - Add simple tests for named expressions using expr and suite --- Lib/test/test_parser.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_parser.py b/Lib/test/test_parser.py index e5d43a015699c0..4a5028a323712e 100644 --- a/Lib/test/test_parser.py +++ b/Lib/test/test_parser.py @@ -14,7 +14,6 @@ # class RoundtripLegalSyntaxTestCase(unittest.TestCase): - def roundtrip(self, f, s): st1 = f(s) t = st1.totuple() @@ -115,6 +114,7 @@ def test_expressions(self): self.check_expr("foo * bar") self.check_expr("foo / bar") self.check_expr("foo // bar") + self.check_expr("(foo := 1)") self.check_expr("lambda: 0") self.check_expr("lambda x: 0") self.check_expr("lambda *y: 0") @@ -421,6 +421,8 @@ def test_dict_comprehensions(self): self.check_expr('{x**2:x[3] for x in seq if condition(x)}') self.check_expr('{x:x for x in seq1 for y in seq2 if condition(x, y)}') + def test_named_expressions(self): + self.check_suite("(a := 1)") # # Second, we take *invalid* trees and make sure we get ParserError From e19a900057a9162377ce89e8f322eae7d31bbbf5 Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Sun, 22 Jul 2018 19:24:01 -0600 Subject: [PATCH 09/67] Tests - Update number of levels for nested expressions to prevent stack overflow --- Lib/test/test_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_parser.py b/Lib/test/test_parser.py index 4a5028a323712e..94a487d1763d77 100644 --- a/Lib/test/test_parser.py +++ b/Lib/test/test_parser.py @@ -779,7 +779,7 @@ def _nested_expression(self, level): def test_deeply_nested_list(self): # XXX used to be 99 levels in 2.x - e = self._nested_expression(93) + e = self._nested_expression(88) st = parser.expr(e) st.compile() From aca4858d36ed07a3f86a665a941ea4333bdeb572 Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Thu, 9 Aug 2018 19:47:23 -0600 Subject: [PATCH 10/67] Update symbol table to handle NamedExpr --- Python/symtable.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Python/symtable.c b/Python/symtable.c index 677b6043438ea7..e2d44f16854d7c 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -1376,6 +1376,10 @@ symtable_visit_expr(struct symtable *st, expr_ty e) VISIT_QUIT(st, 0); } switch (e->kind) { + case NamedExpr_kind: + VISIT(st, expr, e->v.NamedExpr.value); + VISIT(st, expr, e->v.NamedExpr.target); + break; case BoolOp_kind: VISIT_SEQ(st, expr, e->v.BoolOp.values); break; From 2c7b01e801e146ddcb4b0226adae2d5be597d22b Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Fri, 10 Aug 2018 15:18:36 -0600 Subject: [PATCH 11/67] Update Grammar to allow assignment expressions in if statements. Regenerate Python/graminit.c accordingly using `make regen-grammar` --- Grammar/Grammar | 2 +- Python/graminit.c | 96 +++++++++++++++++++++++------------------------ 2 files changed, 49 insertions(+), 49 deletions(-) diff --git a/Grammar/Grammar b/Grammar/Grammar index 3831342e06f2d9..74f646b58ed8a9 100644 --- a/Grammar/Grammar +++ b/Grammar/Grammar @@ -69,7 +69,7 @@ assert_stmt: 'assert' test [',' test] compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt async_stmt: 'async' (funcdef | with_stmt | for_stmt) -if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] +if_stmt: 'if' namedexpr_test ':' suite ('elif' namedexpr_test ':' suite)* ['else' ':' suite] while_stmt: 'while' test ':' suite ['else' ':' suite] for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] try_stmt: ('try' ':' suite diff --git a/Python/graminit.c b/Python/graminit.c index a6c6c749962310..283afd831dac8f 100644 --- a/Python/graminit.c +++ b/Python/graminit.c @@ -891,7 +891,7 @@ static arc arcs_41_0[1] = { {97, 1}, }; static arc arcs_41_1[1] = { - {26, 2}, + {98, 2}, }; static arc arcs_41_2[1] = { {27, 3}, @@ -900,8 +900,8 @@ static arc arcs_41_3[1] = { {28, 4}, }; static arc arcs_41_4[3] = { - {98, 1}, - {99, 5}, + {99, 1}, + {100, 5}, {0, 4}, }; static arc arcs_41_5[1] = { @@ -924,7 +924,7 @@ static state states_41[8] = { {1, arcs_41_7}, }; static arc arcs_42_0[1] = { - {100, 1}, + {101, 1}, }; static arc arcs_42_1[1] = { {26, 2}, @@ -936,7 +936,7 @@ static arc arcs_42_3[1] = { {28, 4}, }; static arc arcs_42_4[2] = { - {99, 5}, + {100, 5}, {0, 4}, }; static arc arcs_42_5[1] = { @@ -959,13 +959,13 @@ static state states_42[8] = { {1, arcs_42_7}, }; static arc arcs_43_0[1] = { - {101, 1}, + {102, 1}, }; static arc arcs_43_1[1] = { {66, 2}, }; static arc arcs_43_2[1] = { - {102, 3}, + {103, 3}, }; static arc arcs_43_3[1] = { {9, 4}, @@ -977,7 +977,7 @@ static arc arcs_43_5[1] = { {28, 6}, }; static arc arcs_43_6[2] = { - {99, 7}, + {100, 7}, {0, 6}, }; static arc arcs_43_7[1] = { @@ -1002,7 +1002,7 @@ static state states_43[10] = { {1, arcs_43_9}, }; static arc arcs_44_0[1] = { - {103, 1}, + {104, 1}, }; static arc arcs_44_1[1] = { {27, 2}, @@ -1011,8 +1011,8 @@ static arc arcs_44_2[1] = { {28, 3}, }; static arc arcs_44_3[2] = { - {104, 4}, - {105, 5}, + {105, 4}, + {106, 5}, }; static arc arcs_44_4[1] = { {27, 6}, @@ -1027,9 +1027,9 @@ static arc arcs_44_7[1] = { {28, 9}, }; static arc arcs_44_8[4] = { - {104, 4}, - {99, 10}, - {105, 5}, + {105, 4}, + {100, 10}, + {106, 5}, {0, 8}, }; static arc arcs_44_9[1] = { @@ -1042,7 +1042,7 @@ static arc arcs_44_11[1] = { {28, 12}, }; static arc arcs_44_12[2] = { - {105, 5}, + {106, 5}, {0, 12}, }; static state states_44[13] = { @@ -1061,10 +1061,10 @@ static state states_44[13] = { {2, arcs_44_12}, }; static arc arcs_45_0[1] = { - {106, 1}, + {107, 1}, }; static arc arcs_45_1[1] = { - {107, 2}, + {108, 2}, }; static arc arcs_45_2[2] = { {32, 1}, @@ -1091,7 +1091,7 @@ static arc arcs_46_1[2] = { {0, 1}, }; static arc arcs_46_2[1] = { - {108, 3}, + {109, 3}, }; static arc arcs_46_3[1] = { {0, 3}, @@ -1103,7 +1103,7 @@ static state states_46[4] = { {1, arcs_46_3}, }; static arc arcs_47_0[1] = { - {109, 1}, + {110, 1}, }; static arc arcs_47_1[2] = { {26, 2}, @@ -1134,14 +1134,14 @@ static arc arcs_48_1[1] = { {0, 1}, }; static arc arcs_48_2[1] = { - {110, 3}, + {111, 3}, }; static arc arcs_48_3[1] = { {6, 4}, }; static arc arcs_48_4[2] = { {6, 4}, - {111, 1}, + {112, 1}, }; static state states_48[5] = { {2, arcs_48_0}, @@ -1184,7 +1184,7 @@ static arc arcs_50_3[1] = { {114, 4}, }; static arc arcs_50_4[1] = { - {99, 5}, + {100, 5}, }; static arc arcs_50_5[1] = { {26, 2}, @@ -1292,7 +1292,7 @@ static state states_56[3] = { {1, arcs_56_2}, }; static arc arcs_57_0[1] = { - {108, 1}, + {109, 1}, }; static arc arcs_57_1[2] = { {125, 0}, @@ -1310,7 +1310,7 @@ static arc arcs_58_0[10] = { {130, 1}, {131, 1}, {132, 1}, - {102, 1}, + {103, 1}, {123, 2}, {133, 3}, }; @@ -1318,7 +1318,7 @@ static arc arcs_58_1[1] = { {0, 1}, }; static arc arcs_58_2[1] = { - {102, 1}, + {103, 1}, }; static arc arcs_58_3[2] = { {123, 1}, @@ -1334,7 +1334,7 @@ static arc arcs_59_0[1] = { {33, 1}, }; static arc arcs_59_1[1] = { - {108, 2}, + {109, 2}, }; static arc arcs_59_2[1] = { {0, 2}, @@ -1521,7 +1521,7 @@ static state states_69[9] = { {1, arcs_69_8}, }; static arc arcs_70_0[2] = { - {112, 1}, + {98, 1}, {51, 1}, }; static arc arcs_70_1[3] = { @@ -1533,7 +1533,7 @@ static arc arcs_70_2[1] = { {0, 2}, }; static arc arcs_70_3[3] = { - {112, 4}, + {98, 4}, {51, 4}, {0, 3}, }; @@ -1640,7 +1640,7 @@ static state states_74[3] = { {1, arcs_74_2}, }; static arc arcs_75_0[2] = { - {108, 1}, + {109, 1}, {51, 1}, }; static arc arcs_75_1[2] = { @@ -1648,7 +1648,7 @@ static arc arcs_75_1[2] = { {0, 1}, }; static arc arcs_75_2[3] = { - {108, 1}, + {109, 1}, {51, 1}, {0, 2}, }; @@ -1685,7 +1685,7 @@ static arc arcs_77_1[4] = { {0, 1}, }; static arc arcs_77_2[1] = { - {108, 7}, + {109, 7}, }; static arc arcs_77_3[3] = { {167, 5}, @@ -1721,7 +1721,7 @@ static arc arcs_77_10[1] = { {27, 12}, }; static arc arcs_77_11[1] = { - {108, 13}, + {109, 13}, }; static arc arcs_77_12[1] = { {26, 13}, @@ -1832,13 +1832,13 @@ static state states_81[2] = { {1, arcs_81_1}, }; static arc arcs_82_0[1] = { - {101, 1}, + {102, 1}, }; static arc arcs_82_1[1] = { {66, 2}, }; static arc arcs_82_2[1] = { - {102, 3}, + {103, 3}, }; static arc arcs_82_3[1] = { {114, 4}, @@ -1934,9 +1934,9 @@ static state states_87[3] = { }; static dfa dfas[88] = { {256, "single_input", 0, 3, states_0, - "\004\050\340\000\002\000\000\000\012\076\011\007\262\004\100\010\000\000\103\242\174\010\002"}, + "\004\050\340\000\002\000\000\000\012\076\011\007\142\011\100\010\000\000\103\242\174\010\002"}, {257, "file_input", 0, 2, states_1, - "\204\050\340\000\002\000\000\000\012\076\011\007\262\004\100\010\000\000\103\242\174\010\002"}, + "\204\050\340\000\002\000\000\000\012\076\011\007\142\011\100\010\000\000\103\242\174\010\002"}, {258, "eval_input", 0, 3, states_2, "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\100\010\000\000\103\242\174\000\000"}, {259, "decorator", 0, 7, states_3, @@ -1960,7 +1960,7 @@ static dfa dfas[88] = { {268, "vfpdef", 0, 2, states_12, "\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {269, "stmt", 0, 2, states_13, - "\000\050\340\000\002\000\000\000\012\076\011\007\262\004\100\010\000\000\103\242\174\010\002"}, + "\000\050\340\000\002\000\000\000\012\076\011\007\142\011\100\010\000\000\103\242\174\010\002"}, {270, "simple_stmt", 0, 4, states_14, "\000\040\200\000\002\000\000\000\012\076\011\007\000\000\100\010\000\000\103\242\174\000\002"}, {271, "small_stmt", 0, 2, states_15, @@ -2012,23 +2012,23 @@ static dfa dfas[88] = { {294, "assert_stmt", 0, 5, states_38, "\000\000\000\000\000\000\000\000\000\000\000\004\000\000\000\000\000\000\000\000\000\000\000"}, {295, "compound_stmt", 0, 2, states_39, - "\000\010\140\000\000\000\000\000\000\000\000\000\262\004\000\000\000\000\000\000\000\010\000"}, + "\000\010\140\000\000\000\000\000\000\000\000\000\142\011\000\000\000\000\000\000\000\010\000"}, {296, "async_stmt", 0, 3, states_40, "\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {297, "if_stmt", 0, 8, states_41, "\000\000\000\000\000\000\000\000\000\000\000\000\002\000\000\000\000\000\000\000\000\000\000"}, {298, "while_stmt", 0, 8, states_42, - "\000\000\000\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000"}, - {299, "for_stmt", 0, 10, states_43, "\000\000\000\000\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\000\000"}, + {299, "for_stmt", 0, 10, states_43, + "\000\000\000\000\000\000\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000\000\000"}, {300, "try_stmt", 0, 13, states_44, - "\000\000\000\000\000\000\000\000\000\000\000\000\200\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\000\000"}, {301, "with_stmt", 0, 5, states_45, - "\000\000\000\000\000\000\000\000\000\000\000\000\000\004\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\000\000"}, {302, "with_item", 0, 4, states_46, "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\100\010\000\000\103\242\174\000\000"}, {303, "except_clause", 0, 5, states_47, - "\000\000\000\000\000\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000\000"}, {304, "suite", 0, 5, states_48, "\004\040\200\000\002\000\000\000\012\076\011\007\000\000\100\010\000\000\103\242\174\000\002"}, {305, "namedexpr_test", 0, 4, states_49, @@ -2050,7 +2050,7 @@ static dfa dfas[88] = { {313, "comparison", 0, 2, states_57, "\000\040\200\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\103\242\174\000\000"}, {314, "comp_op", 0, 4, states_58, - "\000\000\000\000\000\000\000\000\000\000\000\000\100\000\000\310\077\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\200\000\000\310\077\000\000\000\000\000\000"}, {315, "star_expr", 0, 3, states_59, "\000\000\000\000\002\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {316, "expr", 0, 2, states_60, @@ -2096,11 +2096,11 @@ static dfa dfas[88] = { {336, "argument", 0, 4, states_80, "\000\040\200\000\006\000\000\000\000\000\010\000\000\000\100\010\000\000\103\242\174\000\000"}, {337, "comp_iter", 0, 2, states_81, - "\000\000\040\000\000\000\000\000\000\000\000\000\042\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\040\000\000\000\000\000\000\000\000\000\102\000\000\000\000\000\000\000\000\000\000"}, {338, "sync_comp_for", 0, 6, states_82, - "\000\000\000\000\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000\000\000"}, {339, "comp_for", 0, 3, states_83, - "\000\000\040\000\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\000\000"}, + "\000\000\040\000\000\000\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000\000\000"}, {340, "comp_if", 0, 4, states_84, "\000\000\000\000\000\000\000\000\000\000\000\000\002\000\000\000\000\000\000\000\000\000\000"}, {341, "encoding_decl", 0, 2, states_85, @@ -2209,6 +2209,7 @@ static label labels[179] = { {301, 0}, {296, 0}, {1, "if"}, + {305, 0}, {1, "elif"}, {1, "else"}, {1, "while"}, @@ -2223,7 +2224,6 @@ static label labels[179] = { {1, "except"}, {5, 0}, {6, 0}, - {305, 0}, {53, 0}, {310, 0}, {308, 0}, From f0dad890a7d832fab42793cf29fb64daee6180a1 Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Fri, 31 Aug 2018 14:06:49 -0600 Subject: [PATCH 12/67] Tests - Add additional tests for named expressions in RoundtripLegalSyntaxTestCase, based on examples and information directly from PEP 572 Note: failing tests are currently commented out (4 out of 24 tests currently fail) --- Lib/test/test_parser.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/Lib/test/test_parser.py b/Lib/test/test_parser.py index 94a487d1763d77..a451ce339fcbb4 100644 --- a/Lib/test/test_parser.py +++ b/Lib/test/test_parser.py @@ -423,6 +423,34 @@ def test_dict_comprehensions(self): def test_named_expressions(self): self.check_suite("(a := 1)") + self.check_suite("if (match := pattern.search(data)) is None: pass") + self.check_suite("[y := f(x), y**2, y**3]") + self.check_suite("filtered_data = [y for x in data if (y := f(x)) is None]") + self.check_suite("(y := f(x))") + self.check_suite("0 = (y1 := f(x))") + self.check_suite("foo(x=(y := f(x)))") + self.check_suite("def foo(answer=(p := 42)): pass") + self.check_suite("def foo(answer: (p := 42) = 5): pass") + self.check_suite("lambda: (x := 1)") + self.check_suite("(x := lambda: 1)") + self.check_suite("(x := lambda: (y := 1))") # not in PEP + self.check_suite("lambda line: (m := re.match(pattern, line)) and m.group(1)") + self.check_suite("(x := 0)") + self.check_suite("x = (y := 0)") + self.check_suite("(z:=(y:=(x:=0)))") + self.check_suite("(info := (name, phone, *rest))") + self.check_suite("(x:=1,2)") + self.check_suite("(total := total + tax)") + # self.check_suite("len(lines := f.readlines())") + # self.check_suite("foo(x := 3, cat='vector')") + # self.check_suite("foo(cat=(category := 'vector'))") + # self.check_suite("if any(len(longline := l) >= 100 for l in lines): print(longline)") + self.check_suite( + "if env_base := os.environ.get('PYTHONUSERBASE', None): return env_base" + ) + self.check_suite( + "if self._is_special and (ans := self._check_nans(context=context)): return ans" + ) # # Second, we take *invalid* trees and make sure we get ParserError From fbea15fe2bae1e1f729cc370a8df95ea63bdea27 Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Mon, 10 Sep 2018 10:59:57 -0600 Subject: [PATCH 13/67] Tests - Add temporary syntax test failure tests in test_parser.py Note: There is an outstanding TODO for this -- syntax tests need to be moved to a different file (presumably test_syntax.py), but this is covering what needs to be tested at the moment, and it's more convenient to run a single test for the time being --- Lib/test/test_parser.py | 37 +++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_parser.py b/Lib/test/test_parser.py index a451ce339fcbb4..bf18e7a54fc9c7 100644 --- a/Lib/test/test_parser.py +++ b/Lib/test/test_parser.py @@ -453,12 +453,45 @@ def test_named_expressions(self): ) # -# Second, we take *invalid* trees and make sure we get ParserError +# Second, we take *invalid* syntax and make sure we get SyntaxError # rejections for them. +# TODO move to a more appropriate file # -class IllegalSyntaxTestCase(unittest.TestCase): +class RoundtripIllegalSyntaxTestCase(unittest.TestCase): + def syntax_check(self, s): + """ + 1. Generate a syntax tree from the source code + 2. Store the tuple representation of the syntax tree + 3. Round trip the syntax tree from the tuple representation + """ + try: + compile(s, filename="", mode="exec") + except SyntaxError as e: + pass + else: + self.fail("invalid syntax - syntax_check should have failed") + + def test_invalid_named_expressions(self): + self.syntax_check("x := 0") + self.syntax_check("x = y := 0") + self.syntax_check("foo(cat=category := 'vector')") + self.syntax_check("y := f(x)") + self.syntax_check("y0 = y1 := f(x)") + self.syntax_check("foo(x = y := f(x))") + self.syntax_check("def foo(answer = p := 42): pass") + self.syntax_check("def foo(answer: p := 42 = 5): pass") + self.syntax_check("(lambda: x := 1)") + self.syntax_check("z := y := x := 0") # not in PEP + self.syntax_check("loc:=(x,y))") + self.syntax_check("info := (name, phone, *rest)") +# +# Third, we take *invalid* trees and make sure we get ParserError +# rejections for them. +# + +class IllegalSyntaxTestCase(unittest.TestCase): def check_bad_tree(self, tree, label): try: parser.sequence2st(tree) From 3cd271bc670f1f6949fbc09da2763c9c79b9db1e Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Mon, 10 Sep 2018 12:03:45 -0600 Subject: [PATCH 14/67] Add support for allowing assignment expressions as function argument annotations. Uncomment tests for these cases because they all pass now! --- Grammar/Grammar | 1 + Lib/test/test_parser.py | 8 ++++---- Python/graminit.c | 5 +++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Grammar/Grammar b/Grammar/Grammar index 74f646b58ed8a9..f21fa113643223 100644 --- a/Grammar/Grammar +++ b/Grammar/Grammar @@ -135,6 +135,7 @@ arglist: argument (',' argument)* [','] # multiple (test comp_for) arguments are blocked; keyword unpackings # that precede iterable unpackings are blocked; etc. argument: ( test [comp_for] | + test ':=' test | test '=' test | '**' test | '*' test ) diff --git a/Lib/test/test_parser.py b/Lib/test/test_parser.py index bf18e7a54fc9c7..87edfa7eab6f20 100644 --- a/Lib/test/test_parser.py +++ b/Lib/test/test_parser.py @@ -441,10 +441,10 @@ def test_named_expressions(self): self.check_suite("(info := (name, phone, *rest))") self.check_suite("(x:=1,2)") self.check_suite("(total := total + tax)") - # self.check_suite("len(lines := f.readlines())") - # self.check_suite("foo(x := 3, cat='vector')") - # self.check_suite("foo(cat=(category := 'vector'))") - # self.check_suite("if any(len(longline := l) >= 100 for l in lines): print(longline)") + self.check_suite("len(lines := f.readlines())") + self.check_suite("foo(x := 3, cat='vector')") + self.check_suite("foo(cat=(category := 'vector'))") + self.check_suite("if any(len(longline := l) >= 100 for l in lines): print(longline)") self.check_suite( "if env_base := os.environ.get('PYTHONUSERBASE', None): return env_base" ) diff --git a/Python/graminit.c b/Python/graminit.c index 283afd831dac8f..91092f1e0b9ea2 100644 --- a/Python/graminit.c +++ b/Python/graminit.c @@ -1803,8 +1803,9 @@ static arc arcs_80_0[3] = { {34, 2}, {33, 2}, }; -static arc arcs_80_1[3] = { +static arc arcs_80_1[4] = { {167, 3}, + {113, 2}, {31, 2}, {0, 1}, }; @@ -1816,7 +1817,7 @@ static arc arcs_80_3[1] = { }; static state states_80[4] = { {3, arcs_80_0}, - {3, arcs_80_1}, + {4, arcs_80_1}, {1, arcs_80_2}, {1, arcs_80_3}, }; From 7b34033ccd15e0cb924eb93e1007bc4091d18cb6 Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Mon, 10 Sep 2018 12:59:19 -0600 Subject: [PATCH 15/67] Tests - Move existing syntax tests out of test_parser.py and into test_named_expressions.py. Refactor syntax tests to use unittest --- Lib/test/test_named_expressions.py | 75 ++++++++++++++++++++++++++++++ Lib/test/test_parser.py | 36 +------------- 2 files changed, 76 insertions(+), 35 deletions(-) create mode 100644 Lib/test/test_named_expressions.py diff --git a/Lib/test/test_named_expressions.py b/Lib/test/test_named_expressions.py new file mode 100644 index 00000000000000..d96b7fd8f6d610 --- /dev/null +++ b/Lib/test/test_named_expressions.py @@ -0,0 +1,75 @@ + +import unittest + +class NamedExpressionSyntaxTest(unittest.TestCase): + + def test_named_expression_syntax_01(self): + code = '''x = y := 0''' + + with self.assertRaisesRegex(SyntaxError, 'invalid syntax'): + exec(code, {}, {}) + + def test_named_expression_syntax_02(self): + code = '''foo(cat=category := 'vector')''' + + with self.assertRaisesRegex(SyntaxError, 'invalid syntax'): + exec(code, {}, {}) + + def test_named_expression_syntax_03(self): + code = '''y := f(x)''' + + with self.assertRaisesRegex(SyntaxError, 'invalid syntax'): + exec(code, {}, {}) + + def test_named_expression_syntax_04(self): + code = '''y0 = y1 := f(x)''' + + with self.assertRaisesRegex(SyntaxError, 'invalid syntax'): + exec(code, {}, {}) + + def test_named_expression_syntax_05(self): + code = '''foo(x = y := f(x))''' + + with self.assertRaisesRegex(SyntaxError, 'invalid syntax'): + exec(code, {}, {}) + + def test_named_expression_syntax_06(self): + code = '''def foo(answer = p := 42): pass''' + + with self.assertRaisesRegex(SyntaxError, 'invalid syntax'): + exec(code, {}, {}) + + def test_named_expression_syntax_07(self): + code = '''def foo(answer: p := 42 = 5): pass''' + + with self.assertRaisesRegex(SyntaxError, 'invalid syntax'): + exec(code, {}, {}) + + def test_named_expression_syntax_08(self): + code = '''lambda: x := 1)''' + + with self.assertRaisesRegex(SyntaxError, 'invalid syntax'): + exec(code, {}, {}) + + def test_named_expression_syntax_09(self): + code = '''z := y := x := 0") # not in P''' + + with self.assertRaisesRegex(SyntaxError, 'invalid syntax'): + exec(code, {}, {}) + + def test_named_expression_syntax_10(self): + code = '''loc:=(x,y))''' + + with self.assertRaisesRegex(SyntaxError, 'invalid syntax'): + exec(code, {}, {}) + + def test_named_expression_syntax_11(self): + code = '''info := (name, phone, *rest)''' + + with self.assertRaisesRegex(SyntaxError, 'invalid syntax'): + exec(code, {}, {}) + + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_parser.py b/Lib/test/test_parser.py index 87edfa7eab6f20..09282b26011331 100644 --- a/Lib/test/test_parser.py +++ b/Lib/test/test_parser.py @@ -453,41 +453,7 @@ def test_named_expressions(self): ) # -# Second, we take *invalid* syntax and make sure we get SyntaxError -# rejections for them. -# TODO move to a more appropriate file -# - -class RoundtripIllegalSyntaxTestCase(unittest.TestCase): - def syntax_check(self, s): - """ - 1. Generate a syntax tree from the source code - 2. Store the tuple representation of the syntax tree - 3. Round trip the syntax tree from the tuple representation - """ - try: - compile(s, filename="", mode="exec") - except SyntaxError as e: - pass - else: - self.fail("invalid syntax - syntax_check should have failed") - - def test_invalid_named_expressions(self): - self.syntax_check("x := 0") - self.syntax_check("x = y := 0") - self.syntax_check("foo(cat=category := 'vector')") - self.syntax_check("y := f(x)") - self.syntax_check("y0 = y1 := f(x)") - self.syntax_check("foo(x = y := f(x))") - self.syntax_check("def foo(answer = p := 42): pass") - self.syntax_check("def foo(answer: p := 42 = 5): pass") - self.syntax_check("(lambda: x := 1)") - self.syntax_check("z := y := x := 0") # not in PEP - self.syntax_check("loc:=(x,y))") - self.syntax_check("info := (name, phone, *rest)") - -# -# Third, we take *invalid* trees and make sure we get ParserError +# Second, we take *invalid* trees and make sure we get ParserError # rejections for them. # From 13671f7a041f99628ed4fdd4dbcd0b187905ad6a Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Mon, 10 Sep 2018 18:31:31 -0700 Subject: [PATCH 16/67] Add TargetScopeError exception to extend SyntaxError Note: This simply creates the TargetScopeError exception, it is not yet used anywhere --- Objects/exceptions.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 35e1df3ca1faef..5a48d0440bb33f 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -1520,6 +1520,13 @@ MiddlingExtendsException(PyExc_SyntaxError, IndentationError, SyntaxError, "Improper indentation."); +/* + * TargetScopeError extends SyntaxError + */ +MiddlingExtendsException(PyExc_SyntaxError, TargetScopeError, SyntaxError, + "Improper scope target."); + + /* * TabError extends IndentationError */ From 5e4ba891258cb0978ed22551097af950772c5c9f Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Tue, 11 Sep 2018 09:55:59 -0700 Subject: [PATCH 17/67] Tests - Update tests per PEP 572 Continue refactoring test suite: The named expression test suite now checks for any invalid cases that throw exceptions (no longer limited to SyntaxErrors), assignment tests to ensure that variables are properly assigned, and scope tests to ensure that variable availability and values are correct Note: - There are still tests that are marked to skip, as they are not yet implemented - There are approximately 300 lines of the PEP that have not yet been addressed, though these may be deferred --- Lib/test/test_named_expressions.py | 202 ++++++++++++++++++++++++----- Lib/test/test_parser.py | 4 +- 2 files changed, 169 insertions(+), 37 deletions(-) diff --git a/Lib/test/test_named_expressions.py b/Lib/test/test_named_expressions.py index d96b7fd8f6d610..c2b90a23738935 100644 --- a/Lib/test/test_named_expressions.py +++ b/Lib/test/test_named_expressions.py @@ -1,74 +1,206 @@ - +import os import unittest -class NamedExpressionSyntaxTest(unittest.TestCase): - def test_named_expression_syntax_01(self): - code = '''x = y := 0''' +class NamedExpressionInvalidTest(unittest.TestCase): + def test_named_expression_invalid_01(self): + code = """x := 0""" + + with self.assertRaisesRegex(SyntaxError, "invalid syntax"): + exec(code, {}, {}) + + def test_named_expression_invalid_02(self): + code = """x = y := 0""" - with self.assertRaisesRegex(SyntaxError, 'invalid syntax'): + with self.assertRaisesRegex(SyntaxError, "invalid syntax"): exec(code, {}, {}) - def test_named_expression_syntax_02(self): - code = '''foo(cat=category := 'vector')''' + def test_named_expression_invalid_03(self): + code = """foo(cat=category := 'vector')""" - with self.assertRaisesRegex(SyntaxError, 'invalid syntax'): + with self.assertRaisesRegex(SyntaxError, "invalid syntax"): exec(code, {}, {}) - def test_named_expression_syntax_03(self): - code = '''y := f(x)''' + def test_named_expression_invalid_04(self): + code = """y := f(x)""" - with self.assertRaisesRegex(SyntaxError, 'invalid syntax'): + with self.assertRaisesRegex(SyntaxError, "invalid syntax"): exec(code, {}, {}) - def test_named_expression_syntax_04(self): - code = '''y0 = y1 := f(x)''' + def test_named_expression_invalid_05(self): + code = """y0 = y1 := f(x)""" - with self.assertRaisesRegex(SyntaxError, 'invalid syntax'): + with self.assertRaisesRegex(SyntaxError, "invalid syntax"): exec(code, {}, {}) - def test_named_expression_syntax_05(self): - code = '''foo(x = y := f(x))''' + def test_named_expression_invalid_06(self): + code = """foo(x = y := f(x))""" - with self.assertRaisesRegex(SyntaxError, 'invalid syntax'): + with self.assertRaisesRegex(SyntaxError, "invalid syntax"): exec(code, {}, {}) - def test_named_expression_syntax_06(self): - code = '''def foo(answer = p := 42): pass''' + def test_named_expression_invalid_07(self): + code = """def foo(answer = p := 42): pass""" - with self.assertRaisesRegex(SyntaxError, 'invalid syntax'): + with self.assertRaisesRegex(SyntaxError, "invalid syntax"): exec(code, {}, {}) - def test_named_expression_syntax_07(self): - code = '''def foo(answer: p := 42 = 5): pass''' + def test_named_expression_invalid_08(self): + code = """def foo(answer: p := 42 = 5): pass""" - with self.assertRaisesRegex(SyntaxError, 'invalid syntax'): + with self.assertRaisesRegex(SyntaxError, "invalid syntax"): exec(code, {}, {}) - def test_named_expression_syntax_08(self): - code = '''lambda: x := 1)''' + def test_named_expression_invalid_09(self): + code = """(lambda: x := 1)""" - with self.assertRaisesRegex(SyntaxError, 'invalid syntax'): + with self.assertRaisesRegex(SyntaxError, "can't assign to lambda"): exec(code, {}, {}) - def test_named_expression_syntax_09(self): - code = '''z := y := x := 0") # not in P''' + def test_named_expression_invalid_10(self): + code = """z := y := x := 0") # not in P""" - with self.assertRaisesRegex(SyntaxError, 'invalid syntax'): + with self.assertRaisesRegex(SyntaxError, "invalid syntax"): exec(code, {}, {}) - def test_named_expression_syntax_10(self): - code = '''loc:=(x,y))''' + def test_named_expression_invalid_11(self): + code = """info := (name, phone, *rest)""" - with self.assertRaisesRegex(SyntaxError, 'invalid syntax'): + with self.assertRaisesRegex(SyntaxError, "invalid syntax"): exec(code, {}, {}) - def test_named_expression_syntax_11(self): - code = '''info := (name, phone, *rest)''' + def test_named_expression_invalid_12(self): + code = """[[(j := j) for i in range(5)] for j in range(5)]""" - with self.assertRaisesRegex(SyntaxError, 'invalid syntax'): + with self.assertRaisesRegex(UnboundLocalError, "local variable 'j' referenced before assignment"): exec(code, {}, {}) + def test_named_expression_invalid_13(self): + code = "[i + 1 for i in i := [1,2]]" + + with self.assertRaisesRegex(SyntaxError, "invalid syntax"): + exec(code, {}, {}) + + +class NamedExpressionAssignmentTest(unittest.TestCase): + def test_named_expression_assignment_01(self): + (a := 10) + self.assertEqual(a, 10) + + def test_named_expression_assignment_02(self): + a = 20 + (a := a) + self.assertEqual(a, 20) + + def test_named_expression_assignment_03(self): + (total := 1 + 2) + self.assertEqual(total, 3) + + def test_named_expression_assignment_04(self): + if spam := "eggs": + self.assertEqual(spam, "eggs") + else: self.fail("variable was not assigned using named expression") + + def test_named_expression_assignment_05(self): + if True and (spam := True): + self.assertTrue(spam) + else: self.fail("variable was not assigned using named expression") + + def test_named_expression_assignment_06(self): + if (match := 10) is 10: + pass + else: self.fail("variable was not assigned using named expression") + + def test_named_expression_assignment_07(self): + def spam(a): + return a + input_data = [1, 2, 3] + results = [(x, y, x/y) for x in input_data if (y := spam(x)) > 0] + self.assertEqual(results, [(1, 1, 1.0), (2, 2, 1.0), (3, 3, 1.0)]) + + def test_named_expression_assignment_08(self): + def spam(a): + return a + results = [[y := spam(x), x/y] for x in range(1, 5)] + self.assertEqual(results, [[1, 1.0], [2, 1.0], [3, 1.0], [4, 1.0]]) + + def test_named_expression_assignment_09(self): + (x := 1, 2) + self.assertEqual(x, 1) + + def test_named_expression_assignment_10(self): + (z := (y := (x := 0))) + self.assertEqual(x, 0) + self.assertEqual(y, 0) + self.assertEqual(z, 0) + + def test_named_expression_assignment_11(self): + (loc := (1, 2)) + self.assertEqual(loc, (1, 2)) + + def test_named_expression_assignment_12(self): + """ + Where all variables are positive integers, and a is at least as large + as the n'th root of x, this algorithm returns the floor of the n'th + root of x (and roughly doubling the number of accurate bits per + iteration):: + """ + a = 9 + n = 2 + x = 3 + + while a > (d := x // a**(n-1)): + a = ((n-1)*a + d) // n + + self.assertEqual(a, 1) + + @unittest.skip("Not implemented") + def test_named_expression_assignment_13(self): + f = open("../../Lib/test/data/blake2b.txt", "r") + self.assertEqual(len(lines := f.readlines()), 259) + + +class NamedExpressionScopeTest(unittest.TestCase): + def test_named_expression_scope_01(self): + code = 'def foo():\n\n (a := 5)\nprint(a)\n' + with self.assertRaisesRegex(NameError, "name 'a' is not defined"): + exec(code, {}, {}) + + @unittest.skip("Not implemented -- comprehension") + def test_named_expression_scope_02(self): + total = 0 + partial_sums = [total := total + v for v in range(5)] + self.assertEqual(total, 15) + + @unittest.skip("Not implemented -- comprehension") + def test_named_expression_scope_03(self): + """ + This test uses an example data file, Lib/test/data/blake2b.txt, + which starts with a comment followed by a bunch of lines of data. + Thus, this test relies on the existance of said comment in order to + pass. + """ + f = open("../../Lib/test/data/blake2b.txt", "r") + + containsComment = any((comment := line).startswith('#') for line in f.readlines()) + self.assertTrue(containsComment) + self.assertEqual(comment, "# EOF") + + @unittest.skip("Test WIP, Not implemented -- comprehension") + def test_named_expression_scope_04(self): + def spam(a): + return a + results = [[y := spam(x), x/y] for x in range(1, 5)] + self.assertEqual(y, 'something') + + @unittest.skip("Test WIP, Not implemented -- comprehension") + def test_named_expression_scope_05(self): + def spam(a): + return a + input_data = [1, 2, 3] + results = [(x, y, x/y) for x in input_data if (y := spam(x)) > 0] + self.assertEqual(y, 'something') + if __name__ == "__main__": diff --git a/Lib/test/test_parser.py b/Lib/test/test_parser.py index 09282b26011331..3118761056434d 100644 --- a/Lib/test/test_parser.py +++ b/Lib/test/test_parser.py @@ -423,11 +423,12 @@ def test_dict_comprehensions(self): def test_named_expressions(self): self.check_suite("(a := 1)") + self.check_suite("(a := a)") self.check_suite("if (match := pattern.search(data)) is None: pass") self.check_suite("[y := f(x), y**2, y**3]") self.check_suite("filtered_data = [y for x in data if (y := f(x)) is None]") self.check_suite("(y := f(x))") - self.check_suite("0 = (y1 := f(x))") + self.check_suite("y0 = (y1 := f(x))") self.check_suite("foo(x=(y := f(x)))") self.check_suite("def foo(answer=(p := 42)): pass") self.check_suite("def foo(answer: (p := 42) = 5): pass") @@ -435,7 +436,6 @@ def test_named_expressions(self): self.check_suite("(x := lambda: 1)") self.check_suite("(x := lambda: (y := 1))") # not in PEP self.check_suite("lambda line: (m := re.match(pattern, line)) and m.group(1)") - self.check_suite("(x := 0)") self.check_suite("x = (y := 0)") self.check_suite("(z:=(y:=(x:=0)))") self.check_suite("(info := (name, phone, *rest))") From c1c76ea86d973eabd53347e8137ee37b938bda2c Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Tue, 11 Sep 2018 10:16:01 -0700 Subject: [PATCH 18/67] Documentation - Small updates to XXX/todo comments - Remove XXX from child description in ast.c - Add comment with number of previously supported nested expressions for 3.7.X in test_parser.py --- Lib/test/test_parser.py | 2 +- Python/ast.c | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/test/test_parser.py b/Lib/test/test_parser.py index 3118761056434d..0f93b965352f35 100644 --- a/Lib/test/test_parser.py +++ b/Lib/test/test_parser.py @@ -805,7 +805,7 @@ def _nested_expression(self, level): return "["*level+"]"*level def test_deeply_nested_list(self): - # XXX used to be 99 levels in 2.x + # XXX used to be 99 levels in 2.x, used to be 93 levels in 3.7.X e = self._nested_expression(88) st = parser.expr(e) st.compile() diff --git a/Python/ast.c b/Python/ast.c index 4ac44a6ee03efe..3f58ecfe3f1c6b 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -2614,7 +2614,6 @@ ast_for_expr(struct compiling *c, const node *n) switch (TYPE(n)) { case namedexpr_test: // If there are 3 children, continue with named expression - // XXX write better desc of why if (NCH(n) == 3) return ast_for_namedexpr(c, n); case test: From 0ebccb4d4363967f92a342c45313dadbd71372a2 Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Tue, 11 Sep 2018 10:35:17 -0700 Subject: [PATCH 19/67] Fix assert in seq_for_testlist() --- Python/ast.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/ast.c b/Python/ast.c index 3f58ecfe3f1c6b..e7ac2f6df3400b 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -1198,7 +1198,7 @@ seq_for_testlist(struct compiling *c, const node *n) for (i = 0; i < NCH(n); i += 2) { const node *ch = CHILD(n, i); - assert(TYPE(ch) == test || TYPE(ch) == test_nocond || TYPE(ch) == star_expr); + assert(TYPE(ch) == test || TYPE(ch) == test_nocond || TYPE(ch) == star_expr || TYPE(ch) == namedexpr_test); expression = ast_for_expr(c, ch); if (!expression) From 5500849d5cfaa5ea2da6930cc3c24f6d890166cf Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Tue, 11 Sep 2018 11:24:55 -0700 Subject: [PATCH 20/67] Cleanup - Denote "Not implemented -- No keyword args" on failing test case. Fix PEP8 error for blank lines at beginning of test classes in test_parser.py --- Lib/test/test_named_expressions.py | 3 +-- Lib/test/test_parser.py | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_named_expressions.py b/Lib/test/test_named_expressions.py index c2b90a23738935..343eba530f1267 100644 --- a/Lib/test/test_named_expressions.py +++ b/Lib/test/test_named_expressions.py @@ -154,7 +154,7 @@ def test_named_expression_assignment_12(self): self.assertEqual(a, 1) - @unittest.skip("Not implemented") + @unittest.skip("Not implemented -- No keyword args") def test_named_expression_assignment_13(self): f = open("../../Lib/test/data/blake2b.txt", "r") self.assertEqual(len(lines := f.readlines()), 259) @@ -202,6 +202,5 @@ def spam(a): self.assertEqual(y, 'something') - if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_parser.py b/Lib/test/test_parser.py index 0f93b965352f35..3e025729cc92eb 100644 --- a/Lib/test/test_parser.py +++ b/Lib/test/test_parser.py @@ -14,6 +14,7 @@ # class RoundtripLegalSyntaxTestCase(unittest.TestCase): + def roundtrip(self, f, s): st1 = f(s) t = st1.totuple() @@ -458,6 +459,7 @@ def test_named_expressions(self): # class IllegalSyntaxTestCase(unittest.TestCase): + def check_bad_tree(self, tree, label): try: parser.sequence2st(tree) From 38dfaa2a25d6a6eeaaec1fdf64e108558ccecf91 Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Tue, 11 Sep 2018 11:52:40 -0700 Subject: [PATCH 21/67] Tests - Wrap all file opens in `with...as` to ensure files are closed --- Lib/test/test_named_expressions.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_named_expressions.py b/Lib/test/test_named_expressions.py index 343eba530f1267..c452687327d495 100644 --- a/Lib/test/test_named_expressions.py +++ b/Lib/test/test_named_expressions.py @@ -156,8 +156,9 @@ def test_named_expression_assignment_12(self): @unittest.skip("Not implemented -- No keyword args") def test_named_expression_assignment_13(self): - f = open("../../Lib/test/data/blake2b.txt", "r") - self.assertEqual(len(lines := f.readlines()), 259) + # TODO fix import, this breaks depending on how you run it + with open("../../Lib/test/data/blake2b.txt", "r") as f: + self.assertEqual(len(lines := f.readlines()), 259) class NamedExpressionScopeTest(unittest.TestCase): @@ -180,11 +181,10 @@ def test_named_expression_scope_03(self): Thus, this test relies on the existance of said comment in order to pass. """ - f = open("../../Lib/test/data/blake2b.txt", "r") - - containsComment = any((comment := line).startswith('#') for line in f.readlines()) - self.assertTrue(containsComment) - self.assertEqual(comment, "# EOF") + with open("../../Lib/test/data/blake2b.txt", "r") as f: + containsComment = any((comment := line).startswith('#') for line in f.readlines()) + self.assertTrue(containsComment) + self.assertEqual(comment, "# EOF") @unittest.skip("Test WIP, Not implemented -- comprehension") def test_named_expression_scope_04(self): From 3aaddc478ce01a65ac2f55cd346ff87348f65e16 Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Tue, 11 Sep 2018 15:46:40 -0700 Subject: [PATCH 22/67] WIP: handle f(a := 1) --- Python/ast.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Python/ast.c b/Python/ast.c index e7ac2f6df3400b..0ea3f91dea473a 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -2795,6 +2795,10 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func, } else if (TYPE(CHILD(ch, 0)) == STAR) nargs++; + else if (TYPE(CHILD(ch, 1)) == COLONEQUAL) { + printf("COLONEQUAL[1]\n"); + nargs++; + } else /* TYPE(CHILD(ch, 0)) == DOUBLESTAR or keyword argument */ nkeywords++; @@ -2875,6 +2879,14 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func, return NULL; asdl_seq_SET(args, nargs++, e); } + else if (TYPE(CHILD(ch, 1)) == COLONEQUAL) { + printf("COLONEQUAL[2]\n"); + e = ast_for_namedexpr(c, ch); + if (!e) + return NULL; + asdl_seq_SET(args, nargs++, e); + nargs++; + } else { /* a keyword argument */ keyword_ty kw; From 5d525a9ab56d0ff68a75d3f4f62dcaf94af1947d Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Tue, 11 Sep 2018 15:54:06 -0700 Subject: [PATCH 23/67] Tests and Cleanup - No longer skips keyword arg test. Keyword arg test now uses a simpler test case and does not rely on an external file. Remove print statements from ast.c --- Lib/test/test_named_expressions.py | 5 +---- Python/ast.c | 2 -- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/Lib/test/test_named_expressions.py b/Lib/test/test_named_expressions.py index c452687327d495..ba085f47b2c014 100644 --- a/Lib/test/test_named_expressions.py +++ b/Lib/test/test_named_expressions.py @@ -154,11 +154,8 @@ def test_named_expression_assignment_12(self): self.assertEqual(a, 1) - @unittest.skip("Not implemented -- No keyword args") def test_named_expression_assignment_13(self): - # TODO fix import, this breaks depending on how you run it - with open("../../Lib/test/data/blake2b.txt", "r") as f: - self.assertEqual(len(lines := f.readlines()), 259) + self.assertEqual(len(lines := [1, 2]), 2) class NamedExpressionScopeTest(unittest.TestCase): diff --git a/Python/ast.c b/Python/ast.c index 0ea3f91dea473a..f2e87b061024e6 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -2796,7 +2796,6 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func, else if (TYPE(CHILD(ch, 0)) == STAR) nargs++; else if (TYPE(CHILD(ch, 1)) == COLONEQUAL) { - printf("COLONEQUAL[1]\n"); nargs++; } else @@ -2880,7 +2879,6 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func, asdl_seq_SET(args, nargs++, e); } else if (TYPE(CHILD(ch, 1)) == COLONEQUAL) { - printf("COLONEQUAL[2]\n"); e = ast_for_namedexpr(c, ch); if (!e) return NULL; From 32dc443aaa780260aef088acf02e28c9782a5044 Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Tue, 11 Sep 2018 15:55:38 -0700 Subject: [PATCH 24/67] Tests - Refactor last remaining test case that relied on on external file to use a simpler test case without the dependency --- Lib/test/test_named_expressions.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_named_expressions.py b/Lib/test/test_named_expressions.py index ba085f47b2c014..f0e1ab7a4642d7 100644 --- a/Lib/test/test_named_expressions.py +++ b/Lib/test/test_named_expressions.py @@ -172,16 +172,9 @@ def test_named_expression_scope_02(self): @unittest.skip("Not implemented -- comprehension") def test_named_expression_scope_03(self): - """ - This test uses an example data file, Lib/test/data/blake2b.txt, - which starts with a comment followed by a bunch of lines of data. - Thus, this test relies on the existance of said comment in order to - pass. - """ - with open("../../Lib/test/data/blake2b.txt", "r") as f: - containsComment = any((comment := line).startswith('#') for line in f.readlines()) - self.assertTrue(containsComment) - self.assertEqual(comment, "# EOF") + containsOne = any((lastNum := line) == 1 for num in [1, 2, 3]) + self.assertTrue(containsOne) + self.assertEqual(lastNum, 3) @unittest.skip("Test WIP, Not implemented -- comprehension") def test_named_expression_scope_04(self): From d80608883f1f7de79d3d67656d47fe5ccb20d620 Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Tue, 11 Sep 2018 16:00:37 -0700 Subject: [PATCH 25/67] Tests - Add better description of remaning skipped tests. Add test checking scope when using assignment expression in a function argument --- Lib/test/test_named_expressions.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_named_expressions.py b/Lib/test/test_named_expressions.py index f0e1ab7a4642d7..2d5dcd4ff020ca 100644 --- a/Lib/test/test_named_expressions.py +++ b/Lib/test/test_named_expressions.py @@ -164,26 +164,26 @@ def test_named_expression_scope_01(self): with self.assertRaisesRegex(NameError, "name 'a' is not defined"): exec(code, {}, {}) - @unittest.skip("Not implemented -- comprehension") + @unittest.skip("Not implemented -- comprehension scope") def test_named_expression_scope_02(self): total = 0 partial_sums = [total := total + v for v in range(5)] self.assertEqual(total, 15) - @unittest.skip("Not implemented -- comprehension") + @unittest.skip("Not implemented -- comprehension scope") def test_named_expression_scope_03(self): containsOne = any((lastNum := line) == 1 for num in [1, 2, 3]) self.assertTrue(containsOne) self.assertEqual(lastNum, 3) - @unittest.skip("Test WIP, Not implemented -- comprehension") + @unittest.skip("Test WIP, Not implemented -- comprehension scope") def test_named_expression_scope_04(self): def spam(a): return a results = [[y := spam(x), x/y] for x in range(1, 5)] self.assertEqual(y, 'something') - @unittest.skip("Test WIP, Not implemented -- comprehension") + @unittest.skip("Test WIP, Not implemented -- comprehension scope") def test_named_expression_scope_05(self): def spam(a): return a @@ -191,6 +191,10 @@ def spam(a): results = [(x, y, x/y) for x in input_data if (y := spam(x)) > 0] self.assertEqual(y, 'something') + def test_named_expression_scope_06(self): + len(lines := [1, 2]) + self.assertEqual(lines, [1, 2]) + if __name__ == "__main__": unittest.main() From c1469bf5d1d6827bec73c615f3e8ee3b9e7d47df Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Tue, 11 Sep 2018 16:10:57 -0700 Subject: [PATCH 26/67] Tests - Add test for nested comprehension, testing value and scope. Fix variable name in skipped comprehension scope test --- Lib/test/test_named_expressions.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_named_expressions.py b/Lib/test/test_named_expressions.py index 2d5dcd4ff020ca..3bc672a723aaf2 100644 --- a/Lib/test/test_named_expressions.py +++ b/Lib/test/test_named_expressions.py @@ -172,7 +172,7 @@ def test_named_expression_scope_02(self): @unittest.skip("Not implemented -- comprehension scope") def test_named_expression_scope_03(self): - containsOne = any((lastNum := line) == 1 for num in [1, 2, 3]) + containsOne = any((lastNum := num) == 1 for num in [1, 2, 3]) self.assertTrue(containsOne) self.assertEqual(lastNum, 3) @@ -191,7 +191,13 @@ def spam(a): results = [(x, y, x/y) for x in input_data if (y := spam(x)) > 0] self.assertEqual(y, 'something') - def test_named_expression_scope_06(self): + @unittest.skip("Test WIP, Not implemented -- comprehension scope") + def test_named_expression_scope_05(self): + a = [[spam := i for i in range(3)] for j in range(2)] + self.assertEqual(a, [[0, 1, 2], [0, 1, 2]]) + self.assertEqual(spam, 2) + + def test_named_expression_scope_07(self): len(lines := [1, 2]) self.assertEqual(lines, [1, 2]) From e4e63ed0ab20d62760a424d41c5e2d89a2c8fb8b Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Tue, 11 Sep 2018 17:39:43 -0700 Subject: [PATCH 27/67] Handle restriction of LHS for named expressions - can only assign to LHS of type NAME. Specifically, restrict assignment to tuples This adds an alternative set_context specifically for named expressions, set_namedexpr_context. Thus, context is now set differently for standard assignment versus assignment for named expressions in order to handle restrictions. --- Python/ast.c | 122 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 121 insertions(+), 1 deletion(-) diff --git a/Python/ast.c b/Python/ast.c index f2e87b061024e6..4241632636e6cd 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -1087,6 +1087,126 @@ set_context(struct compiling *c, expr_ty e, expr_context_ty ctx, const node *n) return 1; } +/* Set the context ctx for expr_ty e, recursively traversing e. + + Only sets context for expr kinds that "can appear in named assignment context" + (according to ../Parser/Python.asdl). For other expr kinds, it sets + an appropriate syntax error and returns false. + + Very similar to set_context, with additional restrictions specific to named + expressions +*/ + +static int +set_namedexpr_context(struct compiling *c, expr_ty e, expr_context_ty ctx, const node *n) +{ + asdl_seq *s = NULL; + /* If a particular expression type can't be used for assign / delete, + set expr_name to its name and an error message will be generated. + */ + const char* expr_name = NULL; + + /* The ast defines augmented store and load contexts, but the + implementation here doesn't actually use them. The code may be + a little more complex than necessary as a result. It also means + that expressions in an augmented assignment have a Store context. + Consider restructuring so that augmented assignment uses + set_context(), too. + */ + assert(ctx != AugStore && ctx != AugLoad); + + switch (e->kind) { + case Name_kind: + if (ctx == Store) { + if (forbidden_name(c, e->v.Name.id, n, 0)) + return 0; /* forbidden_name() calls ast_error() */ + } + e->v.Name.ctx = ctx; + break; + case Attribute_kind: + expr_name = "attribute"; + break; + case Subscript_kind: + expr_name = "subscript"; + break; + case Starred_kind: + expr_name = "starred"; + break; + case List_kind: + expr_name = "list"; + break; + case Tuple_kind: + expr_name = "tuple"; + break; + case Lambda_kind: + expr_name = "lambda"; + break; + case Call_kind: + expr_name = "function call"; + break; + case BoolOp_kind: + case BinOp_kind: + case UnaryOp_kind: + expr_name = "operator"; + break; + case GeneratorExp_kind: + expr_name = "generator expression"; + break; + case Yield_kind: + case YieldFrom_kind: + expr_name = "yield expression"; + break; + case Await_kind: + expr_name = "await expression"; + break; + case ListComp_kind: + expr_name = "list comprehension"; + break; + case SetComp_kind: + expr_name = "set comprehension"; + break; + case DictComp_kind: + expr_name = "dict comprehension"; + break; + case Dict_kind: + case Set_kind: + case Num_kind: + case Str_kind: + case Bytes_kind: + case JoinedStr_kind: + case FormattedValue_kind: + expr_name = "literal"; + break; + case NameConstant_kind: + expr_name = "keyword"; + break; + case Ellipsis_kind: + expr_name = "Ellipsis"; + break; + case Compare_kind: + expr_name = "comparison"; + break; + case IfExp_kind: + expr_name = "conditional expression"; + break; + default: + PyErr_Format(PyExc_SystemError, + "unexpected expression in named assignment %d (line %d)", + e->kind, e->lineno); + return 0; + } + /* Check for error string set by switch */ + if (expr_name) { + char buf[300]; + PyOS_snprintf(buf, sizeof(buf), + "can't use named assignment with %s", + expr_name); + return ast_error(c, n, buf); + } + + return 1; +} + static operator_ty ast_for_augassign(struct compiling *c, const node *n) { @@ -1705,7 +1825,7 @@ ast_for_namedexpr(struct compiling *c, const node *n) if (!value) return NULL; - if(!set_context(c, target, Store, n)) + if(!set_namedexpr_context(c, target, Store, n)) return NULL; return NamedExpr(target, value, LINENO(n), n->n_col_offset, c->c_arena); From db936c79aad2553f6dca5dc5b01e7e37cce4cab7 Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Tue, 11 Sep 2018 17:44:17 -0700 Subject: [PATCH 28/67] Tests - Update negative test case for assigning to lambda to match new error message. Add negative test case for assigning to tuple --- Lib/test/test_named_expressions.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_named_expressions.py b/Lib/test/test_named_expressions.py index 3bc672a723aaf2..c730b5f9ed2491 100644 --- a/Lib/test/test_named_expressions.py +++ b/Lib/test/test_named_expressions.py @@ -3,6 +3,7 @@ class NamedExpressionInvalidTest(unittest.TestCase): + def test_named_expression_invalid_01(self): code = """x := 0""" @@ -54,7 +55,7 @@ def test_named_expression_invalid_08(self): def test_named_expression_invalid_09(self): code = """(lambda: x := 1)""" - with self.assertRaisesRegex(SyntaxError, "can't assign to lambda"): + with self.assertRaisesRegex(SyntaxError, "can't use named assignment with lambda"): exec(code, {}, {}) def test_named_expression_invalid_10(self): @@ -81,8 +82,15 @@ def test_named_expression_invalid_13(self): with self.assertRaisesRegex(SyntaxError, "invalid syntax"): exec(code, {}, {}) + def test_named_expression_invalid_14(self): + code = """((a, b) := (1, 2))""" + + with self.assertRaisesRegex(SyntaxError, "can't use named assignment with tuple"): + exec(code, {}, {}) + class NamedExpressionAssignmentTest(unittest.TestCase): + def test_named_expression_assignment_01(self): (a := 10) self.assertEqual(a, 10) @@ -159,6 +167,7 @@ def test_named_expression_assignment_13(self): class NamedExpressionScopeTest(unittest.TestCase): + def test_named_expression_scope_01(self): code = 'def foo():\n\n (a := 5)\nprint(a)\n' with self.assertRaisesRegex(NameError, "name 'a' is not defined"): From 47e73675f8051bc27cd7d014dbb85479cde5dc0e Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Tue, 11 Sep 2018 17:45:46 -0700 Subject: [PATCH 29/67] Tests - Reorder test cases to group invalid syntax cases and named assignment target errors --- Lib/test/test_named_expressions.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Lib/test/test_named_expressions.py b/Lib/test/test_named_expressions.py index c730b5f9ed2491..f72a1445991f4f 100644 --- a/Lib/test/test_named_expressions.py +++ b/Lib/test/test_named_expressions.py @@ -53,39 +53,39 @@ def test_named_expression_invalid_08(self): exec(code, {}, {}) def test_named_expression_invalid_09(self): - code = """(lambda: x := 1)""" + code = """z := y := x := 0") # not in P""" - with self.assertRaisesRegex(SyntaxError, "can't use named assignment with lambda"): + with self.assertRaisesRegex(SyntaxError, "invalid syntax"): exec(code, {}, {}) def test_named_expression_invalid_10(self): - code = """z := y := x := 0") # not in P""" + code = """info := (name, phone, *rest)""" with self.assertRaisesRegex(SyntaxError, "invalid syntax"): exec(code, {}, {}) def test_named_expression_invalid_11(self): - code = """info := (name, phone, *rest)""" + code = """[[(j := j) for i in range(5)] for j in range(5)]""" - with self.assertRaisesRegex(SyntaxError, "invalid syntax"): + with self.assertRaisesRegex(UnboundLocalError, "local variable 'j' referenced before assignment"): exec(code, {}, {}) def test_named_expression_invalid_12(self): - code = """[[(j := j) for i in range(5)] for j in range(5)]""" + code = """(lambda: x := 1)""" - with self.assertRaisesRegex(UnboundLocalError, "local variable 'j' referenced before assignment"): + with self.assertRaisesRegex(SyntaxError, "can't use named assignment with lambda"): exec(code, {}, {}) def test_named_expression_invalid_13(self): - code = "[i + 1 for i in i := [1,2]]" + code = """((a, b) := (1, 2))""" - with self.assertRaisesRegex(SyntaxError, "invalid syntax"): + with self.assertRaisesRegex(SyntaxError, "can't use named assignment with tuple"): exec(code, {}, {}) def test_named_expression_invalid_14(self): - code = """((a, b) := (1, 2))""" + code = "[i + 1 for i in i := [1,2]]" - with self.assertRaisesRegex(SyntaxError, "can't use named assignment with tuple"): + with self.assertRaisesRegex(SyntaxError, "invalid syntax"): exec(code, {}, {}) From 787068e8b8dcc66676c0692f2fca773c778fcaf6 Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Tue, 11 Sep 2018 17:49:24 -0700 Subject: [PATCH 30/67] Tests - Update test case for named expression in function argument - check that result and variable are set correctly --- Lib/test/test_named_expressions.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_named_expressions.py b/Lib/test/test_named_expressions.py index f72a1445991f4f..16acab57177b72 100644 --- a/Lib/test/test_named_expressions.py +++ b/Lib/test/test_named_expressions.py @@ -65,27 +65,27 @@ def test_named_expression_invalid_10(self): exec(code, {}, {}) def test_named_expression_invalid_11(self): - code = """[[(j := j) for i in range(5)] for j in range(5)]""" + code = "[i + 1 for i in i := [1,2]]" - with self.assertRaisesRegex(UnboundLocalError, "local variable 'j' referenced before assignment"): + with self.assertRaisesRegex(SyntaxError, "invalid syntax"): exec(code, {}, {}) def test_named_expression_invalid_12(self): - code = """(lambda: x := 1)""" + code = """[[(j := j) for i in range(5)] for j in range(5)]""" - with self.assertRaisesRegex(SyntaxError, "can't use named assignment with lambda"): + with self.assertRaisesRegex(UnboundLocalError, "local variable 'j' referenced before assignment"): exec(code, {}, {}) def test_named_expression_invalid_13(self): - code = """((a, b) := (1, 2))""" + code = """(lambda: x := 1)""" - with self.assertRaisesRegex(SyntaxError, "can't use named assignment with tuple"): + with self.assertRaisesRegex(SyntaxError, "can't use named assignment with lambda"): exec(code, {}, {}) def test_named_expression_invalid_14(self): - code = "[i + 1 for i in i := [1,2]]" + code = """((a, b) := (1, 2))""" - with self.assertRaisesRegex(SyntaxError, "invalid syntax"): + with self.assertRaisesRegex(SyntaxError, "can't use named assignment with tuple"): exec(code, {}, {}) @@ -163,7 +163,9 @@ def test_named_expression_assignment_12(self): self.assertEqual(a, 1) def test_named_expression_assignment_13(self): - self.assertEqual(len(lines := [1, 2]), 2) + length = len(lines := [1, 2]) + self.assertEqual(length, 2) + self.assertEqual(lines, [1,2]) class NamedExpressionScopeTest(unittest.TestCase): From e21f5c88387a53d693a94395240e60a0ccbf7d0a Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Tue, 11 Sep 2018 17:54:08 -0700 Subject: [PATCH 31/67] Todo - Add todo for TargetScopeError based on Guido's comment (https://github.com/python/cpython/commit/2b3acd37bdfc2d35e5094228c6684050d2aa8b0a#r30472562) --- Lib/test/test_named_expressions.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_named_expressions.py b/Lib/test/test_named_expressions.py index 16acab57177b72..50c6a6b7a8d8cd 100644 --- a/Lib/test/test_named_expressions.py +++ b/Lib/test/test_named_expressions.py @@ -73,6 +73,8 @@ def test_named_expression_invalid_11(self): def test_named_expression_invalid_12(self): code = """[[(j := j) for i in range(5)] for j in range(5)]""" + # XXX this fails, but with the wrong exception type - should be TargetScopeError + # with self.assertRaisesRegex(TargetScopeError, ""): with self.assertRaisesRegex(UnboundLocalError, "local variable 'j' referenced before assignment"): exec(code, {}, {}) From acd0f74ea16074c19e2527d1bc970a84b156e637 Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Wed, 12 Sep 2018 09:25:41 -0700 Subject: [PATCH 32/67] Tests - Add named expression tests for assignment operator in function arguments Note: One of two tests are skipped, as function arguments are currently treating an assignment expression inside of parenthesis as one child, which does not properly catch the named expression, nor does it count arguments properly --- Lib/test/test_named_expressions.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Lib/test/test_named_expressions.py b/Lib/test/test_named_expressions.py index 50c6a6b7a8d8cd..d046dfccc4f3fb 100644 --- a/Lib/test/test_named_expressions.py +++ b/Lib/test/test_named_expressions.py @@ -90,6 +90,19 @@ def test_named_expression_invalid_14(self): with self.assertRaisesRegex(SyntaxError, "can't use named assignment with tuple"): exec(code, {}, {}) + @unittest.skip("Not implemented -- custom check for named expr") + def test_named_expression_invalid_15(self): + code = """foo(a=1, b := 2)""" + + with self.assertRaisesRegex(SyntaxError, "positional argument follows keyword argument"): + exec(code, {}, {}) + + def test_named_expression_invalid_16(self): + code = """foo(a=1, (b := 2))""" + + with self.assertRaisesRegex(SyntaxError, "positional argument follows keyword argument"): + exec(code, {}, {}) + class NamedExpressionAssignmentTest(unittest.TestCase): From d0dd983148af30a0bb43b432fdb883ccf15511dd Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Thu, 13 Sep 2018 10:02:29 -0700 Subject: [PATCH 33/67] Add NamedStore to expr_context. Regenerate related code with `make regen-ast` --- Include/Python-ast.h | 28 ++++++++++++++-------------- Parser/Python.asdl | 2 +- Python/Python-ast.c | 21 ++++++++++++++++++++- 3 files changed, 35 insertions(+), 16 deletions(-) diff --git a/Include/Python-ast.h b/Include/Python-ast.h index 5cf814f70fe269..f8394e6c26ad83 100644 --- a/Include/Python-ast.h +++ b/Include/Python-ast.h @@ -3,8 +3,7 @@ #ifndef Py_PYTHON_AST_H #define Py_PYTHON_AST_H #ifdef __cplusplus -extern "C" -{ +extern "C" { #endif #include "asdl.h" @@ -441,11 +440,11 @@ struct _withitem { // Note: these macros affect function definitions, not only call sites. #define Module(a0, a1) _Py_Module(a0, a1) - mod_ty _Py_Module(asdl_seq *body, PyArena *arena); +mod_ty _Py_Module(asdl_seq * body, PyArena *arena); #define Interactive(a0, a1) _Py_Interactive(a0, a1) - mod_ty _Py_Interactive(asdl_seq *body, PyArena *arena); +mod_ty _Py_Interactive(asdl_seq * body, PyArena *arena); #define Expression(a0, a1) _Py_Expression(a0, a1) - mod_ty _Py_Expression(expr_ty body, PyArena *arena); +mod_ty _Py_Expression(expr_ty body, PyArena *arena); #define Suite(a0, a1) _Py_Suite(a0, a1) mod_ty _Py_Suite(asdl_seq * body, PyArena *arena); #define FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) _Py_FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) @@ -631,11 +630,11 @@ expr_ty _Py_Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); #define Slice(a0, a1, a2, a3) _Py_Slice(a0, a1, a2, a3) - slice_ty _Py_Slice(expr_ty lower, expr_ty upper, expr_ty step, PyArena *arena); +slice_ty _Py_Slice(expr_ty lower, expr_ty upper, expr_ty step, PyArena *arena); #define ExtSlice(a0, a1) _Py_ExtSlice(a0, a1) - slice_ty _Py_ExtSlice(asdl_seq *dims, PyArena *arena); +slice_ty _Py_ExtSlice(asdl_seq * dims, PyArena *arena); #define Index(a0, a1) _Py_Index(a0, a1) - slice_ty _Py_Index(expr_ty value, PyArena *arena); +slice_ty _Py_Index(expr_ty value, PyArena *arena); #define comprehension(a0, a1, a2, a3, a4) _Py_comprehension(a0, a1, a2, a3, a4) comprehension_ty _Py_comprehension(expr_ty target, expr_ty iter, asdl_seq * ifs, int is_async, PyArena *arena); @@ -652,15 +651,16 @@ arguments_ty _Py_arguments(asdl_seq * args, arg_ty vararg, asdl_seq * arg_ty _Py_arg(identifier arg, expr_ty annotation, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); #define keyword(a0, a1, a2) _Py_keyword(a0, a1, a2) - keyword_ty _Py_keyword(identifier arg, expr_ty value, PyArena *arena); +keyword_ty _Py_keyword(identifier arg, expr_ty value, PyArena *arena); #define alias(a0, a1, a2) _Py_alias(a0, a1, a2) - alias_ty _Py_alias(identifier name, identifier asname, PyArena *arena); +alias_ty _Py_alias(identifier name, identifier asname, PyArena *arena); #define withitem(a0, a1, a2) _Py_withitem(a0, a1, a2) - withitem_ty _Py_withitem(expr_ty context_expr, expr_ty optional_vars, PyArena *arena); +withitem_ty _Py_withitem(expr_ty context_expr, expr_ty optional_vars, PyArena + *arena); - PyObject *PyAST_mod2obj(mod_ty t); - mod_ty PyAST_obj2mod(PyObject *ast, PyArena *arena, int mode); - int PyAST_Check(PyObject *obj); +PyObject* PyAST_mod2obj(mod_ty t); +mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode); +int PyAST_Check(PyObject* obj); #ifdef __cplusplus } diff --git a/Parser/Python.asdl b/Parser/Python.asdl index b15899a7a0f3c1..7b2a8737ab8020 100644 --- a/Parser/Python.asdl +++ b/Parser/Python.asdl @@ -88,7 +88,7 @@ module Python -- col_offset is the byte offset in the utf8 string the parser uses attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset) - expr_context = Load | Store | Del | AugLoad | AugStore | Param + expr_context = Load | Store | Del | AugLoad | AugStore | Param | NamedStore slice = Slice(expr? lower, expr? upper, expr? step) | ExtSlice(slice* dims) diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 854ffe2117f646..aa0922a0609170 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -349,7 +349,8 @@ static char *Tuple_fields[]={ }; static PyTypeObject *expr_context_type; static PyObject *Load_singleton, *Store_singleton, *Del_singleton, -*AugLoad_singleton, *AugStore_singleton, *Param_singleton; +*AugLoad_singleton, *AugStore_singleton, *Param_singleton, +*NamedStore_singleton; static PyObject* ast2obj_expr_context(expr_context_ty); static PyTypeObject *Load_type; static PyTypeObject *Store_type; @@ -357,6 +358,7 @@ static PyTypeObject *Del_type; static PyTypeObject *AugLoad_type; static PyTypeObject *AugStore_type; static PyTypeObject *Param_type; +static PyTypeObject *NamedStore_type; static PyTypeObject *slice_type; static PyObject* ast2obj_slice(void*); static PyTypeObject *Slice_type; @@ -956,6 +958,10 @@ static int init_types(void) if (!Param_type) return 0; Param_singleton = PyType_GenericNew(Param_type, NULL, NULL); if (!Param_singleton) return 0; + NamedStore_type = make_type("NamedStore", expr_context_type, NULL, 0); + if (!NamedStore_type) return 0; + NamedStore_singleton = PyType_GenericNew(NamedStore_type, NULL, NULL); + if (!NamedStore_singleton) return 0; slice_type = make_type("slice", &AST_type, NULL, 0); if (!slice_type) return 0; if (!add_attributes(slice_type, NULL, 0)) return 0; @@ -3511,6 +3517,9 @@ PyObject* ast2obj_expr_context(expr_context_ty o) case Param: Py_INCREF(Param_singleton); return Param_singleton; + case NamedStore: + Py_INCREF(NamedStore_singleton); + return NamedStore_singleton; default: /* should never happen, but just in case ... */ PyErr_Format(PyExc_SystemError, "unknown expr_context found"); @@ -7241,6 +7250,14 @@ obj2ast_expr_context(PyObject* obj, expr_context_ty* out, PyArena* arena) *out = Param; return 0; } + isinstance = PyObject_IsInstance(obj, (PyObject *)NamedStore_type); + if (isinstance == -1) { + return 1; + } + if (isinstance) { + *out = NamedStore; + return 0; + } PyErr_Format(PyExc_TypeError, "expected some sort of expr_context, but got %R", obj); return 1; @@ -8393,6 +8410,8 @@ PyInit__ast(void) return NULL; if (PyDict_SetItemString(d, "Param", (PyObject*)Param_type) < 0) return NULL; + if (PyDict_SetItemString(d, "NamedStore", (PyObject*)NamedStore_type) < 0) + return NULL; if (PyDict_SetItemString(d, "slice", (PyObject*)slice_type) < 0) return NULL; if (PyDict_SetItemString(d, "Slice", (PyObject*)Slice_type) < 0) return From 9b136d987173b532b2419cdd875ddd01dd898d1c Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Thu, 13 Sep 2018 10:03:34 -0700 Subject: [PATCH 34/67] Add usage of NamedStore to ast_for_named_expr in ast.c. Update occurances of checking for Store to also handle NamedStore where appropriate --- Python/ast.c | 2 +- Python/compile.c | 30 ++++++++++++++++++++++++------ 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/Python/ast.c b/Python/ast.c index 4241632636e6cd..ead50ca818c406 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -1825,7 +1825,7 @@ ast_for_namedexpr(struct compiling *c, const node *n) if (!value) return NULL; - if(!set_namedexpr_context(c, target, Store, n)) + if(!set_namedexpr_context(c, target, NamedStore, n)) return NULL; return NamedExpr(target, value, LINENO(n), n->n_col_offset, c->c_arena); diff --git a/Python/compile.c b/Python/compile.c index a102fd15be792a..486ccd170a0071 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -3428,7 +3428,10 @@ compiler_nameop(struct compiler *c, identifier name, expr_context_ty ctx) case Load: op = (c->u->u_ste->ste_type == ClassBlock) ? LOAD_CLASSDEREF : LOAD_DEREF; break; - case Store: op = STORE_DEREF; break; + case Store: + case NamedStore: + op = STORE_DEREF; + break; case AugLoad: case AugStore: break; @@ -3443,7 +3446,10 @@ compiler_nameop(struct compiler *c, identifier name, expr_context_ty ctx) case OP_FAST: switch (ctx) { case Load: op = LOAD_FAST; break; - case Store: op = STORE_FAST; break; + case Store: + case NamedStore: + op = STORE_FAST; + break; case Del: op = DELETE_FAST; break; case AugLoad: case AugStore: @@ -3459,7 +3465,10 @@ compiler_nameop(struct compiler *c, identifier name, expr_context_ty ctx) case OP_GLOBAL: switch (ctx) { case Load: op = LOAD_GLOBAL; break; - case Store: op = STORE_GLOBAL; break; + case Store: + case NamedStore: + op = STORE_GLOBAL; + break; case Del: op = DELETE_GLOBAL; break; case AugLoad: case AugStore: @@ -3474,7 +3483,10 @@ compiler_nameop(struct compiler *c, identifier name, expr_context_ty ctx) case OP_NAME: switch (ctx) { case Load: op = LOAD_NAME; break; - case Store: op = STORE_NAME; break; + case Store: + case NamedStore: + op = STORE_NAME; + break; case Del: op = DELETE_NAME; break; case AugLoad: case AugStore: @@ -3592,7 +3604,7 @@ static int compiler_list(struct compiler *c, expr_ty e) { asdl_seq *elts = e->v.List.elts; - if (e->v.List.ctx == Store) { + if (e->v.List.ctx == Store || e->v.List.ctx == NamedStore) { return assignment_helper(c, elts); } else if (e->v.List.ctx == Load) { @@ -3608,7 +3620,7 @@ static int compiler_tuple(struct compiler *c, expr_ty e) { asdl_seq *elts = e->v.Tuple.elts; - if (e->v.Tuple.ctx == Store) { + if (e->v.Tuple.ctx == Store || e->v.Tuple.ctx == NamedStore) { return assignment_helper(c, elts); } else if (e->v.Tuple.ctx == Load) { @@ -4663,6 +4675,7 @@ compiler_visit_expr1(struct compiler *c, expr_ty e) ADDOP(c, ROT_TWO); /* Fall through */ case Store: + case NamedStore: ADDOP_NAME(c, STORE_ATTR, e->v.Attribute.attr, names); break; case Del: @@ -4688,6 +4701,10 @@ compiler_visit_expr1(struct compiler *c, expr_ty e) case AugStore: VISIT_SLICE(c, e->v.Subscript.slice, AugStore); break; + case NamedStore: + VISIT(c, expr, e->v.Subscript.value); + VISIT_SLICE(c, e->v.Subscript.slice, NamedStore); + break; case Store: VISIT(c, expr, e->v.Subscript.value); VISIT_SLICE(c, e->v.Subscript.slice, Store); @@ -4706,6 +4723,7 @@ compiler_visit_expr1(struct compiler *c, expr_ty e) case Starred_kind: switch (e->v.Starred.ctx) { case Store: + case NamedStore: /* In all legitimate cases, the Starred node was already replaced * by compiler_list/compiler_tuple. XXX: is that okay? */ return compiler_error(c, From 8fc5c9962a6c434cbf7ac5afda5426da1ed4faf1 Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Thu, 13 Sep 2018 15:01:48 -0700 Subject: [PATCH 35/67] Add ste_comprehension to _symtable_entry to track if the namespace is a comprehension. Initialize ste_comprehension to 0. Set set_comprehension to 1 in symtable_handle_comprehension --- Include/symtable.h | 1 + Python/symtable.c | 3 +++ 2 files changed, 4 insertions(+) diff --git a/Include/symtable.h b/Include/symtable.h index 949022bd6630aa..9392e64387794f 100644 --- a/Include/symtable.h +++ b/Include/symtable.h @@ -50,6 +50,7 @@ typedef struct _symtable_entry { including free refs to globals */ unsigned ste_generator : 1; /* true if namespace is a generator */ unsigned ste_coroutine : 1; /* true if namespace is a coroutine */ + unsigned ste_comprehension : 1; /* true if namespace is a list comprehension */ unsigned ste_varargs : 1; /* true if block has varargs */ unsigned ste_varkeywords : 1; /* true if block has varkeywords */ unsigned ste_returns_value : 1; /* true if namespace uses return with diff --git a/Python/symtable.c b/Python/symtable.c index e2d44f16854d7c..34f316f089f813 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -75,6 +75,7 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block, ste->ste_child_free = 0; ste->ste_generator = 0; ste->ste_coroutine = 0; + ste->ste_comprehension = 0; ste->ste_returns_value = 0; ste->ste_needs_class_closure = 0; @@ -1717,6 +1718,8 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e, if (outermost->is_async) { st->st_cur->ste_coroutine = 1; } + st->st_cur->ste_comprehension = 1; + /* Outermost iter is received as an argument */ if (!symtable_implicit_arg(st, 0)) { symtable_exit_block(st, (void *)e); From 538dea464440906ff0bf33ebe1c019674a3f33fc Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Thu, 13 Sep 2018 15:32:43 -0700 Subject: [PATCH 36/67] s/symtable_add_def/symtable_add_def_helper. Add symtable_add_def to handle grabbing st->st_cur and passing it to symtable_add_def_helper. This now allows us to call the original code from symtable_add_def by instead calling symtable_add_def_helper with a different ste. --- Python/symtable.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/Python/symtable.c b/Python/symtable.c index 34f316f089f813..46067b994abccc 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -973,7 +973,7 @@ symtable_lookup(struct symtable *st, PyObject *name) } static int -symtable_add_def(struct symtable *st, PyObject *name, int flag) +symtable_add_def_helper(struct symtable *st, PyObject *name, int flag, struct _symtable_entry *ste) { PyObject *o; PyObject *dict; @@ -983,15 +983,15 @@ symtable_add_def(struct symtable *st, PyObject *name, int flag) if (!mangled) return 0; - dict = st->st_cur->ste_symbols; + dict = ste->ste_symbols; if ((o = PyDict_GetItem(dict, mangled))) { val = PyLong_AS_LONG(o); if ((flag & DEF_PARAM) && (val & DEF_PARAM)) { /* Is it better to use 'mangled' or 'name' here? */ PyErr_Format(PyExc_SyntaxError, DUPLICATE_ARGUMENT, name); PyErr_SyntaxLocationObject(st->st_filename, - st->st_cur->ste_lineno, - st->st_cur->ste_col_offset + 1); + ste->ste_lineno, + ste->ste_col_offset + 1); goto error; } val |= flag; @@ -1007,7 +1007,7 @@ symtable_add_def(struct symtable *st, PyObject *name, int flag) Py_DECREF(o); if (flag & DEF_PARAM) { - if (PyList_Append(st->st_cur->ste_varnames, mangled) < 0) + if (PyList_Append(ste->ste_varnames, mangled) < 0) goto error; } else if (flag & DEF_GLOBAL) { /* XXX need to update DEF_GLOBAL for other flags too; @@ -1033,6 +1033,11 @@ symtable_add_def(struct symtable *st, PyObject *name, int flag) return 0; } +static int +symtable_add_def(struct symtable *st, PyObject *name, int flag) { + return symtable_add_def_helper(st, name, flag, st->st_cur); +} + /* VISIT, VISIT_SEQ and VIST_SEQ_TAIL take an ASDL type as their second argument. They use the ASDL name to synthesize the name of the C type and the visit function. From ab34b0bbdc51cad9e86bd02b01a5cfc9fa298fe9 Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Thu, 13 Sep 2018 16:39:50 -0700 Subject: [PATCH 37/67] Refactor symtable_record_directive to take lineno and col_offset as arguments instead of stmt_ty. This allows symtable_record_directive to be used for stmt_ty and expr_ty --- Python/symtable.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Python/symtable.c b/Python/symtable.c index 46067b994abccc..50a221e2c71923 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -1088,7 +1088,7 @@ symtable_add_def(struct symtable *st, PyObject *name, int flag) { } static int -symtable_record_directive(struct symtable *st, identifier name, stmt_ty s) +symtable_record_directive(struct symtable *st, identifier name, int lineno, int col_offset) { PyObject *data, *mangled; int res; @@ -1100,7 +1100,7 @@ symtable_record_directive(struct symtable *st, identifier name, stmt_ty s) mangled = _Py_Mangle(st->st_private, name); if (!mangled) return 0; - data = Py_BuildValue("(Nii)", mangled, s->lineno, s->col_offset); + data = Py_BuildValue("(Nii)", mangled, lineno, col_offset); if (!data) return 0; res = PyList_Append(st->st_cur->ste_directives, data); @@ -1286,7 +1286,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) } if (!symtable_add_def(st, name, DEF_GLOBAL)) VISIT_QUIT(st, 0); - if (!symtable_record_directive(st, name, s)) + if (!symtable_record_directive(st, name, s->lineno, s->col_offset)) VISIT_QUIT(st, 0); } break; @@ -1318,7 +1318,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) } if (!symtable_add_def(st, name, DEF_NONLOCAL)) VISIT_QUIT(st, 0); - if (!symtable_record_directive(st, name, s)) + if (!symtable_record_directive(st, name, s->lineno, s->col_offset)) VISIT_QUIT(st, 0); } break; From c04a08d72d28f9377da546a0e3d337c07c2ac5fd Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Thu, 13 Sep 2018 17:30:49 -0700 Subject: [PATCH 38/67] Handle elevating scope for named expressions in comprehensions. --- Python/symtable.c | 50 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/Python/symtable.c b/Python/symtable.c index 50a221e2c71923..a2261e50c3e311 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -1373,6 +1373,50 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT_QUIT(st, 1); } +static int +symtable_extend_namedexpr_scope(struct symtable *st, expr_ty e) { + assert(st->st_stack); + + Py_ssize_t i, size; + struct _symtable_entry *ste; + size = PyList_GET_SIZE(st->st_stack); + assert(size); + + /* Iterate over the stack in reverse and add to the nearest adequate scope */ + for (i = size - 1; i >= 0; i--) { + ste = (struct _symtable_entry *) PyList_GET_ITEM(st->st_stack, i); + + /* If our current entry is a comprehension, skip it */ + if (ste->ste_comprehension) { + continue; + } + + /* If we find a FunctionBlock entry, add as NONLOCAL/LOCAL */ + if (ste->ste_type == FunctionBlock) { + if (!symtable_add_def(st, e->v.Name.id, DEF_NONLOCAL)) + VISIT_QUIT(st, 0); + if (!symtable_record_directive(st, e->v.Name.id, e->lineno, e->col_offset)) + VISIT_QUIT(st, 0); + + return symtable_add_def_helper(st, e->v.Name.id, DEF_LOCAL, ste); + } + /* If we find a ModuleBlock entry, add as GLOBAL */ + if (ste->ste_type == ModuleBlock) { + if (!symtable_add_def(st, e->v.Name.id, DEF_GLOBAL)) + VISIT_QUIT(st, 0); + if (!symtable_record_directive(st, e->v.Name.id, e->lineno, e->col_offset)) + VISIT_QUIT(st, 0); + + return symtable_add_def_helper(st, e->v.Name.id, DEF_GLOBAL, ste); + } + /* XXX handle ClassBlock */ + if (ste->ste_type == ClassBlock) { + printf("XXX: error handling not implemented\n"); + } + } + return 0; +} + static int symtable_visit_expr(struct symtable *st, expr_ty e) { @@ -1486,6 +1530,12 @@ symtable_visit_expr(struct symtable *st, expr_ty e) VISIT(st, expr, e->v.Starred.value); break; case Name_kind: + /* Special-case named expr */ + if (e->v.Name.ctx == NamedStore && st->st_cur->ste_comprehension) { + /* Helper to traverse and find proper scope to add variable to outer scope */ + if(!symtable_extend_namedexpr_scope(st, e)) + VISIT_QUIT(st, 0); + } if (!symtable_add_def(st, e->v.Name.id, e->v.Name.ctx == Load ? USE : DEF_LOCAL)) VISIT_QUIT(st, 0); From f0297f89b42f90ced5709f36b5c1068a73265ed2 Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Thu, 13 Sep 2018 17:31:46 -0700 Subject: [PATCH 39/67] Handle error for usage of named expression inside a class block --- Python/symtable.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Python/symtable.c b/Python/symtable.c index a2261e50c3e311..c959b2be057bb3 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -31,6 +31,9 @@ #define IMPORT_STAR_WARNING "import * only allowed at module level" +#define NAMED_EXPR_IN_CLASS \ +"named expression cannot be used in a class body" + static PySTEntryObject * ste_new(struct symtable *st, identifier name, _Py_block_ty block, void *key, int lineno, int col_offset) @@ -1411,7 +1414,11 @@ symtable_extend_namedexpr_scope(struct symtable *st, expr_ty e) { } /* XXX handle ClassBlock */ if (ste->ste_type == ClassBlock) { - printf("XXX: error handling not implemented\n"); + PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_IN_CLASS, e->v.Name.id); + PyErr_SyntaxLocationObject(st->st_filename, + e->lineno, + e->col_offset); + VISIT_QUIT(st, 0); } } return 0; From ee6d78967a5b419143a16975d1e5f6865eeeafdb Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Fri, 14 Sep 2018 06:49:16 -0700 Subject: [PATCH 40/67] Tests - No longer skip scope tests. Add additional scope tests --- Lib/test/test_named_expressions.py | 73 ++++++++++++++++++++++-------- 1 file changed, 55 insertions(+), 18 deletions(-) diff --git a/Lib/test/test_named_expressions.py b/Lib/test/test_named_expressions.py index d046dfccc4f3fb..1271568575b83b 100644 --- a/Lib/test/test_named_expressions.py +++ b/Lib/test/test_named_expressions.py @@ -75,7 +75,7 @@ def test_named_expression_invalid_12(self): # XXX this fails, but with the wrong exception type - should be TargetScopeError # with self.assertRaisesRegex(TargetScopeError, ""): - with self.assertRaisesRegex(UnboundLocalError, "local variable 'j' referenced before assignment"): + with self.assertRaisesRegex(NameError, "name 'j' is not defined"): exec(code, {}, {}) def test_named_expression_invalid_13(self): @@ -138,14 +138,14 @@ def test_named_expression_assignment_07(self): def spam(a): return a input_data = [1, 2, 3] - results = [(x, y, x/y) for x in input_data if (y := spam(x)) > 0] - self.assertEqual(results, [(1, 1, 1.0), (2, 2, 1.0), (3, 3, 1.0)]) + res = [(x, y, x/y) for x in input_data if (y := spam(x)) > 0] + self.assertEqual(res, [(1, 1, 1.0), (2, 2, 1.0), (3, 3, 1.0)]) def test_named_expression_assignment_08(self): def spam(a): return a - results = [[y := spam(x), x/y] for x in range(1, 5)] - self.assertEqual(results, [[1, 1.0], [2, 1.0], [3, 1.0], [4, 1.0]]) + res = [[y := spam(x), x/y] for x in range(1, 5)] + self.assertEqual(res, [[1, 1.0], [2, 1.0], [3, 1.0], [4, 1.0]]) def test_named_expression_assignment_09(self): (x := 1, 2) @@ -190,43 +190,80 @@ def test_named_expression_scope_01(self): with self.assertRaisesRegex(NameError, "name 'a' is not defined"): exec(code, {}, {}) - @unittest.skip("Not implemented -- comprehension scope") def test_named_expression_scope_02(self): total = 0 partial_sums = [total := total + v for v in range(5)] - self.assertEqual(total, 15) + self.assertEqual(partial_sums, [0, 1, 3, 6, 10]) + self.assertEqual(total, 10) - @unittest.skip("Not implemented -- comprehension scope") def test_named_expression_scope_03(self): containsOne = any((lastNum := num) == 1 for num in [1, 2, 3]) self.assertTrue(containsOne) - self.assertEqual(lastNum, 3) + self.assertEqual(lastNum, 1) - @unittest.skip("Test WIP, Not implemented -- comprehension scope") def test_named_expression_scope_04(self): def spam(a): return a - results = [[y := spam(x), x/y] for x in range(1, 5)] - self.assertEqual(y, 'something') + res = [[y := spam(x), x/y] for x in range(1, 5)] + self.assertEqual(y, 4) - @unittest.skip("Test WIP, Not implemented -- comprehension scope") def test_named_expression_scope_05(self): def spam(a): return a input_data = [1, 2, 3] - results = [(x, y, x/y) for x in input_data if (y := spam(x)) > 0] - self.assertEqual(y, 'something') + res = [(x, y, x/y) for x in input_data if (y := spam(x)) > 0] + self.assertEqual(res, [(1, 1, 1.0), (2, 2, 1.0), (3, 3, 1.0)]) + self.assertEqual(y, 3) - @unittest.skip("Test WIP, Not implemented -- comprehension scope") def test_named_expression_scope_05(self): - a = [[spam := i for i in range(3)] for j in range(2)] - self.assertEqual(a, [[0, 1, 2], [0, 1, 2]]) + res = [[spam := i for i in range(3)] for j in range(2)] + self.assertEqual(res, [[0, 1, 2], [0, 1, 2]]) self.assertEqual(spam, 2) def test_named_expression_scope_07(self): len(lines := [1, 2]) self.assertEqual(lines, [1, 2]) + def test_named_expression_scope_08(self): + def spam(a): + return a + + def eggs(b): + return b * 2 + + res = [spam(a := eggs(b := h)) for h in range(2)] + self.assertEqual(res, [0, 2]) + self.assertEqual(a, 2) + self.assertEqual(b, 1) + + def test_named_expression_scope_09(self): + def spam(a): + return a + + def eggs(b): + return b * 2 + + res = [spam(a := eggs(a := h)) for h in range(2)] + self.assertEqual(res, [0, 2]) + self.assertEqual(a, 2) + + def test_named_expression_scope_10(self): + res = [b := [a := 1 for i in range(2)] for j in range(2)] + + self.assertEqual(res, [[1, 1], [1, 1]]) + self.assertEqual(a, 1) + self.assertEqual(b, [1, 1]) + + + def test_named_expression_scope_11(self): + ns = {} + code = "[j := i for i in range(5)]" + exec(code, ns, {}) + + self.assertEqual(ns["j"], 4) + + + if __name__ == "__main__": unittest.main() From c37e3c1967d4aa9747cd4355a40215186fbbab97 Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Fri, 14 Sep 2018 06:50:35 -0700 Subject: [PATCH 41/67] Cleanup - Update error message for named expression within a comprehension within a class. Update comments. Add assert for symtable_extend_namedexpr_scope to validate that we always find at least a ModuleScope if we don't find a Class or FunctionScope --- Python/symtable.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/Python/symtable.c b/Python/symtable.c index c959b2be057bb3..317fa2470131a0 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -31,8 +31,8 @@ #define IMPORT_STAR_WARNING "import * only allowed at module level" -#define NAMED_EXPR_IN_CLASS \ -"named expression cannot be used in a class body" +#define NAMED_EXPR_COMP_IN_CLASS \ +"named expression within a comprehension cannot be used in a class body" static PySTEntryObject * ste_new(struct symtable *st, identifier name, _Py_block_ty block, @@ -1377,7 +1377,8 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) } static int -symtable_extend_namedexpr_scope(struct symtable *st, expr_ty e) { +symtable_extend_namedexpr_scope(struct symtable *st, expr_ty e) +{ assert(st->st_stack); Py_ssize_t i, size; @@ -1412,15 +1413,20 @@ symtable_extend_namedexpr_scope(struct symtable *st, expr_ty e) { return symtable_add_def_helper(st, e->v.Name.id, DEF_GLOBAL, ste); } - /* XXX handle ClassBlock */ + /* Disallow usage in ClassBlock */ if (ste->ste_type == ClassBlock) { - PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_IN_CLASS, e->v.Name.id); + PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_IN_CLASS, e->v.Name.id); PyErr_SyntaxLocationObject(st->st_filename, e->lineno, e->col_offset); VISIT_QUIT(st, 0); } } + + /* We should always find either a FunctionBlock, ModuleBlock or ClassBlock + and should never fall to this case + */ + assert(0); return 0; } From 1db3fa9452439d1fb770e14c97c800c8e7f7eb14 Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Fri, 14 Sep 2018 07:03:33 -0700 Subject: [PATCH 42/67] Cleanup - Add missing case for NamedStore in expr_context_name. Remove unused var in set_namedexpr_content --- Python/ast.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Python/ast.c b/Python/ast.c index ead50ca818c406..a889f271fdfc5c 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -94,6 +94,8 @@ expr_context_name(expr_context_ty ctx) return "Load"; case Store: return "Store"; + case NamedStore: + return "NamedStore"; case Del: return "Del"; case AugLoad: @@ -1100,7 +1102,6 @@ set_context(struct compiling *c, expr_ty e, expr_context_ty ctx, const node *n) static int set_namedexpr_context(struct compiling *c, expr_ty e, expr_context_ty ctx, const node *n) { - asdl_seq *s = NULL; /* If a particular expression type can't be used for assign / delete, set expr_name to its name and an error message will be generated. */ From 0e29038f7a6520b8f00fe6618c67754827b5cc10 Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Fri, 14 Sep 2018 09:58:55 -0700 Subject: [PATCH 43/67] Refactor - Consolidate set_context and set_namedexpr_context to reduce duplicated code. Special cases for named expressions are handled by checking if ctx is NamedStore --- Python/ast.c | 160 ++++++++++++--------------------------------------- 1 file changed, 38 insertions(+), 122 deletions(-) diff --git a/Python/ast.c b/Python/ast.c index a889f271fdfc5c..978e6c7e15aa90 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -977,14 +977,29 @@ set_context(struct compiling *c, expr_ty e, expr_context_ty ctx, const node *n) switch (e->kind) { case Attribute_kind: + if (ctx == NamedStore) { + expr_name = "attribute"; + break; + } + e->v.Attribute.ctx = ctx; if (ctx == Store && forbidden_name(c, e->v.Attribute.attr, n, 1)) return 0; break; case Subscript_kind: + if (ctx == NamedStore) { + expr_name = "subscript"; + break; + } + e->v.Subscript.ctx = ctx; break; case Starred_kind: + if (ctx == NamedStore) { + expr_name = "starred"; + break; + } + e->v.Starred.ctx = ctx; if (!set_context(c, e->v.Starred.value, ctx, n)) return 0; @@ -997,10 +1012,20 @@ set_context(struct compiling *c, expr_ty e, expr_context_ty ctx, const node *n) e->v.Name.ctx = ctx; break; case List_kind: + if (ctx == NamedStore) { + expr_name = "list"; + break; + } + e->v.List.ctx = ctx; s = e->v.List.elts; break; case Tuple_kind: + if (ctx == NamedStore) { + expr_name = "tuple"; + break; + } + e->v.Tuple.ctx = ctx; s = e->v.Tuple.elts; break; @@ -1062,17 +1087,27 @@ set_context(struct compiling *c, expr_ty e, expr_context_ty ctx, const node *n) case IfExp_kind: expr_name = "conditional expression"; break; + case NamedExpr_kind: + expr_name = "named expression"; + break; default: PyErr_Format(PyExc_SystemError, - "unexpected expression in assignment %d (line %d)", + "unexpected expression in %sassignment %d (line %d)", + ctx == NamedStore ? "named ": "", e->kind, e->lineno); return 0; } /* Check for error string set by switch */ if (expr_name) { - return ast_error(c, n, "cannot %s %s", + if (ctx == NamedStore) { + return ast_error(c, n, "cannot use named assignment with %s", + expr_name); + } + else { + return ast_error(c, n, "cannot %s %s", ctx == Store ? "assign to" : "delete", expr_name); + } } /* If the LHS is a list or tuple, we need to set the assignment @@ -1089,125 +1124,6 @@ set_context(struct compiling *c, expr_ty e, expr_context_ty ctx, const node *n) return 1; } -/* Set the context ctx for expr_ty e, recursively traversing e. - - Only sets context for expr kinds that "can appear in named assignment context" - (according to ../Parser/Python.asdl). For other expr kinds, it sets - an appropriate syntax error and returns false. - - Very similar to set_context, with additional restrictions specific to named - expressions -*/ - -static int -set_namedexpr_context(struct compiling *c, expr_ty e, expr_context_ty ctx, const node *n) -{ - /* If a particular expression type can't be used for assign / delete, - set expr_name to its name and an error message will be generated. - */ - const char* expr_name = NULL; - - /* The ast defines augmented store and load contexts, but the - implementation here doesn't actually use them. The code may be - a little more complex than necessary as a result. It also means - that expressions in an augmented assignment have a Store context. - Consider restructuring so that augmented assignment uses - set_context(), too. - */ - assert(ctx != AugStore && ctx != AugLoad); - - switch (e->kind) { - case Name_kind: - if (ctx == Store) { - if (forbidden_name(c, e->v.Name.id, n, 0)) - return 0; /* forbidden_name() calls ast_error() */ - } - e->v.Name.ctx = ctx; - break; - case Attribute_kind: - expr_name = "attribute"; - break; - case Subscript_kind: - expr_name = "subscript"; - break; - case Starred_kind: - expr_name = "starred"; - break; - case List_kind: - expr_name = "list"; - break; - case Tuple_kind: - expr_name = "tuple"; - break; - case Lambda_kind: - expr_name = "lambda"; - break; - case Call_kind: - expr_name = "function call"; - break; - case BoolOp_kind: - case BinOp_kind: - case UnaryOp_kind: - expr_name = "operator"; - break; - case GeneratorExp_kind: - expr_name = "generator expression"; - break; - case Yield_kind: - case YieldFrom_kind: - expr_name = "yield expression"; - break; - case Await_kind: - expr_name = "await expression"; - break; - case ListComp_kind: - expr_name = "list comprehension"; - break; - case SetComp_kind: - expr_name = "set comprehension"; - break; - case DictComp_kind: - expr_name = "dict comprehension"; - break; - case Dict_kind: - case Set_kind: - case Num_kind: - case Str_kind: - case Bytes_kind: - case JoinedStr_kind: - case FormattedValue_kind: - expr_name = "literal"; - break; - case NameConstant_kind: - expr_name = "keyword"; - break; - case Ellipsis_kind: - expr_name = "Ellipsis"; - break; - case Compare_kind: - expr_name = "comparison"; - break; - case IfExp_kind: - expr_name = "conditional expression"; - break; - default: - PyErr_Format(PyExc_SystemError, - "unexpected expression in named assignment %d (line %d)", - e->kind, e->lineno); - return 0; - } - /* Check for error string set by switch */ - if (expr_name) { - char buf[300]; - PyOS_snprintf(buf, sizeof(buf), - "can't use named assignment with %s", - expr_name); - return ast_error(c, n, buf); - } - - return 1; -} - static operator_ty ast_for_augassign(struct compiling *c, const node *n) { @@ -1826,7 +1742,7 @@ ast_for_namedexpr(struct compiling *c, const node *n) if (!value) return NULL; - if(!set_namedexpr_context(c, target, NamedStore, n)) + if(!set_context(c, target, NamedStore, n)) return NULL; return NamedExpr(target, value, LINENO(n), n->n_col_offset, c->c_arena); From 2ff5744a9ee42698661e49c1d74c0c22401e0df2 Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Fri, 14 Sep 2018 10:01:08 -0700 Subject: [PATCH 44/67] Cleanup - Add additional use cases for ast_for_namedexpr in usage comment. Fix multiple blank lines in test_named_expressions --- Lib/test/test_named_expressions.py | 2 -- Python/ast.c | 10 +++++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_named_expressions.py b/Lib/test/test_named_expressions.py index 1271568575b83b..03244ff720644a 100644 --- a/Lib/test/test_named_expressions.py +++ b/Lib/test/test_named_expressions.py @@ -263,7 +263,5 @@ def test_named_expression_scope_11(self): self.assertEqual(ns["j"], 4) - - if __name__ == "__main__": unittest.main() diff --git a/Python/ast.c b/Python/ast.c index 978e6c7e15aa90..5a8c1287b1658c 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -1731,7 +1731,15 @@ ast_for_decorated(struct compiling *c, const node *n) static expr_ty ast_for_namedexpr(struct compiling *c, const node *n) { - /* namedexpr_test: test [':=' test] */ + /* if_stmt: 'if' namedexpr_test ':' suite ('elif' namedexpr_test ':' suite)* + ['else' ':' suite] + namedexpr_test: test [':=' test] + argument: ( test [comp_for] | + test ':=' test | + test '=' test | + '**' test | + '*' test ) + */ expr_ty target, value; target = ast_for_expr(c, CHILD(n, 0)); From 2494008cba9080717575bf07beb395749e8d7d53 Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Fri, 14 Sep 2018 10:05:07 -0700 Subject: [PATCH 45/67] Tests - Remove unnecessary test case. Renumber test case function names --- Lib/test/test_named_expressions.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/Lib/test/test_named_expressions.py b/Lib/test/test_named_expressions.py index 03244ff720644a..89b3a161d8768e 100644 --- a/Lib/test/test_named_expressions.py +++ b/Lib/test/test_named_expressions.py @@ -71,33 +71,25 @@ def test_named_expression_invalid_11(self): exec(code, {}, {}) def test_named_expression_invalid_12(self): - code = """[[(j := j) for i in range(5)] for j in range(5)]""" - - # XXX this fails, but with the wrong exception type - should be TargetScopeError - # with self.assertRaisesRegex(TargetScopeError, ""): - with self.assertRaisesRegex(NameError, "name 'j' is not defined"): - exec(code, {}, {}) - - def test_named_expression_invalid_13(self): code = """(lambda: x := 1)""" with self.assertRaisesRegex(SyntaxError, "can't use named assignment with lambda"): exec(code, {}, {}) - def test_named_expression_invalid_14(self): + def test_named_expression_invalid_13(self): code = """((a, b) := (1, 2))""" with self.assertRaisesRegex(SyntaxError, "can't use named assignment with tuple"): exec(code, {}, {}) @unittest.skip("Not implemented -- custom check for named expr") - def test_named_expression_invalid_15(self): + def test_named_expression_invalid_14(self): code = """foo(a=1, b := 2)""" with self.assertRaisesRegex(SyntaxError, "positional argument follows keyword argument"): exec(code, {}, {}) - def test_named_expression_invalid_16(self): + def test_named_expression_invalid_15(self): code = """foo(a=1, (b := 2))""" with self.assertRaisesRegex(SyntaxError, "positional argument follows keyword argument"): From 794a1d11442250ad82f9b0daa7b07625cb289b7b Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Fri, 14 Sep 2018 10:06:41 -0700 Subject: [PATCH 46/67] Remove TargetScopeError for now. Will add back if needed --- Objects/exceptions.c | 7 ------- 1 file changed, 7 deletions(-) diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 5a48d0440bb33f..35e1df3ca1faef 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -1520,13 +1520,6 @@ MiddlingExtendsException(PyExc_SyntaxError, IndentationError, SyntaxError, "Improper indentation."); -/* - * TargetScopeError extends SyntaxError - */ -MiddlingExtendsException(PyExc_SyntaxError, TargetScopeError, SyntaxError, - "Improper scope target."); - - /* * TabError extends IndentationError */ From 415d469af1d1268ab13fc28a96054b39b70a8f85 Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Fri, 14 Sep 2018 10:33:25 -0700 Subject: [PATCH 47/67] Cleanup - Small comment nit for consistency --- Python/symtable.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Python/symtable.c b/Python/symtable.c index 317fa2470131a0..d74621386ca758 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -1543,9 +1543,8 @@ symtable_visit_expr(struct symtable *st, expr_ty e) VISIT(st, expr, e->v.Starred.value); break; case Name_kind: - /* Special-case named expr */ + /* Special-case: named expr */ if (e->v.Name.ctx == NamedStore && st->st_cur->ste_comprehension) { - /* Helper to traverse and find proper scope to add variable to outer scope */ if(!symtable_extend_namedexpr_scope(st, e)) VISIT_QUIT(st, 0); } From 162c8c44021c67253b44b9bb8df4a7c62f582ec7 Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Fri, 14 Sep 2018 10:34:11 -0700 Subject: [PATCH 48/67] Handle positional argument check with named expression --- Python/ast.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/Python/ast.c b/Python/ast.c index 5a8c1287b1658c..dbee5b973243dd 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -2924,11 +2924,24 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func, asdl_seq_SET(args, nargs++, e); } else if (TYPE(CHILD(ch, 1)) == COLONEQUAL) { + /* treat colon equal as positional argument */ + if (nkeywords) { + if (ndoublestars) { + ast_error(c, chch, + "positional argument follows " + "keyword argument unpacking"); + } + else { + ast_error(c, chch, + "positional argument follows " + "keyword argument"); + } + return NULL; + } e = ast_for_namedexpr(c, ch); if (!e) return NULL; asdl_seq_SET(args, nargs++, e); - nargs++; } else { /* a keyword argument */ From 68e9e98dadfc67deb70e4d598375aa50218f8016 Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Fri, 14 Sep 2018 11:03:43 -0700 Subject: [PATCH 49/67] Add TargetScopeError exception definition. Add documentation for TargetScopeError in c-api docs. Throw TargetScopeError instead of SyntaxError when using a named expression in a comprehension within a class scope --- Doc/c-api/exceptions.rst | 3 +++ Include/pyerrors.h | 1 + Objects/exceptions.c | 7 +++++++ PC/python3.def | 1 + Python/symtable.c | 2 +- 5 files changed, 13 insertions(+), 1 deletion(-) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index dd1e026cb0716a..6dd0c02c6d7bac 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -800,6 +800,7 @@ the variables: single: PyExc_SystemError single: PyExc_SystemExit single: PyExc_TabError + single: PyExc_TargetScopeError single: PyExc_TimeoutError single: PyExc_TypeError single: PyExc_UnboundLocalError @@ -901,6 +902,8 @@ the variables: +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_TabError` | :exc:`TabError` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_TargetScopeError` | :exc:`TargetScopeError` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_TimeoutError` | :exc:`TimeoutError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_TypeError` | :exc:`TypeError` | | diff --git a/Include/pyerrors.h b/Include/pyerrors.h index efe1c49d2d08d1..5c6751868df489 100644 --- a/Include/pyerrors.h +++ b/Include/pyerrors.h @@ -108,6 +108,7 @@ PyAPI_DATA(PyObject *) PyExc_NotImplementedError; PyAPI_DATA(PyObject *) PyExc_SyntaxError; PyAPI_DATA(PyObject *) PyExc_IndentationError; PyAPI_DATA(PyObject *) PyExc_TabError; +PyAPI_DATA(PyObject *) PyExc_TargetScopeError; PyAPI_DATA(PyObject *) PyExc_ReferenceError; PyAPI_DATA(PyObject *) PyExc_SystemError; PyAPI_DATA(PyObject *) PyExc_SystemExit; diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 35e1df3ca1faef..5a48d0440bb33f 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -1520,6 +1520,13 @@ MiddlingExtendsException(PyExc_SyntaxError, IndentationError, SyntaxError, "Improper indentation."); +/* + * TargetScopeError extends SyntaxError + */ +MiddlingExtendsException(PyExc_SyntaxError, TargetScopeError, SyntaxError, + "Improper scope target."); + + /* * TabError extends IndentationError */ diff --git a/PC/python3.def b/PC/python3.def index 5d93c18af87ec2..e317864d0cd802 100644 --- a/PC/python3.def +++ b/PC/python3.def @@ -235,6 +235,7 @@ EXPORTS PyExc_SystemError=python38.PyExc_SystemError DATA PyExc_SystemExit=python38.PyExc_SystemExit DATA PyExc_TabError=python38.PyExc_TabError DATA + PyExc_TargetScopeError=python38.PyExc_TargetScopeError DATA PyExc_TimeoutError=python38.PyExc_TimeoutError DATA PyExc_TypeError=python38.PyExc_TypeError DATA PyExc_UnboundLocalError=python38.PyExc_UnboundLocalError DATA diff --git a/Python/symtable.c b/Python/symtable.c index d74621386ca758..879e19ab79e048 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -1415,7 +1415,7 @@ symtable_extend_namedexpr_scope(struct symtable *st, expr_ty e) } /* Disallow usage in ClassBlock */ if (ste->ste_type == ClassBlock) { - PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_IN_CLASS, e->v.Name.id); + PyErr_Format(PyExc_TargetScopeError, NAMED_EXPR_COMP_IN_CLASS, e->v.Name.id); PyErr_SyntaxLocationObject(st->st_filename, e->lineno, e->col_offset); From 1ad4b30966664386a190f76325cddc76fee0ea86 Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Fri, 14 Sep 2018 12:03:13 -0700 Subject: [PATCH 50/67] Increase stack size for parser by 200. This is a minimal change (approx. 5kb) and should not have an impact on any systems. Update parser test to allow 99 nested levels again --- Lib/test/test_parser.py | 2 +- Parser/parser.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_parser.py b/Lib/test/test_parser.py index 3e025729cc92eb..5b48f192f014a9 100644 --- a/Lib/test/test_parser.py +++ b/Lib/test/test_parser.py @@ -808,7 +808,7 @@ def _nested_expression(self, level): def test_deeply_nested_list(self): # XXX used to be 99 levels in 2.x, used to be 93 levels in 3.7.X - e = self._nested_expression(88) + e = self._nested_expression(99) st = parser.expr(e) st.compile() diff --git a/Parser/parser.h b/Parser/parser.h index 95cd39d209dd17..aee1c86cb044e9 100644 --- a/Parser/parser.h +++ b/Parser/parser.h @@ -7,7 +7,7 @@ extern "C" { /* Parser interface */ -#define MAXSTACK 1500 +#define MAXSTACK 1700 typedef struct { int s_state; /* State in current DFA */ From 1b1d84dead47b1b47590a98c41bcefd875e13397 Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Fri, 14 Sep 2018 15:33:09 -0700 Subject: [PATCH 51/67] Add TargetScopeError to exception_hierarchy.txt for test_baseexception.py_ --- Lib/test/exception_hierarchy.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/exception_hierarchy.txt b/Lib/test/exception_hierarchy.txt index 763a6c899b48eb..15f4491cf237d2 100644 --- a/Lib/test/exception_hierarchy.txt +++ b/Lib/test/exception_hierarchy.txt @@ -42,6 +42,7 @@ BaseException | +-- NotImplementedError | +-- RecursionError +-- SyntaxError + | +-- TargetScopeError | +-- IndentationError | +-- TabError +-- SystemError From b80108bd3d2c4fb8c81fcfcf218a98e73de9b04f Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Fri, 14 Sep 2018 15:36:10 -0700 Subject: [PATCH 52/67] Tests - Major update for named expression tests, both in test_named_expressions and test_parser - Add test for TargetScopeError - Add tests for named expressions in comprehension scope and edge cases - Add tests for named expressions in function arguments (declarations and call sites) - Reorganize tests to group them more logically --- Lib/test/test_named_expressions.py | 254 +++++++++++++++++++++++------ Lib/test/test_parser.py | 4 + 2 files changed, 207 insertions(+), 51 deletions(-) diff --git a/Lib/test/test_named_expressions.py b/Lib/test/test_named_expressions.py index 89b3a161d8768e..0821094de93404 100644 --- a/Lib/test/test_named_expressions.py +++ b/Lib/test/test_named_expressions.py @@ -17,82 +17,96 @@ def test_named_expression_invalid_02(self): exec(code, {}, {}) def test_named_expression_invalid_03(self): - code = """foo(cat=category := 'vector')""" - - with self.assertRaisesRegex(SyntaxError, "invalid syntax"): - exec(code, {}, {}) - - def test_named_expression_invalid_04(self): code = """y := f(x)""" with self.assertRaisesRegex(SyntaxError, "invalid syntax"): exec(code, {}, {}) - def test_named_expression_invalid_05(self): + def test_named_expression_invalid_04(self): code = """y0 = y1 := f(x)""" with self.assertRaisesRegex(SyntaxError, "invalid syntax"): exec(code, {}, {}) def test_named_expression_invalid_06(self): - code = """foo(x = y := f(x))""" + code = """((a, b) := (1, 2))""" - with self.assertRaisesRegex(SyntaxError, "invalid syntax"): + with self.assertRaisesRegex(SyntaxError, "can't use named assignment with tuple"): exec(code, {}, {}) def test_named_expression_invalid_07(self): - code = """def foo(answer = p := 42): pass""" + code = """def spam(a = b := 42): pass""" with self.assertRaisesRegex(SyntaxError, "invalid syntax"): exec(code, {}, {}) def test_named_expression_invalid_08(self): - code = """def foo(answer: p := 42 = 5): pass""" + code = """def spam(a: b := 42 = 5): pass""" with self.assertRaisesRegex(SyntaxError, "invalid syntax"): exec(code, {}, {}) def test_named_expression_invalid_09(self): - code = """z := y := x := 0") # not in P""" + code = """spam(a=b := 'c')""" with self.assertRaisesRegex(SyntaxError, "invalid syntax"): exec(code, {}, {}) def test_named_expression_invalid_10(self): - code = """info := (name, phone, *rest)""" + code = """spam(x = y := f(x))""" with self.assertRaisesRegex(SyntaxError, "invalid syntax"): exec(code, {}, {}) def test_named_expression_invalid_11(self): - code = "[i + 1 for i in i := [1,2]]" + code = """spam(a=1, b := 2)""" - with self.assertRaisesRegex(SyntaxError, "invalid syntax"): + with self.assertRaisesRegex(SyntaxError, "positional argument follows keyword argument"): exec(code, {}, {}) def test_named_expression_invalid_12(self): - code = """(lambda: x := 1)""" + code = """spam(a=1, (b := 2))""" - with self.assertRaisesRegex(SyntaxError, "can't use named assignment with lambda"): + with self.assertRaisesRegex(SyntaxError, "positional argument follows keyword argument"): exec(code, {}, {}) def test_named_expression_invalid_13(self): - code = """((a, b) := (1, 2))""" + code = """spam(a=1, (b := 2))""" - with self.assertRaisesRegex(SyntaxError, "can't use named assignment with tuple"): + with self.assertRaisesRegex(SyntaxError, "positional argument follows keyword argument"): exec(code, {}, {}) - @unittest.skip("Not implemented -- custom check for named expr") def test_named_expression_invalid_14(self): - code = """foo(a=1, b := 2)""" + code = """(x := lambda: y := 1)""" - with self.assertRaisesRegex(SyntaxError, "positional argument follows keyword argument"): + with self.assertRaisesRegex(SyntaxError, "invalid syntax"): exec(code, {}, {}) def test_named_expression_invalid_15(self): - code = """foo(a=1, (b := 2))""" + code = """(lambda: x := 1)""" - with self.assertRaisesRegex(SyntaxError, "positional argument follows keyword argument"): + with self.assertRaisesRegex(SyntaxError, "can't use named assignment with lambda"): + exec(code, {}, {}) + + def test_named_expression_invalid_16(self): + code = "[i + 1 for i in i := [1,2]]" + + with self.assertRaisesRegex(SyntaxError, "invalid syntax"): + exec(code, {}, {}) + + def test_named_expression_invalid_17(self): + code = "[i := 0, j := 1 for i, j in [(1, 2), (3, 4)]]" + + with self.assertRaisesRegex(SyntaxError, "invalid syntax"): + exec(code, {}, {}) + + def test_named_expression_invalid_18(self): + code = """class Foo(): + [(42, 1 + ((( j := i )))) for i in range(5)] + """ + + with self.assertRaisesRegex(TargetScopeError, + "named expression within a comprehension cannot be used in a class body"): exec(code, {}, {}) @@ -100,60 +114,79 @@ class NamedExpressionAssignmentTest(unittest.TestCase): def test_named_expression_assignment_01(self): (a := 10) + self.assertEqual(a, 10) def test_named_expression_assignment_02(self): a = 20 (a := a) + self.assertEqual(a, 20) def test_named_expression_assignment_03(self): (total := 1 + 2) + self.assertEqual(total, 3) def test_named_expression_assignment_04(self): + (info := (1, 2, 3)) + + self.assertEqual(info, (1, 2, 3)) + + def test_named_expression_assignment_05(self): + (x := 1, 2) + + self.assertEqual(x, 1) + + def test_named_expression_assignment_06(self): + (z := (y := (x := 0))) + + self.assertEqual(x, 0) + self.assertEqual(y, 0) + self.assertEqual(z, 0) + + def test_named_expression_assignment_07(self): + (loc := (1, 2)) + + self.assertEqual(loc, (1, 2)) + + def test_named_expression_assignment_08(self): if spam := "eggs": self.assertEqual(spam, "eggs") else: self.fail("variable was not assigned using named expression") - def test_named_expression_assignment_05(self): + def test_named_expression_assignment_09(self): if True and (spam := True): self.assertTrue(spam) else: self.fail("variable was not assigned using named expression") - def test_named_expression_assignment_06(self): + def test_named_expression_assignment_10(self): if (match := 10) is 10: pass else: self.fail("variable was not assigned using named expression") - def test_named_expression_assignment_07(self): + def test_named_expression_assignment_11(self): def spam(a): return a input_data = [1, 2, 3] res = [(x, y, x/y) for x in input_data if (y := spam(x)) > 0] + self.assertEqual(res, [(1, 1, 1.0), (2, 2, 1.0), (3, 3, 1.0)]) - def test_named_expression_assignment_08(self): + def test_named_expression_assignment_12(self): def spam(a): return a res = [[y := spam(x), x/y] for x in range(1, 5)] - self.assertEqual(res, [[1, 1.0], [2, 1.0], [3, 1.0], [4, 1.0]]) - def test_named_expression_assignment_09(self): - (x := 1, 2) - self.assertEqual(x, 1) + self.assertEqual(res, [[1, 1.0], [2, 1.0], [3, 1.0], [4, 1.0]]) - def test_named_expression_assignment_10(self): - (z := (y := (x := 0))) - self.assertEqual(x, 0) - self.assertEqual(y, 0) - self.assertEqual(z, 0) + def test_named_expression_assignment_13(self): + length = len(lines := [1, 2]) - def test_named_expression_assignment_11(self): - (loc := (1, 2)) - self.assertEqual(loc, (1, 2)) + self.assertEqual(length, 2) + self.assertEqual(lines, [1,2]) - def test_named_expression_assignment_12(self): + def test_named_expression_assignment_14(self): """ Where all variables are positive integers, and a is at least as large as the n'th root of x, this algorithm returns the floor of the n'th @@ -169,27 +202,27 @@ def test_named_expression_assignment_12(self): self.assertEqual(a, 1) - def test_named_expression_assignment_13(self): - length = len(lines := [1, 2]) - self.assertEqual(length, 2) - self.assertEqual(lines, [1,2]) - class NamedExpressionScopeTest(unittest.TestCase): def test_named_expression_scope_01(self): - code = 'def foo():\n\n (a := 5)\nprint(a)\n' + code = """def spam(): + (a := 5) +print(a)""" + with self.assertRaisesRegex(NameError, "name 'a' is not defined"): exec(code, {}, {}) def test_named_expression_scope_02(self): total = 0 partial_sums = [total := total + v for v in range(5)] + self.assertEqual(partial_sums, [0, 1, 3, 6, 10]) self.assertEqual(total, 10) def test_named_expression_scope_03(self): containsOne = any((lastNum := num) == 1 for num in [1, 2, 3]) + self.assertTrue(containsOne) self.assertEqual(lastNum, 1) @@ -197,6 +230,7 @@ def test_named_expression_scope_04(self): def spam(a): return a res = [[y := spam(x), x/y] for x in range(1, 5)] + self.assertEqual(y, 4) def test_named_expression_scope_05(self): @@ -204,16 +238,19 @@ def spam(a): return a input_data = [1, 2, 3] res = [(x, y, x/y) for x in input_data if (y := spam(x)) > 0] + self.assertEqual(res, [(1, 1, 1.0), (2, 2, 1.0), (3, 3, 1.0)]) self.assertEqual(y, 3) - def test_named_expression_scope_05(self): + def test_named_expression_scope_06(self): res = [[spam := i for i in range(3)] for j in range(2)] + self.assertEqual(res, [[0, 1, 2], [0, 1, 2]]) self.assertEqual(spam, 2) def test_named_expression_scope_07(self): len(lines := [1, 2]) + self.assertEqual(lines, [1, 2]) def test_named_expression_scope_08(self): @@ -224,6 +261,7 @@ def eggs(b): return b * 2 res = [spam(a := eggs(b := h)) for h in range(2)] + self.assertEqual(res, [0, 2]) self.assertEqual(a, 2) self.assertEqual(b, 1) @@ -236,6 +274,7 @@ def eggs(b): return b * 2 res = [spam(a := eggs(a := h)) for h in range(2)] + self.assertEqual(res, [0, 2]) self.assertEqual(a, 2) @@ -246,13 +285,126 @@ def test_named_expression_scope_10(self): self.assertEqual(a, 1) self.assertEqual(b, [1, 1]) - def test_named_expression_scope_11(self): + res = [j := i for i in range(5)] + + self.assertEqual(res, [0, 1, 2, 3, 4]) + self.assertEqual(j, 4) + + def test_named_expression_scope_12(self): + res = [i := i for i in range(5)] + + self.assertEqual(res, [0, 1, 2, 3, 4]) + self.assertEqual(i, 4) + + def test_named_expression_scope_13(self): + res = [i := 0 for i, j in [(1, 2), (3, 4)]] + + self.assertEqual(res, [0, 0]) + self.assertEqual(i, 0) + + def test_named_expression_scope_14(self): + res = [(i := 0, j := 1) for i, j in [(1, 2), (3, 4)]] + + self.assertEqual(res, [(0, 1), (0, 1)]) + self.assertEqual(i, 0) + self.assertEqual(j, 1) + + def test_named_expression_scope_15(self): + res = [(i := i, j := j) for i, j in [(1, 2), (3, 4)]] + + self.assertEqual(res, [(1, 2), (3, 4)]) + self.assertEqual(i, 3) + self.assertEqual(j, 4) + + def test_named_expression_scope_16(self): + res = [(i := j, j := i) for i, j in [(1, 2), (3, 4)]] + + self.assertEqual(res, [(2, 2), (4, 4)]) + self.assertEqual(i, 4) + self.assertEqual(j, 4) + + def test_named_expression_scope_17(self): + b = 0 + res = [b := i + b for i in range(5)] + + self.assertEqual(res, [0, 1, 3, 6, 10]) + self.assertEqual(b, 10) + + def test_named_expression_scope_18(self): + def spam(a): + return a + + res = spam(b := 2) + + self.assertEqual(res, 2) + self.assertEqual(b, 2) + + def test_named_expression_scope_19(self): + def spam(a): + return a + + res = spam((b := 2)) + + self.assertEqual(res, 2) + self.assertEqual(b, 2) + + def test_named_expression_scope_20(self): + def spam(a): + return a + + res = spam(a=(b := 2)) + + self.assertEqual(res, 2) + self.assertEqual(b, 2) + + def test_named_expression_scope_21(self): + def spam(a, b): + return a + b + + res = spam(c := 2, b=1) + + self.assertEqual(res, 3) + self.assertEqual(c, 2) + + def test_named_expression_scope_22(self): + def spam(a, b): + return a + b + + res = spam((c := 2), b=1) + + self.assertEqual(res, 3) + self.assertEqual(c, 2) + + def test_named_expression_scope_23(self): + def spam(a, b): + return a + b + + res = spam(b=(c := 2), a=1) + + self.assertEqual(res, 3) + self.assertEqual(c, 2) + + def test_named_expression_scope_24(self): + a = 10 + def spam(): + nonlocal a + (a := 20) + spam() + + self.assertEqual(a, 20) + + def test_named_expression_scope_25(self): ns = {} - code = "[j := i for i in range(5)]" + code = """a = 10 +def spam(): + global a + (a := 20) +spam()""" + exec(code, ns, {}) - self.assertEqual(ns["j"], 4) + self.assertEqual(ns["a"], 20) if __name__ == "__main__": diff --git a/Lib/test/test_parser.py b/Lib/test/test_parser.py index 5b48f192f014a9..9c5fc3ba425d48 100644 --- a/Lib/test/test_parser.py +++ b/Lib/test/test_parser.py @@ -452,6 +452,10 @@ def test_named_expressions(self): self.check_suite( "if self._is_special and (ans := self._check_nans(context=context)): return ans" ) + self.check_suite("foo(b := 2, a=1)") + self.check_suite("foo(b := 2, a=1)") + self.check_suite("foo((b := 2), a=1)") + self.check_suite("foo(c=(b := 2), a=1)") # # Second, we take *invalid* trees and make sure we get ParserError From 6e15f5f6833eba10fd761b16f3f28b2ec491e95f Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Fri, 14 Sep 2018 15:38:33 -0700 Subject: [PATCH 53/67] Cleanup - Remove unnecessary comment --- Python/ast.c | 1 - 1 file changed, 1 deletion(-) diff --git a/Python/ast.c b/Python/ast.c index dbee5b973243dd..d336e1e8ba1beb 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -2658,7 +2658,6 @@ ast_for_expr(struct compiling *c, const node *n) loop: switch (TYPE(n)) { case namedexpr_test: - // If there are 3 children, continue with named expression if (NCH(n) == 3) return ast_for_namedexpr(c, n); case test: From da11136a33b6886009314f3b2ba6f745bcb19ec3 Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Fri, 14 Sep 2018 15:59:06 -0700 Subject: [PATCH 54/67] Cleanup - Comment nitpicks --- Lib/test/test_parser.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_parser.py b/Lib/test/test_parser.py index 9c5fc3ba425d48..ac3899baedb629 100644 --- a/Lib/test/test_parser.py +++ b/Lib/test/test_parser.py @@ -811,7 +811,8 @@ def _nested_expression(self, level): return "["*level+"]"*level def test_deeply_nested_list(self): - # XXX used to be 99 levels in 2.x, used to be 93 levels in 3.7.X + # This has fluctuated between 99 levels in 2.x, down to 93 levels in + # 3.7.X and back up to 99 in 3.8.X. Related to MAXSTACK size in Parser.h e = self._nested_expression(99) st = parser.expr(e) st.compile() From 051ad8c998ed48d25d0b246181801a02d60e843d Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Sat, 15 Sep 2018 13:11:44 -0600 Subject: [PATCH 55/67] Explicitly disallow assignment expressions to a name inside parentheses, e.g.: ((x) := 0) - Add check for LHS types to detect a parenthesis then a name (see note) - Add test for this scenario - Update tests for changed error message for named assignment to a tuple (also, see note) Note: This caused issues with the previous error handling for named assignment to a LHS that contained an expression, such as a tuple. Thus, the check for the LHS of a named expression must be changed to be more specific if we wish to maintain the previous error messages --- Lib/test/test_named_expressions.py | 15 +++++++++- Python/ast.c | 45 +++++++++++++++++++++++++++++- 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_named_expressions.py b/Lib/test/test_named_expressions.py index 0821094de93404..14ce980fd65018 100644 --- a/Lib/test/test_named_expressions.py +++ b/Lib/test/test_named_expressions.py @@ -10,6 +10,13 @@ def test_named_expression_invalid_01(self): with self.assertRaisesRegex(SyntaxError, "invalid syntax"): exec(code, {}, {}) + def test_named_expression_invalid_01(self): + code = """((x) := 0)""" + + with self.assertRaisesRegex(SyntaxError, + "can't use named assignment with expression"): + exec(code, {}, {}) + def test_named_expression_invalid_02(self): code = """x = y := 0""" @@ -28,10 +35,16 @@ def test_named_expression_invalid_04(self): with self.assertRaisesRegex(SyntaxError, "invalid syntax"): exec(code, {}, {}) + # XXX this error message now follows the error message you get with + # keyword args: + # >>> spam((1,2)=1) + # File "", line 1 + # SyntaxError: keyword can't be an expression def test_named_expression_invalid_06(self): code = """((a, b) := (1, 2))""" - with self.assertRaisesRegex(SyntaxError, "can't use named assignment with tuple"): + with self.assertRaisesRegex(SyntaxError, + "can't use named assignment with expression"): exec(code, {}, {}) def test_named_expression_invalid_07(self): diff --git a/Python/ast.c b/Python/ast.c index d336e1e8ba1beb..02c1c1a2c6cf7b 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -1741,8 +1741,51 @@ ast_for_namedexpr(struct compiling *c, const node *n) '*' test ) */ expr_ty target, value; + node *ch = CHILD(n, 0); + + // To remain LL(1), the grammar accepts any test (basically, any + // expression) in the keyword slot of a call site. So, we need + // to manually enforce that the keyword is a NAME here. + static const int name_tree[] = { + namedexpr_test, + test, + or_test, + and_test, + not_test, + comparison, + expr, + xor_expr, + and_expr, + shift_expr, + arith_expr, + term, + factor, + power, + atom_expr, + atom, + }; + + node *expr_node = ch; + for (int i = 1; name_tree[i]; i++) { + if (TYPE(expr_node) != name_tree[i]) + break; + if (TYPE(expr_node) == atom && expr_node->n_nchildren == 3) { + node *expr_node_ch = CHILD(CHILD(expr_node, 1), 0); + for (int i = 0; name_tree[i]; i++) { + if (TYPE(expr_node_ch) != name_tree[i]) + break; + expr_node_ch = CHILD(expr_node_ch, 0); + } + if (TYPE(expr_node_ch) == NAME) { + ast_error(c, ch, + "can't use named assignment with expression"); + return NULL; + } + } + expr_node = CHILD(expr_node, 0); + } - target = ast_for_expr(c, CHILD(n, 0)); + target = ast_for_expr(c, ch); if (!target) return NULL; From f893035b71b51ea3f73817d70f0de6fbafcb45d1 Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Tue, 25 Sep 2018 10:24:38 -0600 Subject: [PATCH 56/67] Cleanup - Wrap lines more strictly in test file --- Lib/test/test_named_expressions.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_named_expressions.py b/Lib/test/test_named_expressions.py index 14ce980fd65018..1de26ddbb163f9 100644 --- a/Lib/test/test_named_expressions.py +++ b/Lib/test/test_named_expressions.py @@ -74,19 +74,22 @@ def test_named_expression_invalid_10(self): def test_named_expression_invalid_11(self): code = """spam(a=1, b := 2)""" - with self.assertRaisesRegex(SyntaxError, "positional argument follows keyword argument"): + with self.assertRaisesRegex(SyntaxError, + "positional argument follows keyword argument"): exec(code, {}, {}) def test_named_expression_invalid_12(self): code = """spam(a=1, (b := 2))""" - with self.assertRaisesRegex(SyntaxError, "positional argument follows keyword argument"): + with self.assertRaisesRegex(SyntaxError, + "positional argument follows keyword argument"): exec(code, {}, {}) def test_named_expression_invalid_13(self): code = """spam(a=1, (b := 2))""" - with self.assertRaisesRegex(SyntaxError, "positional argument follows keyword argument"): + with self.assertRaisesRegex(SyntaxError, + "positional argument follows keyword argument"): exec(code, {}, {}) def test_named_expression_invalid_14(self): @@ -98,7 +101,8 @@ def test_named_expression_invalid_14(self): def test_named_expression_invalid_15(self): code = """(lambda: x := 1)""" - with self.assertRaisesRegex(SyntaxError, "can't use named assignment with lambda"): + with self.assertRaisesRegex(SyntaxError, + "can't use named assignment with lambda"): exec(code, {}, {}) def test_named_expression_invalid_16(self): From 16d71370bc24bfe21cf817d44030043af8ef5b8e Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Tue, 13 Nov 2018 14:18:12 -0700 Subject: [PATCH 57/67] Revert "Explicitly disallow assignment expressions to a name inside parentheses, e.g.: ((x) := 0)" This reverts commit f1531400ca7d7a2d148830c8ac703f041740896d. --- Lib/test/test_named_expressions.py | 15 +--------- Python/ast.c | 45 +----------------------------- 2 files changed, 2 insertions(+), 58 deletions(-) diff --git a/Lib/test/test_named_expressions.py b/Lib/test/test_named_expressions.py index 1de26ddbb163f9..62e5605ee46a1f 100644 --- a/Lib/test/test_named_expressions.py +++ b/Lib/test/test_named_expressions.py @@ -10,13 +10,6 @@ def test_named_expression_invalid_01(self): with self.assertRaisesRegex(SyntaxError, "invalid syntax"): exec(code, {}, {}) - def test_named_expression_invalid_01(self): - code = """((x) := 0)""" - - with self.assertRaisesRegex(SyntaxError, - "can't use named assignment with expression"): - exec(code, {}, {}) - def test_named_expression_invalid_02(self): code = """x = y := 0""" @@ -35,16 +28,10 @@ def test_named_expression_invalid_04(self): with self.assertRaisesRegex(SyntaxError, "invalid syntax"): exec(code, {}, {}) - # XXX this error message now follows the error message you get with - # keyword args: - # >>> spam((1,2)=1) - # File "", line 1 - # SyntaxError: keyword can't be an expression def test_named_expression_invalid_06(self): code = """((a, b) := (1, 2))""" - with self.assertRaisesRegex(SyntaxError, - "can't use named assignment with expression"): + with self.assertRaisesRegex(SyntaxError, "can't use named assignment with tuple"): exec(code, {}, {}) def test_named_expression_invalid_07(self): diff --git a/Python/ast.c b/Python/ast.c index 02c1c1a2c6cf7b..d336e1e8ba1beb 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -1741,51 +1741,8 @@ ast_for_namedexpr(struct compiling *c, const node *n) '*' test ) */ expr_ty target, value; - node *ch = CHILD(n, 0); - - // To remain LL(1), the grammar accepts any test (basically, any - // expression) in the keyword slot of a call site. So, we need - // to manually enforce that the keyword is a NAME here. - static const int name_tree[] = { - namedexpr_test, - test, - or_test, - and_test, - not_test, - comparison, - expr, - xor_expr, - and_expr, - shift_expr, - arith_expr, - term, - factor, - power, - atom_expr, - atom, - }; - - node *expr_node = ch; - for (int i = 1; name_tree[i]; i++) { - if (TYPE(expr_node) != name_tree[i]) - break; - if (TYPE(expr_node) == atom && expr_node->n_nchildren == 3) { - node *expr_node_ch = CHILD(CHILD(expr_node, 1), 0); - for (int i = 0; name_tree[i]; i++) { - if (TYPE(expr_node_ch) != name_tree[i]) - break; - expr_node_ch = CHILD(expr_node_ch, 0); - } - if (TYPE(expr_node_ch) == NAME) { - ast_error(c, ch, - "can't use named assignment with expression"); - return NULL; - } - } - expr_node = CHILD(expr_node, 0); - } - target = ast_for_expr(c, ch); + target = ast_for_expr(c, CHILD(n, 0)); if (!target) return NULL; From 070983b5aabebc92209898781b17c81e1c58f07f Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Tue, 13 Nov 2018 14:28:02 -0700 Subject: [PATCH 58/67] Add NEWS.d entry --- .../Core and Builtins/2018-11-13-14-26-54.bpo-35224.F0B6UQ.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2018-11-13-14-26-54.bpo-35224.F0B6UQ.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-11-13-14-26-54.bpo-35224.F0B6UQ.rst b/Misc/NEWS.d/next/Core and Builtins/2018-11-13-14-26-54.bpo-35224.F0B6UQ.rst new file mode 100644 index 00000000000000..ab446c52614d4c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-11-13-14-26-54.bpo-35224.F0B6UQ.rst @@ -0,0 +1,2 @@ +Implement :pep:`572` (assignment expressions). Documentation and edge cases +will be issued in separate PRs. Patch by Emily Morehouse. From 5133800eaf221c1b6f749f8ea5ac49ee01f1a0ba Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Tue, 13 Nov 2018 14:33:39 -0700 Subject: [PATCH 59/67] Tests - Fix error in test_pickle.test_exceptions by adding TargetScopeError to list of exceptions --- Lib/_compat_pickle.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/_compat_pickle.py b/Lib/_compat_pickle.py index f68496ae639f5f..8bb1cf80afa559 100644 --- a/Lib/_compat_pickle.py +++ b/Lib/_compat_pickle.py @@ -128,6 +128,7 @@ "SystemError", "SystemExit", "TabError", + "TargetScopeError", "TypeError", "UnboundLocalError", "UnicodeDecodeError", From bc2fed3a0f2efa8ae25b6c049677afae90543db1 Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Tue, 20 Nov 2018 11:30:30 -0700 Subject: [PATCH 60/67] Tests - Update error message tests to reflect improved messaging convention (s/can't/cannot) --- Lib/test/test_named_expressions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_named_expressions.py b/Lib/test/test_named_expressions.py index 62e5605ee46a1f..e49fd7de20deb9 100644 --- a/Lib/test/test_named_expressions.py +++ b/Lib/test/test_named_expressions.py @@ -31,7 +31,7 @@ def test_named_expression_invalid_04(self): def test_named_expression_invalid_06(self): code = """((a, b) := (1, 2))""" - with self.assertRaisesRegex(SyntaxError, "can't use named assignment with tuple"): + with self.assertRaisesRegex(SyntaxError, "cannot use named assignment with tuple"): exec(code, {}, {}) def test_named_expression_invalid_07(self): @@ -89,7 +89,7 @@ def test_named_expression_invalid_15(self): code = """(lambda: x := 1)""" with self.assertRaisesRegex(SyntaxError, - "can't use named assignment with lambda"): + "cannot use named assignment with lambda"): exec(code, {}, {}) def test_named_expression_invalid_16(self): From f1cabf8657fea5652ff51d05e6dc5bbe2747bfa7 Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Tue, 27 Nov 2018 11:11:26 -0700 Subject: [PATCH 61/67] Remove cases that cannot be reached in compile.c. Small linting update. --- Python/ast.c | 2 +- Python/compile.c | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/Python/ast.c b/Python/ast.c index d336e1e8ba1beb..7e1fc4908afbc1 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -1750,7 +1750,7 @@ ast_for_namedexpr(struct compiling *c, const node *n) if (!value) return NULL; - if(!set_context(c, target, NamedStore, n)) + if (!set_context(c, target, NamedStore, n)) return NULL; return NamedExpr(target, value, LINENO(n), n->n_col_offset, c->c_arena); diff --git a/Python/compile.c b/Python/compile.c index 486ccd170a0071..d96388b16db523 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -4675,7 +4675,6 @@ compiler_visit_expr1(struct compiler *c, expr_ty e) ADDOP(c, ROT_TWO); /* Fall through */ case Store: - case NamedStore: ADDOP_NAME(c, STORE_ATTR, e->v.Attribute.attr, names); break; case Del: @@ -4701,10 +4700,6 @@ compiler_visit_expr1(struct compiler *c, expr_ty e) case AugStore: VISIT_SLICE(c, e->v.Subscript.slice, AugStore); break; - case NamedStore: - VISIT(c, expr, e->v.Subscript.value); - VISIT_SLICE(c, e->v.Subscript.slice, NamedStore); - break; case Store: VISIT(c, expr, e->v.Subscript.value); VISIT_SLICE(c, e->v.Subscript.slice, Store); @@ -4723,7 +4718,6 @@ compiler_visit_expr1(struct compiler *c, expr_ty e) case Starred_kind: switch (e->v.Starred.ctx) { case Store: - case NamedStore: /* In all legitimate cases, the Starred node was already replaced * by compiler_list/compiler_tuple. XXX: is that okay? */ return compiler_error(c, From 0e0ca07b4d1623f6f9f931dc86961489025daf7d Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Tue, 22 Jan 2019 10:56:34 -0700 Subject: [PATCH 62/67] Update Grammar/Tokens to add COLONEQUAL. Regenerate all files --- Doc/library/token-list.inc | 4 ++ Grammar/Tokens | 1 + Include/token.h | 2 +- Lib/token.py | 109 ++++++++++++++++++------------------- Parser/token.c | 6 ++ 5 files changed, 64 insertions(+), 58 deletions(-) diff --git a/Doc/library/token-list.inc b/Doc/library/token-list.inc index cd6e0f26968eea..3ea9439be859a2 100644 --- a/Doc/library/token-list.inc +++ b/Doc/library/token-list.inc @@ -197,6 +197,10 @@ Token value for ``"..."``. +.. data:: COLONEQUAL + + Token value for ``":="``. + .. data:: OP .. data:: ERRORTOKEN diff --git a/Grammar/Tokens b/Grammar/Tokens index 9595673a5af7a4..f6f303bd5292aa 100644 --- a/Grammar/Tokens +++ b/Grammar/Tokens @@ -52,6 +52,7 @@ AT '@' ATEQUAL '@=' RARROW '->' ELLIPSIS '...' +COLONEQUAL ':=' OP ERRORTOKEN diff --git a/Include/token.h b/Include/token.h index f7a44126feb4a2..b87b84cd966d01 100644 --- a/Include/token.h +++ b/Include/token.h @@ -66,7 +66,7 @@ extern "C" { #define COLONEQUAL 53 #define OP 54 #define ERRORTOKEN 55 -#define N_TOKENS 58 +#define N_TOKENS 59 #define NT_OFFSET 256 /* Special definitions for cooperation with parser */ diff --git a/Lib/token.py b/Lib/token.py index c78c631162a93f..7224eca32fe0a4 100644 --- a/Lib/token.py +++ b/Lib/token.py @@ -1,7 +1,7 @@ """Token constants.""" # Auto-generated by Tools/scripts/generate_token.py -__all__ = ["tok_name", "ISTERMINAL", "ISNONTERMINAL", "ISEOF"] +__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF'] ENDMARKER = 0 NAME = 1 @@ -57,10 +57,9 @@ RARROW = 51 ELLIPSIS = 52 COLONEQUAL = 53 -# Don't forget to update the table _PyParser_TokenNames in tokenizer.c! OP = 54 -ERRORTOKEN = 55 # These aren't used by the C tokenizer but are needed for tokenize.py +ERRORTOKEN = 55 COMMENT = 56 NL = 57 ENCODING = 58 @@ -68,70 +67,66 @@ # Special definitions for cooperation with parser NT_OFFSET = 256 -tok_name = { - value: name - for name, value in globals().items() - if isinstance(value, int) and not name.startswith("_") -} +tok_name = {value: name + for name, value in globals().items() + if isinstance(value, int) and not name.startswith('_')} __all__.extend(tok_name.values()) EXACT_TOKEN_TYPES = { - "!=": NOTEQUAL, - "%": PERCENT, - "%=": PERCENTEQUAL, - "&": AMPER, - "&=": AMPEREQUAL, - "(": LPAR, - ")": RPAR, - "*": STAR, - "**": DOUBLESTAR, - "**=": DOUBLESTAREQUAL, - "*=": STAREQUAL, - "+": PLUS, - "+=": PLUSEQUAL, - ",": COMMA, - "-": MINUS, - "-=": MINEQUAL, - "->": RARROW, - ".": DOT, - "...": ELLIPSIS, - "/": SLASH, - "//": DOUBLESLASH, - "//=": DOUBLESLASHEQUAL, - "/=": SLASHEQUAL, - ":": COLON, - ";": SEMI, - "<": LESS, - "<<": LEFTSHIFT, - "<<=": LEFTSHIFTEQUAL, - "<=": LESSEQUAL, - "=": EQUAL, - "==": EQEQUAL, - ">": GREATER, - ">=": GREATEREQUAL, - ">>": RIGHTSHIFT, - ">>=": RIGHTSHIFTEQUAL, - "@": AT, - "@=": ATEQUAL, - "[": LSQB, - "]": RSQB, - "^": CIRCUMFLEX, - "^=": CIRCUMFLEXEQUAL, - "{": LBRACE, - "|": VBAR, - "|=": VBAREQUAL, - "}": RBRACE, - "~": TILDE, + '!=': NOTEQUAL, + '%': PERCENT, + '%=': PERCENTEQUAL, + '&': AMPER, + '&=': AMPEREQUAL, + '(': LPAR, + ')': RPAR, + '*': STAR, + '**': DOUBLESTAR, + '**=': DOUBLESTAREQUAL, + '*=': STAREQUAL, + '+': PLUS, + '+=': PLUSEQUAL, + ',': COMMA, + '-': MINUS, + '-=': MINEQUAL, + '->': RARROW, + '.': DOT, + '...': ELLIPSIS, + '/': SLASH, + '//': DOUBLESLASH, + '//=': DOUBLESLASHEQUAL, + '/=': SLASHEQUAL, + ':': COLON, + ':=': COLONEQUAL, + ';': SEMI, + '<': LESS, + '<<': LEFTSHIFT, + '<<=': LEFTSHIFTEQUAL, + '<=': LESSEQUAL, + '=': EQUAL, + '==': EQEQUAL, + '>': GREATER, + '>=': GREATEREQUAL, + '>>': RIGHTSHIFT, + '>>=': RIGHTSHIFTEQUAL, + '@': AT, + '@=': ATEQUAL, + '[': LSQB, + ']': RSQB, + '^': CIRCUMFLEX, + '^=': CIRCUMFLEXEQUAL, + '{': LBRACE, + '|': VBAR, + '|=': VBAREQUAL, + '}': RBRACE, + '~': TILDE, } - def ISTERMINAL(x): return x < NT_OFFSET - def ISNONTERMINAL(x): return x >= NT_OFFSET - def ISEOF(x): return x == ENDMARKER diff --git a/Parser/token.c b/Parser/token.c index 35519aa4b61161..d27f98a34d55d1 100644 --- a/Parser/token.c +++ b/Parser/token.c @@ -59,6 +59,7 @@ const char * const _PyParser_TokenNames[] = { "ATEQUAL", "RARROW", "ELLIPSIS", + "COLONEQUAL", "OP", "", "", @@ -142,6 +143,11 @@ PyToken_TwoChars(int c1, int c2) case '=': return SLASHEQUAL; } break; + case ':': + switch (c2) { + case '=': return COLONEQUAL; + } + break; case '<': switch (c2) { case '<': return LEFTSHIFT; From 1153aae9549496f5ef55c8a06bd8e07521b26618 Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Thu, 24 Jan 2019 15:54:09 -0700 Subject: [PATCH 63/67] Update TargetScopeError PRE_INIT and POST_INIT, as this was purposefully left out when fixing rebase conflicts --- Objects/exceptions.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 5a48d0440bb33f..75ede1c9c5de91 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -2538,6 +2538,7 @@ _PyExc_Init(void) PRE_INIT(AttributeError); PRE_INIT(SyntaxError); PRE_INIT(IndentationError); + PRE_INIT(TargetScopeError); PRE_INIT(TabError); PRE_INIT(LookupError); PRE_INIT(IndexError); @@ -2678,6 +2679,7 @@ _PyBuiltins_AddExceptions(PyObject *bltinmod) POST_INIT(AttributeError); POST_INIT(SyntaxError); POST_INIT(IndentationError); + POST_INIT(TargetScopeError); POST_INIT(TabError); POST_INIT(LookupError); POST_INIT(IndexError); From 7775b0dec55271ed9214503c9b40b64c300c270d Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Thu, 24 Jan 2019 15:55:00 -0700 Subject: [PATCH 64/67] Add NamedStore back and regenerate files --- Include/Python-ast.h | 27 ++++++++++++++++++--------- Python/Python-ast.c | 9 ++++++--- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/Include/Python-ast.h b/Include/Python-ast.h index f8394e6c26ad83..3527ae8949e765 100644 --- a/Include/Python-ast.h +++ b/Include/Python-ast.h @@ -17,7 +17,7 @@ typedef struct _stmt *stmt_ty; typedef struct _expr *expr_ty; typedef enum _expr_context { Load=1, Store=2, Del=3, AugLoad=4, AugStore=5, - Param=6 } expr_context_ty; + Param=6, NamedStore=7 } expr_context_ty; typedef struct _slice *slice_ty; @@ -214,14 +214,14 @@ struct _stmt { int end_col_offset; }; -enum _expr_kind {BoolOp_kind=1, BinOp_kind=2, UnaryOp_kind=3, Lambda_kind=4, - IfExp_kind=5, Dict_kind=6, Set_kind=7, ListComp_kind=8, - SetComp_kind=9, DictComp_kind=10, GeneratorExp_kind=11, - Await_kind=12, Yield_kind=13, YieldFrom_kind=14, - Compare_kind=15, Call_kind=16, FormattedValue_kind=17, - JoinedStr_kind=18, Constant_kind=19, Attribute_kind=20, - Subscript_kind=21, Starred_kind=22, Name_kind=23, - List_kind=24, Tuple_kind=25}; +enum _expr_kind {BoolOp_kind=1, NamedExpr_kind=2, BinOp_kind=3, UnaryOp_kind=4, + Lambda_kind=5, IfExp_kind=6, Dict_kind=7, Set_kind=8, + ListComp_kind=9, SetComp_kind=10, DictComp_kind=11, + GeneratorExp_kind=12, Await_kind=13, Yield_kind=14, + YieldFrom_kind=15, Compare_kind=16, Call_kind=17, + FormattedValue_kind=18, JoinedStr_kind=19, Constant_kind=20, + Attribute_kind=21, Subscript_kind=22, Starred_kind=23, + Name_kind=24, List_kind=25, Tuple_kind=26}; struct _expr { enum _expr_kind kind; union { @@ -230,6 +230,11 @@ struct _expr { asdl_seq *values; } BoolOp; + struct { + expr_ty target; + expr_ty value; + } NamedExpr; + struct { expr_ty left; operator_ty op; @@ -541,6 +546,10 @@ stmt_ty _Py_Continue(int lineno, int col_offset, int end_lineno, int #define BoolOp(a0, a1, a2, a3, a4, a5, a6) _Py_BoolOp(a0, a1, a2, a3, a4, a5, a6) expr_ty _Py_BoolOp(boolop_ty op, asdl_seq * values, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); +#define NamedExpr(a0, a1, a2, a3, a4, a5, a6) _Py_NamedExpr(a0, a1, a2, a3, a4, a5, a6) +expr_ty _Py_NamedExpr(expr_ty target, expr_ty value, int lineno, int + col_offset, int end_lineno, int end_col_offset, PyArena + *arena); #define BinOp(a0, a1, a2, a3, a4, a5, a6, a7) _Py_BinOp(a0, a1, a2, a3, a4, a5, a6, a7) expr_ty _Py_BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena diff --git a/Python/Python-ast.c b/Python/Python-ast.c index aa0922a0609170..a333ff95b110d7 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -1786,8 +1786,8 @@ BoolOp(boolop_ty op, asdl_seq * values, int lineno, int col_offset, int } expr_ty -NamedExpr(expr_ty target, expr_ty value, int lineno, int col_offset, PyArena - *arena) +NamedExpr(expr_ty target, expr_ty value, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena) { expr_ty p; if (!target) { @@ -1808,6 +1808,8 @@ NamedExpr(expr_ty target, expr_ty value, int lineno, int col_offset, PyArena p->v.NamedExpr.value = value; p->lineno = lineno; p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } @@ -5985,7 +5987,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = NamedExpr(target, value, lineno, col_offset, arena); + *out = NamedExpr(target, value, lineno, col_offset, end_lineno, + end_col_offset, arena); if (*out == NULL) goto failed; return 0; } From 10a71f81d28e2f7b3581552c061adb1b93462eba Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Thu, 24 Jan 2019 16:08:58 -0700 Subject: [PATCH 65/67] Pass along line number and end col info for named expression --- Python/ast.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Python/ast.c b/Python/ast.c index 7e1fc4908afbc1..f298715256dfc5 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -1753,7 +1753,8 @@ ast_for_namedexpr(struct compiling *c, const node *n) if (!set_context(c, target, NamedStore, n)) return NULL; - return NamedExpr(target, value, LINENO(n), n->n_col_offset, c->c_arena); + return NamedExpr(target, value, LINENO(n), n->n_col_offset, n->n_end_lineno, + n->n_end_col_offset, c->c_arena); } static expr_ty From 2b9a7bc1a5cb2ea69723fa5d2f2aee3181fcf500 Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Thu, 24 Jan 2019 16:09:23 -0700 Subject: [PATCH 66/67] Simplify News entry --- .../Core and Builtins/2018-11-13-14-26-54.bpo-35224.F0B6UQ.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-11-13-14-26-54.bpo-35224.F0B6UQ.rst b/Misc/NEWS.d/next/Core and Builtins/2018-11-13-14-26-54.bpo-35224.F0B6UQ.rst index ab446c52614d4c..fe54f362aef63a 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2018-11-13-14-26-54.bpo-35224.F0B6UQ.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2018-11-13-14-26-54.bpo-35224.F0B6UQ.rst @@ -1,2 +1 @@ -Implement :pep:`572` (assignment expressions). Documentation and edge cases -will be issued in separate PRs. Patch by Emily Morehouse. +Implement :pep:`572` (assignment expressions). Patch by Emily Morehouse. From a63bf06eb78e85b58e9ecde224bbcf438f5b0f7e Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Thu, 24 Jan 2019 16:20:24 -0700 Subject: [PATCH 67/67] Fix compiler warning and explicity mark fallthrough --- Python/ast.c | 1 + Python/compile.c | 1 + 2 files changed, 2 insertions(+) diff --git a/Python/ast.c b/Python/ast.c index f298715256dfc5..6560026109c8b2 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -2661,6 +2661,7 @@ ast_for_expr(struct compiling *c, const node *n) case namedexpr_test: if (NCH(n) == 3) return ast_for_namedexpr(c, n); + /* Fallthrough */ case test: case test_nocond: if (TYPE(CHILD(n, 0)) == lambdef || diff --git a/Python/compile.c b/Python/compile.c index d96388b16db523..eb2c3028b6330a 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -5020,6 +5020,7 @@ compiler_handle_subscr(struct compiler *c, const char *kind, case AugStore:/* fall through to Store */ case Store: op = STORE_SUBSCR; break; case Del: op = DELETE_SUBSCR; break; + case NamedStore: case Param: PyErr_Format(PyExc_SystemError, "invalid %s kind %d in subscript\n",