python · 1st1 · Oct 6, 2017 · Apr 16, 2017 · Apr 16, 2017 · Apr 16, 2017
diff --git a/Doc/library/token.rst b/Doc/library/token.rst
@@ -98,8 +98,6 @@ The token constants are:
          RARROW
          ELLIPSIS
          OP
-          AWAIT
-          ASYNC
          ERRORTOKEN
          N_TOKENS
          NT_OFFSET
@@ -129,9 +127,11 @@ the :mod:`tokenize` module.


 .. versionchanged:: 3.5
-   Added :data:`AWAIT` and :data:`ASYNC` tokens. Starting with
-   Python 3.7, "async" and "await" will be tokenized as :data:`NAME`
-   tokens, and :data:`AWAIT` and :data:`ASYNC` will be removed.
+   Added :data:`AWAIT` and :data:`ASYNC` tokens.

 .. versionchanged:: 3.7
   Added :data:`COMMENT`, :data:`NL` and :data:`ENCODING` tokens.
+
+.. versionchanged:: 3.7
+   Removed :data:`AWAIT` and :data:`ASYNC` tokens. "async" and "await" are
+   now tokenized as :data:`NAME` tokens.
diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py
@@ -272,9 +272,9 @@ def run(self):
 # Support for building "topic help" for pydoc

 pydoc_topic_labels = [
-    'assert', 'assignment', 'atom-identifiers', 'atom-literals',
-    'attribute-access', 'attribute-references', 'augassign', 'binary',
-    'bitwise', 'bltin-code-objects', 'bltin-ellipsis-object',
+    'assert', 'assignment', 'async', 'atom-identifiers', 'atom-literals',
+    'attribute-access', 'attribute-references', 'augassign', 'await',
+    'binary', 'bitwise', 'bltin-code-objects', 'bltin-ellipsis-object',
    'bltin-null-object', 'bltin-type-objects', 'booleans',
    'break', 'callable-types', 'calls', 'class', 'comparisons', 'compound',
    'context-managers', 'continue', 'conversions', 'customization', 'debugger',

diff --git a/Grammar/Grammar b/Grammar/Grammar
@@ -16,7 +16,7 @@ decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
 decorators: decorator+
 decorated: decorators (classdef | funcdef | async_funcdef)

-async_funcdef: ASYNC funcdef
+async_funcdef: 'async' funcdef
 funcdef: 'def' NAME parameters ['->' test] ':' suite

 parameters: '(' [typedargslist] ')'
@@ -68,7 +68,7 @@ nonlocal_stmt: 'nonlocal' NAME (',' NAME)*
 assert_stmt: 'assert' test [',' test]

 compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt
-async_stmt: ASYNC (funcdef | with_stmt | for_stmt)
+async_stmt: 'async' (funcdef | with_stmt | for_stmt)
 if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
 while_stmt: 'while' test ':' suite ['else' ':' suite]
 for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
@@ -103,7 +103,7 @@ arith_expr: term (('+'|'-') term)*
 term: factor (('*'|'@'|'/'|'%'|'//') factor)*
 factor: ('+'|'-'|'~') factor | power
 power: atom_expr ['**' factor]
-atom_expr: [AWAIT] atom trailer*
+atom_expr: ['await'] atom trailer*
 atom: ('(' [yield_expr|testlist_comp] ')' |
       '[' [testlist_comp] ']' |
       '{' [dictorsetmaker] '}' |
@@ -139,7 +139,8 @@ argument: ( test [comp_for] |
            '*' test )

 comp_iter: comp_for | comp_if
-comp_for: [ASYNC] 'for' exprlist 'in' or_test [comp_iter]
+sync_comp_for: 'for' exprlist 'in' or_test [comp_iter]
+comp_for: ['async'] sync_comp_for
 comp_if: 'if' test_nocond [comp_iter]

 # not used in grammar, but may appear in "node" passed from Parser to Compiler

diff --git a/Include/graminit.h b/Include/graminit.h
@@ -81,8 +81,9 @@
 #define arglist 334
 #define argument 335
 #define comp_iter 336
-#define comp_for 337
-#define comp_if 338
-#define encoding_decl 339
-#define yield_expr 340
-#define yield_arg 341
+#define sync_comp_for 337
+#define comp_for 338
+#define comp_if 339
+#define encoding_decl 340
+#define yield_expr 341
+#define yield_arg 342
diff --git a/Include/token.h b/Include/token.h
@@ -9,77 +9,75 @@ extern "C" {

 #undef TILDE   /* Prevent clash of our definition with system macro. Ex AIX, ioctl.h */

-#define ENDMARKER	0
-#define NAME		1
-#define NUMBER		2
-#define STRING		3
-#define NEWLINE		4
-#define INDENT		5
-#define DEDENT		6
-#define LPAR		7
-#define RPAR		8
-#define LSQB		9
-#define RSQB		10
-#define COLON		11
-#define COMMA		12
-#define SEMI		13
-#define PLUS		14
-#define MINUS		15
-#define STAR		16
-#define SLASH		17
-#define VBAR		18
-#define AMPER		19
-#define LESS		20
-#define GREATER		21
-#define EQUAL		22
-#define DOT		23
-#define PERCENT		24
-#define LBRACE		25
-#define RBRACE		26
-#define EQEQUAL		27
-#define NOTEQUAL	28
-#define LESSEQUAL	29
-#define GREATEREQUAL	30
-#define TILDE		31
-#define CIRCUMFLEX	32
-#define LEFTSHIFT	33
-#define RIGHTSHIFT	34
-#define DOUBLESTAR	35
-#define PLUSEQUAL	36
-#define MINEQUAL	37
-#define STAREQUAL	38
-#define SLASHEQUAL	39
-#define PERCENTEQUAL	40
-#define AMPEREQUAL	41
-#define VBAREQUAL	42
-#define CIRCUMFLEXEQUAL	43
-#define LEFTSHIFTEQUAL	44
-#define RIGHTSHIFTEQUAL	45
-#define DOUBLESTAREQUAL	46
-#define DOUBLESLASH	47
+#define ENDMARKER       0
+#define NAME            1
+#define NUMBER          2
+#define STRING          3
+#define NEWLINE         4
+#define INDENT          5
+#define DEDENT          6
+#define LPAR            7
+#define RPAR            8
+#define LSQB            9
+#define RSQB            10
+#define COLON           11
+#define COMMA           12
+#define SEMI            13
+#define PLUS            14
+#define MINUS           15
+#define STAR            16
+#define SLASH           17
+#define VBAR            18
+#define AMPER           19
+#define LESS            20
+#define GREATER         21
+#define EQUAL           22
+#define DOT             23
+#define PERCENT         24
+#define LBRACE          25
+#define RBRACE          26
+#define EQEQUAL         27
+#define NOTEQUAL        28
+#define LESSEQUAL       29
+#define GREATEREQUAL    30
+#define TILDE           31
+#define CIRCUMFLEX      32
+#define LEFTSHIFT       33
+#define RIGHTSHIFT      34
+#define DOUBLESTAR      35
+#define PLUSEQUAL       36
+#define MINEQUAL        37
+#define STAREQUAL       38
+#define SLASHEQUAL      39
+#define PERCENTEQUAL    40
+#define AMPEREQUAL      41
+#define VBAREQUAL       42
+#define CIRCUMFLEXEQUAL 43
+#define LEFTSHIFTEQUAL  44
+#define RIGHTSHIFTEQUAL 45
+#define DOUBLESTAREQUAL 46
+#define DOUBLESLASH     47
 #define DOUBLESLASHEQUAL 48
 #define AT              49
-#define ATEQUAL		50
+#define ATEQUAL         50
 #define RARROW          51
 #define ELLIPSIS        52
 /* Don't forget to update the table _PyParser_TokenNames in tokenizer.c! */
-#define OP		53
-#define AWAIT		54
-#define ASYNC		55
-#define ERRORTOKEN	56
+#define OP              53
+#define ERRORTOKEN      54
 /* These aren't used by the C tokenizer but are needed for tokenize.py */
-#define COMMENT		57
-#define NL		58
-#define ENCODING		59
-#define N_TOKENS	60
+#define COMMENT         55
+#define NL              56
+#define ENCODING                57
+#define N_TOKENS        58

 /* Special definitions for cooperation with parser */

-#define NT_OFFSET		256
+#define NT_OFFSET               256

-#define ISTERMINAL(x)		((x) < NT_OFFSET)
-#define ISNONTERMINAL(x)	((x) >= NT_OFFSET)
-#define ISEOF(x)		((x) == ENDMARKER)
+#define ISTERMINAL(x)           ((x) < NT_OFFSET)
+#define ISNONTERMINAL(x)        ((x) >= NT_OFFSET)
+#define ISEOF(x)                ((x) == ENDMARKER)


 PyAPI_DATA(const char *) _PyParser_TokenNames[]; /* Token names */

diff --git a/Lib/keyword.py b/Lib/keyword.py
@@ -20,6 +20,8 @@
        'and',
        'as',
        'assert',
+        'async',
+        'await',
        'break',
        'class',
        'continue',

diff --git a/Lib/lib2to3/Grammar.txt b/Lib/lib2to3/Grammar.txt
@@ -34,7 +34,7 @@ eval_input: testlist NEWLINE* ENDMARKER
 decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
 decorators: decorator+
 decorated: decorators (classdef | funcdef | async_funcdef)
-async_funcdef: ASYNC funcdef
+async_funcdef: 'async' funcdef
 funcdef: 'def' NAME parameters ['->' test] ':' suite
 parameters: '(' [typedargslist] ')'
 typedargslist: ((tfpdef ['=' test] ',')*
@@ -85,7 +85,7 @@ exec_stmt: 'exec' expr ['in' test [',' test]]
 assert_stmt: 'assert' test [',' test]

 compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt
-async_stmt: ASYNC (funcdef | with_stmt | for_stmt)
+async_stmt: 'async' (funcdef | with_stmt | for_stmt)
 if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
 while_stmt: 'while' test ':' suite ['else' ':' suite]
 for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
@@ -124,7 +124,7 @@ shift_expr: arith_expr (('<<'|'>>') arith_expr)*
 arith_expr: term (('+'|'-') term)*
 term: factor (('*'|'@'|'/'|'%'|'//') factor)*
 factor: ('+'|'-'|'~') factor | power
-power: [AWAIT] atom trailer* ['**' factor]
+power: ['await'] atom trailer* ['**' factor]
 atom: ('(' [yield_expr|testlist_gexp] ')' |
       '[' [listmaker] ']' |
       '{' [dictsetmaker] '}' |
@@ -161,7 +161,7 @@ argument: ( test [comp_for] |
 	    star_expr )

 comp_iter: comp_for | comp_if
-comp_for: [ASYNC] 'for' exprlist 'in' or_test [comp_iter]
+comp_for: ['async'] 'for' exprlist 'in' or_test [comp_iter]
 comp_if: 'if' old_test [comp_iter]

 # As noted above, testlist_safe extends the syntax allowed in list
@@ -180,7 +180,7 @@ comp_if: 'if' old_test [comp_iter]
 #
 # See https://bugs.python.org/issue27494
 old_comp_iter: old_comp_for | old_comp_if
-old_comp_for: [ASYNC] 'for' exprlist 'in' testlist_safe [old_comp_iter]
+old_comp_for: ['async'] 'for' exprlist 'in' testlist_safe [old_comp_iter]
 old_comp_if: 'if' old_test [old_comp_iter]

 testlist1: test (',' test)*

diff --git a/Lib/lib2to3/pgen2/token.py b/Lib/lib2to3/pgen2/token.py
@@ -62,10 +62,8 @@
 COMMENT = 53
 NL = 54
 RARROW = 55
-AWAIT = 56
-ASYNC = 57
-ERRORTOKEN = 58
-N_TOKENS = 59
+ERRORTOKEN = 56
+N_TOKENS = 57
 NT_OFFSET = 256
 #--end constants--