Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit c3b595e

Browse filesBrowse files
carljmiritkatrielerlend-aasland
authored
gh-97933: (PEP 709) inline list/dict/set comprehensions (#101441)
Co-authored-by: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Co-authored-by: Erlend E. Aasland <erlend.aasland@protonmail.com>
1 parent 0aeda29 commit c3b595e
Copy full SHA for c3b595e

27 files changed

+1243
-695
lines changed

‎Doc/library/dis.rst

Copy file name to clipboardExpand all lines: Doc/library/dis.rst
+8Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1196,6 +1196,14 @@ iterations of the loop.
11961196

11971197
.. versionadded:: 3.12
11981198

1199+
.. opcode:: LOAD_FAST_AND_CLEAR (var_num)
1200+
1201+
Pushes a reference to the local ``co_varnames[var_num]`` onto the stack (or
1202+
pushes ``NULL`` onto the stack if the local variable has not been
1203+
initialized) and sets ``co_varnames[var_num]`` to ``NULL``.
1204+
1205+
.. versionadded:: 3.12
1206+
11991207
.. opcode:: STORE_FAST (var_num)
12001208

12011209
Stores ``STACK.pop()`` into the local ``co_varnames[var_num]``.

‎Doc/whatsnew/3.12.rst

Copy file name to clipboardExpand all lines: Doc/whatsnew/3.12.rst
+24Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,30 @@ New Features
153153
In Python 3.14, the default will switch to ``'data'``.
154154
(Contributed by Petr Viktorin in :pep:`706`.)
155155

156+
.. _whatsnew312-pep709:
157+
158+
PEP 709: Comprehension inlining
159+
-------------------------------
160+
161+
Dictionary, list, and set comprehensions are now inlined, rather than creating a
162+
new single-use function object for each execution of the comprehension. This
163+
speeds up execution of a comprehension by up to 2x.
164+
165+
Comprehension iteration variables remain isolated; they don't overwrite a
166+
variable of the same name in the outer scope, nor are they visible after the
167+
comprehension. This isolation is now maintained via stack/locals manipulation,
168+
not via separate function scope.
169+
170+
Inlining does result in a few visible behavior changes:
171+
172+
* There is no longer a separate frame for the comprehension in tracebacks,
173+
and tracing/profiling no longer shows the comprehension as a function call.
174+
* Calling :func:`locals` inside a comprehension now includes variables
175+
from outside the comprehension, and no longer includes the synthetic ``.0``
176+
variable for the comprehension "argument".
177+
178+
Contributed by Carl Meyer and Vladimir Matveev in :pep:`709`.
179+
156180
PEP 688: Making the buffer protocol accessible in Python
157181
--------------------------------------------------------
158182

‎Include/internal/pycore_code.h

Copy file name to clipboardExpand all lines: Include/internal/pycore_code.h
+1Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,7 @@ struct callable_cache {
131131
// Note that these all fit within a byte, as do combinations.
132132
// Later, we will use the smaller numbers to differentiate the different
133133
// kinds of locals (e.g. pos-only arg, varkwargs, local-only).
134+
#define CO_FAST_HIDDEN 0x10
134135
#define CO_FAST_LOCAL 0x20
135136
#define CO_FAST_CELL 0x40
136137
#define CO_FAST_FREE 0x80

‎Include/internal/pycore_compile.h

Copy file name to clipboardExpand all lines: Include/internal/pycore_compile.h
+3Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,9 @@ typedef struct {
7070
PyObject *u_varnames; /* local variables */
7171
PyObject *u_cellvars; /* cell variables */
7272
PyObject *u_freevars; /* free variables */
73+
PyObject *u_fasthidden; /* dict; keys are names that are fast-locals only
74+
temporarily within an inlined comprehension. When
75+
value is True, treat as fast-local. */
7376

7477
Py_ssize_t u_argcount; /* number of arguments for block */
7578
Py_ssize_t u_posonlyargcount; /* number of positional only arguments for block */

‎Include/internal/pycore_flowgraph.h

Copy file name to clipboardExpand all lines: Include/internal/pycore_flowgraph.h
+1-1Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@ _PyCfgInstruction* _PyCfg_BasicblockLastInstr(const _PyCfgBasicblock *b);
9494
int _PyCfg_OptimizeCodeUnit(_PyCfgBuilder *g, PyObject *consts, PyObject *const_cache,
9595
int code_flags, int nlocals, int nparams, int firstlineno);
9696
int _PyCfg_Stackdepth(_PyCfgBasicblock *entryblock, int code_flags);
97-
void _PyCfg_ConvertExceptionHandlersToNops(_PyCfgBasicblock *entryblock);
97+
void _PyCfg_ConvertPseudoOps(_PyCfgBasicblock *entryblock);
9898
int _PyCfg_ResolveJumps(_PyCfgBuilder *g);
9999

100100

‎Include/internal/pycore_opcode.h

Copy file name to clipboardExpand all lines: Include/internal/pycore_opcode.h
+7-6Lines changed: 7 additions & 6 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Include/internal/pycore_symtable.h

Copy file name to clipboardExpand all lines: Include/internal/pycore_symtable.h
+1Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ typedef struct _symtable_entry {
6464
unsigned ste_needs_class_closure : 1; /* for class scopes, true if a
6565
closure over __class__
6666
should be created */
67+
unsigned ste_comp_inlined : 1; /* true if this comprehension is inlined */
6768
unsigned ste_comp_iter_target : 1; /* true if visiting comprehension target */
6869
int ste_comp_iter_expr; /* non-zero if visiting a comprehension range expression */
6970
int ste_lineno; /* first line of block */

‎Include/opcode.h

Copy file name to clipboardExpand all lines: Include/opcode.h
+12-9Lines changed: 12 additions & 9 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Lib/importlib/_bootstrap_external.py

Copy file name to clipboardExpand all lines: Lib/importlib/_bootstrap_external.py
+2-1Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -442,6 +442,7 @@ def _write_atomic(path, data, mode=0o666):
442442
# Python 3.12b1 3526 (Add instrumentation support)
443443
# Python 3.12b1 3527 (Add LOAD_SUPER_ATTR)
444444
# Python 3.12b1 3528 (Add LOAD_SUPER_ATTR_METHOD specialization)
445+
# Python 3.12b1 3529 (Inline list/dict/set comprehensions)
445446

446447
# Python 3.13 will start with 3550
447448

@@ -458,7 +459,7 @@ def _write_atomic(path, data, mode=0o666):
458459
# Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array
459460
# in PC/launcher.c must also be updated.
460461

461-
MAGIC_NUMBER = (3528).to_bytes(2, 'little') + b'\r\n'
462+
MAGIC_NUMBER = (3529).to_bytes(2, 'little') + b'\r\n'
462463

463464
_RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c
464465

‎Lib/opcode.py

Copy file name to clipboardExpand all lines: Lib/opcode.py
+4Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -198,6 +198,8 @@ def pseudo_op(name, op, real_ops):
198198
jrel_op('JUMP_BACKWARD', 140) # Number of words to skip (backwards)
199199
name_op('LOAD_SUPER_ATTR', 141)
200200
def_op('CALL_FUNCTION_EX', 142) # Flags
201+
def_op('LOAD_FAST_AND_CLEAR', 143) # Local variable number
202+
haslocal.append(143)
201203

202204
def_op('EXTENDED_ARG', 144)
203205
EXTENDED_ARG = 144
@@ -268,6 +270,8 @@ def pseudo_op(name, op, real_ops):
268270
pseudo_op('LOAD_ZERO_SUPER_METHOD', 264, ['LOAD_SUPER_ATTR'])
269271
pseudo_op('LOAD_ZERO_SUPER_ATTR', 265, ['LOAD_SUPER_ATTR'])
270272

273+
pseudo_op('STORE_FAST_MAYBE_NULL', 266, ['STORE_FAST'])
274+
271275
MAX_PSEUDO_OPCODE = MIN_PSEUDO_OPCODE + len(_pseudo_ops) - 1
272276

273277
del def_op, name_op, jrel_op, jabs_op, pseudo_op

‎Lib/test/test_compile.py

Copy file name to clipboardExpand all lines: Lib/test/test_compile.py
+6-18Lines changed: 6 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1352,14 +1352,11 @@ def test_multiline_list_comprehension(self):
13521352
and x != 50)]
13531353
""")
13541354
compiled_code, _ = self.check_positions_against_ast(snippet)
1355-
compiled_code = compiled_code.co_consts[0]
13561355
self.assertIsInstance(compiled_code, types.CodeType)
13571356
self.assertOpcodeSourcePositionIs(compiled_code, 'LIST_APPEND',
13581357
line=1, end_line=2, column=1, end_column=8, occurrence=1)
13591358
self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD',
13601359
line=1, end_line=2, column=1, end_column=8, occurrence=1)
1361-
self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE',
1362-
line=1, end_line=6, column=0, end_column=32, occurrence=1)
13631360

13641361
def test_multiline_async_list_comprehension(self):
13651362
snippet = textwrap.dedent("""\
@@ -1374,13 +1371,13 @@ async def f():
13741371
compiled_code, _ = self.check_positions_against_ast(snippet)
13751372
g = {}
13761373
eval(compiled_code, g)
1377-
compiled_code = g['f'].__code__.co_consts[1]
1374+
compiled_code = g['f'].__code__
13781375
self.assertIsInstance(compiled_code, types.CodeType)
13791376
self.assertOpcodeSourcePositionIs(compiled_code, 'LIST_APPEND',
13801377
line=2, end_line=3, column=5, end_column=12, occurrence=1)
13811378
self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD',
13821379
line=2, end_line=3, column=5, end_column=12, occurrence=1)
1383-
self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE',
1380+
self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_CONST',
13841381
line=2, end_line=7, column=4, end_column=36, occurrence=1)
13851382

13861383
def test_multiline_set_comprehension(self):
@@ -1393,14 +1390,11 @@ def test_multiline_set_comprehension(self):
13931390
and x != 50)}
13941391
""")
13951392
compiled_code, _ = self.check_positions_against_ast(snippet)
1396-
compiled_code = compiled_code.co_consts[0]
13971393
self.assertIsInstance(compiled_code, types.CodeType)
13981394
self.assertOpcodeSourcePositionIs(compiled_code, 'SET_ADD',
13991395
line=1, end_line=2, column=1, end_column=8, occurrence=1)
14001396
self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD',
14011397
line=1, end_line=2, column=1, end_column=8, occurrence=1)
1402-
self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE',
1403-
line=1, end_line=6, column=0, end_column=32, occurrence=1)
14041398

14051399
def test_multiline_async_set_comprehension(self):
14061400
snippet = textwrap.dedent("""\
@@ -1415,13 +1409,13 @@ async def f():
14151409
compiled_code, _ = self.check_positions_against_ast(snippet)
14161410
g = {}
14171411
eval(compiled_code, g)
1418-
compiled_code = g['f'].__code__.co_consts[1]
1412+
compiled_code = g['f'].__code__
14191413
self.assertIsInstance(compiled_code, types.CodeType)
14201414
self.assertOpcodeSourcePositionIs(compiled_code, 'SET_ADD',
14211415
line=2, end_line=3, column=5, end_column=12, occurrence=1)
14221416
self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD',
14231417
line=2, end_line=3, column=5, end_column=12, occurrence=1)
1424-
self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE',
1418+
self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_CONST',
14251419
line=2, end_line=7, column=4, end_column=36, occurrence=1)
14261420

14271421
def test_multiline_dict_comprehension(self):
@@ -1434,14 +1428,11 @@ def test_multiline_dict_comprehension(self):
14341428
and x != 50)}
14351429
""")
14361430
compiled_code, _ = self.check_positions_against_ast(snippet)
1437-
compiled_code = compiled_code.co_consts[0]
14381431
self.assertIsInstance(compiled_code, types.CodeType)
14391432
self.assertOpcodeSourcePositionIs(compiled_code, 'MAP_ADD',
14401433
line=1, end_line=2, column=1, end_column=7, occurrence=1)
14411434
self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD',
14421435
line=1, end_line=2, column=1, end_column=7, occurrence=1)
1443-
self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE',
1444-
line=1, end_line=6, column=0, end_column=32, occurrence=1)
14451436

14461437
def test_multiline_async_dict_comprehension(self):
14471438
snippet = textwrap.dedent("""\
@@ -1456,13 +1447,13 @@ async def f():
14561447
compiled_code, _ = self.check_positions_against_ast(snippet)
14571448
g = {}
14581449
eval(compiled_code, g)
1459-
compiled_code = g['f'].__code__.co_consts[1]
1450+
compiled_code = g['f'].__code__
14601451
self.assertIsInstance(compiled_code, types.CodeType)
14611452
self.assertOpcodeSourcePositionIs(compiled_code, 'MAP_ADD',
14621453
line=2, end_line=3, column=5, end_column=11, occurrence=1)
14631454
self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD',
14641455
line=2, end_line=3, column=5, end_column=11, occurrence=1)
1465-
self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE',
1456+
self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_CONST',
14661457
line=2, end_line=7, column=4, end_column=36, occurrence=1)
14671458

14681459
def test_matchcase_sequence(self):
@@ -1711,9 +1702,6 @@ def test_column_offset_deduplication(self):
17111702
for source in [
17121703
"lambda: a",
17131704
"(a for b in c)",
1714-
"[a for b in c]",
1715-
"{a for b in c}",
1716-
"{a: b for c in d}",
17171705
]:
17181706
with self.subTest(source):
17191707
code = compile(f"{source}, {source}", "<test>", "eval")

‎Lib/test/test_compiler_assemble.py

Copy file name to clipboardExpand all lines: Lib/test/test_compiler_assemble.py
+4-1Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ def complete_metadata(self, metadata, filename="myfile.py"):
1616
metadata.setdefault(key, key)
1717
for key in ['consts']:
1818
metadata.setdefault(key, [])
19-
for key in ['names', 'varnames', 'cellvars', 'freevars']:
19+
for key in ['names', 'varnames', 'cellvars', 'freevars', 'fasthidden']:
2020
metadata.setdefault(key, {})
2121
for key in ['argcount', 'posonlyargcount', 'kwonlyargcount']:
2222
metadata.setdefault(key, 0)
@@ -33,6 +33,9 @@ def assemble_test(self, insts, metadata, expected):
3333

3434
expected_metadata = {}
3535
for key, value in metadata.items():
36+
if key == "fasthidden":
37+
# not exposed on code object
38+
continue
3639
if isinstance(value, list):
3740
expected_metadata[key] = tuple(value)
3841
elif isinstance(value, dict):

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.