From 7476442c6dab738226715823b813cd1601a6683b Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 10 Apr 2025 09:43:08 +0100 Subject: [PATCH 01/18] FOR_ITER now pushes NULL as well as the iterator. Preparation for virtual iterators --- Include/internal/pycore_compile.h | 1 + Include/internal/pycore_magic_number.h | 3 +- Include/internal/pycore_opcode_metadata.h | 38 ++-- Include/internal/pycore_uop_ids.h | 6 + Include/internal/pycore_uop_metadata.h | 4 + Lib/test/test_dis.py | 205 +++++++++++----------- Objects/frameobject.c | 4 + Programs/test_frozenmain.h | 2 +- Python/bytecodes.c | 54 +++--- Python/codegen.c | 39 ++-- Python/executor_cases.c.h | 43 +++-- Python/flowgraph.c | 28 ++- Python/generated_cases.c.h | 39 ++-- Python/optimizer_cases.c.h | 11 ++ 14 files changed, 276 insertions(+), 201 deletions(-) diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index a606c2afe0a234..dbf565eb5d7623 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -95,6 +95,7 @@ typedef enum { enum _PyCompile_FBlockType { COMPILE_FBLOCK_WHILE_LOOP, COMPILE_FBLOCK_FOR_LOOP, + COMPILE_FBLOCK_ASYNC_FOR_LOOP, COMPILE_FBLOCK_TRY_EXCEPT, COMPILE_FBLOCK_FINALLY_TRY, COMPILE_FBLOCK_FINALLY_END, diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index f75b05893affc1..668d93a0609b1f 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -275,6 +275,7 @@ Known values: Python 3.14a6 3620 (Optimize bytecode for all/any/tuple called on a genexp) Python 3.14a7 3621 (Optimize LOAD_FAST opcodes into LOAD_FAST_BORROW) Python 3.14a7 3622 (Store annotations in different class dict keys) + Python 3.14a8 3623 (Virtual iterators) Python 3.15 will start with 3650 @@ -287,7 +288,7 @@ PC/launcher.c must also be updated. */ -#define PYC_MAGIC_NUMBER 3622 +#define PYC_MAGIC_NUMBER 3623 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 3b881b30b054f1..44bd6de9da6740 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -199,15 +199,15 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case FORMAT_WITH_SPEC: return 2; case FOR_ITER: - return 1; + return 2; case FOR_ITER_GEN: - return 1; + return 2; case FOR_ITER_LIST: - return 1; + return 2; case FOR_ITER_RANGE: - return 1; + return 2; case FOR_ITER_TUPLE: - return 1; + return 2; case GET_AITER: return 1; case GET_ANEXT: @@ -233,11 +233,11 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case INSTRUMENTED_END_ASYNC_FOR: return 2; case INSTRUMENTED_END_FOR: - return 2; + return 3; case INSTRUMENTED_END_SEND: return 2; case INSTRUMENTED_FOR_ITER: - return 1; + return 2; case INSTRUMENTED_INSTRUCTION: return 0; case INSTRUMENTED_JUMP_BACKWARD: @@ -251,7 +251,7 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case INSTRUMENTED_NOT_TAKEN: return 0; case INSTRUMENTED_POP_ITER: - return 1; + return 2; case INSTRUMENTED_POP_JUMP_IF_FALSE: return 1; case INSTRUMENTED_POP_JUMP_IF_NONE: @@ -393,7 +393,7 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case POP_EXCEPT: return 1; case POP_ITER: - return 1; + return 2; case POP_JUMP_IF_FALSE: return 1; case POP_JUMP_IF_NONE: @@ -680,15 +680,15 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case FORMAT_WITH_SPEC: return 1; case FOR_ITER: - return 2; + return 3; case FOR_ITER_GEN: - return 1; - case FOR_ITER_LIST: return 2; + case FOR_ITER_LIST: + return 3; case FOR_ITER_RANGE: - return 2; + return 3; case FOR_ITER_TUPLE: - return 2; + return 3; case GET_AITER: return 1; case GET_ANEXT: @@ -696,7 +696,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case GET_AWAITABLE: return 1; case GET_ITER: - return 1; + return 2; case GET_LEN: return 2; case GET_YIELD_FROM_ITER: @@ -714,11 +714,11 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case INSTRUMENTED_END_ASYNC_FOR: return 0; case INSTRUMENTED_END_FOR: - return 1; + return 2; case INSTRUMENTED_END_SEND: return 1; case INSTRUMENTED_FOR_ITER: - return 2; + return 3; case INSTRUMENTED_INSTRUCTION: return 0; case INSTRUMENTED_JUMP_BACKWARD: @@ -1237,7 +1237,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [NOP] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [NOT_TAKEN] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [POP_EXCEPT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, - [POP_ITER] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_PURE_FLAG }, + [POP_ITER] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, @@ -1446,7 +1446,7 @@ _PyOpcode_macro_expansion[256] = { [NOP] = { .nuops = 1, .uops = { { _NOP, OPARG_SIMPLE, 0 } } }, [NOT_TAKEN] = { .nuops = 1, .uops = { { _NOP, OPARG_SIMPLE, 0 } } }, [POP_EXCEPT] = { .nuops = 1, .uops = { { _POP_EXCEPT, OPARG_SIMPLE, 0 } } }, - [POP_ITER] = { .nuops = 1, .uops = { { _POP_TOP, OPARG_SIMPLE, 0 } } }, + [POP_ITER] = { .nuops = 1, .uops = { { _POP_ITER, OPARG_SIMPLE, 0 } } }, [POP_JUMP_IF_FALSE] = { .nuops = 1, .uops = { { _POP_JUMP_IF_FALSE, OPARG_REPLACED, 1 } } }, [POP_JUMP_IF_NONE] = { .nuops = 2, .uops = { { _IS_NONE, OPARG_SIMPLE, 1 }, { _POP_JUMP_IF_TRUE, OPARG_REPLACED, 1 } } }, [POP_JUMP_IF_NOT_NONE] = { .nuops = 2, .uops = { { _IS_NONE, OPARG_SIMPLE, 1 }, { _POP_JUMP_IF_FALSE, OPARG_REPLACED, 1 } } }, diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index d6d81f88c8e00c..3155b299f3db4b 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -256,8 +256,14 @@ extern "C" { #define _MONITOR_RESUME 471 #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT +<<<<<<< HEAD #define _POP_JUMP_IF_FALSE 472 #define _POP_JUMP_IF_TRUE 473 +======= +#define _POP_ITER POP_ITER +#define _POP_JUMP_IF_FALSE 465 +#define _POP_JUMP_IF_TRUE 466 +>>>>>>> d908cf7e4c1 (FOR_ITER now pushes NULL as well as the iterator. Preparation for virtual iterators) #define _POP_TOP POP_TOP #define _POP_TOP_LOAD_CONST_INLINE 474 #define _POP_TOP_LOAD_CONST_INLINE_BORROW 475 diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 0ea8e56b6cd0bc..c339888b892f5b 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -66,6 +66,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_POP_TOP] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_PUSH_NULL] = HAS_PURE_FLAG, [_END_FOR] = HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG, + [_POP_ITER] = HAS_ESCAPES_FLAG, [_END_SEND] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_UNARY_NEGATIVE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_UNARY_NOT] = HAS_PURE_FLAG, @@ -534,6 +535,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_MAYBE_EXPAND_METHOD_KW] = "_MAYBE_EXPAND_METHOD_KW", [_NOP] = "_NOP", [_POP_EXCEPT] = "_POP_EXCEPT", + [_POP_ITER] = "_POP_ITER", [_POP_TOP] = "_POP_TOP", [_POP_TOP_LOAD_CONST_INLINE] = "_POP_TOP_LOAD_CONST_INLINE", [_POP_TOP_LOAD_CONST_INLINE_BORROW] = "_POP_TOP_LOAD_CONST_INLINE_BORROW", @@ -694,6 +696,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _END_FOR: return 1; + case _POP_ITER: + return 2; case _END_SEND: return 2; case _UNARY_NEGATIVE: diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index f2586fcee57d87..8495c6b938c845 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -1821,7 +1821,7 @@ def _prepare_test_cases(): make_inst(opname='LOAD_SMALL_INT', arg=10, argval=10, argrepr='', offset=12, start_offset=12, starts_line=False, line_number=3), make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=14, start_offset=14, starts_line=False, line_number=3, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), make_inst(opname='GET_ITER', arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=False, line_number=3), - make_inst(opname='FOR_ITER', arg=32, argval=92, argrepr='to L4', offset=24, start_offset=24, starts_line=False, line_number=3, label=1, cache_info=[('counter', 1, b'\x00\x00')]), + make_inst(opname='FOR_ITER', arg=33, argval=94, argrepr='to L4', offset=24, start_offset=24, starts_line=False, line_number=3, label=1, cache_info=[('counter', 1, b'\x00\x00')]), make_inst(opname='STORE_FAST', arg=0, argval='i', argrepr='i', offset=28, start_offset=28, starts_line=False, line_number=3), make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=True, line_number=4, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=40, start_offset=40, starts_line=False, line_number=4), @@ -1840,110 +1840,111 @@ def _prepare_test_cases(): make_inst(opname='NOT_TAKEN', arg=None, argval=None, argrepr='', offset=82, start_offset=82, starts_line=False, line_number=7), make_inst(opname='JUMP_BACKWARD', arg=32, argval=24, argrepr='to L1', offset=84, start_offset=84, starts_line=False, line_number=7, cache_info=[('counter', 1, b'\x00\x00')]), make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=88, start_offset=88, starts_line=True, line_number=8, label=3), - make_inst(opname='JUMP_FORWARD', arg=13, argval=118, argrepr='to L5', offset=90, start_offset=90, starts_line=False, line_number=8), - make_inst(opname='END_FOR', arg=None, argval=None, argrepr='', offset=92, start_offset=92, starts_line=True, line_number=3, label=4), - make_inst(opname='POP_ITER', arg=None, argval=None, argrepr='', offset=94, start_offset=94, starts_line=False, line_number=3), - make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=96, start_offset=96, starts_line=True, line_number=10, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - make_inst(opname='LOAD_CONST', arg=1, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=106, start_offset=106, starts_line=False, line_number=10), - make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=108, start_offset=108, starts_line=False, line_number=10, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=116, start_offset=116, starts_line=False, line_number=10), - make_inst(opname='LOAD_FAST_CHECK', arg=0, argval='i', argrepr='i', offset=118, start_offset=118, starts_line=True, line_number=11, label=5), - make_inst(opname='TO_BOOL', arg=None, argval=None, argrepr='', offset=120, start_offset=120, starts_line=False, line_number=11, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), - make_inst(opname='POP_JUMP_IF_FALSE', arg=40, argval=212, argrepr='to L8', offset=128, start_offset=128, starts_line=False, line_number=11, cache_info=[('counter', 1, b'\x00\x00')]), - make_inst(opname='NOT_TAKEN', arg=None, argval=None, argrepr='', offset=132, start_offset=132, starts_line=False, line_number=11), - make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=134, start_offset=134, starts_line=True, line_number=12, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=144, start_offset=144, starts_line=False, line_number=12), - make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=146, start_offset=146, starts_line=False, line_number=12, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=154, start_offset=154, starts_line=False, line_number=12), - make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=156, start_offset=156, starts_line=True, line_number=13), - make_inst(opname='LOAD_SMALL_INT', arg=1, argval=1, argrepr='', offset=158, start_offset=158, starts_line=False, line_number=13), - make_inst(opname='BINARY_OP', arg=23, argval=23, argrepr='-=', offset=160, start_offset=160, starts_line=False, line_number=13, cache_info=[('counter', 1, b'\x00\x00'), ('descr', 4, b'\x00\x00\x00\x00\x00\x00\x00\x00')]), - make_inst(opname='STORE_FAST', arg=0, argval='i', argrepr='i', offset=172, start_offset=172, starts_line=False, line_number=13), - make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=174, start_offset=174, starts_line=True, line_number=14), - make_inst(opname='LOAD_SMALL_INT', arg=6, argval=6, argrepr='', offset=176, start_offset=176, starts_line=False, line_number=14), - make_inst(opname='COMPARE_OP', arg=148, argval='>', argrepr='bool(>)', offset=178, start_offset=178, starts_line=False, line_number=14, cache_info=[('counter', 1, b'\x00\x00')]), - make_inst(opname='POP_JUMP_IF_FALSE', arg=3, argval=192, argrepr='to L6', offset=182, start_offset=182, starts_line=False, line_number=14, cache_info=[('counter', 1, b'\x00\x00')]), - make_inst(opname='NOT_TAKEN', arg=None, argval=None, argrepr='', offset=186, start_offset=186, starts_line=False, line_number=14), - make_inst(opname='JUMP_BACKWARD', arg=37, argval=118, argrepr='to L5', offset=188, start_offset=188, starts_line=True, line_number=15, cache_info=[('counter', 1, b'\x00\x00')]), - make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=192, start_offset=192, starts_line=True, line_number=16, label=6), - make_inst(opname='LOAD_SMALL_INT', arg=4, argval=4, argrepr='', offset=194, start_offset=194, starts_line=False, line_number=16), - make_inst(opname='COMPARE_OP', arg=18, argval='<', argrepr='bool(<)', offset=196, start_offset=196, starts_line=False, line_number=16, cache_info=[('counter', 1, b'\x00\x00')]), - make_inst(opname='POP_JUMP_IF_TRUE', arg=3, argval=210, argrepr='to L7', offset=200, start_offset=200, starts_line=False, line_number=16, cache_info=[('counter', 1, b'\x00\x00')]), - make_inst(opname='NOT_TAKEN', arg=None, argval=None, argrepr='', offset=204, start_offset=204, starts_line=False, line_number=16), - make_inst(opname='JUMP_BACKWARD', arg=46, argval=118, argrepr='to L5', offset=206, start_offset=206, starts_line=False, line_number=16, cache_info=[('counter', 1, b'\x00\x00')]), - make_inst(opname='JUMP_FORWARD', arg=11, argval=234, argrepr='to L9', offset=210, start_offset=210, starts_line=True, line_number=17, label=7), - make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=212, start_offset=212, starts_line=True, line_number=19, label=8, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - make_inst(opname='LOAD_CONST', arg=2, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=222, start_offset=222, starts_line=False, line_number=19), - make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=224, start_offset=224, starts_line=False, line_number=19, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=232, start_offset=232, starts_line=False, line_number=19), - make_inst(opname='NOP', arg=None, argval=None, argrepr='', offset=234, start_offset=234, starts_line=True, line_number=20, label=9), - make_inst(opname='LOAD_SMALL_INT', arg=1, argval=1, argrepr='', offset=236, start_offset=236, starts_line=True, line_number=21), - make_inst(opname='LOAD_SMALL_INT', arg=0, argval=0, argrepr='', offset=238, start_offset=238, starts_line=False, line_number=21), - make_inst(opname='BINARY_OP', arg=11, argval=11, argrepr='/', offset=240, start_offset=240, starts_line=False, line_number=21, cache_info=[('counter', 1, b'\x00\x00'), ('descr', 4, b'\x00\x00\x00\x00\x00\x00\x00\x00')]), - make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=252, start_offset=252, starts_line=False, line_number=21), - make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=254, start_offset=254, starts_line=True, line_number=25), - make_inst(opname='COPY', arg=1, argval=1, argrepr='', offset=256, start_offset=256, starts_line=False, line_number=25), - make_inst(opname='LOAD_SPECIAL', arg=1, argval=1, argrepr='__exit__', offset=258, start_offset=258, starts_line=False, line_number=25), - make_inst(opname='SWAP', arg=2, argval=2, argrepr='', offset=260, start_offset=260, starts_line=False, line_number=25), - make_inst(opname='SWAP', arg=3, argval=3, argrepr='', offset=262, start_offset=262, starts_line=False, line_number=25), - make_inst(opname='LOAD_SPECIAL', arg=0, argval=0, argrepr='__enter__', offset=264, start_offset=264, starts_line=False, line_number=25), - make_inst(opname='CALL', arg=0, argval=0, argrepr='', offset=266, start_offset=266, starts_line=False, line_number=25, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - make_inst(opname='STORE_FAST', arg=1, argval='dodgy', argrepr='dodgy', offset=274, start_offset=274, starts_line=False, line_number=25), - make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=276, start_offset=276, starts_line=True, line_number=26, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - make_inst(opname='LOAD_CONST', arg=3, argval='Never reach this', argrepr="'Never reach this'", offset=286, start_offset=286, starts_line=False, line_number=26), - make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=288, start_offset=288, starts_line=False, line_number=26, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=296, start_offset=296, starts_line=False, line_number=26), - make_inst(opname='LOAD_CONST', arg=4, argval=None, argrepr='None', offset=298, start_offset=298, starts_line=True, line_number=25), - make_inst(opname='LOAD_CONST', arg=4, argval=None, argrepr='None', offset=300, start_offset=300, starts_line=False, line_number=25), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=90, start_offset=90, starts_line=False, line_number=8), + make_inst(opname='JUMP_FORWARD', arg=13, argval=120, argrepr='to L5', offset=92, start_offset=92, starts_line=False, line_number=8), + make_inst(opname='END_FOR', arg=None, argval=None, argrepr='', offset=94, start_offset=94, starts_line=True, line_number=3, label=4), + make_inst(opname='POP_ITER', arg=None, argval=None, argrepr='', offset=96, start_offset=96, starts_line=False, line_number=3), + make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=98, start_offset=98, starts_line=True, line_number=10, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + make_inst(opname='LOAD_CONST', arg=1, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=108, start_offset=108, starts_line=False, line_number=10), + make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=110, start_offset=110, starts_line=False, line_number=10, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=118, start_offset=118, starts_line=False, line_number=10), + make_inst(opname='LOAD_FAST_CHECK', arg=0, argval='i', argrepr='i', offset=120, start_offset=120, starts_line=True, line_number=11, label=5), + make_inst(opname='TO_BOOL', arg=None, argval=None, argrepr='', offset=122, start_offset=122, starts_line=False, line_number=11, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='POP_JUMP_IF_FALSE', arg=40, argval=214, argrepr='to L8', offset=130, start_offset=130, starts_line=False, line_number=11, cache_info=[('counter', 1, b'\x00\x00')]), + make_inst(opname='NOT_TAKEN', arg=None, argval=None, argrepr='', offset=134, start_offset=134, starts_line=False, line_number=11), + make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=136, start_offset=136, starts_line=True, line_number=12, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=146, start_offset=146, starts_line=False, line_number=12), + make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=148, start_offset=148, starts_line=False, line_number=12, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=156, start_offset=156, starts_line=False, line_number=12), + make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=158, start_offset=158, starts_line=True, line_number=13), + make_inst(opname='LOAD_SMALL_INT', arg=1, argval=1, argrepr='', offset=160, start_offset=160, starts_line=False, line_number=13), + make_inst(opname='BINARY_OP', arg=23, argval=23, argrepr='-=', offset=162, start_offset=162, starts_line=False, line_number=13, cache_info=[('counter', 1, b'\x00\x00'), ('descr', 4, b'\x00\x00\x00\x00\x00\x00\x00\x00')]), + make_inst(opname='STORE_FAST', arg=0, argval='i', argrepr='i', offset=174, start_offset=174, starts_line=False, line_number=13), + make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=176, start_offset=176, starts_line=True, line_number=14), + make_inst(opname='LOAD_SMALL_INT', arg=6, argval=6, argrepr='', offset=178, start_offset=178, starts_line=False, line_number=14), + make_inst(opname='COMPARE_OP', arg=148, argval='>', argrepr='bool(>)', offset=180, start_offset=180, starts_line=False, line_number=14, cache_info=[('counter', 1, b'\x00\x00')]), + make_inst(opname='POP_JUMP_IF_FALSE', arg=3, argval=194, argrepr='to L6', offset=184, start_offset=184, starts_line=False, line_number=14, cache_info=[('counter', 1, b'\x00\x00')]), + make_inst(opname='NOT_TAKEN', arg=None, argval=None, argrepr='', offset=188, start_offset=188, starts_line=False, line_number=14), + make_inst(opname='JUMP_BACKWARD', arg=37, argval=120, argrepr='to L5', offset=190, start_offset=190, starts_line=True, line_number=15, cache_info=[('counter', 1, b'\x00\x00')]), + make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=194, start_offset=194, starts_line=True, line_number=16, label=6), + make_inst(opname='LOAD_SMALL_INT', arg=4, argval=4, argrepr='', offset=196, start_offset=196, starts_line=False, line_number=16), + make_inst(opname='COMPARE_OP', arg=18, argval='<', argrepr='bool(<)', offset=198, start_offset=198, starts_line=False, line_number=16, cache_info=[('counter', 1, b'\x00\x00')]), + make_inst(opname='POP_JUMP_IF_TRUE', arg=3, argval=212, argrepr='to L7', offset=202, start_offset=202, starts_line=False, line_number=16, cache_info=[('counter', 1, b'\x00\x00')]), + make_inst(opname='NOT_TAKEN', arg=None, argval=None, argrepr='', offset=206, start_offset=206, starts_line=False, line_number=16), + make_inst(opname='JUMP_BACKWARD', arg=46, argval=120, argrepr='to L5', offset=208, start_offset=208, starts_line=False, line_number=16, cache_info=[('counter', 1, b'\x00\x00')]), + make_inst(opname='JUMP_FORWARD', arg=11, argval=236, argrepr='to L9', offset=212, start_offset=212, starts_line=True, line_number=17, label=7), + make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=214, start_offset=214, starts_line=True, line_number=19, label=8, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + make_inst(opname='LOAD_CONST', arg=2, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=224, start_offset=224, starts_line=False, line_number=19), + make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=226, start_offset=226, starts_line=False, line_number=19, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=234, start_offset=234, starts_line=False, line_number=19), + make_inst(opname='NOP', arg=None, argval=None, argrepr='', offset=236, start_offset=236, starts_line=True, line_number=20, label=9), + make_inst(opname='LOAD_SMALL_INT', arg=1, argval=1, argrepr='', offset=238, start_offset=238, starts_line=True, line_number=21), + make_inst(opname='LOAD_SMALL_INT', arg=0, argval=0, argrepr='', offset=240, start_offset=240, starts_line=False, line_number=21), + make_inst(opname='BINARY_OP', arg=11, argval=11, argrepr='/', offset=242, start_offset=242, starts_line=False, line_number=21, cache_info=[('counter', 1, b'\x00\x00'), ('descr', 4, b'\x00\x00\x00\x00\x00\x00\x00\x00')]), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=254, start_offset=254, starts_line=False, line_number=21), + make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=256, start_offset=256, starts_line=True, line_number=25), + make_inst(opname='COPY', arg=1, argval=1, argrepr='', offset=258, start_offset=258, starts_line=False, line_number=25), + make_inst(opname='LOAD_SPECIAL', arg=1, argval=1, argrepr='__exit__', offset=260, start_offset=260, starts_line=False, line_number=25), + make_inst(opname='SWAP', arg=2, argval=2, argrepr='', offset=262, start_offset=262, starts_line=False, line_number=25), + make_inst(opname='SWAP', arg=3, argval=3, argrepr='', offset=264, start_offset=264, starts_line=False, line_number=25), + make_inst(opname='LOAD_SPECIAL', arg=0, argval=0, argrepr='__enter__', offset=266, start_offset=266, starts_line=False, line_number=25), + make_inst(opname='CALL', arg=0, argval=0, argrepr='', offset=268, start_offset=268, starts_line=False, line_number=25, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='STORE_FAST', arg=1, argval='dodgy', argrepr='dodgy', offset=276, start_offset=276, starts_line=False, line_number=25), + make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=278, start_offset=278, starts_line=True, line_number=26, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + make_inst(opname='LOAD_CONST', arg=3, argval='Never reach this', argrepr="'Never reach this'", offset=288, start_offset=288, starts_line=False, line_number=26), + make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=290, start_offset=290, starts_line=False, line_number=26, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=298, start_offset=298, starts_line=False, line_number=26), + make_inst(opname='LOAD_CONST', arg=4, argval=None, argrepr='None', offset=300, start_offset=300, starts_line=True, line_number=25), make_inst(opname='LOAD_CONST', arg=4, argval=None, argrepr='None', offset=302, start_offset=302, starts_line=False, line_number=25), - make_inst(opname='CALL', arg=3, argval=3, argrepr='', offset=304, start_offset=304, starts_line=False, line_number=25, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=312, start_offset=312, starts_line=False, line_number=25), - make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=314, start_offset=314, starts_line=True, line_number=28, label=10, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - make_inst(opname='LOAD_CONST', arg=6, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=324, start_offset=324, starts_line=False, line_number=28), - make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=326, start_offset=326, starts_line=False, line_number=28, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=334, start_offset=334, starts_line=False, line_number=28), - make_inst(opname='LOAD_CONST', arg=4, argval=None, argrepr='None', offset=336, start_offset=336, starts_line=False, line_number=28), - make_inst(opname='RETURN_VALUE', arg=None, argval=None, argrepr='', offset=338, start_offset=338, starts_line=False, line_number=28), - make_inst(opname='PUSH_EXC_INFO', arg=None, argval=None, argrepr='', offset=340, start_offset=340, starts_line=True, line_number=25), - make_inst(opname='WITH_EXCEPT_START', arg=None, argval=None, argrepr='', offset=342, start_offset=342, starts_line=False, line_number=25), - make_inst(opname='TO_BOOL', arg=None, argval=None, argrepr='', offset=344, start_offset=344, starts_line=False, line_number=25, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), - make_inst(opname='POP_JUMP_IF_TRUE', arg=2, argval=360, argrepr='to L11', offset=352, start_offset=352, starts_line=False, line_number=25, cache_info=[('counter', 1, b'\x00\x00')]), - make_inst(opname='NOT_TAKEN', arg=None, argval=None, argrepr='', offset=356, start_offset=356, starts_line=False, line_number=25), - make_inst(opname='RERAISE', arg=2, argval=2, argrepr='', offset=358, start_offset=358, starts_line=False, line_number=25), - make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=360, start_offset=360, starts_line=False, line_number=25, label=11), - make_inst(opname='POP_EXCEPT', arg=None, argval=None, argrepr='', offset=362, start_offset=362, starts_line=False, line_number=25), - make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=364, start_offset=364, starts_line=False, line_number=25), + make_inst(opname='LOAD_CONST', arg=4, argval=None, argrepr='None', offset=304, start_offset=304, starts_line=False, line_number=25), + make_inst(opname='CALL', arg=3, argval=3, argrepr='', offset=306, start_offset=306, starts_line=False, line_number=25, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=314, start_offset=314, starts_line=False, line_number=25), + make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=316, start_offset=316, starts_line=True, line_number=28, label=10, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + make_inst(opname='LOAD_CONST', arg=6, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=326, start_offset=326, starts_line=False, line_number=28), + make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=328, start_offset=328, starts_line=False, line_number=28, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=336, start_offset=336, starts_line=False, line_number=28), + make_inst(opname='LOAD_CONST', arg=4, argval=None, argrepr='None', offset=338, start_offset=338, starts_line=False, line_number=28), + make_inst(opname='RETURN_VALUE', arg=None, argval=None, argrepr='', offset=340, start_offset=340, starts_line=False, line_number=28), + make_inst(opname='PUSH_EXC_INFO', arg=None, argval=None, argrepr='', offset=342, start_offset=342, starts_line=True, line_number=25), + make_inst(opname='WITH_EXCEPT_START', arg=None, argval=None, argrepr='', offset=344, start_offset=344, starts_line=False, line_number=25), + make_inst(opname='TO_BOOL', arg=None, argval=None, argrepr='', offset=346, start_offset=346, starts_line=False, line_number=25, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='POP_JUMP_IF_TRUE', arg=2, argval=362, argrepr='to L11', offset=354, start_offset=354, starts_line=False, line_number=25, cache_info=[('counter', 1, b'\x00\x00')]), + make_inst(opname='NOT_TAKEN', arg=None, argval=None, argrepr='', offset=358, start_offset=358, starts_line=False, line_number=25), + make_inst(opname='RERAISE', arg=2, argval=2, argrepr='', offset=360, start_offset=360, starts_line=False, line_number=25), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=362, start_offset=362, starts_line=False, line_number=25, label=11), + make_inst(opname='POP_EXCEPT', arg=None, argval=None, argrepr='', offset=364, start_offset=364, starts_line=False, line_number=25), make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=366, start_offset=366, starts_line=False, line_number=25), make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=368, start_offset=368, starts_line=False, line_number=25), - make_inst(opname='JUMP_BACKWARD_NO_INTERRUPT', arg=29, argval=314, argrepr='to L10', offset=370, start_offset=370, starts_line=False, line_number=25), - make_inst(opname='COPY', arg=3, argval=3, argrepr='', offset=372, start_offset=372, starts_line=True, line_number=None), - make_inst(opname='POP_EXCEPT', arg=None, argval=None, argrepr='', offset=374, start_offset=374, starts_line=False, line_number=None), - make_inst(opname='RERAISE', arg=1, argval=1, argrepr='', offset=376, start_offset=376, starts_line=False, line_number=None), - make_inst(opname='PUSH_EXC_INFO', arg=None, argval=None, argrepr='', offset=378, start_offset=378, starts_line=False, line_number=None), - make_inst(opname='LOAD_GLOBAL', arg=4, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=380, start_offset=380, starts_line=True, line_number=22, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - make_inst(opname='CHECK_EXC_MATCH', arg=None, argval=None, argrepr='', offset=390, start_offset=390, starts_line=False, line_number=22), - make_inst(opname='POP_JUMP_IF_FALSE', arg=15, argval=426, argrepr='to L12', offset=392, start_offset=392, starts_line=False, line_number=22, cache_info=[('counter', 1, b'\x00\x00')]), - make_inst(opname='NOT_TAKEN', arg=None, argval=None, argrepr='', offset=396, start_offset=396, starts_line=False, line_number=22), - make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=398, start_offset=398, starts_line=False, line_number=22), - make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=400, start_offset=400, starts_line=True, line_number=23, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - make_inst(opname='LOAD_CONST', arg=5, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=410, start_offset=410, starts_line=False, line_number=23), - make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=412, start_offset=412, starts_line=False, line_number=23, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=420, start_offset=420, starts_line=False, line_number=23), - make_inst(opname='POP_EXCEPT', arg=None, argval=None, argrepr='', offset=422, start_offset=422, starts_line=False, line_number=23), - make_inst(opname='JUMP_BACKWARD_NO_INTERRUPT', arg=56, argval=314, argrepr='to L10', offset=424, start_offset=424, starts_line=False, line_number=23), - make_inst(opname='RERAISE', arg=0, argval=0, argrepr='', offset=426, start_offset=426, starts_line=True, line_number=22, label=12), - make_inst(opname='COPY', arg=3, argval=3, argrepr='', offset=428, start_offset=428, starts_line=True, line_number=None), - make_inst(opname='POP_EXCEPT', arg=None, argval=None, argrepr='', offset=430, start_offset=430, starts_line=False, line_number=None), - make_inst(opname='RERAISE', arg=1, argval=1, argrepr='', offset=432, start_offset=432, starts_line=False, line_number=None), - make_inst(opname='PUSH_EXC_INFO', arg=None, argval=None, argrepr='', offset=434, start_offset=434, starts_line=False, line_number=None), - make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=436, start_offset=436, starts_line=True, line_number=28, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - make_inst(opname='LOAD_CONST', arg=6, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=446, start_offset=446, starts_line=False, line_number=28), - make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=448, start_offset=448, starts_line=False, line_number=28, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=456, start_offset=456, starts_line=False, line_number=28), - make_inst(opname='RERAISE', arg=0, argval=0, argrepr='', offset=458, start_offset=458, starts_line=False, line_number=28), - make_inst(opname='COPY', arg=3, argval=3, argrepr='', offset=460, start_offset=460, starts_line=True, line_number=None), - make_inst(opname='POP_EXCEPT', arg=None, argval=None, argrepr='', offset=462, start_offset=462, starts_line=False, line_number=None), - make_inst(opname='RERAISE', arg=1, argval=1, argrepr='', offset=464, start_offset=464, starts_line=False, line_number=None), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=370, start_offset=370, starts_line=False, line_number=25), + make_inst(opname='JUMP_BACKWARD_NO_INTERRUPT', arg=29, argval=316, argrepr='to L10', offset=372, start_offset=372, starts_line=False, line_number=25), + make_inst(opname='COPY', arg=3, argval=3, argrepr='', offset=374, start_offset=374, starts_line=True, line_number=None), + make_inst(opname='POP_EXCEPT', arg=None, argval=None, argrepr='', offset=376, start_offset=376, starts_line=False, line_number=None), + make_inst(opname='RERAISE', arg=1, argval=1, argrepr='', offset=378, start_offset=378, starts_line=False, line_number=None), + make_inst(opname='PUSH_EXC_INFO', arg=None, argval=None, argrepr='', offset=380, start_offset=380, starts_line=False, line_number=None), + make_inst(opname='LOAD_GLOBAL', arg=4, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=382, start_offset=382, starts_line=True, line_number=22, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + make_inst(opname='CHECK_EXC_MATCH', arg=None, argval=None, argrepr='', offset=392, start_offset=392, starts_line=False, line_number=22), + make_inst(opname='POP_JUMP_IF_FALSE', arg=15, argval=428, argrepr='to L12', offset=394, start_offset=394, starts_line=False, line_number=22, cache_info=[('counter', 1, b'\x00\x00')]), + make_inst(opname='NOT_TAKEN', arg=None, argval=None, argrepr='', offset=398, start_offset=398, starts_line=False, line_number=22), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=400, start_offset=400, starts_line=False, line_number=22), + make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=402, start_offset=402, starts_line=True, line_number=23, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + make_inst(opname='LOAD_CONST', arg=5, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=412, start_offset=412, starts_line=False, line_number=23), + make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=414, start_offset=414, starts_line=False, line_number=23, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=422, start_offset=422, starts_line=False, line_number=23), + make_inst(opname='POP_EXCEPT', arg=None, argval=None, argrepr='', offset=424, start_offset=424, starts_line=False, line_number=23), + make_inst(opname='JUMP_BACKWARD_NO_INTERRUPT', arg=56, argval=316, argrepr='to L10', offset=426, start_offset=426, starts_line=False, line_number=23), + make_inst(opname='RERAISE', arg=0, argval=0, argrepr='', offset=428, start_offset=428, starts_line=True, line_number=22, label=12), + make_inst(opname='COPY', arg=3, argval=3, argrepr='', offset=430, start_offset=430, starts_line=True, line_number=None), + make_inst(opname='POP_EXCEPT', arg=None, argval=None, argrepr='', offset=432, start_offset=432, starts_line=False, line_number=None), + make_inst(opname='RERAISE', arg=1, argval=1, argrepr='', offset=434, start_offset=434, starts_line=False, line_number=None), + make_inst(opname='PUSH_EXC_INFO', arg=None, argval=None, argrepr='', offset=436, start_offset=436, starts_line=False, line_number=None), + make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=438, start_offset=438, starts_line=True, line_number=28, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + make_inst(opname='LOAD_CONST', arg=6, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=448, start_offset=448, starts_line=False, line_number=28), + make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=450, start_offset=450, starts_line=False, line_number=28, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=458, start_offset=458, starts_line=False, line_number=28), + make_inst(opname='RERAISE', arg=0, argval=0, argrepr='', offset=460, start_offset=460, starts_line=False, line_number=28), + make_inst(opname='COPY', arg=3, argval=3, argrepr='', offset=462, start_offset=462, starts_line=True, line_number=None), + make_inst(opname='POP_EXCEPT', arg=None, argval=None, argrepr='', offset=464, start_offset=464, starts_line=False, line_number=None), + make_inst(opname='RERAISE', arg=1, argval=1, argrepr='', offset=466, start_offset=466, starts_line=False, line_number=None), ] # One last piece of inspect fodder to check the default line number handling diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 7a62219c139ee5..0d775b7bfa9f42 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1386,6 +1386,10 @@ mark_stacks(PyCodeObject *code_obj, int len) stacks[j] = next_stack; break; case GET_ITER: + next_stack = push_value(pop_value(next_stack), Iterator); + next_stack = push_value(next_stack, Iterator); + stacks[next_i] = next_stack; + break; case GET_AITER: next_stack = push_value(pop_value(next_stack), Iterator); stacks[next_i] = next_stack; diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h index 8cedee31e08a00..bdd2744e266a89 100644 --- a/Programs/test_frozenmain.h +++ b/Programs/test_frozenmain.h @@ -1,6 +1,6 @@ // Auto-generated by Programs/freeze_test_frozenmain.py unsigned char M_test_frozenmain[] = { - 227,0,0,0,0,0,0,0,0,0,0,0,0,8,0,0, + 227,0,0,0,0,0,0,0,0,0,0,0,0,9,0,0, 0,0,0,0,0,243,184,0,0,0,128,0,92,0,80,1, 71,0,114,0,92,0,80,1,71,1,114,1,91,2,32,0, 80,2,50,1,0,0,0,0,0,0,30,0,91,2,32,0, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 6592bc57ed20a1..b6970445384bc4 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -389,7 +389,7 @@ dummy_func( } pure inst(POP_TOP, (value --)) { - PyStackRef_CLOSE(value); + PyStackRef_XCLOSE(value); } pure inst(PUSH_NULL, (-- res)) { @@ -405,9 +405,14 @@ dummy_func( PyStackRef_CLOSE(value); } - macro(POP_ITER) = POP_TOP; - no_save_ip tier1 inst(INSTRUMENTED_END_FOR, (receiver, value -- receiver)) { + inst(POP_ITER, (iter, index_or_null -- )) { + (void)index_or_null; + DEAD(index_or_null); + PyStackRef_CLOSE(iter); + } + + no_save_ip tier1 inst(INSTRUMENTED_END_FOR, (receiver, index_or_null, value -- receiver, index_or_null)) { /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ if (PyStackRef_GenCheck(receiver)) { @@ -419,7 +424,9 @@ dummy_func( PyStackRef_CLOSE(value); } - tier1 inst(INSTRUMENTED_POP_ITER, (iter -- )) { + tier1 inst(INSTRUMENTED_POP_ITER, (iter, index_or_null -- )) { + (void)index_or_null; + DEAD(index_or_null); INSTRUMENTED_JUMP(prev_instr, this_instr+1, PY_MONITORING_EVENT_BRANCH_RIGHT); PyStackRef_CLOSE(iter); } @@ -3019,7 +3026,7 @@ dummy_func( values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); } - inst(GET_ITER, (iterable -- iter)) { + inst(GET_ITER, (iterable -- iter, null)) { #ifdef Py_STATS _Py_GatherStats_GetIter(iterable); #endif @@ -3028,6 +3035,7 @@ dummy_func( PyStackRef_CLOSE(iterable); ERROR_IF(iter_o == NULL, error); iter = PyStackRef_FromPyObjectSteal(iter_o); + null = PyStackRef_NULL; } inst(GET_YIELD_FROM_ITER, (iterable -- iter)) { @@ -3074,7 +3082,7 @@ dummy_func( FOR_ITER_GEN, }; - specializing op(_SPECIALIZE_FOR_ITER, (counter/1, iter -- iter)) { + specializing op(_SPECIALIZE_FOR_ITER, (counter/1, iter, null_or_index -- iter, null_or_index)) { #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; @@ -3086,7 +3094,7 @@ dummy_func( #endif /* ENABLE_SPECIALIZATION_FT */ } - replaced op(_FOR_ITER, (iter -- iter, next)) { + replaced op(_FOR_ITER, (iter, null_or_index -- iter, null_or_index, next)) { /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); @@ -3110,7 +3118,7 @@ dummy_func( // Common case: no jump, leave it to the code generator } - op(_FOR_ITER_TIER_TWO, (iter -- iter, next)) { + op(_FOR_ITER_TIER_TWO, (iter, null_or_index -- iter, null_or_index, next)) { /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); @@ -3134,7 +3142,7 @@ dummy_func( macro(FOR_ITER) = _SPECIALIZE_FOR_ITER + _FOR_ITER; - inst(INSTRUMENTED_FOR_ITER, (unused/1, iter -- iter, next)) { + inst(INSTRUMENTED_FOR_ITER, (unused/1, iter, null_or_index -- iter, null_or_index, next)) { PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); if (next_o != NULL) { @@ -3160,7 +3168,7 @@ dummy_func( } - op(_ITER_CHECK_LIST, (iter -- iter)) { + op(_ITER_CHECK_LIST, (iter, null_or_index -- iter, null_or_index)) { PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); EXIT_IF(Py_TYPE(iter_o) != &PyListIter_Type); #ifdef Py_GIL_DISABLED @@ -3171,7 +3179,7 @@ dummy_func( #endif } - replaced op(_ITER_JUMP_LIST, (iter -- iter)) { + replaced op(_ITER_JUMP_LIST, (iter, null_or_index -- iter, null_or_index)) { PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(iter_o) == &PyListIter_Type); // For free-threaded Python, the loop exit can happen at any point during @@ -3199,7 +3207,7 @@ dummy_func( } // Only used by Tier 2 - op(_GUARD_NOT_EXHAUSTED_LIST, (iter -- iter)) { + op(_GUARD_NOT_EXHAUSTED_LIST, (iter, null_or_index -- iter, null_or_index)) { #ifndef Py_GIL_DISABLED PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); _PyListIterObject *it = (_PyListIterObject *)iter_o; @@ -3213,7 +3221,7 @@ dummy_func( #endif } - replaced op(_ITER_NEXT_LIST, (iter -- iter, next)) { + replaced op(_ITER_NEXT_LIST, (iter, null_or_index -- iter, null_or_index, next)) { PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); _PyListIterObject *it = (_PyListIterObject *)iter_o; assert(Py_TYPE(iter_o) == &PyListIter_Type); @@ -3242,7 +3250,7 @@ dummy_func( } // Only used by Tier 2 - op(_ITER_NEXT_LIST_TIER_TWO, (iter -- iter, next)) { + op(_ITER_NEXT_LIST_TIER_TWO, (iter, null_or_index -- iter, null_or_index, next)) { PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); _PyListIterObject *it = (_PyListIterObject *)iter_o; assert(Py_TYPE(iter_o) == &PyListIter_Type); @@ -3274,7 +3282,7 @@ dummy_func( _ITER_JUMP_LIST + _ITER_NEXT_LIST; - op(_ITER_CHECK_TUPLE, (iter -- iter)) { + op(_ITER_CHECK_TUPLE, (iter, null_or_index -- iter, null_or_index)) { PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); EXIT_IF(Py_TYPE(iter_o) != &PyTupleIter_Type); #ifdef Py_GIL_DISABLED @@ -3282,7 +3290,7 @@ dummy_func( #endif } - replaced op(_ITER_JUMP_TUPLE, (iter -- iter)) { + replaced op(_ITER_JUMP_TUPLE, (iter, null_or_index -- iter, null_or_index)) { PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); (void)iter_o; assert(Py_TYPE(iter_o) == &PyTupleIter_Type); @@ -3306,7 +3314,7 @@ dummy_func( } // Only used by Tier 2 - op(_GUARD_NOT_EXHAUSTED_TUPLE, (iter -- iter)) { + op(_GUARD_NOT_EXHAUSTED_TUPLE, (iter, null_or_index -- iter, null_or_index)) { PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; assert(Py_TYPE(iter_o) == &PyTupleIter_Type); @@ -3318,7 +3326,7 @@ dummy_func( EXIT_IF(it->it_index >= PyTuple_GET_SIZE(seq)); } - op(_ITER_NEXT_TUPLE, (iter -- iter, next)) { + op(_ITER_NEXT_TUPLE, (iter, null_or_index -- iter, null_or_index, next)) { PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; assert(Py_TYPE(iter_o) == &PyTupleIter_Type); @@ -3337,7 +3345,7 @@ dummy_func( _ITER_JUMP_TUPLE + _ITER_NEXT_TUPLE; - op(_ITER_CHECK_RANGE, (iter -- iter)) { + op(_ITER_CHECK_RANGE, (iter, null_or_index -- iter, null_or_index)) { _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); EXIT_IF(Py_TYPE(r) != &PyRangeIter_Type); #ifdef Py_GIL_DISABLED @@ -3345,7 +3353,7 @@ dummy_func( #endif } - replaced op(_ITER_JUMP_RANGE, (iter -- iter)) { + replaced op(_ITER_JUMP_RANGE, (iter, null_or_index -- iter, null_or_index)) { _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); #ifdef Py_GIL_DISABLED @@ -3360,13 +3368,13 @@ dummy_func( } // Only used by Tier 2 - op(_GUARD_NOT_EXHAUSTED_RANGE, (iter -- iter)) { + op(_GUARD_NOT_EXHAUSTED_RANGE, (iter, null_or_index -- iter, null_or_index)) { _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); EXIT_IF(r->len <= 0); } - op(_ITER_NEXT_RANGE, (iter -- iter, next)) { + op(_ITER_NEXT_RANGE, (iter, null_or_index -- iter, null_or_index, next)) { _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); #ifdef Py_GIL_DISABLED @@ -3387,7 +3395,7 @@ dummy_func( _ITER_JUMP_RANGE + _ITER_NEXT_RANGE; - op(_FOR_ITER_GEN_FRAME, (iter -- iter, gen_frame: _PyInterpreterFrame*)) { + op(_FOR_ITER_GEN_FRAME, (iter, null -- iter, null, gen_frame: _PyInterpreterFrame*)) { PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); DEOPT_IF(Py_TYPE(gen) != &PyGen_Type); #ifdef Py_GIL_DISABLED diff --git a/Python/codegen.c b/Python/codegen.c index a7e412f7032c45..a5eeaae0cad6bd 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -525,6 +525,15 @@ codegen_unwind_fblock(compiler *c, location *ploc, return SUCCESS; case COMPILE_FBLOCK_FOR_LOOP: + /* Pop the iterator */ + if (preserve_tos) { + ADDOP_I(c, *ploc, SWAP, 3); + } + ADDOP(c, *ploc, POP_TOP); + ADDOP(c, *ploc, POP_TOP); + return SUCCESS; + + case COMPILE_FBLOCK_ASYNC_FOR_LOOP: /* Pop the iterator */ if (preserve_tos) { ADDOP_I(c, *ploc, SWAP, 2); @@ -629,7 +638,8 @@ codegen_unwind_fblock_stack(compiler *c, location *ploc, c, *ploc, "'break', 'continue' and 'return' cannot appear in an except* block"); } if (loop != NULL && (top->fb_type == COMPILE_FBLOCK_WHILE_LOOP || - top->fb_type == COMPILE_FBLOCK_FOR_LOOP)) { + top->fb_type == COMPILE_FBLOCK_FOR_LOOP || + top->fb_type == COMPILE_FBLOCK_ASYNC_FOR_LOOP)) { *loop = top; return SUCCESS; } @@ -2125,7 +2135,7 @@ codegen_async_for(compiler *c, stmt_ty s) ADDOP(c, LOC(s->v.AsyncFor.iter), GET_AITER); USE_LABEL(c, start); - RETURN_IF_ERROR(_PyCompile_PushFBlock(c, loc, COMPILE_FBLOCK_FOR_LOOP, start, end, NULL)); + RETURN_IF_ERROR(_PyCompile_PushFBlock(c, loc, COMPILE_FBLOCK_ASYNC_FOR_LOOP, start, end, NULL)); /* SETUP_FINALLY to guard the __anext__ call */ ADDOP_JUMP(c, loc, SETUP_FINALLY, except); @@ -2142,7 +2152,7 @@ codegen_async_for(compiler *c, stmt_ty s) /* Mark jump as artificial */ ADDOP_JUMP(c, NO_LOCATION, JUMP, start); - _PyCompile_PopFBlock(c, COMPILE_FBLOCK_FOR_LOOP, start); + _PyCompile_PopFBlock(c, COMPILE_FBLOCK_ASYNC_FOR_LOOP, start); /* Except block for __anext__ */ USE_LABEL(c, except); @@ -3889,10 +3899,11 @@ maybe_optimize_function_call(compiler *c, expr_ty e, jump_target_label end) NEW_JUMP_TARGET_LABEL(c, loop); NEW_JUMP_TARGET_LABEL(c, cleanup); + ADDOP(c, loc, PUSH_NULL); // Push NULL index for loop USE_LABEL(c, loop); ADDOP_JUMP(c, loc, FOR_ITER, cleanup); if (const_oparg == CONSTANT_BUILTIN_TUPLE) { - ADDOP_I(c, loc, LIST_APPEND, 2); + ADDOP_I(c, loc, LIST_APPEND, 3); ADDOP_JUMP(c, loc, JUMP, loop); } else { @@ -4366,13 +4377,12 @@ codegen_sync_comprehension_generator(compiler *c, location loc, } if (IS_JUMP_TARGET_LABEL(start)) { VISIT(c, expr, gen->iter); - ADDOP(c, LOC(gen->iter), GET_ITER); } } } if (IS_JUMP_TARGET_LABEL(start)) { - depth++; + depth += 2; ADDOP(c, LOC(gen->iter), GET_ITER); USE_LABEL(c, start); ADDOP_JUMP(c, LOC(gen->iter), FOR_ITER, anchor); @@ -4681,19 +4691,6 @@ pop_inlined_comprehension_state(compiler *c, location loc, return SUCCESS; } -static inline int -codegen_comprehension_iter(compiler *c, comprehension_ty comp) -{ - VISIT(c, expr, comp->iter); - if (comp->is_async) { - ADDOP(c, LOC(comp->iter), GET_AITER); - } - else { - ADDOP(c, LOC(comp->iter), GET_ITER); - } - return SUCCESS; -} - static int codegen_comprehension(compiler *c, expr_ty e, int type, identifier name, asdl_comprehension_seq *generators, expr_ty elt, @@ -4713,9 +4710,7 @@ codegen_comprehension(compiler *c, expr_ty e, int type, outermost = (comprehension_ty) asdl_seq_GET(generators, 0); if (is_inlined) { - if (codegen_comprehension_iter(c, outermost)) { - goto error; - } + VISIT(c, expr, outermost->iter); if (push_inlined_comprehension_state(c, loc, entry, &inline_state)) { goto error; } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 4e57906c84a154..c97910f07eb847 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -584,7 +584,7 @@ stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); + PyStackRef_XCLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); break; } @@ -609,6 +609,20 @@ break; } + case _POP_ITER: { + _PyStackRef index_or_null; + _PyStackRef iter; + index_or_null = stack_pointer[-1]; + iter = stack_pointer[-2]; + (void)index_or_null; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iter); + stack_pointer = _PyFrame_GetStackPointer(frame); + break; + } + case _END_SEND: { _PyStackRef value; _PyStackRef receiver; @@ -4074,6 +4088,7 @@ case _GET_ITER: { _PyStackRef iterable; _PyStackRef iter; + _PyStackRef null; iterable = stack_pointer[-1]; #ifdef Py_STATS _PyFrame_SetStackPointer(frame, stack_pointer); @@ -4092,8 +4107,10 @@ JUMP_TO_ERROR(); } iter = PyStackRef_FromPyObjectSteal(iter_o); + null = PyStackRef_NULL; stack_pointer[0] = iter; - stack_pointer += 1; + stack_pointer[1] = null; + stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); break; } @@ -4141,7 +4158,7 @@ case _FOR_ITER_TIER_TWO: { _PyStackRef iter; _PyStackRef next; - iter = stack_pointer[-1]; + iter = stack_pointer[-2]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); @@ -4175,7 +4192,7 @@ case _ITER_CHECK_LIST: { _PyStackRef iter; - iter = stack_pointer[-1]; + iter = stack_pointer[-2]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); if (Py_TYPE(iter_o) != &PyListIter_Type) { UOP_STAT_INC(uopcode, miss); @@ -4200,7 +4217,7 @@ case _GUARD_NOT_EXHAUSTED_LIST: { _PyStackRef iter; - iter = stack_pointer[-1]; + iter = stack_pointer[-2]; #ifndef Py_GIL_DISABLED PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); _PyListIterObject *it = (_PyListIterObject *)iter_o; @@ -4226,7 +4243,7 @@ case _ITER_NEXT_LIST_TIER_TWO: { _PyStackRef iter; _PyStackRef next; - iter = stack_pointer[-1]; + iter = stack_pointer[-2]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); _PyListIterObject *it = (_PyListIterObject *)iter_o; assert(Py_TYPE(iter_o) == &PyListIter_Type); @@ -4264,7 +4281,7 @@ case _ITER_CHECK_TUPLE: { _PyStackRef iter; - iter = stack_pointer[-1]; + iter = stack_pointer[-2]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); if (Py_TYPE(iter_o) != &PyTupleIter_Type) { UOP_STAT_INC(uopcode, miss); @@ -4283,7 +4300,7 @@ case _GUARD_NOT_EXHAUSTED_TUPLE: { _PyStackRef iter; - iter = stack_pointer[-1]; + iter = stack_pointer[-2]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; assert(Py_TYPE(iter_o) == &PyTupleIter_Type); @@ -4305,7 +4322,7 @@ case _ITER_NEXT_TUPLE: { _PyStackRef iter; _PyStackRef next; - iter = stack_pointer[-1]; + iter = stack_pointer[-2]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; assert(Py_TYPE(iter_o) == &PyTupleIter_Type); @@ -4324,7 +4341,7 @@ case _ITER_CHECK_RANGE: { _PyStackRef iter; - iter = stack_pointer[-1]; + iter = stack_pointer[-2]; _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); if (Py_TYPE(r) != &PyRangeIter_Type) { UOP_STAT_INC(uopcode, miss); @@ -4343,7 +4360,7 @@ case _GUARD_NOT_EXHAUSTED_RANGE: { _PyStackRef iter; - iter = stack_pointer[-1]; + iter = stack_pointer[-2]; _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); if (r->len <= 0) { @@ -4356,7 +4373,7 @@ case _ITER_NEXT_RANGE: { _PyStackRef iter; _PyStackRef next; - iter = stack_pointer[-1]; + iter = stack_pointer[-2]; _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); #ifdef Py_GIL_DISABLED @@ -4381,7 +4398,7 @@ _PyStackRef iter; _PyInterpreterFrame *gen_frame; oparg = CURRENT_OPARG(); - iter = stack_pointer[-1]; + iter = stack_pointer[-2]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); if (Py_TYPE(gen) != &PyGen_Type) { UOP_STAT_INC(uopcode, miss); diff --git a/Python/flowgraph.c b/Python/flowgraph.c index e0bb5615db3e0b..04b2ae839def9f 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -273,7 +273,7 @@ basicblock_insert_instruction(basicblock *block, int pos, cfg_instr *instr) { } /* For debugging purposes only */ -#if 0 +#if 1 static void dump_instr(cfg_instr *i) { @@ -299,29 +299,43 @@ basicblock_returns(const basicblock *b) { } static void -dump_basicblock(const basicblock *b) +dump_basicblock(const basicblock *b, bool highlight) { const char *b_return = basicblock_returns(b) ? "return " : ""; + if (highlight) { + fprintf(stderr, ">>> "); + } fprintf(stderr, "%d: [EH=%d CLD=%d WRM=%d NO_FT=%d %p] used: %d, depth: %d, preds: %d %s\n", b->b_label.id, b->b_except_handler, b->b_cold, b->b_warm, BB_NO_FALLTHROUGH(b), b, b->b_iused, b->b_startdepth, b->b_predecessors, b_return); + int depth = b->b_startdepth; if (b->b_instr) { int i; for (i = 0; i < b->b_iused; i++) { - fprintf(stderr, " [%02d] ", i); + fprintf(stderr, " [%02d] depth: %d ", i, depth); dump_instr(b->b_instr + i); + + int popped = _PyOpcode_num_popped(b->b_instr[i].i_opcode, b->b_instr[i].i_oparg); + int pushed = _PyOpcode_num_pushed(b->b_instr[i].i_opcode, b->b_instr[i].i_oparg); + depth += (pushed - popped); } } } void -_PyCfgBuilder_DumpGraph(const basicblock *entryblock) +_PyCfgBuilder_DumpGraph(const basicblock *entryblock, const basicblock *mark) { for (const basicblock *b = entryblock; b != NULL; b = b->b_next) { - dump_basicblock(b); + dump_basicblock(b, b == mark); } } +#define DUMP(B, M) _PyCfgBuilder_DumpGraph((B), (M)) + +#else + +#define DUMP(B, M) ((void)0) + #endif @@ -821,6 +835,7 @@ calculate_stackdepth(cfg_builder *g) int maxdepth = 0; basicblock **sp = stack; if (stackdepth_push(&sp, entryblock, 0) < 0) { + DUMP(entryblock, entryblock); goto error; } while (sp != stack) { @@ -841,6 +856,7 @@ calculate_stackdepth(cfg_builder *g) if (new_depth < 0) { PyErr_Format(PyExc_ValueError, "Invalid CFG, stack underflow"); + DUMP(entryblock, b); goto error; } maxdepth = Py_MAX(maxdepth, depth); @@ -855,6 +871,7 @@ calculate_stackdepth(cfg_builder *g) assert(target_depth >= 0); /* invalid code or bug in stackdepth() */ maxdepth = Py_MAX(maxdepth, depth); if (stackdepth_push(&sp, instr->i_target, target_depth) < 0) { + DUMP(entryblock, instr->i_target); goto error; } } @@ -871,6 +888,7 @@ calculate_stackdepth(cfg_builder *g) if (next != NULL) { assert(BB_HAS_FALLTHROUGH(b)); if (stackdepth_push(&sp, next, depth) < 0) { + DUMP(entryblock, next); goto error; } } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 7d3e6c7cbc9eab..1a26aa91482323 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5512,7 +5512,7 @@ _PyStackRef next; // _SPECIALIZE_FOR_ITER { - iter = stack_pointer[-1]; + iter = stack_pointer[-2]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION_FT @@ -5584,7 +5584,7 @@ } // _FOR_ITER_GEN_FRAME { - iter = stack_pointer[-1]; + iter = stack_pointer[-2]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); if (Py_TYPE(gen) != &PyGen_Type) { UPDATE_MISS_STATS(FOR_ITER); @@ -5645,7 +5645,7 @@ /* Skip 1 cache entry */ // _ITER_CHECK_LIST { - iter = stack_pointer[-1]; + iter = stack_pointer[-2]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); if (Py_TYPE(iter_o) != &PyListIter_Type) { UPDATE_MISS_STATS(FOR_ITER); @@ -5744,7 +5744,7 @@ /* Skip 1 cache entry */ // _ITER_CHECK_RANGE { - iter = stack_pointer[-1]; + iter = stack_pointer[-2]; _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); if (Py_TYPE(r) != &PyRangeIter_Type) { UPDATE_MISS_STATS(FOR_ITER); @@ -5811,7 +5811,7 @@ /* Skip 1 cache entry */ // _ITER_CHECK_TUPLE { - iter = stack_pointer[-1]; + iter = stack_pointer[-2]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); if (Py_TYPE(iter_o) != &PyTupleIter_Type) { UPDATE_MISS_STATS(FOR_ITER); @@ -5994,6 +5994,7 @@ INSTRUCTION_STATS(GET_ITER); _PyStackRef iterable; _PyStackRef iter; + _PyStackRef null; iterable = stack_pointer[-1]; #ifdef Py_STATS _PyFrame_SetStackPointer(frame, stack_pointer); @@ -6012,8 +6013,10 @@ JUMP_TO_LABEL(error); } iter = PyStackRef_FromPyObjectSteal(iter_o); + null = PyStackRef_NULL; stack_pointer[0] = iter; - stack_pointer += 1; + stack_pointer[1] = null; + stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -6777,7 +6780,7 @@ _PyStackRef receiver; _PyStackRef value; value = stack_pointer[-1]; - receiver = stack_pointer[-2]; + receiver = stack_pointer[-3]; if (PyStackRef_GenCheck(receiver)) { _PyFrame_SetStackPointer(frame, stack_pointer); int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); @@ -6841,7 +6844,7 @@ _PyStackRef iter; _PyStackRef next; /* Skip 1 cache entry */ - iter = stack_pointer[-1]; + iter = stack_pointer[-2]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); @@ -7134,9 +7137,12 @@ next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_POP_ITER); _PyStackRef iter; - iter = stack_pointer[-1]; + _PyStackRef index_or_null; + index_or_null = stack_pointer[-1]; + iter = stack_pointer[-2]; + (void)index_or_null; INSTRUMENTED_JUMP(prev_instr, this_instr+1, PY_MONITORING_EVENT_BRANCH_RIGHT); - stack_pointer += -1; + stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iter); @@ -9942,12 +9948,15 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(POP_ITER); - _PyStackRef value; - value = stack_pointer[-1]; - stack_pointer += -1; + _PyStackRef iter; + _PyStackRef index_or_null; + index_or_null = stack_pointer[-1]; + iter = stack_pointer[-2]; + (void)index_or_null; + stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); + PyStackRef_CLOSE(iter); stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } @@ -10095,7 +10104,7 @@ stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); + PyStackRef_XCLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 679240b6efa315..2f54a240950b5f 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -131,6 +131,12 @@ break; } + case _POP_ITER: { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + case _END_SEND: { JitOptSymbol *val; val = sym_new_not_null(ctx); @@ -1484,8 +1490,13 @@ case _GET_ITER: { JitOptSymbol *iter; + JitOptSymbol *null; iter = sym_new_not_null(ctx); + null = sym_new_null(ctx); stack_pointer[-1] = iter; + stack_pointer[0] = null; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } From 38a429f679839cb2e548a198c708064e9793ba82 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 11 Apr 2025 18:07:29 +0100 Subject: [PATCH 02/18] use tagged ints for indexes. Work in progress --- Include/internal/pycore_ceval.h | 2 + Include/internal/pycore_code.h | 2 +- Include/internal/pycore_opcode_metadata.h | 2 +- Include/internal/pycore_stackref.h | 18 +- Include/internal/pycore_uop_ids.h | 7 +- Include/internal/pycore_uop_metadata.h | 2 +- Python/bytecodes.c | 253 ++++++++++---------- Python/ceval.c | 18 ++ Python/codegen.c | 2 +- Python/executor_cases.c.h | 152 ++++++------ Python/generated_cases.c.h | 257 +++++++++++---------- Python/optimizer_cases.c.h | 6 +- Python/specialize.c | 57 +++-- Python/stackrefs.c | 8 + Tools/cases_generator/analyzer.py | 5 + Tools/cases_generator/generators_common.py | 1 + 16 files changed, 420 insertions(+), 372 deletions(-) diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 115dd40b8bdfce..63784da16cb8c7 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -377,6 +377,8 @@ PyAPI_FUNC(_PyStackRef) _PyFloat_FromDouble_ConsumeInputs(_PyStackRef left, _PyS extern int _PyRunRemoteDebugger(PyThreadState *tstate); #endif +_PyStackRef _PyForIter_NextWithIndex(PyObject *seq, _PyStackRef index); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 635d2b24f4bdff..c11d0e104a9094 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -313,7 +313,7 @@ extern void _Py_Specialize_CompareOp(_PyStackRef lhs, _PyStackRef rhs, _Py_CODEUNIT *instr, int oparg); extern void _Py_Specialize_UnpackSequence(_PyStackRef seq, _Py_CODEUNIT *instr, int oparg); -extern void _Py_Specialize_ForIter(_PyStackRef iter, _Py_CODEUNIT *instr, int oparg); +extern void _Py_Specialize_ForIter(_PyStackRef iter, _PyStackRef null_or_index, _Py_CODEUNIT *instr, int oparg); extern void _Py_Specialize_Send(_PyStackRef receiver, _Py_CODEUNIT *instr); extern void _Py_Specialize_ToBool(_PyStackRef value, _Py_CODEUNIT *instr); extern void _Py_Specialize_ContainsOp(_PyStackRef value, _Py_CODEUNIT *instr); diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 44bd6de9da6740..34060b7133f24c 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1150,7 +1150,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG }, [GET_AITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [GET_ANEXT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [GET_AWAITABLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index a2acf311ff4c65..dbd7a061ad3771 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -233,6 +233,8 @@ extern intptr_t PyStackRef_UntagInt(_PyStackRef ref); extern _PyStackRef PyStackRef_TagInt(intptr_t i); +extern _PyStackRef PyStackRef_IncrementTaggedInt(_PyStackRef ref); + extern bool PyStackRef_IsNullOrInt(_PyStackRef ref); @@ -262,6 +264,14 @@ PyStackRef_UntagInt(_PyStackRef i) } +static inline _PyStackRef +PyStackRef_IncrementTaggedInt(_PyStackRef ref) +{ + assert(ref.bits != (uintptr_t)-1); // Overflow + return (_PyStackRef){ .bits = ref.bits + 4 }; +} + + #ifdef Py_GIL_DISABLED #define Py_TAG_DEFERRED (1) @@ -686,7 +696,13 @@ PyStackRef_XCLOSE(_PyStackRef ref) #endif // !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG) -#define PyStackRef_TYPE(stackref) Py_TYPE(PyStackRef_AsPyObjectBorrow(stackref)) +static inline PyTypeObject * +PyStackRef_TYPE(_PyStackRef stackref) { + if (PyStackRef_IsTaggedInt(stackref)) { + return &PyLong_Type; + } + return Py_TYPE(PyStackRef_AsPyObjectBorrow(stackref)); +} // Converts a PyStackRef back to a PyObject *, converting the // stackref to a new reference. diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 3155b299f3db4b..754f43cb5aaca1 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -256,14 +256,9 @@ extern "C" { #define _MONITOR_RESUME 471 #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT -<<<<<<< HEAD +#define _POP_ITER POP_ITER #define _POP_JUMP_IF_FALSE 472 #define _POP_JUMP_IF_TRUE 473 -======= -#define _POP_ITER POP_ITER -#define _POP_JUMP_IF_FALSE 465 -#define _POP_JUMP_IF_TRUE 466 ->>>>>>> d908cf7e4c1 (FOR_ITER now pushes NULL as well as the iterator. Preparation for virtual iterators) #define _POP_TOP POP_TOP #define _POP_TOP_LOAD_CONST_INLINE 474 #define _POP_TOP_LOAD_CONST_INLINE_BORROW 475 diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index c339888b892f5b..798f7c6bce9a75 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -202,7 +202,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_FOR_ITER_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_ITER_CHECK_LIST] = HAS_EXIT_FLAG, [_GUARD_NOT_EXHAUSTED_LIST] = HAS_EXIT_FLAG, - [_ITER_NEXT_LIST_TIER_TWO] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, + [_ITER_NEXT_LIST_TIER_TWO] = HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_ITER_CHECK_TUPLE] = HAS_EXIT_FLAG, [_GUARD_NOT_EXHAUSTED_TUPLE] = HAS_EXIT_FLAG, [_ITER_NEXT_TUPLE] = 0, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index b6970445384bc4..deefe5c9b7b46e 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3026,16 +3026,24 @@ dummy_func( values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); } - inst(GET_ITER, (iterable -- iter, null)) { + inst(GET_ITER, (iterable -- iter, index_or_null)) { #ifdef Py_STATS _Py_GatherStats_GetIter(iterable); #endif /* before: [obj]; after [getiter(obj)] */ - PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); - PyStackRef_CLOSE(iterable); - ERROR_IF(iter_o == NULL, error); - iter = PyStackRef_FromPyObjectSteal(iter_o); - null = PyStackRef_NULL; + PyTypeObject *tp = PyStackRef_TYPE(iterable); + if (tp == &PyTuple_Type || tp == &PyList_Type) { + iter = iterable; + DEAD(iterable); + index_or_null = PyStackRef_TagInt(0); + } + else { + PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); + PyStackRef_CLOSE(iterable); + ERROR_IF(iter_o == NULL, error); + iter = PyStackRef_FromPyObjectSteal(iter_o); + index_or_null = PyStackRef_NULL; + } } inst(GET_YIELD_FROM_ITER, (iterable -- iter)) { @@ -3086,7 +3094,7 @@ dummy_func( #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - _Py_Specialize_ForIter(iter, next_instr, oparg); + _Py_Specialize_ForIter(iter, null_or_index, next_instr, oparg); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(FOR_ITER); @@ -3097,30 +3105,41 @@ dummy_func( replaced op(_FOR_ITER, (iter, null_or_index -- iter, null_or_index, next)) { /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); - if (next_o == NULL) { - if (_PyErr_Occurred(tstate)) { - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - if (!matches) { - ERROR_NO_POP(); + if (PyStackRef_IsTaggedInt(null_or_index)) { + next = _PyForIter_NextWithIndex(iter_o, null_or_index); + if (PyStackRef_IsNull(next)) { + null_or_index = PyStackRef_TagInt(-1); + JUMPBY(oparg + 1); + DISPATCH(); + } + null_or_index = PyStackRef_IncrementTaggedInt(null_or_index); + } + else { + PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); + if (next_o == NULL) { + if (_PyErr_Occurred(tstate)) { + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + if (!matches) { + ERROR_NO_POP(); + } + _PyEval_MonitorRaise(tstate, frame, this_instr); + _PyErr_Clear(tstate); } - _PyEval_MonitorRaise(tstate, frame, this_instr); - _PyErr_Clear(tstate); + /* iterator ended normally */ + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + /* Jump forward oparg, then skip following END_FOR */ + JUMPBY(oparg + 1); + DISPATCH(); } - /* iterator ended normally */ - assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - /* Jump forward oparg, then skip following END_FOR */ - JUMPBY(oparg + 1); - DISPATCH(); + next = PyStackRef_FromPyObjectSteal(next_o); } - next = PyStackRef_FromPyObjectSteal(next_o); - // Common case: no jump, leave it to the code generator } op(_FOR_ITER_TIER_TWO, (iter, null_or_index -- iter, null_or_index, next)) { /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + EXIT_IF(!PyStackRef_IsNull(null_or_index)); PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); if (next_o == NULL) { if (_PyErr_Occurred(tstate)) { @@ -3144,61 +3163,62 @@ dummy_func( inst(INSTRUMENTED_FOR_ITER, (unused/1, iter, null_or_index -- iter, null_or_index, next)) { PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); - if (next_o != NULL) { - next = PyStackRef_FromPyObjectSteal(next_o); + if (PyStackRef_IsTaggedInt(null_or_index)) { + next = _PyForIter_NextWithIndex(iter_o, null_or_index); + if (PyStackRef_IsNull(next)) { + JUMPBY(oparg + 1); + DISPATCH(); + } INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); } else { - if (_PyErr_Occurred(tstate)) { - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - if (!matches) { - ERROR_NO_POP(); + PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); + if (next_o != NULL) { + next = PyStackRef_FromPyObjectSteal(next_o); + INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); + } + else { + if (_PyErr_Occurred(tstate)) { + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + if (!matches) { + ERROR_NO_POP(); + } + _PyEval_MonitorRaise(tstate, frame, this_instr); + _PyErr_Clear(tstate); } - _PyEval_MonitorRaise(tstate, frame, this_instr); - _PyErr_Clear(tstate); + /* iterator ended normally */ + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + /* Skip END_FOR */ + JUMPBY(oparg + 1); + DISPATCH(); } - /* iterator ended normally */ - assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - /* Skip END_FOR */ - JUMPBY(oparg + 1); - DISPATCH(); } } op(_ITER_CHECK_LIST, (iter, null_or_index -- iter, null_or_index)) { PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - EXIT_IF(Py_TYPE(iter_o) != &PyListIter_Type); + EXIT_IF(Py_TYPE(iter_o) != &PyList_Type); + assert(PyStackRef_IsTaggedInt(null_or_index)); #ifdef Py_GIL_DISABLED - EXIT_IF(!_PyObject_IsUniquelyReferenced(iter_o)); - _PyListIterObject *it = (_PyListIterObject *)iter_o; - EXIT_IF(!_Py_IsOwnedByCurrentThread((PyObject *)it->it_seq) || - !_PyObject_GC_IS_SHARED(it->it_seq)); + EXIT_IF(!_Py_IsOwnedByCurrentThread(iter_o) || + !_PyObject_GC_IS_SHARED(iter_o)); #endif } replaced op(_ITER_JUMP_LIST, (iter, null_or_index -- iter, null_or_index)) { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(iter_o) == &PyListIter_Type); -// For free-threaded Python, the loop exit can happen at any point during -// item retrieval, so it doesn't make much sense to check and jump -// separately before item retrieval. Any length check we do here can be -// invalid by the time we actually try to fetch the item. #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced(iter_o)); - (void)iter_o; + // For free-threaded Python, the loop exit can happen at any point during + // item retrieval, so it doesn't make much sense to check and jump + // separately before item retrieval. Any length check we do here can be + // invalid by the time we actually try to fetch the item. #else - _PyListIterObject *it = (_PyListIterObject *)iter_o; + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(list_o) == &PyList_Type); STAT_INC(FOR_ITER, hit); - PyListObject *seq = it->it_seq; - if (seq == NULL || (size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) { - it->it_index = -1; - if (seq != NULL) { - it->it_seq = NULL; - Py_DECREF(seq); - } + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) { + null_or_index = PyStackRef_TagInt(-1); /* Jump forward oparg, then skip following END_FOR instruction */ JUMPBY(oparg + 1); DISPATCH(); @@ -3209,71 +3229,61 @@ dummy_func( // Only used by Tier 2 op(_GUARD_NOT_EXHAUSTED_LIST, (iter, null_or_index -- iter, null_or_index)) { #ifndef Py_GIL_DISABLED - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyListIterObject *it = (_PyListIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyListIter_Type); - PyListObject *seq = it->it_seq; - EXIT_IF(seq == NULL); - if ((size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) { - it->it_index = -1; - EXIT_IF(1); - } + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(list_o) == &PyList_Type); + EXIT_IF((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)); #endif } replaced op(_ITER_NEXT_LIST, (iter, null_or_index -- iter, null_or_index, next)) { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyListIterObject *it = (_PyListIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyListIter_Type); - PyListObject *seq = it->it_seq; - assert(seq); + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(PyList_CheckExact(list_o)); #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced(iter_o)); - assert(_Py_IsOwnedByCurrentThread((PyObject *)seq) || - _PyObject_GC_IS_SHARED(seq)); + assert(_Py_IsOwnedByCurrentThread(list_o) || + _PyObject_GC_IS_SHARED(list_o)); STAT_INC(FOR_ITER, hit); - int result = _PyList_GetItemRefNoLock(seq, it->it_index, &next); + int result = _PyList_GetItemRefNoLock((PyListObject *)list_o, PyStackRef_UntagInt(null_or_index), &next); // A negative result means we lost a race with another thread // and we need to take the slow path. DEOPT_IF(result < 0); if (result == 0) { - it->it_index = -1; + null_or_index = PyStackRef_TagInt(-1); /* Jump forward oparg, then skip following END_FOR instruction */ JUMPBY(oparg + 1); DISPATCH(); } - it->it_index++; #else - assert(it->it_index < PyList_GET_SIZE(seq)); - next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++)); + intptr_t i = PyStackRef_UntagInt(null_or_index); + assert(i < PyList_GET_SIZE(list_o)); + next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, i)); #endif + null_or_index = PyStackRef_IncrementTaggedInt(null_or_index); + assert(PyStackRef_UntagInt(null_or_index) == i + 1); } // Only used by Tier 2 op(_ITER_NEXT_LIST_TIER_TWO, (iter, null_or_index -- iter, null_or_index, next)) { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyListIterObject *it = (_PyListIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyListIter_Type); - PyListObject *seq = it->it_seq; - assert(seq); + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(PyList_CheckExact(list_o)); #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced(iter_o)); - assert(_Py_IsOwnedByCurrentThread((PyObject *)seq) || - _PyObject_GC_IS_SHARED(seq)); + assert(_Py_IsOwnedByCurrentThread((PyObject *)list_o) || + _PyObject_GC_IS_SHARED(list_o)); STAT_INC(FOR_ITER, hit); - int result = _PyList_GetItemRefNoLock(seq, it->it_index, &next); + int result = _PyList_GetItemRefNoLock((PyListObject *)list_o, PyStackRef_UntagInt(null_or_index), &next); // A negative result means we lost a race with another thread // and we need to take the slow path. - EXIT_IF(result < 0); + DEOPT_IF(result < 0); if (result == 0) { - it->it_index = -1; - EXIT_IF(1); + null_or_index = PyStackRef_TagInt(-1); + /* Jump forward oparg, then skip following END_FOR instruction */ + JUMPBY(oparg + 1); + DISPATCH(); } - it->it_index++; #else - assert(it->it_index < PyList_GET_SIZE(seq)); - next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++)); + assert(PyStackRef_UntagInt(null_or_index) < PyList_GET_SIZE(list_o)); + next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index))); #endif + null_or_index = PyStackRef_IncrementTaggedInt(null_or_index); } macro(FOR_ITER_LIST) = @@ -3284,29 +3294,20 @@ dummy_func( op(_ITER_CHECK_TUPLE, (iter, null_or_index -- iter, null_or_index)) { PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - EXIT_IF(Py_TYPE(iter_o) != &PyTupleIter_Type); + EXIT_IF(Py_TYPE(iter_o) != &PyTuple_Type); + assert(PyStackRef_IsTaggedInt(null_or_index)); #ifdef Py_GIL_DISABLED EXIT_IF(!_PyObject_IsUniquelyReferenced(iter_o)); #endif } replaced op(_ITER_JUMP_TUPLE, (iter, null_or_index -- iter, null_or_index)) { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - (void)iter_o; - assert(Py_TYPE(iter_o) == &PyTupleIter_Type); -#ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced(iter_o)); -#endif - _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + (void)tuple_o; + assert(Py_TYPE(tuple_o) == &PyTuple_Type); STAT_INC(FOR_ITER, hit); - PyTupleObject *seq = it->it_seq; - if (seq == NULL || (size_t)it->it_index >= (size_t)PyTuple_GET_SIZE(seq)) { -#ifndef Py_GIL_DISABLED - if (seq != NULL) { - it->it_seq = NULL; - Py_DECREF(seq); - } -#endif + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { + null_or_index = PyStackRef_TagInt(-1); /* Jump forward oparg, then skip following END_FOR instruction */ JUMPBY(oparg + 1); DISPATCH(); @@ -3315,28 +3316,18 @@ dummy_func( // Only used by Tier 2 op(_GUARD_NOT_EXHAUSTED_TUPLE, (iter, null_or_index -- iter, null_or_index)) { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyTupleIter_Type); -#ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced(iter_o)); -#endif - PyTupleObject *seq = it->it_seq; - EXIT_IF(seq == NULL); - EXIT_IF(it->it_index >= PyTuple_GET_SIZE(seq)); + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(tuple_o) == &PyTuple_Type); + EXIT_IF((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)); } op(_ITER_NEXT_TUPLE, (iter, null_or_index -- iter, null_or_index, next)) { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyTupleIter_Type); - PyTupleObject *seq = it->it_seq; -#ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced(iter_o)); -#endif - assert(seq); - assert(it->it_index < PyTuple_GET_SIZE(seq)); - next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++)); + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(tuple_o) == &PyTuple_Type); + uintptr_t i = PyStackRef_UntagInt(null_or_index); + assert((size_t)i < (size_t)PyTuple_GET_SIZE(tuple_o)); + next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(tuple_o, i)); + null_or_index = PyStackRef_IncrementTaggedInt(null_or_index); } macro(FOR_ITER_TUPLE) = diff --git a/Python/ceval.c b/Python/ceval.c index 19a1c9529dd9aa..9e2204378cf1f8 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -155,6 +155,9 @@ dump_item(_PyStackRef item) printf(""); return; } + if (PyList_CheckExact(obj)) { + printf("len=%ld ", Py_SIZE(obj)); + } // Don't call __repr__(), it might recurse into the interpreter. printf("<%s at %p>", Py_TYPE(obj)->tp_name, (void *)obj); } @@ -271,6 +274,7 @@ maybe_lltrace_resume_frame(_PyInterpreterFrame *frame, PyObject *globals) lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that } } + // lltrace = 5; if (lltrace >= 5) { lltrace_resume_frame(frame); } @@ -3421,6 +3425,20 @@ _PyEval_LoadName(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject *na return value; } +_PyStackRef +_PyForIter_NextWithIndex(PyObject *seq, _PyStackRef index) +{ + assert(PyStackRef_IsTaggedInt(index)); + assert(Py_TYPE(seq) == &PyTuple_Type || Py_TYPE(seq) == &PyList_Type); + size_t size = Py_SIZE(seq); + intptr_t i = PyStackRef_UntagInt(index); + if ((size_t)i >= size) { + return PyStackRef_NULL; + } + PyObject *next_o = PySequence_Fast_GET_ITEM(seq, i); + return PyStackRef_FromPyObjectNew(next_o); +} + /* Check if a 'cls' provides the given special method. */ static inline int type_has_special_method(PyTypeObject *cls, PyObject *name) diff --git a/Python/codegen.c b/Python/codegen.c index a5eeaae0cad6bd..9e662552ccca14 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -4477,9 +4477,9 @@ codegen_async_comprehension_generator(compiler *c, location loc, else { /* Sub-iter - calculate on the fly */ VISIT(c, expr, gen->iter); - ADDOP(c, LOC(gen->iter), GET_AITER); } } + ADDOP(c, LOC(gen->iter), GET_AITER); USE_LABEL(c, start); /* Runtime will push a block here, so we need to account for that */ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index c97910f07eb847..aa179bdd60f548 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4088,29 +4088,38 @@ case _GET_ITER: { _PyStackRef iterable; _PyStackRef iter; - _PyStackRef null; + _PyStackRef index_or_null; iterable = stack_pointer[-1]; #ifdef Py_STATS _PyFrame_SetStackPointer(frame, stack_pointer); _Py_GatherStats_GetIter(iterable); stack_pointer = _PyFrame_GetStackPointer(frame); #endif - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (iter_o == NULL) { - JUMP_TO_ERROR(); + + PyTypeObject *tp = PyStackRef_TYPE(iterable); + if (tp == &PyTuple_Type || tp == &PyList_Type) { + iter = iterable; + index_or_null = PyStackRef_TagInt(0); } - iter = PyStackRef_FromPyObjectSteal(iter_o); - null = PyStackRef_NULL; - stack_pointer[0] = iter; - stack_pointer[1] = null; - stack_pointer += 2; + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (iter_o == NULL) { + JUMP_TO_ERROR(); + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + index_or_null = PyStackRef_NULL; + stack_pointer += 1; + } + stack_pointer[-1] = iter; + stack_pointer[0] = index_or_null; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -4156,10 +4165,16 @@ /* _FOR_ITER is not a viable micro-op for tier 2 because it is replaced */ case _FOR_ITER_TIER_TWO: { + _PyStackRef null_or_index; _PyStackRef iter; _PyStackRef next; + null_or_index = stack_pointer[-1]; iter = stack_pointer[-2]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (!PyStackRef_IsNull(null_or_index)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -4191,21 +4206,19 @@ /* _INSTRUMENTED_FOR_ITER is not a viable micro-op for tier 2 because it is instrumented */ case _ITER_CHECK_LIST: { + _PyStackRef null_or_index; _PyStackRef iter; + null_or_index = stack_pointer[-1]; iter = stack_pointer[-2]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(iter_o) != &PyListIter_Type) { + if (Py_TYPE(iter_o) != &PyList_Type) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } + assert(PyStackRef_IsTaggedInt(null_or_index)); #ifdef Py_GIL_DISABLED - if (!_PyObject_IsUniquelyReferenced(iter_o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - _PyListIterObject *it = (_PyListIterObject *)iter_o; - if (!_Py_IsOwnedByCurrentThread((PyObject *)it->it_seq) || - !_PyObject_GC_IS_SHARED(it->it_seq)) { + if (!_Py_IsOwnedByCurrentThread(iter_o) || + !_PyObject_GC_IS_SHARED(iter_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -4216,24 +4229,17 @@ /* _ITER_JUMP_LIST is not a viable micro-op for tier 2 because it is replaced */ case _GUARD_NOT_EXHAUSTED_LIST: { + _PyStackRef null_or_index; _PyStackRef iter; + null_or_index = stack_pointer[-1]; iter = stack_pointer[-2]; #ifndef Py_GIL_DISABLED - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyListIterObject *it = (_PyListIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyListIter_Type); - PyListObject *seq = it->it_seq; - if (seq == NULL) { + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(list_o) == &PyList_Type); + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - if ((size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) { - it->it_index = -1; - if (1) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - } #endif break; } @@ -4241,38 +4247,37 @@ /* _ITER_NEXT_LIST is not a viable micro-op for tier 2 because it is replaced */ case _ITER_NEXT_LIST_TIER_TWO: { + _PyStackRef null_or_index; _PyStackRef iter; _PyStackRef next; + oparg = CURRENT_OPARG(); + null_or_index = stack_pointer[-1]; iter = stack_pointer[-2]; - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyListIterObject *it = (_PyListIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyListIter_Type); - PyListObject *seq = it->it_seq; - assert(seq); + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(PyList_CheckExact(list_o)); #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced(iter_o)); - assert(_Py_IsOwnedByCurrentThread((PyObject *)seq) || - _PyObject_GC_IS_SHARED(seq)); + assert(_Py_IsOwnedByCurrentThread((PyObject *)list_o) || + _PyObject_GC_IS_SHARED(list_o)); STAT_INC(FOR_ITER, hit); _PyFrame_SetStackPointer(frame, stack_pointer); - int result = _PyList_GetItemRefNoLock(seq, it->it_index, &next); + int result = _PyList_GetItemRefNoLock((PyListObject *)list_o, PyStackRef_UntagInt(null_or_index), &next); stack_pointer = _PyFrame_GetStackPointer(frame); if (result < 0) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } if (result == 0) { - it->it_index = -1; - if (1) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + null_or_index = PyStackRef_TagInt(-1); + JUMPBY(oparg + 1); + stack_pointer[-1] = null_or_index; + DISPATCH(); } - it->it_index++; #else - assert(it->it_index < PyList_GET_SIZE(seq)); - next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++)); + assert(PyStackRef_UntagInt(null_or_index) < PyList_GET_SIZE(list_o)); + next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index))); #endif + null_or_index = PyStackRef_IncrementTaggedInt(null_or_index); + stack_pointer[-1] = null_or_index; stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -4280,13 +4285,16 @@ } case _ITER_CHECK_TUPLE: { + _PyStackRef null_or_index; _PyStackRef iter; + null_or_index = stack_pointer[-1]; iter = stack_pointer[-2]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(iter_o) != &PyTupleIter_Type) { + if (Py_TYPE(iter_o) != &PyTuple_Type) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } + assert(PyStackRef_IsTaggedInt(null_or_index)); #ifdef Py_GIL_DISABLED if (!_PyObject_IsUniquelyReferenced(iter_o)) { UOP_STAT_INC(uopcode, miss); @@ -4299,20 +4307,13 @@ /* _ITER_JUMP_TUPLE is not a viable micro-op for tier 2 because it is replaced */ case _GUARD_NOT_EXHAUSTED_TUPLE: { + _PyStackRef null_or_index; _PyStackRef iter; + null_or_index = stack_pointer[-1]; iter = stack_pointer[-2]; - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyTupleIter_Type); - #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced(iter_o)); - #endif - PyTupleObject *seq = it->it_seq; - if (seq == NULL) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - if (it->it_index >= PyTuple_GET_SIZE(seq)) { + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(tuple_o) == &PyTuple_Type); + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -4320,19 +4321,18 @@ } case _ITER_NEXT_TUPLE: { + _PyStackRef null_or_index; _PyStackRef iter; _PyStackRef next; + null_or_index = stack_pointer[-1]; iter = stack_pointer[-2]; - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyTupleIter_Type); - PyTupleObject *seq = it->it_seq; - #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced(iter_o)); - #endif - assert(seq); - assert(it->it_index < PyTuple_GET_SIZE(seq)); - next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++)); + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(tuple_o) == &PyTuple_Type); + uintptr_t i = PyStackRef_UntagInt(null_or_index); + assert((size_t)i < (size_t)PyTuple_GET_SIZE(tuple_o)); + next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(tuple_o, i)); + null_or_index = PyStackRef_IncrementTaggedInt(null_or_index); + stack_pointer[-1] = null_or_index; stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 1a26aa91482323..1b7327c80b731e 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5509,9 +5509,11 @@ _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef iter; + _PyStackRef null_or_index; _PyStackRef next; // _SPECIALIZE_FOR_ITER { + null_or_index = stack_pointer[-1]; iter = stack_pointer[-2]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; @@ -5519,7 +5521,7 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_ForIter(iter, next_instr, oparg); + _Py_Specialize_ForIter(iter, null_or_index, next_instr, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } @@ -5530,29 +5532,44 @@ // _FOR_ITER { PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (next_o == NULL) { - if (_PyErr_Occurred(tstate)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!matches) { - JUMP_TO_LABEL(error); + if (PyStackRef_IsTaggedInt(null_or_index)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + next = _PyForIter_NextWithIndex(iter_o, null_or_index); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (PyStackRef_IsNull(next)) { + null_or_index = PyStackRef_TagInt(-1); + JUMPBY(oparg + 1); + stack_pointer[-1] = null_or_index; + DISPATCH(); + } + null_or_index = PyStackRef_IncrementTaggedInt(null_or_index); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (next_o == NULL) { + if (_PyErr_Occurred(tstate)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!matches) { + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_MonitorRaise(tstate, frame, this_instr); + _PyErr_Clear(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_MonitorRaise(tstate, frame, this_instr); - _PyErr_Clear(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + JUMPBY(oparg + 1); + DISPATCH(); } - assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - JUMPBY(oparg + 1); - DISPATCH(); + next = PyStackRef_FromPyObjectSteal(next_o); } - next = PyStackRef_FromPyObjectSteal(next_o); } + stack_pointer[-1] = null_or_index; stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -5641,26 +5658,23 @@ INSTRUCTION_STATS(FOR_ITER_LIST); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef iter; + _PyStackRef null_or_index; _PyStackRef next; /* Skip 1 cache entry */ // _ITER_CHECK_LIST { + null_or_index = stack_pointer[-1]; iter = stack_pointer[-2]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(iter_o) != &PyListIter_Type) { + if (Py_TYPE(iter_o) != &PyList_Type) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); JUMP_TO_PREDICTED(FOR_ITER); } + assert(PyStackRef_IsTaggedInt(null_or_index)); #ifdef Py_GIL_DISABLED - if (!_PyObject_IsUniquelyReferenced(iter_o)) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(FOR_ITER); - } - _PyListIterObject *it = (_PyListIterObject *)iter_o; - if (!_Py_IsOwnedByCurrentThread((PyObject *)it->it_seq) || - !_PyObject_GC_IS_SHARED(it->it_seq)) { + if (!_Py_IsOwnedByCurrentThread(iter_o) || + !_PyObject_GC_IS_SHARED(iter_o)) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); JUMP_TO_PREDICTED(FOR_ITER); @@ -5669,42 +5683,30 @@ } // _ITER_JUMP_LIST { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(iter_o) == &PyListIter_Type); #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced(iter_o)); - (void)iter_o; + #else - _PyListIterObject *it = (_PyListIterObject *)iter_o; + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(list_o) == &PyList_Type); STAT_INC(FOR_ITER, hit); - PyListObject *seq = it->it_seq; - if (seq == NULL || (size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) { - it->it_index = -1; - if (seq != NULL) { - it->it_seq = NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); - } + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) { + null_or_index = PyStackRef_TagInt(-1); JUMPBY(oparg + 1); + stack_pointer[-1] = null_or_index; DISPATCH(); } #endif } // _ITER_NEXT_LIST { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyListIterObject *it = (_PyListIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyListIter_Type); - PyListObject *seq = it->it_seq; - assert(seq); + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(PyList_CheckExact(list_o)); #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced(iter_o)); - assert(_Py_IsOwnedByCurrentThread((PyObject *)seq) || - _PyObject_GC_IS_SHARED(seq)); + assert(_Py_IsOwnedByCurrentThread(list_o) || + _PyObject_GC_IS_SHARED(list_o)); STAT_INC(FOR_ITER, hit); _PyFrame_SetStackPointer(frame, stack_pointer); - int result = _PyList_GetItemRefNoLock(seq, it->it_index, &next); + int result = _PyList_GetItemRefNoLock((PyListObject *)list_o, PyStackRef_UntagInt(null_or_index), &next); stack_pointer = _PyFrame_GetStackPointer(frame); if (result < 0) { UPDATE_MISS_STATS(FOR_ITER); @@ -5712,16 +5714,20 @@ JUMP_TO_PREDICTED(FOR_ITER); } if (result == 0) { - it->it_index = -1; + null_or_index = PyStackRef_TagInt(-1); JUMPBY(oparg + 1); + stack_pointer[-1] = null_or_index; DISPATCH(); } - it->it_index++; #else - assert(it->it_index < PyList_GET_SIZE(seq)); - next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++)); + intptr_t i = PyStackRef_UntagInt(null_or_index); + assert(i < PyList_GET_SIZE(list_o)); + next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, i)); #endif + null_or_index = PyStackRef_IncrementTaggedInt(null_or_index); + assert(PyStackRef_UntagInt(null_or_index) == i + 1); } + stack_pointer[-1] = null_or_index; stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -5807,17 +5813,20 @@ INSTRUCTION_STATS(FOR_ITER_TUPLE); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef iter; + _PyStackRef null_or_index; _PyStackRef next; /* Skip 1 cache entry */ // _ITER_CHECK_TUPLE { + null_or_index = stack_pointer[-1]; iter = stack_pointer[-2]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(iter_o) != &PyTupleIter_Type) { + if (Py_TYPE(iter_o) != &PyTuple_Type) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); JUMP_TO_PREDICTED(FOR_ITER); } + assert(PyStackRef_IsTaggedInt(null_or_index)); #ifdef Py_GIL_DISABLED if (!_PyObject_IsUniquelyReferenced(iter_o)) { UPDATE_MISS_STATS(FOR_ITER); @@ -5828,42 +5837,27 @@ } // _ITER_JUMP_TUPLE { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - (void)iter_o; - assert(Py_TYPE(iter_o) == &PyTupleIter_Type); - #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced(iter_o)); - #endif - _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + (void)tuple_o; + assert(Py_TYPE(tuple_o) == &PyTuple_Type); STAT_INC(FOR_ITER, hit); - PyTupleObject *seq = it->it_seq; - if (seq == NULL || (size_t)it->it_index >= (size_t)PyTuple_GET_SIZE(seq)) { - #ifndef Py_GIL_DISABLED - if (seq != NULL) { - it->it_seq = NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - #endif - + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { + null_or_index = PyStackRef_TagInt(-1); JUMPBY(oparg + 1); + stack_pointer[-1] = null_or_index; DISPATCH(); } } // _ITER_NEXT_TUPLE { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyTupleIter_Type); - PyTupleObject *seq = it->it_seq; - #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced(iter_o)); - #endif - assert(seq); - assert(it->it_index < PyTuple_GET_SIZE(seq)); - next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++)); + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(tuple_o) == &PyTuple_Type); + uintptr_t i = PyStackRef_UntagInt(null_or_index); + assert((size_t)i < (size_t)PyTuple_GET_SIZE(tuple_o)); + next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(tuple_o, i)); + null_or_index = PyStackRef_IncrementTaggedInt(null_or_index); } + stack_pointer[-1] = null_or_index; stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -5994,29 +5988,38 @@ INSTRUCTION_STATS(GET_ITER); _PyStackRef iterable; _PyStackRef iter; - _PyStackRef null; + _PyStackRef index_or_null; iterable = stack_pointer[-1]; #ifdef Py_STATS _PyFrame_SetStackPointer(frame, stack_pointer); _Py_GatherStats_GetIter(iterable); stack_pointer = _PyFrame_GetStackPointer(frame); #endif - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (iter_o == NULL) { - JUMP_TO_LABEL(error); + + PyTypeObject *tp = PyStackRef_TYPE(iterable); + if (tp == &PyTuple_Type || tp == &PyList_Type) { + iter = iterable; + index_or_null = PyStackRef_TagInt(0); } - iter = PyStackRef_FromPyObjectSteal(iter_o); - null = PyStackRef_NULL; - stack_pointer[0] = iter; - stack_pointer[1] = null; - stack_pointer += 2; + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (iter_o == NULL) { + JUMP_TO_LABEL(error); + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + index_or_null = PyStackRef_NULL; + stack_pointer += 1; + } + stack_pointer[-1] = iter; + stack_pointer[0] = index_or_null; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -6842,34 +6845,48 @@ next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); _PyStackRef iter; + _PyStackRef null_or_index; _PyStackRef next; /* Skip 1 cache entry */ + null_or_index = stack_pointer[-1]; iter = stack_pointer[-2]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (next_o != NULL) { - next = PyStackRef_FromPyObjectSteal(next_o); + if (PyStackRef_IsTaggedInt(null_or_index)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + next = _PyForIter_NextWithIndex(iter_o, null_or_index); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (PyStackRef_IsNull(next)) { + JUMPBY(oparg + 1); + DISPATCH(); + } INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); } else { - if (_PyErr_Occurred(tstate)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!matches) { - JUMP_TO_LABEL(error); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (next_o != NULL) { + next = PyStackRef_FromPyObjectSteal(next_o); + INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); + } + else { + if (_PyErr_Occurred(tstate)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!matches) { + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_MonitorRaise(tstate, frame, this_instr); + _PyErr_Clear(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_MonitorRaise(tstate, frame, this_instr); - _PyErr_Clear(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + JUMPBY(oparg + 1); + DISPATCH(); } - assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - JUMPBY(oparg + 1); - DISPATCH(); } stack_pointer[0] = next; stack_pointer += 1; diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 2f54a240950b5f..1bf59c8c0bec80 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1490,11 +1490,11 @@ case _GET_ITER: { JitOptSymbol *iter; - JitOptSymbol *null; + JitOptSymbol *index_or_null; iter = sym_new_not_null(ctx); - null = sym_new_null(ctx); + index_or_null = sym_new_not_null(ctx); stack_pointer[-1] = iter; - stack_pointer[0] = null; + stack_pointer[0] = index_or_null; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; diff --git a/Python/specialize.c b/Python/specialize.c index 562e4a19d64a7f..bcaad21bfddc26 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -2898,7 +2898,7 @@ int #endif // Py_STATS Py_NO_INLINE void -_Py_Specialize_ForIter(_PyStackRef iter, _Py_CODEUNIT *instr, int oparg) +_Py_Specialize_ForIter(_PyStackRef iter, _PyStackRef null_or_index, _Py_CODEUNIT *instr, int oparg) { assert(ENABLE_SPECIALIZATION_FT); assert(_PyOpcode_Caches[FOR_ITER] == INLINE_CACHE_ENTRIES_FOR_ITER); @@ -2913,38 +2913,33 @@ _Py_Specialize_ForIter(_PyStackRef iter, _Py_CODEUNIT *instr, int oparg) if (!_PyObject_IsUniquelyReferenced(iter_o)) goto failure; #endif - if (tp == &PyListIter_Type) { -#ifdef Py_GIL_DISABLED - _PyListIterObject *it = (_PyListIterObject *)iter_o; - if (!_Py_IsOwnedByCurrentThread((PyObject *)it->it_seq) && - !_PyObject_GC_IS_SHARED(it->it_seq)) { - // Maybe this should just set GC_IS_SHARED in a critical - // section, instead of leaving it to the first iteration? - goto failure; + if (PyStackRef_IsNull(null_or_index)) { + if (tp == &PyRangeIter_Type) { + specialize(instr, FOR_ITER_RANGE); + return; + } + else if (tp == &PyGen_Type && oparg <= SHRT_MAX) { + // Generators are very much not thread-safe, so don't worry about + // the specialization not being thread-safe. + assert(instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1].op.code == END_FOR || + instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1].op.code == INSTRUMENTED_END_FOR + ); + /* Don't specialize if PEP 523 is active */ + if (_PyInterpreterState_GET()->eval_frame) + goto failure; + specialize(instr, FOR_ITER_GEN); + return; } -#endif - specialize(instr, FOR_ITER_LIST); - return; - } - else if (tp == &PyTupleIter_Type) { - specialize(instr, FOR_ITER_TUPLE); - return; - } - else if (tp == &PyRangeIter_Type) { - specialize(instr, FOR_ITER_RANGE); - return; } - else if (tp == &PyGen_Type && oparg <= SHRT_MAX) { - // Generators are very much not thread-safe, so don't worry about - // the specialization not being thread-safe. - assert(instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1].op.code == END_FOR || - instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1].op.code == INSTRUMENTED_END_FOR - ); - /* Don't specialize if PEP 523 is active */ - if (_PyInterpreterState_GET()->eval_frame) - goto failure; - specialize(instr, FOR_ITER_GEN); - return; + else { + if (tp == &PyList_Type) { + specialize(instr, FOR_ITER_LIST); + return; + } + else if (tp == &PyTuple_Type) { + specialize(instr, FOR_ITER_TUPLE); + return; + } } failure: SPECIALIZATION_FAIL(FOR_ITER, diff --git a/Python/stackrefs.c b/Python/stackrefs.c index 979a6b1c62820a..c64525b21d35fe 100644 --- a/Python/stackrefs.c +++ b/Python/stackrefs.c @@ -216,4 +216,12 @@ PyStackRef_IsNullOrInt(_PyStackRef ref) return PyStackRef_IsNull(ref) || PyStackRef_IsTaggedInt(ref); } +_PyStackRef +PyStackRef_IncrementTaggedInt(_PyStackRef ref) +{ + assert(ref.index != (uintptr_t)-1); // Overflow + return (_PyStackRef){ .index = ref.index + 2 }; +} + + #endif diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index dddbf2cf872e3d..8463ca4f979126 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -602,6 +602,10 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "PyStackRef_None", "PyStackRef_TYPE", "PyStackRef_True", + "PyStackRef_TagInt", + "PyStackRef_UntagInt", + "PyStackRef_IsTaggedInt", + "PyStackRef_IncrementTaggedInt", "PyTuple_GET_ITEM", "PyTuple_GET_SIZE", "PyType_HasFeature", @@ -683,6 +687,7 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "PyStackRef_UntagInt", ) + def check_escaping_calls(instr: parser.CodeDef, escapes: dict[SimpleStmt, EscapingCall]) -> None: error: lexer.Token | None = None calls = {e.call for e in escapes.values()} diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 9ba0767cba35a0..025dd4dd3b2910 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -140,6 +140,7 @@ def dispatch( ) -> bool: if storage.spilled: raise analysis_error("stack_pointer needs reloading before dispatch", tkn) + storage.stack.flush(self.out) self.emit(tkn) return False From 2caf56f7384ed9909464d6c0c339594fbb611551 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 15 Apr 2025 16:50:57 +0100 Subject: [PATCH 03/18] Remove debug code --- Python/bytecodes.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index deefe5c9b7b46e..72cdfa47838fc0 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3253,12 +3253,9 @@ dummy_func( DISPATCH(); } #else - intptr_t i = PyStackRef_UntagInt(null_or_index); - assert(i < PyList_GET_SIZE(list_o)); - next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, i)); + next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index))); #endif null_or_index = PyStackRef_IncrementTaggedInt(null_or_index); - assert(PyStackRef_UntagInt(null_or_index) == i + 1); } // Only used by Tier 2 From 588943a2cd1e24154a379901de8381b3626c8fc4 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 15 Apr 2025 18:07:06 +0100 Subject: [PATCH 04/18] Regenerate files --- Python/generated_cases.c.h | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 1b7327c80b731e..7ee23f0a7fd5f0 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5720,12 +5720,9 @@ DISPATCH(); } #else - intptr_t i = PyStackRef_UntagInt(null_or_index); - assert(i < PyList_GET_SIZE(list_o)); - next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, i)); + next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index))); #endif null_or_index = PyStackRef_IncrementTaggedInt(null_or_index); - assert(PyStackRef_UntagInt(null_or_index) == i + 1); } stack_pointer[-1] = null_or_index; stack_pointer[0] = next; From aa62e3976b843c8ef4adf987509e86a0337c4eed Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 16 Apr 2025 09:37:36 +0100 Subject: [PATCH 05/18] Tidy up list of non-escaping functions --- Tools/cases_generator/analyzer.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 8463ca4f979126..c97d96c9e7f7b1 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -602,10 +602,6 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "PyStackRef_None", "PyStackRef_TYPE", "PyStackRef_True", - "PyStackRef_TagInt", - "PyStackRef_UntagInt", - "PyStackRef_IsTaggedInt", - "PyStackRef_IncrementTaggedInt", "PyTuple_GET_ITEM", "PyTuple_GET_SIZE", "PyType_HasFeature", @@ -685,6 +681,8 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "PyStackRef_IsTaggedInt", "PyStackRef_TagInt", "PyStackRef_UntagInt", + "PyStackRef_IncrementTaggedInt", + "PyStackRef_IsNullOrInt", ) From 88562ec76b46e50b4202d89f99efdf331529082a Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 16 Apr 2025 09:58:22 +0100 Subject: [PATCH 06/18] Fix up list iteration for FT build --- Python/bytecodes.c | 3 +-- Python/executor_cases.c.h | 3 +-- Python/generated_cases.c.h | 3 +-- Python/specialize.c | 9 +++------ 4 files changed, 6 insertions(+), 12 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 72cdfa47838fc0..05ff8e62a87b1f 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3202,8 +3202,7 @@ dummy_func( EXIT_IF(Py_TYPE(iter_o) != &PyList_Type); assert(PyStackRef_IsTaggedInt(null_or_index)); #ifdef Py_GIL_DISABLED - EXIT_IF(!_Py_IsOwnedByCurrentThread(iter_o) || - !_PyObject_GC_IS_SHARED(iter_o)); + EXIT_IF(!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)); #endif } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index aa179bdd60f548..11775e612e6975 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4217,8 +4217,7 @@ } assert(PyStackRef_IsTaggedInt(null_or_index)); #ifdef Py_GIL_DISABLED - if (!_Py_IsOwnedByCurrentThread(iter_o) || - !_PyObject_GC_IS_SHARED(iter_o)) { + if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 7ee23f0a7fd5f0..998e981662448e 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5673,8 +5673,7 @@ } assert(PyStackRef_IsTaggedInt(null_or_index)); #ifdef Py_GIL_DISABLED - if (!_Py_IsOwnedByCurrentThread(iter_o) || - !_PyObject_GC_IS_SHARED(iter_o)) { + if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); JUMP_TO_PREDICTED(FOR_ITER); diff --git a/Python/specialize.c b/Python/specialize.c index bcaad21bfddc26..3e1fe5a7070fdc 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -2905,13 +2905,10 @@ _Py_Specialize_ForIter(_PyStackRef iter, _PyStackRef null_or_index, _Py_CODEUNIT PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); PyTypeObject *tp = Py_TYPE(iter_o); #ifdef Py_GIL_DISABLED - // Only specialize for uniquely referenced iterators, so that we know - // they're only referenced by this one thread. This is more limiting - // than we need (even `it = iter(mylist); for item in it:` won't get - // specialized) but we don't have a way to check whether we're the only - // _thread_ who has access to the object. - if (!_PyObject_IsUniquelyReferenced(iter_o)) + // Only specialize for lists owned by this thread or shared + if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { goto failure; + } #endif if (PyStackRef_IsNull(null_or_index)) { if (tp == &PyRangeIter_Type) { From 025049dab078f288c1ee75e9ee0775bbf320c03b Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 16 Apr 2025 10:47:00 +0100 Subject: [PATCH 07/18] Fix tier 2 FT build --- Include/internal/pycore_uop_metadata.h | 2 +- Python/bytecodes.c | 8 +------- Python/executor_cases.c.h | 9 +-------- 3 files changed, 3 insertions(+), 16 deletions(-) diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 798f7c6bce9a75..ea47602dea37e7 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -202,7 +202,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_FOR_ITER_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_ITER_CHECK_LIST] = HAS_EXIT_FLAG, [_GUARD_NOT_EXHAUSTED_LIST] = HAS_EXIT_FLAG, - [_ITER_NEXT_LIST_TIER_TWO] = HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_ITER_NEXT_LIST_TIER_TWO] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_ITER_CHECK_TUPLE] = HAS_EXIT_FLAG, [_GUARD_NOT_EXHAUSTED_TUPLE] = HAS_EXIT_FLAG, [_ITER_NEXT_TUPLE] = 0, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 05ff8e62a87b1f..da45b9aa66e1db 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3268,13 +3268,7 @@ dummy_func( int result = _PyList_GetItemRefNoLock((PyListObject *)list_o, PyStackRef_UntagInt(null_or_index), &next); // A negative result means we lost a race with another thread // and we need to take the slow path. - DEOPT_IF(result < 0); - if (result == 0) { - null_or_index = PyStackRef_TagInt(-1); - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); - DISPATCH(); - } + DEOPT_IF(result <= 0); #else assert(PyStackRef_UntagInt(null_or_index) < PyList_GET_SIZE(list_o)); next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index))); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 11775e612e6975..1b1ab6efdb6a72 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4249,7 +4249,6 @@ _PyStackRef null_or_index; _PyStackRef iter; _PyStackRef next; - oparg = CURRENT_OPARG(); null_or_index = stack_pointer[-1]; iter = stack_pointer[-2]; PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); @@ -4261,16 +4260,10 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int result = _PyList_GetItemRefNoLock((PyListObject *)list_o, PyStackRef_UntagInt(null_or_index), &next); stack_pointer = _PyFrame_GetStackPointer(frame); - if (result < 0) { + if (result <= 0) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - if (result == 0) { - null_or_index = PyStackRef_TagInt(-1); - JUMPBY(oparg + 1); - stack_pointer[-1] = null_or_index; - DISPATCH(); - } #else assert(PyStackRef_UntagInt(null_or_index) < PyList_GET_SIZE(list_o)); next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index))); From 35e389ddf5f918da615f0979357c9b94aaa29bbb Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 30 Apr 2025 10:11:19 +0100 Subject: [PATCH 08/18] Remove debugging code --- Python/ceval.c | 4 ---- Python/flowgraph.c | 12 +----------- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index 9e2204378cf1f8..dfe8abbbc70164 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -155,9 +155,6 @@ dump_item(_PyStackRef item) printf(""); return; } - if (PyList_CheckExact(obj)) { - printf("len=%ld ", Py_SIZE(obj)); - } // Don't call __repr__(), it might recurse into the interpreter. printf("<%s at %p>", Py_TYPE(obj)->tp_name, (void *)obj); } @@ -274,7 +271,6 @@ maybe_lltrace_resume_frame(_PyInterpreterFrame *frame, PyObject *globals) lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that } } - // lltrace = 5; if (lltrace >= 5) { lltrace_resume_frame(frame); } diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 04b2ae839def9f..c3ac1513cc130b 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -273,7 +273,7 @@ basicblock_insert_instruction(basicblock *block, int pos, cfg_instr *instr) { } /* For debugging purposes only */ -#if 1 +#if 0 static void dump_instr(cfg_instr *i) { @@ -330,12 +330,6 @@ _PyCfgBuilder_DumpGraph(const basicblock *entryblock, const basicblock *mark) } } -#define DUMP(B, M) _PyCfgBuilder_DumpGraph((B), (M)) - -#else - -#define DUMP(B, M) ((void)0) - #endif @@ -835,7 +829,6 @@ calculate_stackdepth(cfg_builder *g) int maxdepth = 0; basicblock **sp = stack; if (stackdepth_push(&sp, entryblock, 0) < 0) { - DUMP(entryblock, entryblock); goto error; } while (sp != stack) { @@ -856,7 +849,6 @@ calculate_stackdepth(cfg_builder *g) if (new_depth < 0) { PyErr_Format(PyExc_ValueError, "Invalid CFG, stack underflow"); - DUMP(entryblock, b); goto error; } maxdepth = Py_MAX(maxdepth, depth); @@ -871,7 +863,6 @@ calculate_stackdepth(cfg_builder *g) assert(target_depth >= 0); /* invalid code or bug in stackdepth() */ maxdepth = Py_MAX(maxdepth, depth); if (stackdepth_push(&sp, instr->i_target, target_depth) < 0) { - DUMP(entryblock, instr->i_target); goto error; } } @@ -888,7 +879,6 @@ calculate_stackdepth(cfg_builder *g) if (next != NULL) { assert(BB_HAS_FALLTHROUGH(b)); if (stackdepth_push(&sp, next, depth) < 0) { - DUMP(entryblock, next); goto error; } } From ed89950013bd93a8fd363a3b2b346cf40e7f4389 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 30 Apr 2025 10:19:42 +0100 Subject: [PATCH 09/18] Rename function --- Include/internal/pycore_stackref.h | 7 ++++--- Python/bytecodes.c | 8 ++++---- Python/executor_cases.c.h | 4 ++-- Python/generated_cases.c.h | 6 +++--- Python/stackrefs.c | 2 +- Tools/cases_generator/analyzer.py | 2 +- 6 files changed, 15 insertions(+), 14 deletions(-) diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index dbd7a061ad3771..ad3699fef1d58c 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -233,7 +233,8 @@ extern intptr_t PyStackRef_UntagInt(_PyStackRef ref); extern _PyStackRef PyStackRef_TagInt(intptr_t i); -extern _PyStackRef PyStackRef_IncrementTaggedInt(_PyStackRef ref); +/* Increments a tagged int, but does not check for overflow */ +extern _PyStackRef PyStackRef_IncrementTaggedIntNoOverflow(_PyStackRef ref); extern bool PyStackRef_IsNullOrInt(_PyStackRef ref); @@ -265,9 +266,9 @@ PyStackRef_UntagInt(_PyStackRef i) static inline _PyStackRef -PyStackRef_IncrementTaggedInt(_PyStackRef ref) +PyStackRef_IncrementTaggedIntNoOverflow(_PyStackRef ref) { - assert(ref.bits != (uintptr_t)-1); // Overflow + assert(ref.bits != (uintptr_t)-1); // Deosn't overflow return (_PyStackRef){ .bits = ref.bits + 4 }; } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 8a5056dcd35ed9..d42a1c4b703485 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3112,7 +3112,7 @@ dummy_func( JUMPBY(oparg + 1); DISPATCH(); } - null_or_index = PyStackRef_IncrementTaggedInt(null_or_index); + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); } else { PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); @@ -3254,7 +3254,7 @@ dummy_func( #else next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index))); #endif - null_or_index = PyStackRef_IncrementTaggedInt(null_or_index); + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); } // Only used by Tier 2 @@ -3273,7 +3273,7 @@ dummy_func( assert(PyStackRef_UntagInt(null_or_index) < PyList_GET_SIZE(list_o)); next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index))); #endif - null_or_index = PyStackRef_IncrementTaggedInt(null_or_index); + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); } macro(FOR_ITER_LIST) = @@ -3317,7 +3317,7 @@ dummy_func( uintptr_t i = PyStackRef_UntagInt(null_or_index); assert((size_t)i < (size_t)PyTuple_GET_SIZE(tuple_o)); next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(tuple_o, i)); - null_or_index = PyStackRef_IncrementTaggedInt(null_or_index); + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); } macro(FOR_ITER_TUPLE) = diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 1b1ab6efdb6a72..3c9544bf266ec3 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4268,7 +4268,7 @@ assert(PyStackRef_UntagInt(null_or_index) < PyList_GET_SIZE(list_o)); next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index))); #endif - null_or_index = PyStackRef_IncrementTaggedInt(null_or_index); + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); stack_pointer[-1] = null_or_index; stack_pointer[0] = next; stack_pointer += 1; @@ -4323,7 +4323,7 @@ uintptr_t i = PyStackRef_UntagInt(null_or_index); assert((size_t)i < (size_t)PyTuple_GET_SIZE(tuple_o)); next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(tuple_o, i)); - null_or_index = PyStackRef_IncrementTaggedInt(null_or_index); + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); stack_pointer[-1] = null_or_index; stack_pointer[0] = next; stack_pointer += 1; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index e9d8266e2ad11c..eed7c908f75585 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5542,7 +5542,7 @@ stack_pointer[-1] = null_or_index; DISPATCH(); } - null_or_index = PyStackRef_IncrementTaggedInt(null_or_index); + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); } else { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -5721,7 +5721,7 @@ #else next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index))); #endif - null_or_index = PyStackRef_IncrementTaggedInt(null_or_index); + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); } stack_pointer[-1] = null_or_index; stack_pointer[0] = next; @@ -5851,7 +5851,7 @@ uintptr_t i = PyStackRef_UntagInt(null_or_index); assert((size_t)i < (size_t)PyTuple_GET_SIZE(tuple_o)); next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(tuple_o, i)); - null_or_index = PyStackRef_IncrementTaggedInt(null_or_index); + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); } stack_pointer[-1] = null_or_index; stack_pointer[0] = next; diff --git a/Python/stackrefs.c b/Python/stackrefs.c index c64525b21d35fe..7671e40cfeb8e5 100644 --- a/Python/stackrefs.c +++ b/Python/stackrefs.c @@ -217,7 +217,7 @@ PyStackRef_IsNullOrInt(_PyStackRef ref) } _PyStackRef -PyStackRef_IncrementTaggedInt(_PyStackRef ref) +PyStackRef_IncrementTaggedIntNoOverflow(_PyStackRef ref) { assert(ref.index != (uintptr_t)-1); // Overflow return (_PyStackRef){ .index = ref.index + 2 }; diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index ddc1d8d7a5ee5b..f0455e923e7b2a 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -679,7 +679,7 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "PyStackRef_IsTaggedInt", "PyStackRef_TagInt", "PyStackRef_UntagInt", - "PyStackRef_IncrementTaggedInt", + "PyStackRef_IncrementTaggedIntNoOverflow", "PyStackRef_IsNullOrInt", ) From ec8d7970c69fa36b19ff20cb38a1ee015ca9b75d Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 30 Apr 2025 11:32:00 +0100 Subject: [PATCH 10/18] Restrict specialization in free-threaded build --- Python/bytecodes.c | 3 --- Python/executor_cases.c.h | 6 ------ Python/generated_cases.c.h | 7 ------- Python/specialize.c | 21 ++++++++++++++++----- 4 files changed, 16 insertions(+), 21 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index d42a1c4b703485..f4474ac90f79e8 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3286,9 +3286,6 @@ dummy_func( PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); EXIT_IF(Py_TYPE(iter_o) != &PyTuple_Type); assert(PyStackRef_IsTaggedInt(null_or_index)); -#ifdef Py_GIL_DISABLED - EXIT_IF(!_PyObject_IsUniquelyReferenced(iter_o)); -#endif } replaced op(_ITER_JUMP_TUPLE, (iter, null_or_index -- iter, null_or_index)) { diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 3c9544bf266ec3..769bc2c84bb621 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4287,12 +4287,6 @@ JUMP_TO_JUMP_TARGET(); } assert(PyStackRef_IsTaggedInt(null_or_index)); - #ifdef Py_GIL_DISABLED - if (!_PyObject_IsUniquelyReferenced(iter_o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - #endif break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index eed7c908f75585..b1d74a9303c4fe 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5823,13 +5823,6 @@ JUMP_TO_PREDICTED(FOR_ITER); } assert(PyStackRef_IsTaggedInt(null_or_index)); - #ifdef Py_GIL_DISABLED - if (!_PyObject_IsUniquelyReferenced(iter_o)) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(FOR_ITER); - } - #endif } // _ITER_JUMP_TUPLE { diff --git a/Python/specialize.c b/Python/specialize.c index 3e1fe5a7070fdc..1da692ebe065ef 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -2904,13 +2904,18 @@ _Py_Specialize_ForIter(_PyStackRef iter, _PyStackRef null_or_index, _Py_CODEUNIT assert(_PyOpcode_Caches[FOR_ITER] == INLINE_CACHE_ENTRIES_FOR_ITER); PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); PyTypeObject *tp = Py_TYPE(iter_o); + + if (PyStackRef_IsNull(null_or_index)) { #ifdef Py_GIL_DISABLED - // Only specialize for lists owned by this thread or shared - if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { - goto failure; - } + // Only specialize for uniquely referenced iterators, so that we know + // they're only referenced by this one thread. This is more limiting + // than we need (even `it = iter(mylist); for item in it:` won't get + // specialized) but we don't have a way to check whether we're the only + // _thread_ who has access to the object. + if (!_PyObject_IsUniquelyReferenced(iter_o)) { + goto failure; + } #endif - if (PyStackRef_IsNull(null_or_index)) { if (tp == &PyRangeIter_Type) { specialize(instr, FOR_ITER_RANGE); return; @@ -2930,6 +2935,12 @@ _Py_Specialize_ForIter(_PyStackRef iter, _PyStackRef null_or_index, _Py_CODEUNIT } else { if (tp == &PyList_Type) { +#ifdef Py_GIL_DISABLED + // Only specialize for lists owned by this thread or shared + if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { + goto failure; + } +#endif specialize(instr, FOR_ITER_LIST); return; } From 428735b5bc6d5095219e35114b26e65ec730d674 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 30 Apr 2025 14:13:15 +0100 Subject: [PATCH 11/18] Add news --- .../2025-04-30-14-13-01.gh-issue-132554.GqQaUp.rst | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-04-30-14-13-01.gh-issue-132554.GqQaUp.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-04-30-14-13-01.gh-issue-132554.GqQaUp.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-04-30-14-13-01.gh-issue-132554.GqQaUp.rst new file mode 100644 index 00000000000000..bfe2d633309191 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-04-30-14-13-01.gh-issue-132554.GqQaUp.rst @@ -0,0 +1,4 @@ +Change iteration to use "virtual iterators" for sequences. Instead of +creating an iterator, a tagged integer representing the next index is pushed +to the stack above the iterable. For non-sequence iterators, ``NULL`` is +pushed. From 4c83848fb32a585e4befdb8a5667831de9e6e58b Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 30 Apr 2025 17:12:00 +0100 Subject: [PATCH 12/18] handle tagged ints when doing type checks --- Include/internal/pycore_stackref.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index ad3699fef1d58c..211f65afde1457 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -714,18 +714,27 @@ PyStackRef_TYPE(_PyStackRef stackref) { static inline bool PyStackRef_GenCheck(_PyStackRef stackref) { + if (PyStackRef_IsTaggedInt(stackref)) { + return false; + } return PyGen_Check(PyStackRef_AsPyObjectBorrow(stackref)); } static inline bool PyStackRef_BoolCheck(_PyStackRef stackref) { + if (PyStackRef_IsTaggedInt(stackref)) { + return false; + } return PyBool_Check(PyStackRef_AsPyObjectBorrow(stackref)); } static inline bool PyStackRef_LongCheck(_PyStackRef stackref) { + if (PyStackRef_IsTaggedInt(stackref)) { + return false; + } return PyLong_Check(PyStackRef_AsPyObjectBorrow(stackref)); } @@ -738,12 +747,18 @@ PyStackRef_ExceptionInstanceCheck(_PyStackRef stackref) static inline bool PyStackRef_CodeCheck(_PyStackRef stackref) { + if (PyStackRef_IsTaggedInt(stackref)) { + return false; + } return PyCode_Check(PyStackRef_AsPyObjectBorrow(stackref)); } static inline bool PyStackRef_FunctionCheck(_PyStackRef stackref) { + if (PyStackRef_IsTaggedInt(stackref)) { + return false; + } return PyFunction_Check(PyStackRef_AsPyObjectBorrow(stackref)); } From f43ccc3b46b7ff29bb48b81e9a5eff69340b55d7 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 30 Apr 2025 17:22:32 +0100 Subject: [PATCH 13/18] Fix long check --- Include/internal/pycore_stackref.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index 211f65afde1457..83e8b84f0622e8 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -733,7 +733,7 @@ static inline bool PyStackRef_LongCheck(_PyStackRef stackref) { if (PyStackRef_IsTaggedInt(stackref)) { - return false; + return true; } return PyLong_Check(PyStackRef_AsPyObjectBorrow(stackref)); } From 3fd46c3dcd22759805d36925b35e36ada1371c13 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 30 Apr 2025 18:31:47 +0100 Subject: [PATCH 14/18] Attempt to make _PyForIter_NextWithIndex thread safe. --- Include/internal/pycore_stackref.h | 3 +++ Python/ceval.c | 22 ++++++++++++++++++---- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index 83e8b84f0622e8..7f3a5725a8dd23 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -741,6 +741,9 @@ PyStackRef_LongCheck(_PyStackRef stackref) static inline bool PyStackRef_ExceptionInstanceCheck(_PyStackRef stackref) { + if (PyStackRef_IsTaggedInt(stackref)) { + return false; + } return PyExceptionInstance_Check(PyStackRef_AsPyObjectBorrow(stackref)); } diff --git a/Python/ceval.c b/Python/ceval.c index 0805b589cd7320..a8a8340f0f5f35 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -3421,14 +3421,28 @@ _PyStackRef _PyForIter_NextWithIndex(PyObject *seq, _PyStackRef index) { assert(PyStackRef_IsTaggedInt(index)); - assert(Py_TYPE(seq) == &PyTuple_Type || Py_TYPE(seq) == &PyList_Type); - size_t size = Py_SIZE(seq); + assert(PyTuple_CheckExact(seq) || PyList_CheckExact(seq)); intptr_t i = PyStackRef_UntagInt(index); + if (PyTuple_CheckExact(seq)) { + size_t size = PyTuple_GET_SIZE(seq); + if ((size_t)i >= size) { + return PyStackRef_NULL; + } + return PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, i)); + } + size_t size = PyList_GET_SIZE(seq); if ((size_t)i >= size) { return PyStackRef_NULL; } - PyObject *next_o = PySequence_Fast_GET_ITEM(seq, i); - return PyStackRef_FromPyObjectNew(next_o); +#ifdef Py_GIL_DISABLED + PyObject *item = _PyList_GetItemRef((PyListObject *)seq, i); + if (item == NULL) { + return PyStackRef_NULL; + } + return PyStackRef_FromPyObjectSteal(item); +#else + return PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, i)); +#endif } /* Check if a 'cls' provides the given special method. */ From 433282fbae085cee4722e8abd01c73cf07dbb4e9 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 30 Apr 2025 20:20:39 +0100 Subject: [PATCH 15/18] Add review comment --- Python/specialize.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Python/specialize.c b/Python/specialize.c index 1da692ebe065ef..44fce2bebd851c 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -2927,8 +2927,9 @@ _Py_Specialize_ForIter(_PyStackRef iter, _PyStackRef null_or_index, _Py_CODEUNIT instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1].op.code == INSTRUMENTED_END_FOR ); /* Don't specialize if PEP 523 is active */ - if (_PyInterpreterState_GET()->eval_frame) + if (_PyInterpreterState_GET()->eval_frame) { goto failure; + } specialize(instr, FOR_ITER_GEN); return; } From e915a05ce2261ee9edf9bf482e2719a6240f1b83 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 8 May 2025 10:55:30 +0100 Subject: [PATCH 16/18] Simplify list indexing code --- Python/ceval.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index a8a8340f0f5f35..30b195ae47cab3 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -3430,19 +3430,11 @@ _PyForIter_NextWithIndex(PyObject *seq, _PyStackRef index) } return PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, i)); } - size_t size = PyList_GET_SIZE(seq); - if ((size_t)i >= size) { - return PyStackRef_NULL; - } -#ifdef Py_GIL_DISABLED PyObject *item = _PyList_GetItemRef((PyListObject *)seq, i); if (item == NULL) { return PyStackRef_NULL; } return PyStackRef_FromPyObjectSteal(item); -#else - return PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, i)); -#endif } /* Check if a 'cls' provides the given special method. */ From 6d1b93e760b1312c2b938f89c87066be6195b303 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 8 May 2025 15:36:08 +0100 Subject: [PATCH 17/18] GET_ITER may leave the iterable on the stack --- Lib/test/test_list.py | 14 ++++++++++++++ Python/flowgraph.c | 1 + 2 files changed, 15 insertions(+) diff --git a/Lib/test/test_list.py b/Lib/test/test_list.py index 6894fba2ad1b00..450fc9024e4bac 100644 --- a/Lib/test/test_list.py +++ b/Lib/test/test_list.py @@ -365,5 +365,19 @@ def lappend(l, x, y): rc, _, _ = assert_python_ok("-c", code) self.assertEqual(rc, 0) + def test_list_overwrite_local(self): + """Test that overwriting the last reference to the + iterable doesn't prematurely free the iterable""" + + def foo(x): + r = 0 + for i in x: + r += i + x = None + return r + + self.assertEqual(foo(list(range(10))), 45) + + if __name__ == "__main__": unittest.main() diff --git a/Python/flowgraph.c b/Python/flowgraph.c index dabaf9bdc7cbb6..67ccf350b72ed6 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -2871,6 +2871,7 @@ optimize_load_fast(cfg_builder *g) // Opcodes that consume no inputs case GET_ANEXT: + case GET_ITER: case GET_LEN: case IMPORT_FROM: case MATCH_KEYS: From 37ca28565accb04e6be039bde9b72c7cf25ae8f5 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 8 May 2025 15:49:41 +0100 Subject: [PATCH 18/18] Fix test_dis --- Lib/test/test_dis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 2968b910110fb1..7967c9d9886962 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -851,7 +851,7 @@ def foo(x): %4d RETURN_GENERATOR POP_TOP L1: RESUME 0 - LOAD_FAST_BORROW 0 (.0) + LOAD_FAST 0 (.0) GET_ITER L2: FOR_ITER 14 (to L3) STORE_FAST 1 (z)