gh-105481: add haslocal to _opcode.py. Generate most oplists in opcod…

…e.py with the functions in _opcode.py
python · iritkatriel · Jul 18, 2023 · Jul 14, 2023 · Jul 14, 2023 · Jul 15, 2023
commit 36a7d56e5ae06a4269052f6961dbe1154ccc57ef
diff --git a/Include/cpython/compile.h b/Include/cpython/compile.h
@@ -71,5 +71,6 @@ PyAPI_FUNC(int) PyCompile_OpcodeStackEffectWithJump(int opcode, int oparg, int j
 PyAPI_FUNC(int) PyUnstable_OpcodeIsValid(int opcode);
 PyAPI_FUNC(int) PyUnstable_OpcodeHasArg(int opcode);
 PyAPI_FUNC(int) PyUnstable_OpcodeHasConst(int opcode);
-PyAPI_FUNC(int) PyUnstable_OpcodeHasName(int opcode);
 PyAPI_FUNC(int) PyUnstable_OpcodeHasJump(int opcode);
+PyAPI_FUNC(int) PyUnstable_OpcodeHasLocal(int opcode);
+PyAPI_FUNC(int) PyUnstable_OpcodeHasName(int opcode);
diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h
@@ -952,10 +952,12 @@ enum InstructionFormat { INSTR_FMT_IB, INSTR_FMT_IBC, INSTR_FMT_IBC00, INSTR_FMT
 #define HAS_CONST_FLAG (2)
 #define HAS_NAME_FLAG (4)
 #define HAS_JUMP_FLAG (8)
+#define HAS_LOCAL_FLAG (16)
 #define OPCODE_HAS_ARG(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ARG_FLAG))
 #define OPCODE_HAS_CONST(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_CONST_FLAG))
 #define OPCODE_HAS_NAME(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NAME_FLAG))
 #define OPCODE_HAS_JUMP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_JUMP_FLAG))
+#define OPCODE_HAS_LOCAL(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_LOCAL_FLAG))

 struct opcode_metadata {
    bool valid_entry;
@@ -992,16 +994,16 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = {
    [NOP] = { true, INSTR_FMT_IX, 0 },
    [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
    [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
-    [LOAD_CLOSURE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
-    [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
-    [LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
-    [LOAD_FAST_AND_CLEAR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
-    [LOAD_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
+    [LOAD_CLOSURE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG },
+    [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG },
+    [LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG },
+    [LOAD_FAST_AND_CLEAR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG },
+    [LOAD_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG },
    [LOAD_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG },
-    [STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
-    [STORE_FAST_MAYBE_NULL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
-    [STORE_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
-    [STORE_FAST_STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
+    [STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG },
+    [STORE_FAST_MAYBE_NULL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG },
+    [STORE_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG },
+    [STORE_FAST_STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG },
    [POP_TOP] = { true, INSTR_FMT_IX, 0 },
    [PUSH_NULL] = { true, INSTR_FMT_IX, 0 },
    [END_FOR] = { true, INSTR_FMT_IB, 0 },
@@ -1025,7 +1027,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = {
    [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IBC, 0 },
    [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IBC, 0 },
    [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IBC, 0 },
-    [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IB, 0 },
+    [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IB, HAS_LOCAL_FLAG },
    [BINARY_SUBSCR] = { true, INSTR_FMT_IXC, 0 },
    [BINARY_SLICE] = { true, INSTR_FMT_IX, 0 },
    [STORE_SLICE] = { true, INSTR_FMT_IX, 0 },
@@ -1077,12 +1079,12 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = {
    [LOAD_GLOBAL] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG },
    [LOAD_GLOBAL_MODULE] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG },
    [LOAD_GLOBAL_BUILTIN] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG },
-    [DELETE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
-    [MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
-    [DELETE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
-    [LOAD_FROM_DICT_OR_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
-    [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
-    [STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
+    [DELETE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG },
+    [MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG },
+    [DELETE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG },
+    [LOAD_FROM_DICT_OR_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG },
+    [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG },
+    [STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG },
    [COPY_FREE_VARS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
    [BUILD_STRING] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
    [BUILD_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },

diff --git a/Lib/opcode.py b/Lib/opcode.py
@@ -29,12 +29,6 @@

 cmp_op = ('<', '<=', '==', '!=', '>', '>=')

-hasarg = []
-hasconst = []
-hasname = []
-hasjrel = []
-hasjabs = []
-haslocal = []
 hascompare = []
 hasfree = []
 hasexc = []
@@ -45,8 +39,7 @@
 def is_pseudo(op):
    return op >= MIN_PSEUDO_OPCODE and op <= MAX_PSEUDO_OPCODE

-oplists = [hasarg, hasconst, hasname, hasjrel, hasjabs,
-           haslocal, hascompare, hasfree, hasexc]
+oplists = [hascompare, hasfree, hasexc]

 opmap = {}

@@ -57,18 +50,6 @@ def is_pseudo(op):
 def def_op(name, op):
    opmap[name] = op

-def name_op(name, op):
-    def_op(name, op)
-    hasname.append(op)
-
-def jrel_op(name, op):
-    def_op(name, op)
-    hasjrel.append(op)
-
-def jabs_op(name, op):
-    def_op(name, op)
-    hasjabs.append(op)
-
 def pseudo_op(name, op, real_ops):
    def_op(name, op)
    _pseudo_ops[name] = real_ops
@@ -149,55 +130,49 @@ def pseudo_op(name, op, real_ops):

 HAVE_ARGUMENT = 90             # real opcodes from here have an argument:

-name_op('STORE_NAME', 90)       # Index in name list
-name_op('DELETE_NAME', 91)      # ""
+def_op('STORE_NAME', 90)       # Index in name list
+def_op('DELETE_NAME', 91)      # ""
 def_op('UNPACK_SEQUENCE', 92)   # Number of tuple items
-jrel_op('FOR_ITER', 93)
+def_op('FOR_ITER', 93)
 def_op('UNPACK_EX', 94)
-name_op('STORE_ATTR', 95)       # Index in name list
-name_op('DELETE_ATTR', 96)      # ""
-name_op('STORE_GLOBAL', 97)     # ""
-name_op('DELETE_GLOBAL', 98)    # ""
+def_op('STORE_ATTR', 95)       # Index in name list
+def_op('DELETE_ATTR', 96)      # ""
+def_op('STORE_GLOBAL', 97)     # ""
+def_op('DELETE_GLOBAL', 98)    # ""
 def_op('SWAP', 99)
 def_op('LOAD_CONST', 100)       # Index in const list
-hasconst.append(100)
-name_op('LOAD_NAME', 101)       # Index in name list
+def_op('LOAD_NAME', 101)       # Index in name list
 def_op('BUILD_TUPLE', 102)      # Number of tuple items
 def_op('BUILD_LIST', 103)       # Number of list items
 def_op('BUILD_SET', 104)        # Number of set items
 def_op('BUILD_MAP', 105)        # Number of dict entries
-name_op('LOAD_ATTR', 106)       # Index in name list
+def_op('LOAD_ATTR', 106)       # Index in name list
 def_op('COMPARE_OP', 107)       # Comparison operator
 hascompare.append(107)
-name_op('IMPORT_NAME', 108)     # Index in name list
-name_op('IMPORT_FROM', 109)     # Index in name list
-jrel_op('JUMP_FORWARD', 110)    # Number of words to skip
+def_op('IMPORT_NAME', 108)     # Index in name list
+def_op('IMPORT_FROM', 109)     # Index in name list
+def_op('JUMP_FORWARD', 110)    # Number of words to skip

-jrel_op('POP_JUMP_IF_FALSE', 114)
-jrel_op('POP_JUMP_IF_TRUE', 115)
-name_op('LOAD_GLOBAL', 116)     # Index in name list
+def_op('POP_JUMP_IF_FALSE', 114)
+def_op('POP_JUMP_IF_TRUE', 115)
+def_op('LOAD_GLOBAL', 116)     # Index in name list
 def_op('IS_OP', 117)
 def_op('CONTAINS_OP', 118)
 def_op('RERAISE', 119)
 def_op('COPY', 120)
 def_op('RETURN_CONST', 121)
-hasconst.append(121)
 def_op('BINARY_OP', 122)
-jrel_op('SEND', 123)            # Number of words to skip
+def_op('SEND', 123)            # Number of words to skip
 def_op('LOAD_FAST', 124)        # Local variable number, no null check
-haslocal.append(124)
 def_op('STORE_FAST', 125)       # Local variable number
-haslocal.append(125)
 def_op('DELETE_FAST', 126)      # Local variable number
-haslocal.append(126)
 def_op('LOAD_FAST_CHECK', 127)  # Local variable number
-haslocal.append(127)
-jrel_op('POP_JUMP_IF_NOT_NONE', 128)
-jrel_op('POP_JUMP_IF_NONE', 129)
+def_op('POP_JUMP_IF_NOT_NONE', 128)
+def_op('POP_JUMP_IF_NONE', 129)
 def_op('RAISE_VARARGS', 130)    # Number of raise arguments (1, 2, or 3)
 def_op('GET_AWAITABLE', 131)
 def_op('BUILD_SLICE', 133)      # Number of items
-jrel_op('JUMP_BACKWARD_NO_INTERRUPT', 134) # Number of words to skip (backwards)
+def_op('JUMP_BACKWARD_NO_INTERRUPT', 134) # Number of words to skip (backwards)
 def_op('MAKE_CELL', 135)
 hasfree.append(135)
 def_op('LOAD_DEREF', 137)
@@ -206,11 +181,10 @@ def pseudo_op(name, op, real_ops):
 hasfree.append(138)
 def_op('DELETE_DEREF', 139)
 hasfree.append(139)
-jrel_op('JUMP_BACKWARD', 140)    # Number of words to skip (backwards)
-name_op('LOAD_SUPER_ATTR', 141)
+def_op('JUMP_BACKWARD', 140)    # Number of words to skip (backwards)
+def_op('LOAD_SUPER_ATTR', 141)
 def_op('CALL_FUNCTION_EX', 142)  # Flags
 def_op('LOAD_FAST_AND_CLEAR', 143)  # Local variable number
-haslocal.append(143)
 def_op('EXTENDED_ARG', 144)
 EXTENDED_ARG = 144
 def_op('LIST_APPEND', 145)
@@ -236,10 +210,9 @@ def pseudo_op(name, op, real_ops):
 def_op('STORE_FAST_STORE_FAST', 170)
 def_op('CALL', 171)
 def_op('KW_NAMES', 172)
-hasconst.append(172)
 def_op('CALL_INTRINSIC_1', 173)
 def_op('CALL_INTRINSIC_2', 174)
-name_op('LOAD_FROM_DICT_OR_GLOBALS', 175)
+def_op('LOAD_FROM_DICT_OR_GLOBALS', 175)
 def_op('LOAD_FROM_DICT_OR_DEREF', 176)
 hasfree.append(176)
 def_op('SET_FUNCTION_ATTRIBUTE', 177)    # Attribute
@@ -270,7 +243,6 @@ def pseudo_op(name, op, real_ops):
 def_op('INSTRUMENTED_LINE', 254)
 # 255 is reserved

-hasarg.extend([op for op in opmap.values() if op >= HAVE_ARGUMENT])

 MIN_PSEUDO_OPCODE = 256

@@ -295,12 +267,29 @@ def pseudo_op(name, op, real_ops):

 MAX_PSEUDO_OPCODE = MIN_PSEUDO_OPCODE + len(_pseudo_ops) - 1

-del def_op, name_op, jrel_op, jabs_op, pseudo_op
+del def_op, pseudo_op

 opname = ['<%r>' % (op,) for op in range(MAX_PSEUDO_OPCODE + 1)]
 for op, i in opmap.items():
    opname[i] = op

+# _opcode may not be ready during early stages of the build
+try:
+    import _opcode
+except ImportError:
+    hasarg = []
+    hasconst = []
+    hasname = []
+    hasjrel = []
+    hasjabs = []
+    haslocal = []
+else:
+    hasarg = [op for op in opmap.values() if _opcode.has_arg(op)]
+    hasconst = [op for op in opmap.values() if _opcode.has_const(op)]
+    hasname = [op for op in opmap.values() if _opcode.has_name(op)]
+    hasjrel = [op for op in opmap.values() if _opcode.has_jump(op)]
+    hasjabs = []
+    haslocal = [op for op in opmap.values() if _opcode.has_local(op)]

 _nb_ops = [
    ("NB_ADD", "+"),

diff --git a/Lib/test/test__opcode.py b/Lib/test/test__opcode.py
@@ -24,6 +24,7 @@ def test_invalid_opcodes(self):
        self.check_bool_function_result(_opcode.has_const, invalid, False)
        self.check_bool_function_result(_opcode.has_name, invalid, False)
        self.check_bool_function_result(_opcode.has_jump, invalid, False)
+        self.check_bool_function_result(_opcode.has_local, invalid, False)

    def test_is_valid(self):
        names = [
@@ -61,17 +62,12 @@ def test_has_jump(self):
        self.check_bool_function_result(_opcode.has_jump, has_jump, True)
        self.check_bool_function_result(_opcode.has_jump, no_jump, False)

-    # the following test is part of the refactor, it will be removed soon
-    def test_against_legacy_bool_values(self):
-        # limiting to ops up to ENTER_EXECUTOR, because everything after that
-        # is not currently categorized correctly in opcode.py.
-        for op in range(0, opcode.opmap['ENTER_EXECUTOR']):
-            with self.subTest(op=op):
-                if opcode.opname[op] != f'<{op}>':
-                    self.assertEqual(op in dis.hasarg, _opcode.has_arg(op))
-                    self.assertEqual(op in dis.hasconst, _opcode.has_const(op))
-                    self.assertEqual(op in dis.hasname, _opcode.has_name(op))
-                    self.assertEqual(op in dis.hasjrel, _opcode.has_jump(op))
+    def test_has_local(self):
+        has_local = ['LOAD_FAST', 'LOAD_FAST_CHECK', 'LOAD_FAST_AND_CLEAR',
+                     'STORE_FAST_MAYBE_NULL', 'LOAD_CLOSURE']
+        no_local = ['SETUP_WITH', 'POP_TOP', 'NOP', 'CACHE']
+        self.check_bool_function_result(_opcode.has_local, has_local, True)
+        self.check_bool_function_result(_opcode.has_local, no_local, False)

    def test_stack_effect(self):
        self.assertEqual(stack_effect(dis.opmap['POP_TOP']), -1)

diff --git a/Modules/_opcode.c b/Modules/_opcode.c
@@ -145,6 +145,25 @@ _opcode_has_jump_impl(PyObject *module, int opcode)

 }

+/*[clinic input]
+
+_opcode.has_local -> bool
+
+  opcode: int
+
+Return True if the opcode accesses a local variable, False otherwise.
+[clinic start generated code]*/
+
+static int
+_opcode_has_local_impl(PyObject *module, int opcode)
+/*[clinic end generated code: output=da5a8616b7a5097b input=9a798ee24aaef49d]*/
+{
+    return PyUnstable_OpcodeIsValid(opcode) &&
+           PyUnstable_OpcodeHasLocal(opcode);
+
+}
+
+
 /*[clinic input]

 _opcode.get_specialization_stats
@@ -171,6 +190,7 @@ opcode_functions[] =  {
    _OPCODE_HAS_CONST_METHODDEF
    _OPCODE_HAS_NAME_METHODDEF
    _OPCODE_HAS_JUMP_METHODDEF
+    _OPCODE_HAS_LOCAL_METHODDEF
    _OPCODE_GET_SPECIALIZATION_STATS_METHODDEF
    {NULL, NULL, 0, NULL}
 };

diff --git a/Modules/clinic/_opcode.c.h b/Modules/clinic/_opcode.c.h