Get compact debug table (almost) working.

python · markshannon · Apr 21, 2022 · Apr 8, 2022 · Apr 14, 2022 · Apr 14, 2022
commit f603e3f851ce9e2c6fc3bf48a45bb6e06fe8745b
diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py
@@ -131,16 +131,16 @@ def test_different_filenames(self):
    @requires_debug_ranges()
    def test_no_columntable_and_endlinetable_with_no_debug_ranges(self):
        # Make sure when demarshalling objects with `-X no_debug_ranges`
-        # that the columntable and endlinetable are None.
+        # that the columns are None.
        co = ExceptionTestCase.test_exceptions.__code__
        code = textwrap.dedent("""
        import sys
        import marshal
        with open(sys.argv[1], 'rb') as f:
            co = marshal.load(f)
-
-            assert co.co_endlinetable is None
-            assert co.co_columntable is None
+            positions = list(co.co_positions())
+            assert positions[0][2] is None
+            assert positions[0][3] is None
        """)

        try:

@@ -349,6 +349,118 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con)
           PyBytes_GET_SIZE(con->code));
 }

+static int
+scan_varint(const uint8_t *ptr)
+{
+    int read = *ptr++;
+    int val = read & 63;
+    int shift = 0;
+    while (read & 64) {
+        read = *ptr++;
+        shift += 6;
+        val |= (read & 63) << shift;
+    }
+    return val;
+}
+
+static int
+scan_signed_varint(const uint8_t *ptr)
+{
+    int uval = scan_varint(ptr);
+    if (uval & 1) {
+        return -(int)(uval >> 1);
+    }
+    else {
+        return uval >> 1;
+    }
+}
+
+static int
+get_line_delta(const uint8_t *ptr)
+{
+    int code = ((*ptr) >> 3) & 15;
+    switch (code) {
+        case 15:
+            return 0;
+        case 13: /* No column */
+        case 14: /* Long form */
+            return scan_signed_varint(ptr+1);
+        case 10:
+        case 11:
+        case 12:
+            /* One line forms */
+            return code - 10;
+        default:
+            /* Same line */
+            return 0;
+    }
+}
+
+static uint8_t *
+write_varint(uint8_t *ptr, unsigned int val)
+{
+    while (val >= 64) {
+        *ptr++ = 64 | (val & 63);
+        val >>= 6;
+    }
+    *ptr++ = val;
+    return ptr;
+}
+
+static uint8_t *
+write_signed_varint(uint8_t *ptr, int val)
+{
+    if (val < 0) {
+        val = ((-val)<<1) | 1;
+    }
+    else {
+        val = val << 1;
+    }
+    return write_varint(ptr, val);
+}
+
+static PyObject *
+remove_column_info(PyObject *locations)
+{
+    int offset = 0;
+    const uint8_t *data = (const uint8_t *)PyBytes_AS_STRING(locations);
+    PyObject *res = PyBytes_FromStringAndSize(NULL, 32);
+    if (res == NULL) {
+        PyErr_NoMemory();
+        return NULL;
+    }
+    uint8_t *output = (uint8_t *)PyBytes_AS_STRING(res);
+    while (offset < PyBytes_GET_SIZE(locations)) {
+        Py_ssize_t len = PyBytes_GET_SIZE(res);
+        Py_ssize_t write_offset = output - (uint8_t *)PyBytes_AS_STRING(res);
+        if (write_offset + 16 >= PyBytes_GET_SIZE(res)) {
+            if (_PyBytes_Resize(&res, len * 2) < 0) {
+                return NULL;
+            }
+            output = (uint8_t *)PyBytes_AS_STRING(res) + write_offset;
+        }
+        int blength = data[offset] & 7;
+        int code = (data[offset] >> 3) & 15;
+        if (code == 15) {
+            *output++ = 0xf8 | blength;
+        }
+        else {
+            int ldelta = get_line_delta(&data[offset]);
+            *output++ = 0xe8 | blength;
+            output = write_signed_varint(output, ldelta);
+        }
+        offset++;
+        while ((data[offset] & 128) == 0) {
+            offset++;
+        }
+    }
+    Py_ssize_t write_offset = output - (uint8_t *)PyBytes_AS_STRING(res);
+    if (_PyBytes_Resize(&res, write_offset)) {
+        return NULL;
+    }
+    return res;
+}
+
 /* The caller is responsible for ensuring that the given data is valid. */

 PyCodeObject *
@@ -375,11 +487,15 @@ _PyCode_New(struct _PyCodeConstructor *con)
        return NULL;
    }

+    PyObject *replacement_locations = NULL;
    // Discard the endlinetable and columntable if we are opted out of debug
    // ranges.
    if (!_Py_GetConfig()->code_debug_ranges) {
-        con->endlinetable = Py_None;
-        con->columntable = Py_None;
+        replacement_locations = remove_column_info(con->locationtable);
+        if (replacement_locations == NULL) {
+            return NULL;
+        }
+        con->locationtable = replacement_locations;
    }

    Py_ssize_t size = PyBytes_GET_SIZE(con->code) / sizeof(_Py_CODEUNIT);
@@ -389,7 +505,7 @@ _PyCode_New(struct _PyCodeConstructor *con)
        return NULL;
    }
    init_code(co, con);
-
+    Py_XDECREF(replacement_locations);
    return co;
 }

@@ -651,57 +767,10 @@ _PyCode_CheckLineNumber(int lasti, PyCodeAddressRange *bounds)
    return bounds->ar_line;
 }

-static int
-scan_varint(const uint8_t *ptr)
-{
-    int read = *ptr++;
-    int val = read & 63;
-    int shift = 0;
-    while (read & 64) {
-        read = *ptr++;
-        shift += 6;
-        val |= (read & 63) << shift;
-    }
-    return val;
-}
-
-static int
-scan_signed_varint(const uint8_t *ptr)
-{
-    int uval = scan_varint(ptr);
-    if (uval & 1) {
-        return -(int)(uval >> 1);
-    }
-    else {
-        return uval >> 1;
-    }
-}
-
-static int
-get_next_line_delta(PyCodeAddressRange *bounds)
-{
-    int code = ((*bounds->opaque.lo_next) >> 3) & 15;
-    switch (code) {
-        case 15:
-            return 0;
-        case 13: /* No column */
-        case 14: /* Long form */
-            return scan_signed_varint(bounds->opaque.lo_next+1);
-        case 10:
-        case 11:
-        case 12:
-            /* One line forms */
-            return code - 10;
-        default:
-            /* Same line */
-            return 0;
-    }
-}
-
 static int
 is_no_line_marker(uint8_t b)
 {
-    return (b >> 3) == 31;
+    return (b >> 3) == 0x1f;
 }


@@ -718,6 +787,16 @@ next_code_delta(PyCodeAddressRange *bounds)
    return (((*bounds->opaque.lo_next) & 7) + 1) * sizeof(_Py_CODEUNIT);
 }

+static int
+previous_code_delta(PyCodeAddressRange *bounds)
+{
+    const uint8_t *ptr = bounds->opaque.lo_next-1;
+    while (((*ptr) & 128) == 0) {
+        ptr--;
+    }
+    return (((*ptr) & 7) + 1) * sizeof(_Py_CODEUNIT);
+}
+
 static int
 read_byte(PyCodeAddressRange *bounds)
 {
@@ -758,9 +837,9 @@ retreat(PyCodeAddressRange *bounds)
    do {
        bounds->opaque.lo_next--;
    } while (((*bounds->opaque.lo_next) & 128) == 0);
-    bounds->opaque.computed_line -= get_next_line_delta(bounds);
+    bounds->opaque.computed_line -= get_line_delta(bounds->opaque.lo_next);
    bounds->ar_end = bounds->ar_start;
-    bounds->ar_start -= next_code_delta(bounds);
+    bounds->ar_start -= previous_code_delta(bounds);
    if (is_no_line_marker(bounds->opaque.lo_next[-1])) {
        bounds->ar_line = -1;
    }
@@ -774,7 +853,7 @@ static void
 advance(PyCodeAddressRange *bounds)
 {
    ASSERT_VALID_BOUNDS(bounds);
-    bounds->opaque.computed_line += get_next_line_delta(bounds);
+    bounds->opaque.computed_line += get_line_delta(bounds->opaque.lo_next);
    if (is_no_line_marker(*bounds->opaque.lo_next)) {
        bounds->ar_line = -1;
    }
@@ -846,7 +925,6 @@ advance_with_locations(PyCodeAddressRange *bounds, int *endline, int *column, in
    }
    ASSERT_VALID_BOUNDS(bounds);
 }
-
 int
 PyCode_Addr2Location(PyCodeObject *co, int addrq,
                     int *start_line, int *start_column,

@@ -367,6 +367,7 @@ marklines(PyCodeObject *code, int len)
    PyCodeAddressRange bounds;
    _PyCode_InitAddressRange(code, &bounds);
    assert (bounds.ar_end == 0);
+    int last_line = -1;

    int *linestarts = PyMem_New(int, len);
    if (linestarts == NULL) {
@@ -378,7 +379,10 @@ marklines(PyCodeObject *code, int len)

    while (_PyLineTable_NextAddressRange(&bounds)) {
        assert(bounds.ar_start / (int)sizeof(_Py_CODEUNIT) < len);
-        linestarts[bounds.ar_start / sizeof(_Py_CODEUNIT)] = bounds.ar_line;
+        if (bounds.ar_line != last_line && bounds.ar_line != -1) {
+            linestarts[bounds.ar_start / sizeof(_Py_CODEUNIT)] = bounds.ar_line;
+            last_line = bounds.ar_line;
+        }
    }
    return linestarts;
 }

@@ -7175,6 +7175,7 @@ assemble_init(struct assembler *a, int nblocks, int firstlineno)
    Py_XDECREF(a->a_lnotab);
    Py_XDECREF(a->a_enotab);
    Py_XDECREF(a->a_cnotab);
+    Py_XDECREF(a->a_locationtable);
    Py_XDECREF(a->a_except_table);
    return 0;
 }
@@ -7186,6 +7187,7 @@ assemble_free(struct assembler *a)
    Py_XDECREF(a->a_lnotab);
    Py_XDECREF(a->a_enotab);
    Py_XDECREF(a->a_cnotab);
+    Py_XDECREF(a->a_locationtable);
    Py_XDECREF(a->a_except_table);
 }


diff --git a/Python/marshal.c b/Python/marshal.c
@@ -1481,6 +1481,7 @@ r_object(RFILE *p)
            Py_XDECREF(linetable);
            Py_XDECREF(endlinetable);
            Py_XDECREF(columntable);
+            Py_XDECREF(locationtable);
            Py_XDECREF(exceptiontable);
        }
        retval = v;