Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

GH-132508: Use tagged integers on the evaluation stack for the last instruction offset #132545

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 14 commits into from
Apr 29, 2025
Merged

GH-132508: Use tagged integers on the evaluation stack for the last instruction offset #132545

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
ba30e9a
Use tagged integers on the evaluation stack for lasti
markshannon Apr 11, 2025
1d223c3
Add support for Py_STACKREF_DEBUG
markshannon Apr 14, 2025
246199b
Add news
markshannon Apr 15, 2025
b1b1252
Fix dump_item
markshannon Apr 15, 2025
8d3c3ea
Clarify division between pre-defined stackref indexes and others
markshannon Apr 15, 2025
aa4ddf0
Cast before shifting
markshannon Apr 15, 2025
e223e36
Integer tagging doesn't escape
markshannon Apr 15, 2025
9e11cc7
Push assert up to caller
markshannon Apr 15, 2025
79ed469
Avoid side effects in macro
markshannon Apr 16, 2025
b491544
Remove debug code
markshannon Apr 16, 2025
f6126b1
Handle negative ints properly in Py_STACKREF_DEBUG build
markshannon Apr 16, 2025
1b4aba6
Brandt's fix to the stencils
markshannon Apr 28, 2025
d84cd2d
Put Brandt's patch in the right place :)
markshannon Apr 29, 2025
fb89cd9
Replace is_tagged_int with PyStackRef_IsTaggedInt
markshannon Apr 29, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 65 additions & 10 deletions 75 Include/internal/pycore_stackref.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,11 +63,11 @@ extern void _Py_stackref_associate(PyInterpreterState *interp, PyObject *obj, _P

static const _PyStackRef PyStackRef_NULL = { .index = 0 };

#define PyStackRef_None ((_PyStackRef){ .index = 1 } )
markshannon marked this conversation as resolved.
Show resolved Hide resolved
#define PyStackRef_False ((_PyStackRef){ .index = 2 })
#define PyStackRef_True ((_PyStackRef){ .index = 3 })
#define PyStackRef_None ((_PyStackRef){ .index = 2 } )
#define PyStackRef_False ((_PyStackRef){ .index = 4 })
#define PyStackRef_True ((_PyStackRef){ .index = 6 })

#define LAST_PREDEFINED_STACKREF_INDEX 3
#define LAST_PREDEFINED_STACKREF_INDEX 6

static inline int
PyStackRef_IsNull(_PyStackRef ref)
Expand Down Expand Up @@ -96,6 +96,7 @@ PyStackRef_IsNone(_PyStackRef ref)
static inline PyObject *
_PyStackRef_AsPyObjectBorrow(_PyStackRef ref, const char *filename, int linenumber)
{
assert((ref.index & 1) == 0);
iritkatriel marked this conversation as resolved.
Show resolved Hide resolved
_Py_stackref_record_borrow(ref, filename, linenumber);
return _Py_stackref_get_object(ref);
}
Expand Down Expand Up @@ -132,31 +133,45 @@ _PyStackRef_FromPyObjectImmortal(PyObject *obj, const char *filename, int linenu
}
#define PyStackRef_FromPyObjectImmortal(obj) _PyStackRef_FromPyObjectImmortal(_PyObject_CAST(obj), __FILE__, __LINE__)

static inline bool
is_tagged_int(_PyStackRef ref)
{
return (ref.index & 1) == 1;
markshannon marked this conversation as resolved.
Show resolved Hide resolved
}

static inline void
_PyStackRef_CLOSE(_PyStackRef ref, const char *filename, int linenumber)
{
if (is_tagged_int(ref)) {
return;
}
PyObject *obj = _Py_stackref_close(ref, filename, linenumber);
Py_DECREF(obj);
}
#define PyStackRef_CLOSE(REF) _PyStackRef_CLOSE((REF), __FILE__, __LINE__)


static inline void
_PyStackRef_XCLOSE(_PyStackRef ref, const char *filename, int linenumber)
{
if (PyStackRef_IsNull(ref)) {
return;
}
PyObject *obj = _Py_stackref_close(ref, filename, linenumber);
Py_DECREF(obj);
_PyStackRef_CLOSE(ref, filename, linenumber);
}
#define PyStackRef_XCLOSE(REF) _PyStackRef_XCLOSE((REF), __FILE__, __LINE__)

static inline _PyStackRef
_PyStackRef_DUP(_PyStackRef ref, const char *filename, int linenumber)
{
PyObject *obj = _Py_stackref_get_object(ref);
Py_INCREF(obj);
return _Py_stackref_create(obj, filename, linenumber);
if (ref.index & 1) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
if (ref.index & 1) {
if (is_tagged_int(ref)) {

return ref;
}
else {
PyObject *obj = _Py_stackref_get_object(ref);
Py_INCREF(obj);
return _Py_stackref_create(obj, filename, linenumber);
}
}
#define PyStackRef_DUP(REF) _PyStackRef_DUP(REF, __FILE__, __LINE__)

Expand Down Expand Up @@ -210,8 +225,39 @@ _PyStackRef_FromPyObjectNewMortal(PyObject *obj, const char *filename, int linen

extern int PyStackRef_Is(_PyStackRef a, _PyStackRef b);

extern bool PyStackRef_IsTaggedInt(_PyStackRef ref);

extern intptr_t PyStackRef_UntagInt(_PyStackRef ref);

extern _PyStackRef PyStackRef_TagInt(intptr_t i);

extern bool
PyStackRef_IsNullOrInt(_PyStackRef ref);

#else

#define Py_INT_TAG 3

static inline bool
PyStackRef_IsTaggedInt(_PyStackRef i)
{
return (i.bits & Py_INT_TAG) == Py_INT_TAG;
}

static inline _PyStackRef
PyStackRef_TagInt(intptr_t i)
{
assert(Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, (i << 2), 2) == i);
Fidget-Spinner marked this conversation as resolved.
Show resolved Hide resolved
return (_PyStackRef){ .bits = ((((uintptr_t)i) << 2) | Py_INT_TAG) };
}

static inline intptr_t
PyStackRef_UntagInt(_PyStackRef i)
{
assert((i.bits & Py_INT_TAG) == Py_INT_TAG);
return Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, i.bits, 2);
}


#ifdef Py_GIL_DISABLED

Expand All @@ -232,6 +278,8 @@ static const _PyStackRef PyStackRef_NULL = { .bits = Py_TAG_DEFERRED};
#define PyStackRef_IsTrue(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_True)
#define PyStackRef_IsFalse(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_False)

#define PyStackRef_IsNullOrInt(stackref) (PyStackRef_IsNull(stackref) || PyStackRef_IsTaggedInt(stackref))

static inline PyObject *
PyStackRef_AsPyObjectBorrow(_PyStackRef stackref)
{
Expand Down Expand Up @@ -451,6 +499,7 @@ PyStackRef_RefcountOnObject(_PyStackRef ref)
static inline PyObject *
PyStackRef_AsPyObjectBorrow(_PyStackRef ref)
{
assert(!PyStackRef_IsTaggedInt(ref));
return BITS_TO_PTR_MASKED(ref);
}

Expand Down Expand Up @@ -587,6 +636,12 @@ PyStackRef_CLOSE(_PyStackRef ref)
}
#endif

static inline bool
PyStackRef_IsNullOrInt(_PyStackRef ref)
{
return PyStackRef_IsNull(ref) || PyStackRef_IsTaggedInt(ref);
}

static inline void
PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct)
{
Expand Down Expand Up @@ -726,7 +781,7 @@ _Py_TryXGetStackRef(PyObject **src, _PyStackRef *out)
// Like Py_VISIT but for _PyStackRef fields
#define _Py_VISIT_STACKREF(ref) \
do { \
if (!PyStackRef_IsNull(ref)) { \
if (!PyStackRef_IsNullOrInt(ref)) { \
int vret = _PyGC_VisitStackRef(&(ref), visit, arg); \
if (vret) \
return vret; \
Expand Down
3 changes: 3 additions & 0 deletions 3 Misc/NEWS.d/next/Core_and_Builtins/2025-04-15-10-09-49.gh-issue-132508.zVe3iI.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Uses tagged integers on the evaluation stack to represent the instruction
offsets when reraising an exception. This avoids the need to box the integer
which could fail in low memory conditions.
20 changes: 4 additions & 16 deletions 20 Python/bytecodes.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1385,16 +1385,7 @@ dummy_func(

assert(oparg >= 0 && oparg <= 2);
if (oparg) {
PyObject *lasti = PyStackRef_AsPyObjectBorrow(values[0]);
if (PyLong_Check(lasti)) {
frame->instr_ptr = _PyFrame_GetBytecode(frame) + PyLong_AsLong(lasti);
assert(!_PyErr_Occurred(tstate));
}
else {
_PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int");
Py_DECREF(exc);
ERROR_NO_POP();
}
frame->instr_ptr = _PyFrame_GetBytecode(frame) + PyStackRef_UntagInt(values[0]);
}
assert(exc && PyExceptionInstance_Check(exc));
_PyErr_SetRaisedException(tstate, exc);
Expand Down Expand Up @@ -3458,7 +3449,7 @@ dummy_func(
if (tb == NULL) {
tb = Py_None;
}
assert(PyStackRef_LongCheck(lasti));
assert(PyStackRef_IsTaggedInt(lasti));
(void)lasti; // Shut up compiler warning if asserts are off
PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb};
int has_self = !PyStackRef_IsNull(exit_self);
Expand Down Expand Up @@ -5344,11 +5335,8 @@ dummy_func(
}
if (lasti) {
int frame_lasti = _PyInterpreterFrame_LASTI(frame);
PyObject *lasti = PyLong_FromLong(frame_lasti);
if (lasti == NULL) {
goto exception_unwind;
}
_PyFrame_StackPush(frame, PyStackRef_FromPyObjectSteal(lasti));
_PyStackRef lasti = PyStackRef_TagInt(frame_lasti);
_PyFrame_StackPush(frame, lasti);
}

/* Make the raw exception data
Expand Down
4 changes: 4 additions & 0 deletions 4 Python/ceval.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,10 @@ dump_item(_PyStackRef item)
printf("<NULL>");
return;
}
if (PyStackRef_IsTaggedInt(item)) {
printf("%" PRId64, (int64_t)PyStackRef_UntagInt(item));
return;
}
PyObject *obj = PyStackRef_AsPyObjectBorrow(item);
if (obj == NULL) {
printf("<nil>");
Expand Down
4 changes: 3 additions & 1 deletion 4 Python/executor_cases.c.h
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

10 changes: 8 additions & 2 deletions 10 Python/gc.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -547,6 +547,7 @@ _PyGC_VisitStackRef(_PyStackRef *ref, visitproc visit, void *arg)
// This is a bit tricky! We want to ignore stackrefs with embedded
// refcounts when computing the incoming references, but otherwise treat
// them like normal.
assert(!PyStackRef_IsTaggedInt(*ref));
if (!PyStackRef_RefcountOnObject(*ref) && (visit == visit_decref)) {
return 0;
}
Expand All @@ -560,7 +561,9 @@ _PyGC_VisitFrameStack(_PyInterpreterFrame *frame, visitproc visit, void *arg)
_PyStackRef *ref = _PyFrame_GetLocalsArray(frame);
/* locals and stack */
for (; ref < frame->stackpointer; ref++) {
_Py_VISIT_STACKREF(*ref);
if (!PyStackRef_IsTaggedInt(*ref)) {
_Py_VISIT_STACKREF(*ref);
}
}
return 0;
}
Expand Down Expand Up @@ -1495,8 +1498,11 @@ mark_stacks(PyInterpreterState *interp, PyGC_Head *visited, int visited_space, b
objects_marked += move_to_reachable(func, &reachable, visited_space);
while (sp > locals) {
sp--;
if (PyStackRef_IsNullOrInt(*sp)) {
continue;
}
PyObject *op = PyStackRef_AsPyObjectBorrow(*sp);
if (op == NULL || _Py_IsImmortal(op)) {
if (_Py_IsImmortal(op)) {
continue;
}
if (_PyObject_IS_GC(op)) {
Expand Down
7 changes: 4 additions & 3 deletions 7 Python/gc_free_threading.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -265,7 +265,7 @@ frame_disable_deferred_refcounting(_PyInterpreterFrame *frame)

frame->f_funcobj = PyStackRef_AsStrongReference(frame->f_funcobj);
for (_PyStackRef *ref = frame->localsplus; ref < frame->stackpointer; ref++) {
if (!PyStackRef_IsNull(*ref) && PyStackRef_IsDeferred(*ref)) {
if (!PyStackRef_IsNullOrInt(*ref) && PyStackRef_IsDeferred(*ref)) {
*ref = PyStackRef_AsStrongReference(*ref);
}
}
Expand Down Expand Up @@ -420,7 +420,7 @@ gc_visit_heaps(PyInterpreterState *interp, mi_block_visit_fun *visitor,
static inline void
gc_visit_stackref(_PyStackRef stackref)
{
if (PyStackRef_IsDeferred(stackref) && !PyStackRef_IsNull(stackref)) {
if (PyStackRef_IsDeferred(stackref) && !PyStackRef_IsNullOrInt(stackref)) {
PyObject *obj = PyStackRef_AsPyObjectBorrow(stackref);
if (_PyObject_GC_IS_TRACKED(obj) && !gc_is_frozen(obj)) {
gc_add_refs(obj, 1);
Expand Down Expand Up @@ -817,7 +817,7 @@ gc_abort_mark_alive(PyInterpreterState *interp,
static int
gc_visit_stackref_mark_alive(gc_mark_args_t *args, _PyStackRef stackref)
{
if (!PyStackRef_IsNull(stackref)) {
if (!PyStackRef_IsNullOrInt(stackref)) {
PyObject *op = PyStackRef_AsPyObjectBorrow(stackref);
if (gc_mark_enqueue(op, args) < 0) {
return -1;
Expand Down Expand Up @@ -1706,6 +1706,7 @@ _PyGC_VisitStackRef(_PyStackRef *ref, visitproc visit, void *arg)
// This is a bit tricky! We want to ignore deferred references when
// computing the incoming references, but otherwise treat them like
// regular references.
assert(!PyStackRef_IsTaggedInt(*ref));
if (!PyStackRef_IsDeferred(*ref) ||
(visit != visit_decref && visit != visit_decref_unreachable))
{
Expand Down
31 changes: 11 additions & 20 deletions 31 Python/generated_cases.c.h
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion 2 Python/pystate.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -673,7 +673,7 @@ init_interpreter(PyInterpreterState *interp,
interp->dtoa = (struct _dtoa_state)_dtoa_state_INIT(interp);
}
#if !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG)
interp->next_stackref = 1;
interp->next_stackref = 2;
markshannon marked this conversation as resolved.
Show resolved Hide resolved
_Py_hashtable_allocator_t alloc = {
.malloc = malloc,
.free = free,
Expand Down
28 changes: 26 additions & 2 deletions 28 Python/stackrefs.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,8 @@ _Py_stackref_create(PyObject *obj, const char *filename, int linenumber)
Py_FatalError("Cannot create a stackref for NULL");
}
PyInterpreterState *interp = PyInterpreterState_Get();
uint64_t new_id = interp->next_stackref++;
uint64_t new_id = interp->next_stackref;
interp->next_stackref = new_id + 2;
TableEntry *entry = make_table_entry(obj, filename, linenumber);
if (entry == NULL) {
Py_FatalError("No memory left for stackref debug table");
Expand Down Expand Up @@ -152,7 +153,7 @@ void
_Py_stackref_associate(PyInterpreterState *interp, PyObject *obj, _PyStackRef ref)
{
assert(interp->next_stackref >= ref.index);
interp->next_stackref = ref.index+1;
interp->next_stackref = ref.index+2;
TableEntry *entry = make_table_entry(obj, "builtin-object", 0);
if (entry == NULL) {
Py_FatalError("No memory left for stackref debug table");
Expand Down Expand Up @@ -197,4 +198,27 @@ _PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct, const char *
_Py_DECREF_SPECIALIZED(obj, destruct);
}

_PyStackRef PyStackRef_TagInt(intptr_t i)
{
return (_PyStackRef){ .index = (i << 1) + 1 };
Fidget-Spinner marked this conversation as resolved.
Show resolved Hide resolved
}

intptr_t
PyStackRef_UntagInt(_PyStackRef i)
{
assert(is_tagged_int(i));
return Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, i.index, 1);
}

bool PyStackRef_IsTaggedInt(_PyStackRef ref)
{
return is_tagged_int(ref);
}

bool
PyStackRef_IsNullOrInt(_PyStackRef ref)
{
return PyStackRef_IsNull(ref) || PyStackRef_IsTaggedInt(ref);
}

#endif
Loading
Morty Proxy This is a proxified and sanitized view of the page, visit original site.