Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit ef29104

Browse filesBrowse files
encukouencukou
authored
pythonGH-91079: Revert "pythonGH-91079: Implement C stack limits using addresses, not counters. (pythonGH-130007)" for now (GH130413)
Revert "pythonGH-91079: Implement C stack limits using addresses, not counters. (pythonGH-130007)" for now Unfortunatlely, the change broke some buildbots. This reverts commit 2498c22.
1 parent 0ff1611 commit ef29104
Copy full SHA for ef29104

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.
Dismiss banner

47 files changed

+1464
-1218
lines changed

‎Doc/c-api/exceptions.rst

Copy file name to clipboardExpand all lines: Doc/c-api/exceptions.rst
+5-1Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -921,7 +921,11 @@ because the :ref:`call protocol <call>` takes care of recursion handling.
921921
922922
Marks a point where a recursive C-level call is about to be performed.
923923
924-
The function then checks if the stack limit is reached. If this is the
924+
If :c:macro:`!USE_STACKCHECK` is defined, this function checks if the OS
925+
stack overflowed using :c:func:`PyOS_CheckStack`. If this is the case, it
926+
sets a :exc:`MemoryError` and returns a nonzero value.
927+
928+
The function then checks if the recursion limit is reached. If this is the
925929
case, a :exc:`RecursionError` is set and a nonzero value is returned.
926930
Otherwise, zero is returned.
927931

‎Include/cpython/object.h

Copy file name to clipboardExpand all lines: Include/cpython/object.h
+5-6Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -487,19 +487,18 @@ PyAPI_FUNC(void) _PyTrash_thread_destroy_chain(PyThreadState *tstate);
487487
* we have headroom above the trigger limit */
488488
#define Py_TRASHCAN_HEADROOM 50
489489

490-
/* Helper function for Py_TRASHCAN_BEGIN */
491-
PyAPI_FUNC(int) _Py_ReachedRecursionLimitWithMargin(PyThreadState *tstate, int margin_count);
492-
493490
#define Py_TRASHCAN_BEGIN(op, dealloc) \
494491
do { \
495492
PyThreadState *tstate = PyThreadState_Get(); \
496-
if (_Py_ReachedRecursionLimitWithMargin(tstate, 1) && Py_TYPE(op)->tp_dealloc == (destructor)dealloc) { \
493+
if (tstate->c_recursion_remaining <= Py_TRASHCAN_HEADROOM && Py_TYPE(op)->tp_dealloc == (destructor)dealloc) { \
497494
_PyTrash_thread_deposit_object(tstate, (PyObject *)op); \
498495
break; \
499-
}
496+
} \
497+
tstate->c_recursion_remaining--;
500498
/* The body of the deallocator is here. */
501499
#define Py_TRASHCAN_END \
502-
if (tstate->delete_later && !_Py_ReachedRecursionLimitWithMargin(tstate, 2)) { \
500+
tstate->c_recursion_remaining++; \
501+
if (tstate->delete_later && tstate->c_recursion_remaining > (Py_TRASHCAN_HEADROOM*2)) { \
503502
_PyTrash_thread_destroy_chain(tstate); \
504503
} \
505504
} while (0);

‎Include/cpython/pystate.h

Copy file name to clipboardExpand all lines: Include/cpython/pystate.h
+32-2Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ struct _ts {
112112
int py_recursion_remaining;
113113
int py_recursion_limit;
114114

115-
int c_recursion_remaining; /* Retained for backwards compatibility. Do not use */
115+
int c_recursion_remaining;
116116
int recursion_headroom; /* Allow 50 more calls to handle any errors. */
117117

118118
/* 'tracing' keeps track of the execution depth when tracing/profiling.
@@ -202,7 +202,36 @@ struct _ts {
202202
PyObject *threading_local_sentinel;
203203
};
204204

205-
# define Py_C_RECURSION_LIMIT 5000
205+
#ifdef Py_DEBUG
206+
// A debug build is likely built with low optimization level which implies
207+
// higher stack memory usage than a release build: use a lower limit.
208+
# define Py_C_RECURSION_LIMIT 500
209+
#elif defined(__s390x__)
210+
# define Py_C_RECURSION_LIMIT 800
211+
#elif defined(_WIN32) && defined(_M_ARM64)
212+
# define Py_C_RECURSION_LIMIT 1000
213+
#elif defined(_WIN32)
214+
# define Py_C_RECURSION_LIMIT 3000
215+
#elif defined(__ANDROID__)
216+
// On an ARM64 emulator, API level 34 was OK with 10000, but API level 21
217+
// crashed in test_compiler_recursion_limit.
218+
# define Py_C_RECURSION_LIMIT 3000
219+
#elif defined(_Py_ADDRESS_SANITIZER)
220+
# define Py_C_RECURSION_LIMIT 4000
221+
#elif defined(__sparc__)
222+
// test_descr crashed on sparc64 with >7000 but let's keep a margin of error.
223+
# define Py_C_RECURSION_LIMIT 4000
224+
#elif defined(__wasi__)
225+
// Based on wasmtime 16.
226+
# define Py_C_RECURSION_LIMIT 5000
227+
#elif defined(__hppa__) || defined(__powerpc64__)
228+
// test_descr crashed with >8000 but let's keep a margin of error.
229+
# define Py_C_RECURSION_LIMIT 5000
230+
#else
231+
// This value is duplicated in Lib/test/support/__init__.py
232+
# define Py_C_RECURSION_LIMIT 10000
233+
#endif
234+
206235

207236
/* other API */
208237

@@ -217,6 +246,7 @@ _PyThreadState_UncheckedGet(void)
217246
return PyThreadState_GetUnchecked();
218247
}
219248

249+
220250
// Disable tracing and profiling.
221251
PyAPI_FUNC(void) PyThreadState_EnterTracing(PyThreadState *tstate);
222252

‎Include/internal/pycore_ceval.h

Copy file name to clipboardExpand all lines: Include/internal/pycore_ceval.h
+20-21Lines changed: 20 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -193,12 +193,18 @@ extern void _PyEval_DeactivateOpCache(void);
193193

194194
/* --- _Py_EnterRecursiveCall() ----------------------------------------- */
195195

196+
#ifdef USE_STACKCHECK
197+
/* With USE_STACKCHECK macro defined, trigger stack checks in
198+
_Py_CheckRecursiveCall() on every 64th call to _Py_EnterRecursiveCall. */
196199
static inline int _Py_MakeRecCheck(PyThreadState *tstate) {
197-
char here;
198-
uintptr_t here_addr = (uintptr_t)&here;
199-
_PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate;
200-
return here_addr < _tstate->c_stack_soft_limit;
200+
return (tstate->c_recursion_remaining-- < 0
201+
|| (tstate->c_recursion_remaining & 63) == 0);
201202
}
203+
#else
204+
static inline int _Py_MakeRecCheck(PyThreadState *tstate) {
205+
return tstate->c_recursion_remaining-- < 0;
206+
}
207+
#endif
202208

203209
// Export for '_json' shared extension, used via _Py_EnterRecursiveCall()
204210
// static inline function.
@@ -214,31 +220,23 @@ static inline int _Py_EnterRecursiveCallTstate(PyThreadState *tstate,
214220
return (_Py_MakeRecCheck(tstate) && _Py_CheckRecursiveCall(tstate, where));
215221
}
216222

223+
static inline void _Py_EnterRecursiveCallTstateUnchecked(PyThreadState *tstate) {
224+
assert(tstate->c_recursion_remaining > 0);
225+
tstate->c_recursion_remaining--;
226+
}
227+
217228
static inline int _Py_EnterRecursiveCall(const char *where) {
218229
PyThreadState *tstate = _PyThreadState_GET();
219230
return _Py_EnterRecursiveCallTstate(tstate, where);
220231
}
221232

222-
static inline void _Py_LeaveRecursiveCallTstate(PyThreadState *tstate) {
223-
(void)tstate;
224-
}
225-
226-
PyAPI_FUNC(void) _Py_InitializeRecursionLimits(PyThreadState *tstate);
227-
228-
static inline int _Py_ReachedRecursionLimit(PyThreadState *tstate) {
229-
char here;
230-
uintptr_t here_addr = (uintptr_t)&here;
231-
_PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate;
232-
if (here_addr > _tstate->c_stack_soft_limit) {
233-
return 0;
234-
}
235-
if (_tstate->c_stack_hard_limit == 0) {
236-
_Py_InitializeRecursionLimits(tstate);
237-
}
238-
return here_addr <= _tstate->c_stack_soft_limit;
233+
static inline void _Py_LeaveRecursiveCallTstate(PyThreadState *tstate) {
234+
tstate->c_recursion_remaining++;
239235
}
240236

241237
static inline void _Py_LeaveRecursiveCall(void) {
238+
PyThreadState *tstate = _PyThreadState_GET();
239+
_Py_LeaveRecursiveCallTstate(tstate);
242240
}
243241

244242
extern struct _PyInterpreterFrame* _PyEval_GetFrame(void);
@@ -329,6 +327,7 @@ void _Py_unset_eval_breaker_bit_all(PyInterpreterState *interp, uintptr_t bit);
329327

330328
PyAPI_FUNC(PyObject *) _PyFloat_FromDouble_ConsumeInputs(_PyStackRef left, _PyStackRef right, double value);
331329

330+
332331
#ifdef __cplusplus
333332
}
334333
#endif

‎Include/internal/pycore_symtable.h

Copy file name to clipboardExpand all lines: Include/internal/pycore_symtable.h
+2Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,8 @@ struct symtable {
8282
PyObject *st_private; /* name of current class or NULL */
8383
_PyFutureFeatures *st_future; /* module's future features that affect
8484
the symbol table */
85+
int recursion_depth; /* current recursion depth */
86+
int recursion_limit; /* recursion limit */
8587
};
8688

8789
typedef struct _symtable_entry {

‎Include/internal/pycore_tstate.h

Copy file name to clipboardExpand all lines: Include/internal/pycore_tstate.h
-5Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,6 @@ typedef struct _PyThreadStateImpl {
2121
// semi-public fields are in PyThreadState.
2222
PyThreadState base;
2323

24-
// These are addresses, but we need to convert to ints to avoid UB.
25-
uintptr_t c_stack_top;
26-
uintptr_t c_stack_soft_limit;
27-
uintptr_t c_stack_hard_limit;
28-
2924
PyObject *asyncio_running_loop; // Strong reference
3025
PyObject *asyncio_running_task; // Strong reference
3126

‎Include/pythonrun.h

Copy file name to clipboardExpand all lines: Include/pythonrun.h
+8-12Lines changed: 8 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -21,18 +21,14 @@ PyAPI_FUNC(void) PyErr_DisplayException(PyObject *);
2121
/* Stuff with no proper home (yet) */
2222
PyAPI_DATA(int) (*PyOS_InputHook)(void);
2323

24-
/* Stack size, in "pointers". This must be large enough, so
25-
* no two calls to check recursion depth are more than this far
26-
* apart. In practice, that means it must be larger than the C
27-
* stack consumption of PyEval_EvalDefault */
28-
#if defined(Py_DEBUG) && defined(WIN32)
29-
# define PYOS_STACK_MARGIN 3072
30-
#else
31-
# define PYOS_STACK_MARGIN 2048
32-
#endif
33-
#define PYOS_STACK_MARGIN_BYTES (PYOS_STACK_MARGIN * sizeof(void *))
34-
35-
#if defined(WIN32)
24+
/* Stack size, in "pointers" (so we get extra safety margins
25+
on 64-bit platforms). On a 32-bit platform, this translates
26+
to an 8k margin. */
27+
#define PYOS_STACK_MARGIN 2048
28+
29+
#if defined(WIN32) && !defined(MS_WIN64) && !defined(_M_ARM) && defined(_MSC_VER) && _MSC_VER >= 1300
30+
/* Enable stack checking under Microsoft C */
31+
// When changing the platforms, ensure PyOS_CheckStack() docs are still correct
3632
#define USE_STACKCHECK
3733
#endif
3834

‎Lib/test/list_tests.py

Copy file name to clipboardExpand all lines: Lib/test/list_tests.py
+2-4Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,7 @@
66
from functools import cmp_to_key
77

88
from test import seq_tests
9-
from test.support import ALWAYS_EQ, NEVER_EQ
10-
from test.support import skip_emscripten_stack_overflow, skip_wasi_stack_overflow
9+
from test.support import ALWAYS_EQ, NEVER_EQ, get_c_recursion_limit, skip_emscripten_stack_overflow
1110

1211

1312
class CommonTest(seq_tests.CommonTest):
@@ -60,11 +59,10 @@ def test_repr(self):
6059
self.assertEqual(str(a2), "[0, 1, 2, [...], 3]")
6160
self.assertEqual(repr(a2), "[0, 1, 2, [...], 3]")
6261

63-
@skip_wasi_stack_overflow()
6462
@skip_emscripten_stack_overflow()
6563
def test_repr_deep(self):
6664
a = self.type2test([])
67-
for i in range(100_000):
65+
for i in range(get_c_recursion_limit() + 1):
6866
a = self.type2test([a])
6967
self.assertRaises(RecursionError, repr, a)
7068

‎Lib/test/mapping_tests.py

Copy file name to clipboardExpand all lines: Lib/test/mapping_tests.py
+3-4Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# tests common to dict and UserDict
22
import unittest
33
import collections
4-
from test import support
4+
from test.support import get_c_recursion_limit, skip_emscripten_stack_overflow
55

66

77
class BasicTestMappingProtocol(unittest.TestCase):
@@ -622,11 +622,10 @@ def __repr__(self):
622622
d = self._full_mapping({1: BadRepr()})
623623
self.assertRaises(Exc, repr, d)
624624

625-
@support.skip_wasi_stack_overflow()
626-
@support.skip_emscripten_stack_overflow()
625+
@skip_emscripten_stack_overflow()
627626
def test_repr_deep(self):
628627
d = self._empty_mapping()
629-
for i in range(support.exceeds_recursion_limit()):
628+
for i in range(get_c_recursion_limit() + 1):
630629
d0 = d
631630
d = self._empty_mapping()
632631
d[1] = d0

‎Lib/test/pythoninfo.py

Copy file name to clipboardExpand all lines: Lib/test/pythoninfo.py
+1Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -684,6 +684,7 @@ def collect_testcapi(info_add):
684684
for name in (
685685
'LONG_MAX', # always 32-bit on Windows, 64-bit on 64-bit Unix
686686
'PY_SSIZE_T_MAX',
687+
'Py_C_RECURSION_LIMIT',
687688
'SIZEOF_TIME_T', # 32-bit or 64-bit depending on the platform
688689
'SIZEOF_WCHAR_T', # 16-bit or 32-bit depending on the platform
689690
):

‎Lib/test/support/__init__.py

Copy file name to clipboardExpand all lines: Lib/test/support/__init__.py
+11-5Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,8 @@
5656
"run_with_tz", "PGO", "missing_compiler_executable",
5757
"ALWAYS_EQ", "NEVER_EQ", "LARGEST", "SMALLEST",
5858
"LOOPBACK_TIMEOUT", "INTERNET_TIMEOUT", "SHORT_TIMEOUT", "LONG_TIMEOUT",
59-
"Py_DEBUG", "exceeds_recursion_limit", "skip_on_s390x",
59+
"Py_DEBUG", "exceeds_recursion_limit", "get_c_recursion_limit",
60+
"skip_on_s390x",
6061
"requires_jit_enabled",
6162
"requires_jit_disabled",
6263
"force_not_colorized",
@@ -557,9 +558,6 @@ def skip_android_selinux(name):
557558
def skip_emscripten_stack_overflow():
558559
return unittest.skipIf(is_emscripten, "Exhausts limited stack on Emscripten")
559560

560-
def skip_wasi_stack_overflow():
561-
return unittest.skipIf(is_wasi, "Exhausts stack on WASI")
562-
563561
is_apple_mobile = sys.platform in {"ios", "tvos", "watchos"}
564562
is_apple = is_apple_mobile or sys.platform == "darwin"
565563

@@ -2626,9 +2624,17 @@ def adjust_int_max_str_digits(max_digits):
26262624
sys.set_int_max_str_digits(current)
26272625

26282626

2627+
def get_c_recursion_limit():
2628+
try:
2629+
import _testcapi
2630+
return _testcapi.Py_C_RECURSION_LIMIT
2631+
except ImportError:
2632+
raise unittest.SkipTest('requires _testcapi')
2633+
2634+
26292635
def exceeds_recursion_limit():
26302636
"""For recursion tests, easily exceeds default recursion limit."""
2631-
return 100_000
2637+
return get_c_recursion_limit() * 3
26322638

26332639

26342640
# Windows doesn't have os.uname() but it doesn't support s390x.

‎Lib/test/test_ast/test_ast.py

Copy file name to clipboardExpand all lines: Lib/test/test_ast/test_ast.py
+11-12Lines changed: 11 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,7 @@
1818
_testinternalcapi = None
1919

2020
from test import support
21-
from test.support import os_helper, script_helper
22-
from test.support import skip_emscripten_stack_overflow, skip_wasi_stack_overflow
21+
from test.support import os_helper, script_helper, skip_emscripten_stack_overflow
2322
from test.support.ast_helper import ASTTestMixin
2423
from test.test_ast.utils import to_tuple
2524
from test.test_ast.snippets import (
@@ -751,25 +750,25 @@ def next(self):
751750
enum._test_simple_enum(_Precedence, ast._Precedence)
752751

753752
@support.cpython_only
754-
@skip_wasi_stack_overflow()
755753
@skip_emscripten_stack_overflow()
756754
def test_ast_recursion_limit(self):
757-
crash_depth = 200_000
758-
success_depth = 200
755+
fail_depth = support.exceeds_recursion_limit()
756+
crash_depth = 100_000
757+
success_depth = int(support.get_c_recursion_limit() * 0.8)
759758
if _testinternalcapi is not None:
760759
remaining = _testinternalcapi.get_c_recursion_remaining()
761760
success_depth = min(success_depth, remaining)
762761

763762
def check_limit(prefix, repeated):
764763
expect_ok = prefix + repeated * success_depth
765764
ast.parse(expect_ok)
766-
767-
broken = prefix + repeated * crash_depth
768-
details = "Compiling ({!r} + {!r} * {})".format(
769-
prefix, repeated, crash_depth)
770-
with self.assertRaises(RecursionError, msg=details):
771-
with support.infinite_recursion():
772-
ast.parse(broken)
765+
for depth in (fail_depth, crash_depth):
766+
broken = prefix + repeated * depth
767+
details = "Compiling ({!r} + {!r} * {})".format(
768+
prefix, repeated, depth)
769+
with self.assertRaises(RecursionError, msg=details):
770+
with support.infinite_recursion():
771+
ast.parse(broken)
773772

774773
check_limit("a", "()")
775774
check_limit("a", ".b")

‎Lib/test/test_call.py

Copy file name to clipboardExpand all lines: Lib/test/test_call.py
+3-3Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import unittest
22
from test.support import (cpython_only, is_wasi, requires_limited_api, Py_DEBUG,
3-
set_recursion_limit, skip_on_s390x, exceeds_recursion_limit, skip_emscripten_stack_overflow,
3+
set_recursion_limit, skip_on_s390x, skip_emscripten_stack_overflow,
44
skip_if_sanitizer, import_helper)
55
try:
66
import _testcapi
@@ -1064,10 +1064,10 @@ def c_py_recurse(m):
10641064
recurse(90_000)
10651065
with self.assertRaises(RecursionError):
10661066
recurse(101_000)
1067-
c_recurse(50)
1067+
c_recurse(100)
10681068
with self.assertRaises(RecursionError):
10691069
c_recurse(90_000)
1070-
c_py_recurse(50)
1070+
c_py_recurse(90)
10711071
with self.assertRaises(RecursionError):
10721072
c_py_recurse(100_000)
10731073

‎Lib/test/test_capi/test_misc.py

Copy file name to clipboardExpand all lines: Lib/test/test_capi/test_misc.py
+1-1Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -408,7 +408,7 @@ def test_trashcan_subclass(self):
408408
# activated when its tp_dealloc is being called by a subclass
409409
from _testcapi import MyList
410410
L = None
411-
for i in range(100):
411+
for i in range(1000):
412412
L = MyList((L,))
413413

414414
@support.requires_resource('cpu')

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.