From b11469a4d20877e07f7e11b2ee96d596692eb5d4 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Thu, 1 May 2025 23:25:19 +0200 Subject: [PATCH 01/31] GH-91048: Add utils for printing the call stack for asyncio tasks --- Lib/asyncio/tools.py | 145 ++++++++++++++++++ Lib/test/test_external_inspection.py | 10 +- Lib/test/test_sys.py | 2 +- Modules/Setup | 2 +- Modules/Setup.stdlib.in | 2 +- ...linspection.c => _remotedebuggingmodule.c} | 96 ++++++++---- ...ction.vcxproj => _remotedebugging.vcxproj} | 4 +- ...lters => _remotedebugging.vcxproj.filters} | 2 +- PCbuild/pcbuild.proj | 4 +- PCbuild/pcbuild.sln | 2 +- Tools/build/generate_stdlib_module_names.py | 2 +- configure | 40 ++--- configure.ac | 4 +- 13 files changed, 244 insertions(+), 71 deletions(-) create mode 100644 Lib/asyncio/tools.py rename Modules/{_testexternalinspection.c => _remotedebuggingmodule.c} (95%) rename PCbuild/{_testexternalinspection.vcxproj => _remotedebugging.vcxproj} (97%) rename PCbuild/{_testexternalinspection.vcxproj.filters => _remotedebugging.vcxproj.filters} (90%) diff --git a/Lib/asyncio/tools.py b/Lib/asyncio/tools.py new file mode 100644 index 00000000000000..9721a1951cfdbf --- /dev/null +++ b/Lib/asyncio/tools.py @@ -0,0 +1,145 @@ +import argparse +from collections import defaultdict +from itertools import count +from enum import Enum +import sys +from _remotedebugging import get_all_awaited_by + + +class NodeType(Enum): + COROUTINE = 1 + TASK = 2 + + +# ─── indexing helpers ─────────────────────────────────────────── +def _index(result): + id2name, awaits = {}, [] + for _thr_id, tasks in result: + for tid, tname, awaited in tasks: + id2name[tid] = tname + for stack, parent_id in awaited: + awaits.append((parent_id, stack, tid)) + return id2name, awaits + + +def _build_tree(id2name, awaits): + id2label = {(NodeType.TASK, tid): name for tid, name in id2name.items()} + children = defaultdict(list) + cor_names = defaultdict(dict) # (parent) -> {frame: node} + cor_id_seq = count(1) + + def _cor_node(parent_key, frame_name): + """Return an existing or new (NodeType.COROUTINE, …) node under *parent_key*.""" + bucket = cor_names[parent_key] + if frame_name in bucket: + return bucket[frame_name] + node_key = (NodeType.COROUTINE, f"c{next(cor_id_seq)}") + id2label[node_key] = frame_name + children[parent_key].append(node_key) + bucket[frame_name] = node_key + return node_key + + # touch every task so it’s present even if it awaits nobody + for tid in id2name: + children[(NodeType.TASK, tid)] + + # lay down parent ➜ …frames… ➜ child paths + for parent_id, stack, child_id in awaits: + cur = (NodeType.TASK, parent_id) + for frame in reversed(stack): # outer-most → inner-most + cur = _cor_node(cur, frame) + child_key = (NodeType.TASK, child_id) + if child_key not in children[cur]: + children[cur].append(child_key) + + return id2label, children + + +def _roots(id2label, children): + roots = [n for n, lbl in id2label.items() if lbl == "Task-1"] + if roots: + return roots + all_children = {c for kids in children.values() for c in kids} + return [n for n in id2label if n not in all_children] + + +# ─── PRINT TREE FUNCTION ─────────────────────────────────────── +def print_async_tree(result, task_emoji="(T)", cor_emoji="", printer=print): + """ + Pretty-print the async call tree produced by `get_all_async_stacks()`, + prefixing tasks with *task_emoji* and coroutine frames with *cor_emoji*. + """ + id2name, awaits = _index(result) + labels, children = _build_tree(id2name, awaits) + + def pretty(node): + flag = task_emoji if node[0] == NodeType.TASK else cor_emoji + return f"{flag} {labels[node]}" + + def render(node, prefix="", last=True, buf=None): + if buf is None: + buf = [] + buf.append(f"{prefix}{'└── ' if last else '├── '}{pretty(node)}") + new_pref = prefix + (" " if last else "│ ") + kids = children.get(node, []) + for i, kid in enumerate(kids): + render(kid, new_pref, i == len(kids) - 1, buf) + return buf + + result = [] + for r, root in enumerate(_roots(labels, children)): + result.append(render(root)) + return result + + +def build_task_table(result): + id2name, awaits = _index(result) + table = [] + for tid, tasks in result: + for task_id, task_name, awaited in tasks: + for stack, awaiter_id in awaited: + coroutine_chain = " -> ".join(stack) + awaiter_name = id2name.get(awaiter_id, "Unknown") + table.append( + [ + tid, + hex(task_id), + task_name, + coroutine_chain, + awaiter_name, + hex(awaiter_id), + ] + ) + + return table + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Show Python async tasks in a process") + parser.add_argument("pid", type=int, help="Process ID(s) to inspect.") + parser.add_argument( + "--tree", "-t", action="store_true", help="Display tasks in a tree format" + ) + args = parser.parse_args() + + try: + tasks = get_all_awaited_by(args.pid) + except RuntimeError as e: + print(f"Error retrieving tasks: {e}") + sys.exit(1) + + if args.tree: + # Print the async call tree + result = print_async_tree(tasks) + for tree in result: + print("\n".join(tree)) + else: + # Build and print the task table + table = build_task_table(tasks) + # Print the table in a simple tabular format + print( + f"{'tid':<10} {'task id':<20} {'task name':<20} {'coroutine chain':<50} {'awaiter name':<20} {'awaiter id':<15}" + ) + print("-" * 135) + for row in table: + print(f"{row[0]:<10} {row[1]:<20} {row[2]:<20} {row[3]:<50} {row[4]:<20} {row[5]:<15}") diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index aa05db972f068d..452b0174dfe911 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -13,13 +13,13 @@ PROCESS_VM_READV_SUPPORTED = False try: - from _testexternalinspection import PROCESS_VM_READV_SUPPORTED - from _testexternalinspection import get_stack_trace - from _testexternalinspection import get_async_stack_trace - from _testexternalinspection import get_all_awaited_by + from _remotedebuggingg import PROCESS_VM_READV_SUPPORTED + from _remotedebuggingg import get_stack_trace + from _remotedebuggingg import get_async_stack_trace + from _remotedebuggingg import get_all_awaited_by except ImportError: raise unittest.SkipTest( - "Test only runs when _testexternalinspection is available") + "Test only runs when _remotedebuggingmodule is available") def _make_test_script(script_dir, script_basename, source): to_return = make_script(script_dir, script_basename, source) diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 56413d00823f4a..10c3e0e9a1d2bb 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1960,7 +1960,7 @@ def _supports_remote_attaching(): PROCESS_VM_READV_SUPPORTED = False try: - from _testexternalinspection import PROCESS_VM_READV_SUPPORTED + from _remotedebuggingmodule import PROCESS_VM_READV_SUPPORTED except ImportError: pass diff --git a/Modules/Setup b/Modules/Setup index 65c22d48ba0bb7..530ce6d79b8918 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -286,7 +286,7 @@ PYTHONPATH=$(COREPYTHONPATH) #_testcapi _testcapimodule.c #_testimportmultiple _testimportmultiple.c #_testmultiphase _testmultiphase.c -#_testexternalinspection _testexternalinspection.c +#_remotedebuggingmodule _remotedebuggingmodule.c #_testsinglephase _testsinglephase.c # --- diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 33e60f37d19922..be4fb513e592e1 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -33,6 +33,7 @@ # Modules that should always be present (POSIX and Windows): @MODULE_ARRAY_TRUE@array arraymodule.c @MODULE__ASYNCIO_TRUE@_asyncio _asynciomodule.c +@MODULE__REMOTEDEBUGGING_TRUE@_remotedebugging _remotedebuggingmodule.c @MODULE__BISECT_TRUE@_bisect _bisectmodule.c @MODULE__CSV_TRUE@_csv _csv.c @MODULE__HEAPQ_TRUE@_heapq _heapqmodule.c @@ -186,7 +187,6 @@ @MODULE__TESTIMPORTMULTIPLE_TRUE@_testimportmultiple _testimportmultiple.c @MODULE__TESTMULTIPHASE_TRUE@_testmultiphase _testmultiphase.c @MODULE__TESTSINGLEPHASE_TRUE@_testsinglephase _testsinglephase.c -@MODULE__TESTEXTERNALINSPECTION_TRUE@_testexternalinspection _testexternalinspection.c @MODULE__CTYPES_TEST_TRUE@_ctypes_test _ctypes/_ctypes_test.c # Limited API template modules; must be built as shared modules. diff --git a/Modules/_testexternalinspection.c b/Modules/_remotedebuggingmodule.c similarity index 95% rename from Modules/_testexternalinspection.c rename to Modules/_remotedebuggingmodule.c index b65c5821443ebf..3cf14542ff0330 100644 --- a/Modules/_testexternalinspection.c +++ b/Modules/_remotedebuggingmodule.c @@ -152,9 +152,9 @@ read_char(proc_handle_t *handle, uintptr_t address, char *result) } static int -read_int(proc_handle_t *handle, uintptr_t address, int *result) +read_sized_int(proc_handle_t *handle, uintptr_t address, void *result, size_t size) { - int res = _Py_RemoteDebug_ReadRemoteMemory(handle, address, sizeof(int), result); + int res = _Py_RemoteDebug_ReadRemoteMemory(handle, address, size, result); if (res < 0) { return -1; } @@ -376,11 +376,13 @@ parse_coro_chain( } Py_DECREF(name); - int gi_frame_state; - err = read_int( + int8_t gi_frame_state; + err = read_sized_int( handle, coro_address + offsets->gen_object.gi_frame_state, - &gi_frame_state); + &gi_frame_state, + sizeof(int8_t) + ); if (err) { return -1; } @@ -470,7 +472,8 @@ parse_task_awaited_by( struct _Py_DebugOffsets* offsets, struct _Py_AsyncioModuleDebugOffsets* async_offsets, uintptr_t task_address, - PyObject *awaited_by + PyObject *awaited_by, + int recurse_task ); @@ -480,7 +483,8 @@ parse_task( struct _Py_DebugOffsets* offsets, struct _Py_AsyncioModuleDebugOffsets* async_offsets, uintptr_t task_address, - PyObject *render_to + PyObject *render_to, + int recurse_task ) { char is_task; int err = read_char( @@ -508,8 +512,13 @@ parse_task( Py_DECREF(call_stack); if (is_task) { - PyObject *tn = parse_task_name( - handle, offsets, async_offsets, task_address); + PyObject *tn = NULL; + if (recurse_task) { + tn = parse_task_name( + handle, offsets, async_offsets, task_address); + } else { + tn = PyLong_FromUnsignedLong(task_address); + } if (tn == NULL) { goto err; } @@ -550,21 +559,23 @@ parse_task( goto err; } - PyObject *awaited_by = PyList_New(0); - if (awaited_by == NULL) { - goto err; - } - if (PyList_Append(result, awaited_by)) { + if (recurse_task) { + PyObject *awaited_by = PyList_New(0); + if (awaited_by == NULL) { + goto err; + } + if (PyList_Append(result, awaited_by)) { + Py_DECREF(awaited_by); + goto err; + } + /* we can operate on a borrowed one to simplify cleanup */ Py_DECREF(awaited_by); - goto err; - } - /* we can operate on a borrowed one to simplify cleanup */ - Py_DECREF(awaited_by); - if (parse_task_awaited_by(handle, offsets, async_offsets, - task_address, awaited_by) - ) { - goto err; + if (parse_task_awaited_by(handle, offsets, async_offsets, + task_address, awaited_by, 1) + ) { + goto err; + } } Py_DECREF(result); @@ -581,7 +592,8 @@ parse_tasks_in_set( struct _Py_DebugOffsets* offsets, struct _Py_AsyncioModuleDebugOffsets* async_offsets, uintptr_t set_addr, - PyObject *awaited_by + PyObject *awaited_by, + int recurse_task ) { uintptr_t set_obj; if (read_py_ptr( @@ -642,7 +654,9 @@ parse_tasks_in_set( offsets, async_offsets, key_addr, - awaited_by) + awaited_by, + recurse_task + ) ) { return -1; } @@ -666,7 +680,8 @@ parse_task_awaited_by( struct _Py_DebugOffsets* offsets, struct _Py_AsyncioModuleDebugOffsets* async_offsets, uintptr_t task_address, - PyObject *awaited_by + PyObject *awaited_by, + int recurse_task ) { uintptr_t task_ab_addr; int err = read_py_ptr( @@ -696,7 +711,9 @@ parse_task_awaited_by( offsets, async_offsets, task_address + async_offsets->asyncio_task_object.task_awaited_by, - awaited_by) + awaited_by, + recurse_task + ) ) { return -1; } @@ -715,7 +732,9 @@ parse_task_awaited_by( offsets, async_offsets, sub_task, - awaited_by) + awaited_by, + recurse_task + ) ) { return -1; } @@ -1060,15 +1079,24 @@ append_awaited_by_for_thread( return -1; } - PyObject *result_item = PyTuple_New(2); + PyObject* task_id = PyLong_FromUnsignedLong(task_addr); + if (task_id == NULL) { + Py_DECREF(tn); + Py_DECREF(current_awaited_by); + return -1; + } + + PyObject *result_item = PyTuple_New(3); if (result_item == NULL) { Py_DECREF(tn); Py_DECREF(current_awaited_by); + Py_DECREF(task_id); return -1; } - PyTuple_SET_ITEM(result_item, 0, tn); // steals ref - PyTuple_SET_ITEM(result_item, 1, current_awaited_by); // steals ref + PyTuple_SET_ITEM(result_item, 0, task_id); // steals ref + PyTuple_SET_ITEM(result_item, 1, tn); // steals ref + PyTuple_SET_ITEM(result_item, 2, current_awaited_by); // steals ref if (PyList_Append(result, result_item)) { Py_DECREF(result_item); return -1; @@ -1076,7 +1104,7 @@ append_awaited_by_for_thread( Py_DECREF(result_item); if (parse_task_awaited_by(handle, debug_offsets, async_offsets, - task_addr, current_awaited_by)) + task_addr, current_awaited_by, 0)) { return -1; } @@ -1499,7 +1527,7 @@ get_async_stack_trace(PyObject* self, PyObject* args) if (parse_task_awaited_by( handle, &local_debug_offsets, &local_async_debug, - running_task_addr, awaited_by) + running_task_addr, awaited_by, 1) ) { goto result_err; } @@ -1526,13 +1554,13 @@ static PyMethodDef methods[] = { static struct PyModuleDef module = { .m_base = PyModuleDef_HEAD_INIT, - .m_name = "_testexternalinspection", + .m_name = "_remotedebuggingmodule", .m_size = -1, .m_methods = methods, }; PyMODINIT_FUNC -PyInit__testexternalinspection(void) +PyInit__remotedebugging(void) { PyObject* mod = PyModule_Create(&module); if (mod == NULL) { diff --git a/PCbuild/_testexternalinspection.vcxproj b/PCbuild/_remotedebugging.vcxproj similarity index 97% rename from PCbuild/_testexternalinspection.vcxproj rename to PCbuild/_remotedebugging.vcxproj index d5f347ecfec2c7..a16079f7c6c869 100644 --- a/PCbuild/_testexternalinspection.vcxproj +++ b/PCbuild/_remotedebugging.vcxproj @@ -68,7 +68,7 @@ {4D7C112F-3083-4D9E-9754-9341C14D9B39} - _testexternalinspection + _remotedebugging Win32Proj false @@ -93,7 +93,7 @@ <_ProjectFileVersion>10.0.30319.1 - + diff --git a/PCbuild/_testexternalinspection.vcxproj.filters b/PCbuild/_remotedebugging.vcxproj.filters similarity index 90% rename from PCbuild/_testexternalinspection.vcxproj.filters rename to PCbuild/_remotedebugging.vcxproj.filters index feb4343e5c2b8c..888e2cd478aa4e 100644 --- a/PCbuild/_testexternalinspection.vcxproj.filters +++ b/PCbuild/_remotedebugging.vcxproj.filters @@ -9,7 +9,7 @@ - + diff --git a/PCbuild/pcbuild.proj b/PCbuild/pcbuild.proj index 1bf430e03debc8..eec213d7bac612 100644 --- a/PCbuild/pcbuild.proj +++ b/PCbuild/pcbuild.proj @@ -66,7 +66,7 @@ - + @@ -79,7 +79,7 @@ - + diff --git a/PCbuild/pcbuild.sln b/PCbuild/pcbuild.sln index 803bb149c905cb..d2bfb9472b10ee 100644 --- a/PCbuild/pcbuild.sln +++ b/PCbuild/pcbuild.sln @@ -81,7 +81,7 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testclinic", "_testclinic. EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testinternalcapi", "_testinternalcapi.vcxproj", "{900342D7-516A-4469-B1AD-59A66E49A25F}" EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testexternalinspection", "_testexternalinspection.vcxproj", "{4D7C112F-3083-4D9E-9754-9341C14D9B39}" +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_remotedebugging", "_remotedebugging.vcxproj", "{4D7C112F-3083-4D9E-9754-9341C14D9B39}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testimportmultiple", "_testimportmultiple.vcxproj", "{36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}" EndProject diff --git a/Tools/build/generate_stdlib_module_names.py b/Tools/build/generate_stdlib_module_names.py index 9873890837fa8e..8feb6c317d343e 100644 --- a/Tools/build/generate_stdlib_module_names.py +++ b/Tools/build/generate_stdlib_module_names.py @@ -34,7 +34,7 @@ '_testlimitedcapi', '_testmultiphase', '_testsinglephase', - '_testexternalinspection', + '_remotedebuggingmodule', '_xxtestfuzz', 'idlelib.idle_test', 'test', diff --git a/configure b/configure index 7dbb35f9f45f4b..82e699c745f7b9 100755 --- a/configure +++ b/configure @@ -654,8 +654,8 @@ MODULE__XXTESTFUZZ_FALSE MODULE__XXTESTFUZZ_TRUE MODULE_XXSUBTYPE_FALSE MODULE_XXSUBTYPE_TRUE -MODULE__TESTEXTERNALINSPECTION_FALSE -MODULE__TESTEXTERNALINSPECTION_TRUE +MODULE__REMOTEDEBUGGING_FALSE +MODULE__REMOTEDEBUGGING_TRUE MODULE__TESTSINGLEPHASE_FALSE MODULE__TESTSINGLEPHASE_TRUE MODULE__TESTMULTIPHASE_FALSE @@ -30684,7 +30684,7 @@ case $ac_sys_system in #( py_cv_module__ctypes_test=n/a - py_cv_module__testexternalinspection=n/a + py_cv_module__remotedebuggingmodule=n/a py_cv_module__testimportmultiple=n/a py_cv_module__testmultiphase=n/a py_cv_module__testsinglephase=n/a @@ -33449,44 +33449,44 @@ fi printf "%s\n" "$py_cv_module__testsinglephase" >&6; } - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _testexternalinspection" >&5 -printf %s "checking for stdlib extension module _testexternalinspection... " >&6; } - if test "$py_cv_module__testexternalinspection" != "n/a" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _remotedebuggingmodule" >&5 +printf %s "checking for stdlib extension module _remotedebuggingmodule... " >&6; } + if test "$py_cv_module__remotedebuggingmodule" != "n/a" then : if test "$TEST_MODULES" = yes then : if true then : - py_cv_module__testexternalinspection=yes + py_cv_module__remotedebuggingmodule=yes else case e in #( - e) py_cv_module__testexternalinspection=missing ;; + e) py_cv_module__remotedebuggingmodule=missing ;; esac fi else case e in #( - e) py_cv_module__testexternalinspection=disabled ;; + e) py_cv_module__remotedebuggingmodule=disabled ;; esac fi fi - as_fn_append MODULE_BLOCK "MODULE__TESTEXTERNALINSPECTION_STATE=$py_cv_module__testexternalinspection$as_nl" - if test "x$py_cv_module__testexternalinspection" = xyes + as_fn_append MODULE_BLOCK "MODULE__REMOTEDEBUGGING_STATE=$py_cv_module__remotedebuggingmodule$as_nl" + if test "x$py_cv_module__remotedebuggingmodule" = xyes then : fi - if test "$py_cv_module__testexternalinspection" = yes; then - MODULE__TESTEXTERNALINSPECTION_TRUE= - MODULE__TESTEXTERNALINSPECTION_FALSE='#' + if test "$py_cv_module__remotedebuggingmodule" = yes; then + MODULE__REMOTEDEBUGGING_TRUE= + MODULE__REMOTEDEBUGGING_FALSE='#' else - MODULE__TESTEXTERNALINSPECTION_TRUE='#' - MODULE__TESTEXTERNALINSPECTION_FALSE= + MODULE__REMOTEDEBUGGING_TRUE='#' + MODULE__REMOTEDEBUGGING_FALSE= fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $py_cv_module__testexternalinspection" >&5 -printf "%s\n" "$py_cv_module__testexternalinspection" >&6; } + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $py_cv_module__remotedebuggingmodule" >&5 +printf "%s\n" "$py_cv_module__remotedebuggingmodule" >&6; } { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module xxsubtype" >&5 @@ -34119,8 +34119,8 @@ if test -z "${MODULE__TESTSINGLEPHASE_TRUE}" && test -z "${MODULE__TESTSINGLEPHA as_fn_error $? "conditional \"MODULE__TESTSINGLEPHASE\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi -if test -z "${MODULE__TESTEXTERNALINSPECTION_TRUE}" && test -z "${MODULE__TESTEXTERNALINSPECTION_FALSE}"; then - as_fn_error $? "conditional \"MODULE__TESTEXTERNALINSPECTION\" was never defined. +if test -z "${MODULE__REMOTEDEBUGGING_TRUE}" && test -z "${MODULE__REMOTEDEBUGGING_FALSE}"; then + as_fn_error $? "conditional \"MODULE__REMOTEDEBUGGING\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi if test -z "${MODULE_XXSUBTYPE_TRUE}" && test -z "${MODULE_XXSUBTYPE_FALSE}"; then diff --git a/configure.ac b/configure.ac index 65f265045ba318..fa538da673f770 100644 --- a/configure.ac +++ b/configure.ac @@ -7720,7 +7720,7 @@ AS_CASE([$ac_sys_system], dnl (see Modules/Setup.stdlib.in). PY_STDLIB_MOD_SET_NA( [_ctypes_test], - [_testexternalinspection], + [_remotedebuggingmodule], [_testimportmultiple], [_testmultiphase], [_testsinglephase], @@ -8082,7 +8082,7 @@ PY_STDLIB_MOD([_testbuffer], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testimportmultiple], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) PY_STDLIB_MOD([_testmultiphase], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) PY_STDLIB_MOD([_testsinglephase], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) -PY_STDLIB_MOD([_testexternalinspection], [test "$TEST_MODULES" = yes]) +PY_STDLIB_MOD([_remotedebuggingmodule], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([xxsubtype], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_xxtestfuzz], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_ctypes_test], From 7f800e8f50ac6d61ba4ab400f9760d54ffa58d40 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 2 May 2025 14:16:16 +0200 Subject: [PATCH 02/31] Maybe --- Lib/asyncio/tools.py | 105 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 95 insertions(+), 10 deletions(-) diff --git a/Lib/asyncio/tools.py b/Lib/asyncio/tools.py index 9721a1951cfdbf..492a11eada0b3f 100644 --- a/Lib/asyncio/tools.py +++ b/Lib/asyncio/tools.py @@ -56,18 +56,93 @@ def _cor_node(parent_key, frame_name): def _roots(id2label, children): - roots = [n for n, lbl in id2label.items() if lbl == "Task-1"] - if roots: - return roots + """Return every task that is *not awaited by anybody*.""" all_children = {c for kids in children.values() for c in kids} return [n for n in id2label if n not in all_children] +# ─── helpers for _roots() ───────────────────────────────────────── +from collections import defaultdict + +def _roots(id2label, children): + """ + Return one root per *source* strongly-connected component (SCC). + + • Build the graph that contains **only tasks** as nodes and edges + parent-task ─▶ child-task (ignore coroutine frames). + + • Collapse it into SCCs with Tarjan (linear time). + + • For every component whose condensation-DAG in-degree is 0, choose a + stable representative (lexicographically-smallest label, fallback to + smallest object-id) and return that list. + """ + TASK = NodeType.TASK + task_nodes = [n for n in id2label if n[0] == TASK] + + # ------------ adjacency list among *tasks only* ----------------- + adj = defaultdict(list) + for p in task_nodes: + adj[p] = [c for c in children.get(p, []) if c[0] == TASK] + + # ------------ Tarjan’s algorithm -------------------------------- + index = 0 + stack, on_stack = [], set() + node_index, low = {}, {} + comp_of = {} # node -> comp-id + comps = defaultdict(list) # comp-id -> [members] + + def strong(v): + nonlocal index + node_index[v] = low[v] = index + index += 1 + stack.append(v) + on_stack.add(v) + + for w in adj[v]: + if w not in node_index: + strong(w) + low[v] = min(low[v], low[w]) + elif w in on_stack: + low[v] = min(low[v], node_index[w]) + + if low[v] == node_index[v]: # root of an SCC + while True: + w = stack.pop() + on_stack.remove(w) + comp_of[w] = v # use root-node as comp-id + comps[v].append(w) + if w == v: + break + + for v in task_nodes: + if v not in node_index: + strong(v) + + # ------------ condensation DAG in-degrees ----------------------- + indeg = defaultdict(int) + for p in task_nodes: + cp = comp_of[p] + for q in adj[p]: + cq = comp_of[q] + if cp != cq: + indeg[cq] += 1 + + # ------------ choose one representative per source-SCC ---------- + roots = [] + for cid, members in comps.items(): + if indeg[cid] == 0: # source component + roots.append(min( + members, + key=lambda n: (id2label[n], n[1]) # stable pick + )) + return roots + # ─── PRINT TREE FUNCTION ─────────────────────────────────────── def print_async_tree(result, task_emoji="(T)", cor_emoji="", printer=print): """ Pretty-print the async call tree produced by `get_all_async_stacks()`, - prefixing tasks with *task_emoji* and coroutine frames with *cor_emoji*. + coping safely with cycles. """ id2name, awaits = _index(result) labels, children = _build_tree(id2name, awaits) @@ -76,20 +151,29 @@ def pretty(node): flag = task_emoji if node[0] == NodeType.TASK else cor_emoji return f"{flag} {labels[node]}" - def render(node, prefix="", last=True, buf=None): + def render(node, prefix="", last=True, buf=None, ancestry=frozenset()): + """ + DFS renderer that stops if *node* already occurs in *ancestry* + (i.e. we just found a cycle). + """ if buf is None: buf = [] + + if node in ancestry: + buf.append(f"{prefix}{'└── ' if last else '├── '}↺ {pretty(node)} (cycle)") + return buf + buf.append(f"{prefix}{'└── ' if last else '├── '}{pretty(node)}") new_pref = prefix + (" " if last else "│ ") kids = children.get(node, []) for i, kid in enumerate(kids): - render(kid, new_pref, i == len(kids) - 1, buf) + render(kid, new_pref, i == len(kids) - 1, buf, ancestry | {node}) return buf - result = [] - for r, root in enumerate(_roots(labels, children)): - result.append(render(root)) - return result + forest = [] + for root in _roots(labels, children): + forest.append(render(root)) + return forest def build_task_table(result): @@ -124,6 +208,7 @@ def build_task_table(result): try: tasks = get_all_awaited_by(args.pid) + print(tasks) except RuntimeError as e: print(f"Error retrieving tasks: {e}") sys.exit(1) From c5e4efe5d3bdb38d749e8886c80cc628712dc81e Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 2 May 2025 14:27:57 +0200 Subject: [PATCH 03/31] Maybe --- Lib/asyncio/tools.py | 148 ++++++++++++++++++++++--------------------- 1 file changed, 75 insertions(+), 73 deletions(-) diff --git a/Lib/asyncio/tools.py b/Lib/asyncio/tools.py index 492a11eada0b3f..f25156ad339ea4 100644 --- a/Lib/asyncio/tools.py +++ b/Lib/asyncio/tools.py @@ -56,89 +56,80 @@ def _cor_node(parent_key, frame_name): def _roots(id2label, children): - """Return every task that is *not awaited by anybody*.""" + roots = [n for n, lbl in id2label.items() if lbl == "Task-1"] + if roots: + return roots all_children = {c for kids in children.values() for c in kids} return [n for n in id2label if n not in all_children] -# ─── helpers for _roots() ───────────────────────────────────────── -from collections import defaultdict +# ─── detect cycles in the task-to-task graph ─────────────────────── +def _task_graph(awaits): + """Return {parent_task_id: {child_task_id, …}, …}.""" + from collections import defaultdict + g = defaultdict(set) + for parent_id, _stack, child_id in awaits: + g[parent_id].add(child_id) + return g -def _roots(id2label, children): + +def _find_cycles(graph): """ - Return one root per *source* strongly-connected component (SCC). + Depth-first search for back-edges. - • Build the graph that contains **only tasks** as nodes and edges - parent-task ─▶ child-task (ignore coroutine frames). + Returns a list of cycles (each cycle is a list of task-ids) or an + empty list if the graph is acyclic. + """ + WHITE, GREY, BLACK = 0, 1, 2 + color = {n: WHITE for n in graph} + path, cycles = [], [] + + def dfs(v): + color[v] = GREY + path.append(v) + for w in graph.get(v, ()): + if color[w] == WHITE: + dfs(w) + elif color[w] == GREY: # back-edge → cycle! + i = path.index(w) + cycles.append(path[i:] + [w]) # make a copy + color[v] = BLACK + path.pop() + + for v in list(graph): + if color[v] == WHITE: + dfs(v) + return cycles - • Collapse it into SCCs with Tarjan (linear time). - • For every component whose condensation-DAG in-degree is 0, choose a - stable representative (lexicographically-smallest label, fallback to - smallest object-id) and return that list. +# ─── PRINT TREE FUNCTION ─────────────────────────────────────── +def print_async_tree(result, task_emoji="(T)", cor_emoji="", printer=print): """ - TASK = NodeType.TASK - task_nodes = [n for n in id2label if n[0] == TASK] - - # ------------ adjacency list among *tasks only* ----------------- - adj = defaultdict(list) - for p in task_nodes: - adj[p] = [c for c in children.get(p, []) if c[0] == TASK] - - # ------------ Tarjan’s algorithm -------------------------------- - index = 0 - stack, on_stack = [], set() - node_index, low = {}, {} - comp_of = {} # node -> comp-id - comps = defaultdict(list) # comp-id -> [members] - - def strong(v): - nonlocal index - node_index[v] = low[v] = index - index += 1 - stack.append(v) - on_stack.add(v) - - for w in adj[v]: - if w not in node_index: - strong(w) - low[v] = min(low[v], low[w]) - elif w in on_stack: - low[v] = min(low[v], node_index[w]) - - if low[v] == node_index[v]: # root of an SCC - while True: - w = stack.pop() - on_stack.remove(w) - comp_of[w] = v # use root-node as comp-id - comps[v].append(w) - if w == v: - break - - for v in task_nodes: - if v not in node_index: - strong(v) - - # ------------ condensation DAG in-degrees ----------------------- - indeg = defaultdict(int) - for p in task_nodes: - cp = comp_of[p] - for q in adj[p]: - cq = comp_of[q] - if cp != cq: - indeg[cq] += 1 - - # ------------ choose one representative per source-SCC ---------- - roots = [] - for cid, members in comps.items(): - if indeg[cid] == 0: # source component - roots.append(min( - members, - key=lambda n: (id2label[n], n[1]) # stable pick - )) - return roots + Pretty-print the async call tree produced by `get_all_async_stacks()`, + prefixing tasks with *task_emoji* and coroutine frames with *cor_emoji*. + """ + id2name, awaits = _index(result) + labels, children = _build_tree(id2name, awaits) + + def pretty(node): + flag = task_emoji if node[0] == NodeType.TASK else cor_emoji + return f"{flag} {labels[node]}" + + def render(node, prefix="", last=True, buf=None): + if buf is None: + buf = [] + buf.append(f"{prefix}{'└── ' if last else '├── '}{pretty(node)}") + new_pref = prefix + (" " if last else "│ ") + kids = children.get(node, []) + for i, kid in enumerate(kids): + render(kid, new_pref, i == len(kids) - 1, buf) + return buf + + result = [] + for r, root in enumerate(_roots(labels, children)): + result.append(render(root)) + return result -# ─── PRINT TREE FUNCTION ─────────────────────────────────────── def print_async_tree(result, task_emoji="(T)", cor_emoji="", printer=print): """ Pretty-print the async call tree produced by `get_all_async_stacks()`, @@ -208,13 +199,24 @@ def build_task_table(result): try: tasks = get_all_awaited_by(args.pid) - print(tasks) except RuntimeError as e: print(f"Error retrieving tasks: {e}") sys.exit(1) if args.tree: # Print the async call tree + id2name, awaits = _index(tasks) + g = _task_graph(awaits) + cycles = _find_cycles(g) + + if cycles: + print("ERROR: await-graph contains cycles – cannot print a tree!\n") + for c in cycles: + # pretty-print task names instead of bare ids + names = " → ".join(id2name.get(tid, hex(tid)) for tid in c) + print(f" cycle: {names}") + sys.exit(1) + result = print_async_tree(tasks) for tree in result: print("\n".join(tree)) From 1c982b1f98d6b2da5ad597ec80845c0c2d436c93 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 2 May 2025 14:31:35 +0200 Subject: [PATCH 04/31] Maybe --- Lib/asyncio/tools.py | 37 ------------------------------------- 1 file changed, 37 deletions(-) diff --git a/Lib/asyncio/tools.py b/Lib/asyncio/tools.py index f25156ad339ea4..8bf24d169dbcbb 100644 --- a/Lib/asyncio/tools.py +++ b/Lib/asyncio/tools.py @@ -130,43 +130,6 @@ def render(node, prefix="", last=True, buf=None): return result -def print_async_tree(result, task_emoji="(T)", cor_emoji="", printer=print): - """ - Pretty-print the async call tree produced by `get_all_async_stacks()`, - coping safely with cycles. - """ - id2name, awaits = _index(result) - labels, children = _build_tree(id2name, awaits) - - def pretty(node): - flag = task_emoji if node[0] == NodeType.TASK else cor_emoji - return f"{flag} {labels[node]}" - - def render(node, prefix="", last=True, buf=None, ancestry=frozenset()): - """ - DFS renderer that stops if *node* already occurs in *ancestry* - (i.e. we just found a cycle). - """ - if buf is None: - buf = [] - - if node in ancestry: - buf.append(f"{prefix}{'└── ' if last else '├── '}↺ {pretty(node)} (cycle)") - return buf - - buf.append(f"{prefix}{'└── ' if last else '├── '}{pretty(node)}") - new_pref = prefix + (" " if last else "│ ") - kids = children.get(node, []) - for i, kid in enumerate(kids): - render(kid, new_pref, i == len(kids) - 1, buf, ancestry | {node}) - return buf - - forest = [] - for root in _roots(labels, children): - forest.append(render(root)) - return forest - - def build_task_table(result): id2name, awaits = _index(result) table = [] From 2d94cde8902a35cdb7ba9e827e1ba0fd9bb7e70a Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 2 May 2025 19:53:26 +0200 Subject: [PATCH 05/31] fix configure --- configure | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) mode change 100755 => 100644 configure diff --git a/configure b/configure old mode 100755 new mode 100644 index 82e699c745f7b9..2c77775099c099 --- a/configure +++ b/configure @@ -654,8 +654,8 @@ MODULE__XXTESTFUZZ_FALSE MODULE__XXTESTFUZZ_TRUE MODULE_XXSUBTYPE_FALSE MODULE_XXSUBTYPE_TRUE -MODULE__REMOTEDEBUGGING_FALSE -MODULE__REMOTEDEBUGGING_TRUE +MODULE__REMOTEDEBUGGINGMODULE_FALSE +MODULE__REMOTEDEBUGGINGMODULE_TRUE MODULE__TESTSINGLEPHASE_FALSE MODULE__TESTSINGLEPHASE_TRUE MODULE__TESTMULTIPHASE_FALSE @@ -33469,7 +33469,7 @@ esac fi fi - as_fn_append MODULE_BLOCK "MODULE__REMOTEDEBUGGING_STATE=$py_cv_module__remotedebuggingmodule$as_nl" + as_fn_append MODULE_BLOCK "MODULE__REMOTEDEBUGGINGMODULE_STATE=$py_cv_module__remotedebuggingmodule$as_nl" if test "x$py_cv_module__remotedebuggingmodule" = xyes then : @@ -33478,11 +33478,11 @@ then : fi if test "$py_cv_module__remotedebuggingmodule" = yes; then - MODULE__REMOTEDEBUGGING_TRUE= - MODULE__REMOTEDEBUGGING_FALSE='#' + MODULE__REMOTEDEBUGGINGMODULE_TRUE= + MODULE__REMOTEDEBUGGINGMODULE_FALSE='#' else - MODULE__REMOTEDEBUGGING_TRUE='#' - MODULE__REMOTEDEBUGGING_FALSE= + MODULE__REMOTEDEBUGGINGMODULE_TRUE='#' + MODULE__REMOTEDEBUGGINGMODULE_FALSE= fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $py_cv_module__remotedebuggingmodule" >&5 @@ -34119,8 +34119,8 @@ if test -z "${MODULE__TESTSINGLEPHASE_TRUE}" && test -z "${MODULE__TESTSINGLEPHA as_fn_error $? "conditional \"MODULE__TESTSINGLEPHASE\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi -if test -z "${MODULE__REMOTEDEBUGGING_TRUE}" && test -z "${MODULE__REMOTEDEBUGGING_FALSE}"; then - as_fn_error $? "conditional \"MODULE__REMOTEDEBUGGING\" was never defined. +if test -z "${MODULE__REMOTEDEBUGGINGMODULE_TRUE}" && test -z "${MODULE__REMOTEDEBUGGINGMODULE_FALSE}"; then + as_fn_error $? "conditional \"MODULE__REMOTEDEBUGGINGMODULE\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi if test -z "${MODULE_XXSUBTYPE_TRUE}" && test -z "${MODULE_XXSUBTYPE_FALSE}"; then From 0a9a49628177f078768968ddd0985358aaf9a725 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 2 May 2025 20:02:39 +0200 Subject: [PATCH 06/31] fix configure --- Modules/Setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/Setup b/Modules/Setup index 530ce6d79b8918..c3e0d9eb9344a9 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -286,7 +286,7 @@ PYTHONPATH=$(COREPYTHONPATH) #_testcapi _testcapimodule.c #_testimportmultiple _testimportmultiple.c #_testmultiphase _testmultiphase.c -#_remotedebuggingmodule _remotedebuggingmodule.c +#_remotedebugging _remotedebuggingmodule.c #_testsinglephase _testsinglephase.c # --- From db47ff39f47bef43d4a7206ccf1160f99271d2a7 Mon Sep 17 00:00:00 2001 From: Marta Gomez Macias Date: Fri, 2 May 2025 21:03:39 +0000 Subject: [PATCH 07/31] fix configure --- Lib/test/test_external_inspection.py | 8 ++++---- Modules/_remotedebuggingmodule.c | 2 +- Tools/build/generate_stdlib_module_names.py | 2 +- configure | 19 +++++++++---------- configure.ac | 4 ++-- 5 files changed, 17 insertions(+), 18 deletions(-) mode change 100644 => 100755 configure diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index 452b0174dfe911..3535eb306e1958 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -13,10 +13,10 @@ PROCESS_VM_READV_SUPPORTED = False try: - from _remotedebuggingg import PROCESS_VM_READV_SUPPORTED - from _remotedebuggingg import get_stack_trace - from _remotedebuggingg import get_async_stack_trace - from _remotedebuggingg import get_all_awaited_by + from _remotedebugging import PROCESS_VM_READV_SUPPORTED + from _remotedebugging import get_stack_trace + from _remotedebugging import get_async_stack_trace + from _remotedebugging import get_all_awaited_by except ImportError: raise unittest.SkipTest( "Test only runs when _remotedebuggingmodule is available") diff --git a/Modules/_remotedebuggingmodule.c b/Modules/_remotedebuggingmodule.c index 3cf14542ff0330..e027ffb28d316f 100644 --- a/Modules/_remotedebuggingmodule.c +++ b/Modules/_remotedebuggingmodule.c @@ -1554,7 +1554,7 @@ static PyMethodDef methods[] = { static struct PyModuleDef module = { .m_base = PyModuleDef_HEAD_INIT, - .m_name = "_remotedebuggingmodule", + .m_name = "_remotedebugging", .m_size = -1, .m_methods = methods, }; diff --git a/Tools/build/generate_stdlib_module_names.py b/Tools/build/generate_stdlib_module_names.py index 8feb6c317d343e..761eecba96f291 100644 --- a/Tools/build/generate_stdlib_module_names.py +++ b/Tools/build/generate_stdlib_module_names.py @@ -34,7 +34,7 @@ '_testlimitedcapi', '_testmultiphase', '_testsinglephase', - '_remotedebuggingmodule', + '_remotedebugging', '_xxtestfuzz', 'idlelib.idle_test', 'test', diff --git a/configure b/configure old mode 100644 new mode 100755 index 2c77775099c099..cea1c45ae33484 --- a/configure +++ b/configure @@ -654,8 +654,8 @@ MODULE__XXTESTFUZZ_FALSE MODULE__XXTESTFUZZ_TRUE MODULE_XXSUBTYPE_FALSE MODULE_XXSUBTYPE_TRUE -MODULE__REMOTEDEBUGGINGMODULE_FALSE -MODULE__REMOTEDEBUGGINGMODULE_TRUE +MODULE__REMOTEDEBUGGING_FALSE +MODULE__REMOTEDEBUGGING_TRUE MODULE__TESTSINGLEPHASE_FALSE MODULE__TESTSINGLEPHASE_TRUE MODULE__TESTMULTIPHASE_FALSE @@ -33469,7 +33469,7 @@ esac fi fi - as_fn_append MODULE_BLOCK "MODULE__REMOTEDEBUGGINGMODULE_STATE=$py_cv_module__remotedebuggingmodule$as_nl" + as_fn_append MODULE_BLOCK "MODULE__REMOTEDEBUGGING_STATE=$py_cv_module__remotedebuggingmodule$as_nl" if test "x$py_cv_module__remotedebuggingmodule" = xyes then : @@ -33478,11 +33478,11 @@ then : fi if test "$py_cv_module__remotedebuggingmodule" = yes; then - MODULE__REMOTEDEBUGGINGMODULE_TRUE= - MODULE__REMOTEDEBUGGINGMODULE_FALSE='#' + MODULE__REMOTEDEBUGGING_TRUE= + MODULE__REMOTEDEBUGGING_FALSE='#' else - MODULE__REMOTEDEBUGGINGMODULE_TRUE='#' - MODULE__REMOTEDEBUGGINGMODULE_FALSE= + MODULE__REMOTEDEBUGGING_TRUE='#' + MODULE__REMOTEDEBUGGING_FALSE= fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $py_cv_module__remotedebuggingmodule" >&5 @@ -34119,8 +34119,8 @@ if test -z "${MODULE__TESTSINGLEPHASE_TRUE}" && test -z "${MODULE__TESTSINGLEPHA as_fn_error $? "conditional \"MODULE__TESTSINGLEPHASE\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi -if test -z "${MODULE__REMOTEDEBUGGINGMODULE_TRUE}" && test -z "${MODULE__REMOTEDEBUGGINGMODULE_FALSE}"; then - as_fn_error $? "conditional \"MODULE__REMOTEDEBUGGINGMODULE\" was never defined. +if test -z "${MODULE__REMOTEDEBUGGING_TRUE}" && test -z "${MODULE__REMOTEDEBUGGING_FALSE}"; then + as_fn_error $? "conditional \"MODULE__REMOTEDEBUGGING\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi if test -z "${MODULE_XXSUBTYPE_TRUE}" && test -z "${MODULE_XXSUBTYPE_FALSE}"; then @@ -35388,4 +35388,3 @@ if test "$ac_cv_header_stdatomic_h" != "yes"; then { printf "%s\n" "$as_me:${as_lineno-$LINENO}: Your compiler or platform does have a working C11 stdatomic.h. A future version of Python may require stdatomic.h." >&5 printf "%s\n" "$as_me: Your compiler or platform does have a working C11 stdatomic.h. A future version of Python may require stdatomic.h." >&6;} fi - diff --git a/configure.ac b/configure.ac index fa538da673f770..ed5c65ecbcc2be 100644 --- a/configure.ac +++ b/configure.ac @@ -7720,7 +7720,7 @@ AS_CASE([$ac_sys_system], dnl (see Modules/Setup.stdlib.in). PY_STDLIB_MOD_SET_NA( [_ctypes_test], - [_remotedebuggingmodule], + [_remotedebugging], [_testimportmultiple], [_testmultiphase], [_testsinglephase], @@ -8082,7 +8082,7 @@ PY_STDLIB_MOD([_testbuffer], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testimportmultiple], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) PY_STDLIB_MOD([_testmultiphase], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) PY_STDLIB_MOD([_testsinglephase], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) -PY_STDLIB_MOD([_remotedebuggingmodule], [test "$TEST_MODULES" = yes]) +PY_STDLIB_MOD([_remotedebugging], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([xxsubtype], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_xxtestfuzz], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_ctypes_test], From 6f8bd4c79beee4a845469b28227178b1f628e061 Mon Sep 17 00:00:00 2001 From: Marta Gomez Macias Date: Fri, 2 May 2025 21:23:04 +0000 Subject: [PATCH 08/31] some tests + fixes --- Lib/asyncio/tools.py | 35 ++- Lib/test/test_asyncio/test_tools.py | 382 ++++++++++++++++++++++++++++ 2 files changed, 404 insertions(+), 13 deletions(-) create mode 100644 Lib/test/test_asyncio/test_tools.py diff --git a/Lib/asyncio/tools.py b/Lib/asyncio/tools.py index 8bf24d169dbcbb..1228c787b0980e 100644 --- a/Lib/asyncio/tools.py +++ b/Lib/asyncio/tools.py @@ -11,6 +11,23 @@ class NodeType(Enum): TASK = 2 +class CycleFoundException(Exception): + """Raised when there is a cycle when drawing the call tree.""" + + def __init__(self, cycles, id2name): + super().__init__() + self.cycles = cycles + self.id2name = id2name + + def __str__(self): + for c in self.cycles: + names = " → ".join(self.id2name.get(tid, hex(tid)) for tid in c) + return ( + "ERROR: await-graph contains cycles – cannot print a tree!\n" + f"cycle: {names}" + ) + + # ─── indexing helpers ─────────────────────────────────────────── def _index(result): id2name, awaits = {}, [] @@ -80,7 +97,7 @@ def _find_cycles(graph): empty list if the graph is acyclic. """ WHITE, GREY, BLACK = 0, 1, 2 - color = {n: WHITE for n in graph} + color = defaultdict(lambda: WHITE) path, cycles = [], [] def dfs(v): @@ -108,6 +125,10 @@ def print_async_tree(result, task_emoji="(T)", cor_emoji="", printer=print): prefixing tasks with *task_emoji* and coroutine frames with *cor_emoji*. """ id2name, awaits = _index(result) + g = _task_graph(awaits) + cycles = _find_cycles(g) + if cycles: + raise CycleFoundException(cycles, id2name) labels, children = _build_tree(id2name, awaits) def pretty(node): @@ -168,18 +189,6 @@ def build_task_table(result): if args.tree: # Print the async call tree - id2name, awaits = _index(tasks) - g = _task_graph(awaits) - cycles = _find_cycles(g) - - if cycles: - print("ERROR: await-graph contains cycles – cannot print a tree!\n") - for c in cycles: - # pretty-print task names instead of bare ids - names = " → ".join(id2name.get(tid, hex(tid)) for tid in c) - print(f" cycle: {names}") - sys.exit(1) - result = print_async_tree(tasks) for tree in result: print("\n".join(tree)) diff --git a/Lib/test/test_asyncio/test_tools.py b/Lib/test/test_asyncio/test_tools.py new file mode 100644 index 00000000000000..149c6db6d5ba71 --- /dev/null +++ b/Lib/test/test_asyncio/test_tools.py @@ -0,0 +1,382 @@ +"""Tests for the asyncio tools script.""" + +from Lib.asyncio import tools +from test.test_asyncio import utils as test_utils + + +# mock output of get_all_awaited_by function. +TEST_INPUTS = [ + [ + # test case containing a task called timer being awaited in two + # different subtasks part of a TaskGroup (root1 and root2) which call + # awaiter functions. + ( + ( + 1, + [ + (2, "Task-1", []), + ( + 3, + "timer", + [ + [["awaiter3", "awaiter2", "awaiter"], 4], + [["awaiter1_3", "awaiter1_2", "awaiter1"], 5], + [["awaiter1_3", "awaiter1_2", "awaiter1"], 6], + [["awaiter3", "awaiter2", "awaiter"], 7], + ], + ), + ( + 8, + "root1", + [[["_aexit", "__aexit__", "main"], 2]], + ), + ( + 9, + "root2", + [[["_aexit", "__aexit__", "main"], 2]], + ), + ( + 4, + "child1_1", + [ + [ + ["_aexit", "__aexit__", "blocho_caller", "bloch"], + 8, + ] + ], + ), + ( + 6, + "child2_1", + [ + [ + ["_aexit", "__aexit__", "blocho_caller", "bloch"], + 8, + ] + ], + ), + ( + 7, + "child1_2", + [ + [ + ["_aexit", "__aexit__", "blocho_caller", "bloch"], + 9, + ] + ], + ), + ( + 5, + "child2_2", + [ + [ + ["_aexit", "__aexit__", "blocho_caller", "bloch"], + 9, + ] + ], + ), + ], + ), + (0, []), + ), + ([]), + ( + [ + [ + "└── (T) Task-1", + " └── main", + " └── __aexit__", + " └── _aexit", + " ├── (T) root1", + " │ └── bloch", + " │ └── blocho_caller", + " │ └── __aexit__", + " │ └── _aexit", + " │ ├── (T) child1_1", + " │ │ └── awaiter", + " │ │ └── awaiter2", + " │ │ └── awaiter3", + " │ │ └── (T) timer", + " │ └── (T) child2_1", + " │ └── awaiter1", + " │ └── awaiter1_2", + " │ └── awaiter1_3", + " │ └── (T) timer", + " └── (T) root2", + " └── bloch", + " └── blocho_caller", + " └── __aexit__", + " └── _aexit", + " ├── (T) child1_2", + " │ └── awaiter", + " │ └── awaiter2", + " │ └── awaiter3", + " │ └── (T) timer", + " └── (T) child2_2", + " └── awaiter1", + " └── awaiter1_2", + " └── awaiter1_3", + " └── (T) timer", + ] + ] + ), + ( + [ + [ + 1, + "0x3", + "timer", + "awaiter3 -> awaiter2 -> awaiter", + "child1_1", + "0x4", + ], + [ + 1, + "0x3", + "timer", + "awaiter1_3 -> awaiter1_2 -> awaiter1", + "child2_2", + "0x5", + ], + [ + 1, + "0x3", + "timer", + "awaiter1_3 -> awaiter1_2 -> awaiter1", + "child2_1", + "0x6", + ], + [ + 1, + "0x3", + "timer", + "awaiter3 -> awaiter2 -> awaiter", + "child1_2", + "0x7", + ], + [ + 1, + "0x8", + "root1", + "_aexit -> __aexit__ -> main", + "Task-1", + "0x2", + ], + [ + 1, + "0x9", + "root2", + "_aexit -> __aexit__ -> main", + "Task-1", + "0x2", + ], + [ + 1, + "0x4", + "child1_1", + "_aexit -> __aexit__ -> blocho_caller -> bloch", + "root1", + "0x8", + ], + [ + 1, + "0x6", + "child2_1", + "_aexit -> __aexit__ -> blocho_caller -> bloch", + "root1", + "0x8", + ], + [ + 1, + "0x7", + "child1_2", + "_aexit -> __aexit__ -> blocho_caller -> bloch", + "root2", + "0x9", + ], + [ + 1, + "0x5", + "child2_2", + "_aexit -> __aexit__ -> blocho_caller -> bloch", + "root2", + "0x9", + ], + ] + ), + ], + [ + # test case containing two roots + ( + ( + 1, + [ + (2, "Task-1", []), + (3, "timer", [[["awaiter"], 4]]), + (4, "subtask1", [[["main"], 2]]), + ], + ), + ( + 5, + [ + (6, "Task-1", []), + (7, "sleeper", [[["awaiter2"], 8]]), + (8, "subtask2", [[["main"], 6]]), + ], + ), + (0, []), + ), + ([]), + ( + [ + [ + "└── (T) Task-1", + " └── main", + " └── (T) subtask1", + " └── awaiter", + " └── (T) timer", + ], + [ + "└── (T) Task-1", + " └── main", + " └── (T) subtask2", + " └── awaiter2", + " └── (T) sleeper", + ], + ] + ), + ( + [ + [1, "0x3", "timer", "awaiter", "subtask1", "0x4"], + [1, "0x4", "subtask1", "main", "Task-1", "0x2"], + [5, "0x7", "sleeper", "awaiter2", "subtask2", "0x8"], + [5, "0x8", "subtask2", "main", "Task-1", "0x6"], + ] + ), + ], + # Tests cycle detection. + [ + # this test case contains a cycle: two tasks awaiting each other. + ( + [ + ( + 1, + [ + (2, "Task-1", []), + ( + 3, + "a", + [[["awaiter2"], 4], [["main"], 2]], + ), + (4, "b", [[["awaiter"], 3]]), + ], + ), + (0, []), + ] + ), + ([[4, 3, 4]]), + ([]), + ( + [ + [1, "0x3", "a", "awaiter2", "b", "0x4"], + [1, "0x3", "a", "main", "Task-1", "0x2"], + [1, "0x4", "b", "awaiter", "a", "0x3"], + ] + ), + ], + [ + # this test case contains two cycles + ( + [ + ( + 1, + [ + (2, "Task-1", []), + ( + 3, + "A", + [[["nested", "nested", "task_b"], 4]], + ), + ( + 4, + "B", + [ + [["nested", "nested", "task_c"], 5], + [["nested", "nested", "task_a"], 3], + ], + ), + (5, "C", [[["nested", "nested"], 6]]), + ( + 6, + "Task-2", + [[["nested", "nested", "task_b"], 4]], + ), + ], + ), + (0, []), + ] + ), + ([[4, 3, 4], [4, 6, 5, 4]]), + ([]), + ( + [ + [ + 1, + "0x3", + "A", + "nested -> nested -> task_b", + "B", + "0x4", + ], + [ + 1, + "0x4", + "B", + "nested -> nested -> task_c", + "C", + "0x5", + ], + [ + 1, + "0x4", + "B", + "nested -> nested -> task_a", + "A", + "0x3", + ], + [ + 1, + "0x5", + "C", + "nested -> nested", + "Task-2", + "0x6", + ], + [ + 1, + "0x6", + "Task-2", + "nested -> nested -> task_b", + "B", + "0x4", + ], + ] + ), + ], +] + + +class TestAsyncioTools(test_utils.TestCase): + + def test_asyncio_utils(self): + for input_, cycles, tree, table in TEST_INPUTS: + if cycles: + try: + tools.print_async_tree(input_) + except tools.CycleFoundException as e: + self.assertEqual(e.cycles, cycles) + else: + print(tools.print_async_tree(input_)) + self.assertEqual(tools.print_async_tree(input_), tree) + print(tools.build_task_table(input_)) + self.assertEqual(tools.build_task_table(input_), table) From 152b3d703c1a939a144c3dc601a556d67a9a2001 Mon Sep 17 00:00:00 2001 From: Marta Gomez Macias Date: Fri, 2 May 2025 23:39:05 +0000 Subject: [PATCH 09/31] improve tests --- Lib/asyncio/tools.py | 7 ++-- Lib/test/test_asyncio/test_tools.py | 52 ++++++++++++++++------------- 2 files changed, 32 insertions(+), 27 deletions(-) diff --git a/Lib/asyncio/tools.py b/Lib/asyncio/tools.py index 1228c787b0980e..ccfd4893ac43ab 100644 --- a/Lib/asyncio/tools.py +++ b/Lib/asyncio/tools.py @@ -73,9 +73,9 @@ def _cor_node(parent_key, frame_name): def _roots(id2label, children): - roots = [n for n, lbl in id2label.items() if lbl == "Task-1"] - if roots: - return roots + # roots = [n for n, lbl in id2label.items() if lbl == "Task-1"] + # if roots: + # return roots all_children = {c for kids in children.values() for c in kids} return [n for n in id2label if n not in all_children] @@ -187,6 +187,7 @@ def build_task_table(result): print(f"Error retrieving tasks: {e}") sys.exit(1) + print(tasks) if args.tree: # Print the async call tree result = print_async_tree(tasks) diff --git a/Lib/test/test_asyncio/test_tools.py b/Lib/test/test_asyncio/test_tools.py index 149c6db6d5ba71..61ce4d8eb01785 100644 --- a/Lib/test/test_asyncio/test_tools.py +++ b/Lib/test/test_asyncio/test_tools.py @@ -1,7 +1,8 @@ """Tests for the asyncio tools script.""" +import unittest + from Lib.asyncio import tools -from test.test_asyncio import utils as test_utils # mock output of get_all_awaited_by function. @@ -209,48 +210,53 @@ # test case containing two roots ( ( - 1, + 9, [ - (2, "Task-1", []), - (3, "timer", [[["awaiter"], 4]]), - (4, "subtask1", [[["main"], 2]]), + (5, "Task-5", []), + (6, "Task-6", [[["main2"], 5]]), + (7, "Task-7", [[["main2"], 5]]), + (8, "Task-8", [[["main2"], 5]]), ], ), ( - 5, + 10, [ - (6, "Task-1", []), - (7, "sleeper", [[["awaiter2"], 8]]), - (8, "subtask2", [[["main"], 6]]), + (1, "Task-1", []), + (2, "Task-2", [[["main"], 1]]), + (3, "Task-3", [[["main"], 1]]), + (4, "Task-4", [[["main"], 1]]), ], ), + (11, []), (0, []), ), ([]), ( [ [ - "└── (T) Task-1", - " └── main", - " └── (T) subtask1", - " └── awaiter", - " └── (T) timer", + "└── (T) Task-5", + " └── main2", + " ├── (T) Task-6", + " ├── (T) Task-7", + " └── (T) Task-8", ], [ "└── (T) Task-1", " └── main", - " └── (T) subtask2", - " └── awaiter2", - " └── (T) sleeper", + " ├── (T) Task-2", + " ├── (T) Task-3", + " └── (T) Task-4", ], ] ), ( [ - [1, "0x3", "timer", "awaiter", "subtask1", "0x4"], - [1, "0x4", "subtask1", "main", "Task-1", "0x2"], - [5, "0x7", "sleeper", "awaiter2", "subtask2", "0x8"], - [5, "0x8", "subtask2", "main", "Task-1", "0x6"], + [9, "0x6", "Task-6", "main2", "Task-5", "0x5"], + [9, "0x7", "Task-7", "main2", "Task-5", "0x5"], + [9, "0x8", "Task-8", "main2", "Task-5", "0x5"], + [10, "0x2", "Task-2", "main", "Task-1", "0x1"], + [10, "0x3", "Task-3", "main", "Task-1", "0x1"], + [10, "0x4", "Task-4", "main", "Task-1", "0x1"], ] ), ], @@ -366,7 +372,7 @@ ] -class TestAsyncioTools(test_utils.TestCase): +class TestAsyncioTools(unittest.TestCase): def test_asyncio_utils(self): for input_, cycles, tree, table in TEST_INPUTS: @@ -376,7 +382,5 @@ def test_asyncio_utils(self): except tools.CycleFoundException as e: self.assertEqual(e.cycles, cycles) else: - print(tools.print_async_tree(input_)) self.assertEqual(tools.print_async_tree(input_), tree) - print(tools.build_task_table(input_)) self.assertEqual(tools.build_task_table(input_), table) From 955ef27415d3276b852e9b4d79124bc68144cf94 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 3 May 2025 01:45:16 +0200 Subject: [PATCH 10/31] dsf --- configure | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/configure b/configure index cea1c45ae33484..3b74554d5a2e64 100755 --- a/configure +++ b/configure @@ -30684,7 +30684,7 @@ case $ac_sys_system in #( py_cv_module__ctypes_test=n/a - py_cv_module__remotedebuggingmodule=n/a + py_cv_module__remotedebugging=n/a py_cv_module__testimportmultiple=n/a py_cv_module__testmultiphase=n/a py_cv_module__testsinglephase=n/a @@ -33449,35 +33449,35 @@ fi printf "%s\n" "$py_cv_module__testsinglephase" >&6; } - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _remotedebuggingmodule" >&5 -printf %s "checking for stdlib extension module _remotedebuggingmodule... " >&6; } - if test "$py_cv_module__remotedebuggingmodule" != "n/a" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _remotedebugging" >&5 +printf %s "checking for stdlib extension module _remotedebugging... " >&6; } + if test "$py_cv_module__remotedebugging" != "n/a" then : if test "$TEST_MODULES" = yes then : if true then : - py_cv_module__remotedebuggingmodule=yes + py_cv_module__remotedebugging=yes else case e in #( - e) py_cv_module__remotedebuggingmodule=missing ;; + e) py_cv_module__remotedebugging=missing ;; esac fi else case e in #( - e) py_cv_module__remotedebuggingmodule=disabled ;; + e) py_cv_module__remotedebugging=disabled ;; esac fi fi - as_fn_append MODULE_BLOCK "MODULE__REMOTEDEBUGGING_STATE=$py_cv_module__remotedebuggingmodule$as_nl" - if test "x$py_cv_module__remotedebuggingmodule" = xyes + as_fn_append MODULE_BLOCK "MODULE__REMOTEDEBUGGING_STATE=$py_cv_module__remotedebugging$as_nl" + if test "x$py_cv_module__remotedebugging" = xyes then : fi - if test "$py_cv_module__remotedebuggingmodule" = yes; then + if test "$py_cv_module__remotedebugging" = yes; then MODULE__REMOTEDEBUGGING_TRUE= MODULE__REMOTEDEBUGGING_FALSE='#' else @@ -33485,8 +33485,8 @@ else MODULE__REMOTEDEBUGGING_FALSE= fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $py_cv_module__remotedebuggingmodule" >&5 -printf "%s\n" "$py_cv_module__remotedebuggingmodule" >&6; } + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $py_cv_module__remotedebugging" >&5 +printf "%s\n" "$py_cv_module__remotedebugging" >&6; } { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module xxsubtype" >&5 @@ -35388,3 +35388,4 @@ if test "$ac_cv_header_stdatomic_h" != "yes"; then { printf "%s\n" "$as_me:${as_lineno-$LINENO}: Your compiler or platform does have a working C11 stdatomic.h. A future version of Python may require stdatomic.h." >&5 printf "%s\n" "$as_me: Your compiler or platform does have a working C11 stdatomic.h. A future version of Python may require stdatomic.h." >&6;} fi + From 65aee3cfe449178d8aed96755396718881b2e27c Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 3 May 2025 01:59:55 +0200 Subject: [PATCH 11/31] dsf --- Lib/test/test_external_inspection.py | 31 +++++++++++++++------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index 3535eb306e1958..e7f3e922f75a45 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -4,6 +4,7 @@ import importlib import sys import socket +from unittest.mock import ANY from test.support import os_helper, SHORT_TIMEOUT, busy_retry from test.support.script_helper import make_script from test.support.socket_helper import find_unused_port @@ -184,13 +185,13 @@ def new_eager_loop(): root_task = "Task-1" expected_stack_trace = [ - ["c5", "c4", "c3", "c2"], - "c2_root", + ['c5', 'c4', 'c3', 'c2'], + 'c2_root', [ - [["main"], root_task, []], - [["c1"], "sub_main_1", [[["main"], root_task, []]]], - [["c1"], "sub_main_2", [[["main"], root_task, []]]], - ], + [['_aexit', '__aexit__', 'main'], root_task, []], + [['c1'], 'sub_main_1', [[['_aexit', '__aexit__', 'main'], root_task, []]]], + [['c1'], 'sub_main_2', [[['_aexit', '__aexit__', 'main'], root_task, []]]], + ] ] self.assertEqual(stack_trace, expected_stack_trace) @@ -397,8 +398,10 @@ async def main(): # sets are unordered, so we want to sort "awaited_by"s stack_trace[2].sort(key=lambda x: x[1]) - expected_stack_trace = [ - ['deep', 'c1', 'run_one_coro'], 'Task-2', [[['main'], 'Task-1', []]] + expected_stack_trace = [ + ['deep', 'c1', 'run_one_coro'], + 'Task-2', + [[['staggered_race', 'main'], 'Task-1', []]] ] self.assertEqual(stack_trace, expected_stack_trace) @@ -516,19 +519,19 @@ async def main(): # expected: at least 1000 pending tasks self.assertGreaterEqual(len(entries), 1000) # the first three tasks stem from the code structure - self.assertIn(('Task-1', []), entries) - self.assertIn(('server task', [[['main'], 'Task-1', []]]), entries) - self.assertIn(('echo client spam', [[['main'], 'Task-1', []]]), entries) + self.assertIn((ANY, 'Task-1', []), entries) + self.assertIn((ANY, 'server task', [[['_aexit', '__aexit__', 'main'], ANY]]), entries) + self.assertIn((ANY, 'echo client spam', [[['_aexit', '__aexit__', 'main'], ANY]]), entries) - expected_stack = [[['echo_client_spam'], 'echo client spam', [[['main'], 'Task-1', []]]]] - tasks_with_stack = [task for task in entries if task[1] == expected_stack] + expected_stack = [[['_aexit', '__aexit__', 'echo_client_spam'], ANY]] + tasks_with_stack = [task for task in entries if task[2] == expected_stack] self.assertGreaterEqual(len(tasks_with_stack), 1000) # the final task will have some random number, but it should for # sure be one of the echo client spam horde (In windows this is not true # for some reason) if sys.platform != "win32": - self.assertEqual([[['echo_client_spam'], 'echo client spam', [[['main'], 'Task-1', []]]]], entries[-1][1]) + self.assertEqual([[['_aexit', '__aexit__', 'echo_client_spam'], ANY]], entries[-1][2]) except PermissionError: self.skipTest( "Insufficient permissions to read the stack trace") From 51e689ed04fd38afb7e637ff88b564eaefd76732 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 3 May 2025 02:12:15 +0200 Subject: [PATCH 12/31] test fixes --- Lib/test/test_asyncio/test_tools.py | 227 ++++++++++++++++++++++++---- 1 file changed, 195 insertions(+), 32 deletions(-) diff --git a/Lib/test/test_asyncio/test_tools.py b/Lib/test/test_asyncio/test_tools.py index 61ce4d8eb01785..59cdca4ce6bfec 100644 --- a/Lib/test/test_asyncio/test_tools.py +++ b/Lib/test/test_asyncio/test_tools.py @@ -6,7 +6,7 @@ # mock output of get_all_awaited_by function. -TEST_INPUTS = [ +TEST_INPUTS_TREE = [ [ # test case containing a task called timer being awaited in two # different subtasks part of a TaskGroup (root1 and root2) which call @@ -80,7 +80,6 @@ ), (0, []), ), - ([]), ( [ [ @@ -121,6 +120,184 @@ ] ] ), + ], + [ + # test case containing two roots + ( + ( + 9, + [ + (5, "Task-5", []), + (6, "Task-6", [[["main2"], 5]]), + (7, "Task-7", [[["main2"], 5]]), + (8, "Task-8", [[["main2"], 5]]), + ], + ), + ( + 10, + [ + (1, "Task-1", []), + (2, "Task-2", [[["main"], 1]]), + (3, "Task-3", [[["main"], 1]]), + (4, "Task-4", [[["main"], 1]]), + ], + ), + (11, []), + (0, []), + ), + ( + [ + [ + "└── (T) Task-5", + " └── main2", + " ├── (T) Task-6", + " ├── (T) Task-7", + " └── (T) Task-8", + ], + [ + "└── (T) Task-1", + " └── main", + " ├── (T) Task-2", + " ├── (T) Task-3", + " └── (T) Task-4", + ], + ] + ), + ], +] + +TEST_INPUTS_CYCLES_TREE = [ + [ + # this test case contains a cycle: two tasks awaiting each other. + ( + [ + ( + 1, + [ + (2, "Task-1", []), + ( + 3, + "a", + [[["awaiter2"], 4], [["main"], 2]], + ), + (4, "b", [[["awaiter"], 3]]), + ], + ), + (0, []), + ] + ), + ([[4, 3, 4]]), + ], + [ + # this test case contains two cycles + ( + [ + ( + 1, + [ + (2, "Task-1", []), + ( + 3, + "A", + [[["nested", "nested", "task_b"], 4]], + ), + ( + 4, + "B", + [ + [["nested", "nested", "task_c"], 5], + [["nested", "nested", "task_a"], 3], + ], + ), + (5, "C", [[["nested", "nested"], 6]]), + ( + 6, + "Task-2", + [[["nested", "nested", "task_b"], 4]], + ), + ], + ), + (0, []), + ] + ), + ([[4, 3, 4], [4, 6, 5, 4]]), + ], +] + +TEST_INPUTS_TABLE = [ + [ + # test case containing a task called timer being awaited in two + # different subtasks part of a TaskGroup (root1 and root2) which call + # awaiter functions. + ( + ( + 1, + [ + (2, "Task-1", []), + ( + 3, + "timer", + [ + [["awaiter3", "awaiter2", "awaiter"], 4], + [["awaiter1_3", "awaiter1_2", "awaiter1"], 5], + [["awaiter1_3", "awaiter1_2", "awaiter1"], 6], + [["awaiter3", "awaiter2", "awaiter"], 7], + ], + ), + ( + 8, + "root1", + [[["_aexit", "__aexit__", "main"], 2]], + ), + ( + 9, + "root2", + [[["_aexit", "__aexit__", "main"], 2]], + ), + ( + 4, + "child1_1", + [ + [ + ["_aexit", "__aexit__", "blocho_caller", "bloch"], + 8, + ] + ], + ), + ( + 6, + "child2_1", + [ + [ + ["_aexit", "__aexit__", "blocho_caller", "bloch"], + 8, + ] + ], + ), + ( + 7, + "child1_2", + [ + [ + ["_aexit", "__aexit__", "blocho_caller", "bloch"], + 9, + ] + ], + ), + ( + 5, + "child2_2", + [ + [ + ["_aexit", "__aexit__", "blocho_caller", "bloch"], + 9, + ] + ], + ), + ], + ), + (0, []), + ), ( [ [ @@ -230,25 +407,6 @@ (11, []), (0, []), ), - ([]), - ( - [ - [ - "└── (T) Task-5", - " └── main2", - " ├── (T) Task-6", - " ├── (T) Task-7", - " └── (T) Task-8", - ], - [ - "└── (T) Task-1", - " └── main", - " ├── (T) Task-2", - " ├── (T) Task-3", - " └── (T) Task-4", - ], - ] - ), ( [ [9, "0x6", "Task-6", "main2", "Task-5", "0x5"], @@ -260,7 +418,7 @@ ] ), ], - # Tests cycle detection. + # CASES WITH CYCLES [ # this test case contains a cycle: two tasks awaiting each other. ( @@ -280,8 +438,6 @@ (0, []), ] ), - ([[4, 3, 4]]), - ([]), ( [ [1, "0x3", "a", "awaiter2", "b", "0x4"], @@ -322,8 +478,6 @@ (0, []), ] ), - ([[4, 3, 4], [4, 6, 5, 4]]), - ([]), ( [ [ @@ -372,15 +526,24 @@ ] -class TestAsyncioTools(unittest.TestCase): +class TestAsyncioToolsTree(unittest.TestCase): def test_asyncio_utils(self): - for input_, cycles, tree, table in TEST_INPUTS: - if cycles: + for input_, tree in TEST_INPUTS_TREE: + with self.subTest(input_): + self.assertEqual(tools.print_async_tree(input_), tree) + + def test_asyncio_utils_cycles(self): + for input_, cycles in TEST_INPUTS_CYCLES_TREE: + with self.subTest(input_): try: tools.print_async_tree(input_) except tools.CycleFoundException as e: self.assertEqual(e.cycles, cycles) - else: - self.assertEqual(tools.print_async_tree(input_), tree) - self.assertEqual(tools.build_task_table(input_), table) + + +class TestAsyncioToolsTable(unittest.TestCase): + def test_asyncio_utils(self): + for input_, table in TEST_INPUTS_TABLE: + with self.subTest(input_): + self.assertEqual(tools.build_task_table(input_), table) From 1d2734864a914e5e87b4cac665b7ed099103e212 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 3 May 2025 02:25:01 +0200 Subject: [PATCH 13/31] test fixes --- Lib/asyncio/tools.py | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/Lib/asyncio/tools.py b/Lib/asyncio/tools.py index ccfd4893ac43ab..d966c35c1f5135 100644 --- a/Lib/asyncio/tools.py +++ b/Lib/asyncio/tools.py @@ -1,4 +1,5 @@ import argparse +from dataclasses import dataclass from collections import defaultdict from itertools import count from enum import Enum @@ -11,21 +12,11 @@ class NodeType(Enum): TASK = 2 +@dataclass(frozen=True) class CycleFoundException(Exception): """Raised when there is a cycle when drawing the call tree.""" - - def __init__(self, cycles, id2name): - super().__init__() - self.cycles = cycles - self.id2name = id2name - - def __str__(self): - for c in self.cycles: - names = " → ".join(self.id2name.get(tid, hex(tid)) for tid in c) - return ( - "ERROR: await-graph contains cycles – cannot print a tree!\n" - f"cycle: {names}" - ) + cycles: list[list[int]] + id2name: dict[int, str] # ─── indexing helpers ─────────────────────────────────────────── @@ -172,6 +163,14 @@ def build_task_table(result): return table +def _print_cycle_exception(exception: CycleFoundException): + print("ERROR: await-graph contains cycles – cannot print a tree!", file=sys.stderr) + print("", file=sys.stderr) + for c in exception.cycles: + inames = " → ".join(exception.id2name.get(tid, hex(tid)) for tid in c) + print(f"cycle: {inames}", file=sys.stderr) + + if __name__ == "__main__": parser = argparse.ArgumentParser(description="Show Python async tasks in a process") @@ -184,13 +183,19 @@ def build_task_table(result): try: tasks = get_all_awaited_by(args.pid) except RuntimeError as e: + while e.__cause__ is not None: + e = e.__cause__ print(f"Error retrieving tasks: {e}") sys.exit(1) - print(tasks) if args.tree: # Print the async call tree - result = print_async_tree(tasks) + try: + result = print_async_tree(tasks) + except CycleFoundException as e: + _print_cycle_exception(e) + sys.exit(1) + for tree in result: print("\n".join(tree)) else: From 1d1b0e9b2ebf739ea7be5886c390e46624205725 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 3 May 2025 02:27:35 +0200 Subject: [PATCH 14/31] test fixes --- Lib/asyncio/tools.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Lib/asyncio/tools.py b/Lib/asyncio/tools.py index d966c35c1f5135..1719821a1ffe6e 100644 --- a/Lib/asyncio/tools.py +++ b/Lib/asyncio/tools.py @@ -64,9 +64,6 @@ def _cor_node(parent_key, frame_name): def _roots(id2label, children): - # roots = [n for n, lbl in id2label.items() if lbl == "Task-1"] - # if roots: - # return roots all_children = {c for kids in children.values() for c in kids} return [n for n in id2label if n not in all_children] @@ -183,8 +180,8 @@ def _print_cycle_exception(exception: CycleFoundException): try: tasks = get_all_awaited_by(args.pid) except RuntimeError as e: - while e.__cause__ is not None: - e = e.__cause__ + while e.__context__ is not None: + e = e.__context__ print(f"Error retrieving tasks: {e}") sys.exit(1) From edad4d1f983f3b075188b438e25fa0ee21416806 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 3 May 2025 03:18:48 +0200 Subject: [PATCH 15/31] test fixes --- Lib/test/test_asyncio/test_tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_asyncio/test_tools.py b/Lib/test/test_asyncio/test_tools.py index 59cdca4ce6bfec..6ad47ef9e874bf 100644 --- a/Lib/test/test_asyncio/test_tools.py +++ b/Lib/test/test_asyncio/test_tools.py @@ -2,7 +2,7 @@ import unittest -from Lib.asyncio import tools +from asyncio import tools # mock output of get_all_awaited_by function. From 199589ceb14d1e72202bb58032e5a037d842c8b7 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 3 May 2025 13:19:16 +0200 Subject: [PATCH 16/31] Fix free threading offsets --- Modules/_remotedebuggingmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_remotedebuggingmodule.c b/Modules/_remotedebuggingmodule.c index e027ffb28d316f..84d97858fe2ed7 100644 --- a/Modules/_remotedebuggingmodule.c +++ b/Modules/_remotedebuggingmodule.c @@ -345,7 +345,7 @@ parse_coro_chain( uintptr_t gen_type_addr; int err = read_ptr( handle, - coro_address + sizeof(void*), + coro_address + offsets->pyobject.ob_type, &gen_type_addr); if (err) { return -1; From 9e87032d62c6f2c6c0bbd5f36caefd16ac969902 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 3 May 2025 13:32:08 +0200 Subject: [PATCH 17/31] Fix free threading offsets AGAIN --- Modules/_remotedebuggingmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_remotedebuggingmodule.c b/Modules/_remotedebuggingmodule.c index 84d97858fe2ed7..f6e8135a6c5800 100644 --- a/Modules/_remotedebuggingmodule.c +++ b/Modules/_remotedebuggingmodule.c @@ -429,7 +429,7 @@ parse_coro_chain( uintptr_t gi_await_addr_type_addr; int err = read_ptr( handle, - gi_await_addr + sizeof(void*), + gi_await_addr + offsets->pyobject.ob_type, &gi_await_addr_type_addr); if (err) { return -1; From 69e9221f54965da54bcfe40c82929f9705ad6ef4 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 3 May 2025 18:36:18 +0200 Subject: [PATCH 18/31] Debugging --- Lib/test/test_external_inspection.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index e7f3e922f75a45..0c8fc0395b65cc 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -517,6 +517,7 @@ async def main(): self.assertEqual(all_awaited_by[1], (0, [])) entries = all_awaited_by[0][1] # expected: at least 1000 pending tasks + print(entries, file=sys.stderr) self.assertGreaterEqual(len(entries), 1000) # the first three tasks stem from the code structure self.assertIn((ANY, 'Task-1', []), entries) From b6cb609812f6ce27fcc1552c05e248a6b6f1e591 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 3 May 2025 18:54:44 +0200 Subject: [PATCH 19/31] More tests --- Lib/test/test_asyncio/test_tools.py | 224 ++++++++++++++++++++++++++++ 1 file changed, 224 insertions(+) diff --git a/Lib/test/test_asyncio/test_tools.py b/Lib/test/test_asyncio/test_tools.py index 6ad47ef9e874bf..2f2b5feac43d74 100644 --- a/Lib/test/test_asyncio/test_tools.py +++ b/Lib/test/test_asyncio/test_tools.py @@ -547,3 +547,227 @@ def test_asyncio_utils(self): for input_, table in TEST_INPUTS_TABLE: with self.subTest(input_): self.assertEqual(tools.build_task_table(input_), table) + + +class TestAsyncioToolsBasic(unittest.TestCase): + def test_empty_input_tree(self): + """Test print_async_tree with empty input.""" + result = [] + expected_output = [] + self.assertEqual(tools.print_async_tree(result), expected_output) + + def test_empty_input_table(self): + """Test build_task_table with empty input.""" + result = [] + expected_output = [] + self.assertEqual(tools.build_task_table(result), expected_output) + + def test_only_independent_tasks_tree(self): + input_ = [(1, [(10, "taskA", []), (11, "taskB", [])])] + expected = [["└── (T) taskA"], ["└── (T) taskB"]] + result = tools.print_async_tree(input_) + self.assertEqual(sorted(result), sorted(expected)) + + def test_only_independent_tasks_table(self): + input_ = [(1, [(10, "taskA", []), (11, "taskB", [])])] + self.assertEqual(tools.build_task_table(input_), []) + + def test_single_task_tree(self): + """Test print_async_tree with a single task and no awaits.""" + result = [ + ( + 1, + [ + (2, "Task-1", []), + ], + ) + ] + expected_output = [ + [ + "└── (T) Task-1", + ] + ] + self.assertEqual(tools.print_async_tree(result), expected_output) + + def test_single_task_table(self): + """Test build_task_table with a single task and no awaits.""" + result = [ + ( + 1, + [ + (2, "Task-1", []), + ], + ) + ] + expected_output = [] + self.assertEqual(tools.build_task_table(result), expected_output) + + def test_cycle_detection(self): + """Test print_async_tree raises CycleFoundException for cyclic input.""" + result = [ + ( + 1, + [ + (2, "Task-1", [[["main"], 3]]), + (3, "Task-2", [[["main"], 2]]), + ], + ) + ] + with self.assertRaises(tools.CycleFoundException) as context: + tools.print_async_tree(result) + self.assertEqual(context.exception.cycles, [[3, 2, 3]]) + + def test_complex_tree(self): + """Test print_async_tree with a more complex tree structure.""" + result = [ + ( + 1, + [ + (2, "Task-1", []), + (3, "Task-2", [[["main"], 2]]), + (4, "Task-3", [[["main"], 3]]), + ], + ) + ] + expected_output = [ + [ + "└── (T) Task-1", + " └── main", + " └── (T) Task-2", + " └── main", + " └── (T) Task-3", + ] + ] + self.assertEqual(tools.print_async_tree(result), expected_output) + + def test_complex_table(self): + """Test build_task_table with a more complex tree structure.""" + result = [ + ( + 1, + [ + (2, "Task-1", []), + (3, "Task-2", [[["main"], 2]]), + (4, "Task-3", [[["main"], 3]]), + ], + ) + ] + expected_output = [ + [1, "0x3", "Task-2", "main", "Task-1", "0x2"], + [1, "0x4", "Task-3", "main", "Task-2", "0x3"], + ] + self.assertEqual(tools.build_task_table(result), expected_output) + + def test_deep_coroutine_chain(self): + input_ = [ + ( + 1, + [ + (10, "leaf", [[["c1", "c2", "c3", "c4", "c5"], 11]]), + (11, "root", []), + ], + ) + ] + expected = [ + [ + "└── (T) root", + " └── c5", + " └── c4", + " └── c3", + " └── c2", + " └── c1", + " └── (T) leaf", + ] + ] + result = tools.print_async_tree(input_) + self.assertEqual(result, expected) + + def test_multiple_cycles_same_node(self): + input_ = [ + ( + 1, + [ + (1, "Task-A", [[["call1"], 2]]), + (2, "Task-B", [[["call2"], 3]]), + (3, "Task-C", [[["call3"], 1], [["call4"], 2]]), + ], + ) + ] + with self.assertRaises(tools.CycleFoundException) as ctx: + tools.print_async_tree(input_) + cycles = ctx.exception.cycles + self.assertTrue(any(set(c) == {1, 2, 3} for c in cycles)) + + def test_table_output_format(self): + input_ = [(1, [(1, "Task-A", [[["foo"], 2]]), (2, "Task-B", [])])] + table = tools.build_task_table(input_) + for row in table: + self.assertEqual(len(row), 6) + self.assertIsInstance(row[0], int) # thread ID + self.assertTrue( + isinstance(row[1], str) and row[1].startswith("0x") + ) # hex task ID + self.assertIsInstance(row[2], str) # task name + self.assertIsInstance(row[3], str) # coroutine chain + self.assertIsInstance(row[4], str) # awaiter name + self.assertTrue( + isinstance(row[5], str) and row[5].startswith("0x") + ) # hex awaiter ID + + +class TestAsyncioToolsEdgeCases(unittest.TestCase): + + def test_task_awaits_self(self): + """A task directly awaits itself – should raise a cycle.""" + input_ = [(1, [(1, "Self-Awaiter", [[["loopback"], 1]])])] + with self.assertRaises(tools.CycleFoundException) as ctx: + tools.print_async_tree(input_) + self.assertIn([1, 1], ctx.exception.cycles) + + def test_task_with_missing_awaiter_id(self): + """Awaiter ID not in task list – should not crash, just show 'Unknown'.""" + input_ = [(1, [(1, "Task-A", [[["coro"], 999]])])] # 999 not defined + table = tools.build_task_table(input_) + self.assertEqual(len(table), 1) + self.assertEqual(table[0][4], "Unknown") + + def test_duplicate_coroutine_frames(self): + """Same coroutine frame repeated under a parent – should deduplicate.""" + input_ = [ + ( + 1, + [ + (1, "Task-1", [[["frameA"], 2], [["frameA"], 3]]), + (2, "Task-2", []), + (3, "Task-3", []), + ], + ) + ] + tree = tools.print_async_tree(input_) + # Both children should be under the same coroutine node + flat = "\n".join(tree[0]) + self.assertIn("frameA", flat) + self.assertIn("Task-2", flat) + self.assertIn("Task-1", flat) + + flat = "\n".join(tree[1]) + self.assertIn("frameA", flat) + self.assertIn("Task-3", flat) + self.assertIn("Task-1", flat) + + def test_task_with_no_name(self): + """Task with no name in id2name – should still render with fallback.""" + input_ = [(1, [(1, "root", [[["f1"], 2]]), (2, None, [])])] + # If name is None, fallback to string should not crash + tree = tools.print_async_tree(input_) + self.assertIn("(T) None", "\n".join(tree[0])) + + def test_tree_rendering_with_custom_emojis(self): + """Pass custom emojis to the tree renderer.""" + input_ = [(1, [(1, "MainTask", [[["f1", "f2"], 2]]), (2, "SubTask", [])])] + tree = tools.print_async_tree(input_, task_emoji="🧵", cor_emoji="🔁") + flat = "\n".join(tree[0]) + self.assertIn("🧵 MainTask", flat) + self.assertIn("🔁 f1", flat) + self.assertIn("🔁 f2", flat) + self.assertIn("🧵 SubTask", flat) From 2dd34521a9429903246a04441c63ddcf55fb64ba Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 3 May 2025 19:04:29 +0200 Subject: [PATCH 20/31] Add news entry --- Doc/whatsnew/3.14.rst | 90 +++++++++++++++++++ ...5-05-03-19-04-03.gh-issue-91048.S8QWSw.rst | 4 + 2 files changed, 94 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-03-19-04-03.gh-issue-91048.S8QWSw.rst diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 2f8b652d47e428..dca311880f9826 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -508,6 +508,96 @@ configuration mechanisms). .. seealso:: :pep:`741`. +.. _whatsnew314-asyncio-introspection + +Asyncio introspection capabilities +---------------------------------- + +Added a new command-line interface to inspect running Python processes using +asynchronous tasks, available via: + +.. code-block:: bash + python -m asyncio.tools [--tree] PID + +This tool inspects the given process ID (PID) and displays information about +currently running asyncio tasks. By default, it outputs a task table: a flat +listing of all tasks, their names, their coroutine stacks, and which tasks are +awaiting them. + +With the ``--tree`` option, it instead renders a visual async call tree, +showing coroutine relationships in a hierarchical format. This command is +particularly useful for debugging long-running or stuck asynchronous programs. +It can help developers quickly identify where a program is blocked, what tasks +are pending, and how coroutines are chained together. + +For example given this code: + +.. code-block:: python + + import asyncio + + async def sleeper(name, delay): + await asyncio.sleep(delay) + print(f"{name} is done sleeping.") + + async def inner(name): + await asyncio.sleep(0.1) + await sleeper(name, 1) + + async def task_group(name): + await asyncio.gather( + inner(f"{name}-1"), + inner(f"{name}-2"), + ) + + async def main(): + # Start two separate task groups + t1 = asyncio.create_task(task_group("groupA")) + t2 = asyncio.create_task(task_group("groupB")) + await t1 + await t2 + + if __name__ == "__main__": + asyncio.run(main()) + +Executing the new tool on the running process will yield a table like this: + +.. code-block:: bash + + python -m asyncio.tools 12345 + + tid task id task name coroutine chain awaiter name awaiter id + --------------------------------------------------------------------------------------------------------------------------------------- + 6826911 0x200013c0220 Task-2 main Task-1 0x200013b0020 + 6826911 0x200013c0620 Task-4 task_group Task-2 0x200013c0220 + 6826911 0x200013c0820 Task-5 task_group Task-2 0x200013c0220 + 6826911 0x200013c0c20 Task-6 task_group Task-3 0x200013c0420 + 6826911 0x200013c0e20 Task-7 task_group Task-3 0x200013c0420 + + +and with the ``--tree`` option: + +.. code-block:: bash + + python -m asyncio.tools --tree 12345 + + └── (T) Task-1 + └── main + └── (T) Task-2 + └── task_group + ├── (T) Task-4 + └── (T) Task-5 + └── (T) Task-3 + └── task_group + ├── (T) Task-6 + └── (T) Task-7 + +If a cycle is detected in the async await graph (which could indicate a +programming issue), the tool raises an error and lists the cycle paths that +prevent tree construction. + +(Contributed by Pablo Galindo, Łukasz Langa and Marta Gomez Macias in :gh:`91048`.) + .. _whatsnew314-tail-call: A new type of interpreter diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-03-19-04-03.gh-issue-91048.S8QWSw.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-03-19-04-03.gh-issue-91048.S8QWSw.rst new file mode 100644 index 00000000000000..c33810614d4d56 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-03-19-04-03.gh-issue-91048.S8QWSw.rst @@ -0,0 +1,4 @@ +Add a new ``python -m asyncio.tools`` command-line interface to inspect +asyncio tasks in a running Python process. Displays a flat table of await +relationships or a tree view with ``--tree``, useful for debugging async +code. Patch by Pablo Galindo, Łukasz Langa and Marta Gomez Macias. From a84a171054042c4d678d1001019c50cc544b9ca4 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 3 May 2025 19:08:24 +0200 Subject: [PATCH 21/31] Doc fixes --- Doc/whatsnew/3.14.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index dca311880f9826..f91a9066871f06 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -517,6 +517,7 @@ Added a new command-line interface to inspect running Python processes using asynchronous tasks, available via: .. code-block:: bash + python -m asyncio.tools [--tree] PID This tool inspects the given process ID (PID) and displays information about From 0f75edcc0e6a3a5e6e7687c1dbee889613560736 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sat, 3 May 2025 19:47:39 +0200 Subject: [PATCH 22/31] Fix doc build --- Doc/whatsnew/3.14.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index f91a9066871f06..0e19461d6e20a4 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -508,7 +508,7 @@ configuration mechanisms). .. seealso:: :pep:`741`. -.. _whatsnew314-asyncio-introspection +.. _whatsnew314-asyncio-introspection: Asyncio introspection capabilities ---------------------------------- @@ -597,7 +597,7 @@ If a cycle is detected in the async await graph (which could indicate a programming issue), the tool raises an error and lists the cycle paths that prevent tree construction. -(Contributed by Pablo Galindo, Łukasz Langa and Marta Gomez Macias in :gh:`91048`.) +(Contributed by Pablo Galindo, Łukasz Langa, and Marta Gomez Macias in :gh:`91048`.) .. _whatsnew314-tail-call: From c3a6bcbbda1a631496695e966bc3922480aa2135 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sat, 3 May 2025 20:01:11 +0200 Subject: [PATCH 23/31] Add Yury --- Doc/whatsnew/3.14.rst | 3 ++- .../2025-05-03-19-04-03.gh-issue-91048.S8QWSw.rst | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 0e19461d6e20a4..5bb482de3685f5 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -597,7 +597,8 @@ If a cycle is detected in the async await graph (which could indicate a programming issue), the tool raises an error and lists the cycle paths that prevent tree construction. -(Contributed by Pablo Galindo, Łukasz Langa, and Marta Gomez Macias in :gh:`91048`.) +(Contributed by Pablo Galindo, Łukasz Langa, Yury Selivanov, and Marta +Gomez Macias in :gh:`91048`.) .. _whatsnew314-tail-call: diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-03-19-04-03.gh-issue-91048.S8QWSw.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-03-19-04-03.gh-issue-91048.S8QWSw.rst index c33810614d4d56..0d5e7734f212d6 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-03-19-04-03.gh-issue-91048.S8QWSw.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-03-19-04-03.gh-issue-91048.S8QWSw.rst @@ -1,4 +1,5 @@ Add a new ``python -m asyncio.tools`` command-line interface to inspect asyncio tasks in a running Python process. Displays a flat table of await relationships or a tree view with ``--tree``, useful for debugging async -code. Patch by Pablo Galindo, Łukasz Langa and Marta Gomez Macias. +code. Patch by Pablo Galindo, Łukasz Langa, Yury Selivanov, and Marta +Gomez Macias. From 5e1cb87c1d9cd3a1a4dadfedb570ee353f468011 Mon Sep 17 00:00:00 2001 From: Marta Gomez Macias Date: Sat, 3 May 2025 18:04:37 +0000 Subject: [PATCH 24/31] fix: Show independent tasks in the table --- Lib/asyncio/tools.py | 11 +++++ Lib/test/test_asyncio/test_tools.py | 72 ++++++++++++++++++++++++++++- 2 files changed, 81 insertions(+), 2 deletions(-) diff --git a/Lib/asyncio/tools.py b/Lib/asyncio/tools.py index 1719821a1ffe6e..4daace72471465 100644 --- a/Lib/asyncio/tools.py +++ b/Lib/asyncio/tools.py @@ -144,6 +144,17 @@ def build_task_table(result): table = [] for tid, tasks in result: for task_id, task_name, awaited in tasks: + if not awaited: + table.append( + [ + tid, + hex(task_id), + task_name, + "", + "", + "0x0" + ] + ) for stack, awaiter_id in awaited: coroutine_chain = " -> ".join(stack) awaiter_name = id2name.get(awaiter_id, "Unknown") diff --git a/Lib/test/test_asyncio/test_tools.py b/Lib/test/test_asyncio/test_tools.py index 2f2b5feac43d74..c9abdc60387310 100644 --- a/Lib/test/test_asyncio/test_tools.py +++ b/Lib/test/test_asyncio/test_tools.py @@ -164,6 +164,37 @@ ] ), ], + [ + # test case containing two roots, one of them without subtasks + ( + [ + (1, [(2, "Task-5", [])]), + ( + 3, + [ + (4, "Task-1", []), + (5, "Task-2", [[["main"], 4]]), + (6, "Task-3", [[["main"], 4]]), + (7, "Task-4", [[["main"], 4]]), + ], + ), + (8, []), + (0, []), + ] + ), + ( + [ + ["└── (T) Task-5"], + [ + "└── (T) Task-1", + " └── main", + " ├── (T) Task-2", + " ├── (T) Task-3", + " └── (T) Task-4", + ], + ] + ), + ], ] TEST_INPUTS_CYCLES_TREE = [ @@ -300,6 +331,7 @@ ), ( [ + [1, "0x2", "Task-1", "", "", "0x0"], [ 1, "0x3", @@ -409,15 +441,45 @@ ), ( [ + [9, "0x5", "Task-5", "", "", "0x0"], [9, "0x6", "Task-6", "main2", "Task-5", "0x5"], [9, "0x7", "Task-7", "main2", "Task-5", "0x5"], [9, "0x8", "Task-8", "main2", "Task-5", "0x5"], + [10, "0x1", "Task-1", "", "", "0x0"], [10, "0x2", "Task-2", "main", "Task-1", "0x1"], [10, "0x3", "Task-3", "main", "Task-1", "0x1"], [10, "0x4", "Task-4", "main", "Task-1", "0x1"], ] ), ], + [ + # test case containing two roots, one of them without subtasks + ( + [ + (1, [(2, "Task-5", [])]), + ( + 3, + [ + (4, "Task-1", []), + (5, "Task-2", [[["main"], 4]]), + (6, "Task-3", [[["main"], 4]]), + (7, "Task-4", [[["main"], 4]]), + ], + ), + (8, []), + (0, []), + ] + ), + ( + [ + [1, "0x2", "Task-5", "", "", "0x0"], + [3, "0x4", "Task-1", "", "", "0x0"], + [3, "0x5", "Task-2", "main", "Task-1", "0x4"], + [3, "0x6", "Task-3", "main", "Task-1", "0x4"], + [3, "0x7", "Task-4", "main", "Task-1", "0x4"], + ] + ), + ], # CASES WITH CYCLES [ # this test case contains a cycle: two tasks awaiting each other. @@ -440,6 +502,7 @@ ), ( [ + [1, "0x2", "Task-1", "", "", "0x0"], [1, "0x3", "a", "awaiter2", "b", "0x4"], [1, "0x3", "a", "main", "Task-1", "0x2"], [1, "0x4", "b", "awaiter", "a", "0x3"], @@ -480,6 +543,7 @@ ), ( [ + [1, "0x2", "Task-1", "", "", "0x0"], [ 1, "0x3", @@ -570,7 +634,10 @@ def test_only_independent_tasks_tree(self): def test_only_independent_tasks_table(self): input_ = [(1, [(10, "taskA", []), (11, "taskB", [])])] - self.assertEqual(tools.build_task_table(input_), []) + self.assertEqual( + tools.build_task_table(input_), + [[1, "0xa", "taskA", "", "", "0x0"], [1, "0xb", "taskB", "", "", "0x0"]], + ) def test_single_task_tree(self): """Test print_async_tree with a single task and no awaits.""" @@ -599,7 +666,7 @@ def test_single_task_table(self): ], ) ] - expected_output = [] + expected_output = [[1, "0x2", "Task-1", "", "", "0x0"]] self.assertEqual(tools.build_task_table(result), expected_output) def test_cycle_detection(self): @@ -653,6 +720,7 @@ def test_complex_table(self): ) ] expected_output = [ + [1, "0x2", "Task-1", "", "", "0x0"], [1, "0x3", "Task-2", "main", "Task-1", "0x2"], [1, "0x4", "Task-3", "main", "Task-2", "0x3"], ] From af6a8bf0210fef0ad18b593e320fa521a4080e36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sat, 3 May 2025 22:46:48 +0200 Subject: [PATCH 25/31] Temporarily skip test_async_global_awaited_by on free-threading --- Lib/test/test_external_inspection.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index 0c8fc0395b65cc..4e82f567e1f429 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -5,7 +5,7 @@ import sys import socket from unittest.mock import ANY -from test.support import os_helper, SHORT_TIMEOUT, busy_retry +from test.support import os_helper, SHORT_TIMEOUT, busy_retry, requires_gil_enabled from test.support.script_helper import make_script from test.support.socket_helper import find_unused_port @@ -406,6 +406,7 @@ async def main(): self.assertEqual(stack_trace, expected_stack_trace) @skip_if_not_supported + @requires_gil_enabled("gh-133359: occasionally flaky on AMD64") @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, "Test only runs on Linux with process_vm_readv support") def test_async_global_awaited_by(self): @@ -517,7 +518,6 @@ async def main(): self.assertEqual(all_awaited_by[1], (0, [])) entries = all_awaited_by[0][1] # expected: at least 1000 pending tasks - print(entries, file=sys.stderr) self.assertGreaterEqual(len(entries), 1000) # the first three tasks stem from the code structure self.assertIn((ANY, 'Task-1', []), entries) @@ -548,7 +548,6 @@ async def main(): "Test only runs on Linux with process_vm_readv support") def test_self_trace(self): stack_trace = get_stack_trace(os.getpid()) - print(stack_trace) self.assertEqual(stack_trace[0], "test_self_trace") if __name__ == "__main__": From 8db5dbe3f11030f8f8811fa0629eac710786bf15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sat, 3 May 2025 23:25:58 +0200 Subject: [PATCH 26/31] Drop the `tools`. It's cleaner. --- Doc/whatsnew/3.14.rst | 16 +++-- Lib/asyncio/__main__.py | 32 +++++++++ Lib/asyncio/tools.py | 68 +++++++++---------- Lib/test/test_asyncio/test_tools.py | 36 +++++----- ...5-05-03-19-04-03.gh-issue-91048.S8QWSw.rst | 5 +- 5 files changed, 96 insertions(+), 61 deletions(-) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 5bb482de3685f5..65d0a319a07cf2 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -518,14 +518,18 @@ asynchronous tasks, available via: .. code-block:: bash - python -m asyncio.tools [--tree] PID + python -m asyncio ps PID This tool inspects the given process ID (PID) and displays information about -currently running asyncio tasks. By default, it outputs a task table: a flat +currently running asyncio tasks. It outputs a task table: a flat listing of all tasks, their names, their coroutine stacks, and which tasks are awaiting them. -With the ``--tree`` option, it instead renders a visual async call tree, +.. code-block:: bash + + python -m asyncio pstree PID + +This tool fetches the same information, but renders a visual async call tree, showing coroutine relationships in a hierarchical format. This command is particularly useful for debugging long-running or stuck asynchronous programs. It can help developers quickly identify where a program is blocked, what tasks @@ -565,7 +569,7 @@ Executing the new tool on the running process will yield a table like this: .. code-block:: bash - python -m asyncio.tools 12345 + python -m asyncio ps 12345 tid task id task name coroutine chain awaiter name awaiter id --------------------------------------------------------------------------------------------------------------------------------------- @@ -576,11 +580,11 @@ Executing the new tool on the running process will yield a table like this: 6826911 0x200013c0e20 Task-7 task_group Task-3 0x200013c0420 -and with the ``--tree`` option: +or: .. code-block:: bash - python -m asyncio.tools --tree 12345 + python -m asyncio pstree 12345 └── (T) Task-1 └── main diff --git a/Lib/asyncio/__main__.py b/Lib/asyncio/__main__.py index 69f5a30cfe5095..7d980bc401ae3b 100644 --- a/Lib/asyncio/__main__.py +++ b/Lib/asyncio/__main__.py @@ -1,5 +1,7 @@ +import argparse import ast import asyncio +import asyncio.tools import concurrent.futures import contextvars import inspect @@ -140,6 +142,36 @@ def interrupt(self) -> None: if __name__ == '__main__': + parser = argparse.ArgumentParser( + prog="python3 -m asyncio", + description="Interactive asyncio shell and CLI tools", + ) + subparsers = parser.add_subparsers(help="sub-commands", dest="command") + ps = subparsers.add_parser( + "ps", help="Display a table of all pending tasks in a process" + ) + ps.add_argument("pid", type=int, help="Process ID to inspect") + pstree = subparsers.add_parser( + "pstree", help="Display a tree of all pending tasks in a process" + ) + pstree.add_argument("pid", type=int, help="Process ID to inspect") + args = parser.parse_args() + match args.command: + case "ps": + asyncio.tools.display_awaited_by_tasks_table(args.pid) + sys.exit(0) + case "pstree": + asyncio.tools.display_awaited_by_tasks_tree(args.pid) + sys.exit(0) + case None: + pass # continue to the interactive shell + case _: + # shouldn't happen as an invalid command-line wouldn't parse + # but let's keep it for the next person adding a command + print(f"error: unhandled command {args.command}", file=sys.stderr) + parser.print_usage(file=sys.stderr) + sys.exit(1) + sys.audit("cpython.run_stdin") if os.getenv('PYTHON_BASIC_REPL'): diff --git a/Lib/asyncio/tools.py b/Lib/asyncio/tools.py index 4daace72471465..6d59ea4ebebf1c 100644 --- a/Lib/asyncio/tools.py +++ b/Lib/asyncio/tools.py @@ -1,4 +1,3 @@ -import argparse from dataclasses import dataclass from collections import defaultdict from itertools import count @@ -107,10 +106,12 @@ def dfs(v): # ─── PRINT TREE FUNCTION ─────────────────────────────────────── -def print_async_tree(result, task_emoji="(T)", cor_emoji="", printer=print): +def build_async_tree(result, task_emoji="(T)", cor_emoji="", printer=print): """ - Pretty-print the async call tree produced by `get_all_async_stacks()`, - prefixing tasks with *task_emoji* and coroutine frames with *cor_emoji*. + Build a list of strings for pretty-print a async call tree. + + The call tree is produced by `get_all_async_stacks()`, prefixing tasks + with `task_emoji` and coroutine frames with `cor_emoji`. """ id2name, awaits = _index(result) g = _task_graph(awaits) @@ -179,40 +180,39 @@ def _print_cycle_exception(exception: CycleFoundException): print(f"cycle: {inames}", file=sys.stderr) - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Show Python async tasks in a process") - parser.add_argument("pid", type=int, help="Process ID(s) to inspect.") - parser.add_argument( - "--tree", "-t", action="store_true", help="Display tasks in a tree format" - ) - args = parser.parse_args() - +def _get_awaited_by_tasks(pid: int) -> list: try: - tasks = get_all_awaited_by(args.pid) + return get_all_awaited_by(pid) except RuntimeError as e: while e.__context__ is not None: e = e.__context__ print(f"Error retrieving tasks: {e}") sys.exit(1) - if args.tree: - # Print the async call tree - try: - result = print_async_tree(tasks) - except CycleFoundException as e: - _print_cycle_exception(e) - sys.exit(1) - - for tree in result: - print("\n".join(tree)) - else: - # Build and print the task table - table = build_task_table(tasks) - # Print the table in a simple tabular format - print( - f"{'tid':<10} {'task id':<20} {'task name':<20} {'coroutine chain':<50} {'awaiter name':<20} {'awaiter id':<15}" - ) - print("-" * 135) - for row in table: - print(f"{row[0]:<10} {row[1]:<20} {row[2]:<20} {row[3]:<50} {row[4]:<20} {row[5]:<15}") + +def display_awaited_by_tasks_table(pid: int) -> None: + """Build and print a table of all pending tasks under `pid`.""" + + tasks = _get_awaited_by_tasks(pid) + table = build_task_table(tasks) + # Print the table in a simple tabular format + print( + f"{'tid':<10} {'task id':<20} {'task name':<20} {'coroutine chain':<50} {'awaiter name':<20} {'awaiter id':<15}" + ) + print("-" * 135) + for row in table: + print(f"{row[0]:<10} {row[1]:<20} {row[2]:<20} {row[3]:<50} {row[4]:<20} {row[5]:<15}") + + +def display_awaited_by_tasks_tree(pid: int) -> None: + """Build and print a tree of all pending tasks under `pid`.""" + + tasks = _get_awaited_by_tasks(pid) + try: + result = print_async_tree(tasks) + except CycleFoundException as e: + _print_cycle_exception(e) + sys.exit(1) + + for tree in result: + print("\n".join(tree)) diff --git a/Lib/test/test_asyncio/test_tools.py b/Lib/test/test_asyncio/test_tools.py index c9abdc60387310..2caf56172c9193 100644 --- a/Lib/test/test_asyncio/test_tools.py +++ b/Lib/test/test_asyncio/test_tools.py @@ -1,5 +1,3 @@ -"""Tests for the asyncio tools script.""" - import unittest from asyncio import tools @@ -595,13 +593,13 @@ class TestAsyncioToolsTree(unittest.TestCase): def test_asyncio_utils(self): for input_, tree in TEST_INPUTS_TREE: with self.subTest(input_): - self.assertEqual(tools.print_async_tree(input_), tree) + self.assertEqual(tools.build_async_tree(input_), tree) def test_asyncio_utils_cycles(self): for input_, cycles in TEST_INPUTS_CYCLES_TREE: with self.subTest(input_): try: - tools.print_async_tree(input_) + tools.build_async_tree(input_) except tools.CycleFoundException as e: self.assertEqual(e.cycles, cycles) @@ -615,10 +613,10 @@ def test_asyncio_utils(self): class TestAsyncioToolsBasic(unittest.TestCase): def test_empty_input_tree(self): - """Test print_async_tree with empty input.""" + """Test build_async_tree with empty input.""" result = [] expected_output = [] - self.assertEqual(tools.print_async_tree(result), expected_output) + self.assertEqual(tools.build_async_tree(result), expected_output) def test_empty_input_table(self): """Test build_task_table with empty input.""" @@ -629,7 +627,7 @@ def test_empty_input_table(self): def test_only_independent_tasks_tree(self): input_ = [(1, [(10, "taskA", []), (11, "taskB", [])])] expected = [["└── (T) taskA"], ["└── (T) taskB"]] - result = tools.print_async_tree(input_) + result = tools.build_async_tree(input_) self.assertEqual(sorted(result), sorted(expected)) def test_only_independent_tasks_table(self): @@ -640,7 +638,7 @@ def test_only_independent_tasks_table(self): ) def test_single_task_tree(self): - """Test print_async_tree with a single task and no awaits.""" + """Test build_async_tree with a single task and no awaits.""" result = [ ( 1, @@ -654,7 +652,7 @@ def test_single_task_tree(self): "└── (T) Task-1", ] ] - self.assertEqual(tools.print_async_tree(result), expected_output) + self.assertEqual(tools.build_async_tree(result), expected_output) def test_single_task_table(self): """Test build_task_table with a single task and no awaits.""" @@ -670,7 +668,7 @@ def test_single_task_table(self): self.assertEqual(tools.build_task_table(result), expected_output) def test_cycle_detection(self): - """Test print_async_tree raises CycleFoundException for cyclic input.""" + """Test build_async_tree raises CycleFoundException for cyclic input.""" result = [ ( 1, @@ -681,11 +679,11 @@ def test_cycle_detection(self): ) ] with self.assertRaises(tools.CycleFoundException) as context: - tools.print_async_tree(result) + tools.build_async_tree(result) self.assertEqual(context.exception.cycles, [[3, 2, 3]]) def test_complex_tree(self): - """Test print_async_tree with a more complex tree structure.""" + """Test build_async_tree with a more complex tree structure.""" result = [ ( 1, @@ -705,7 +703,7 @@ def test_complex_tree(self): " └── (T) Task-3", ] ] - self.assertEqual(tools.print_async_tree(result), expected_output) + self.assertEqual(tools.build_async_tree(result), expected_output) def test_complex_table(self): """Test build_task_table with a more complex tree structure.""" @@ -747,7 +745,7 @@ def test_deep_coroutine_chain(self): " └── (T) leaf", ] ] - result = tools.print_async_tree(input_) + result = tools.build_async_tree(input_) self.assertEqual(result, expected) def test_multiple_cycles_same_node(self): @@ -762,7 +760,7 @@ def test_multiple_cycles_same_node(self): ) ] with self.assertRaises(tools.CycleFoundException) as ctx: - tools.print_async_tree(input_) + tools.build_async_tree(input_) cycles = ctx.exception.cycles self.assertTrue(any(set(c) == {1, 2, 3} for c in cycles)) @@ -789,7 +787,7 @@ def test_task_awaits_self(self): """A task directly awaits itself – should raise a cycle.""" input_ = [(1, [(1, "Self-Awaiter", [[["loopback"], 1]])])] with self.assertRaises(tools.CycleFoundException) as ctx: - tools.print_async_tree(input_) + tools.build_async_tree(input_) self.assertIn([1, 1], ctx.exception.cycles) def test_task_with_missing_awaiter_id(self): @@ -811,7 +809,7 @@ def test_duplicate_coroutine_frames(self): ], ) ] - tree = tools.print_async_tree(input_) + tree = tools.build_async_tree(input_) # Both children should be under the same coroutine node flat = "\n".join(tree[0]) self.assertIn("frameA", flat) @@ -827,13 +825,13 @@ def test_task_with_no_name(self): """Task with no name in id2name – should still render with fallback.""" input_ = [(1, [(1, "root", [[["f1"], 2]]), (2, None, [])])] # If name is None, fallback to string should not crash - tree = tools.print_async_tree(input_) + tree = tools.build_async_tree(input_) self.assertIn("(T) None", "\n".join(tree[0])) def test_tree_rendering_with_custom_emojis(self): """Pass custom emojis to the tree renderer.""" input_ = [(1, [(1, "MainTask", [[["f1", "f2"], 2]]), (2, "SubTask", [])])] - tree = tools.print_async_tree(input_, task_emoji="🧵", cor_emoji="🔁") + tree = tools.build_async_tree(input_, task_emoji="🧵", cor_emoji="🔁") flat = "\n".join(tree[0]) self.assertIn("🧵 MainTask", flat) self.assertIn("🔁 f1", flat) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-03-19-04-03.gh-issue-91048.S8QWSw.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-03-19-04-03.gh-issue-91048.S8QWSw.rst index 0d5e7734f212d6..1d45868b7b27bc 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-03-19-04-03.gh-issue-91048.S8QWSw.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-03-19-04-03.gh-issue-91048.S8QWSw.rst @@ -1,5 +1,6 @@ -Add a new ``python -m asyncio.tools`` command-line interface to inspect +Add a new ``python -m asyncio ps PID`` command-line interface to inspect asyncio tasks in a running Python process. Displays a flat table of await -relationships or a tree view with ``--tree``, useful for debugging async +relationships. A variant showing a tree view is also available as +``python -m asyncio pstree PID``. Both are useful for debugging async code. Patch by Pablo Galindo, Łukasz Langa, Yury Selivanov, and Marta Gomez Macias. From 6f8aa6b441d95b9ffbeb9f360e007c469433b488 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sat, 3 May 2025 23:31:41 +0200 Subject: [PATCH 27/31] Satisfy the linting gods --- Lib/asyncio/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/asyncio/tools.py b/Lib/asyncio/tools.py index 6d59ea4ebebf1c..8b63b80061203a 100644 --- a/Lib/asyncio/tools.py +++ b/Lib/asyncio/tools.py @@ -109,7 +109,7 @@ def dfs(v): def build_async_tree(result, task_emoji="(T)", cor_emoji="", printer=print): """ Build a list of strings for pretty-print a async call tree. - + The call tree is produced by `get_all_async_stacks()`, prefixing tasks with `task_emoji` and coroutine frames with `cor_emoji`. """ From 8d566c6bc915578e98f44196fa3b4d1753b225a1 Mon Sep 17 00:00:00 2001 From: Marta Gomez Macias Date: Sun, 4 May 2025 00:00:22 +0000 Subject: [PATCH 28/31] chore: Refactor --- Lib/asyncio/tools.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/Lib/asyncio/tools.py b/Lib/asyncio/tools.py index 8b63b80061203a..ee96dd80b8f04f 100644 --- a/Lib/asyncio/tools.py +++ b/Lib/asyncio/tools.py @@ -1,3 +1,5 @@ +"""Tool to analyze tasks running in a asyncio script.""" + from dataclasses import dataclass from collections import defaultdict from itertools import count @@ -46,10 +48,6 @@ def _cor_node(parent_key, frame_name): bucket[frame_name] = node_key return node_key - # touch every task so it’s present even if it awaits nobody - for tid in id2name: - children[(NodeType.TASK, tid)] - # lay down parent ➜ …frames… ➜ child paths for parent_id, stack, child_id in awaits: cur = (NodeType.TASK, parent_id) @@ -69,7 +67,6 @@ def _roots(id2label, children): # ─── detect cycles in the task-to-task graph ─────────────────────── def _task_graph(awaits): """Return {parent_task_id: {child_task_id, …}, …}.""" - from collections import defaultdict g = defaultdict(set) for parent_id, _stack, child_id in awaits: g[parent_id].add(child_id) @@ -106,7 +103,7 @@ def dfs(v): # ─── PRINT TREE FUNCTION ─────────────────────────────────────── -def build_async_tree(result, task_emoji="(T)", cor_emoji="", printer=print): +def build_async_tree(result, task_emoji="(T)", cor_emoji=""): """ Build a list of strings for pretty-print a async call tree. @@ -134,10 +131,7 @@ def render(node, prefix="", last=True, buf=None): render(kid, new_pref, i == len(kids) - 1, buf) return buf - result = [] - for r, root in enumerate(_roots(labels, children)): - result.append(render(root)) - return result + return [render(root) for root in _roots(labels, children)] def build_task_table(result): @@ -209,7 +203,7 @@ def display_awaited_by_tasks_tree(pid: int) -> None: tasks = _get_awaited_by_tasks(pid) try: - result = print_async_tree(tasks) + result = build_async_tree(tasks) except CycleFoundException as e: _print_cycle_exception(e) sys.exit(1) From 9dbe00d276b3bd8eb1201e1450a2db63239739bf Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sun, 4 May 2025 02:17:02 +0200 Subject: [PATCH 29/31] Doc fixes --- Doc/whatsnew/3.14.rst | 67 ++++++++++++++++++++++--------------------- 1 file changed, 35 insertions(+), 32 deletions(-) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 65d0a319a07cf2..33bfc783480973 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -541,26 +541,21 @@ For example given this code: import asyncio - async def sleeper(name, delay): - await asyncio.sleep(delay) - print(f"{name} is done sleeping.") + async def play(track): + await asyncio.sleep(5) + print(f"🎵 Finished: {track}") - async def inner(name): - await asyncio.sleep(0.1) - await sleeper(name, 1) - - async def task_group(name): - await asyncio.gather( - inner(f"{name}-1"), - inner(f"{name}-2"), - ) + async def album(name, tracks): + async with asyncio.TaskGroup() as tg: + for track in tracks: + tg.create_task(play(track), name=track) async def main(): - # Start two separate task groups - t1 = asyncio.create_task(task_group("groupA")) - t2 = asyncio.create_task(task_group("groupB")) - await t1 - await t2 + async with asyncio.TaskGroup() as tg: + tg.create_task( + album("Sundowning", ["TNDNBTG", "Levitate"]), name="Sundowning") + tg.create_task( + album("TMBTE", ["DYWTYLM", "Aqua Regia"]), name="TMBTE") if __name__ == "__main__": asyncio.run(main()) @@ -571,13 +566,15 @@ Executing the new tool on the running process will yield a table like this: python -m asyncio ps 12345 - tid task id task name coroutine chain awaiter name awaiter id + tid task id task name coroutine chain awaiter name awaiter id --------------------------------------------------------------------------------------------------------------------------------------- - 6826911 0x200013c0220 Task-2 main Task-1 0x200013b0020 - 6826911 0x200013c0620 Task-4 task_group Task-2 0x200013c0220 - 6826911 0x200013c0820 Task-5 task_group Task-2 0x200013c0220 - 6826911 0x200013c0c20 Task-6 task_group Task-3 0x200013c0420 - 6826911 0x200013c0e20 Task-7 task_group Task-3 0x200013c0420 + 8138752 0x564bd3d0210 Task-1 0x0 + 8138752 0x564bd3d0410 Sundowning _aexit -> __aexit__ -> main Task-1 0x564bd3d0210 + 8138752 0x564bd3d0610 TMBTE _aexit -> __aexit__ -> main Task-1 0x564bd3d0210 + 8138752 0x564bd3d0810 TNDNBTG _aexit -> __aexit__ -> album Sundowning 0x564bd3d0410 + 8138752 0x564bd3d0a10 Levitate _aexit -> __aexit__ -> album Sundowning 0x564bd3d0410 + 8138752 0x564bd3e0550 DYWTYLM _aexit -> __aexit__ -> album TMBTE 0x564bd3d0610 + 8138752 0x564bd3e0710 Aqua Regia _aexit -> __aexit__ -> album TMBTE 0x564bd3d0610 or: @@ -587,15 +584,21 @@ or: python -m asyncio pstree 12345 └── (T) Task-1 - └── main - └── (T) Task-2 - └── task_group - ├── (T) Task-4 - └── (T) Task-5 - └── (T) Task-3 - └── task_group - ├── (T) Task-6 - └── (T) Task-7 + └── main + └── __aexit__ + └── _aexit + ├── (T) Sundowning + │ └── album + │ └── __aexit__ + │ └── _aexit + │ ├── (T) TNDNBTG + │ └── (T) Levitate + └── (T) TMBTE + └── album + └── __aexit__ + └── _aexit + ├── (T) DYWTYLM + └── (T) Aqua Regia If a cycle is detected in the async await graph (which could indicate a programming issue), the tool raises an error and lists the cycle paths that From c56782beac2a0d255be2b4933091d56e39311eab Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sun, 4 May 2025 02:21:45 +0200 Subject: [PATCH 30/31] Type fixes --- Modules/_remotedebuggingmodule.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_remotedebuggingmodule.c b/Modules/_remotedebuggingmodule.c index f6e8135a6c5800..0e055ae1604d5f 100644 --- a/Modules/_remotedebuggingmodule.c +++ b/Modules/_remotedebuggingmodule.c @@ -517,7 +517,7 @@ parse_task( tn = parse_task_name( handle, offsets, async_offsets, task_address); } else { - tn = PyLong_FromUnsignedLong(task_address); + tn = PyLong_FromUnsignedLongLong(task_address); } if (tn == NULL) { goto err; @@ -1079,7 +1079,7 @@ append_awaited_by_for_thread( return -1; } - PyObject* task_id = PyLong_FromUnsignedLong(task_addr); + PyObject* task_id = PyLong_FromUnsignedLongLong(task_addr); if (task_id == NULL) { Py_DECREF(tn); Py_DECREF(current_awaited_by); From 293337f560b4df15ea4f7d67c4c937cf66717ca1 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sun, 4 May 2025 02:24:33 +0200 Subject: [PATCH 31/31] Type fixes --- Lib/asyncio/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/asyncio/tools.py b/Lib/asyncio/tools.py index ee96dd80b8f04f..16440b594ad993 100644 --- a/Lib/asyncio/tools.py +++ b/Lib/asyncio/tools.py @@ -1,4 +1,4 @@ -"""Tool to analyze tasks running in a asyncio script.""" +"""Tools to analyze tasks running in asyncio programs.""" from dataclasses import dataclass from collections import defaultdict