Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit ea25f32

Browse filesBrowse files
zoobazooba
authored
gh-89240: Enable multiprocessing on Windows to use large process pools (GH-107873)
We add _winapi.BatchedWaitForMultipleObjects to wait for larger numbers of handles. This is an internal module, hence undocumented, and should be used with caution. Check the docstring for info before using BatchedWaitForMultipleObjects.
1 parent 2f07786 commit ea25f32
Copy full SHA for ea25f32

File tree

Expand file treeCollapse file tree

12 files changed

+1195
-6
lines changed
Filter options
Expand file treeCollapse file tree

12 files changed

+1195
-6
lines changed

‎Include/internal/pycore_global_objects_fini_generated.h

Copy file name to clipboardExpand all lines: Include/internal/pycore_global_objects_fini_generated.h
+10Lines changed: 10 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Include/internal/pycore_global_strings.h

Copy file name to clipboardExpand all lines: Include/internal/pycore_global_strings.h
+10Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -372,6 +372,7 @@ struct _Py_global_strings {
372372
STRUCT_FOR_ID(defaultaction)
373373
STRUCT_FOR_ID(delete)
374374
STRUCT_FOR_ID(depth)
375+
STRUCT_FOR_ID(desired_access)
375376
STRUCT_FOR_ID(detect_types)
376377
STRUCT_FOR_ID(deterministic)
377378
STRUCT_FOR_ID(device)
@@ -462,6 +463,7 @@ struct _Py_global_strings {
462463
STRUCT_FOR_ID(groups)
463464
STRUCT_FOR_ID(h)
464465
STRUCT_FOR_ID(handle)
466+
STRUCT_FOR_ID(handle_seq)
465467
STRUCT_FOR_ID(hash_name)
466468
STRUCT_FOR_ID(header)
467469
STRUCT_FOR_ID(headers)
@@ -479,9 +481,12 @@ struct _Py_global_strings {
479481
STRUCT_FOR_ID(indexgroup)
480482
STRUCT_FOR_ID(inf)
481483
STRUCT_FOR_ID(infer_variance)
484+
STRUCT_FOR_ID(inherit_handle)
482485
STRUCT_FOR_ID(inheritable)
483486
STRUCT_FOR_ID(initial)
484487
STRUCT_FOR_ID(initial_bytes)
488+
STRUCT_FOR_ID(initial_owner)
489+
STRUCT_FOR_ID(initial_state)
485490
STRUCT_FOR_ID(initial_value)
486491
STRUCT_FOR_ID(initval)
487492
STRUCT_FOR_ID(inner_size)
@@ -537,6 +542,7 @@ struct _Py_global_strings {
537542
STRUCT_FOR_ID(locals)
538543
STRUCT_FOR_ID(logoption)
539544
STRUCT_FOR_ID(loop)
545+
STRUCT_FOR_ID(manual_reset)
540546
STRUCT_FOR_ID(mapping)
541547
STRUCT_FOR_ID(match)
542548
STRUCT_FOR_ID(max_length)
@@ -553,6 +559,7 @@ struct _Py_global_strings {
553559
STRUCT_FOR_ID(metadata)
554560
STRUCT_FOR_ID(method)
555561
STRUCT_FOR_ID(microsecond)
562+
STRUCT_FOR_ID(milliseconds)
556563
STRUCT_FOR_ID(minute)
557564
STRUCT_FOR_ID(mod)
558565
STRUCT_FOR_ID(mode)
@@ -562,6 +569,7 @@ struct _Py_global_strings {
562569
STRUCT_FOR_ID(month)
563570
STRUCT_FOR_ID(mro)
564571
STRUCT_FOR_ID(msg)
572+
STRUCT_FOR_ID(mutex)
565573
STRUCT_FOR_ID(mycmp)
566574
STRUCT_FOR_ID(n)
567575
STRUCT_FOR_ID(n_arg)
@@ -665,6 +673,7 @@ struct _Py_global_strings {
665673
STRUCT_FOR_ID(sched_priority)
666674
STRUCT_FOR_ID(scheduler)
667675
STRUCT_FOR_ID(second)
676+
STRUCT_FOR_ID(security_attributes)
668677
STRUCT_FOR_ID(seek)
669678
STRUCT_FOR_ID(seekable)
670679
STRUCT_FOR_ID(selectors)
@@ -752,6 +761,7 @@ struct _Py_global_strings {
752761
STRUCT_FOR_ID(values)
753762
STRUCT_FOR_ID(version)
754763
STRUCT_FOR_ID(volume)
764+
STRUCT_FOR_ID(wait_all)
755765
STRUCT_FOR_ID(warnings)
756766
STRUCT_FOR_ID(warnoptions)
757767
STRUCT_FOR_ID(wbits)

‎Include/internal/pycore_runtime_init_generated.h

Copy file name to clipboardExpand all lines: Include/internal/pycore_runtime_init_generated.h
+10Lines changed: 10 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Include/internal/pycore_unicodeobject_generated.h

Copy file name to clipboardExpand all lines: Include/internal/pycore_unicodeobject_generated.h
+30Lines changed: 30 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Lib/multiprocessing/connection.py

Copy file name to clipboardExpand all lines: Lib/multiprocessing/connection.py
+13-1Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1011,8 +1011,20 @@ def _exhaustive_wait(handles, timeout):
10111011
# returning the first signalled might create starvation issues.)
10121012
L = list(handles)
10131013
ready = []
1014+
# Windows limits WaitForMultipleObjects at 64 handles, and we use a
1015+
# few for synchronisation, so we switch to batched waits at 60.
1016+
if len(L) > 60:
1017+
try:
1018+
res = _winapi.BatchedWaitForMultipleObjects(L, False, timeout)
1019+
except TimeoutError:
1020+
return []
1021+
ready.extend(L[i] for i in res)
1022+
if res:
1023+
L = [h for i, h in enumerate(L) if i > res[0] & i not in res]
1024+
timeout = 0
10141025
while L:
1015-
res = _winapi.WaitForMultipleObjects(L, False, timeout)
1026+
short_L = L[:60] if len(L) > 60 else L
1027+
res = _winapi.WaitForMultipleObjects(short_L, False, timeout)
10161028
if res == WAIT_TIMEOUT:
10171029
break
10181030
elif WAIT_OBJECT_0 <= res < WAIT_OBJECT_0 + len(L):

‎Lib/test/_test_multiprocessing.py

Copy file name to clipboardExpand all lines: Lib/test/_test_multiprocessing.py
+18Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6113,6 +6113,24 @@ def test_spawn_sys_executable_none_allows_import(self):
61136113
self.assertEqual(rc, 0)
61146114
self.assertFalse(err, msg=err.decode('utf-8'))
61156115

6116+
def test_large_pool(self):
6117+
#
6118+
# gh-89240: Check that large pools are always okay
6119+
#
6120+
testfn = os_helper.TESTFN
6121+
self.addCleanup(os_helper.unlink, testfn)
6122+
with open(testfn, 'w', encoding='utf-8') as f:
6123+
f.write(textwrap.dedent('''\
6124+
import multiprocessing
6125+
def f(x): return x*x
6126+
if __name__ == '__main__':
6127+
with multiprocessing.Pool(200) as p:
6128+
print(sum(p.map(f, range(1000))))
6129+
'''))
6130+
rc, out, err = script_helper.assert_python_ok(testfn)
6131+
self.assertEqual("332833500", out.decode('utf-8').strip())
6132+
self.assertFalse(err, msg=err.decode('utf-8'))
6133+
61166134

61176135
#
61186136
# Mixins

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.