From a528ab9f3efc8fcff1976b65eb117efba3267993 Mon Sep 17 00:00:00 2001 From: Jake Tesler Date: Thu, 31 Oct 2019 12:22:27 -0700 Subject: [PATCH 1/9] Implement fix for MainThread.native_id for multiprocessing.Process objects created via fork start method --- Lib/multiprocessing/process.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/multiprocessing/process.py b/Lib/multiprocessing/process.py index c62c826cff9580..be13c079bb89b3 100644 --- a/Lib/multiprocessing/process.py +++ b/Lib/multiprocessing/process.py @@ -301,6 +301,8 @@ def _bootstrap(self, parent_sentinel=None): _current_process = self _parent_process = _ParentProcess( self._parent_name, self._parent_pid, parent_sentinel) + if threading._HAVE_THREAD_NATIVE_ID: + threading.main_thread()._set_native_id() try: util._finalizer_registry.clear() util._run_after_forkers() From 462b906f12aaeae66351706dca2664b82cb62f9b Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Fri, 8 Nov 2019 00:36:13 +0000 Subject: [PATCH 2/9] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20blu?= =?UTF-8?q?rb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Core and Builtins/2019-11-08-00-36-10.bpo-38707.SZL036.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2019-11-08-00-36-10.bpo-38707.SZL036.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-11-08-00-36-10.bpo-38707.SZL036.rst b/Misc/NEWS.d/next/Core and Builtins/2019-11-08-00-36-10.bpo-38707.SZL036.rst new file mode 100644 index 00000000000000..23a2a4e1718401 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2019-11-08-00-36-10.bpo-38707.SZL036.rst @@ -0,0 +1 @@ +`multiprocessing.Process` objects now retain their correct `MainThread.native_id` attribute \ No newline at end of file From 52c89898e437f56b174b642dc64b0951e5383ed6 Mon Sep 17 00:00:00 2001 From: Jake Tesler Date: Fri, 8 Nov 2019 10:51:03 -0800 Subject: [PATCH 3/9] Update Misc/NEWS.d/next/Core and Builtins/2019-11-08-00-36-10.bpo-38707.SZL036.rst Co-Authored-By: Brandt Bucher --- .../Core and Builtins/2019-11-08-00-36-10.bpo-38707.SZL036.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-11-08-00-36-10.bpo-38707.SZL036.rst b/Misc/NEWS.d/next/Core and Builtins/2019-11-08-00-36-10.bpo-38707.SZL036.rst index 23a2a4e1718401..c24118dcf28a05 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2019-11-08-00-36-10.bpo-38707.SZL036.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2019-11-08-00-36-10.bpo-38707.SZL036.rst @@ -1 +1 @@ -`multiprocessing.Process` objects now retain their correct `MainThread.native_id` attribute \ No newline at end of file +:class:`multiprocessing.Process` objects now retain their correct ``MainThread.native_id`` attribute. From c98cca0890c9bdee856a51a0f82df0e59072d069 Mon Sep 17 00:00:00 2001 From: Jake Tesler Date: Fri, 8 Nov 2019 11:54:44 -0800 Subject: [PATCH 4/9] Add multiprocessing native_id test --- Lib/test/_test_multiprocessing.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index f7bebc6e8370be..aed7ae03ad032b 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -361,6 +361,24 @@ def test_process(self): self.assertNotIn(p, self.active_children()) close_queue(q) + if threading._HAVE_THREAD_NATIVE_ID: + def test_process_thread_attributes(self): + current_mainthread_native_id = threading.main_thread().native_id + + q = self.Queue(1) + p = self.Process(target=self._test_process_thread_attributes, args=(q,)) + p.daemon = True + p.start() + + child_mainthread_native_id = q.get() + self.assertNotEqual(current_mainthread_native_id, child_mainthread_native_id) + close_queue(q) + + @classmethod + def _test_process_thread_attributes(cls, q): + mainthread_native_id = threading.main_thread().native_id + q.put(mainthread_native_id) + @classmethod def _sleep_some(cls): time.sleep(100) From e71cb78520481c55f3b231cd36401ee623d69bf6 Mon Sep 17 00:00:00 2001 From: Jake Tesler Date: Fri, 8 Nov 2019 12:22:49 -0800 Subject: [PATCH 5/9] Skip test if type is 'threads' Since Dummy Processes are simply Thread objects, they will share a MainThread and their native_id attribute will match. --- Lib/test/_test_multiprocessing.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index aed7ae03ad032b..cef564ba392a20 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -363,6 +363,9 @@ def test_process(self): if threading._HAVE_THREAD_NATIVE_ID: def test_process_thread_attributes(self): + if self.TYPE == 'threads': + self.skipTest('test not appropriate for {}'.format(self.TYPE)) + current_mainthread_native_id = threading.main_thread().native_id q = self.Queue(1) From e510a5e16b1b6f189142afe597996185ebf0fff6 Mon Sep 17 00:00:00 2001 From: Jake Tesler Date: Fri, 8 Nov 2019 12:34:29 -0800 Subject: [PATCH 6/9] Minor code cleanup --- Lib/test/_test_multiprocessing.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index cef564ba392a20..b81d9426984c33 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -374,9 +374,10 @@ def test_process_thread_attributes(self): p.start() child_mainthread_native_id = q.get() - self.assertNotEqual(current_mainthread_native_id, child_mainthread_native_id) close_queue(q) + self.assertNotEqual(current_mainthread_native_id, child_mainthread_native_id) + @classmethod def _test_process_thread_attributes(cls, q): mainthread_native_id = threading.main_thread().native_id From 455042d18ffa55257e3f4850a068b471da8296e7 Mon Sep 17 00:00:00 2001 From: Jake Tesler Date: Fri, 8 Nov 2019 12:59:33 -0800 Subject: [PATCH 7/9] Refactoring per suggestions from @brandtbucher --- Lib/test/_test_multiprocessing.py | 32 +++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index b81d9426984c33..b5778170367b29 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -361,27 +361,27 @@ def test_process(self): self.assertNotIn(p, self.active_children()) close_queue(q) - if threading._HAVE_THREAD_NATIVE_ID: - def test_process_thread_attributes(self): - if self.TYPE == 'threads': - self.skipTest('test not appropriate for {}'.format(self.TYPE)) + @unittest.skipUnless(threading._HAVE_THREAD_NATIVE_ID, "needs native_id") + def test_process_mainthread_native_id(self): + if self.TYPE == 'threads': + self.skipTest('test not appropriate for {}'.format(self.TYPE)) - current_mainthread_native_id = threading.main_thread().native_id + current_mainthread_native_id = threading.main_thread().native_id - q = self.Queue(1) - p = self.Process(target=self._test_process_thread_attributes, args=(q,)) - p.daemon = True - p.start() + q = self.Queue(1) + p = self.Process(target=self._test_process_mainthread_native_id, args=(q,)) + p.daemon = True + p.start() - child_mainthread_native_id = q.get() - close_queue(q) + child_mainthread_native_id = q.get() + close_queue(q) - self.assertNotEqual(current_mainthread_native_id, child_mainthread_native_id) + self.assertNotEqual(current_mainthread_native_id, child_mainthread_native_id) - @classmethod - def _test_process_thread_attributes(cls, q): - mainthread_native_id = threading.main_thread().native_id - q.put(mainthread_native_id) + @classmethod + def _test_process_mainthread_native_id(cls, q): + mainthread_native_id = threading.main_thread().native_id + q.put(mainthread_native_id) @classmethod def _sleep_some(cls): From 91ff94f663d665313b9c8ba8aad939e13c7c6feb Mon Sep 17 00:00:00 2001 From: Jake Tesler Date: Wed, 13 Nov 2019 17:58:24 -0800 Subject: [PATCH 8/9] Modified tests' child Process from daemon to using join() per @pitrou --- Lib/test/_test_multiprocessing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index b5778170367b29..611291cbbaf93b 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -370,10 +370,10 @@ def test_process_mainthread_native_id(self): q = self.Queue(1) p = self.Process(target=self._test_process_mainthread_native_id, args=(q,)) - p.daemon = True p.start() child_mainthread_native_id = q.get() + p.join() close_queue(q) self.assertNotEqual(current_mainthread_native_id, child_mainthread_native_id) From 22f6163edd51d301d733eb625da238c6240f090b Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 19 Nov 2019 20:28:08 +0100 Subject: [PATCH 9/9] Improve NEWS message --- .../Core and Builtins/2019-11-08-00-36-10.bpo-38707.SZL036.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-11-08-00-36-10.bpo-38707.SZL036.rst b/Misc/NEWS.d/next/Core and Builtins/2019-11-08-00-36-10.bpo-38707.SZL036.rst index c24118dcf28a05..4ef9ed81931b72 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2019-11-08-00-36-10.bpo-38707.SZL036.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2019-11-08-00-36-10.bpo-38707.SZL036.rst @@ -1 +1 @@ -:class:`multiprocessing.Process` objects now retain their correct ``MainThread.native_id`` attribute. +``MainThread.native_id`` is now correctly reset in child processes spawned using :class:`multiprocessing.Process`, instead of retaining the parent's value.