From 1c358465fdb2fb611e163e965426c7c1331931d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 15 Feb 2023 23:40:25 +0100 Subject: [PATCH 01/11] gh-88110: clear concurrent.futures.thread._threads_queues after fork Threads are gone after fork, so clear the queues too. Otherwise the child process (here created via multiprocessing.Process) crashes on interpreter exit with: Traceback (most recent call last): File "/usr/lib64/python3.11/multiprocessing/popen_fork.py", line 72, in _launch code = process_obj._bootstrap(parent_sentinel=child_r) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/lib64/python3.11/multiprocessing/process.py", line 332, in _bootstrap threading._shutdown() File "/usr/lib64/python3.11/threading.py", line 1561, in _shutdown atexit_call() File "/usr/lib64/python3.11/concurrent/futures/thread.py", line 31, in _python_exit t.join() File "/usr/lib64/python3.11/threading.py", line 1109, in join raise RuntimeError("cannot join current thread") RuntimeError: cannot join current thread Fixes #88110 --- Lib/concurrent/futures/thread.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/concurrent/futures/thread.py b/Lib/concurrent/futures/thread.py index 16cc5533d429ef..909359b648709f 100644 --- a/Lib/concurrent/futures/thread.py +++ b/Lib/concurrent/futures/thread.py @@ -41,6 +41,7 @@ def _python_exit(): os.register_at_fork(before=_global_shutdown_lock.acquire, after_in_child=_global_shutdown_lock._at_fork_reinit, after_in_parent=_global_shutdown_lock.release) + os.register_at_fork(after_in_child=_threads_queues.clear) class WorkerContext: From 556481102ded0f5959bacc74b62a1d62fb7f8758 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 15 Feb 2023 23:54:57 +0100 Subject: [PATCH 02/11] Add NEWS entry --- .../next/Library/2023-02-15-23-54-42.gh-issue-88110.KU6erv.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-02-15-23-54-42.gh-issue-88110.KU6erv.rst diff --git a/Misc/NEWS.d/next/Library/2023-02-15-23-54-42.gh-issue-88110.KU6erv.rst b/Misc/NEWS.d/next/Library/2023-02-15-23-54-42.gh-issue-88110.KU6erv.rst new file mode 100644 index 00000000000000..9f6003a1f0556a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-15-23-54-42.gh-issue-88110.KU6erv.rst @@ -0,0 +1,2 @@ +Fix multiprocessing.Process exiting with code 1 even on successful run when +threads are used in the same application too. From d035c6b4f8ff731cbfd8e1ce8fa6ae628e3b5b8d Mon Sep 17 00:00:00 2001 From: Andrei Bodrov Date: Tue, 29 Oct 2024 02:50:12 +0300 Subject: [PATCH 03/11] Add test --- .../test_thread_pool.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Lib/test/test_concurrent_futures/test_thread_pool.py b/Lib/test/test_concurrent_futures/test_thread_pool.py index 2b5bea9f4055a2..c980bbe68f737f 100644 --- a/Lib/test/test_concurrent_futures/test_thread_pool.py +++ b/Lib/test/test_concurrent_futures/test_thread_pool.py @@ -66,6 +66,25 @@ def submit(pool): with futures.ProcessPoolExecutor(1, mp_context=mp.get_context('fork')) as workers: workers.submit(tuple) + @support.requires_fork() + @unittest.skipUnless(hasattr(os, 'register_at_fork'), 'need os.register_at_fork') + @support.requires_resource('cpu') + def test_process_fork_from_a_threadpool(self): + # bpo-43944: clear concurrent.futures.thread._threads_queues after fork, + # otherwise child process will try to join parent thread + def fork_process_and_return_exitcode(): + p = mp.get_context('fork').Process(target=lambda: 1) + p.start() + p.join() + return p.exitcode + + with futures.ThreadPoolExecutor(1) as pool: + process_exitcode = pool.submit(fork_process_and_return_exitcode).result() + + self.assertEqual(process_exitcode, 0) + + + def test_executor_map_current_future_cancel(self): stop_event = threading.Event() log = [] From cf1d9f1659fb123405096ade683e7ab54554d1a3 Mon Sep 17 00:00:00 2001 From: Andrei Bodrov Date: Tue, 29 Oct 2024 02:58:10 +0300 Subject: [PATCH 04/11] Fix indentation --- Lib/test/test_concurrent_futures/test_thread_pool.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Lib/test/test_concurrent_futures/test_thread_pool.py b/Lib/test/test_concurrent_futures/test_thread_pool.py index c980bbe68f737f..20b05b16377415 100644 --- a/Lib/test/test_concurrent_futures/test_thread_pool.py +++ b/Lib/test/test_concurrent_futures/test_thread_pool.py @@ -80,10 +80,8 @@ def fork_process_and_return_exitcode(): with futures.ThreadPoolExecutor(1) as pool: process_exitcode = pool.submit(fork_process_and_return_exitcode).result() - - self.assertEqual(process_exitcode, 0) - + self.assertEqual(process_exitcode, 0) def test_executor_map_current_future_cancel(self): stop_event = threading.Event() From 18b91ac39bba755c1be8b0f381e0f638a05a365d Mon Sep 17 00:00:00 2001 From: Andrei Bodrov Date: Tue, 29 Oct 2024 15:35:50 +0300 Subject: [PATCH 05/11] Fix formatting in description Co-authored-by: RUANG (James Roy) --- .../next/Library/2023-02-15-23-54-42.gh-issue-88110.KU6erv.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2023-02-15-23-54-42.gh-issue-88110.KU6erv.rst b/Misc/NEWS.d/next/Library/2023-02-15-23-54-42.gh-issue-88110.KU6erv.rst index 9f6003a1f0556a..a5813ca2d3a95e 100644 --- a/Misc/NEWS.d/next/Library/2023-02-15-23-54-42.gh-issue-88110.KU6erv.rst +++ b/Misc/NEWS.d/next/Library/2023-02-15-23-54-42.gh-issue-88110.KU6erv.rst @@ -1,2 +1,2 @@ -Fix multiprocessing.Process exiting with code 1 even on successful run when +Fix :class:`multiprocessing.Process` exiting with code 1 even on successful run when threads are used in the same application too. From 85d7b8fa7a59cfbd513eafd34ff71ba6cc0b321f Mon Sep 17 00:00:00 2001 From: Andrei Bodrov Date: Sun, 17 Nov 2024 23:34:00 +0300 Subject: [PATCH 06/11] Ignore the warning about fork with threads --- .../test_concurrent_futures/test_thread_pool.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_concurrent_futures/test_thread_pool.py b/Lib/test/test_concurrent_futures/test_thread_pool.py index 20b05b16377415..7d3220982f4944 100644 --- a/Lib/test/test_concurrent_futures/test_thread_pool.py +++ b/Lib/test/test_concurrent_futures/test_thread_pool.py @@ -4,6 +4,7 @@ import multiprocessing.util import os import threading +import warnings import unittest from concurrent import futures from test import support @@ -67,20 +68,22 @@ def submit(pool): workers.submit(tuple) @support.requires_fork() - @unittest.skipUnless(hasattr(os, 'register_at_fork'), 'need os.register_at_fork') - @support.requires_resource('cpu') + @unittest.skipUnless(hasattr(os, "register_at_fork"), "need os.register_at_fork") + @support.requires_resource("cpu") def test_process_fork_from_a_threadpool(self): # bpo-43944: clear concurrent.futures.thread._threads_queues after fork, # otherwise child process will try to join parent thread def fork_process_and_return_exitcode(): - p = mp.get_context('fork').Process(target=lambda: 1) - p.start() + # Ignore the warning about fork with threads. + with warnings.catch_warnings(category=DeprecationWarning, action="iframe.php?url=https%3A%2F%2Fgithub.com%2Fignore"): + p = mp.get_context("fork").Process(target=lambda: 1) + p.start() p.join() return p.exitcode - + with futures.ThreadPoolExecutor(1) as pool: process_exitcode = pool.submit(fork_process_and_return_exitcode).result() - + self.assertEqual(process_exitcode, 0) def test_executor_map_current_future_cancel(self): From ce72fb3148c5a0cf5908c9e76722e65bab6d4d65 Mon Sep 17 00:00:00 2001 From: Andrei Bodrov Date: Sun, 17 Nov 2024 23:45:13 +0300 Subject: [PATCH 07/11] Fix lint --- Lib/test/test_concurrent_futures/test_thread_pool.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_concurrent_futures/test_thread_pool.py b/Lib/test/test_concurrent_futures/test_thread_pool.py index 7d3220982f4944..420064a7180f73 100644 --- a/Lib/test/test_concurrent_futures/test_thread_pool.py +++ b/Lib/test/test_concurrent_futures/test_thread_pool.py @@ -80,10 +80,10 @@ def fork_process_and_return_exitcode(): p.start() p.join() return p.exitcode - + with futures.ThreadPoolExecutor(1) as pool: process_exitcode = pool.submit(fork_process_and_return_exitcode).result() - + self.assertEqual(process_exitcode, 0) def test_executor_map_current_future_cancel(self): From b9dcbe0a11f9c2b0ab6c3a2422020ab912a7fbd3 Mon Sep 17 00:00:00 2001 From: Andrei Bodrov Date: Wed, 20 Nov 2024 15:08:11 +0300 Subject: [PATCH 08/11] Fix comments --- Lib/test/test_concurrent_futures/test_thread_pool.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_concurrent_futures/test_thread_pool.py b/Lib/test/test_concurrent_futures/test_thread_pool.py index 420064a7180f73..cfa875f9c25eee 100644 --- a/Lib/test/test_concurrent_futures/test_thread_pool.py +++ b/Lib/test/test_concurrent_futures/test_thread_pool.py @@ -4,7 +4,6 @@ import multiprocessing.util import os import threading -import warnings import unittest from concurrent import futures from test import support @@ -68,15 +67,15 @@ def submit(pool): workers.submit(tuple) @support.requires_fork() - @unittest.skipUnless(hasattr(os, "register_at_fork"), "need os.register_at_fork") - @support.requires_resource("cpu") + @unittest.skipUnless(hasattr(os, 'register_at_fork'), 'need os.register_at_fork') + @support.requires_resource('cpu') def test_process_fork_from_a_threadpool(self): # bpo-43944: clear concurrent.futures.thread._threads_queues after fork, # otherwise child process will try to join parent thread def fork_process_and_return_exitcode(): # Ignore the warning about fork with threads. - with warnings.catch_warnings(category=DeprecationWarning, action="iframe.php?url=https%3A%2F%2Fgithub.com%2Fignore"): - p = mp.get_context("fork").Process(target=lambda: 1) + with self.assertWarnsRegex(DeprecationWarning, msg="use of fork() may lead to deadlocks in child"): + p = mp.get_context('fork').Process(target=lambda: 1) p.start() p.join() return p.exitcode From 209260ef2d73923b25c97edbe6f35384def2cc99 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Wed, 20 Nov 2024 08:43:36 -0800 Subject: [PATCH 09/11] reword the NEWS entry --- .../Library/2023-02-15-23-54-42.gh-issue-88110.KU6erv.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Misc/NEWS.d/next/Library/2023-02-15-23-54-42.gh-issue-88110.KU6erv.rst b/Misc/NEWS.d/next/Library/2023-02-15-23-54-42.gh-issue-88110.KU6erv.rst index a5813ca2d3a95e..0ae97cca795797 100644 --- a/Misc/NEWS.d/next/Library/2023-02-15-23-54-42.gh-issue-88110.KU6erv.rst +++ b/Misc/NEWS.d/next/Library/2023-02-15-23-54-42.gh-issue-88110.KU6erv.rst @@ -1,2 +1,2 @@ -Fix :class:`multiprocessing.Process` exiting with code 1 even on successful run when -threads are used in the same application too. +Fixed :class:`multiprocessing.Process` exiting with code 1 even on success when +it was used from a :class:`concurrent.futures.ThreadPoolExecutor` task. From b8d3f31981ab510a740f226cb9913fe14ed72c20 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Wed, 20 Nov 2024 08:48:59 -0800 Subject: [PATCH 10/11] more NEWS wordsmithing for accuracy Colonel Mustard did it in the Library with the Lead Pipe. --- .../Library/2023-02-15-23-54-42.gh-issue-88110.KU6erv.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Misc/NEWS.d/next/Library/2023-02-15-23-54-42.gh-issue-88110.KU6erv.rst b/Misc/NEWS.d/next/Library/2023-02-15-23-54-42.gh-issue-88110.KU6erv.rst index 0ae97cca795797..42a83edc3ba68d 100644 --- a/Misc/NEWS.d/next/Library/2023-02-15-23-54-42.gh-issue-88110.KU6erv.rst +++ b/Misc/NEWS.d/next/Library/2023-02-15-23-54-42.gh-issue-88110.KU6erv.rst @@ -1,2 +1,2 @@ -Fixed :class:`multiprocessing.Process` exiting with code 1 even on success when -it was used from a :class:`concurrent.futures.ThreadPoolExecutor` task. +Fixed :class:`multiprocessing.Process` reporting a ``.exitcode`` of 1 even on success when +using the ``"fork"`` start method while using a :class:`concurrent.futures.ThreadPoolExecutor`. From bbaad99d9c7ae74c23889f5720541d557fa3fdca Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 22 Nov 2024 17:23:45 +0200 Subject: [PATCH 11/11] Fix the test. --- Lib/test/test_concurrent_futures/test_thread_pool.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_concurrent_futures/test_thread_pool.py b/Lib/test/test_concurrent_futures/test_thread_pool.py index cfa875f9c25eee..4324241b374967 100644 --- a/Lib/test/test_concurrent_futures/test_thread_pool.py +++ b/Lib/test/test_concurrent_futures/test_thread_pool.py @@ -68,13 +68,13 @@ def submit(pool): @support.requires_fork() @unittest.skipUnless(hasattr(os, 'register_at_fork'), 'need os.register_at_fork') - @support.requires_resource('cpu') def test_process_fork_from_a_threadpool(self): # bpo-43944: clear concurrent.futures.thread._threads_queues after fork, # otherwise child process will try to join parent thread def fork_process_and_return_exitcode(): # Ignore the warning about fork with threads. - with self.assertWarnsRegex(DeprecationWarning, msg="use of fork() may lead to deadlocks in child"): + with self.assertWarnsRegex(DeprecationWarning, + r"use of fork\(\) may lead to deadlocks in the child"): p = mp.get_context('fork').Process(target=lambda: 1) p.start() p.join()