gh-98360: multiprocessing now spawns children on Windows with correct argv[0] in virtual environments (GH-98462)

(cherry picked from commit e48f9b2)

Co-authored-by: Steve Dower <[email protected]>

Author: 2 people

Lib/multiprocessing/popen_spawn_win32.py

@@ -54,19 +54,20 @@ def __init__(self, process_obj):
         wfd = msvcrt.open_osfhandle(whandle, 0)
         cmd = spawn.get_command_line(parent_pid=os.getpid(),
                                      pipe_handle=rhandle)
-        cmd = ' '.join('"%s"' % x for x in cmd)
 
         python_exe = spawn.get_executable()
 
         # bpo-35797: When running in a venv, we bypass the redirect
         # executor and launch our base Python.
         if WINENV and _path_eq(python_exe, sys.executable):
-            python_exe = sys._base_executable
+            cmd[0] = python_exe = sys._base_executable
             env = os.environ.copy()
             env["__PYVENV_LAUNCHER__"] = sys.executable
         else:
             env = None
 
+        cmd = ' '.join('"%s"' % x for x in cmd)
+
         with open(wfd, 'wb', closefd=True) as to_child:
             # start process
             try:

Lib/test/_test_venv_multiprocessing.py

@@ -0,0 +1,40 @@
+import multiprocessing
+import random
+import sys
+import time
+
+def fill_queue(queue, code):
+    queue.put(code)
+
+
+def drain_queue(queue, code):
+    if code != queue.get():
+        sys.exit(1)
+
+
+def test_func():
+    code = random.randrange(0, 1000)
+    queue = multiprocessing.Queue()
+    fill_pool = multiprocessing.Process(
+        target=fill_queue,
+        args=(queue, code)
+    )
+    drain_pool = multiprocessing.Process(
+        target=drain_queue,
+        args=(queue, code)
+    )
+    drain_pool.start()
+    fill_pool.start()
+    fill_pool.join()
+    drain_pool.join()
+
+
+def main():
+    test_pool = multiprocessing.Process(target=test_func)
+    test_pool.start()
+    test_pool.join()
+    sys.exit(test_pool.exitcode)
+
+
+if __name__ == "__main__":
+    main()

Lib/test/test_venv.py

@@ -20,7 +20,7 @@
 from test.support import (captured_stdout, captured_stderr, requires_zlib,
                           skip_if_broken_multiprocessing_synchronize, verbose,
                           requires_subprocess, is_emscripten, is_wasi,
-                          requires_venv_with_pip)
+                          requires_venv_with_pip, TEST_HOME_DIR)
 from test.support.os_helper import (can_symlink, EnvironmentVarGuard, rmtree)
 import unittest
 import venv
@@ -482,6 +482,20 @@ def test_multiprocessing(self):
             'pool.terminate()'])
         self.assertEqual(out.strip(), "python".encode())
 
+    @requireVenvCreate
+    def test_multiprocessing_recursion(self):
+        """
+        Test that the multiprocessing is able to spawn itself
+        """
+        skip_if_broken_multiprocessing_synchronize()
+
+        rmtree(self.env_dir)
+        self.run_with_capture(venv.create, self.env_dir)
+        envpy = os.path.join(os.path.realpath(self.env_dir),
+                             self.bindir, self.exe)
+        script = os.path.join(TEST_HOME_DIR, '_test_venv_multiprocessing.py')
+        subprocess.check_call([envpy, script])
+
     @unittest.skipIf(os.name == 'nt', 'not relevant on Windows')
     def test_deactivate_with_strict_bash_opts(self):
         bash = shutil.which("bash")

Misc/NEWS.d/next/Windows/2022-10-19-20-00-28.gh-issue-98360.O2m6YG.rst

@@ -0,0 +1,4 @@
+Fixes :mod:`multiprocessing` spawning child processes on Windows from a
+virtual environment to ensure that child processes that also use
+:mod:`multiprocessing` to spawn more children will recognize that they are
+in a virtual environment.