From 489699ca05bed5cfd10e847d8580840812b476cd Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Jun 2021 14:13:27 +0200 Subject: [PATCH] bpo-44441: _PyImport_Fini2() resets PyImport_Inittab (GH-26874) Py_RunMain() now resets PyImport_Inittab to its initial value at exit. It must be possible to call PyImport_AppendInittab() or PyImport_ExtendInittab() at each Python initialization. --- Doc/c-api/import.rst | 6 +- Doc/c-api/init_config.rst | 5 +- Lib/test/test_embed.py | 17 +++++- .../2021-06-23-12-12-04.bpo-44441.3p14JB.rst | 4 ++ Programs/_testembed.c | 58 ++++++++++++++++++- Python/import.c | 3 + 6 files changed, 85 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2021-06-23-12-12-04.bpo-44441.3p14JB.rst diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index c6fc33076f0..d2ae6b6d4e4 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -299,4 +299,8 @@ Importing Modules field; failure to provide the sentinel value can result in a memory fault. Returns ``0`` on success or ``-1`` if insufficient memory could be allocated to extend the internal table. In the event of failure, no modules are added to the - internal table. This should be called before :c:func:`Py_Initialize`. + internal table. This must be called before :c:func:`Py_Initialize`. + + If Python is initialized multiple times, :c:func:`PyImport_AppendInittab` or + :c:func:`PyImport_ExtendInittab` must be called before each Python + initialization. diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index de850294811..fe5b83aa8dc 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -1193,7 +1193,10 @@ The caller is responsible to handle exceptions (error or exit) using If :c:func:`PyImport_FrozenModules`, :c:func:`PyImport_AppendInittab` or :c:func:`PyImport_ExtendInittab` are used, they must be set or called after -Python preinitialization and before the Python initialization. +Python preinitialization and before the Python initialization. If Python is +initialized multiple times, :c:func:`PyImport_AppendInittab` or +:c:func:`PyImport_ExtendInittab` must be called before each Python +initialization. The current configuration (``PyConfig`` type) is stored in ``PyInterpreterState.config``. diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 373fcf0ffac..c50defd9a6a 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -30,6 +30,7 @@ # _PyCoreConfig_InitIsolatedConfig() API_ISOLATED = 3 +INIT_LOOPS = 16 MAX_HASH_SEED = 4294967295 @@ -111,13 +112,13 @@ def run_repeated_init_and_subinterpreters(self): self.assertEqual(err, "") # The output from _testembed looks like this: - # --- Pass 0 --- + # --- Pass 1 --- # interp 0 <0x1cf9330>, thread state <0x1cf9700>: id(modules) = 139650431942728 # interp 1 <0x1d4f690>, thread state <0x1d35350>: id(modules) = 139650431165784 # interp 2 <0x1d5a690>, thread state <0x1d99ed0>: id(modules) = 139650413140368 # interp 3 <0x1d4f690>, thread state <0x1dc3340>: id(modules) = 139650412862200 # interp 0 <0x1cf9330>, thread state <0x1cf9700>: id(modules) = 139650431942728 - # --- Pass 1 --- + # --- Pass 2 --- # ... interp_pat = (r"^interp (\d+) <(0x[\dA-F]+)>, " @@ -125,7 +126,7 @@ def run_repeated_init_and_subinterpreters(self): r"id\(modules\) = ([\d]+)$") Interp = namedtuple("Interp", "id interp tstate modules") - numloops = 0 + numloops = 1 current_run = [] for line in out.splitlines(): if line == "--- Pass {} ---".format(numloops): @@ -159,6 +160,8 @@ def run_repeated_init_and_subinterpreters(self): class EmbeddingTests(EmbeddingTestsMixin, unittest.TestCase): + maxDiff = 100 * 50 + def test_subinterps_main(self): for run in self.run_repeated_init_and_subinterpreters(): main = run[0] @@ -194,6 +197,14 @@ def test_subinterps_distinct_state(self): self.assertNotEqual(sub.tstate, main.tstate) self.assertNotEqual(sub.modules, main.modules) + def test_repeated_init_and_inittab(self): + out, err = self.run_embedded_interpreter("test_repeated_init_and_inittab") + self.assertEqual(err, "") + + lines = [f"--- Pass {i} ---" for i in range(1, INIT_LOOPS+1)] + lines = "\n".join(lines) + "\n" + self.assertEqual(out, lines) + def test_forced_io_encoding(self): # Checks forced configuration of embedded interpreter IO streams env = dict(os.environ, PYTHONIOENCODING="utf-8:surrogateescape") diff --git a/Misc/NEWS.d/next/C API/2021-06-23-12-12-04.bpo-44441.3p14JB.rst b/Misc/NEWS.d/next/C API/2021-06-23-12-12-04.bpo-44441.3p14JB.rst new file mode 100644 index 00000000000..80c4282c18e --- /dev/null +++ b/Misc/NEWS.d/next/C API/2021-06-23-12-12-04.bpo-44441.3p14JB.rst @@ -0,0 +1,4 @@ +:c:func:`Py_RunMain` now resets :c:data:`PyImport_Inittab` to its initial value +at exit. It must be possible to call :c:func:`PyImport_AppendInittab` or +:c:func:`PyImport_ExtendInittab` at each Python initialization. +Patch by Victor Stinner. diff --git a/Programs/_testembed.c b/Programs/_testembed.c index a5ae7c1d863..d963cb3dc7e 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -22,6 +22,8 @@ /* Use path starting with "./" avoids a search along the PATH */ #define PROGRAM_NAME L"./_testembed" +#define INIT_LOOPS 16 + // Ignore Py_DEPRECATED() compiler warnings: deprecated functions are // tested on purpose here. _Py_COMP_DIAG_PUSH @@ -67,9 +69,8 @@ static int test_repeated_init_and_subinterpreters(void) { PyThreadState *mainstate, *substate; PyGILState_STATE gilstate; - int i, j; - for (i=0; i<15; i++) { + for (int i=1; i <= INIT_LOOPS; i++) { printf("--- Pass %d ---\n", i); _testembed_Py_Initialize(); mainstate = PyThreadState_Get(); @@ -80,7 +81,7 @@ static int test_repeated_init_and_subinterpreters(void) print_subinterp(); PyThreadState_Swap(NULL); - for (j=0; j<3; j++) { + for (int j=0; j<3; j++) { substate = Py_NewInterpreter(); print_subinterp(); Py_EndInterpreter(substate); @@ -96,6 +97,20 @@ static int test_repeated_init_and_subinterpreters(void) return 0; } +#define EMBEDDED_EXT_NAME "embedded_ext" + +static PyModuleDef embedded_ext = { + PyModuleDef_HEAD_INIT, + .m_name = EMBEDDED_EXT_NAME, + .m_size = 0, +}; + +static PyObject* +PyInit_embedded_ext(void) +{ + return PyModule_Create(&embedded_ext); +} + /***************************************************** * Test forcing a particular IO encoding *****************************************************/ @@ -1790,6 +1805,38 @@ static int list_frozen(void) } +static int test_repeated_init_and_inittab(void) +{ + // bpo-44441: Py_RunMain() must reset PyImport_Inittab at exit. + // It must be possible to call PyImport_AppendInittab() or + // PyImport_ExtendInittab() before each Python initialization. + for (int i=1; i <= INIT_LOOPS; i++) { + printf("--- Pass %d ---\n", i); + + // Call PyImport_AppendInittab() at each iteration + if (PyImport_AppendInittab(EMBEDDED_EXT_NAME, + &PyInit_embedded_ext) != 0) { + fprintf(stderr, "PyImport_AppendInittab() failed\n"); + return 1; + } + + // Initialize Python + wchar_t* argv[] = {PROGRAM_NAME, L"-c", L"pass"}; + PyConfig config; + PyConfig_InitPythonConfig(&config); + config.isolated = 1; + config_set_argv(&config, Py_ARRAY_LENGTH(argv), argv); + init_from_config_clear(&config); + + // Py_RunMain() calls _PyImport_Fini2() which resets PyImport_Inittab + int exitcode = Py_RunMain(); + if (exitcode != 0) { + return exitcode; + } + } + return 0; +} + /* ********************************************************* * List of test cases and the function that implements it. @@ -1810,8 +1857,10 @@ struct TestCase }; static struct TestCase TestCases[] = { + // Python initialization {"test_forced_io_encoding", test_forced_io_encoding}, {"test_repeated_init_and_subinterpreters", test_repeated_init_and_subinterpreters}, + {"test_repeated_init_and_inittab", test_repeated_init_and_inittab}, {"test_pre_initialization_api", test_pre_initialization_api}, {"test_pre_initialization_sys_options", test_pre_initialization_sys_options}, {"test_bpo20891", test_bpo20891}, @@ -1851,6 +1900,7 @@ static struct TestCase TestCases[] = { {"test_run_main", test_run_main}, {"test_get_argc_argv", test_get_argc_argv}, + // Audit {"test_open_code_hook", test_open_code_hook}, {"test_audit", test_audit}, {"test_audit_subinterpreter", test_audit_subinterpreter}, @@ -1860,11 +1910,13 @@ static struct TestCase TestCases[] = { {"test_audit_run_startup", test_audit_run_startup}, {"test_audit_run_stdin", test_audit_run_stdin}, + // Specific C API {"test_unicode_id_init", test_unicode_id_init}, #ifndef MS_WINDOWS {"test_frozenmain", test_frozenmain}, #endif + // Command {"list_frozen", list_frozen}, {NULL, NULL} }; diff --git a/Python/import.c b/Python/import.c index c4878c628be..7301fccb9fa 100644 --- a/Python/import.c +++ b/Python/import.c @@ -255,6 +255,9 @@ _PyImport_Fini2(void) PyMemAllocatorEx old_alloc; _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + // Reset PyImport_Inittab + PyImport_Inittab = _PyImport_Inittab; + /* Free memory allocated by PyImport_ExtendInittab() */ PyMem_RawFree(inittab_copy); inittab_copy = NULL;