From e81f6e687d0f04a45f2389d0b43fafd6d8491624 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 8 Jun 2020 18:12:59 +0200 Subject: [PATCH] bpo-40910: Export Py_GetArgcArgv() function (GH-20721) Export explicitly the Py_GetArgcArgv() function to the C API and document the function. Previously, it was exported implicitly which no longer works since Python is built with -fvisibility=hidden. * Add PyConfig._orig_argv member. * Py_InitializeFromConfig() no longer calls _PyConfig_Write() twice. * PyConfig_Read() no longer initializes Py_GetArgcArgv(): it is now _PyConfig_Write() responsibility. * _PyConfig_Write() result type becomes PyStatus instead of void. * Write an unit test on Py_GetArgcArgv(). --- Doc/c-api/init_config.rst | 9 ++++ Include/cpython/initconfig.h | 16 +++++++ Include/internal/pycore_initconfig.h | 2 +- Lib/test/test_embed.py | 34 +++++++++++++-- .../2020-06-08-15-59-06.bpo-40910.L56oI0.rst | 3 ++ PC/python3.def | 1 + Programs/_testembed.c | 42 +++++++++++++++++++ Python/bootstrap_hash.c | 2 +- Python/initconfig.c | 30 +++++++------ Python/pylifecycle.c | 14 ++++--- 10 files changed, 131 insertions(+), 22 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2020-06-08-15-59-06.bpo-40910.L56oI0.rst diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 7b8e894fe22..c51b157bbb3 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -43,6 +43,7 @@ Functions: * :c:func:`Py_PreInitializeFromArgs` * :c:func:`Py_PreInitializeFromBytesArgs` * :c:func:`Py_RunMain` +* :c:func:`Py_GetArgcArgv` The preconfiguration (``PyPreConfig`` type) is stored in ``_PyRuntime.preconfig`` and the configuration (``PyConfig`` type) is stored in @@ -984,6 +985,14 @@ customized Python always running in isolated mode using :c:func:`Py_RunMain`. +Py_GetArgcArgv() +---------------- + +.. c:function:: void Py_GetArgcArgv(int *argc, wchar_t ***argv) + + Get the original command line arguments, before Python modified them. + + Multi-Phase Initialization Private Provisional API -------------------------------------------------- diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index 563c2bacfa4..57933211bb9 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -411,6 +411,14 @@ typedef struct { /* If non-zero, disallow threads, subprocesses, and fork. Default: 0. */ int _isolated_interpreter; + + /* Original command line arguments. If _orig_argv is empty and _argv is + not equal to [''], PyConfig_Read() copies the configuration 'argv' list + into '_orig_argv' list before modifying 'argv' list (if parse_argv + is non-zero). + + _PyConfig_Write() initializes Py_GetArgcArgv() to this list. */ + PyWideStringList _orig_argv; } PyConfig; PyAPI_FUNC(void) PyConfig_InitPythonConfig(PyConfig *config); @@ -436,5 +444,13 @@ PyAPI_FUNC(PyStatus) PyConfig_SetWideStringList(PyConfig *config, PyWideStringList *list, Py_ssize_t length, wchar_t **items); + +/* --- Helper functions --------------------------------------- */ + +/* Get the original command line arguments, before Python modified them. + + See also PyConfig._orig_argv. */ +PyAPI_FUNC(void) Py_GetArgcArgv(int *argc, wchar_t ***argv); + #endif /* !Py_LIMITED_API */ #endif /* !Py_PYCORECONFIG_H */ diff --git a/Include/internal/pycore_initconfig.h b/Include/internal/pycore_initconfig.h index 8c6706c95cb..457a005860b 100644 --- a/Include/internal/pycore_initconfig.h +++ b/Include/internal/pycore_initconfig.h @@ -150,7 +150,7 @@ extern PyStatus _PyConfig_Copy( PyConfig *config, const PyConfig *config2); extern PyStatus _PyConfig_InitPathConfig(PyConfig *config); -extern void _PyConfig_Write(const PyConfig *config, +extern PyStatus _PyConfig_Write(const PyConfig *config, struct pyruntimestate *runtime); extern PyStatus _PyConfig_SetPyArgv( PyConfig *config, diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index f1371db8669..b7b70589da5 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -366,6 +366,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'program_name': GET_DEFAULT_CONFIG, 'parse_argv': 0, 'argv': [""], + '_orig_argv': [], 'xoptions': [], 'warnoptions': [], @@ -739,7 +740,12 @@ def test_init_from_config(self): 'pycache_prefix': 'conf_pycache_prefix', 'program_name': './conf_program_name', - 'argv': ['-c', 'arg2', ], + 'argv': ['-c', 'arg2'], + '_orig_argv': ['python3', + '-W', 'cmdline_warnoption', + '-X', 'cmdline_xoption', + '-c', 'pass', + 'arg2'], 'parse_argv': 1, 'xoptions': [ 'config_xoption1=3', @@ -872,6 +878,7 @@ def test_preinit_parse_argv(self): } config = { 'argv': ['script.py'], + '_orig_argv': ['python3', '-X', 'dev', 'script.py'], 'run_filename': os.path.abspath('script.py'), 'dev_mode': 1, 'faulthandler': 1, @@ -886,9 +893,14 @@ def test_preinit_dont_parse_argv(self): preconfig = { 'isolated': 0, } + argv = ["python3", + "-E", "-I", + "-X", "dev", + "-X", "utf8", + "script.py"] config = { - 'argv': ["python3", "-E", "-I", - "-X", "dev", "-X", "utf8", "script.py"], + 'argv': argv, + '_orig_argv': argv, 'isolated': 0, } self.check_all_configs("test_preinit_dont_parse_argv", config, preconfig, @@ -967,6 +979,9 @@ def test_init_sys_add(self): 'ignore:::sysadd_warnoption', 'ignore:::config_warnoption', ], + '_orig_argv': ['python3', + '-W', 'ignore:::cmdline_warnoption', + '-X', 'cmdline_xoption'], } self.check_all_configs("test_init_sys_add", config, api=API_PYTHON) @@ -975,6 +990,7 @@ def test_init_run_main(self): 'print(json.dumps(_testinternalcapi.get_configs()))') config = { 'argv': ['-c', 'arg2'], + '_orig_argv': ['python3', '-c', code, 'arg2'], 'program_name': './python3', 'run_command': code + '\n', 'parse_argv': 1, @@ -986,6 +1002,9 @@ def test_init_main(self): 'print(json.dumps(_testinternalcapi.get_configs()))') config = { 'argv': ['-c', 'arg2'], + '_orig_argv': ['python3', + '-c', code, + 'arg2'], 'program_name': './python3', 'run_command': code + '\n', 'parse_argv': 1, @@ -999,6 +1018,7 @@ def test_init_parse_argv(self): config = { 'parse_argv': 1, 'argv': ['-c', 'arg1', '-v', 'arg3'], + '_orig_argv': ['./argv0', '-E', '-c', 'pass', 'arg1', '-v', 'arg3'], 'program_name': './argv0', 'run_command': 'pass\n', 'use_environment': 0, @@ -1012,6 +1032,7 @@ def test_init_dont_parse_argv(self): config = { 'parse_argv': 0, 'argv': ['./argv0', '-E', '-c', 'pass', 'arg1', '-v', 'arg3'], + '_orig_argv': ['./argv0', '-E', '-c', 'pass', 'arg1', '-v', 'arg3'], 'program_name': './argv0', } self.check_all_configs("test_init_dont_parse_argv", config, pre_config, @@ -1299,10 +1320,17 @@ def test_init_warnoptions(self): 'faulthandler': 1, 'bytes_warning': 1, 'warnoptions': warnoptions, + '_orig_argv': ['python3', + '-Wignore:::cmdline1', + '-Wignore:::cmdline2'], } self.check_all_configs("test_init_warnoptions", config, preconfig, api=API_PYTHON) + def test_get_argc_argv(self): + self.run_embedded_interpreter("test_get_argc_argv") + # ignore output + class AuditingTests(EmbeddingTestsMixin, unittest.TestCase): def test_open_code_hook(self): diff --git a/Misc/NEWS.d/next/C API/2020-06-08-15-59-06.bpo-40910.L56oI0.rst b/Misc/NEWS.d/next/C API/2020-06-08-15-59-06.bpo-40910.L56oI0.rst new file mode 100644 index 00000000000..1d0cb0b0235 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2020-06-08-15-59-06.bpo-40910.L56oI0.rst @@ -0,0 +1,3 @@ +Export explicitly the :c:func:`Py_GetArgcArgv` function to the C API and +document the function. Previously, it was exported implicitly which no +longer works since Python is built with ``-fvisibility=hidden``. diff --git a/PC/python3.def b/PC/python3.def index 6d54d4eaf71..2a6aaf4331e 100644 --- a/PC/python3.def +++ b/PC/python3.def @@ -734,6 +734,7 @@ EXPORTS Py_FinalizeEx=python310.Py_FinalizeEx Py_GenericAlias=python310.Py_GenericAlias Py_GenericAliasType=python310.Py_GenericAliasType + Py_GetArgcArgv=python310.Py_GetArgcArgv Py_GetBuildInfo=python310.Py_GetBuildInfo Py_GetCompiler=python310.Py_GetCompiler Py_GetCopyright=python310.Py_GetCopyright diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 11524dfbc0d..d89f6be6570 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -1334,6 +1334,7 @@ static int test_init_read_set(void) return 0; fail: + PyConfig_Clear(&config); Py_ExitStatusException(status); } @@ -1592,6 +1593,46 @@ static int test_run_main(void) } +static int test_get_argc_argv(void) +{ + PyConfig config; + PyConfig_InitPythonConfig(&config); + + wchar_t *argv[] = {L"python3", L"-c", + (L"import sys; " + L"print(f'Py_RunMain(): sys.argv={sys.argv}')"), + L"arg2"}; + config_set_argv(&config, Py_ARRAY_LENGTH(argv), argv); + config_set_string(&config, &config.program_name, L"./python3"); + + // Calling PyConfig_Read() twice must not change Py_GetArgcArgv() result. + // The second call is done by Py_InitializeFromConfig(). + PyStatus status = PyConfig_Read(&config); + if (PyStatus_Exception(status)) { + PyConfig_Clear(&config); + Py_ExitStatusException(status); + } + + init_from_config_clear(&config); + + int get_argc; + wchar_t **get_argv; + Py_GetArgcArgv(&get_argc, &get_argv); + printf("argc: %i\n", get_argc); + assert(get_argc == Py_ARRAY_LENGTH(argv)); + for (int i=0; i < get_argc; i++) { + printf("argv[%i]: %ls\n", i, get_argv[i]); + assert(wcscmp(get_argv[i], argv[i]) == 0); + } + + Py_Finalize(); + + printf("\n"); + printf("test ok\n"); + return 0; +} + + /* ********************************************************* * List of test cases and the function that implements it. * @@ -1649,6 +1690,7 @@ static struct TestCase TestCases[] = { {"test_init_setpythonhome", test_init_setpythonhome}, {"test_init_warnoptions", test_init_warnoptions}, {"test_run_main", test_run_main}, + {"test_get_argc_argv", test_get_argc_argv}, {"test_open_code_hook", test_open_code_hook}, {"test_audit", test_audit}, diff --git a/Python/bootstrap_hash.c b/Python/bootstrap_hash.c index b2109275014..47369305ee8 100644 --- a/Python/bootstrap_hash.c +++ b/Python/bootstrap_hash.c @@ -580,7 +580,7 @@ _Py_HashRandomization_Init(const PyConfig *config) res = pyurandom(secret, secret_size, 0, 0); if (res < 0) { return _PyStatus_ERR("failed to get random numbers " - "to initialize Python"); + "to initialize Python"); } } return _PyStatus_OK(); diff --git a/Python/initconfig.c b/Python/initconfig.c index 834b8ed9430..998ceb7bbfa 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -548,8 +548,6 @@ _Py_SetArgcArgv(Py_ssize_t argc, wchar_t * const *argv) } -/* Make the *original* argc/argv available to other modules. - This is rare, but it is needed by the secureware extension. */ void Py_GetArgcArgv(int *argc, wchar_t ***argv) { @@ -859,6 +857,7 @@ _PyConfig_Copy(PyConfig *config, const PyConfig *config2) COPY_ATTR(pathconfig_warnings); COPY_ATTR(_init_main); COPY_ATTR(_isolated_interpreter); + COPY_WSTRLIST(_orig_argv); #undef COPY_ATTR #undef COPY_WSTR_ATTR @@ -960,6 +959,7 @@ config_as_dict(const PyConfig *config) SET_ITEM_INT(pathconfig_warnings); SET_ITEM_INT(_init_main); SET_ITEM_INT(_isolated_interpreter); + SET_ITEM_WSTRLIST(_orig_argv); return dict; @@ -1856,7 +1856,7 @@ config_init_stdio(const PyConfig *config) - set Py_xxx global configuration variables - initialize C standard streams (stdin, stdout, stderr) */ -void +PyStatus _PyConfig_Write(const PyConfig *config, _PyRuntimeState *runtime) { config_set_global_vars(config); @@ -1870,6 +1870,13 @@ _PyConfig_Write(const PyConfig *config, _PyRuntimeState *runtime) preconfig->isolated = config->isolated; preconfig->use_environment = config->use_environment; preconfig->dev_mode = config->dev_mode; + + if (_Py_SetArgcArgv(config->_orig_argv.length, + config->_orig_argv.items) < 0) + { + return _PyStatus_NO_MEMORY(); + } + return _PyStatus_OK(); } @@ -2493,7 +2500,6 @@ PyStatus PyConfig_Read(PyConfig *config) { PyStatus status; - PyWideStringList orig_argv = _PyWideStringList_INIT; status = _Py_PreInitializeFromConfig(config, NULL); if (_PyStatus_EXCEPTION(status)) { @@ -2502,8 +2508,13 @@ PyConfig_Read(PyConfig *config) config_get_global_vars(config); - if (_PyWideStringList_Copy(&orig_argv, &config->argv) < 0) { - return _PyStatus_NO_MEMORY(); + if (config->_orig_argv.length == 0 + && !(config->argv.length == 1 + && wcscmp(config->argv.items[0], L"") == 0)) + { + if (_PyWideStringList_Copy(&config->_orig_argv, &config->argv) < 0) { + return _PyStatus_NO_MEMORY(); + } } _PyPreCmdline precmdline = _PyPreCmdline_INIT; @@ -2534,11 +2545,6 @@ PyConfig_Read(PyConfig *config) goto done; } - if (_Py_SetArgcArgv(orig_argv.length, orig_argv.items) < 0) { - status = _PyStatus_NO_MEMORY(); - goto done; - } - /* Check config consistency */ assert(config->isolated >= 0); assert(config->use_environment >= 0); @@ -2591,11 +2597,11 @@ PyConfig_Read(PyConfig *config) assert(config->check_hash_pycs_mode != NULL); assert(config->_install_importlib >= 0); assert(config->pathconfig_warnings >= 0); + assert(_PyWideStringList_CheckConsistency(&config->_orig_argv)); status = _PyStatus_OK(); done: - _PyWideStringList_Clear(&orig_argv); _PyPreCmdline_Clear(&precmdline); return status; } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index d730a98d3e5..f2f7d585c80 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -460,7 +460,10 @@ pyinit_core_reconfigure(_PyRuntimeState *runtime, return _PyStatus_ERR("can't make main interpreter"); } - _PyConfig_Write(config, runtime); + status = _PyConfig_Write(config, runtime); + if (_PyStatus_EXCEPTION(status)) { + return status; + } status = _PyInterpreterState_SetConfig(interp, config); if (_PyStatus_EXCEPTION(status)) { @@ -486,7 +489,10 @@ pycore_init_runtime(_PyRuntimeState *runtime, return _PyStatus_ERR("main interpreter already initialized"); } - _PyConfig_Write(config, runtime); + PyStatus status = _PyConfig_Write(config, runtime); + if (_PyStatus_EXCEPTION(status)) { + return status; + } /* Py_Finalize leaves _Py_Finalizing set in order to help daemon * threads behave a little more gracefully at interpreter shutdown. @@ -499,7 +505,7 @@ pycore_init_runtime(_PyRuntimeState *runtime, */ _PyRuntimeState_SetFinalizing(runtime, NULL); - PyStatus status = _Py_HashRandomization_Init(config); + status = _Py_HashRandomization_Init(config); if (_PyStatus_EXCEPTION(status)) { return status; } @@ -746,8 +752,6 @@ pyinit_config(_PyRuntimeState *runtime, PyThreadState **tstate_p, const PyConfig *config) { - _PyConfig_Write(config, runtime); - PyStatus status = pycore_init_runtime(runtime, config); if (_PyStatus_EXCEPTION(status)) { return status;