From 8bf39b606ef7b02c0279a80789f3c4824b0da5e9 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 26 Sep 2019 02:22:35 +0200 Subject: [PATCH] bpo-38234: Add test_init_setpath_config() to test_embed (GH-16402) * Add test_embed.test_init_setpath_config(): test Py_SetPath() with PyConfig. * test_init_setpath() and test_init_setpythonhome() no longer call Py_SetProgramName(), but use the default program name. * _PyPathConfig: isolated, site_import and base_executable fields are now only available on Windows. * If executable is set explicitly in the configuration, ignore calculated base_executable: _PyConfig_InitPathConfig() copies executable to base_executable. * Complete path config documentation. --- Doc/c-api/init_config.rst | 19 ++++++--- Include/internal/pycore_pathconfig.h | 15 +++++-- Lib/test/test_embed.py | 59 ++++++++++++++++++++++++---- PC/getpathp.c | 6 +-- Programs/_testembed.c | 52 ++++++++++++------------ Python/pathconfig.c | 33 +++++++++++++--- 6 files changed, 132 insertions(+), 52 deletions(-) diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index bc24fa08131..d98bcda2074 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -864,29 +864,38 @@ Path Configuration :c:type:`PyConfig` contains multiple fields for the path configuration: -* Path configuration input fields: +* Path configuration inputs: * :c:member:`PyConfig.home` * :c:member:`PyConfig.pathconfig_warnings` * :c:member:`PyConfig.program_name` * :c:member:`PyConfig.pythonpath_env` + * current working directory: to get absolute paths + * ``PATH`` environment variable to get the program full path + (from :c:member:`PyConfig.program_name`) + * ``__PYVENV_LAUNCHER__`` environment variable + * (Windows only) Application paths in the registry under + "Software\Python\PythonCore\X.Y\PythonPath" of HKEY_CURRENT_USER and + HKEY_LOCAL_MACHINE (where X.Y is the Python version). * Path configuration output fields: + * :c:member:`PyConfig.base_exec_prefix` * :c:member:`PyConfig.base_executable` + * :c:member:`PyConfig.base_prefix` * :c:member:`PyConfig.exec_prefix` * :c:member:`PyConfig.executable` - * :c:member:`PyConfig.prefix` * :c:member:`PyConfig.module_search_paths_set`, :c:member:`PyConfig.module_search_paths` + * :c:member:`PyConfig.prefix` -If at least one "output field" is not set, Python computes the path +If at least one "output field" is not set, Python calculates the path configuration to fill unset fields. If :c:member:`~PyConfig.module_search_paths_set` is equal to 0, :c:member:`~PyConfig.module_search_paths` is overridden and :c:member:`~PyConfig.module_search_paths_set` is set to 1. -It is possible to completely ignore the function computing the default +It is possible to completely ignore the function calculating the default path configuration by setting explicitly all path configuration output fields listed above. A string is considered as set even if it is non-empty. ``module_search_paths`` is considered as set if @@ -894,7 +903,7 @@ fields listed above. A string is considered as set even if it is non-empty. configuration input fields are ignored as well. Set :c:member:`~PyConfig.pathconfig_warnings` to 0 to suppress warnings when -computing the path configuration (Unix only, Windows does not log any warning). +calculating the path configuration (Unix only, Windows does not log any warning). If :c:member:`~PyConfig.base_prefix` or :c:member:`~PyConfig.base_exec_prefix` fields are not set, they inherit their value from :c:member:`~PyConfig.prefix` diff --git a/Include/internal/pycore_pathconfig.h b/Include/internal/pycore_pathconfig.h index 61b3790fe1f..b5d11447365 100644 --- a/Include/internal/pycore_pathconfig.h +++ b/Include/internal/pycore_pathconfig.h @@ -19,6 +19,7 @@ typedef struct _PyPathConfig { wchar_t *program_name; /* Set by Py_SetPythonHome() or PYTHONHOME environment variable */ wchar_t *home; +#ifdef MS_WINDOWS /* isolated and site_import are used to set Py_IsolatedFlag and Py_NoSiteFlag flags on Windows in read_pth_file(). These fields are ignored when their value are equal to -1 (unset). */ @@ -26,12 +27,18 @@ typedef struct _PyPathConfig { int site_import; /* Set when a venv is detected */ wchar_t *base_executable; +#endif } _PyPathConfig; -#define _PyPathConfig_INIT \ - {.module_search_path = NULL, \ - .isolated = -1, \ - .site_import = -1} +#ifdef MS_WINDOWS +# define _PyPathConfig_INIT \ + {.module_search_path = NULL, \ + .isolated = -1, \ + .site_import = -1} +#else +# define _PyPathConfig_INIT \ + {.module_search_path = NULL} +#endif /* Note: _PyPathConfig_INIT sets other fields to 0/NULL */ PyAPI_DATA(_PyPathConfig) _Py_path_config; diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index ed90fc0cbed..8855e907c25 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -635,16 +635,19 @@ def check_global_config(self, configs): self.assertEqual(configs['global_config'], expected) def check_all_configs(self, testname, expected_config=None, - expected_preconfig=None, modify_path_cb=None, stderr=None, - *, api, env=None, ignore_stderr=False, cwd=None): + expected_preconfig=None, modify_path_cb=None, + stderr=None, *, api, preconfig_api=None, + env=None, ignore_stderr=False, cwd=None): new_env = remove_python_envvars() if env is not None: new_env.update(env) env = new_env - if api == API_ISOLATED: + if preconfig_api is None: + preconfig_api = api + if preconfig_api == API_ISOLATED: default_preconfig = self.PRE_CONFIG_ISOLATED - elif api == API_PYTHON: + elif preconfig_api == API_PYTHON: default_preconfig = self.PRE_CONFIG_PYTHON else: default_preconfig = self.PRE_CONFIG_COMPAT @@ -1002,8 +1005,21 @@ def test_init_dont_parse_argv(self): self.check_all_configs("test_init_dont_parse_argv", config, pre_config, api=API_PYTHON) + def default_program_name(self, config): + if MS_WINDOWS: + program_name = 'python' + executable = self.test_exe + else: + program_name = 'python3' + executable = shutil.which(program_name) or '' + config.update({ + 'program_name': program_name, + 'base_executable': executable, + 'executable': executable, + }) + def test_init_setpath(self): - # Test Py_SetProgramName() + Py_SetPath() + # Test Py_SetPath() config = self._get_expected_config() paths = config['config']['module_search_paths'] @@ -1014,11 +1030,38 @@ def test_init_setpath(self): 'exec_prefix': '', 'base_exec_prefix': '', } + self.default_program_name(config) env = {'TESTPATH': os.path.pathsep.join(paths)} self.check_all_configs("test_init_setpath", config, api=API_COMPAT, env=env, ignore_stderr=True) + def test_init_setpath_config(self): + # Test Py_SetPath() with PyConfig + config = self._get_expected_config() + paths = config['config']['module_search_paths'] + + config = { + # set by Py_SetPath() + 'module_search_paths': paths, + 'prefix': '', + 'base_prefix': '', + 'exec_prefix': '', + 'base_exec_prefix': '', + # overriden by PyConfig + 'program_name': 'conf_program_name', + 'base_executable': 'conf_executable', + 'executable': 'conf_executable', + } + env = {'TESTPATH': os.path.pathsep.join(paths)} + # Py_SetPath() preinitialized Python using the compat API, + # so we need preconfig_api=API_COMPAT. + self.check_all_configs("test_init_setpath_config", config, + api=API_PYTHON, + preconfig_api=API_COMPAT, + env=env, + ignore_stderr=True) + def module_search_paths(self, prefix=None, exec_prefix=None): config = self._get_expected_config() if prefix is None: @@ -1067,8 +1110,7 @@ def tmpdir_with_python(self): yield tmpdir def test_init_setpythonhome(self): - # Test Py_SetPythonHome(home) + PYTHONPATH env var - # + Py_SetProgramName() + # Test Py_SetPythonHome(home) with PYTHONPATH env var config = self._get_expected_config() paths = config['config']['module_search_paths'] paths_str = os.path.pathsep.join(paths) @@ -1095,7 +1137,8 @@ def test_init_setpythonhome(self): 'base_exec_prefix': exec_prefix, 'pythonpath_env': paths_str, } - env = {'TESTHOME': home, 'TESTPATH': paths_str} + self.default_program_name(config) + env = {'TESTHOME': home, 'PYTHONPATH': paths_str} self.check_all_configs("test_init_setpythonhome", config, api=API_COMPAT, env=env) diff --git a/PC/getpathp.c b/PC/getpathp.c index 0eb75b8daf3..8bac592aefd 100644 --- a/PC/getpathp.c +++ b/PC/getpathp.c @@ -1099,11 +1099,11 @@ calculate_free(PyCalculatePath *calculate) - __PYVENV_LAUNCHER__ environment variable - GetModuleFileNameW(NULL): fully qualified path of the executable file of the current process - - .pth configuration file + - ._pth configuration file - pyvenv.cfg configuration file - Registry key "Software\Python\PythonCore\X.Y\PythonPath" - of HKEY_LOCAL_MACHINE and HKEY_CURRENT_USER where X.Y is the Python - version (major.minor). + of HKEY_CURRENT_USER and HKEY_LOCAL_MACHINE where X.Y is the Python + version. Outputs, 'pathconfig' fields: diff --git a/Programs/_testembed.c b/Programs/_testembed.c index ed07606398d..14fca24f318 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -1425,8 +1425,6 @@ static int test_init_sys_add(void) static int test_init_setpath(void) { - Py_SetProgramName(PROGRAM_NAME); - char *env = getenv("TESTPATH"); if (!env) { fprintf(stderr, "missing TESTPATH env var\n"); @@ -1448,23 +1446,35 @@ static int test_init_setpath(void) } -static int mysetenv(const char *name, const char *value) +static int test_init_setpath_config(void) { - size_t len = strlen(name) + 1 + strlen(value) + 1; - char *env = PyMem_RawMalloc(len); - if (env == NULL) { - fprintf(stderr, "out of memory\n"); - return -1; + char *env = getenv("TESTPATH"); + if (!env) { + fprintf(stderr, "missing TESTPATH env var\n"); + return 1; } - strcpy(env, name); - strcat(env, "="); - strcat(env, value); + wchar_t *path = Py_DecodeLocale(env, NULL); + if (path == NULL) { + fprintf(stderr, "failed to decode TESTPATH\n"); + return 1; + } + Py_SetPath(path); + PyMem_RawFree(path); + putenv("TESTPATH="); - putenv(env); + PyStatus status; + PyConfig config; - /* Don't call PyMem_RawFree(env), but leak env memory block: - putenv() does not copy the string. */ + status = PyConfig_InitPythonConfig(&config); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + config_set_string(&config, &config.program_name, L"conf_program_name"); + config_set_string(&config, &config.executable, L"conf_executable"); + init_from_config_clear(&config); + dump_config(); + Py_Finalize(); return 0; } @@ -1485,19 +1495,6 @@ static int test_init_setpythonhome(void) PyMem_RawFree(home); putenv("TESTHOME="); - char *path = getenv("TESTPATH"); - if (!path) { - fprintf(stderr, "missing TESTPATH env var\n"); - return 1; - } - - if (mysetenv("PYTHONPATH", path) < 0) { - return 1; - } - putenv("TESTPATH="); - - Py_SetProgramName(PROGRAM_NAME); - Py_Initialize(); dump_config(); Py_Finalize(); @@ -1642,6 +1639,7 @@ static struct TestCase TestCases[] = { {"test_init_main", test_init_main}, {"test_init_sys_add", test_init_sys_add}, {"test_init_setpath", test_init_setpath}, + {"test_init_setpath_config", test_init_setpath_config}, {"test_init_setpythonhome", test_init_setpythonhome}, {"test_run_main", test_run_main}, diff --git a/Python/pathconfig.c b/Python/pathconfig.c index 8f76fa50c97..f4e14986687 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -58,7 +58,10 @@ pathconfig_clear(_PyPathConfig *config) CLEAR(config->module_search_path); CLEAR(config->program_name); CLEAR(config->home); +#ifdef MS_WINDOWS CLEAR(config->base_executable); +#endif + #undef CLEAR PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); @@ -83,9 +86,11 @@ pathconfig_copy(_PyPathConfig *config, const _PyPathConfig *config2) COPY_ATTR(module_search_path); COPY_ATTR(program_name); COPY_ATTR(home); +#ifdef MS_WINDOWS config->isolated = config2->isolated; config->site_import = config2->site_import; COPY_ATTR(base_executable); +#endif #undef COPY_ATTR @@ -189,12 +194,14 @@ pathconfig_set_from_config(_PyPathConfig *pathconfig, const PyConfig *config) } \ } - COPY_CONFIG(base_executable, base_executable); COPY_CONFIG(program_full_path, executable); COPY_CONFIG(prefix, prefix); COPY_CONFIG(exec_prefix, exec_prefix); COPY_CONFIG(program_name, program_name); COPY_CONFIG(home, home); +#ifdef MS_WINDOWS + COPY_CONFIG(base_executable, base_executable); +#endif #undef COPY_CONFIG @@ -330,18 +337,32 @@ config_calculate_pathconfig(PyConfig *config) } \ } +#ifdef MS_WINDOWS + if (config->executable != NULL && config->base_executable == NULL) { + /* If executable is set explicitly in the configuration, + ignore calculated base_executable: _PyConfig_InitPathConfig() + will copy executable to base_executable */ + } + else { + COPY_ATTR(base_executable, base_executable); + } +#endif + COPY_ATTR(program_full_path, executable); COPY_ATTR(prefix, prefix); COPY_ATTR(exec_prefix, exec_prefix); - COPY_ATTR(base_executable, base_executable); + #undef COPY_ATTR +#ifdef MS_WINDOWS + /* If a ._pth file is found: isolated and site_import are overriden */ if (pathconfig.isolated != -1) { config->isolated = pathconfig.isolated; } if (pathconfig.site_import != -1) { config->site_import = pathconfig.site_import; } +#endif status = _PyStatus_OK(); goto done; @@ -360,9 +381,9 @@ _PyConfig_InitPathConfig(PyConfig *config) { /* Do we need to calculate the path? */ if (!config->module_search_paths_set - || (config->executable == NULL) - || (config->prefix == NULL) - || (config->exec_prefix == NULL)) + || config->executable == NULL + || config->prefix == NULL + || config->exec_prefix == NULL) { PyStatus status = config_calculate_pathconfig(config); if (_PyStatus_EXCEPTION(status)) { @@ -442,7 +463,9 @@ pathconfig_global_init(void) assert(_Py_path_config.module_search_path != NULL); assert(_Py_path_config.program_name != NULL); /* home can be NULL */ +#ifdef MS_WINDOWS assert(_Py_path_config.base_executable != NULL); +#endif }