From b9cdd0fb9c463c2503a4d854bb6529a9db58fe1b Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sat, 16 Oct 2021 13:16:08 -0600 Subject: [PATCH] bpo-45020: Default to using frozen modules unless running from source tree. (gh-28940) The default was "off". Switching it to "on" means users get the benefit of frozen stdlib modules without having to do anything. There's a special-case for running-in-source-tree, so contributors don't get surprised when their stdlib changes don't get used. https://bugs.python.org/issue45020 --- Doc/using/cmdline.rst | 3 +- Include/internal/pycore_fileutils.h | 1 + Lib/site.py | 4 +- Lib/test/test_embed.py | 14 +++-- Misc/NEWS.d/3.11.0a1.rst | 2 +- Python/fileutils.c | 12 +++++ Python/initconfig.c | 80 ++++++++++++++++++++++------- 7 files changed, 91 insertions(+), 25 deletions(-) diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 23a645a036a..d341ea8bb43 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -483,7 +483,8 @@ Miscellaneous options * ``-X frozen_modules`` determines whether or not frozen modules are ignored by the import machinery. A value of "on" means they get imported and "off" means they are ignored. The default is "on" - for non-debug builds (the normal case) and "off" for debug builds. + if this is an installed Python (the normal case). If it's under + development (running from the source tree) then the default is "off". Note that the "importlib_bootstrap" and "importlib_bootstrap_external" frozen modules are always used, even if this flag is set to "off". diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index 3464477bce5..ab436ae9b00 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -79,6 +79,7 @@ extern wchar_t * _Py_join_relfile(const wchar_t *dirname, extern int _Py_add_relfile(wchar_t *dirname, const wchar_t *relfile, size_t bufsize); +extern size_t _Py_find_basename(const wchar_t *filename); // Macros to protect CRT calls against instant termination when passed an // invalid parameter (bpo-23524). IPH stands for Invalid Parameter Handler. diff --git a/Lib/site.py b/Lib/site.py index 939893eb5ee..e129f3b4851 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -418,8 +418,10 @@ def setcopyright(): files, dirs = [], [] # Not all modules are required to have a __file__ attribute. See # PEP 420 for more details. - if hasattr(os, '__file__'): + here = getattr(sys, '_stdlib_dir', None) + if not here and hasattr(os, '__file__'): here = os.path.dirname(os.__file__) + if here: files.extend(["LICENSE.txt", "LICENSE"]) dirs.extend([os.path.join(here, os.pardir), here, os.curdir]) builtins.license = _sitebuiltins._Printer( diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 4cbb4c2c1ce..4b4396efb5c 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -53,12 +53,13 @@ def remove_python_envvars(): class EmbeddingTestsMixin: def setUp(self): exename = "_testembed" + builddir = os.path.dirname(sys.executable) if MS_WINDOWS: ext = ("_d" if debug_build(sys.executable) else "") + ".exe" exename += ext - exepath = os.path.dirname(sys.executable) + exepath = builddir else: - exepath = os.path.join(support.REPO_ROOT, "Programs") + exepath = os.path.join(builddir, 'Programs') self.test_exe = exe = os.path.join(exepath, exename) if not os.path.exists(exe): self.skipTest("%r doesn't exist" % exe) @@ -434,7 +435,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'pathconfig_warnings': 1, '_init_main': 1, '_isolated_interpreter': 0, - 'use_frozen_modules': 0, + 'use_frozen_modules': 1, } if MS_WINDOWS: CONFIG_COMPAT.update({ @@ -1146,6 +1147,7 @@ def test_init_setpath(self): # The current getpath.c doesn't determine the stdlib dir # in this case. 'stdlib_dir': '', + 'use_frozen_modules': -1, } self.default_program_name(config) env = {'TESTPATH': os.path.pathsep.join(paths)} @@ -1169,6 +1171,7 @@ def test_init_setpath_config(self): # The current getpath.c doesn't determine the stdlib dir # in this case. 'stdlib_dir': '', + 'use_frozen_modules': -1, # overridden by PyConfig 'program_name': 'conf_program_name', 'base_executable': 'conf_executable', @@ -1265,6 +1268,8 @@ def test_init_setpythonhome(self): 'stdlib_dir': stdlib, } self.default_program_name(config) + if not config['executable']: + config['use_frozen_modules'] = -1 env = {'TESTHOME': home, 'PYTHONPATH': paths_str} self.check_all_configs("test_init_setpythonhome", config, api=API_COMPAT, env=env) @@ -1303,6 +1308,7 @@ def test_init_pybuilddir(self): # The current getpath.c doesn't determine the stdlib dir # in this case. 'stdlib_dir': None, + 'use_frozen_modules': -1, } env = self.copy_paths_by_env(config) self.check_all_configs("test_init_compat_config", config, @@ -1361,6 +1367,7 @@ def test_init_pyvenv_cfg(self): config['base_prefix'] = pyvenv_home config['prefix'] = pyvenv_home config['stdlib_dir'] = os.path.join(pyvenv_home, 'lib') + config['use_frozen_modules'] = 1 ver = sys.version_info dll = f'python{ver.major}' @@ -1373,6 +1380,7 @@ def test_init_pyvenv_cfg(self): # The current getpath.c doesn't determine the stdlib dir # in this case. config['stdlib_dir'] = None + config['use_frozen_modules'] = -1 env = self.copy_paths_by_env(config) self.check_all_configs("test_init_compat_config", config, diff --git a/Misc/NEWS.d/3.11.0a1.rst b/Misc/NEWS.d/3.11.0a1.rst index e3d2acc4999..a64a3e74ccb 100644 --- a/Misc/NEWS.d/3.11.0a1.rst +++ b/Misc/NEWS.d/3.11.0a1.rst @@ -262,7 +262,7 @@ Compiler now removes trailing unused constants from co_consts. Add a new command line option, "-X frozen_modules=[on|off]" to opt out of (or into) using optional frozen modules. This defaults to "on" (or "off" if -it's a debug build). +it's running out of the source tree). .. diff --git a/Python/fileutils.c b/Python/fileutils.c index 173d34dd23f..3d8f3a4f163 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -2169,6 +2169,18 @@ _Py_add_relfile(wchar_t *dirname, const wchar_t *relfile, size_t bufsize) } +size_t +_Py_find_basename(const wchar_t *filename) +{ + for (size_t i = wcslen(filename); i > 0; --i) { + if (filename[i] == SEP) { + return i + 1; + } + } + return 0; +} + + /* Get the current directory. buflen is the buffer size in wide characters including the null character. Decode the path from the locale encoding. diff --git a/Python/initconfig.c b/Python/initconfig.c index b0d54b0472f..c916e2f7c07 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -739,6 +739,7 @@ _PyConfig_InitCompatConfig(PyConfig *config) #ifdef MS_WINDOWS config->legacy_windows_stdio = -1; #endif + config->use_frozen_modules = -1; } @@ -2090,6 +2091,44 @@ config_init_fs_encoding(PyConfig *config, const PyPreConfig *preconfig) } +/* Determine if the current build is a "development" build (e.g. running + out of the source tree) or not. + + A return value of -1 indicates that we do not know. + */ +static int +is_dev_env(PyConfig *config) +{ + // This should only ever get called early in runtime initialization, + // before the global path config is written. Otherwise we would + // use Py_GetProgramFullPath() and _Py_GetStdlibDir(). + assert(config != NULL); + + const wchar_t *executable = config->executable; + const wchar_t *stdlib = config->stdlib_dir; + if (executable == NULL || *executable == L'\0' || + stdlib == NULL || *stdlib == L'\0') { + // _PyPathConfig_Calculate() hasn't run yet. + return -1; + } + size_t len = _Py_find_basename(executable); + if (wcscmp(executable + len, L"python") != 0 && + wcscmp(executable + len, L"python.exe") != 0) { + return 0; + } + /* If dirname() is the same for both then it is a dev build. */ + if (len != _Py_find_basename(stdlib)) { + return 0; + } + // We do not bother normalizing the two filenames first since + // for config_init_import() is does the right thing as-is. + if (wcsncmp(stdlib, executable, len) != 0) { + return 0; + } + return 1; +} + + static PyStatus config_init_import(PyConfig *config, int compute_path_config) { @@ -2101,25 +2140,28 @@ config_init_import(PyConfig *config, int compute_path_config) } /* -X frozen_modules=[on|off] */ - const wchar_t *value = config_get_xoption_value(config, L"frozen_modules"); - if (value == NULL) { - // For now we always default to "off". - // In the near future we will be factoring in PGO and in-development. - config->use_frozen_modules = 0; - } - else if (wcscmp(value, L"on") == 0) { - config->use_frozen_modules = 1; - } - else if (wcscmp(value, L"off") == 0) { - config->use_frozen_modules = 0; - } - else if (wcslen(value) == 0) { - // "-X frozen_modules" and "-X frozen_modules=" both imply "on". - config->use_frozen_modules = 1; - } - else { - return PyStatus_Error("bad value for option -X frozen_modules " - "(expected \"on\" or \"off\")"); + if (config->use_frozen_modules < 0) { + const wchar_t *value = config_get_xoption_value(config, L"frozen_modules"); + if (value == NULL) { + int isdev = is_dev_env(config); + if (isdev >= 0) { + config->use_frozen_modules = !isdev; + } + } + else if (wcscmp(value, L"on") == 0) { + config->use_frozen_modules = 1; + } + else if (wcscmp(value, L"off") == 0) { + config->use_frozen_modules = 0; + } + else if (wcslen(value) == 0) { + // "-X frozen_modules" and "-X frozen_modules=" both imply "on". + config->use_frozen_modules = 1; + } + else { + return PyStatus_Error("bad value for option -X frozen_modules " + "(expected \"on\" or \"off\")"); + } } return _PyStatus_OK();