Merge with 3.6

Doc/using/windows.rst

@@ -720,15 +720,24 @@ installation directory.  So, if you had installed Python to
 :file:`C:\\Python\\Lib\\` and third-party modules should be stored in
 :file:`C:\\Python\\Lib\\site-packages\\`.
 
-To completely override :data:`sys.path`, create a text file named ``'sys.path'``
-containing a list of paths alongside the Python executable. This will ignore all
-registry settings and environment variables, enable isolated mode, disable
-importing :mod:`site`, and fill :data:`sys.path` with exactly the paths listed
-in the file. Paths may be absolute or relative to the directory containing the
-file.
+To completely override :data:`sys.path`, create a ``._pth`` file with the same
+name as the DLL (``python36._pth``) or the executable (``python._pth``) and
+specify one line for each path to add to :data:`sys.path`. The file based on the
+DLL name overrides the one based on the executable, which allows paths to be
+restricted for any program loading the runtime if desired.
 
-When the ``'sys.path'`` file is missing, this is how :data:`sys.path` is
-populated on Windows:
+When the file exists, all registry and environment variables are ignored,
+isolated mode is enabled, and :mod:`site` is not imported unless one line in the
+file specifies ``import site``. Blank paths and lines starting with ``#`` are
+ignored. Each path may be absolute or relative to the location of the file.
+Import statements other than to ``site`` are not permitted, and arbitrary code
+cannot be specified.
+
+Note that ``.pth`` files (without leading underscore) will be processed normally
+by the :mod:`site` module.
+
+When no ``._pth`` file is found, this is how :data:`sys.path` is populated on
+Windows:
 
 * An empty entry is added at the start, which corresponds to the current
   directory.
@@ -782,9 +791,10 @@ The end result of all this is:
 For those who want to bundle Python into their application or distribution, the
 following advice will prevent conflicts with other installations:
 
-* Include a ``sys.path`` file alongside your executable containing the
-  directories to include. This will ignore user site-packages and other paths
-  listed in the registry or in environment variables.
+* Include a ``._pth`` file alongside your executable containing the
+  directories to include. This will ignore paths listed in the registry and
+  environment variables, and also ignore :mod:`site` unless ``import site`` is
+  listed.
 
 * If you are loading :file:`python3.dll` or :file:`python36.dll` in your own
   executable, explicitly call :c:func:`Py_SetPath` or (at least)

Doc/whatsnew/3.6.rst

@@ -108,7 +108,7 @@ Windows improvements:
   which means that when the 260 character path limit may no longer apply.
   See :ref:`removing the MAX_PATH limitation <max-path>` for details.
 
-* A ``sys.path`` file can be added to force isolated mode and fully specify
+* A ``._pth`` file can be added to force isolated mode and fully specify
   all search paths to avoid registry and environment lookup. See
   :ref:`the documentation <finding_modules>` for more information.
 

Lib/site.py

@@ -60,7 +60,8 @@
 
 The readline module is also automatically configured to enable
 completion for systems that support it.  This can be overridden in
-sitecustomize, usercustomize or PYTHONSTARTUP.
+sitecustomize, usercustomize or PYTHONSTARTUP.  Starting Python in
+isolated mode (-I) disables automatic readline configuration.
 
 After these operations, an attempt is made to import a module
 named sitecustomize, which can perform arbitrary additional
@@ -491,7 +492,7 @@ def execsitecustomize():
             else:
                 raise
     except Exception as err:
-        if os.environ.get("PYTHONVERBOSE"):
+        if sys.flags.verbose:
             sys.excepthook(*sys.exc_info())
         else:
             sys.stderr.write(
@@ -511,7 +512,7 @@ def execusercustomize():
             else:
                 raise
     except Exception as err:
-        if os.environ.get("PYTHONVERBOSE"):
+        if sys.flags.verbose:
             sys.excepthook(*sys.exc_info())
         else:
             sys.stderr.write(
@@ -538,12 +539,13 @@ def main():
     setquit()
     setcopyright()
     sethelper()
-    enablerlcompleter()
+    if not sys.flags.isolated:
+        enablerlcompleter()
     execsitecustomize()
     if ENABLE_USER_SITE:
         execusercustomize()
 
-# Prevent edition of sys.path when python was started with -S and
+# Prevent extending of sys.path when python was started with -S and
 # site is imported later.
 if not sys.flags.no_site:
     main()

Lib/test/test_site.py

@@ -140,8 +140,6 @@ def test_addpackage_import_bad_exec(self):
         self.assertRegex(err_out.getvalue(), 'Traceback')
         self.assertRegex(err_out.getvalue(), 'ModuleNotFoundError')
 
-    @unittest.skipIf(sys.platform == "win32", "Windows does not raise an "
-                      "error for file paths containing null characters")
     def test_addpackage_import_bad_pth_file(self):
         # Issue 5258
         pth_dir, pth_fn = self.make_pth("abc\x00def\n")
@@ -447,10 +445,9 @@ def test_startup_imports(self):
         popen = subprocess.Popen([sys.executable, '-I', '-v', '-c',
                                   'import sys; print(set(sys.modules))'],
                                  stdout=subprocess.PIPE,
-                                 stderr=subprocess.PIPE)
+                                 stderr=subprocess.PIPE,
+                                 encoding='utf-8')
         stdout, stderr = popen.communicate()
-        stdout = stdout.decode('utf-8')
-        stderr = stderr.decode('utf-8')
         modules = eval(stdout)
 
         self.assertIn('site', modules)
@@ -474,6 +471,5 @@ def test_startup_imports(self):
         if sys.platform != 'darwin':
             self.assertFalse(modules.intersection(collection_mods), stderr)
 
-
 if __name__ == "__main__":
     unittest.main()

Misc/NEWS

@@ -10,6 +10,8 @@ What's New in Python 3.7.0 alpha 1
 Core and Builtins
 -----------------
 
+- Issue #28192: Don't import readline in isolated mode.
+
 - Issue #27441: Remove some redundant assignments to ob_size in longobject.c.
   Thanks Oren Milman.
 
@@ -70,6 +72,13 @@ Library
 - Issue #27759: Fix selectors incorrectly retain invalid file descriptors.
   Patch by Mark Williams.
 
+Windows
+-------
+
+- Issue #28137: Renames Windows path file to ._pth
+
+- Issue #28138: Windows ._pth file should allow import site
+
 Build
 -----
 

Modules/main.c

@@ -703,7 +703,8 @@ Py_Main(int argc, wchar_t **argv)
     PySys_SetArgv(argc-_PyOS_optind, argv+_PyOS_optind);
 
     if ((Py_InspectFlag || (command == NULL && filename == NULL && module == NULL)) &&
-        isatty(fileno(stdin))) {
+        isatty(fileno(stdin)) &&
+        !Py_IsolatedFlag) {
         PyObject *v;
         v = PyImport_ImportModule("readline");
         if (v == NULL)

PC/getpathp.c

@@ -6,8 +6,9 @@
    PATH RULES FOR WINDOWS:
    This describes how sys.path is formed on Windows.  It describes the
    functionality, not the implementation (ie, the order in which these
-   are actually fetched is different). The presence of a sys.path file
-   alongside the program overrides these rules - see below.
+   are actually fetched is different). The presence of a python._pth or
+   pythonXY._pth file alongside the program overrides these rules - see
+   below.
 
    * Python always adds an empty entry at the start, which corresponds
      to the current directory.
@@ -37,11 +38,21 @@
      used (eg. .\Lib;.\DLLs, etc)
 
 
-   If a sys.path file exists adjacent to python.exe, it must contain a
-   list of paths to add to sys.path, one per line (like a .pth file but without
-   the ability to execute arbitrary code). Each path is relative to the
-   directory containing the file. No other paths are added to the search path,
-   and the registry finder is not enabled.
+   If a '._pth' file exists adjacent to the executable with the same base name
+   (e.g. python._pth adjacent to python.exe) or adjacent to the shared library
+   (e.g. python36._pth adjacent to python36.dll), it is used in preference to
+   the above process. The shared library file takes precedence over the
+   executable. The path file must contain a list of paths to add to sys.path,
+   one per line. Each path is relative to the directory containing the file.
+   Blank lines and comments beginning with '#' are permitted.
+
+   In the presence of this ._pth file, no other paths are added to the search
+   path, the registry finder is not enabled, site.py is not imported and
+   isolated mode is enabled. The site package can be enabled by including a
+   line reading "import site"; no other imports are recognized. Any invalid
+   entry (other than directories that do not exist) will result in immediate
+   termination of the program.
+
 
   The end result of all this is:
   * When running python.exe, or any other .exe in the main Python directory
@@ -61,8 +72,9 @@
   * An embedding application can use Py_SetPath() to override all of
     these automatic path computations.
 
-  * An isolation install of Python can disable all implicit paths by
-    providing a sys.path file.
+  * An install of Python can fully specify the contents of sys.path using
+    either a 'EXENAME._pth' or 'DLLNAME._pth' file, optionally including
+    "import site" to enable the site module.
 
    ---------------------------------------------------------------- */
 
@@ -135,6 +147,33 @@ reduce(wchar_t *dir)
     dir[i] = '\0';
 }
 
+static int
+change_ext(wchar_t *dest, const wchar_t *src, const wchar_t *ext)
+{
+    size_t src_len = wcsnlen_s(src, MAXPATHLEN+1);
+    size_t i = src_len;
+    if (i >= MAXPATHLEN+1)
+        Py_FatalError("buffer overflow in getpathp.c's reduce()");
+
+    while (i > 0 && src[i] != '.' && !is_sep(src[i]))
+        --i;
+
+    if (i == 0) {
+        dest[0] = '\0';
+        return -1;
+    }
+
+    if (is_sep(src[i]))
+        i = src_len;
+
+    if (wcsncpy_s(dest, MAXPATHLEN+1, src, i) ||
+        wcscat_s(dest, MAXPATHLEN+1, ext)) {
+        dest[0] = '\0';
+        return -1;
+    }
+
+    return 0;
+}
 
 static int
 exists(wchar_t *filename)
@@ -499,12 +538,17 @@ find_env_config_value(FILE * env_file, const wchar_t * key, wchar_t * value)
 }
 
 static int
-read_sys_path_file(const wchar_t *path, const wchar_t *prefix)
+read_pth_file(const wchar_t *path, wchar_t *prefix, int *isolated, int *nosite)
 {
     FILE *sp_file = _Py_wfopen(path, L"r");
     if (sp_file == NULL)
         return -1;
 
+    wcscpy_s(prefix, MAXPATHLEN+1, path);
+    reduce(prefix);
+    *isolated = 1;
+    *nosite = 1;
+
     size_t bufsiz = MAXPATHLEN;
     size_t prefixlen = wcslen(prefix);
 
@@ -516,16 +560,25 @@ read_sys_path_file(const wchar_t *path, const wchar_t *prefix)
         char *p = fgets(line, MAXPATHLEN + 1, sp_file);
         if (!p)
             break;
+        if (*p == '\0' || *p == '#')
+            continue;
+        while (*++p) {
+            if (*p == '\r' || *p == '\n') {
+                *p = '\0';
+                break;
+            }
+        }
 
-        DWORD n = strlen(line);
-        if (n == 0 || p[n - 1] != '\n')
-            break;
-        if (n > 2 && p[n - 1] == '\r')
-            --n;
+        if (strcmp(line, "import site") == 0) {
+            *nosite = 0;
+            continue;
+        } else if (strncmp(line, "import ", 7) == 0) {
+            Py_FatalError("only 'import site' is supported in ._pth file");
+        }
 
-        DWORD wn = MultiByteToWideChar(CP_UTF8, 0, line, n - 1, NULL, 0);
+        DWORD wn = MultiByteToWideChar(CP_UTF8, 0, line, -1, NULL, 0);
         wchar_t *wline = (wchar_t*)PyMem_RawMalloc((wn + 1) * sizeof(wchar_t));
-        wn = MultiByteToWideChar(CP_UTF8, 0, line, n - 1, wline, wn);
+        wn = MultiByteToWideChar(CP_UTF8, 0, line, -1, wline, wn + 1);
         wline[wn] = '\0';
 
         while (wn + prefixlen + 4 > bufsiz) {
@@ -539,8 +592,8 @@ read_sys_path_file(const wchar_t *path, const wchar_t *prefix)
 
         if (buf[0])
             wcscat_s(buf, bufsiz, L";");
+
         wchar_t *b = &buf[wcslen(buf)];
-        
         wcscat_s(buf, bufsiz, prefix);
         join(b, wline);
 
@@ -586,13 +639,12 @@ calculate_path(void)
     {
         wchar_t spbuffer[MAXPATHLEN+1];
 
-        wcscpy_s(spbuffer, MAXPATHLEN+1, argv0_path);
-        join(spbuffer, L"sys.path");
-        if (exists(spbuffer) && read_sys_path_file(spbuffer, argv0_path) == 0) {
-            wcscpy_s(prefix, MAXPATHLEN + 1, argv0_path);
-            Py_IsolatedFlag = 1;
-            Py_NoSiteFlag = 1;
-            return;
+        if ((dllpath[0] && !change_ext(spbuffer, dllpath, L"._pth") && exists(spbuffer)) ||
+            (progpath[0] && !change_ext(spbuffer, progpath, L"._pth") && exists(spbuffer))) {
+
+            if (!read_pth_file(spbuffer, prefix, &Py_IsolatedFlag, &Py_NoSiteFlag)) {
+                return;
+            }
         }
     }
 
@@ -631,16 +683,7 @@ calculate_path(void)
     }
 
     /* Calculate zip archive path from DLL or exe path */
-    if (wcscpy_s(zip_path, MAXPATHLEN + 1, dllpath[0] ? dllpath : progpath)) {
-        /* exceeded buffer length - ignore zip_path */
-        zip_path[0] = '\0';
-    } else {
-        wchar_t *dot = wcsrchr(zip_path, '.');
-        if (!dot || wcscpy_s(dot, MAXPATHLEN + 1 - (dot - zip_path), L".zip")) {
-            /* exceeded buffer length - ignore zip_path */
-            zip_path[0] = L'\0';
-        }
-    }
+    change_ext(zip_path, dllpath[0] ? dllpath : progpath, L".zip");
 
     if (pythonhome == NULL || *pythonhome == '\0') {
         if (zip_path[0] && exists(zip_path)) {

Tools/msi/make_zip.py

@@ -91,11 +91,13 @@ def include_in_tools(p):
 
     return p.suffix.lower() in {'.py', '.pyw', '.txt'}
 
+BASE_NAME = 'python{0.major}{0.minor}'.format(sys.version_info)
+
 FULL_LAYOUT = [
     ('/', 'PCBuild/$arch', 'python.exe', is_not_debug),
     ('/', 'PCBuild/$arch', 'pythonw.exe', is_not_debug),
-    ('/', 'PCBuild/$arch', 'python{0.major}.dll'.format(sys.version_info), is_not_debug),
-    ('/', 'PCBuild/$arch', 'python{0.major}{0.minor}.dll'.format(sys.version_info), is_not_debug),
+    ('/', 'PCBuild/$arch', 'python{}.dll'.format(sys.version_info.major), is_not_debug),
+    ('/', 'PCBuild/$arch', '{}.dll'.format(BASE_NAME), is_not_debug),
     ('DLLs/', 'PCBuild/$arch', '*.pyd', is_not_debug),
     ('DLLs/', 'PCBuild/$arch', '*.dll', is_not_debug_or_python),
     ('include/', 'include', '*.h', None),
@@ -109,7 +111,7 @@ def include_in_tools(p):
     ('/', 'PCBuild/$arch', 'python*.exe', is_not_debug),
     ('/', 'PCBuild/$arch', '*.pyd', is_not_debug),
     ('/', 'PCBuild/$arch', '*.dll', is_not_debug),
-    ('python{0.major}{0.minor}.zip'.format(sys.version_info), 'Lib', '**/*', include_in_lib),
+    ('{}.zip'.format(BASE_NAME), 'Lib', '**/*', include_in_lib),
 ]
 
 if os.getenv('DOC_FILENAME'):
@@ -209,9 +211,12 @@ def main():
             print('Copied {} files'.format(copied))
 
         if ns.embed:
-            with open(str(temp / 'sys.path'), 'w') as f:
-                print('python{0.major}{0.minor}.zip'.format(sys.version_info), file=f)
+            with open(str(temp / (BASE_NAME + '._pth')), 'w') as f:
+                print(BASE_NAME + '.zip', file=f)
                 print('.', file=f)
+                print('', file=f)
+                print('# Uncomment to run site.main() automatically', file=f)
+                print('#import site', file=f)
 
         if out:
             total = copy_to_layout(out, rglob(temp, '**/*', None))