diff --git a/Include/cpython/fileutils.h b/Include/cpython/fileutils.h index 7ab2290184a..b386ad107bd 100644 --- a/Include/cpython/fileutils.h +++ b/Include/cpython/fileutils.h @@ -2,165 +2,7 @@ # error "this header file must not be included directly" #endif -typedef enum { - _Py_ERROR_UNKNOWN=0, - _Py_ERROR_STRICT, - _Py_ERROR_SURROGATEESCAPE, - _Py_ERROR_REPLACE, - _Py_ERROR_IGNORE, - _Py_ERROR_BACKSLASHREPLACE, - _Py_ERROR_SURROGATEPASS, - _Py_ERROR_XMLCHARREFREPLACE, - _Py_ERROR_OTHER -} _Py_error_handler; - -PyAPI_FUNC(_Py_error_handler) _Py_GetErrorHandler(const char *errors); - -PyAPI_FUNC(int) _Py_DecodeLocaleEx( - const char *arg, - wchar_t **wstr, - size_t *wlen, - const char **reason, - int current_locale, - _Py_error_handler errors); - -PyAPI_FUNC(int) _Py_EncodeLocaleEx( - const wchar_t *text, - char **str, - size_t *error_pos, - const char **reason, - int current_locale, - _Py_error_handler errors); - -PyAPI_FUNC(char*) _Py_EncodeLocaleRaw( - const wchar_t *text, - size_t *error_pos); - -PyAPI_FUNC(PyObject *) _Py_device_encoding(int); - -#if defined(MS_WINDOWS) || defined(__APPLE__) - /* On Windows, the count parameter of read() is an int (bpo-9015, bpo-9611). - On macOS 10.13, read() and write() with more than INT_MAX bytes - fail with EINVAL (bpo-24658). */ -# define _PY_READ_MAX INT_MAX -# define _PY_WRITE_MAX INT_MAX -#else - /* write() should truncate the input to PY_SSIZE_T_MAX bytes, - but it's safer to do it ourself to have a portable behaviour */ -# define _PY_READ_MAX PY_SSIZE_T_MAX -# define _PY_WRITE_MAX PY_SSIZE_T_MAX -#endif - -#ifdef MS_WINDOWS -struct _Py_stat_struct { - unsigned long st_dev; - uint64_t st_ino; - unsigned short st_mode; - int st_nlink; - int st_uid; - int st_gid; - unsigned long st_rdev; - __int64 st_size; - time_t st_atime; - int st_atime_nsec; - time_t st_mtime; - int st_mtime_nsec; - time_t st_ctime; - int st_ctime_nsec; - unsigned long st_file_attributes; - unsigned long st_reparse_tag; -}; -#else -# define _Py_stat_struct stat -#endif - -PyAPI_FUNC(int) _Py_fstat( - int fd, - struct _Py_stat_struct *status); - -PyAPI_FUNC(int) _Py_fstat_noraise( - int fd, - struct _Py_stat_struct *status); - -PyAPI_FUNC(int) _Py_stat( - PyObject *path, - struct stat *status); - -PyAPI_FUNC(int) _Py_open( - const char *pathname, - int flags); - -PyAPI_FUNC(int) _Py_open_noraise( - const char *pathname, - int flags); - -PyAPI_FUNC(FILE *) _Py_wfopen( - const wchar_t *path, - const wchar_t *mode); - +// Used by _testcapi which must not use the internal C API PyAPI_FUNC(FILE*) _Py_fopen_obj( PyObject *path, const char *mode); - -PyAPI_FUNC(Py_ssize_t) _Py_read( - int fd, - void *buf, - size_t count); - -PyAPI_FUNC(Py_ssize_t) _Py_write( - int fd, - const void *buf, - size_t count); - -PyAPI_FUNC(Py_ssize_t) _Py_write_noraise( - int fd, - const void *buf, - size_t count); - -#ifdef HAVE_READLINK -PyAPI_FUNC(int) _Py_wreadlink( - const wchar_t *path, - wchar_t *buf, - /* Number of characters of 'buf' buffer - including the trailing NUL character */ - size_t buflen); -#endif - -#ifdef HAVE_REALPATH -PyAPI_FUNC(wchar_t*) _Py_wrealpath( - const wchar_t *path, - wchar_t *resolved_path, - /* Number of characters of 'resolved_path' buffer - including the trailing NUL character */ - size_t resolved_path_len); -#endif - -PyAPI_FUNC(wchar_t*) _Py_wgetcwd( - wchar_t *buf, - /* Number of characters of 'buf' buffer - including the trailing NUL character */ - size_t buflen); - -PyAPI_FUNC(int) _Py_get_inheritable(int fd); - -PyAPI_FUNC(int) _Py_set_inheritable(int fd, int inheritable, - int *atomic_flag_works); - -PyAPI_FUNC(int) _Py_set_inheritable_async_safe(int fd, int inheritable, - int *atomic_flag_works); - -PyAPI_FUNC(int) _Py_dup(int fd); - -#ifndef MS_WINDOWS -PyAPI_FUNC(int) _Py_get_blocking(int fd); - -PyAPI_FUNC(int) _Py_set_blocking(int fd, int blocking); -#else /* MS_WINDOWS */ -PyAPI_FUNC(void*) _Py_get_osfhandle_noraise(int fd); - -PyAPI_FUNC(void*) _Py_get_osfhandle(int fd); - -PyAPI_FUNC(int) _Py_open_osfhandle_noraise(void *handle, int flags); - -PyAPI_FUNC(int) _Py_open_osfhandle(void *handle, int flags); -#endif /* MS_WINDOWS */ diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index 38df73b745b..61c11a8b2d3 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -10,6 +10,165 @@ extern "C" { #include /* struct lconv */ +typedef enum { + _Py_ERROR_UNKNOWN=0, + _Py_ERROR_STRICT, + _Py_ERROR_SURROGATEESCAPE, + _Py_ERROR_REPLACE, + _Py_ERROR_IGNORE, + _Py_ERROR_BACKSLASHREPLACE, + _Py_ERROR_SURROGATEPASS, + _Py_ERROR_XMLCHARREFREPLACE, + _Py_ERROR_OTHER +} _Py_error_handler; + +PyAPI_FUNC(_Py_error_handler) _Py_GetErrorHandler(const char *errors); + +PyAPI_FUNC(int) _Py_DecodeLocaleEx( + const char *arg, + wchar_t **wstr, + size_t *wlen, + const char **reason, + int current_locale, + _Py_error_handler errors); + +PyAPI_FUNC(int) _Py_EncodeLocaleEx( + const wchar_t *text, + char **str, + size_t *error_pos, + const char **reason, + int current_locale, + _Py_error_handler errors); + +PyAPI_FUNC(char*) _Py_EncodeLocaleRaw( + const wchar_t *text, + size_t *error_pos); + +PyAPI_FUNC(PyObject *) _Py_device_encoding(int); + +#if defined(MS_WINDOWS) || defined(__APPLE__) + /* On Windows, the count parameter of read() is an int (bpo-9015, bpo-9611). + On macOS 10.13, read() and write() with more than INT_MAX bytes + fail with EINVAL (bpo-24658). */ +# define _PY_READ_MAX INT_MAX +# define _PY_WRITE_MAX INT_MAX +#else + /* write() should truncate the input to PY_SSIZE_T_MAX bytes, + but it's safer to do it ourself to have a portable behaviour */ +# define _PY_READ_MAX PY_SSIZE_T_MAX +# define _PY_WRITE_MAX PY_SSIZE_T_MAX +#endif + +#ifdef MS_WINDOWS +struct _Py_stat_struct { + unsigned long st_dev; + uint64_t st_ino; + unsigned short st_mode; + int st_nlink; + int st_uid; + int st_gid; + unsigned long st_rdev; + __int64 st_size; + time_t st_atime; + int st_atime_nsec; + time_t st_mtime; + int st_mtime_nsec; + time_t st_ctime; + int st_ctime_nsec; + unsigned long st_file_attributes; + unsigned long st_reparse_tag; +}; +#else +# define _Py_stat_struct stat +#endif + +PyAPI_FUNC(int) _Py_fstat( + int fd, + struct _Py_stat_struct *status); + +PyAPI_FUNC(int) _Py_fstat_noraise( + int fd, + struct _Py_stat_struct *status); + +PyAPI_FUNC(int) _Py_stat( + PyObject *path, + struct stat *status); + +PyAPI_FUNC(int) _Py_open( + const char *pathname, + int flags); + +PyAPI_FUNC(int) _Py_open_noraise( + const char *pathname, + int flags); + +PyAPI_FUNC(FILE *) _Py_wfopen( + const wchar_t *path, + const wchar_t *mode); + +PyAPI_FUNC(Py_ssize_t) _Py_read( + int fd, + void *buf, + size_t count); + +PyAPI_FUNC(Py_ssize_t) _Py_write( + int fd, + const void *buf, + size_t count); + +PyAPI_FUNC(Py_ssize_t) _Py_write_noraise( + int fd, + const void *buf, + size_t count); + +#ifdef HAVE_READLINK +PyAPI_FUNC(int) _Py_wreadlink( + const wchar_t *path, + wchar_t *buf, + /* Number of characters of 'buf' buffer + including the trailing NUL character */ + size_t buflen); +#endif + +#ifdef HAVE_REALPATH +PyAPI_FUNC(wchar_t*) _Py_wrealpath( + const wchar_t *path, + wchar_t *resolved_path, + /* Number of characters of 'resolved_path' buffer + including the trailing NUL character */ + size_t resolved_path_len); +#endif + +PyAPI_FUNC(wchar_t*) _Py_wgetcwd( + wchar_t *buf, + /* Number of characters of 'buf' buffer + including the trailing NUL character */ + size_t buflen); + +PyAPI_FUNC(int) _Py_get_inheritable(int fd); + +PyAPI_FUNC(int) _Py_set_inheritable(int fd, int inheritable, + int *atomic_flag_works); + +PyAPI_FUNC(int) _Py_set_inheritable_async_safe(int fd, int inheritable, + int *atomic_flag_works); + +PyAPI_FUNC(int) _Py_dup(int fd); + +#ifndef MS_WINDOWS +PyAPI_FUNC(int) _Py_get_blocking(int fd); + +PyAPI_FUNC(int) _Py_set_blocking(int fd, int blocking); +#else /* MS_WINDOWS */ +PyAPI_FUNC(void*) _Py_get_osfhandle_noraise(int fd); + +PyAPI_FUNC(void*) _Py_get_osfhandle(int fd); + +PyAPI_FUNC(int) _Py_open_osfhandle_noraise(void *handle, int flags); + +PyAPI_FUNC(int) _Py_open_osfhandle(void *handle, int flags); +#endif /* MS_WINDOWS */ + // This is used after getting NULL back from Py_DecodeLocale(). #define DECODE_LOCALE_ERR(NAME, LEN) \ ((LEN) == (size_t)-2) \ diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index 97d11aeb820..3b6dfe9dbba 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -8,6 +8,8 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_fileutils.h" // _Py_error_handler + /* runtime lifecycle */ diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index f924826db94..4ad24dbb9a9 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -15,6 +15,10 @@ import _testcapi except ImportError: _testcapi = None +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = None try: import ctypes @@ -3345,7 +3349,7 @@ def test_seeking_write(self): self.assertEqual(sr.readline(), b'789\n') -@unittest.skipIf(_testcapi is None, 'need _testcapi module') +@unittest.skipIf(_testinternalcapi is None, 'need _testinternalcapi module') class LocaleCodecTest(unittest.TestCase): """ Test indirectly _Py_DecodeUTF8Ex() and _Py_EncodeUTF8Ex(). @@ -3359,7 +3363,7 @@ class LocaleCodecTest(unittest.TestCase): SURROGATES = "\uDC80\uDCFF" def encode(self, text, errors="strict"): - return _testcapi.EncodeLocaleEx(text, 0, errors) + return _testinternalcapi.EncodeLocaleEx(text, 0, errors) def check_encode_strings(self, errors): for text in self.STRINGS: @@ -3399,7 +3403,7 @@ def test_encode_unsupported_error_handler(self): self.assertEqual(str(cm.exception), 'unsupported error handler') def decode(self, encoded, errors="strict"): - return _testcapi.DecodeLocaleEx(encoded, 0, errors) + return _testinternalcapi.DecodeLocaleEx(encoded, 0, errors) def check_decode_strings(self, errors): is_utf8 = (self.ENCODING == "utf-8") diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index be40d68b40b..ea9c048554d 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -5417,98 +5417,6 @@ bad_get(PyObject *module, PyObject *const *args, Py_ssize_t nargs) } -static PyObject * -encode_locale_ex(PyObject *self, PyObject *args) -{ - PyObject *unicode; - int current_locale = 0; - wchar_t *wstr; - PyObject *res = NULL; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "U|is", &unicode, ¤t_locale, &errors)) { - return NULL; - } - wstr = PyUnicode_AsWideCharString(unicode, NULL); - if (wstr == NULL) { - return NULL; - } - _Py_error_handler error_handler = _Py_GetErrorHandler(errors); - - char *str = NULL; - size_t error_pos; - const char *reason = NULL; - int ret = _Py_EncodeLocaleEx(wstr, - &str, &error_pos, &reason, - current_locale, error_handler); - PyMem_Free(wstr); - - switch(ret) { - case 0: - res = PyBytes_FromString(str); - PyMem_RawFree(str); - break; - case -1: - PyErr_NoMemory(); - break; - case -2: - PyErr_Format(PyExc_RuntimeError, "encode error: pos=%zu, reason=%s", - error_pos, reason); - break; - case -3: - PyErr_SetString(PyExc_ValueError, "unsupported error handler"); - break; - default: - PyErr_SetString(PyExc_ValueError, "unknown error code"); - break; - } - return res; -} - - -static PyObject * -decode_locale_ex(PyObject *self, PyObject *args) -{ - char *str; - int current_locale = 0; - PyObject *res = NULL; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "y|is", &str, ¤t_locale, &errors)) { - return NULL; - } - _Py_error_handler error_handler = _Py_GetErrorHandler(errors); - - wchar_t *wstr = NULL; - size_t wlen = 0; - const char *reason = NULL; - int ret = _Py_DecodeLocaleEx(str, - &wstr, &wlen, &reason, - current_locale, error_handler); - - switch(ret) { - case 0: - res = PyUnicode_FromWideChar(wstr, wlen); - PyMem_RawFree(wstr); - break; - case -1: - PyErr_NoMemory(); - break; - case -2: - PyErr_Format(PyExc_RuntimeError, "decode error: pos=%zu, reason=%s", - wlen, reason); - break; - case -3: - PyErr_SetString(PyExc_ValueError, "unsupported error handler"); - break; - default: - PyErr_SetString(PyExc_ValueError, "unknown error code"); - break; - } - return res; -} - - #ifdef Py_REF_DEBUG static PyObject * negative_refcount(PyObject *self, PyObject *Py_UNUSED(args)) @@ -6125,8 +6033,6 @@ static PyMethodDef TestMethods[] = { {"test_pythread_tss_key_state", test_pythread_tss_key_state, METH_VARARGS}, {"hamt", new_hamt, METH_NOARGS}, {"bad_get", (PyCFunction)(void(*)(void))bad_get, METH_FASTCALL}, - {"EncodeLocaleEx", encode_locale_ex, METH_VARARGS}, - {"DecodeLocaleEx", decode_locale_ex, METH_VARARGS}, #ifdef Py_REF_DEBUG {"negative_refcount", negative_refcount, METH_NOARGS}, #endif diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 19babb06f56..9deba3558bf 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -399,6 +399,98 @@ get_getpath_codeobject(PyObject *self, PyObject *Py_UNUSED(args)) { } +static PyObject * +encode_locale_ex(PyObject *self, PyObject *args) +{ + PyObject *unicode; + int current_locale = 0; + wchar_t *wstr; + PyObject *res = NULL; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "U|is", &unicode, ¤t_locale, &errors)) { + return NULL; + } + wstr = PyUnicode_AsWideCharString(unicode, NULL); + if (wstr == NULL) { + return NULL; + } + _Py_error_handler error_handler = _Py_GetErrorHandler(errors); + + char *str = NULL; + size_t error_pos; + const char *reason = NULL; + int ret = _Py_EncodeLocaleEx(wstr, + &str, &error_pos, &reason, + current_locale, error_handler); + PyMem_Free(wstr); + + switch(ret) { + case 0: + res = PyBytes_FromString(str); + PyMem_RawFree(str); + break; + case -1: + PyErr_NoMemory(); + break; + case -2: + PyErr_Format(PyExc_RuntimeError, "encode error: pos=%zu, reason=%s", + error_pos, reason); + break; + case -3: + PyErr_SetString(PyExc_ValueError, "unsupported error handler"); + break; + default: + PyErr_SetString(PyExc_ValueError, "unknown error code"); + break; + } + return res; +} + + +static PyObject * +decode_locale_ex(PyObject *self, PyObject *args) +{ + char *str; + int current_locale = 0; + PyObject *res = NULL; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y|is", &str, ¤t_locale, &errors)) { + return NULL; + } + _Py_error_handler error_handler = _Py_GetErrorHandler(errors); + + wchar_t *wstr = NULL; + size_t wlen = 0; + const char *reason = NULL; + int ret = _Py_DecodeLocaleEx(str, + &wstr, &wlen, &reason, + current_locale, error_handler); + + switch(ret) { + case 0: + res = PyUnicode_FromWideChar(wstr, wlen); + PyMem_RawFree(wstr); + break; + case -1: + PyErr_NoMemory(); + break; + case -2: + PyErr_Format(PyExc_RuntimeError, "decode error: pos=%zu, reason=%s", + wlen, reason); + break; + case -3: + PyErr_SetString(PyExc_ValueError, "unsupported error handler"); + break; + default: + PyErr_SetString(PyExc_ValueError, "unknown error code"); + break; + } + return res; +} + + static PyMethodDef TestMethods[] = { {"get_configs", get_configs, METH_NOARGS}, {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, @@ -413,6 +505,8 @@ static PyMethodDef TestMethods[] = { {"test_edit_cost", test_edit_cost, METH_NOARGS}, {"normalize_path", normalize_path, METH_O, NULL}, {"get_getpath_codeobject", get_getpath_codeobject, METH_NOARGS, NULL}, + {"EncodeLocaleEx", encode_locale_ex, METH_VARARGS}, + {"DecodeLocaleEx", decode_locale_ex, METH_VARARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 68e5f0d52dd..b838439a9cb 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -1,8 +1,9 @@ #include "Python.h" +#include "pycore_fileutils.h" // _Py_write_noraise() #include "pycore_gc.h" // PyGC_Head +#include "pycore_hashtable.h" // _Py_hashtable_t #include "pycore_pymem.h" // _Py_tracemalloc_config #include "pycore_traceback.h" -#include "pycore_hashtable.h" #include #include // malloc() diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 742bcb3d145..7c9c28f7fab 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -18,8 +18,13 @@ / ftp://squirl.nightmare.com/pub/python/python-ext. */ +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + #define PY_SSIZE_T_CLEAN #include +#include "pycore_fileutils.h" // _Py_stat_struct #include "structmember.h" // PyMemberDef #include // offsetof() diff --git a/Modules/ossaudiodev.c b/Modules/ossaudiodev.c index 4bab9a58eb1..c9e788fd704 100644 --- a/Modules/ossaudiodev.c +++ b/Modules/ossaudiodev.c @@ -17,8 +17,13 @@ * $Id$ */ +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + #define PY_SSIZE_T_CLEAN #include "Python.h" +#include "pycore_fileutils.h" // _Py_write() #include "structmember.h" // PyMemberDef #include // getenv() diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index ff1c028d0d6..367e299f83a 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -4,11 +4,16 @@ have any value except INVALID_SOCKET. */ +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + #if defined(HAVE_POLL_H) && !defined(_GNU_SOURCE) -#define _GNU_SOURCE +# define _GNU_SOURCE #endif #include "Python.h" +#include "pycore_fileutils.h" // _Py_set_inheritable() #include "structmember.h" // PyMemberDef #ifdef HAVE_SYS_DEVPOLL_H diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 89e93c58187..ed83f5c8262 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -85,6 +85,10 @@ Local naming conventions: */ +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + #ifdef __APPLE__ // Issue #35569: Expose RFC 3542 socket options. #define __APPLE_USE_RFC_3542 1 @@ -103,6 +107,7 @@ Local naming conventions: #define PY_SSIZE_T_CLEAN #include "Python.h" +#include "pycore_fileutils.h" // _Py_set_inheritable() #include "structmember.h" // PyMemberDef #ifdef _Py_MEMORY_SANITIZER diff --git a/Programs/_freeze_module.c b/Programs/_freeze_module.c index d50787666f8..b2f1a24016f 100644 --- a/Programs/_freeze_module.c +++ b/Programs/_freeze_module.c @@ -11,6 +11,7 @@ #include #include +#include "pycore_fileutils.h" // _Py_stat_struct #include #include diff --git a/Python/bootstrap_hash.c b/Python/bootstrap_hash.c index 144f7cb4a21..3a2a7318086 100644 --- a/Python/bootstrap_hash.c +++ b/Python/bootstrap_hash.c @@ -1,5 +1,7 @@ #include "Python.h" #include "pycore_initconfig.h" +#include "pycore_fileutils.h" // _Py_fstat_noraise() + #ifdef MS_WINDOWS # include # include