bpo-26836: Add os.memfd_create() (#13567)

* bpo-26836: Add os.memfd_create()

* Use the glibc wrapper for memfd_create()

Co-Authored-By: Christian Heimes <christian@python.org>

* Fix deletions caused by autoreconf.

* Use MFD_CLOEXEC as the default value for *flags*.

* Add memset_s to configure.ac.

* Revert memset_s changes.

* Apply the requested changes.

* Tweak the docs.
This commit is contained in:
Zackery Spytz 2019-05-29 13:57:07 -06:00 committed by Christian Heimes
parent 0c2f930564
commit 43fdbd2729
10 changed files with 259 additions and 12 deletions

View file

@ -2982,6 +2982,44 @@ features:
Added support for :class:`bytes` paths.
.. function:: memfd_create(name[, flags=os.MFD_CLOEXEC])
Create an anonymous file and return a file descriptor that refers to it.
*flags* must be one of the ``os.MFD_*`` constants available on the system
(or a bitwise ORed combination of them). By default, the new file
descriptor is :ref:`non-inheritable <fd_inheritance>`.
.. availability:: Linux 3.17 or newer with glibc 2.27 or newer.
.. versionadded:: 3.8
.. data:: MFD_CLOEXEC
MFD_ALLOW_SEALING
MFD_HUGETLB
MFD_HUGE_SHIFT
MFD_HUGE_MASK
MFD_HUGE_64KB
MFD_HUGE_512KB
MFD_HUGE_1MB
MFD_HUGE_2MB
MFD_HUGE_8MB
MFD_HUGE_16MB
MFD_HUGE_32MB
MFD_HUGE_256MB
MFD_HUGE_512MB
MFD_HUGE_1GB
MFD_HUGE_2GB
MFD_HUGE_16GB
These flags can be passed to :func:`memfd_create`.
.. availability:: Linux 3.17 or newer with glibc 2.27 or newer. The
``MFD_HUGE*`` flags are only available since Linux 4.14.
.. versionadded:: 3.8
Linux extended attributes
~~~~~~~~~~~~~~~~~~~~~~~~~

View file

@ -486,6 +486,10 @@ Added new function :func:`~os.add_dll_directory` on Windows for providing
additional search paths for native dependencies when importing extension
modules or loading DLLs using :mod:`ctypes`.
A new :func:`os.memfd_create` function was added to wrap the
``memfd_create()`` syscall.
(Contributed by Zackery Spytz and Christian Heimes in :issue:`26836`.)
os.path
-------

View file

@ -3122,6 +3122,22 @@ def test_stty_match(self):
self.assertEqual(expected, actual)
@unittest.skipUnless(hasattr(os, 'memfd_create'), 'requires os.memfd_create')
class MemfdCreateTests(unittest.TestCase):
def test_memfd_create(self):
fd = os.memfd_create("Hi", os.MFD_CLOEXEC)
self.assertNotEqual(fd, -1)
self.addCleanup(os.close, fd)
self.assertFalse(os.get_inheritable(fd))
with open(fd, "wb", closefd=False) as f:
f.write(b'memfd_create')
self.assertEqual(f.tell(), 12)
fd2 = os.memfd_create("Hi")
self.addCleanup(os.close, fd2)
self.assertFalse(os.get_inheritable(fd2))
class OSErrorTests(unittest.TestCase):
def setUp(self):
class Str(str):

View file

@ -0,0 +1 @@
Add :func:`os.memfd_create`.

View file

@ -7343,6 +7343,61 @@ exit:
return return_value;
}
#if defined(HAVE_MEMFD_CREATE)
PyDoc_STRVAR(os_memfd_create__doc__,
"memfd_create($module, /, name, flags=MFD_CLOEXEC)\n"
"--\n"
"\n");
#define OS_MEMFD_CREATE_METHODDEF \
{"memfd_create", (PyCFunction)(void(*)(void))os_memfd_create, METH_FASTCALL|METH_KEYWORDS, os_memfd_create__doc__},
static PyObject *
os_memfd_create_impl(PyObject *module, PyObject *name, unsigned int flags);
static PyObject *
os_memfd_create(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
{
PyObject *return_value = NULL;
static const char * const _keywords[] = {"name", "flags", NULL};
static _PyArg_Parser _parser = {NULL, _keywords, "memfd_create", 0};
PyObject *argsbuf[2];
Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1;
PyObject *name = NULL;
unsigned int flags = MFD_CLOEXEC;
args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf);
if (!args) {
goto exit;
}
if (!PyUnicode_FSConverter(args[0], &name)) {
goto exit;
}
if (!noptargs) {
goto skip_optional_pos;
}
if (PyFloat_Check(args[1])) {
PyErr_SetString(PyExc_TypeError,
"integer argument expected, got float" );
goto exit;
}
flags = (unsigned int)PyLong_AsUnsignedLongMask(args[1]);
if (flags == (unsigned int)-1 && PyErr_Occurred()) {
goto exit;
}
skip_optional_pos:
return_value = os_memfd_create_impl(module, name, flags);
exit:
/* Cleanup for name */
Py_XDECREF(name);
return return_value;
}
#endif /* defined(HAVE_MEMFD_CREATE) */
PyDoc_STRVAR(os_cpu_count__doc__,
"cpu_count($module, /)\n"
"--\n"
@ -8549,6 +8604,10 @@ exit:
#define OS_LISTXATTR_METHODDEF
#endif /* !defined(OS_LISTXATTR_METHODDEF) */
#ifndef OS_MEMFD_CREATE_METHODDEF
#define OS_MEMFD_CREATE_METHODDEF
#endif /* !defined(OS_MEMFD_CREATE_METHODDEF) */
#ifndef OS_GET_HANDLE_INHERITABLE_METHODDEF
#define OS_GET_HANDLE_INHERITABLE_METHODDEF
#endif /* !defined(OS_GET_HANDLE_INHERITABLE_METHODDEF) */
@ -8576,4 +8635,4 @@ exit:
#ifndef OS__REMOVE_DLL_DIRECTORY_METHODDEF
#define OS__REMOVE_DLL_DIRECTORY_METHODDEF
#endif /* !defined(OS__REMOVE_DLL_DIRECTORY_METHODDEF) */
/*[clinic end generated code: output=5ee9420fb2e7aa2c input=a9049054013a1b77]*/
/*[clinic end generated code: output=855b81aafd05beed input=a9049054013a1b77]*/

View file

@ -389,6 +389,19 @@ extern char *ctermid_r(char *);
#define HAVE_STRUCT_STAT_ST_FSTYPE 1
#endif
/* memfd_create is either defined in sys/mman.h or sys/memfd.h
* linux/memfd.h defines additional flags
*/
#ifdef HAVE_SYS_MMAN_H
#include <sys/mman.h>
#endif
#ifdef HAVE_SYS_MEMFD_H
#include <sys/memfd.h>
#endif
#ifdef HAVE_LINUX_MEMFD_H
#include <linux/memfd.h>
#endif
#ifdef _Py_MEMORY_SANITIZER
# include <sanitizer/msan_interface.h>
#endif
@ -11897,6 +11910,31 @@ os_urandom_impl(PyObject *module, Py_ssize_t size)
return bytes;
}
#ifdef HAVE_MEMFD_CREATE
/*[clinic input]
os.memfd_create
name: FSConverter
flags: unsigned_int(bitwise=True, c_default="MFD_CLOEXEC") = MFD_CLOEXEC
[clinic start generated code]*/
static PyObject *
os_memfd_create_impl(PyObject *module, PyObject *name, unsigned int flags)
/*[clinic end generated code: output=6681ede983bdb9a6 input=a42cfc199bcd56e9]*/
{
int fd;
const char *bytes = PyBytes_AS_STRING(name);
Py_BEGIN_ALLOW_THREADS
fd = memfd_create(bytes, flags);
Py_END_ALLOW_THREADS
if (fd == -1) {
return PyErr_SetFromErrno(PyExc_OSError);
}
return PyLong_FromLong(fd);
}
#endif
/* Terminal size querying */
static PyTypeObject* TerminalSizeType;
@ -13554,6 +13592,7 @@ static PyMethodDef posix_methods[] = {
OS_SCANDIR_METHODDEF
OS_FSPATH_METHODDEF
OS_GETRANDOM_METHODDEF
OS_MEMFD_CREATE_METHODDEF
#ifdef MS_WINDOWS
OS__ADD_DLL_DIRECTORY_METHODDEF
OS__REMOVE_DLL_DIRECTORY_METHODDEF
@ -14003,6 +14042,27 @@ all_ins(PyObject *m)
if (PyModule_AddIntMacro(m, GRND_RANDOM)) return -1;
if (PyModule_AddIntMacro(m, GRND_NONBLOCK)) return -1;
#endif
#ifdef HAVE_MEMFD_CREATE
if (PyModule_AddIntMacro(m, MFD_CLOEXEC)) return -1;
if (PyModule_AddIntMacro(m, MFD_ALLOW_SEALING)) return -1;
#ifdef MFD_HUGETLB
if (PyModule_AddIntMacro(m, MFD_HUGETLB)) return -1;
if (PyModule_AddIntMacro(m, MFD_HUGE_SHIFT)) return -1;
if (PyModule_AddIntMacro(m, MFD_HUGE_MASK)) return -1;
if (PyModule_AddIntMacro(m, MFD_HUGE_64KB)) return -1;
if (PyModule_AddIntMacro(m, MFD_HUGE_512KB)) return -1;
if (PyModule_AddIntMacro(m, MFD_HUGE_1MB)) return -1;
if (PyModule_AddIntMacro(m, MFD_HUGE_2MB)) return -1;
if (PyModule_AddIntMacro(m, MFD_HUGE_8MB)) return -1;
if (PyModule_AddIntMacro(m, MFD_HUGE_16MB)) return -1;
if (PyModule_AddIntMacro(m, MFD_HUGE_32MB)) return -1;
if (PyModule_AddIntMacro(m, MFD_HUGE_256MB)) return -1;
if (PyModule_AddIntMacro(m, MFD_HUGE_512MB)) return -1;
if (PyModule_AddIntMacro(m, MFD_HUGE_1GB)) return -1;
if (PyModule_AddIntMacro(m, MFD_HUGE_2GB)) return -1;
if (PyModule_AddIntMacro(m, MFD_HUGE_16GB)) return -1;
#endif
#endif
#if defined(__APPLE__)
if (PyModule_AddIntConstant(m, "_COPYFILE_DATA", COPYFILE_DATA)) return -1;
@ -14119,6 +14179,10 @@ static const char * const have_functions[] = {
"HAVE_LUTIMES",
#endif
#ifdef HAVE_MEMFD_CREATE
"HAVE_MEMFD_CREATE",
#endif
#ifdef HAVE_MKDIRAT
"HAVE_MKDIRAT",
#endif

10
aclocal.m4 vendored
View file

@ -55,7 +55,7 @@ dnl
dnl See the "Since" comment for each macro you use to see what version
dnl of the macros you require.
m4_defun([PKG_PREREQ],
[m4_define([PKG_MACROS_VERSION], [0.29.2])
[m4_define([PKG_MACROS_VERSION], [0.29.1])
m4_if(m4_version_compare(PKG_MACROS_VERSION, [$1]), -1,
[m4_fatal([pkg.m4 version $1 or higher is required but ]PKG_MACROS_VERSION[ found])])
])dnl PKG_PREREQ
@ -156,7 +156,7 @@ AC_ARG_VAR([$1][_CFLAGS], [C compiler flags for $1, overriding pkg-config])dnl
AC_ARG_VAR([$1][_LIBS], [linker flags for $1, overriding pkg-config])dnl
pkg_failed=no
AC_MSG_CHECKING([for $2])
AC_MSG_CHECKING([for $1])
_PKG_CONFIG([$1][_CFLAGS], [cflags], [$2])
_PKG_CONFIG([$1][_LIBS], [libs], [$2])
@ -166,11 +166,11 @@ and $1[]_LIBS to avoid the need to call pkg-config.
See the pkg-config man page for more details.])
if test $pkg_failed = yes; then
AC_MSG_RESULT([no])
AC_MSG_RESULT([no])
_PKG_SHORT_ERRORS_SUPPORTED
if test $_pkg_short_errors_supported = yes; then
$1[]_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "$2" 2>&1`
else
else
$1[]_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "$2" 2>&1`
fi
# Put the nasty error message in config.log where it belongs
@ -187,7 +187,7 @@ installed software in a non-standard prefix.
_PKG_TEXT])[]dnl
])
elif test $pkg_failed = untried; then
AC_MSG_RESULT([no])
AC_MSG_RESULT([no])
m4_default([$4], [AC_MSG_FAILURE(
[The pkg-config script could not be found or is too old. Make sure it
is in your PATH or set the PKG_CONFIG environment variable to the full

49
configure vendored
View file

@ -785,6 +785,7 @@ infodir
docdir
oldincludedir
includedir
runstatedir
localstatedir
sharedstatedir
sysconfdir
@ -897,6 +898,7 @@ datadir='${datarootdir}'
sysconfdir='${prefix}/etc'
sharedstatedir='${prefix}/com'
localstatedir='${prefix}/var'
runstatedir='${localstatedir}/run'
includedir='${prefix}/include'
oldincludedir='/usr/include'
docdir='${datarootdir}/doc/${PACKAGE_TARNAME}'
@ -1149,6 +1151,15 @@ do
| -silent | --silent | --silen | --sile | --sil)
silent=yes ;;
-runstatedir | --runstatedir | --runstatedi | --runstated \
| --runstate | --runstat | --runsta | --runst | --runs \
| --run | --ru | --r)
ac_prev=runstatedir ;;
-runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \
| --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \
| --run=* | --ru=* | --r=*)
runstatedir=$ac_optarg ;;
-sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb)
ac_prev=sbindir ;;
-sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \
@ -1286,7 +1297,7 @@ fi
for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \
datadir sysconfdir sharedstatedir localstatedir includedir \
oldincludedir docdir infodir htmldir dvidir pdfdir psdir \
libdir localedir mandir
libdir localedir mandir runstatedir
do
eval ac_val=\$$ac_var
# Remove trailing slashes.
@ -1439,6 +1450,7 @@ Fine tuning of the installation directories:
--sysconfdir=DIR read-only single-machine data [PREFIX/etc]
--sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com]
--localstatedir=DIR modifiable single-machine data [PREFIX/var]
--runstatedir=DIR modifiable per-process data [LOCALSTATEDIR/run]
--libdir=DIR object code libraries [EPREFIX/lib]
--includedir=DIR C header files [PREFIX/include]
--oldincludedir=DIR C header files for non-gcc [/usr/include]
@ -7892,7 +7904,7 @@ sys/stat.h sys/syscall.h sys/sys_domain.h sys/termio.h sys/time.h \
sys/times.h sys/types.h sys/uio.h sys/un.h sys/utsname.h sys/wait.h pty.h \
libutil.h sys/resource.h netpacket/packet.h sysexits.h bluetooth.h \
linux/tipc.h linux/random.h spawn.h util.h alloca.h endian.h \
sys/endian.h sys/sysmacros.h
sys/endian.h sys/sysmacros.h linux/memfd.h sys/memfd.h sys/mman.h
do :
as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh`
ac_fn_c_check_header_mongrel "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default"
@ -11785,6 +11797,39 @@ $as_echo "no" >&6; }
fi
rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for memfd_create" >&5
$as_echo_n "checking for memfd_create... " >&6; }
cat confdefs.h - <<_ACEOF >conftest.$ac_ext
/* end confdefs.h. */
#ifdef HAVE_SYS_MMAN_H
#include <sys/mman.h>
#endif
#ifdef HAVE_SYS_MEMFD_H
#include <sys/memfd.h>
#endif
int
main ()
{
void *x=memfd_create
;
return 0;
}
_ACEOF
if ac_fn_c_try_compile "$LINENO"; then :
$as_echo "#define HAVE_MEMFD_CREATE 1" >>confdefs.h
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
$as_echo "yes" >&6; }
else
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
$as_echo "no" >&6; }
fi
rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
# On some systems (eg. FreeBSD 5), we would find a definition of the
# functions ctermid_r, setgroups in the library, but no prototype
# (e.g. because we use _XOPEN_SOURCE). See whether we can take their

View file

@ -2133,7 +2133,7 @@ sys/stat.h sys/syscall.h sys/sys_domain.h sys/termio.h sys/time.h \
sys/times.h sys/types.h sys/uio.h sys/un.h sys/utsname.h sys/wait.h pty.h \
libutil.h sys/resource.h netpacket/packet.h sysexits.h bluetooth.h \
linux/tipc.h linux/random.h spawn.h util.h alloca.h endian.h \
sys/endian.h sys/sysmacros.h)
sys/endian.h sys/sysmacros.h linux/memfd.h sys/memfd.h sys/mman.h)
AC_HEADER_DIRENT
AC_HEADER_MAJOR
@ -3626,6 +3626,20 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
[AC_MSG_RESULT(no)
])
AC_MSG_CHECKING(for memfd_create)
AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
#ifdef HAVE_SYS_MMAN_H
#include <sys/mman.h>
#endif
#ifdef HAVE_SYS_MEMFD_H
#include <sys/memfd.h>
#endif
]], [[void *x=memfd_create]])],
[AC_DEFINE(HAVE_MEMFD_CREATE, 1, Define if you have the 'memfd_create' function.)
AC_MSG_RESULT(yes)],
[AC_MSG_RESULT(no)
])
# On some systems (eg. FreeBSD 5), we would find a definition of the
# functions ctermid_r, setgroups in the library, but no prototype
# (e.g. because we use _XOPEN_SOURCE). See whether we can take their

View file

@ -625,6 +625,9 @@
/* Define to 1 if you have the <linux/can/raw.h> header file. */
#undef HAVE_LINUX_CAN_RAW_H
/* Define to 1 if you have the <linux/memfd.h> header file. */
#undef HAVE_LINUX_MEMFD_H
/* Define to 1 if you have the <linux/netlink.h> header file. */
#undef HAVE_LINUX_NETLINK_H
@ -667,15 +670,15 @@
/* Define to 1 if you have the `mbrtowc' function. */
#undef HAVE_MBRTOWC
/* Define if you have the 'memfd_create' function. */
#undef HAVE_MEMFD_CREATE
/* Define to 1 if you have the <memory.h> header file. */
#undef HAVE_MEMORY_H
/* Define to 1 if you have the `memrchr' function. */
#undef HAVE_MEMRCHR
/* Define to 1 if you have the `memset_s' function. */
#undef HAVE_MEMSET_S
/* Define to 1 if you have the `mkdirat' function. */
#undef HAVE_MKDIRAT
@ -1119,6 +1122,9 @@
/* Define to 1 if you have the <sys/lock.h> header file. */
#undef HAVE_SYS_LOCK_H
/* Define to 1 if you have the <sys/memfd.h> header file. */
#undef HAVE_SYS_MEMFD_H
/* Define to 1 if you have the <sys/mkdev.h> header file. */
#undef HAVE_SYS_MKDEV_H