diff --git a/Doc/library/os.rst b/Doc/library/os.rst index b53fd71e65b..107764ba4d5 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -707,6 +707,28 @@ as internal buffering of data. pass +.. function:: copy_file_range(src, dst, count, offset_src=None, offset_dst=None) + + Copy *count* bytes from file descriptor *src*, starting from offset + *offset_src*, to file descriptor *dst*, starting from offset *offset_dst*. + If *offset_src* is None, then *src* is read from the current position; + respectively for *offset_dst*. The files pointed by *src* and *dst* + must reside in the same filesystem, otherwise an :exc:`OSError` is + raised with :attr:`~OSError.errno` set to :data:`errno.EXDEV`. + + This copy is done without the additional cost of transferring data + from the kernel to user space and then back into the kernel. Additionally, + some filesystems could implement extra optimizations. The copy is done as if + both files are opened as binary. + + The return value is the amount of bytes copied. This could be less than the + amount requested. + + .. availability:: Linux kernel >= 4.5 or glibc >= 2.27. + + .. versionadded:: 3.8 + + .. function:: device_encoding(fd) Return a string describing the encoding of the device associated with *fd* diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index f17a19a7585..a8eae616205 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -231,6 +231,89 @@ def test_symlink_keywords(self): except (NotImplementedError, OSError): pass # No OS support or unprivileged user + @unittest.skipUnless(hasattr(os, 'copy_file_range'), 'test needs os.copy_file_range()') + def test_copy_file_range_invalid_values(self): + with self.assertRaises(ValueError): + os.copy_file_range(0, 1, -10) + + @unittest.skipUnless(hasattr(os, 'copy_file_range'), 'test needs os.copy_file_range()') + def test_copy_file_range(self): + TESTFN2 = support.TESTFN + ".3" + data = b'0123456789' + + create_file(support.TESTFN, data) + self.addCleanup(support.unlink, support.TESTFN) + + in_file = open(support.TESTFN, 'rb') + self.addCleanup(in_file.close) + in_fd = in_file.fileno() + + out_file = open(TESTFN2, 'w+b') + self.addCleanup(support.unlink, TESTFN2) + self.addCleanup(out_file.close) + out_fd = out_file.fileno() + + try: + i = os.copy_file_range(in_fd, out_fd, 5) + except OSError as e: + # Handle the case in which Python was compiled + # in a system with the syscall but without support + # in the kernel. + if e.errno != errno.ENOSYS: + raise + self.skipTest(e) + else: + # The number of copied bytes can be less than + # the number of bytes originally requested. + self.assertIn(i, range(0, 6)); + + with open(TESTFN2, 'rb') as in_file: + self.assertEqual(in_file.read(), data[:i]) + + @unittest.skipUnless(hasattr(os, 'copy_file_range'), 'test needs os.copy_file_range()') + def test_copy_file_range_offset(self): + TESTFN4 = support.TESTFN + ".4" + data = b'0123456789' + bytes_to_copy = 6 + in_skip = 3 + out_seek = 5 + + create_file(support.TESTFN, data) + self.addCleanup(support.unlink, support.TESTFN) + + in_file = open(support.TESTFN, 'rb') + self.addCleanup(in_file.close) + in_fd = in_file.fileno() + + out_file = open(TESTFN4, 'w+b') + self.addCleanup(support.unlink, TESTFN4) + self.addCleanup(out_file.close) + out_fd = out_file.fileno() + + try: + i = os.copy_file_range(in_fd, out_fd, bytes_to_copy, + offset_src=in_skip, + offset_dst=out_seek) + except OSError as e: + # Handle the case in which Python was compiled + # in a system with the syscall but without support + # in the kernel. + if e.errno != errno.ENOSYS: + raise + self.skipTest(e) + else: + # The number of copied bytes can be less than + # the number of bytes originally requested. + self.assertIn(i, range(0, bytes_to_copy+1)); + + with open(TESTFN4, 'rb') as in_file: + read = in_file.read() + # seeked bytes (5) are zero'ed + self.assertEqual(read[:out_seek], b'\x00'*out_seek) + # 012 are skipped (in_skip) + # 345678 are copied in the file (in_skip + bytes_to_copy) + self.assertEqual(read[out_seek:], + data[in_skip:in_skip+i]) # Test attributes on return values from os.*stat* family. class StatAttributeTests(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-05-30-23-43-03.bpo-26826.NkRzjb.rst b/Misc/NEWS.d/next/Core and Builtins/2018-05-30-23-43-03.bpo-26826.NkRzjb.rst new file mode 100644 index 00000000000..27d7f82a672 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-05-30-23-43-03.bpo-26826.NkRzjb.rst @@ -0,0 +1 @@ +Expose :func:`copy_file_range` as a low level API in the :mod:`os` module. diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 13f25460b4f..22cb94761de 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -5395,6 +5395,108 @@ exit: #endif /* (defined(HAVE_PWRITEV) || defined (HAVE_PWRITEV2)) */ +#if defined(HAVE_COPY_FILE_RANGE) + +PyDoc_STRVAR(os_copy_file_range__doc__, +"copy_file_range($module, /, src, dst, count, offset_src=None,\n" +" offset_dst=None)\n" +"--\n" +"\n" +"Copy count bytes from one file descriptor to another.\n" +"\n" +" src\n" +" Source file descriptor.\n" +" dst\n" +" Destination file descriptor.\n" +" count\n" +" Number of bytes to copy.\n" +" offset_src\n" +" Starting offset in src.\n" +" offset_dst\n" +" Starting offset in dst.\n" +"\n" +"If offset_src is None, then src is read from the current position;\n" +"respectively for offset_dst."); + +#define OS_COPY_FILE_RANGE_METHODDEF \ + {"copy_file_range", (PyCFunction)(void(*)(void))os_copy_file_range, METH_FASTCALL|METH_KEYWORDS, os_copy_file_range__doc__}, + +static PyObject * +os_copy_file_range_impl(PyObject *module, int src, int dst, Py_ssize_t count, + PyObject *offset_src, PyObject *offset_dst); + +static PyObject * +os_copy_file_range(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"src", "dst", "count", "offset_src", "offset_dst", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "copy_file_range", 0}; + PyObject *argsbuf[5]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; + int src; + int dst; + Py_ssize_t count; + PyObject *offset_src = Py_None; + PyObject *offset_dst = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 5, 0, argsbuf); + if (!args) { + goto exit; + } + if (PyFloat_Check(args[0])) { + PyErr_SetString(PyExc_TypeError, + "integer argument expected, got float" ); + goto exit; + } + src = _PyLong_AsInt(args[0]); + if (src == -1 && PyErr_Occurred()) { + goto exit; + } + if (PyFloat_Check(args[1])) { + PyErr_SetString(PyExc_TypeError, + "integer argument expected, got float" ); + goto exit; + } + dst = _PyLong_AsInt(args[1]); + if (dst == -1 && PyErr_Occurred()) { + goto exit; + } + if (PyFloat_Check(args[2])) { + PyErr_SetString(PyExc_TypeError, + "integer argument expected, got float" ); + goto exit; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = PyNumber_Index(args[2]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + count = ival; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[3]) { + offset_src = args[3]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + offset_dst = args[4]; +skip_optional_pos: + return_value = os_copy_file_range_impl(module, src, dst, count, offset_src, offset_dst); + +exit: + return return_value; +} + +#endif /* defined(HAVE_COPY_FILE_RANGE) */ + #if defined(HAVE_MKFIFO) PyDoc_STRVAR(os_mkfifo__doc__, @@ -8460,6 +8562,10 @@ exit: #define OS_PWRITEV_METHODDEF #endif /* !defined(OS_PWRITEV_METHODDEF) */ +#ifndef OS_COPY_FILE_RANGE_METHODDEF + #define OS_COPY_FILE_RANGE_METHODDEF +#endif /* !defined(OS_COPY_FILE_RANGE_METHODDEF) */ + #ifndef OS_MKFIFO_METHODDEF #define OS_MKFIFO_METHODDEF #endif /* !defined(OS_MKFIFO_METHODDEF) */ @@ -8635,4 +8741,4 @@ exit: #ifndef OS__REMOVE_DLL_DIRECTORY_METHODDEF #define OS__REMOVE_DLL_DIRECTORY_METHODDEF #endif /* !defined(OS__REMOVE_DLL_DIRECTORY_METHODDEF) */ -/*[clinic end generated code: output=855b81aafd05beed input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b3ae8afd275ea5cd input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 77a3700ab22..8f6cffffcdf 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -117,6 +117,10 @@ corresponding Unix manual entries for more information on calls."); #include #endif +#ifdef HAVE_COPY_FILE_RANGE +#include +#endif + #if !defined(CPU_ALLOC) && defined(HAVE_SCHED_SETAFFINITY) #undef HAVE_SCHED_SETAFFINITY #endif @@ -9455,8 +9459,74 @@ os_pwritev_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, } #endif /* HAVE_PWRITEV */ +#ifdef HAVE_COPY_FILE_RANGE +/*[clinic input] + +os.copy_file_range + src: int + Source file descriptor. + dst: int + Destination file descriptor. + count: Py_ssize_t + Number of bytes to copy. + offset_src: object = None + Starting offset in src. + offset_dst: object = None + Starting offset in dst. + +Copy count bytes from one file descriptor to another. + +If offset_src is None, then src is read from the current position; +respectively for offset_dst. +[clinic start generated code]*/ + +static PyObject * +os_copy_file_range_impl(PyObject *module, int src, int dst, Py_ssize_t count, + PyObject *offset_src, PyObject *offset_dst) +/*[clinic end generated code: output=1a91713a1d99fc7a input=42fdce72681b25a9]*/ +{ + off_t offset_src_val, offset_dst_val; + off_t *p_offset_src = NULL; + off_t *p_offset_dst = NULL; + Py_ssize_t ret; + int async_err = 0; + /* The flags argument is provided to allow + * for future extensions and currently must be to 0. */ + int flags = 0; + if (count < 0) { + PyErr_SetString(PyExc_ValueError, "negative value for 'count' not allowed"); + return NULL; + } + + if (offset_src != Py_None) { + if (!Py_off_t_converter(offset_src, &offset_src_val)) { + return NULL; + } + p_offset_src = &offset_src_val; + } + + if (offset_dst != Py_None) { + if (!Py_off_t_converter(offset_dst, &offset_dst_val)) { + return NULL; + } + p_offset_dst = &offset_dst_val; + } + + do { + Py_BEGIN_ALLOW_THREADS + ret = copy_file_range(src, p_offset_src, dst, p_offset_dst, count, flags); + Py_END_ALLOW_THREADS + } while (ret < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); + + if (ret < 0) { + return (!async_err) ? posix_error() : NULL; + } + + return PyLong_FromSsize_t(ret); +} +#endif /* HAVE_COPY_FILE_RANGE*/ #ifdef HAVE_MKFIFO /*[clinic input] @@ -13432,6 +13502,7 @@ static PyMethodDef posix_methods[] = { OS_POSIX_SPAWN_METHODDEF OS_POSIX_SPAWNP_METHODDEF OS_READLINK_METHODDEF + OS_COPY_FILE_RANGE_METHODDEF OS_RENAME_METHODDEF OS_REPLACE_METHODDEF OS_RMDIR_METHODDEF diff --git a/aclocal.m4 b/aclocal.m4 index 85f00dd5fac..3d6b1a375fd 100644 --- a/aclocal.m4 +++ b/aclocal.m4 @@ -12,9 +12,9 @@ # PARTICULAR PURPOSE. m4_ifndef([AC_CONFIG_MACRO_DIRS], [m4_defun([_AM_CONFIG_MACRO_DIRS], [])m4_defun([AC_CONFIG_MACRO_DIRS], [_AM_CONFIG_MACRO_DIRS($@)])]) -dnl pkg.m4 - Macros to locate and utilise pkg-config. -*- Autoconf -*- -dnl serial 11 (pkg-config-0.29.1) -dnl +# pkg.m4 - Macros to locate and utilise pkg-config. -*- Autoconf -*- +# serial 11 (pkg-config-0.29.1) + dnl Copyright © 2004 Scott James Remnant . dnl Copyright © 2012-2015 Dan Nicholson dnl @@ -288,5 +288,73 @@ AS_VAR_COPY([$1], [pkg_cv_][$1]) AS_VAR_IF([$1], [""], [$5], [$4])dnl ])dnl PKG_CHECK_VAR +dnl PKG_WITH_MODULES(VARIABLE-PREFIX, MODULES, +dnl [ACTION-IF-FOUND],[ACTION-IF-NOT-FOUND], +dnl [DESCRIPTION], [DEFAULT]) +dnl ------------------------------------------ +dnl +dnl Prepare a "--with-" configure option using the lowercase +dnl [VARIABLE-PREFIX] name, merging the behaviour of AC_ARG_WITH and +dnl PKG_CHECK_MODULES in a single macro. +AC_DEFUN([PKG_WITH_MODULES], +[ +m4_pushdef([with_arg], m4_tolower([$1])) + +m4_pushdef([description], + [m4_default([$5], [build with ]with_arg[ support])]) + +m4_pushdef([def_arg], [m4_default([$6], [auto])]) +m4_pushdef([def_action_if_found], [AS_TR_SH([with_]with_arg)=yes]) +m4_pushdef([def_action_if_not_found], [AS_TR_SH([with_]with_arg)=no]) + +m4_case(def_arg, + [yes],[m4_pushdef([with_without], [--without-]with_arg)], + [m4_pushdef([with_without],[--with-]with_arg)]) + +AC_ARG_WITH(with_arg, + AS_HELP_STRING(with_without, description[ @<:@default=]def_arg[@:>@]),, + [AS_TR_SH([with_]with_arg)=def_arg]) + +AS_CASE([$AS_TR_SH([with_]with_arg)], + [yes],[PKG_CHECK_MODULES([$1],[$2],$3,$4)], + [auto],[PKG_CHECK_MODULES([$1],[$2], + [m4_n([def_action_if_found]) $3], + [m4_n([def_action_if_not_found]) $4])]) + +m4_popdef([with_arg]) +m4_popdef([description]) +m4_popdef([def_arg]) + +])dnl PKG_WITH_MODULES + +dnl PKG_HAVE_WITH_MODULES(VARIABLE-PREFIX, MODULES, +dnl [DESCRIPTION], [DEFAULT]) +dnl ----------------------------------------------- +dnl +dnl Convenience macro to trigger AM_CONDITIONAL after PKG_WITH_MODULES +dnl check._[VARIABLE-PREFIX] is exported as make variable. +AC_DEFUN([PKG_HAVE_WITH_MODULES], +[ +PKG_WITH_MODULES([$1],[$2],,,[$3],[$4]) + +AM_CONDITIONAL([HAVE_][$1], + [test "$AS_TR_SH([with_]m4_tolower([$1]))" = "yes"]) +])dnl PKG_HAVE_WITH_MODULES + +dnl PKG_HAVE_DEFINE_WITH_MODULES(VARIABLE-PREFIX, MODULES, +dnl [DESCRIPTION], [DEFAULT]) +dnl ------------------------------------------------------ +dnl +dnl Convenience macro to run AM_CONDITIONAL and AC_DEFINE after +dnl PKG_WITH_MODULES check. HAVE_[VARIABLE-PREFIX] is exported as make +dnl and preprocessor variable. +AC_DEFUN([PKG_HAVE_DEFINE_WITH_MODULES], +[ +PKG_HAVE_WITH_MODULES([$1],[$2],[$3],[$4]) + +AS_IF([test "$AS_TR_SH([with_]m4_tolower([$1]))" = "yes"], + [AC_DEFINE([HAVE_][$1], 1, [Enable ]m4_tolower([$1])[ support])]) +])dnl PKG_HAVE_DEFINE_WITH_MODULES + m4_include([m4/ax_c_float_words_bigendian.m4]) m4_include([m4/ax_check_openssl.m4]) diff --git a/configure b/configure index cacf9fc4189..b606fc808c1 100755 --- a/configure +++ b/configure @@ -785,7 +785,6 @@ infodir docdir oldincludedir includedir -runstatedir localstatedir sharedstatedir sysconfdir @@ -898,7 +897,6 @@ datadir='${datarootdir}' sysconfdir='${prefix}/etc' sharedstatedir='${prefix}/com' localstatedir='${prefix}/var' -runstatedir='${localstatedir}/run' includedir='${prefix}/include' oldincludedir='/usr/include' docdir='${datarootdir}/doc/${PACKAGE_TARNAME}' @@ -1151,15 +1149,6 @@ do | -silent | --silent | --silen | --sile | --sil) silent=yes ;; - -runstatedir | --runstatedir | --runstatedi | --runstated \ - | --runstate | --runstat | --runsta | --runst | --runs \ - | --run | --ru | --r) - ac_prev=runstatedir ;; - -runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \ - | --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \ - | --run=* | --ru=* | --r=*) - runstatedir=$ac_optarg ;; - -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb) ac_prev=sbindir ;; -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \ @@ -1297,7 +1286,7 @@ fi for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \ datadir sysconfdir sharedstatedir localstatedir includedir \ oldincludedir docdir infodir htmldir dvidir pdfdir psdir \ - libdir localedir mandir runstatedir + libdir localedir mandir do eval ac_val=\$$ac_var # Remove trailing slashes. @@ -1450,7 +1439,6 @@ Fine tuning of the installation directories: --sysconfdir=DIR read-only single-machine data [PREFIX/etc] --sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com] --localstatedir=DIR modifiable single-machine data [PREFIX/var] - --runstatedir=DIR modifiable per-process data [LOCALSTATEDIR/run] --libdir=DIR object code libraries [EPREFIX/lib] --includedir=DIR C header files [PREFIX/include] --oldincludedir=DIR C header files for non-gcc [/usr/include] @@ -11476,7 +11464,8 @@ fi # checks for library functions for ac_func in alarm accept4 setitimer getitimer bind_textdomain_codeset chown \ - clock confstr ctermid dup3 execv explicit_bzero explicit_memset faccessat fchmod fchmodat fchown fchownat \ + clock confstr copy_file_range ctermid dup3 execv explicit_bzero explicit_memset \ + faccessat fchmod fchmodat fchown fchownat \ fexecve fdopendir fork fpathconf fstatat ftime ftruncate futimesat \ futimens futimes gai_strerror getentropy \ getgrgid_r getgrnam_r \ diff --git a/configure.ac b/configure.ac index 1190b37e9f9..3d589ac2589 100644 --- a/configure.ac +++ b/configure.ac @@ -3520,7 +3520,8 @@ fi # checks for library functions AC_CHECK_FUNCS(alarm accept4 setitimer getitimer bind_textdomain_codeset chown \ - clock confstr ctermid dup3 execv explicit_bzero explicit_memset faccessat fchmod fchmodat fchown fchownat \ + clock confstr copy_file_range ctermid dup3 execv explicit_bzero explicit_memset \ + faccessat fchmod fchmodat fchown fchownat \ fexecve fdopendir fork fpathconf fstatat ftime ftruncate futimesat \ futimens futimes gai_strerror getentropy \ getgrgid_r getgrnam_r \ diff --git a/pyconfig.h.in b/pyconfig.h.in index b9bb3ffa6f6..20cc901e473 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -148,6 +148,9 @@ /* Define to 1 if you have the `copysign' function. */ #undef HAVE_COPYSIGN +/* Define to 1 if you have the `copy_file_range' function. */ +#undef HAVE_COPY_FILE_RANGE + /* Define to 1 if you have the header file. */ #undef HAVE_CRYPT_H