From ea21128d44dc31bc1cd2f9c285dbcf8e3ad1fe18 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 19 Jan 2018 01:53:41 +0000 Subject: [PATCH 01/14] Expose preadv2 in os module --- Lib/test/test_posix.py | 13 +++++++ Modules/clinic/posixmodule.c.h | 50 +++++++++++++++++++++++++- Modules/posixmodule.c | 66 ++++++++++++++++++++++++++++++++++ 3 files changed, 128 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 44b8d6a7ad0796..b3c07ff863d32b 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -272,6 +272,19 @@ def test_pread(self): finally: os.close(fd) + @unittest.skipUnless(hasattr(posix, 'pread'), "test needs posix.pread()") + def test_preadv2(self): + fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT) + try: + os.write(fd, b'test1tt2t3') + os.lseek(fd, 0, os.SEEK_SET) + buf = [bytearray(i) for i in [5, 3, 2]] + self.assertEqual(posix.preadv2(fd, buf, 0, os.RWF_SYNC), 10) + self.assertEqual([b'test1', b'tt2', b't3'], [bytes(i) for i in buf]) + finally: + os.close(fd) + + @unittest.skipUnless(hasattr(posix, 'pwrite'), "test needs posix.pwrite()") def test_pwrite(self): fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT) diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 6f4c028e7efef5..dd3b9b7faa6835 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -3705,6 +3705,50 @@ os_pread(PyObject *module, PyObject *const *args, Py_ssize_t nargs) #endif /* defined(HAVE_PREAD) */ +#if defined(HAVE_PREAD) + +PyDoc_STRVAR(os_preadv2__doc__, +"preadv2($module, fd, buffers, offset, flags, /)\n" +"--\n" +"\n" +"Read a number of bytes from a file descriptor starting at a particular offset.\n" +"\n" +"Read length bytes from file descriptor fd, starting at offset bytes from\n" +"the beginning of the file. The file offset remains unchanged."); + +#define OS_PREADV2_METHODDEF \ + {"preadv2", (PyCFunction)os_preadv2, METH_FASTCALL, os_preadv2__doc__}, + +static Py_ssize_t +os_preadv2_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, + int flags); + +static PyObject * +os_preadv2(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int fd; + PyObject *buffers; + Py_off_t offset; + int flags; + Py_ssize_t _return_value; + + if (!_PyArg_ParseStack(args, nargs, "iOO&i:preadv2", + &fd, &buffers, Py_off_t_converter, &offset, &flags)) { + goto exit; + } + _return_value = os_preadv2_impl(module, fd, buffers, offset, flags); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromSsize_t(_return_value); + +exit: + return return_value; +} + +#endif /* defined(HAVE_PREAD) */ + PyDoc_STRVAR(os_write__doc__, "write($module, fd, data, /)\n" "--\n" @@ -6239,6 +6283,10 @@ os_getrandom(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject #define OS_PREAD_METHODDEF #endif /* !defined(OS_PREAD_METHODDEF) */ +#ifndef OS_PREADV2_METHODDEF + #define OS_PREADV2_METHODDEF +#endif /* !defined(OS_PREADV2_METHODDEF) */ + #ifndef OS_PIPE_METHODDEF #define OS_PIPE_METHODDEF #endif /* !defined(OS_PIPE_METHODDEF) */ @@ -6410,4 +6458,4 @@ os_getrandom(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject #ifndef OS_GETRANDOM_METHODDEF #define OS_GETRANDOM_METHODDEF #endif /* !defined(OS_GETRANDOM_METHODDEF) */ -/*[clinic end generated code: output=6345053cd5992caf input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2b1fdec599a15ea9 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index b0e48dabbd55cf..264e1362183f75 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -8163,6 +8163,64 @@ os_pread_impl(PyObject *module, int fd, int length, Py_off_t offset) } #endif /* HAVE_PREAD */ +#ifdef HAVE_PREAD +/*[clinic input] +os.preadv2 -> Py_ssize_t + + fd: int + buffers: object + offset: Py_off_t + flags: int + / + +Read a number of bytes from a file descriptor starting at a particular offset. + +Read length bytes from file descriptor fd, starting at offset bytes from +the beginning of the file. The file offset remains unchanged. +[clinic start generated code]*/ + +static Py_ssize_t +os_preadv2_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, + int flags) +/*[clinic end generated code: output=fd040c20cf504631 input=f4a633d90005eb65]*/ +{ + Py_ssize_t cnt, n; + int async_err = 0; + struct iovec *iov; + Py_buffer *buf; + + if (!PySequence_Check(buffers)) { + PyErr_SetString(PyExc_TypeError, + "preadv2() arg 2 must be a sequence"); + return -1; + } + + cnt = PySequence_Size(buffers); + if (cnt < 0) + return -1; + + if (iov_setup(&iov, &buf, buffers, cnt, PyBUF_WRITABLE) < 0) + return -1; + + do { + Py_BEGIN_ALLOW_THREADS + n = preadv2(fd, iov, cnt, offset, flags); + Py_END_ALLOW_THREADS + } while (n < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); + + iov_cleanup(iov, buf, cnt); + if (n < 0) { + if (!async_err) + posix_error(); + return -1; + } + + return n; +} + + +#endif /* HAVE PREADV2 */ + /*[clinic input] os.write -> Py_ssize_t @@ -12506,6 +12564,7 @@ static PyMethodDef posix_methods[] = { OS_READ_METHODDEF OS_READV_METHODDEF OS_PREAD_METHODDEF + OS_PREADV2_METHODDEF OS_WRITE_METHODDEF OS_WRITEV_METHODDEF OS_PWRITE_METHODDEF @@ -12953,6 +13012,13 @@ all_ins(PyObject *m) if (PyModule_AddIntMacro(m, F_TEST)) return -1; #endif +#ifdef HAVE_READV + if (PyModule_AddIntConstant(m, "RWF_DSYNC", RWF_DSYNC)) return -1; + if (PyModule_AddIntConstant(m, "RWF_HIPRI", RWF_HIPRI)) return -1; + if (PyModule_AddIntConstant(m, "RWF_SYNC", RWF_SYNC)) return -1; + if (PyModule_AddIntConstant(m, "RWF_NOWAIT", RWF_NOWAIT)) return -1; +#endif + #ifdef HAVE_SPAWNV if (PyModule_AddIntConstant(m, "P_WAIT", _P_WAIT)) return -1; if (PyModule_AddIntConstant(m, "P_NOWAIT", _P_NOWAIT)) return -1; From 4570976db32e695f94ab0ccd07d187eb870b7084 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 19 Jan 2018 01:54:32 +0000 Subject: [PATCH 02/14] Add News entry --- .../Core and Builtins/2018-01-19-01-54-22.bpo-31368.kzKqUR.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2018-01-19-01-54-22.bpo-31368.kzKqUR.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-01-19-01-54-22.bpo-31368.kzKqUR.rst b/Misc/NEWS.d/next/Core and Builtins/2018-01-19-01-54-22.bpo-31368.kzKqUR.rst new file mode 100644 index 00000000000000..a614ab64628423 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-01-19-01-54-22.bpo-31368.kzKqUR.rst @@ -0,0 +1 @@ +Expose preadv2 system call in the os module. Patch by Pablo Galindo From 21a3004ea437ae8f036b866460a43d20e475e460 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 19 Jan 2018 02:00:11 +0000 Subject: [PATCH 03/14] Add correct guards for the presence of preadv2 --- Lib/test/test_posix.py | 2 +- Modules/clinic/posixmodule.c.h | 6 +++--- Modules/posixmodule.c | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index b3c07ff863d32b..947de6e3efd1db 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -272,7 +272,7 @@ def test_pread(self): finally: os.close(fd) - @unittest.skipUnless(hasattr(posix, 'pread'), "test needs posix.pread()") + @unittest.skipUnless(hasattr(posix, 'preadv2'), "test needs posix.preadv2()") def test_preadv2(self): fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT) try: diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index dd3b9b7faa6835..c609dba89beaa8 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -3705,7 +3705,7 @@ os_pread(PyObject *module, PyObject *const *args, Py_ssize_t nargs) #endif /* defined(HAVE_PREAD) */ -#if defined(HAVE_PREAD) +#if defined(HAVE_PREADV2) PyDoc_STRVAR(os_preadv2__doc__, "preadv2($module, fd, buffers, offset, flags, /)\n" @@ -3747,7 +3747,7 @@ os_preadv2(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } -#endif /* defined(HAVE_PREAD) */ +#endif /* defined(HAVE_PREADV2) */ PyDoc_STRVAR(os_write__doc__, "write($module, fd, data, /)\n" @@ -6458,4 +6458,4 @@ os_getrandom(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject #ifndef OS_GETRANDOM_METHODDEF #define OS_GETRANDOM_METHODDEF #endif /* !defined(OS_GETRANDOM_METHODDEF) */ -/*[clinic end generated code: output=2b1fdec599a15ea9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=efbe9448c1bba1a2 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 264e1362183f75..ea0841cdacb16e 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -8163,7 +8163,7 @@ os_pread_impl(PyObject *module, int fd, int length, Py_off_t offset) } #endif /* HAVE_PREAD */ -#ifdef HAVE_PREAD +#ifdef HAVE_PREADV2 /*[clinic input] os.preadv2 -> Py_ssize_t @@ -13012,7 +13012,7 @@ all_ins(PyObject *m) if (PyModule_AddIntMacro(m, F_TEST)) return -1; #endif -#ifdef HAVE_READV +#ifdef HAVE_PREADV2 if (PyModule_AddIntConstant(m, "RWF_DSYNC", RWF_DSYNC)) return -1; if (PyModule_AddIntConstant(m, "RWF_HIPRI", RWF_HIPRI)) return -1; if (PyModule_AddIntConstant(m, "RWF_SYNC", RWF_SYNC)) return -1; From 8a2a2a31db2af445f0cfdb987aad42a03f3d1ed1 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 19 Jan 2018 21:07:35 +0000 Subject: [PATCH 04/14] Implement check for individual constants --- Modules/posixmodule.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index ea0841cdacb16e..3033accac4bdb8 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -13012,10 +13012,16 @@ all_ins(PyObject *m) if (PyModule_AddIntMacro(m, F_TEST)) return -1; #endif -#ifdef HAVE_PREADV2 +#ifdef RWF_DSYNC if (PyModule_AddIntConstant(m, "RWF_DSYNC", RWF_DSYNC)) return -1; +#endif +#ifdef RWF_HIPRI if (PyModule_AddIntConstant(m, "RWF_HIPRI", RWF_HIPRI)) return -1; +#endif +#ifdef RWF_SYNC if (PyModule_AddIntConstant(m, "RWF_SYNC", RWF_SYNC)) return -1; +#endif +#ifdef RWF_NOWAIT if (PyModule_AddIntConstant(m, "RWF_NOWAIT", RWF_NOWAIT)) return -1; #endif From c0c53dc91daabaa8d722607dd49c1b713f40ce64 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 19 Jan 2018 22:06:14 +0000 Subject: [PATCH 05/14] Implemented preadv and join implementation to preadv2 --- Lib/test/test_posix.py | 17 ++++++-- Modules/clinic/posixmodule.c.h | 32 +++++++------- Modules/posixmodule.c | 31 ++++++++----- aclocal.m4 | 80 +++------------------------------- configure | 2 +- configure.ac | 2 +- pyconfig.h.in | 6 +++ 7 files changed, 64 insertions(+), 106 deletions(-) diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 947de6e3efd1db..0e86cda7e991f2 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -272,18 +272,29 @@ def test_pread(self): finally: os.close(fd) - @unittest.skipUnless(hasattr(posix, 'preadv2'), "test needs posix.preadv2()") - def test_preadv2(self): + @unittest.skipUnless(hasattr(posix, 'preadv'), "test needs posix.preadv()") + def test_preadv(self): fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT) try: os.write(fd, b'test1tt2t3') os.lseek(fd, 0, os.SEEK_SET) buf = [bytearray(i) for i in [5, 3, 2]] - self.assertEqual(posix.preadv2(fd, buf, 0, os.RWF_SYNC), 10) + self.assertEqual(posix.preadv(fd, buf, 0), 10) self.assertEqual([b'test1', b'tt2', b't3'], [bytes(i) for i in buf]) finally: os.close(fd) + @unittest.skipUnless(hasattr(posix, 'RWF_SYNC'), "test needs posix.preadv2()") + def test_preadv2(self): + fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT) + try: + os.write(fd, b'test1tt2t3') + os.lseek(fd, 0, os.SEEK_SET) + buf = [bytearray(i) for i in [5, 3, 2]] + self.assertEqual(posix.preadv(fd, buf, 0, os.RWF_SYNC), 10) + self.assertEqual([b'test1', b'tt2', b't3'], [bytes(i) for i in buf]) + finally: + os.close(fd) @unittest.skipUnless(hasattr(posix, 'pwrite'), "test needs posix.pwrite()") def test_pwrite(self): diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index c609dba89beaa8..c2b7f327346b9c 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -3705,10 +3705,10 @@ os_pread(PyObject *module, PyObject *const *args, Py_ssize_t nargs) #endif /* defined(HAVE_PREAD) */ -#if defined(HAVE_PREADV2) +#if (defined(HAVE_PREADV) || defined (HAVE_PREADV2)) -PyDoc_STRVAR(os_preadv2__doc__, -"preadv2($module, fd, buffers, offset, flags, /)\n" +PyDoc_STRVAR(os_preadv__doc__, +"preadv($module, fd, buffers, offset, flags=0, /)\n" "--\n" "\n" "Read a number of bytes from a file descriptor starting at a particular offset.\n" @@ -3716,28 +3716,28 @@ PyDoc_STRVAR(os_preadv2__doc__, "Read length bytes from file descriptor fd, starting at offset bytes from\n" "the beginning of the file. The file offset remains unchanged."); -#define OS_PREADV2_METHODDEF \ - {"preadv2", (PyCFunction)os_preadv2, METH_FASTCALL, os_preadv2__doc__}, +#define OS_PREADV_METHODDEF \ + {"preadv", (PyCFunction)os_preadv, METH_FASTCALL, os_preadv__doc__}, static Py_ssize_t -os_preadv2_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, - int flags); +os_preadv_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, + int flags); static PyObject * -os_preadv2(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +os_preadv(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; int fd; PyObject *buffers; Py_off_t offset; - int flags; + int flags = 0; Py_ssize_t _return_value; - if (!_PyArg_ParseStack(args, nargs, "iOO&i:preadv2", + if (!_PyArg_ParseStack(args, nargs, "iOO&|i:preadv", &fd, &buffers, Py_off_t_converter, &offset, &flags)) { goto exit; } - _return_value = os_preadv2_impl(module, fd, buffers, offset, flags); + _return_value = os_preadv_impl(module, fd, buffers, offset, flags); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; } @@ -3747,7 +3747,7 @@ os_preadv2(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } -#endif /* defined(HAVE_PREADV2) */ +#endif /* (defined(HAVE_PREADV) || defined (HAVE_PREADV2)) */ PyDoc_STRVAR(os_write__doc__, "write($module, fd, data, /)\n" @@ -6283,9 +6283,9 @@ os_getrandom(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject #define OS_PREAD_METHODDEF #endif /* !defined(OS_PREAD_METHODDEF) */ -#ifndef OS_PREADV2_METHODDEF - #define OS_PREADV2_METHODDEF -#endif /* !defined(OS_PREADV2_METHODDEF) */ +#ifndef OS_PREADV_METHODDEF + #define OS_PREADV_METHODDEF +#endif /* !defined(OS_PREADV_METHODDEF) */ #ifndef OS_PIPE_METHODDEF #define OS_PIPE_METHODDEF @@ -6458,4 +6458,4 @@ os_getrandom(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject #ifndef OS_GETRANDOM_METHODDEF #define OS_GETRANDOM_METHODDEF #endif /* !defined(OS_GETRANDOM_METHODDEF) */ -/*[clinic end generated code: output=efbe9448c1bba1a2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=52cbdec1822e34f9 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 3033accac4bdb8..7a514275c7fc70 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -8163,14 +8163,14 @@ os_pread_impl(PyObject *module, int fd, int length, Py_off_t offset) } #endif /* HAVE_PREAD */ -#ifdef HAVE_PREADV2 +#if defined(HAVE_PREADV) || defined (HAVE_PREADV2) /*[clinic input] -os.preadv2 -> Py_ssize_t +os.preadv -> Py_ssize_t fd: int buffers: object offset: Py_off_t - flags: int + flags: int = 0 / Read a number of bytes from a file descriptor starting at a particular offset. @@ -8180,9 +8180,9 @@ the beginning of the file. The file offset remains unchanged. [clinic start generated code]*/ static Py_ssize_t -os_preadv2_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, - int flags) -/*[clinic end generated code: output=fd040c20cf504631 input=f4a633d90005eb65]*/ +os_preadv_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, + int flags) +/*[clinic end generated code: output=26fc9c6e58e7ada5 input=5b07b7e2d1825627]*/ { Py_ssize_t cnt, n; int async_err = 0; @@ -8201,12 +8201,23 @@ os_preadv2_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, if (iov_setup(&iov, &buf, buffers, cnt, PyBUF_WRITABLE) < 0) return -1; - + #ifdef HAVE_PREADV2 do { Py_BEGIN_ALLOW_THREADS n = preadv2(fd, iov, cnt, offset, flags); Py_END_ALLOW_THREADS } while (n < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); + #else + if(flags != 0){ + PyErr_SetString(PyExc_OSError, "preadv2() not available in this system"); + return -1; + } + do { + Py_BEGIN_ALLOW_THREADS + n = preadv(fd, iov, cnt, offset); + Py_END_ALLOW_THREADS + } while (n < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); + #endif iov_cleanup(iov, buf, cnt); if (n < 0) { @@ -8217,9 +8228,7 @@ os_preadv2_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, return n; } - - -#endif /* HAVE PREADV2 */ +#endif /* HAVE_PREADV2 */ /*[clinic input] @@ -12564,7 +12573,7 @@ static PyMethodDef posix_methods[] = { OS_READ_METHODDEF OS_READV_METHODDEF OS_PREAD_METHODDEF - OS_PREADV2_METHODDEF + OS_PREADV_METHODDEF OS_WRITE_METHODDEF OS_WRITEV_METHODDEF OS_PWRITE_METHODDEF diff --git a/aclocal.m4 b/aclocal.m4 index 09694962914b79..8ed232fb515f90 100644 --- a/aclocal.m4 +++ b/aclocal.m4 @@ -13,7 +13,7 @@ m4_ifndef([AC_CONFIG_MACRO_DIRS], [m4_defun([_AM_CONFIG_MACRO_DIRS], [])m4_defun([AC_CONFIG_MACRO_DIRS], [_AM_CONFIG_MACRO_DIRS($@)])]) # pkg.m4 - Macros to locate and utilise pkg-config. -*- Autoconf -*- -# serial 11 (pkg-config-0.29.1) +# serial 12 (pkg-config-0.29.2) dnl Copyright © 2004 Scott James Remnant . dnl Copyright © 2012-2015 Dan Nicholson @@ -55,7 +55,7 @@ dnl dnl See the "Since" comment for each macro you use to see what version dnl of the macros you require. m4_defun([PKG_PREREQ], -[m4_define([PKG_MACROS_VERSION], [0.29.1]) +[m4_define([PKG_MACROS_VERSION], [0.29.2]) m4_if(m4_version_compare(PKG_MACROS_VERSION, [$1]), -1, [m4_fatal([pkg.m4 version $1 or higher is required but ]PKG_MACROS_VERSION[ found])]) ])dnl PKG_PREREQ @@ -156,7 +156,7 @@ AC_ARG_VAR([$1][_CFLAGS], [C compiler flags for $1, overriding pkg-config])dnl AC_ARG_VAR([$1][_LIBS], [linker flags for $1, overriding pkg-config])dnl pkg_failed=no -AC_MSG_CHECKING([for $1]) +AC_MSG_CHECKING([for $2]) _PKG_CONFIG([$1][_CFLAGS], [cflags], [$2]) _PKG_CONFIG([$1][_LIBS], [libs], [$2]) @@ -166,11 +166,11 @@ and $1[]_LIBS to avoid the need to call pkg-config. See the pkg-config man page for more details.]) if test $pkg_failed = yes; then - AC_MSG_RESULT([no]) + AC_MSG_RESULT([no]) _PKG_SHORT_ERRORS_SUPPORTED if test $_pkg_short_errors_supported = yes; then $1[]_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "$2" 2>&1` - else + else $1[]_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "$2" 2>&1` fi # Put the nasty error message in config.log where it belongs @@ -187,7 +187,7 @@ installed software in a non-standard prefix. _PKG_TEXT])[]dnl ]) elif test $pkg_failed = untried; then - AC_MSG_RESULT([no]) + AC_MSG_RESULT([no]) m4_default([$4], [AC_MSG_FAILURE( [The pkg-config script could not be found or is too old. Make sure it is in your PATH or set the PKG_CONFIG environment variable to the full @@ -288,72 +288,4 @@ AS_VAR_COPY([$1], [pkg_cv_][$1]) AS_VAR_IF([$1], [""], [$5], [$4])dnl ])dnl PKG_CHECK_VAR -dnl PKG_WITH_MODULES(VARIABLE-PREFIX, MODULES, -dnl [ACTION-IF-FOUND],[ACTION-IF-NOT-FOUND], -dnl [DESCRIPTION], [DEFAULT]) -dnl ------------------------------------------ -dnl -dnl Prepare a "--with-" configure option using the lowercase -dnl [VARIABLE-PREFIX] name, merging the behaviour of AC_ARG_WITH and -dnl PKG_CHECK_MODULES in a single macro. -AC_DEFUN([PKG_WITH_MODULES], -[ -m4_pushdef([with_arg], m4_tolower([$1])) - -m4_pushdef([description], - [m4_default([$5], [build with ]with_arg[ support])]) - -m4_pushdef([def_arg], [m4_default([$6], [auto])]) -m4_pushdef([def_action_if_found], [AS_TR_SH([with_]with_arg)=yes]) -m4_pushdef([def_action_if_not_found], [AS_TR_SH([with_]with_arg)=no]) - -m4_case(def_arg, - [yes],[m4_pushdef([with_without], [--without-]with_arg)], - [m4_pushdef([with_without],[--with-]with_arg)]) - -AC_ARG_WITH(with_arg, - AS_HELP_STRING(with_without, description[ @<:@default=]def_arg[@:>@]),, - [AS_TR_SH([with_]with_arg)=def_arg]) - -AS_CASE([$AS_TR_SH([with_]with_arg)], - [yes],[PKG_CHECK_MODULES([$1],[$2],$3,$4)], - [auto],[PKG_CHECK_MODULES([$1],[$2], - [m4_n([def_action_if_found]) $3], - [m4_n([def_action_if_not_found]) $4])]) - -m4_popdef([with_arg]) -m4_popdef([description]) -m4_popdef([def_arg]) - -])dnl PKG_WITH_MODULES - -dnl PKG_HAVE_WITH_MODULES(VARIABLE-PREFIX, MODULES, -dnl [DESCRIPTION], [DEFAULT]) -dnl ----------------------------------------------- -dnl -dnl Convenience macro to trigger AM_CONDITIONAL after PKG_WITH_MODULES -dnl check._[VARIABLE-PREFIX] is exported as make variable. -AC_DEFUN([PKG_HAVE_WITH_MODULES], -[ -PKG_WITH_MODULES([$1],[$2],,,[$3],[$4]) - -AM_CONDITIONAL([HAVE_][$1], - [test "$AS_TR_SH([with_]m4_tolower([$1]))" = "yes"]) -])dnl PKG_HAVE_WITH_MODULES - -dnl PKG_HAVE_DEFINE_WITH_MODULES(VARIABLE-PREFIX, MODULES, -dnl [DESCRIPTION], [DEFAULT]) -dnl ------------------------------------------------------ -dnl -dnl Convenience macro to run AM_CONDITIONAL and AC_DEFINE after -dnl PKG_WITH_MODULES check. HAVE_[VARIABLE-PREFIX] is exported as make -dnl and preprocessor variable. -AC_DEFUN([PKG_HAVE_DEFINE_WITH_MODULES], -[ -PKG_HAVE_WITH_MODULES([$1],[$2],[$3],[$4]) - -AS_IF([test "$AS_TR_SH([with_]m4_tolower([$1]))" = "yes"], - [AC_DEFINE([HAVE_][$1], 1, [Enable ]m4_tolower([$1])[ support])]) -])dnl PKG_HAVE_DEFINE_WITH_MODULES - m4_include([m4/ax_check_openssl.m4]) diff --git a/configure b/configure index 420c07639b5cc4..ca1557ef118beb 100755 --- a/configure +++ b/configure @@ -11201,7 +11201,7 @@ for ac_func in alarm accept4 setitimer getitimer bind_textdomain_codeset chown \ initgroups kill killpg lchmod lchown lockf linkat lstat lutimes mmap \ memrchr mbrtowc mkdirat mkfifo \ mkfifoat mknod mknodat mktime mremap nice openat pathconf pause pipe2 plock poll \ - posix_fallocate posix_fadvise pread \ + posix_fallocate posix_fadvise pread preadv preadv2 \ pthread_init pthread_kill putenv pwrite readlink readlinkat readv realpath renameat \ sem_open sem_timedwait sem_getvalue sem_unlink sendfile setegid seteuid \ setgid sethostname \ diff --git a/configure.ac b/configure.ac index 39e2e8e769e973..df92ea8dd33c30 100644 --- a/configure.ac +++ b/configure.ac @@ -3435,7 +3435,7 @@ AC_CHECK_FUNCS(alarm accept4 setitimer getitimer bind_textdomain_codeset chown \ initgroups kill killpg lchmod lchown lockf linkat lstat lutimes mmap \ memrchr mbrtowc mkdirat mkfifo \ mkfifoat mknod mknodat mktime mremap nice openat pathconf pause pipe2 plock poll \ - posix_fallocate posix_fadvise pread \ + posix_fallocate posix_fadvise pread preadv preadv2 \ pthread_init pthread_kill putenv pwrite readlink readlinkat readv realpath renameat \ sem_open sem_timedwait sem_getvalue sem_unlink sendfile setegid seteuid \ setgid sethostname \ diff --git a/pyconfig.h.in b/pyconfig.h.in index dd7c62bad1bacb..d98d6e7cb2628a 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -707,6 +707,12 @@ /* Define to 1 if you have the `pread' function. */ #undef HAVE_PREAD +/* Define to 1 if you have the `preadv' function. */ +#undef HAVE_PREADV + +/* Define to 1 if you have the `preadv2' function. */ +#undef HAVE_PREADV2 + /* Define if you have the 'prlimit' functions. */ #undef HAVE_PRLIMIT From 4ddbca92005c9a3876cc54685083a4363fb2b020 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 20 Jan 2018 16:32:34 +0000 Subject: [PATCH 06/14] Clean iov on error --- Modules/posixmodule.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 7a514275c7fc70..d146a00c98b466 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -8201,15 +8201,16 @@ os_preadv_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, if (iov_setup(&iov, &buf, buffers, cnt, PyBUF_WRITABLE) < 0) return -1; - #ifdef HAVE_PREADV2 +#ifdef HAVE_PREADV2 do { Py_BEGIN_ALLOW_THREADS n = preadv2(fd, iov, cnt, offset, flags); Py_END_ALLOW_THREADS } while (n < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); - #else +#else if(flags != 0){ PyErr_SetString(PyExc_OSError, "preadv2() not available in this system"); + iov_cleanup(iov, buf, cnt); return -1; } do { @@ -8217,7 +8218,7 @@ os_preadv_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, n = preadv(fd, iov, cnt, offset); Py_END_ALLOW_THREADS } while (n < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); - #endif +#endif iov_cleanup(iov, buf, cnt); if (n < 0) { @@ -8228,7 +8229,7 @@ os_preadv_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, return n; } -#endif /* HAVE_PREADV2 */ +#endif /* HAVE_PREADV */ /*[clinic input] From cac62acc1dd6d5f994ad420f8602ef3cbc52a7d6 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Mon, 22 Jan 2018 20:30:20 +0000 Subject: [PATCH 07/14] Add documentation in the os module --- Doc/library/os.rst | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index ee08853087f71e..49221743223644 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1195,6 +1195,32 @@ or `the MSDN `_ on Windo .. versionadded:: 3.3 +.. function:: preadv(fd, buffers, offset, flags = None) + + Combines the functionality of :func:`os.readv` and :func:`os.pread`. It + performs the same task as :func:`os.readv`, but adds a fourth argument, + offset, which specifies the file offset at which the input operation is + to be performed. + + If the *flags* argument is provided it will invoke the *preadv2* system + call, which modifies the behavior on a per-call basis based on the value + of the *flags* argument. If this argument is ommited, the *preadv* system + call will be called instead. + + The flags argument contains a bitwise OR of zero or more of the following + flags: + +.. data:: RWF_HIPRI + RWF_NOWAIT + + The above constants are available on Linux Kernel 4.11 and 4.6 (or newer) + respectively. + + Availability: Unix. + + .. versionadded:: 3.7 + + .. function:: tcgetpgrp(fd) Return the process group associated with the terminal given by *fd* (an open From c7b7285c2d9473537553c4d8ceeae7e9e317c999 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Mon, 22 Jan 2018 22:00:37 +0000 Subject: [PATCH 08/14] Add information about the rest of the parameters --- Doc/library/os.rst | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 49221743223644..9765f10a44a9d4 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1195,12 +1195,16 @@ or `the MSDN `_ on Windo .. versionadded:: 3.3 -.. function:: preadv(fd, buffers, offset, flags = None) +.. function:: preadv(fd, buffers, offset, flags=0) Combines the functionality of :func:`os.readv` and :func:`os.pread`. It - performs the same task as :func:`os.readv`, but adds a fourth argument, - offset, which specifies the file offset at which the input operation is - to be performed. + reads from a file descriptor *fd* into a number of mutable :term:`bytes-like + objects ` *buffers*. As :func:`os.readv`, it will transfer + data into each buffer until it is full and then move on to the next buffer in + the sequence to hold the rest of the data. Its fourth argument, *offset*, + specifies the file offset at which the input operation is to be performed. + :func:`~os.preadv` return the total number of bytes read (which may be less than + the total capacity of all the objects). If the *flags* argument is provided it will invoke the *preadv2* system call, which modifies the behavior on a per-call basis based on the value From 58910d018cd56b814cc1fe2e0025838180c31b31 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Tue, 23 Jan 2018 00:08:10 +0000 Subject: [PATCH 09/14] Expose pwritev and pwritev2 in os module --- Doc/library/os.rst | 29 +++++++ Lib/test/test_posix.py | 32 +++++++- .../2018-01-19-01-54-22.bpo-31368.kzKqUR.rst | 3 +- Modules/clinic/posixmodule.c.h | 51 +++++++++++- Modules/posixmodule.c | 81 +++++++++++++++++++ configure | 2 +- configure.ac | 2 +- pyconfig.h.in | 6 ++ 8 files changed, 200 insertions(+), 6 deletions(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 9765f10a44a9d4..739726e96717c8 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1101,6 +1101,35 @@ or `the MSDN `_ on Windo .. versionadded:: 3.3 +.. function:: pwritev(fd, buffers, offset, flags=0) + + Combines the functionality of :func:`os.writev` and :func:`os.pwrite`. It + writes the contents of *buffers* to file descriptor *fd*. *buffers* must be + a sequence of :term:`bytes-like objects `. Buffers are + processed in array order. Entire contents of first buffer is written before + proceeding to second, and so on. The operating system may set a limit + (sysconf() value SC_IOV_MAX) on the number of buffers that can be used. + :func:`~os.pwritev` writes the contents of each object to the file descriptor + and returns the total number of bytes written. + + If the *flags* argument is provided it will invoke the *pwritev2* system + call, which modifies the behavior on a per-call basis based on the value + of the *flags* argument. If this argument is ommited, the *pwritev* system + call will be called instead. + + The flags argument contains a bitwise OR of zero or more of the following + flags: + +.. data:: RWF_DSYNC + RWF_SYNC + + The above constants are available on Linux Kernel 4.7 (or newer). + + Availability: Unix. + + .. versionadded:: 3.7 + + .. function:: read(fd, n) Read at most *n* bytes from file descriptor *fd*. Return a bytestring containing the diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 0e86cda7e991f2..ddf576a58955b1 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -284,14 +284,14 @@ def test_preadv(self): finally: os.close(fd) - @unittest.skipUnless(hasattr(posix, 'RWF_SYNC'), "test needs posix.preadv2()") + @unittest.skipUnless(hasattr(posix, 'RWF_HIPRI'), "test needs posix.preadv2()") def test_preadv2(self): fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT) try: os.write(fd, b'test1tt2t3') os.lseek(fd, 0, os.SEEK_SET) buf = [bytearray(i) for i in [5, 3, 2]] - self.assertEqual(posix.preadv(fd, buf, 0, os.RWF_SYNC), 10) + self.assertEqual(posix.preadv(fd, buf, 0, os.RWF_HIPRI), 10) self.assertEqual([b'test1', b'tt2', b't3'], [bytes(i) for i in buf]) finally: os.close(fd) @@ -307,6 +307,34 @@ def test_pwrite(self): finally: os.close(fd) + @unittest.skipUnless(hasattr(posix, 'pwritev'), "test needs posix.pwritev()") + def test_pwritev(self): + fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT) + try: + n = os.pwritev(fd, [b'test1tt2t3'],2) + self.assertEqual(n, 10) + + os.lseek(fd, 0, os.SEEK_SET) + self.assertEqual(b'\x00\x00test1tt2', posix.read(fd, 10)) + + finally: + os.close(fd) + + + @unittest.skipUnless(hasattr(posix, 'os.RWF_SYNC'), "test needs posix.pwritev2()") + def test_pwritev2(self): + fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT) + try: + n = os.pwritev(fd, [b'test1tt2t3'],2, os.RWF_SYNC) + self.assertEqual(n, 10) + + os.lseek(fd, 0, os.SEEK_SET) + self.assertEqual(b'\x00\x00test1tt2', posix.read(fd, 10)) + + finally: + os.close(fd) + + @unittest.skipUnless(hasattr(posix, 'posix_fallocate'), "test needs posix.posix_fallocate()") def test_posix_fallocate(self): diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-01-19-01-54-22.bpo-31368.kzKqUR.rst b/Misc/NEWS.d/next/Core and Builtins/2018-01-19-01-54-22.bpo-31368.kzKqUR.rst index a614ab64628423..97314d833cda0f 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2018-01-19-01-54-22.bpo-31368.kzKqUR.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2018-01-19-01-54-22.bpo-31368.kzKqUR.rst @@ -1 +1,2 @@ -Expose preadv2 system call in the os module. Patch by Pablo Galindo +Expose preadv (preadv2) and pwritev (pwritev2) system calls in the os module. +Patch by Pablo Galindo diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index c2b7f327346b9c..dfbbf1f44be62f 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -4007,6 +4007,51 @@ os_pwrite(PyObject *module, PyObject *const *args, Py_ssize_t nargs) #endif /* defined(HAVE_PWRITE) */ +#if (defined(HAVE_PWRITEV) || defined (HAVE_PWRITEV2)) + +PyDoc_STRVAR(os_pwritev__doc__, +"pwritev($module, fd, buffers, offset, flags=0, /)\n" +"--\n" +"\n" +"Write bytes to a file descriptor starting at a particular offset.\n" +"\n" +"Write buffer to fd, starting at offset bytes from the beginning of\n" +"the file. Returns the number of bytes writte. Does not change the\n" +"current file offset."); + +#define OS_PWRITEV_METHODDEF \ + {"pwritev", (PyCFunction)os_pwritev, METH_FASTCALL, os_pwritev__doc__}, + +static Py_ssize_t +os_pwritev_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, + int flags); + +static PyObject * +os_pwritev(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int fd; + PyObject *buffers; + Py_off_t offset; + int flags = 0; + Py_ssize_t _return_value; + + if (!_PyArg_ParseStack(args, nargs, "iOO&|i:pwritev", + &fd, &buffers, Py_off_t_converter, &offset, &flags)) { + goto exit; + } + _return_value = os_pwritev_impl(module, fd, buffers, offset, flags); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromSsize_t(_return_value); + +exit: + return return_value; +} + +#endif /* (defined(HAVE_PWRITEV) || defined (HAVE_PWRITEV2)) */ + #if defined(HAVE_MKFIFO) PyDoc_STRVAR(os_mkfifo__doc__, @@ -6303,6 +6348,10 @@ os_getrandom(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject #define OS_PWRITE_METHODDEF #endif /* !defined(OS_PWRITE_METHODDEF) */ +#ifndef OS_PWRITEV_METHODDEF + #define OS_PWRITEV_METHODDEF +#endif /* !defined(OS_PWRITEV_METHODDEF) */ + #ifndef OS_MKFIFO_METHODDEF #define OS_MKFIFO_METHODDEF #endif /* !defined(OS_MKFIFO_METHODDEF) */ @@ -6458,4 +6507,4 @@ os_getrandom(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject #ifndef OS_GETRANDOM_METHODDEF #define OS_GETRANDOM_METHODDEF #endif /* !defined(OS_GETRANDOM_METHODDEF) */ -/*[clinic end generated code: output=52cbdec1822e34f9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d50830cf566cb297 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index d146a00c98b466..baf9c8a0fa7ee6 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -8204,7 +8204,9 @@ os_preadv_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, #ifdef HAVE_PREADV2 do { Py_BEGIN_ALLOW_THREADS + _Py_BEGIN_SUPPRESS_IPH n = preadv2(fd, iov, cnt, offset, flags); + _Py_END_SUPPRESS_IPH Py_END_ALLOW_THREADS } while (n < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); #else @@ -8215,7 +8217,9 @@ os_preadv_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, } do { Py_BEGIN_ALLOW_THREADS + _Py_BEGIN_SUPPRESS_IPH n = preadv(fd, iov, cnt, offset); + _Py_END_SUPPRESS_IPH Py_END_ALLOW_THREADS } while (n < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); #endif @@ -8681,6 +8685,82 @@ os_pwrite_impl(PyObject *module, int fd, Py_buffer *buffer, Py_off_t offset) } #endif /* HAVE_PWRITE */ +#if defined(HAVE_PWRITEV) || defined (HAVE_PWRITEV2) +/*[clinic input] +os.pwritev -> Py_ssize_t + + fd: int + buffers: object + offset: Py_off_t + flags: int = 0 + / + +Write bytes to a file descriptor starting at a particular offset. + +Write buffer to fd, starting at offset bytes from the beginning of +the file. Returns the number of bytes writte. Does not change the +current file offset. +[clinic start generated code]*/ + +static Py_ssize_t +os_pwritev_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, + int flags) +/*[clinic end generated code: output=e3dd3e9d11a6a5c7 input=76ec1699089ae61c]*/ +{ + Py_ssize_t cnt; + Py_ssize_t result; + int async_err = 0; + struct iovec *iov; + Py_buffer *buf; + + if (!PySequence_Check(buffers)) { + PyErr_SetString(PyExc_TypeError, + "pwritev() arg 2 must be a sequence"); + return -1; + } + + cnt = PySequence_Size(buffers); + if (cnt < 0) + return -1; + + if (iov_setup(&iov, &buf, buffers, cnt, PyBUF_SIMPLE) < 0) + return -1; +#ifdef HAVE_PWRITEV2 + do { + Py_BEGIN_ALLOW_THREADS + _Py_BEGIN_SUPPRESS_IPH + result = pwritev2(fd, iov, cnt, offset, flags); + _Py_END_SUPPRESS_IPH + Py_END_ALLOW_THREADS + } while (result < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); +#else + if(flags != 0){ + PyErr_SetString(PyExc_OSError, "pwritev2() not available in this system"); + iov_cleanup(iov, buf, cnt); + return -1; + } + do { + Py_BEGIN_ALLOW_THREADS + _Py_BEGIN_SUPPRESS_IPH + result = pwritev(fd, iov, cnt, offset); + _Py_END_SUPPRESS_IPH + Py_END_ALLOW_THREADS + } while (result < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); +#endif + + iov_cleanup(iov, buf, cnt); + if (result < 0) { + if (!async_err) + posix_error(); + return -1; + } + + return result; +} +#endif /* HAVE_PWRITEV */ + + + #ifdef HAVE_MKFIFO /*[clinic input] @@ -12578,6 +12658,7 @@ static PyMethodDef posix_methods[] = { OS_WRITE_METHODDEF OS_WRITEV_METHODDEF OS_PWRITE_METHODDEF + OS_PWRITEV_METHODDEF #ifdef HAVE_SENDFILE {"sendfile", (PyCFunction)posix_sendfile, METH_VARARGS | METH_KEYWORDS, posix_sendfile__doc__}, diff --git a/configure b/configure index ca1557ef118beb..2603cd9dbc5082 100755 --- a/configure +++ b/configure @@ -11202,7 +11202,7 @@ for ac_func in alarm accept4 setitimer getitimer bind_textdomain_codeset chown \ memrchr mbrtowc mkdirat mkfifo \ mkfifoat mknod mknodat mktime mremap nice openat pathconf pause pipe2 plock poll \ posix_fallocate posix_fadvise pread preadv preadv2 \ - pthread_init pthread_kill putenv pwrite readlink readlinkat readv realpath renameat \ + pthread_init pthread_kill putenv pwrite pwritev pwritev2 readlink readlinkat readv realpath renameat \ sem_open sem_timedwait sem_getvalue sem_unlink sendfile setegid seteuid \ setgid sethostname \ setlocale setregid setreuid setresuid setresgid setsid setpgid setpgrp setpriority setuid setvbuf \ diff --git a/configure.ac b/configure.ac index df92ea8dd33c30..1002788cd9e587 100644 --- a/configure.ac +++ b/configure.ac @@ -3436,7 +3436,7 @@ AC_CHECK_FUNCS(alarm accept4 setitimer getitimer bind_textdomain_codeset chown \ memrchr mbrtowc mkdirat mkfifo \ mkfifoat mknod mknodat mktime mremap nice openat pathconf pause pipe2 plock poll \ posix_fallocate posix_fadvise pread preadv preadv2 \ - pthread_init pthread_kill putenv pwrite readlink readlinkat readv realpath renameat \ + pthread_init pthread_kill putenv pwrite pwritev pwritev2 readlink readlinkat readv realpath renameat \ sem_open sem_timedwait sem_getvalue sem_unlink sendfile setegid seteuid \ setgid sethostname \ setlocale setregid setreuid setresuid setresgid setsid setpgid setpgrp setpriority setuid setvbuf \ diff --git a/pyconfig.h.in b/pyconfig.h.in index d98d6e7cb2628a..03beda6623e978 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -749,6 +749,12 @@ /* Define to 1 if you have the `pwrite' function. */ #undef HAVE_PWRITE +/* Define to 1 if you have the `pwritev' function. */ +#undef HAVE_PWRITEV + +/* Define to 1 if you have the `pwritev2' function. */ +#undef HAVE_PWRITEV2 + /* Define to 1 if you have the `readlink' function. */ #undef HAVE_READLINK From f7cdf227c9c96a1216b9ecaec55164f7de89f63e Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Tue, 23 Jan 2018 01:12:48 +0000 Subject: [PATCH 10/14] Fix style issues and Docs --- Doc/library/os.rst | 34 ++++++++++++++++++++++++++-------- Modules/posixmodule.c | 36 ++++++++++++++++++++++-------------- 2 files changed, 48 insertions(+), 22 deletions(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 739726e96717c8..b6a411f9b5f98b 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1112,14 +1112,23 @@ or `the MSDN `_ on Windo :func:`~os.pwritev` writes the contents of each object to the file descriptor and returns the total number of bytes written. - If the *flags* argument is provided it will invoke the *pwritev2* system - call, which modifies the behavior on a per-call basis based on the value - of the *flags* argument. If this argument is ommited, the *pwritev* system - call will be called instead. + :func:`~os.pwritev` will call *pwritev2* system call if available, which modifies + the behavior on a per-call basis based on the value of the *flags* argument. If + the *flags* argument is provided and different than zero, and *pwritev2* is not + available, it will raise a :exc:`NotImplementedError`. + + *pwritev2* and flags different than zero are available since Linux version + 4.7 and newer. The flags argument contains a bitwise OR of zero or more of the following flags: + Availability: Unix (version 2.6.30), FreeBSD (version 6.0 and newer), + OpenBSD (version 2.7 and newer). + + .. versionadded:: 3.7 + + .. data:: RWF_DSYNC RWF_SYNC @@ -1235,14 +1244,23 @@ or `the MSDN `_ on Windo :func:`~os.preadv` return the total number of bytes read (which may be less than the total capacity of all the objects). - If the *flags* argument is provided it will invoke the *preadv2* system - call, which modifies the behavior on a per-call basis based on the value - of the *flags* argument. If this argument is ommited, the *preadv* system - call will be called instead. + :func:`~os.preadv` will call *preadv2* system call if available, which modifies + the behavior on a per-call basis based on the value of the *flags* argument. If + the *flags* argument is provided and different than zero, and *preadv2* is not + available, it will raise a :exc:`NotImplementedError`. + + *preadv2* and flags different than zero are available since Linux version + 4.6 and newer. The flags argument contains a bitwise OR of zero or more of the following flags: + Availability: Unix (version 2.6.30), FreeBSD (version 6.0 and newer), + OpenBSD (version 2.7 and newer). + + .. versionadded:: 3.7 + + .. data:: RWF_HIPRI RWF_NOWAIT diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index baf9c8a0fa7ee6..d416de8c635520 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -8196,11 +8196,20 @@ os_preadv_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, } cnt = PySequence_Size(buffers); - if (cnt < 0) + if (cnt < 0){ return -1; + } - if (iov_setup(&iov, &buf, buffers, cnt, PyBUF_WRITABLE) < 0) +#ifndef HAVE_PREADV2 + if(flags != 0){ + argument_unavailable_error("preadv2", "preadv2() not available in this system"); + return -1; + } +#endif + + if (iov_setup(&iov, &buf, buffers, cnt, PyBUF_WRITABLE) < 0){ return -1; + } #ifdef HAVE_PREADV2 do { Py_BEGIN_ALLOW_THREADS @@ -8210,11 +8219,6 @@ os_preadv_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, Py_END_ALLOW_THREADS } while (n < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); #else - if(flags != 0){ - PyErr_SetString(PyExc_OSError, "preadv2() not available in this system"); - iov_cleanup(iov, buf, cnt); - return -1; - } do { Py_BEGIN_ALLOW_THREADS _Py_BEGIN_SUPPRESS_IPH @@ -8720,11 +8724,20 @@ os_pwritev_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, } cnt = PySequence_Size(buffers); - if (cnt < 0) + if (cnt < 0){ return -1; + } - if (iov_setup(&iov, &buf, buffers, cnt, PyBUF_SIMPLE) < 0) +#ifndef HAVE_PWRITEV2 + if(flags != 0){ + argument_unavailable_error("pwritev2", "pwritev2() not available in this system"); return -1; + } +#endif + + if (iov_setup(&iov, &buf, buffers, cnt, PyBUF_SIMPLE) < 0){ + return -1; + } #ifdef HAVE_PWRITEV2 do { Py_BEGIN_ALLOW_THREADS @@ -8734,11 +8747,6 @@ os_pwritev_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, Py_END_ALLOW_THREADS } while (result < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); #else - if(flags != 0){ - PyErr_SetString(PyExc_OSError, "pwritev2() not available in this system"); - iov_cleanup(iov, buf, cnt); - return -1; - } do { Py_BEGIN_ALLOW_THREADS _Py_BEGIN_SUPPRESS_IPH From 47f65b53ae5c27a040b48ba666f015464119b9cf Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Tue, 23 Jan 2018 01:50:41 +0000 Subject: [PATCH 11/14] Correct style, update docs, fix misused API --- Doc/library/os.rst | 39 +++++++++++++++++++++++---------------- Modules/posixmodule.c | 16 ++++++++-------- 2 files changed, 31 insertions(+), 24 deletions(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index b6a411f9b5f98b..f978d1f71be4db 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1112,18 +1112,23 @@ or `the MSDN `_ on Windo :func:`~os.pwritev` writes the contents of each object to the file descriptor and returns the total number of bytes written. + + The *flags* argument contains a bitwise OR of zero or more of the following + flags: + + - RWF_DSYNC + - RWF_SYNC + :func:`~os.pwritev` will call *pwritev2* system call if available, which modifies - the behavior on a per-call basis based on the value of the *flags* argument. If - the *flags* argument is provided and different than zero, and *pwritev2* is not - available, it will raise a :exc:`NotImplementedError`. + the behavior on a per-call basis based on the value of the *flags* argument. The + :func:`~os.pwritev2` function is required to pass flags. - *pwritev2* and flags different than zero are available since Linux version + *pwritev2* and *flags* different than zero are available since Linux version 4.7 and newer. - The flags argument contains a bitwise OR of zero or more of the following - flags: + The :func:`~os.pwritev2` function is required to pass flags. - Availability: Unix (version 2.6.30), FreeBSD (version 6.0 and newer), + Availability: Linux (version 2.6.30), FreeBSD 6.0 and newer, OpenBSD (version 2.7 and newer). .. versionadded:: 3.7 @@ -1134,7 +1139,7 @@ or `the MSDN `_ on Windo The above constants are available on Linux Kernel 4.7 (or newer). - Availability: Unix. + Availability: Linux. .. versionadded:: 3.7 @@ -1244,18 +1249,20 @@ or `the MSDN `_ on Windo :func:`~os.preadv` return the total number of bytes read (which may be less than the total capacity of all the objects). + The flags argument contains a bitwise OR of zero or more of the following + flags: + + - RWF_HIPRI + - RWF_NOWAIT + :func:`~os.preadv` will call *preadv2* system call if available, which modifies - the behavior on a per-call basis based on the value of the *flags* argument. If - the *flags* argument is provided and different than zero, and *preadv2* is not - available, it will raise a :exc:`NotImplementedError`. + the behavior on a per-call basis based on the value of the *flags* argument. The + :func:`~os.preadv2` function is required to pass flags. *preadv2* and flags different than zero are available since Linux version 4.6 and newer. - The flags argument contains a bitwise OR of zero or more of the following - flags: - - Availability: Unix (version 2.6.30), FreeBSD (version 6.0 and newer), + Availability: Linux (version 2.6.30), FreeBSD 6.0 and newer, OpenBSD (version 2.7 and newer). .. versionadded:: 3.7 @@ -1267,7 +1274,7 @@ or `the MSDN `_ on Windo The above constants are available on Linux Kernel 4.11 and 4.6 (or newer) respectively. - Availability: Unix. + Availability: Linux. .. versionadded:: 3.7 diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index d416de8c635520..10e6aaf5b6745d 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -8196,18 +8196,18 @@ os_preadv_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, } cnt = PySequence_Size(buffers); - if (cnt < 0){ + if (cnt < 0) { return -1; } #ifndef HAVE_PREADV2 - if(flags != 0){ - argument_unavailable_error("preadv2", "preadv2() not available in this system"); + if(flags != 0) { + argument_unavailable_error("preadv2", "flags"); return -1; } #endif - if (iov_setup(&iov, &buf, buffers, cnt, PyBUF_WRITABLE) < 0){ + if (iov_setup(&iov, &buf, buffers, cnt, PyBUF_WRITABLE) < 0) { return -1; } #ifdef HAVE_PREADV2 @@ -8724,18 +8724,18 @@ os_pwritev_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, } cnt = PySequence_Size(buffers); - if (cnt < 0){ + if (cnt < 0) { return -1; } #ifndef HAVE_PWRITEV2 - if(flags != 0){ - argument_unavailable_error("pwritev2", "pwritev2() not available in this system"); + if(flags != 0) { + argument_unavailable_error("pwritev2", "flags"); return -1; } #endif - if (iov_setup(&iov, &buf, buffers, cnt, PyBUF_SIMPLE) < 0){ + if (iov_setup(&iov, &buf, buffers, cnt, PyBUF_SIMPLE) < 0) { return -1; } #ifdef HAVE_PWRITEV2 From 87c69701afdab34eaf1492a9821262441da4b96e Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 26 Jan 2018 11:57:12 -0500 Subject: [PATCH 12/14] Improve tests, clean names and correct style --- Lib/test/test_posix.py | 35 ++++++++++++++++++----------------- Modules/posixmodule.c | 6 ++++-- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index ddf576a58955b1..362b826149c651 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -276,23 +276,21 @@ def test_pread(self): def test_preadv(self): fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT) try: - os.write(fd, b'test1tt2t3') - os.lseek(fd, 0, os.SEEK_SET) + os.write(fd, b'test1tt2t3t5t6t6t8') buf = [bytearray(i) for i in [5, 3, 2]] - self.assertEqual(posix.preadv(fd, buf, 0), 10) - self.assertEqual([b'test1', b'tt2', b't3'], [bytes(i) for i in buf]) + self.assertEqual(posix.preadv(fd, buf, 3), 10) + self.assertEqual([b't1tt2', b't3t', b'5t'], list(buf)) finally: os.close(fd) - @unittest.skipUnless(hasattr(posix, 'RWF_HIPRI'), "test needs posix.preadv2()") - def test_preadv2(self): + @unittest.skipUnless(hasattr(posix, 'RWF_HIPRI'), "test needs posix.RWF_HIPRI") + def test_preadv_flags(self): fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT) try: - os.write(fd, b'test1tt2t3') - os.lseek(fd, 0, os.SEEK_SET) + os.write(fd, b'test1tt2t3t5t6t6t8') buf = [bytearray(i) for i in [5, 3, 2]] - self.assertEqual(posix.preadv(fd, buf, 0, os.RWF_HIPRI), 10) - self.assertEqual([b'test1', b'tt2', b't3'], [bytes(i) for i in buf]) + self.assertEqual(posix.preadv(fd, buf, 3, os.RWF_HIPRI), 10) + self.assertEqual([b't1tt2', b't3t', b'5t'], list(buf)) finally: os.close(fd) @@ -311,26 +309,29 @@ def test_pwrite(self): def test_pwritev(self): fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT) try: - n = os.pwritev(fd, [b'test1tt2t3'],2) + os.write(fd,b"xx") + os.lseek(fd, 0, os.SEEK_SET) + n = os.pwritev(fd, [b'test1', b'tt2', b't3'], 2) self.assertEqual(n, 10) os.lseek(fd, 0, os.SEEK_SET) - self.assertEqual(b'\x00\x00test1tt2', posix.read(fd, 10)) + self.assertEqual(b'xxtest1tt2', posix.read(fd, 10)) finally: os.close(fd) - @unittest.skipUnless(hasattr(posix, 'os.RWF_SYNC'), "test needs posix.pwritev2()") - def test_pwritev2(self): + @unittest.skipUnless(hasattr(posix, 'os.RWF_SYNC'), "test needs os.RWF_SYNC") + def test_pwritev_flags(self): fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT) try: - n = os.pwritev(fd, [b'test1tt2t3'],2, os.RWF_SYNC) + os.write(fd,b"xx") + os.lseek(fd, 0, os.SEEK_SET) + n = os.pwritev(fd, [b'test1', b'tt2', b't3'], 2, os.RWF_SYNC) self.assertEqual(n, 10) os.lseek(fd, 0, os.SEEK_SET) - self.assertEqual(b'\x00\x00test1tt2', posix.read(fd, 10)) - + self.assertEqual(b'xxtest1tt2', posix.read(fd, 10)) finally: os.close(fd) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 10e6aaf5b6745d..4f0e7fdcb3a9d7 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -8230,8 +8230,9 @@ os_preadv_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, iov_cleanup(iov, buf, cnt); if (n < 0) { - if (!async_err) + if (!async_err) { posix_error(); + } return -1; } @@ -8758,8 +8759,9 @@ os_pwritev_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, iov_cleanup(iov, buf, cnt); if (result < 0) { - if (!async_err) + if (!async_err) { posix_error(); + } return -1; } From cb74d38bb0f4ff1a9e28279afeca908d92cf0634 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 26 Jan 2018 12:03:01 -0500 Subject: [PATCH 13/14] Correct documentation --- Doc/library/os.rst | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index f978d1f71be4db..49a5ad043b516f 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1104,10 +1104,10 @@ or `the MSDN `_ on Windo .. function:: pwritev(fd, buffers, offset, flags=0) Combines the functionality of :func:`os.writev` and :func:`os.pwrite`. It - writes the contents of *buffers* to file descriptor *fd*. *buffers* must be - a sequence of :term:`bytes-like objects `. Buffers are - processed in array order. Entire contents of first buffer is written before - proceeding to second, and so on. The operating system may set a limit + writes the contents of *buffers* to file descriptor *fd* at offset *offset*. + *buffers* must be a sequence of :term:`bytes-like objects `. + Buffers are processed in array order. Entire contents of first buffer is written + before proceeding to second, and so on. The operating system may set a limit (sysconf() value SC_IOV_MAX) on the number of buffers that can be used. :func:`~os.pwritev` writes the contents of each object to the file descriptor and returns the total number of bytes written. @@ -1116,18 +1116,18 @@ or `the MSDN `_ on Windo The *flags* argument contains a bitwise OR of zero or more of the following flags: - - RWF_DSYNC - - RWF_SYNC + - RWF_DSYNC + - RWF_SYNC :func:`~os.pwritev` will call *pwritev2* system call if available, which modifies the behavior on a per-call basis based on the value of the *flags* argument. The :func:`~os.pwritev2` function is required to pass flags. - *pwritev2* and *flags* different than zero are available since Linux version - 4.7 and newer. - The :func:`~os.pwritev2` function is required to pass flags. + *pwritev2* and *flags* different than zero are available since Linux version + 4.7. + Availability: Linux (version 2.6.30), FreeBSD 6.0 and newer, OpenBSD (version 2.7 and newer). @@ -1246,7 +1246,7 @@ or `the MSDN `_ on Windo data into each buffer until it is full and then move on to the next buffer in the sequence to hold the rest of the data. Its fourth argument, *offset*, specifies the file offset at which the input operation is to be performed. - :func:`~os.preadv` return the total number of bytes read (which may be less than + :func:`~os.preadv` return the total number of bytes read (which can be less than the total capacity of all the objects). The flags argument contains a bitwise OR of zero or more of the following @@ -1257,7 +1257,7 @@ or `the MSDN `_ on Windo :func:`~os.preadv` will call *preadv2* system call if available, which modifies the behavior on a per-call basis based on the value of the *flags* argument. The - :func:`~os.preadv2` function is required to pass flags. + :c:func:`preadv2` function is required to pass flags. *preadv2* and flags different than zero are available since Linux version 4.6 and newer. From 5563b8db0c784e0cf545bdcba1fc6b7075e87701 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 27 Jan 2018 12:51:05 +0000 Subject: [PATCH 14/14] Correct typos, docstring and improve Docs --- Doc/library/os.rst | 54 +++++++++---------- Lib/test/test_posix.py | 9 ++-- .../2018-01-19-01-54-22.bpo-31368.kzKqUR.rst | 3 +- Modules/clinic/posixmodule.c.h | 37 ++++++++++--- Modules/posixmodule.c | 39 ++++++++++---- 5 files changed, 90 insertions(+), 52 deletions(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 49a5ad043b516f..6ea8cdce678678 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1112,34 +1112,30 @@ or `the MSDN `_ on Windo :func:`~os.pwritev` writes the contents of each object to the file descriptor and returns the total number of bytes written. - The *flags* argument contains a bitwise OR of zero or more of the following flags: - RWF_DSYNC - RWF_SYNC - :func:`~os.pwritev` will call *pwritev2* system call if available, which modifies - the behavior on a per-call basis based on the value of the *flags* argument. The - :func:`~os.pwritev2` function is required to pass flags. - - The :func:`~os.pwritev2` function is required to pass flags. - - *pwritev2* and *flags* different than zero are available since Linux version - 4.7. + Using non-zero flags requires Linux 4.7 or newer. Availability: Linux (version 2.6.30), FreeBSD 6.0 and newer, OpenBSD (version 2.7 and newer). .. versionadded:: 3.7 +.. data:: RWF_DSYNC (since Linux 4.7) + Provide a per-write equivalent of the O_DSYNC open(2) flag. This flag + is meaningful only for pwritev2(), and its effect applies only to the + data range written by the system call. -.. data:: RWF_DSYNC - RWF_SYNC - - The above constants are available on Linux Kernel 4.7 (or newer). + .. versionadded:: 3.7 - Availability: Linux. +.. data:: RWF_SYNC (since Linux 4.7) + Provide a per-write equivalent of the O_SYNC open(2) flag. This flag is + meaningful only for pwritev2(), and its effect applies only to the data + range written by the system call. .. versionadded:: 3.7 @@ -1252,15 +1248,10 @@ or `the MSDN `_ on Windo The flags argument contains a bitwise OR of zero or more of the following flags: - - RWF_HIPRI - - RWF_NOWAIT + - RWF_HIPRI + - RWF_NOWAIT - :func:`~os.preadv` will call *preadv2* system call if available, which modifies - the behavior on a per-call basis based on the value of the *flags* argument. The - :c:func:`preadv2` function is required to pass flags. - - *preadv2* and flags different than zero are available since Linux version - 4.6 and newer. + Using non-zero flags requires Linux 4.6 or newer. Availability: Linux (version 2.6.30), FreeBSD 6.0 and newer, OpenBSD (version 2.7 and newer). @@ -1268,13 +1259,22 @@ or `the MSDN `_ on Windo .. versionadded:: 3.7 -.. data:: RWF_HIPRI - RWF_NOWAIT +.. data:: RWF_HIPRI (since Linux 4.6) + High priority read/write. Allows block-based filesystems to use polling + of the device, which provides lower latency, but may use additional + resources. (Currently, this feature is usable only on a file descriptor + opened using the O_DIRECT flag.) + + .. versionadded:: 3.7 - The above constants are available on Linux Kernel 4.11 and 4.6 (or newer) - respectively. - Availability: Linux. +.. data:: RWF_NOWAIT (since Linux 4.14) + Do not wait for data which is not immediately available. If this flag + is specified, the preadv2() system call will return instantly + if it would have to read data from the backing storage or wait for a lock. + If some data was successfully read, it will return the number of bytes + read. If no bytes were read, it will return -1 and set errno to EAGAIN. + Currently, this flag is meaningful only for preadv2(). .. versionadded:: 3.7 diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 362b826149c651..f2d63666c750af 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -309,18 +309,16 @@ def test_pwrite(self): def test_pwritev(self): fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT) try: - os.write(fd,b"xx") + os.write(fd, b"xx") os.lseek(fd, 0, os.SEEK_SET) n = os.pwritev(fd, [b'test1', b'tt2', b't3'], 2) self.assertEqual(n, 10) os.lseek(fd, 0, os.SEEK_SET) - self.assertEqual(b'xxtest1tt2', posix.read(fd, 10)) - + self.assertEqual(b'xxtest1tt2t3', posix.read(fd, 100)) finally: os.close(fd) - @unittest.skipUnless(hasattr(posix, 'os.RWF_SYNC'), "test needs os.RWF_SYNC") def test_pwritev_flags(self): fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT) @@ -331,11 +329,10 @@ def test_pwritev_flags(self): self.assertEqual(n, 10) os.lseek(fd, 0, os.SEEK_SET) - self.assertEqual(b'xxtest1tt2', posix.read(fd, 10)) + self.assertEqual(b'xxtest1tt2', posix.read(fd, 100)) finally: os.close(fd) - @unittest.skipUnless(hasattr(posix, 'posix_fallocate'), "test needs posix.posix_fallocate()") def test_posix_fallocate(self): diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-01-19-01-54-22.bpo-31368.kzKqUR.rst b/Misc/NEWS.d/next/Core and Builtins/2018-01-19-01-54-22.bpo-31368.kzKqUR.rst index 97314d833cda0f..e13a58bb2c208a 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2018-01-19-01-54-22.bpo-31368.kzKqUR.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2018-01-19-01-54-22.bpo-31368.kzKqUR.rst @@ -1,2 +1 @@ -Expose preadv (preadv2) and pwritev (pwritev2) system calls in the os module. -Patch by Pablo Galindo +Expose preadv and pwritev system calls in the os module. Patch by Pablo Galindo diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index dfbbf1f44be62f..f432437e99efe1 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -3711,10 +3711,21 @@ PyDoc_STRVAR(os_preadv__doc__, "preadv($module, fd, buffers, offset, flags=0, /)\n" "--\n" "\n" -"Read a number of bytes from a file descriptor starting at a particular offset.\n" +"Reads from a file descriptor into a number of mutable bytes-like objects.\n" "\n" -"Read length bytes from file descriptor fd, starting at offset bytes from\n" -"the beginning of the file. The file offset remains unchanged."); +"Combines the functionality of readv() and pread(). As readv(), it will\n" +"transfer data into each buffer until it is full and then move on to the next\n" +"buffer in the sequence to hold the rest of the data. Its fourth argument,\n" +"specifies the file offset at which the input operation is to be performed. It\n" +"will return the total number of bytes read (which can be less than the total\n" +"capacity of all the objects).\n" +"\n" +"The flags argument contains a bitwise OR of zero or more of the following flags:\n" +"\n" +"- RWF_HIPRI\n" +"- RWF_NOWAIT\n" +"\n" +"Using non-zero flags requires Linux 4.6 or newer."); #define OS_PREADV_METHODDEF \ {"preadv", (PyCFunction)os_preadv, METH_FASTCALL, os_preadv__doc__}, @@ -4013,11 +4024,21 @@ PyDoc_STRVAR(os_pwritev__doc__, "pwritev($module, fd, buffers, offset, flags=0, /)\n" "--\n" "\n" -"Write bytes to a file descriptor starting at a particular offset.\n" +"Writes the contents of bytes-like objects to a file descriptor at a given offset.\n" "\n" -"Write buffer to fd, starting at offset bytes from the beginning of\n" -"the file. Returns the number of bytes writte. Does not change the\n" -"current file offset."); +"Combines the functionality of writev() and pwrite(). All buffers must be a sequence\n" +"of bytes-like objects. Buffers are processed in array order. Entire contents of first\n" +"buffer is written before proceeding to second, and so on. The operating system may\n" +"set a limit (sysconf() value SC_IOV_MAX) on the number of buffers that can be used.\n" +"This function writes the contents of each object to the file descriptor and returns\n" +"the total number of bytes written.\n" +"\n" +"The flags argument contains a bitwise OR of zero or more of the following flags:\n" +"\n" +"- RWF_DSYNC\n" +"- RWF_SYNC\n" +"\n" +"Using non-zero flags requires Linux 4.7 or newer."); #define OS_PWRITEV_METHODDEF \ {"pwritev", (PyCFunction)os_pwritev, METH_FASTCALL, os_pwritev__doc__}, @@ -6507,4 +6528,4 @@ os_getrandom(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject #ifndef OS_GETRANDOM_METHODDEF #define OS_GETRANDOM_METHODDEF #endif /* !defined(OS_GETRANDOM_METHODDEF) */ -/*[clinic end generated code: output=d50830cf566cb297 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=06ace805893aa10c input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 4f0e7fdcb3a9d7..a1a38ebda387f5 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -8173,16 +8173,27 @@ os.preadv -> Py_ssize_t flags: int = 0 / -Read a number of bytes from a file descriptor starting at a particular offset. +Reads from a file descriptor into a number of mutable bytes-like objects. -Read length bytes from file descriptor fd, starting at offset bytes from -the beginning of the file. The file offset remains unchanged. +Combines the functionality of readv() and pread(). As readv(), it will +transfer data into each buffer until it is full and then move on to the next +buffer in the sequence to hold the rest of the data. Its fourth argument, +specifies the file offset at which the input operation is to be performed. It +will return the total number of bytes read (which can be less than the total +capacity of all the objects). + +The flags argument contains a bitwise OR of zero or more of the following flags: + +- RWF_HIPRI +- RWF_NOWAIT + +Using non-zero flags requires Linux 4.6 or newer. [clinic start generated code]*/ static Py_ssize_t os_preadv_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, int flags) -/*[clinic end generated code: output=26fc9c6e58e7ada5 input=5b07b7e2d1825627]*/ +/*[clinic end generated code: output=26fc9c6e58e7ada5 input=4173919dc1f7ed99]*/ { Py_ssize_t cnt, n; int async_err = 0; @@ -8700,17 +8711,27 @@ os.pwritev -> Py_ssize_t flags: int = 0 / -Write bytes to a file descriptor starting at a particular offset. +Writes the contents of bytes-like objects to a file descriptor at a given offset. -Write buffer to fd, starting at offset bytes from the beginning of -the file. Returns the number of bytes writte. Does not change the -current file offset. +Combines the functionality of writev() and pwrite(). All buffers must be a sequence +of bytes-like objects. Buffers are processed in array order. Entire contents of first +buffer is written before proceeding to second, and so on. The operating system may +set a limit (sysconf() value SC_IOV_MAX) on the number of buffers that can be used. +This function writes the contents of each object to the file descriptor and returns +the total number of bytes written. + +The flags argument contains a bitwise OR of zero or more of the following flags: + +- RWF_DSYNC +- RWF_SYNC + +Using non-zero flags requires Linux 4.7 or newer. [clinic start generated code]*/ static Py_ssize_t os_pwritev_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, int flags) -/*[clinic end generated code: output=e3dd3e9d11a6a5c7 input=76ec1699089ae61c]*/ +/*[clinic end generated code: output=e3dd3e9d11a6a5c7 input=803dc5ddbf0cfd3b]*/ { Py_ssize_t cnt; Py_ssize_t result;