mirror of
https://github.com/python/cpython
synced 2024-09-23 03:03:41 +00:00
Issue #12591: Allow io.TextIOWrapper to work with raw IO objects (without
a read1() method), and add a *write_through* parameter to mandate unbuffered writes.
This commit is contained in:
commit
d42c1d09e9
|
@ -1478,7 +1478,7 @@ class TextIOWrapper(TextIOBase):
|
||||||
_CHUNK_SIZE = 2048
|
_CHUNK_SIZE = 2048
|
||||||
|
|
||||||
def __init__(self, buffer, encoding=None, errors=None, newline=None,
|
def __init__(self, buffer, encoding=None, errors=None, newline=None,
|
||||||
line_buffering=False):
|
line_buffering=False, write_through=False):
|
||||||
if newline is not None and not isinstance(newline, str):
|
if newline is not None and not isinstance(newline, str):
|
||||||
raise TypeError("illegal newline type: %r" % (type(newline),))
|
raise TypeError("illegal newline type: %r" % (type(newline),))
|
||||||
if newline not in (None, "", "\n", "\r", "\r\n"):
|
if newline not in (None, "", "\n", "\r", "\r\n"):
|
||||||
|
@ -1521,6 +1521,7 @@ def __init__(self, buffer, encoding=None, errors=None, newline=None,
|
||||||
self._decoded_chars_used = 0 # offset into _decoded_chars for read()
|
self._decoded_chars_used = 0 # offset into _decoded_chars for read()
|
||||||
self._snapshot = None # info for reconstructing decoder state
|
self._snapshot = None # info for reconstructing decoder state
|
||||||
self._seekable = self._telling = self.buffer.seekable()
|
self._seekable = self._telling = self.buffer.seekable()
|
||||||
|
self._has_read1 = hasattr(self.buffer, 'read1')
|
||||||
self._b2cratio = 0.0
|
self._b2cratio = 0.0
|
||||||
|
|
||||||
if self._seekable and self.writable():
|
if self._seekable and self.writable():
|
||||||
|
@ -1687,7 +1688,10 @@ def _read_chunk(self):
|
||||||
# len(dec_buffer) bytes ago with decoder state (b'', dec_flags).
|
# len(dec_buffer) bytes ago with decoder state (b'', dec_flags).
|
||||||
|
|
||||||
# Read a chunk, decode it, and put the result in self._decoded_chars.
|
# Read a chunk, decode it, and put the result in self._decoded_chars.
|
||||||
input_chunk = self.buffer.read1(self._CHUNK_SIZE)
|
if self._has_read1:
|
||||||
|
input_chunk = self.buffer.read1(self._CHUNK_SIZE)
|
||||||
|
else:
|
||||||
|
input_chunk = self.buffer.read(self._CHUNK_SIZE)
|
||||||
eof = not input_chunk
|
eof = not input_chunk
|
||||||
decoded_chars = self._decoder.decode(input_chunk, eof)
|
decoded_chars = self._decoder.decode(input_chunk, eof)
|
||||||
self._set_decoded_chars(decoded_chars)
|
self._set_decoded_chars(decoded_chars)
|
||||||
|
|
|
@ -2314,6 +2314,27 @@ def test_readonly_attributes(self):
|
||||||
with self.assertRaises(AttributeError):
|
with self.assertRaises(AttributeError):
|
||||||
txt.buffer = buf
|
txt.buffer = buf
|
||||||
|
|
||||||
|
def test_rawio(self):
|
||||||
|
# Issue #12591: TextIOWrapper must work with raw I/O objects, so
|
||||||
|
# that subprocess.Popen() can have the required unbuffered
|
||||||
|
# semantics with universal_newlines=True.
|
||||||
|
raw = self.MockRawIO([b'abc', b'def', b'ghi\njkl\nopq\n'])
|
||||||
|
txt = self.TextIOWrapper(raw, encoding='ascii', newline='\n')
|
||||||
|
# Reads
|
||||||
|
self.assertEqual(txt.read(4), 'abcd')
|
||||||
|
self.assertEqual(txt.readline(), 'efghi\n')
|
||||||
|
self.assertEqual(list(txt), ['jkl\n', 'opq\n'])
|
||||||
|
|
||||||
|
def test_rawio_write_through(self):
|
||||||
|
# Issue #12591: with write_through=True, writes don't need a flush
|
||||||
|
raw = self.MockRawIO([b'abc', b'def', b'ghi\njkl\nopq\n'])
|
||||||
|
txt = self.TextIOWrapper(raw, encoding='ascii', newline='\n',
|
||||||
|
write_through=True)
|
||||||
|
txt.write('1')
|
||||||
|
txt.write('23\n4')
|
||||||
|
txt.write('5')
|
||||||
|
self.assertEqual(b''.join(raw._write_stack), b'123\n45')
|
||||||
|
|
||||||
class CTextIOWrapperTest(TextIOWrapperTest):
|
class CTextIOWrapperTest(TextIOWrapperTest):
|
||||||
|
|
||||||
def test_initialization(self):
|
def test_initialization(self):
|
||||||
|
|
|
@ -237,6 +237,10 @@ Core and Builtins
|
||||||
Library
|
Library
|
||||||
-------
|
-------
|
||||||
|
|
||||||
|
- Issue #12591: Allow io.TextIOWrapper to work with raw IO objects (without
|
||||||
|
a read1() method), and add a *write_through* parameter to mandate
|
||||||
|
unbuffered writes.
|
||||||
|
|
||||||
- Issue #10883: Fix socket leaks in urllib.request when using FTP.
|
- Issue #10883: Fix socket leaks in urllib.request when using FTP.
|
||||||
|
|
||||||
- Issue #12592: Make Python build on OpenBSD 5 (and future major releases).
|
- Issue #12592: Make Python build on OpenBSD 5 (and future major releases).
|
||||||
|
|
|
@ -653,10 +653,12 @@ typedef struct
|
||||||
PyObject *errors;
|
PyObject *errors;
|
||||||
const char *writenl; /* utf-8 encoded, NULL stands for \n */
|
const char *writenl; /* utf-8 encoded, NULL stands for \n */
|
||||||
char line_buffering;
|
char line_buffering;
|
||||||
|
char write_through;
|
||||||
char readuniversal;
|
char readuniversal;
|
||||||
char readtranslate;
|
char readtranslate;
|
||||||
char writetranslate;
|
char writetranslate;
|
||||||
char seekable;
|
char seekable;
|
||||||
|
char has_read1;
|
||||||
char telling;
|
char telling;
|
||||||
char deallocating;
|
char deallocating;
|
||||||
/* Specialized encoding func (see below) */
|
/* Specialized encoding func (see below) */
|
||||||
|
@ -813,13 +815,13 @@ static int
|
||||||
textiowrapper_init(textio *self, PyObject *args, PyObject *kwds)
|
textiowrapper_init(textio *self, PyObject *args, PyObject *kwds)
|
||||||
{
|
{
|
||||||
char *kwlist[] = {"buffer", "encoding", "errors",
|
char *kwlist[] = {"buffer", "encoding", "errors",
|
||||||
"newline", "line_buffering",
|
"newline", "line_buffering", "write_through",
|
||||||
NULL};
|
NULL};
|
||||||
PyObject *buffer, *raw;
|
PyObject *buffer, *raw;
|
||||||
char *encoding = NULL;
|
char *encoding = NULL;
|
||||||
char *errors = NULL;
|
char *errors = NULL;
|
||||||
char *newline = NULL;
|
char *newline = NULL;
|
||||||
int line_buffering = 0;
|
int line_buffering = 0, write_through = 0;
|
||||||
_PyIO_State *state = IO_STATE;
|
_PyIO_State *state = IO_STATE;
|
||||||
|
|
||||||
PyObject *res;
|
PyObject *res;
|
||||||
|
@ -827,9 +829,9 @@ textiowrapper_init(textio *self, PyObject *args, PyObject *kwds)
|
||||||
|
|
||||||
self->ok = 0;
|
self->ok = 0;
|
||||||
self->detached = 0;
|
self->detached = 0;
|
||||||
if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|zzzi:fileio",
|
if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|zzzii:fileio",
|
||||||
kwlist, &buffer, &encoding, &errors,
|
kwlist, &buffer, &encoding, &errors,
|
||||||
&newline, &line_buffering))
|
&newline, &line_buffering, &write_through))
|
||||||
return -1;
|
return -1;
|
||||||
|
|
||||||
if (newline && newline[0] != '\0'
|
if (newline && newline[0] != '\0'
|
||||||
|
@ -935,6 +937,7 @@ textiowrapper_init(textio *self, PyObject *args, PyObject *kwds)
|
||||||
self->chunk_size = 8192;
|
self->chunk_size = 8192;
|
||||||
self->readuniversal = (newline == NULL || newline[0] == '\0');
|
self->readuniversal = (newline == NULL || newline[0] == '\0');
|
||||||
self->line_buffering = line_buffering;
|
self->line_buffering = line_buffering;
|
||||||
|
self->write_through = write_through;
|
||||||
self->readtranslate = (newline == NULL);
|
self->readtranslate = (newline == NULL);
|
||||||
if (newline) {
|
if (newline) {
|
||||||
self->readnl = PyUnicode_FromString(newline);
|
self->readnl = PyUnicode_FromString(newline);
|
||||||
|
@ -1044,6 +1047,8 @@ textiowrapper_init(textio *self, PyObject *args, PyObject *kwds)
|
||||||
self->seekable = self->telling = PyObject_IsTrue(res);
|
self->seekable = self->telling = PyObject_IsTrue(res);
|
||||||
Py_DECREF(res);
|
Py_DECREF(res);
|
||||||
|
|
||||||
|
self->has_read1 = PyObject_HasAttrString(buffer, "read1");
|
||||||
|
|
||||||
self->encoding_start_of_stream = 0;
|
self->encoding_start_of_stream = 0;
|
||||||
if (self->seekable && self->encoder) {
|
if (self->seekable && self->encoder) {
|
||||||
PyObject *cookieObj;
|
PyObject *cookieObj;
|
||||||
|
@ -1287,7 +1292,9 @@ textiowrapper_write(textio *self, PyObject *args)
|
||||||
text = newtext;
|
text = newtext;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (self->line_buffering &&
|
if (self->write_through)
|
||||||
|
needflush = 1;
|
||||||
|
else if (self->line_buffering &&
|
||||||
(haslf ||
|
(haslf ||
|
||||||
findchar(PyUnicode_AS_UNICODE(text),
|
findchar(PyUnicode_AS_UNICODE(text),
|
||||||
PyUnicode_GET_SIZE(text), '\r')))
|
PyUnicode_GET_SIZE(text), '\r')))
|
||||||
|
@ -1435,7 +1442,8 @@ textiowrapper_read_chunk(textio *self)
|
||||||
if (chunk_size == NULL)
|
if (chunk_size == NULL)
|
||||||
goto fail;
|
goto fail;
|
||||||
input_chunk = PyObject_CallMethodObjArgs(self->buffer,
|
input_chunk = PyObject_CallMethodObjArgs(self->buffer,
|
||||||
_PyIO_str_read1, chunk_size, NULL);
|
(self->has_read1 ? _PyIO_str_read1: _PyIO_str_read),
|
||||||
|
chunk_size, NULL);
|
||||||
Py_DECREF(chunk_size);
|
Py_DECREF(chunk_size);
|
||||||
if (input_chunk == NULL)
|
if (input_chunk == NULL)
|
||||||
goto fail;
|
goto fail;
|
||||||
|
|
Loading…
Reference in a new issue