mirror of
https://github.com/python/cpython
synced 2024-09-16 04:09:56 +00:00
bpo-36311: Fixes decoding multibyte characters around chunk boundaries and improves decoding performance (GH-15083)
This commit is contained in:
parent
df0c21ff46
commit
7ebdda0dbe
|
@ -3075,13 +3075,13 @@ def test_mbcs_alias(self):
|
||||||
self.assertEqual(codec.name, 'mbcs')
|
self.assertEqual(codec.name, 'mbcs')
|
||||||
|
|
||||||
@support.bigmemtest(size=2**31, memuse=7, dry_run=False)
|
@support.bigmemtest(size=2**31, memuse=7, dry_run=False)
|
||||||
def test_large_input(self):
|
def test_large_input(self, size):
|
||||||
# Test input longer than INT_MAX.
|
# Test input longer than INT_MAX.
|
||||||
# Input should contain undecodable bytes before and after
|
# Input should contain undecodable bytes before and after
|
||||||
# the INT_MAX limit.
|
# the INT_MAX limit.
|
||||||
encoded = (b'01234567' * (2**28-1) +
|
encoded = (b'01234567' * ((size//8)-1) +
|
||||||
b'\x85\x86\xea\xeb\xec\xef\xfc\xfd\xfe\xff')
|
b'\x85\x86\xea\xeb\xec\xef\xfc\xfd\xfe\xff')
|
||||||
self.assertEqual(len(encoded), 2**31+2)
|
self.assertEqual(len(encoded), size+2)
|
||||||
decoded = codecs.code_page_decode(932, encoded, 'surrogateescape', True)
|
decoded = codecs.code_page_decode(932, encoded, 'surrogateescape', True)
|
||||||
self.assertEqual(decoded[1], len(encoded))
|
self.assertEqual(decoded[1], len(encoded))
|
||||||
del encoded
|
del encoded
|
||||||
|
@ -3092,6 +3092,20 @@ def test_large_input(self):
|
||||||
'\udc85\udc86\udcea\udceb\udcec'
|
'\udc85\udc86\udcea\udceb\udcec'
|
||||||
'\udcef\udcfc\udcfd\udcfe\udcff')
|
'\udcef\udcfc\udcfd\udcfe\udcff')
|
||||||
|
|
||||||
|
@support.bigmemtest(size=2**31, memuse=6, dry_run=False)
|
||||||
|
def test_large_utf8_input(self, size):
|
||||||
|
# Test input longer than INT_MAX.
|
||||||
|
# Input should contain a decodable multi-byte character
|
||||||
|
# surrounding INT_MAX
|
||||||
|
encoded = (b'0123456\xed\x84\x80' * (size//8))
|
||||||
|
self.assertEqual(len(encoded), size // 8 * 10)
|
||||||
|
decoded = codecs.code_page_decode(65001, encoded, 'ignore', True)
|
||||||
|
self.assertEqual(decoded[1], len(encoded))
|
||||||
|
del encoded
|
||||||
|
self.assertEqual(len(decoded[0]), size)
|
||||||
|
self.assertEqual(decoded[0][:10], '0123456\ud10001')
|
||||||
|
self.assertEqual(decoded[0][-11:], '56\ud1000123456\ud100')
|
||||||
|
|
||||||
|
|
||||||
class ASCIITest(unittest.TestCase):
|
class ASCIITest(unittest.TestCase):
|
||||||
def test_encode(self):
|
def test_encode(self):
|
||||||
|
|
|
@ -0,0 +1,2 @@
|
||||||
|
Decoding bytes objects larger than 2GiB is faster and no longer fails when a
|
||||||
|
multibyte characters spans a chunk boundary.
|
|
@ -7186,6 +7186,12 @@ PyUnicode_AsASCIIString(PyObject *unicode)
|
||||||
#define NEED_RETRY
|
#define NEED_RETRY
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
/* INT_MAX is the theoretical largest chunk (or INT_MAX / 2 when
|
||||||
|
transcoding from UTF-16), but INT_MAX / 4 perfoms better in
|
||||||
|
both cases also and avoids partial characters overrunning the
|
||||||
|
length limit in MultiByteToWideChar on Windows */
|
||||||
|
#define DECODING_CHUNK_SIZE (INT_MAX/4)
|
||||||
|
|
||||||
#ifndef WC_ERR_INVALID_CHARS
|
#ifndef WC_ERR_INVALID_CHARS
|
||||||
# define WC_ERR_INVALID_CHARS 0x0080
|
# define WC_ERR_INVALID_CHARS 0x0080
|
||||||
#endif
|
#endif
|
||||||
|
@ -7422,8 +7428,8 @@ decode_code_page_stateful(int code_page,
|
||||||
do
|
do
|
||||||
{
|
{
|
||||||
#ifdef NEED_RETRY
|
#ifdef NEED_RETRY
|
||||||
if (size > INT_MAX) {
|
if (size > DECODING_CHUNK_SIZE) {
|
||||||
chunk_size = INT_MAX;
|
chunk_size = DECODING_CHUNK_SIZE;
|
||||||
final = 0;
|
final = 0;
|
||||||
done = 0;
|
done = 0;
|
||||||
}
|
}
|
||||||
|
@ -7827,10 +7833,8 @@ encode_code_page(int code_page,
|
||||||
do
|
do
|
||||||
{
|
{
|
||||||
#ifdef NEED_RETRY
|
#ifdef NEED_RETRY
|
||||||
/* UTF-16 encoding may double the size, so use only INT_MAX/2
|
if (len > DECODING_CHUNK_SIZE) {
|
||||||
chunks. */
|
chunk_len = DECODING_CHUNK_SIZE;
|
||||||
if (len > INT_MAX/2) {
|
|
||||||
chunk_len = INT_MAX/2;
|
|
||||||
done = 0;
|
done = 0;
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
|
|
Loading…
Reference in a new issue