Got test_pickletools and test_pickle working.

(Alas, test_cpickle is still broken.)

Lib/pickle.py

@@ -465,7 +465,7 @@ def save_long(self, obj, pack=struct.pack):
                     self.write(BININT1 + bytes([obj]))
                     return
                 if obj <= 0xffff:
-                    self.write(BININT2, bytes([obj&0xff, obj>>8]))
+                    self.write(BININT2 + bytes([obj&0xff, obj>>8]))
                     return
             # Next check for 4-byte signed ints:
             high_bits = obj >> 31  # note that Python shift sign-extends
@@ -820,6 +820,7 @@ def load(self):
                 key = read(1)
                 if not key:
                     raise EOFError
+                assert isinstance(key, bytes)
                 dispatch[key[0]](self)
         except _Stop as stopinst:
             return stopinst.value
@@ -892,7 +893,7 @@ def load_binint1(self):
     dispatch[BININT1[0]] = load_binint1
 
     def load_binint2(self):
-        self.append(mloads(b'i' + self.read(2) + '\000\000'))
+        self.append(mloads(b'i' + self.read(2) + b'\000\000'))
     dispatch[BININT2[0]] = load_binint2
 
     def load_long(self):
@@ -1111,7 +1112,7 @@ def load_dup(self):
     dispatch[DUP[0]] = load_dup
 
     def load_get(self):
-        self.append(self.memo[self.readline()[:-1]])
+        self.append(self.memo[str8(self.readline())[:-1]])
     dispatch[GET[0]] = load_get
 
     def load_binget(self):
@@ -1226,24 +1227,24 @@ def encode_long(x):
     byte in the LONG1 pickling context.
 
     >>> encode_long(0)
-    ''
+    b''
     >>> encode_long(255)
-    '\xff\x00'
+    b'\xff\x00'
     >>> encode_long(32767)
-    '\xff\x7f'
+    b'\xff\x7f'
     >>> encode_long(-256)
-    '\x00\xff'
+    b'\x00\xff'
     >>> encode_long(-32768)
-    '\x00\x80'
+    b'\x00\x80'
     >>> encode_long(-128)
-    '\x80'
+    b'\x80'
     >>> encode_long(127)
-    '\x7f'
+    b'\x7f'
     >>>
     """
 
     if x == 0:
-        return ''
+        return b''
     if x > 0:
         ashex = hex(x)
         assert ashex.startswith("0x")
@@ -1284,24 +1285,24 @@ def encode_long(x):
         ashex = ashex[2:]
     assert len(ashex) & 1 == 0, (x, ashex)
     binary = _binascii.unhexlify(ashex)
-    return binary[::-1]
+    return bytes(binary[::-1])
 
 def decode_long(data):
     r"""Decode a long from a two's complement little-endian binary string.
 
-    >>> decode_long('')
+    >>> decode_long(b'')
     0
-    >>> decode_long("\xff\x00")
+    >>> decode_long(b"\xff\x00")
     255
-    >>> decode_long("\xff\x7f")
+    >>> decode_long(b"\xff\x7f")
     32767
-    >>> decode_long("\x00\xff")
+    >>> decode_long(b"\x00\xff")
     -256
-    >>> decode_long("\x00\x80")
+    >>> decode_long(b"\x00\x80")
     -32768
-    >>> decode_long("\x80")
+    >>> decode_long(b"\x80")
     -128
-    >>> decode_long("\x7f")
+    >>> decode_long(b"\x7f")
     127
     """
 
@@ -1310,7 +1311,7 @@ def decode_long(data):
         return 0
     ashex = _binascii.hexlify(data[::-1])
     n = int(ashex, 16) # quadratic time before Python 2.3; linear now
-    if data[-1] >= '\x80':
+    if data[-1] >= 0x80:
         n -= 1 << (nbytes * 8)
     return n
 
@@ -1320,15 +1321,19 @@ def dump(obj, file, protocol=None):
     Pickler(file, protocol).dump(obj)
 
 def dumps(obj, protocol=None):
-    file = io.BytesIO()
-    Pickler(file, protocol).dump(obj)
-    return file.getvalue()
+    f = io.BytesIO()
+    Pickler(f, protocol).dump(obj)
+    res = f.getvalue()
+    assert isinstance(res, bytes)
+    return res
 
 def load(file):
     return Unpickler(file).load()
 
-def loads(str):
-    file = io.BytesIO(str)
+def loads(s):
+    if isinstance(s, str):
+        raise TypeError("Can't load pickle from unicode string")
+    file = io.BytesIO(s)
     return Unpickler(file).load()
 
 # Doctest

Lib/pickletools.py

@@ -202,14 +202,14 @@ def __init__(self, name, n, reader, doc):
 
 def read_uint1(f):
     r"""
-    >>> import StringIO
-    >>> read_uint1(StringIO.StringIO('\xff'))
+    >>> import io
+    >>> read_uint1(io.BytesIO(b'\xff'))
     255
     """
 
     data = f.read(1)
     if data:
-        return ord(data)
+        return data[0]
     raise ValueError("not enough data in stream to read uint1")
 
 uint1 = ArgumentDescriptor(
@@ -221,10 +221,10 @@ def read_uint1(f):
 
 def read_uint2(f):
     r"""
-    >>> import StringIO
-    >>> read_uint2(StringIO.StringIO('\xff\x00'))
+    >>> import io
+    >>> read_uint2(io.BytesIO(b'\xff\x00'))
     255
-    >>> read_uint2(StringIO.StringIO('\xff\xff'))
+    >>> read_uint2(io.BytesIO(b'\xff\xff'))
     65535
     """
 
@@ -242,10 +242,10 @@ def read_uint2(f):
 
 def read_int4(f):
     r"""
-    >>> import StringIO
-    >>> read_int4(StringIO.StringIO('\xff\x00\x00\x00'))
+    >>> import io
+    >>> read_int4(io.BytesIO(b'\xff\x00\x00\x00'))
     255
-    >>> read_int4(StringIO.StringIO('\x00\x00\x00\x80')) == -(2**31)
+    >>> read_int4(io.BytesIO(b'\x00\x00\x00\x80')) == -(2**31)
     True
     """
 
@@ -261,34 +261,48 @@ def read_int4(f):
            doc="Four-byte signed integer, little-endian, 2's complement.")
 
 
+def readline(f):
+    """Read a line from a binary file."""
+    # XXX Slow but at least correct
+    b = bytes()
+    while True:
+        c = f.read(1)
+        if not c:
+            break
+        b += c
+        if c == b'\n':
+            break
+    return b
+
+
 def read_stringnl(f, decode=True, stripquotes=True):
     r"""
-    >>> import StringIO
-    >>> read_stringnl(StringIO.StringIO("'abcd'\nefg\n"))
+    >>> import io
+    >>> read_stringnl(io.BytesIO(b"'abcd'\nefg\n"))
     'abcd'
 
-    >>> read_stringnl(StringIO.StringIO("\n"))
+    >>> read_stringnl(io.BytesIO(b"\n"))
     Traceback (most recent call last):
     ...
-    ValueError: no string quotes around ''
+    ValueError: no string quotes around b''
 
-    >>> read_stringnl(StringIO.StringIO("\n"), stripquotes=False)
+    >>> read_stringnl(io.BytesIO(b"\n"), stripquotes=False)
     ''
 
-    >>> read_stringnl(StringIO.StringIO("''\n"))
+    >>> read_stringnl(io.BytesIO(b"''\n"))
     ''
 
-    >>> read_stringnl(StringIO.StringIO('"abcd"'))
+    >>> read_stringnl(io.BytesIO(b'"abcd"'))
     Traceback (most recent call last):
     ...
     ValueError: no newline found when trying to read stringnl
 
     Embedded escapes are undone in the result.
-    >>> read_stringnl(StringIO.StringIO(r"'a\n\\b\x00c\td'" + "\n'e'"))
+    >>> read_stringnl(io.BytesIO(br"'a\n\\b\x00c\td'" + b"\n'e'"))
     'a\n\\b\x00c\td'
     """
 
-    data = f.readline()
+    data = readline(f)
     if not data.endswith('\n'):
         raise ValueError("no newline found when trying to read stringnl")
     data = data[:-1]    # lose the newline
@@ -336,8 +350,8 @@ def read_stringnl_noescape(f):
 
 def read_stringnl_noescape_pair(f):
     r"""
-    >>> import StringIO
-    >>> read_stringnl_noescape_pair(StringIO.StringIO("Queue\nEmpty\njunk"))
+    >>> import io
+    >>> read_stringnl_noescape_pair(io.BytesIO(b"Queue\nEmpty\njunk"))
     'Queue Empty'
     """
 
@@ -358,12 +372,12 @@ def read_stringnl_noescape_pair(f):
 
 def read_string4(f):
     r"""
-    >>> import StringIO
-    >>> read_string4(StringIO.StringIO("\x00\x00\x00\x00abc"))
+    >>> import io
+    >>> read_string4(io.BytesIO(b"\x00\x00\x00\x00abc"))
     ''
-    >>> read_string4(StringIO.StringIO("\x03\x00\x00\x00abcdef"))
+    >>> read_string4(io.BytesIO(b"\x03\x00\x00\x00abcdef"))
     'abc'
-    >>> read_string4(StringIO.StringIO("\x00\x00\x00\x03abcdef"))
+    >>> read_string4(io.BytesIO(b"\x00\x00\x00\x03abcdef"))
     Traceback (most recent call last):
     ...
     ValueError: expected 50331648 bytes in a string4, but only 6 remain
@@ -374,7 +388,7 @@ def read_string4(f):
         raise ValueError("string4 byte count < 0: %d" % n)
     data = f.read(n)
     if len(data) == n:
-        return data
+        return data.decode("latin-1")
     raise ValueError("expected %d bytes in a string4, but only %d remain" %
                      (n, len(data)))
 
@@ -392,10 +406,10 @@ def read_string4(f):
 
 def read_string1(f):
     r"""
-    >>> import StringIO
-    >>> read_string1(StringIO.StringIO("\x00"))
+    >>> import io
+    >>> read_string1(io.BytesIO(b"\x00"))
     ''
-    >>> read_string1(StringIO.StringIO("\x03abcdef"))
+    >>> read_string1(io.BytesIO(b"\x03abcdef"))
     'abc'
     """
 
@@ -403,7 +417,7 @@ def read_string1(f):
     assert n >= 0
     data = f.read(n)
     if len(data) == n:
-        return data
+        return data.decode("latin-1")
     raise ValueError("expected %d bytes in a string1, but only %d remain" %
                      (n, len(data)))
 
@@ -421,12 +435,12 @@ def read_string1(f):
 
 def read_unicodestringnl(f):
     r"""
-    >>> import StringIO
-    >>> read_unicodestringnl(StringIO.StringIO("abc\uabcd\njunk"))
-    u'abc\uabcd'
+    >>> import io
+    >>> read_unicodestringnl(io.BytesIO(b"abc\\uabcd\njunk")) == 'abc\uabcd'
+    True
     """
 
-    data = f.readline()
+    data = readline(f)
     if not data.endswith('\n'):
         raise ValueError("no newline found when trying to read "
                          "unicodestringnl")
@@ -446,17 +460,17 @@ def read_unicodestringnl(f):
 
 def read_unicodestring4(f):
     r"""
-    >>> import StringIO
-    >>> s = u'abcd\uabcd'
+    >>> import io
+    >>> s = 'abcd\uabcd'
     >>> enc = s.encode('utf-8')
     >>> enc
-    'abcd\xea\xaf\x8d'
-    >>> n = chr(len(enc)) + chr(0) * 3  # little-endian 4-byte length
-    >>> t = read_unicodestring4(StringIO.StringIO(n + enc + 'junk'))
+    b'abcd\xea\xaf\x8d'
+    >>> n = bytes([len(enc), 0, 0, 0])  # little-endian 4-byte length
+    >>> t = read_unicodestring4(io.BytesIO(n + enc + b'junk'))
     >>> s == t
     True
 
-    >>> read_unicodestring4(StringIO.StringIO(n + enc[:-1]))
+    >>> read_unicodestring4(io.BytesIO(n + enc[:-1]))
     Traceback (most recent call last):
     ...
     ValueError: expected 7 bytes in a unicodestring4, but only 6 remain
@@ -486,14 +500,14 @@ def read_unicodestring4(f):
 
 def read_decimalnl_short(f):
     r"""
-    >>> import StringIO
-    >>> read_decimalnl_short(StringIO.StringIO("1234\n56"))
+    >>> import io
+    >>> read_decimalnl_short(io.BytesIO(b"1234\n56"))
     1234
 
-    >>> read_decimalnl_short(StringIO.StringIO("1234L\n56"))
+    >>> read_decimalnl_short(io.BytesIO(b"1234L\n56"))
     Traceback (most recent call last):
     ...
-    ValueError: trailing 'L' not allowed in '1234L'
+    ValueError: trailing 'L' not allowed in b'1234L'
     """
 
     s = read_stringnl(f, decode=False, stripquotes=False)
@@ -515,12 +529,12 @@ def read_decimalnl_short(f):
 
 def read_decimalnl_long(f):
     r"""
-    >>> import StringIO
+    >>> import io
 
-    >>> read_decimalnl_long(StringIO.StringIO("1234L\n56"))
+    >>> read_decimalnl_long(io.BytesIO(b"1234L\n56"))
     1234
 
-    >>> read_decimalnl_long(StringIO.StringIO("123456789012345678901234L\n6"))
+    >>> read_decimalnl_long(io.BytesIO(b"123456789012345678901234L\n6"))
     123456789012345678901234
     """
 
@@ -554,8 +568,8 @@ def read_decimalnl_long(f):
 
 def read_floatnl(f):
     r"""
-    >>> import StringIO
-    >>> read_floatnl(StringIO.StringIO("-1.25\n6"))
+    >>> import io
+    >>> read_floatnl(io.BytesIO(b"-1.25\n6"))
     -1.25
     """
     s = read_stringnl(f, decode=False, stripquotes=False)
@@ -576,11 +590,11 @@ def read_floatnl(f):
 
 def read_float8(f):
     r"""
-    >>> import StringIO, struct
+    >>> import io, struct
     >>> raw = struct.pack(">d", -1.25)
     >>> raw
-    '\xbf\xf4\x00\x00\x00\x00\x00\x00'
-    >>> read_float8(StringIO.StringIO(raw + "\n"))
+    b'\xbf\xf4\x00\x00\x00\x00\x00\x00'
+    >>> read_float8(io.BytesIO(raw + b"\n"))
     -1.25
     """
 
@@ -614,16 +628,16 @@ def read_float8(f):
 
 def read_long1(f):
     r"""
-    >>> import StringIO
-    >>> read_long1(StringIO.StringIO("\x00"))
+    >>> import io
+    >>> read_long1(io.BytesIO(b"\x00"))
     0
-    >>> read_long1(StringIO.StringIO("\x02\xff\x00"))
+    >>> read_long1(io.BytesIO(b"\x02\xff\x00"))
     255
-    >>> read_long1(StringIO.StringIO("\x02\xff\x7f"))
+    >>> read_long1(io.BytesIO(b"\x02\xff\x7f"))
     32767
-    >>> read_long1(StringIO.StringIO("\x02\x00\xff"))
+    >>> read_long1(io.BytesIO(b"\x02\x00\xff"))
     -256
-    >>> read_long1(StringIO.StringIO("\x02\x00\x80"))
+    >>> read_long1(io.BytesIO(b"\x02\x00\x80"))
     -32768
     """
 
@@ -646,16 +660,16 @@ def read_long1(f):
 
 def read_long4(f):
     r"""
-    >>> import StringIO
-    >>> read_long4(StringIO.StringIO("\x02\x00\x00\x00\xff\x00"))
+    >>> import io
+    >>> read_long4(io.BytesIO(b"\x02\x00\x00\x00\xff\x00"))
     255
-    >>> read_long4(StringIO.StringIO("\x02\x00\x00\x00\xff\x7f"))
+    >>> read_long4(io.BytesIO(b"\x02\x00\x00\x00\xff\x7f"))
     32767
-    >>> read_long4(StringIO.StringIO("\x02\x00\x00\x00\x00\xff"))
+    >>> read_long4(io.BytesIO(b"\x02\x00\x00\x00\x00\xff"))
     -256
-    >>> read_long4(StringIO.StringIO("\x02\x00\x00\x00\x00\x80"))
+    >>> read_long4(io.BytesIO(b"\x02\x00\x00\x00\x00\x80"))
     -32768
-    >>> read_long1(StringIO.StringIO("\x00\x00\x00\x00"))
+    >>> read_long1(io.BytesIO(b"\x00\x00\x00\x00"))
     0
     """
 
@@ -701,7 +715,7 @@ class StackObject(object):
     )
 
     def __init__(self, name, obtype, doc):
-        assert isinstance(name, str)
+        assert isinstance(name, basestring)
         self.name = name
 
         assert isinstance(obtype, type) or isinstance(obtype, tuple)
@@ -710,7 +724,7 @@ def __init__(self, name, obtype, doc):
                 assert isinstance(contained, type)
         self.obtype = obtype
 
-        assert isinstance(doc, str)
+        assert isinstance(doc, basestring)
         self.doc = doc
 
     def __repr__(self):
@@ -846,10 +860,10 @@ class OpcodeInfo(object):
 
     def __init__(self, name, code, arg,
                  stack_before, stack_after, proto, doc):
-        assert isinstance(name, str)
+        assert isinstance(name, basestring)
         self.name = name
 
-        assert isinstance(code, str)
+        assert isinstance(code, basestring)
         assert len(code) == 1
         self.code = code
 
@@ -869,7 +883,7 @@ def __init__(self, name, code, arg,
         assert isinstance(proto, int) and 0 <= proto <= 2
         self.proto = proto
 
-        assert isinstance(doc, str)
+        assert isinstance(doc, basestring)
         self.doc = doc
 
 I = OpcodeInfo
@@ -1819,10 +1833,9 @@ def genops(pickle):
     to query its current position) pos is None.
     """
 
-    import cStringIO as StringIO
-
-    if isinstance(pickle, str):
-        pickle = StringIO.StringIO(pickle)
+    if isinstance(pickle, bytes):
+        import io
+        pickle = io.BytesIO(pickle)
 
     if hasattr(pickle, "tell"):
         getpos = pickle.tell
@@ -1832,9 +1845,9 @@ def genops(pickle):
     while True:
         pos = getpos()
         code = pickle.read(1)
-        opcode = code2op.get(code)
+        opcode = code2op.get(code.decode("latin-1"))
         if opcode is None:
-            if code == "":
+            if code == b"":
                 raise ValueError("pickle exhausted before seeing STOP")
             else:
                 raise ValueError("at position %s, opcode %r unknown" % (
@@ -1845,7 +1858,7 @@ def genops(pickle):
         else:
             arg = opcode.arg.reader(pickle)
         yield opcode, arg, pos
-        if code == '.':
+        if code == b'.':
             assert opcode.name == 'STOP'
             break
 
@@ -1995,7 +2008,7 @@ def __init__(self, value):
 
 _dis_test = r"""
 >>> import pickle
->>> x = [1, 2, (3, 4), {'abc': u"def"}]
+>>> x = [1, 2, (3, 4), {str8('abc'): "def"}]
 >>> pkl = pickle.dumps(x, 0)
 >>> dis(pkl)
     0: (    MARK
@@ -2016,7 +2029,7 @@ def __init__(self, value):
    27: p    PUT        2
    30: S    STRING     'abc'
    37: p    PUT        3
-   40: V    UNICODE    u'def'
+   40: V    UNICODE    'def'
    45: p    PUT        4
    48: s    SETITEM
    49: a    APPEND
@@ -2041,7 +2054,7 @@ def __init__(self, value):
    17: q        BINPUT     2
    19: U        SHORT_BINSTRING 'abc'
    24: q        BINPUT     3
-   26: X        BINUNICODE u'def'
+   26: X        BINUNICODE 'def'
    34: q        BINPUT     4
    36: s        SETITEM
    37: e        APPENDS    (MARK at 3)
@@ -2216,13 +2229,14 @@ def __init__(self, value):
 
 _memo_test = r"""
 >>> import pickle
->>> from StringIO import StringIO
->>> f = StringIO()
+>>> import io
+>>> f = io.BytesIO()
 >>> p = pickle.Pickler(f, 2)
 >>> x = [1, 2, 3]
 >>> p.dump(x)
 >>> p.dump(x)
 >>> f.seek(0)
+0
 >>> memo = {}
 >>> dis(f, memo=memo)
     0: \x80 PROTO      2

Lib/test/pickletester.py

@@ -21,7 +21,7 @@
 # Return True if opcode code appears in the pickle, else False.
 def opcode_in_pickle(code, pickle):
     for op, dummy, dummy in pickletools.genops(pickle):
-        if op.code == code:
+        if op.code == code.decode("latin-1"):
             return True
     return False
 
@@ -29,7 +29,7 @@ def opcode_in_pickle(code, pickle):
 def count_opcode(code, pickle):
     n = 0
     for op, dummy, dummy in pickletools.genops(pickle):
-        if op.code == code:
+        if op.code == code.decode("latin-1"):
             n += 1
     return n
 
@@ -95,7 +95,7 @@ class use_metaclass(object, metaclass=metaclass):
 # the object returned by create_data().
 
 # break into multiple strings to avoid confusing font-lock-mode
-DATA0 = """(lp1
+DATA0 = b"""(lp1
 I0
 aL1L
 aF2
@@ -103,7 +103,7 @@ class use_metaclass(object, metaclass=metaclass):
 complex
 p2
 """ + \
-"""(F3
+b"""(F3
 F0
 tRp3
 aI1
@@ -118,15 +118,15 @@ class use_metaclass(object, metaclass=metaclass):
 aI-2147483647
 aI-2147483648
 a""" + \
-"""(S'abc'
+b"""(S'abc'
 p4
 g4
 """ + \
-"""(i__main__
+b"""(i__main__
 C
 p5
 """ + \
-"""(dp6
+b"""(dp6
 S'foo'
 p7
 I1
@@ -213,14 +213,14 @@ class use_metaclass(object, metaclass=metaclass):
 highest protocol among opcodes = 0
 """
 
-DATA1 = (']q\x01(K\x00L1L\nG@\x00\x00\x00\x00\x00\x00\x00'
-         'c__builtin__\ncomplex\nq\x02(G@\x08\x00\x00\x00\x00\x00'
-         '\x00G\x00\x00\x00\x00\x00\x00\x00\x00tRq\x03K\x01J\xff\xff'
-         '\xff\xffK\xffJ\x01\xff\xff\xffJ\x00\xff\xff\xffM\xff\xff'
-         'J\x01\x00\xff\xffJ\x00\x00\xff\xffJ\xff\xff\xff\x7fJ\x01\x00'
-         '\x00\x80J\x00\x00\x00\x80(U\x03abcq\x04h\x04(c__main__\n'
-         'C\nq\x05oq\x06}q\x07(U\x03fooq\x08K\x01U\x03barq\tK\x02ubh'
-         '\x06tq\nh\nK\x05e.'
+DATA1 = (b']q\x01(K\x00L1L\nG@\x00\x00\x00\x00\x00\x00\x00'
+         b'c__builtin__\ncomplex\nq\x02(G@\x08\x00\x00\x00\x00\x00'
+         b'\x00G\x00\x00\x00\x00\x00\x00\x00\x00tRq\x03K\x01J\xff\xff'
+         b'\xff\xffK\xffJ\x01\xff\xff\xffJ\x00\xff\xff\xffM\xff\xff'
+         b'J\x01\x00\xff\xffJ\x00\x00\xff\xffJ\xff\xff\xff\x7fJ\x01\x00'
+         b'\x00\x80J\x00\x00\x00\x80(U\x03abcq\x04h\x04(c__main__\n'
+         b'C\nq\x05oq\x06}q\x07(U\x03fooq\x08K\x01U\x03barq\tK\x02ubh'
+         b'\x06tq\nh\nK\x05e.'
         )
 
 # Disassembly of DATA1.
@@ -280,13 +280,13 @@ class use_metaclass(object, metaclass=metaclass):
 highest protocol among opcodes = 1
 """
 
-DATA2 = ('\x80\x02]q\x01(K\x00\x8a\x01\x01G@\x00\x00\x00\x00\x00\x00\x00'
-         'c__builtin__\ncomplex\nq\x02G@\x08\x00\x00\x00\x00\x00\x00G\x00'
-         '\x00\x00\x00\x00\x00\x00\x00\x86Rq\x03K\x01J\xff\xff\xff\xffK'
-         '\xffJ\x01\xff\xff\xffJ\x00\xff\xff\xffM\xff\xffJ\x01\x00\xff\xff'
-         'J\x00\x00\xff\xffJ\xff\xff\xff\x7fJ\x01\x00\x00\x80J\x00\x00\x00'
-         '\x80(U\x03abcq\x04h\x04(c__main__\nC\nq\x05oq\x06}q\x07(U\x03foo'
-         'q\x08K\x01U\x03barq\tK\x02ubh\x06tq\nh\nK\x05e.')
+DATA2 = (b'\x80\x02]q\x01(K\x00\x8a\x01\x01G@\x00\x00\x00\x00\x00\x00\x00'
+         b'c__builtin__\ncomplex\nq\x02G@\x08\x00\x00\x00\x00\x00\x00G\x00'
+         b'\x00\x00\x00\x00\x00\x00\x00\x86Rq\x03K\x01J\xff\xff\xff\xffK'
+         b'\xffJ\x01\xff\xff\xffJ\x00\xff\xff\xffM\xff\xffJ\x01\x00\xff\xff'
+         b'J\x00\x00\xff\xffJ\xff\xff\xff\x7fJ\x01\x00\x00\x80J\x00\x00\x00'
+         b'\x80(U\x03abcq\x04h\x04(c__main__\nC\nq\x05oq\x06}q\x07(U\x03foo'
+         b'q\x08K\x01U\x03barq\tK\x02ubh\x06tq\nh\nK\x05e.')
 
 # Disassembly of DATA2.
 DATA2_DIS = """\
@@ -465,7 +465,7 @@ def test_recursive_multi(self):
             self.assert_(x[0].attr[1] is x)
 
     def test_garyp(self):
-        self.assertRaises(self.error, self.loads, 'garyp')
+        self.assertRaises(self.error, self.loads, b'garyp')
 
     def test_insecure_strings(self):
         insecure = ["abc", "2 + 2", # not quoted
@@ -479,7 +479,7 @@ def test_insecure_strings(self):
                     #"'\\\\a\'\'\'\\\'\\\\\''",
                     ]
         for s in insecure:
-            buf = "S" + s + "\012p0\012."
+            buf = b"S" + bytes(s) + b"\012p0\012."
             self.assertRaises(ValueError, self.loads, buf)
 
     if have_unicode:
@@ -505,12 +505,12 @@ def test_ints(self):
 
     def test_maxint64(self):
         maxint64 = (1 << 63) - 1
-        data = 'I' + str(maxint64) + '\n.'
+        data = b'I' + bytes(str(maxint64)) + b'\n.'
         got = self.loads(data)
         self.assertEqual(got, maxint64)
 
         # Try too with a bogus literal.
-        data = 'I' + str(maxint64) + 'JUNK\n.'
+        data = b'I' + bytes(str(maxint64)) + b'JUNK\n.'
         self.assertRaises(ValueError, self.loads, data)
 
     def test_long(self):
@@ -535,7 +535,7 @@ def test_long(self):
     @run_with_locale('LC_ALL', 'de_DE', 'fr_FR')
     def test_float_format(self):
         # make sure that floats are formatted locale independent
-        self.assertEqual(self.dumps(1.2)[0:3], 'F1.')
+        self.assertEqual(self.dumps(1.2)[0:3], b'F1.')
 
     def test_reduce(self):
         pass
@@ -577,12 +577,12 @@ def test_proto(self):
         for proto in protocols:
             expected = build_none
             if proto >= 2:
-                expected = pickle.PROTO + chr(proto) + expected
+                expected = pickle.PROTO + bytes([proto]) + expected
             p = self.dumps(None, proto)
             self.assertEqual(p, expected)
 
         oob = protocols[-1] + 1     # a future protocol
-        badpickle = pickle.PROTO + chr(oob) + build_none
+        badpickle = pickle.PROTO + bytes([oob]) + build_none
         try:
             self.loads(badpickle)
         except ValueError as detail:
@@ -708,8 +708,8 @@ def produce_global_ext(self, extcode, opcode):
 
             # Dump using protocol 1 for comparison.
             s1 = self.dumps(x, 1)
-            self.assert_(__name__ in s1)
-            self.assert_("MyList" in s1)
+            self.assert_(bytes(__name__) in s1)
+            self.assert_(b"MyList" in s1)
             self.assertEqual(opcode_in_pickle(opcode, s1), False)
 
             y = self.loads(s1)
@@ -718,9 +718,9 @@ def produce_global_ext(self, extcode, opcode):
 
             # Dump using protocol 2 for test.
             s2 = self.dumps(x, 2)
-            self.assert_(__name__ not in s2)
-            self.assert_("MyList" not in s2)
-            self.assertEqual(opcode_in_pickle(opcode, s2), True)
+            self.assert_(bytes(__name__) not in s2)
+            self.assert_(b"MyList" not in s2)
+            self.assertEqual(opcode_in_pickle(opcode, s2), True, repr(s2))
 
             y = self.loads(s2)
             self.assertEqual(list(x), list(y))
@@ -770,6 +770,7 @@ def test_dict_chunking(self):
         x = dict.fromkeys(range(n))
         for proto in protocols:
             s = self.dumps(x, proto)
+            assert isinstance(s, bytes)
             y = self.loads(s)
             self.assertEqual(x, y)
             num_setitems = count_opcode(pickle.SETITEMS, s)

Lib/test/test_pickle.py

@@ -1,6 +1,6 @@
 import pickle
 import unittest
-from cStringIO import StringIO
+import io
 
 from test import test_support
 
@@ -26,16 +26,16 @@ class PicklerTests(AbstractPickleTests):
     error = KeyError
 
     def dumps(self, arg, proto=0, fast=0):
-        f = StringIO()
+        f = io.BytesIO()
         p = pickle.Pickler(f, proto)
         if fast:
             p.fast = fast
         p.dump(arg)
         f.seek(0)
-        return f.read()
+        return bytes(f.read())
 
     def loads(self, buf):
-        f = StringIO(buf)
+        f = io.BytesIO(buf)
         u = pickle.Unpickler(f)
         return u.load()
 
@@ -45,7 +45,7 @@ def dumps(self, arg, proto=0, fast=0):
         class PersPickler(pickle.Pickler):
             def persistent_id(subself, obj):
                 return self.persistent_id(obj)
-        f = StringIO()
+        f = io.BytesIO()
         p = PersPickler(f, proto)
         if fast:
             p.fast = fast
@@ -57,7 +57,7 @@ def loads(self, buf):
         class PersUnpickler(pickle.Unpickler):
             def persistent_load(subself, obj):
                 return self.persistent_load(obj)
-        f = StringIO(buf)
+        f = io.BytesIO(buf)
         u = PersUnpickler(f)
         return u.load()
 

Modules/cPickle.c

@@ -5241,6 +5241,13 @@ cpm_dumps(PyObject *self, PyObject *args, PyObject *kwds)
 		goto finally;
 
 	res = PycStringIO->cgetvalue(file);
+        if (res == NULL)
+		goto finally;
+	if (!PyBytes_Check(res)) {
+		PyObject *tmp = res;
+		res = PyBytes_FromObject(res);
+		Py_DECREF(tmp);
+	}
 
   finally:
 	Py_XDECREF(pickler);