tokenizer.c: make coding markup work again.

io.open() now takes all positional parameters (so we can conveniently
call it from C code).

test_tarfile.py no longer uses u"..." literals, but is otherwise still
badly broken.

This is a checkpoint; some more stuff now breaks.

Lib/io.py

@@ -49,7 +49,7 @@ def __init__(self, errno, strerror, characters_written=0):
         self.characters_written = characters_written
 
 
-def open(file, mode="r", buffering=None, *, encoding=None, newline=None):
+def open(file, mode="r", buffering=None, encoding=None, newline=None):
     """Replacement for the built-in open function.
 
     Args:
@@ -59,7 +59,6 @@ def open(file, mode="r", buffering=None, *, encoding=None, newline=None):
       buffering: optional int >= 0 giving the buffer size; values
                  can be: 0 = unbuffered, 1 = line buffered,
                  larger = fully buffered.
-    Keywords (for text modes only; *must* be given as keyword arguments):
       encoding: optional string giving the text encoding.
       newline: optional newlines specifier; must be None, '\n' or '\r\n';
                specifies the line ending expected on input and written on

Lib/test/test_tarfile.py

@@ -432,17 +432,17 @@ def test_pax_global_headers(self):
         tarinfo = tar.getmember("pax/regtype1")
         self.assertEqual(tarinfo.uname, "foo")
         self.assertEqual(tarinfo.gname, "bar")
-        self.assertEqual(tarinfo.pax_headers.get("VENDOR.umlauts"), u"ÄÖÜäöüß")
+        self.assertEqual(tarinfo.pax_headers.get("VENDOR.umlauts"), "ÄÖÜäöüß")
 
         tarinfo = tar.getmember("pax/regtype2")
         self.assertEqual(tarinfo.uname, "")
         self.assertEqual(tarinfo.gname, "bar")
-        self.assertEqual(tarinfo.pax_headers.get("VENDOR.umlauts"), u"ÄÖÜäöüß")
+        self.assertEqual(tarinfo.pax_headers.get("VENDOR.umlauts"), "ÄÖÜäöüß")
 
         tarinfo = tar.getmember("pax/regtype3")
         self.assertEqual(tarinfo.uname, "tarfile")
         self.assertEqual(tarinfo.gname, "tarfile")
-        self.assertEqual(tarinfo.pax_headers.get("VENDOR.umlauts"), u"ÄÖÜäöüß")
+        self.assertEqual(tarinfo.pax_headers.get("VENDOR.umlauts"), "ÄÖÜäöüß")
 
     def test_pax_number_fields(self):
         # All following number fields are read from the pax header.
@@ -727,11 +727,11 @@ def _test(self, name, link=None):
 
     def test_pax_global_header(self):
         pax_headers = {
-                u"foo": u"bar",
-                u"uid": u"0",
-                u"mtime": u"1.23",
-                u"test": u"äöü",
-                u"äöü": u"test"}
+                "foo": "bar",
+                "uid": "0",
+                "mtime": "1.23",
+                "test": "äöü",
+                "äöü": "test"}
 
         tar = tarfile.open(tmpname, "w", format=tarfile.PAX_FORMAT, \
                 pax_headers=pax_headers)
@@ -756,11 +756,11 @@ def test_pax_global_header(self):
     def test_pax_extended_header(self):
         # The fields from the pax header have priority over the
         # TarInfo.
-        pax_headers = {u"path": u"foo", u"uid": u"123"}
+        pax_headers = {"path": "foo", "uid": "123"}
 
         tar = tarfile.open(tmpname, "w", format=tarfile.PAX_FORMAT, encoding="iso8859-1")
         t = tarfile.TarInfo()
-        t.name = u"äöü"     # non-ASCII
+        t.name = "äöü"     # non-ASCII
         t.uid = 8**8        # too large
         t.pax_headers = pax_headers
         tar.addfile(t)
@@ -808,11 +808,11 @@ def test_unicode_filename_error(self):
         else:
             tar.addfile(tarinfo)
 
-        tarinfo.name = u"äöü"
+        tarinfo.name = "äöü"
         self.assertRaises(UnicodeError, tar.addfile, tarinfo)
 
         tarinfo.name = "foo"
-        tarinfo.uname = u"äöü"
+        tarinfo.uname = "äöü"
         self.assertRaises(UnicodeError, tar.addfile, tarinfo)
 
     def test_unicode_argument(self):
@@ -825,7 +825,7 @@ def test_unicode_argument(self):
         tar.close()
 
     def test_uname_unicode(self):
-        for name in (u"äöü", "äöü"):
+        for name in ("äöü", "äöü"):
             t = tarfile.TarInfo("foo")
             t.uname = name
             t.gname = name
@@ -860,9 +860,9 @@ def _create_unicode_name(self, name):
     def test_error_handlers(self):
         # Test if the unicode error handlers work correctly for characters
         # that cannot be expressed in a given encoding.
-        self._create_unicode_name(u"äöü")
+        self._create_unicode_name("äöü")
 
-        for handler, name in (("utf-8", u"äöü".encode("utf8")),
+        for handler, name in (("utf-8", "äöü".encode("utf8")),
                     ("replace", "???"), ("ignore", "")):
             tar = tarfile.open(tmpname, format=self.format, encoding="ascii",
                     errors=handler)
@@ -874,11 +874,11 @@ def test_error_handlers(self):
     def test_error_handler_utf8(self):
         # Create a pathname that has one component representable using
         # iso8859-1 and the other only in iso8859-15.
-        self._create_unicode_name(u"äöü/¤")
+        self._create_unicode_name("äöü/¤")
 
         tar = tarfile.open(tmpname, format=self.format, encoding="iso8859-1",
                 errors="utf-8")
-        self.assertEqual(tar.getnames()[0], "äöü/" + u"¤".encode("utf8"))
+        self.assertEqual(tar.getnames()[0], "äöü/" + "¤".encode("utf8"))
 
 
 class AppendTest(unittest.TestCase):

Parser/tokenizer.c

@@ -396,25 +396,29 @@ fp_readl(char *s, int size, struct tok_state *tok)
 static int
 fp_setreadl(struct tok_state *tok, const char* enc)
 {
-	PyObject *reader, *stream, *readline;
+	PyObject *readline = NULL, *stream = NULL, *io = NULL;
+	int ok = 0;
 
-	/* XXX: constify filename argument. */
-	stream = PyFile_FromFile(tok->fp, (char*)tok->filename, "rb", NULL);
+	io = PyImport_ImportModule("io");
+	if (io == NULL)
+		goto cleanup;
+
+	stream = PyObject_CallMethod(io, "open", "ssis",
+				     tok->filename, "r", -1, enc);
 	if (stream == NULL)
-		return 0;
+		goto cleanup;
 
-	reader = PyCodec_StreamReader(enc, stream, NULL);
-	Py_DECREF(stream);
-	if (reader == NULL)
-		return 0;
-
-	readline = PyObject_GetAttrString(reader, "readline");
-	Py_DECREF(reader);
+	readline = PyObject_GetAttrString(stream, "readline");
 	if (readline == NULL)
-		return 0;
+		goto cleanup;
 
 	tok->decoding_readline = readline;
-	return 1;
+	ok = 1;
+
+  cleanup:
+	Py_XDECREF(stream);
+	Py_XDECREF(io);
+	return ok;
 }
 
 /* Fetch the next byte from TOK. */