[WIP] gh-85235: Always specify encoding for open() text files.

Lib/idlelib/idle_test/test_config.py

@@ -191,7 +191,7 @@ def setUpClass(cls):
             idle_dir = os.path.abspath(sys.path[0])
         for ctype in conf.config_types:
             config_path = os.path.join(idle_dir, '../config-%s.def' % ctype)
-            with open(config_path, 'r') as f:
+            with open(config_path, 'r', encoding='ascii') as f:
                 cls.config_string[ctype] = f.read()
 
         cls.orig_warn = config._warn

Lib/idlelib/idle_test/test_outwin.py

@@ -159,7 +159,7 @@ def test_file_line_helper(self, mock_open):
         for line, expected_output in test_lines:
             self.assertEqual(flh(line), expected_output)
             if expected_output:
-                mock_open.assert_called_with(expected_output[0], 'r')
+                mock_open.assert_called_with(expected_output[0], 'rb')
 
 
 if __name__ == '__main__':

Lib/idlelib/idle_test/test_textview.py

@@ -223,7 +223,7 @@ def _command():
         self.assertEqual(self.called, True)
         self.assertEqual(self.view.title(), 'TITLE_FILE')
         get = self.view.viewframe.textframe.text.get
-        with open(__file__) as f:
+        with open(__file__, encoding='ascii') as f:
             self.assertEqual(get('1.0', '1.end'), f.readline().strip())
             f.readline()
             self.assertEqual(get('3.0', '3.end'), f.readline().strip())

Lib/idlelib/outwin.py

@@ -42,7 +42,7 @@ def file_line_helper(line):
         if match:
             filename, lineno = match.group(1, 2)
             try:
-                f = open(filename, "r")
+                f = open(filename, "rb")
                 f.close()
                 break
             except OSError:

Lib/mailcap.py

@@ -42,7 +42,7 @@ def getcaps():
     lineno = 0
     for mailcap in listmailcapfiles():
         try:
-            fp = open(mailcap, 'r')
+            fp = open(mailcap, 'r', encoding='utf-8')
         except OSError:
             continue
         with fp:

Lib/test/_test_eintr.py

@@ -344,13 +344,13 @@ def _test_open(self, do_open_close_reader, do_open_close_writer):
             self.assertEqual(proc.wait(), 0)
 
     def python_open(self, path):
-        fp = open(path, 'w')
+        fp = open(path, 'wb')
         fp.close()
 
     @unittest.skipIf(sys.platform == "darwin",
                      "hangs under macOS; see bpo-25234, bpo-35363")
     def test_open(self):
-        self._test_open("fp = open(path, 'r')\nfp.close()",
+        self._test_open("fp = open(path, 'rb')\nfp.close()",
                         self.python_open)
 
     def os_open(self, path):

Lib/test/libregrtest/cmdline.py

@@ -428,13 +428,13 @@ def _parse_args(args, **kwargs):
     if ns.match_filename:
         if ns.match_tests is None:
             ns.match_tests = []
-        with open(ns.match_filename) as fp:
+        with open(ns.match_filename, encoding='utf-8') as fp:
             for line in fp:
                 ns.match_tests.append(line.strip())
     if ns.ignore_filename:
         if ns.ignore_tests is None:
             ns.ignore_tests = []
-        with open(ns.ignore_filename) as fp:
+        with open(ns.ignore_filename, encoding='utf-8') as fp:
             for line in fp:
                 ns.ignore_tests.append(line.strip())
     if ns.forever:

Lib/test/libregrtest/main.py

@@ -207,7 +207,7 @@ def find_tests(self, tests):
         if self.ns.single:
             self.next_single_filename = os.path.join(self.tmp_dir, 'pynexttest')
             try:
-                with open(self.next_single_filename, 'r') as fp:
+                with open(self.next_single_filename, 'r', encoding='utf-8') as fp:
                     next_test = fp.read().strip()
                     self.tests = [next_test]
             except OSError:
@@ -218,7 +218,8 @@ def find_tests(self, tests):
             # regex to match 'test_builtin' in line:
             # '0:00:00 [  4/400] test_builtin -- test_dict took 1 sec'
             regex = re.compile(r'\btest_[a-zA-Z0-9_]+\b')
-            with open(os.path.join(os_helper.SAVEDCWD, self.ns.fromfile)) as fp:
+            filepath = os.path.join(os_helper.SAVEDCWD, self.ns.fromfile)
+            with open(filepath, encoding='utf-8', errors='replace') as fp:
                 for line in fp:
                     line = line.split('#', 1)[0]
                     line = line.strip()
@@ -566,7 +567,7 @@ def run_tests(self):
     def finalize(self):
         if self.next_single_filename:
             if self.next_single_test:
-                with open(self.next_single_filename, 'w') as fp:
+                with open(self.next_single_filename, 'w', encoding='utf-8') as fp:
                     fp.write(self.next_single_test + '\n')
             else:
                 os.unlink(self.next_single_filename)

Lib/test/pickletester.py

@@ -3429,7 +3429,7 @@ def test_callapi(self):
         self.Pickler(f, protocol=-1)
 
     def test_dump_text_file(self):
-        f = open(TESTFN, "w")
+        f = open(TESTFN, "w", encoding='ascii')
         try:
             for proto in protocols:
                 self.assertRaises(TypeError, self.dump, 123, f, proto)

Lib/test/test_builtin.py

@@ -2112,7 +2112,7 @@ def _run_child(self, child, terminal_input):
                 # Make sure we don't get stuck if there's a problem
                 signal.alarm(2)
                 os.close(r)
-                with open(w, "w") as wpipe:
+                with open(w, "w", encoding='ascii') as wpipe:
                     child(wpipe)
             except:
                 traceback.print_exc()
@@ -2125,7 +2125,7 @@ def _run_child(self, child, terminal_input):
         os.write(fd, terminal_input)
 
         # Get results from the pipe
-        with open(r, encoding="utf-8") as rpipe:
+        with open(r, encoding="ascii") as rpipe:
             lines = []
             while True:
                 line = rpipe.readline().strip()

Lib/test/test_contextlib.py

@@ -345,7 +345,7 @@ class C:
 
 class FileContextTestCase(unittest.TestCase):
 
-    def testWithOpen(self):
+    def testWithOpenText(self):
         tfn = tempfile.mktemp()
         try:
             f = None
@@ -363,6 +363,24 @@ def testWithOpen(self):
         finally:
             os_helper.unlink(tfn)
 
+    def testWithOpenBinary(self):
+        tfn = tempfile.mktemp()
+        try:
+            f = None
+            with open(tfn, "wb") as f:
+                self.assertFalse(f.closed)
+                f.write(b"Booh\n")
+            self.assertTrue(f.closed)
+            f = None
+            with self.assertRaises(ZeroDivisionError):
+                with open(tfn, "rb") as f:
+                    self.assertFalse(f.closed)
+                    self.assertEqual(f.read(), b"Booh\n")
+                    1 / 0
+            self.assertTrue(f.closed)
+        finally:
+            os_helper.unlink(tfn)
+
 class LockContextTestCase(unittest.TestCase):
 
     def boilerPlate(self, lock, locked):

Lib/test/test_difflib.py

@@ -238,7 +238,7 @@ def test_html_diff(self):
         actual = full.replace('</body>','\n%s\n</body>' % tables)
 
         # temporarily uncomment next two lines to baseline this test
-        #with open('test_difflib_expect.html','w') as fp:
+        #with open('test_difflib_expect.html','w', encoding='utf-8') as fp:
         #    fp.write(actual)
 
         with open(findfile('test_difflib_expect.html'), encoding="utf-8") as fp:

Lib/test/test_exception_hierarchy.py

@@ -103,7 +103,7 @@ def test_try_except(self):
         # PyErr_SetFromErrnoWithFilenameObject was called.
         # (it is therefore deliberate that it doesn't use assertRaises)
         try:
-            open(filename)
+            open(filename, 'rb')
         except FileNotFoundError:
             pass
         else:

Lib/test/test_exceptions.py

@@ -70,7 +70,7 @@ def testRaising(self):
             unlink(TESTFN)
 
         self.raise_catch(OSError, "OSError")
-        self.assertRaises(OSError, open, 'this file does not exist', 'r')
+        self.assertRaises(OSError, open, 'this file does not exist', 'rb')
 
         self.raise_catch(ImportError, "ImportError")
         self.assertRaises(ImportError, __import__, "undefined_module")

Lib/test/test_frame.py

@@ -253,7 +253,7 @@ def __del__(self):
             def gen():
                 yield
 
-            fd = open({__file__!r})
+            fd = open({__file__!r}, 'rb')
             l = [fd, GCHello()]
             l.append(l)
             del fd

Lib/test/test_hashlib.py

@@ -1163,12 +1163,12 @@ def test_file_digest(self):
         with self.assertRaises(ValueError):
             hashlib.file_digest(None, "sha256")
 
-        with self.assertRaises(ValueError):
-            with open(os_helper.TESTFN, "r") as f:
+        with open(os_helper.TESTFN, "r", encoding="latin1") as f:
+            with self.assertRaises(ValueError):
                 hashlib.file_digest(f, "sha256")
 
-        with self.assertRaises(ValueError):
-            with open(os_helper.TESTFN, "wb") as f:
+        with open(os_helper.TESTFN, "wb") as f:
+            with self.assertRaises(ValueError):
                 hashlib.file_digest(f, "sha256")
 
 

Lib/test/test_interpreters.py

@@ -46,7 +46,7 @@ def _running(interp):
     def run():
         interp.run(dedent(f"""
             # wait for "signal"
-            with open({r}) as rpipe:
+            with open({r}, 'rb') as rpipe:
                 rpipe.read()
             """))
 
@@ -55,8 +55,8 @@ def run():
 
     yield
 
-    with open(w, 'w') as spipe:
-        spipe.write('done')
+    with open(w, 'wb') as spipe:
+        spipe.write(b'done')
     t.join()
 
 

Lib/test/test_lib2to3/test_main.py

@@ -62,14 +62,14 @@ def setup_test_source_trees(self):
         os.mkdir(self.py3_dest_dir)
         # Turn it into a package with a few files.
         self.setup_files = []
-        open(os.path.join(self.py2_src_dir, "__init__.py"), "w").close()
+        open(os.path.join(self.py2_src_dir, "__init__.py"), "wb").close()
         self.setup_files.append("__init__.py")
         shutil.copy(PY2_TEST_MODULE, self.py2_src_dir)
         self.setup_files.append(os.path.basename(PY2_TEST_MODULE))
         self.trivial_py2_file = os.path.join(self.py2_src_dir, "trivial.py")
         self.init_py2_file = os.path.join(self.py2_src_dir, "__init__.py")
-        with open(self.trivial_py2_file, "w") as trivial:
-            trivial.write("print 'I need a simple conversion.'")
+        with open(self.trivial_py2_file, "wb") as trivial:
+            trivial.write(b"print 'I need a simple conversion.'")
         self.setup_files.append("trivial.py")
 
     def test_filename_changing_on_output_single_dir(self):

Lib/test/test_logging.py

@@ -5566,7 +5566,7 @@ def test_emit_after_closing_in_write_mode(self):
         fh.emit(self.next_rec())    # '1'
         fh.close()
         fh.emit(self.next_rec())    # '2'
-        with open(self.fn) as fp:
+        with open(self.fn, encoding='ascii') as fp:
             self.assertEqual(fp.read().strip(), '1')
 
 class RotatingFileHandlerTest(BaseFileTest):
@@ -5727,7 +5727,7 @@ def test_rollover(self):
             for f in files:
                 print('Contents of %s:' % f)
                 path = os.path.join(dn, f)
-                with open(path, 'r') as tf:
+                with open(path, 'r', encoding='latin1') as tf:
                     print(tf.read())
         self.assertTrue(found, msg=msg)
 

Lib/test/test_mailbox.py

@@ -2171,7 +2171,7 @@ def createMessage(self, dir, mbox=False):
         try:
             os.link(tmpname, newname)
         except (AttributeError, PermissionError):
-            with open(newname, "w") as fp:
+            with open(newname, "w", encoding='ascii') as fp:
                 fp.write(DUMMY_MESSAGE)
         self._msgfiles.append(newname)
         return tmpname

Lib/test/test_mailcap.py

@@ -93,7 +93,7 @@ def test_listmailcapfiles(self):
 
     def test_readmailcapfile(self):
         # Test readmailcapfile() using test file. It should match MAILCAPDICT.
-        with open(MAILCAPFILE, 'r') as mcf:
+        with open(MAILCAPFILE, 'r', encoding='utf-8') as mcf:
             with self.assertWarns(DeprecationWarning):
                 d = mailcap.readmailcapfile(mcf)
         self.assertDictEqual(d, MAILCAPDICT_DEPRECATED)

Lib/test/test_netrc.py

@@ -280,7 +280,7 @@ def test_security(self):
         os.mkdir(d)
         self.addCleanup(os_helper.rmtree, d)
         fn = os.path.join(d, '.netrc')
-        with open(fn, 'wt') as f:
+        with open(fn, 'w', encoding='ascii') as f:
             f.write("""\
                 machine foo.domain.com login bar password pass
                 default login foo password pass
@@ -293,7 +293,7 @@ def test_security(self):
                              ('bar', '', 'pass'))
             os.chmod(fn, 0o622)
             self.assertRaises(netrc.NetrcParseError, netrc.netrc)
-        with open(fn, 'wt') as f:
+        with open(fn, 'wt', encoding='ascii') as f:
             f.write("""\
                 machine foo.domain.com login anonymous password pass
                 default login foo password pass

Lib/test/test_patma.py

@@ -19,7 +19,7 @@ def test_refleaks(self):
         # Hunting for leaks using -R doesn't catch leaks in the compiler itself,
         # just the code under test. This test ensures that if there are leaks in
         # the pattern compiler, those runs will fail:
-        with open(__file__) as file:
+        with open(__file__, 'rb') as file:
             compile(file.read(), __file__, "exec")