From 6d1330d17d78b3bcb12031acb94ecfd1b449a735 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 23 Jun 2022 11:14:40 +0300 Subject: [PATCH 1/3] gh-85308: Add argparse tests for reading non-ASCII arguments from file --- Lib/test/test_argparse.py | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 299eb30b4332bf..33ada515d4726b 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -1505,14 +1505,15 @@ class TestArgumentsFromFile(TempDirMixin, ParserTestCase): def setUp(self): super(TestArgumentsFromFile, self).setUp() file_texts = [ - ('hello', 'hello world!\n'), - ('recursive', '-a\n' - 'A\n' - '@hello'), - ('invalid', '@no-such-path\n'), + ('hello', os.fsencode(self.hello) + b'\n'), + ('recursive', b'-a\n' + b'A\n' + b'@hello'), + ('invalid', b'@no-such-path\n'), + ('undecodable', self.undecodable + b'\n'), ] for path, text in file_texts: - with open(path, 'w', encoding="utf-8") as file: + with open(path, 'wb') as file: file.write(text) parser_signature = Sig(fromfile_prefix_chars='@') @@ -1522,15 +1523,24 @@ def setUp(self): Sig('y', nargs='+'), ] failures = ['', '-b', 'X', '@invalid', '@missing'] + hello = 'hello world!' + os_helper.FS_NONASCII successes = [ ('X Y', NS(a=None, x='X', y=['Y'])), ('X -a A Y Z', NS(a='A', x='X', y=['Y', 'Z'])), - ('@hello X', NS(a=None, x='hello world!', y=['X'])), - ('X @hello', NS(a=None, x='X', y=['hello world!'])), - ('-a B @recursive Y Z', NS(a='A', x='hello world!', y=['Y', 'Z'])), - ('X @recursive Z -a B', NS(a='B', x='X', y=['hello world!', 'Z'])), + ('@hello X', NS(a=None, x=hello, y=['X'])), + ('X @hello', NS(a=None, x='X', y=[hello])), + ('-a B @recursive Y Z', NS(a='A', x=hello, y=['Y', 'Z'])), + ('X @recursive Z -a B', NS(a='B', x='X', y=[hello, 'Z'])), (["-a", "", "X", "Y"], NS(a='', x='X', y=['Y'])), ] + if os_helper.TESTFN_UNDECODABLE: + undecodable = os_helper.TESTFN_UNDECODABLE.lstrip(b'@') + successes += [ + ('@undecodable X', NS(a=None, x=os.fsdecode(undecodable), y=['X'])), + ('X @undecodable', NS(a=None, x='X', y=[os.fsdecode(undecodable)])), + ] + else: + undecodable = b'' class TestArgumentsFromFileConverter(TempDirMixin, ParserTestCase): @@ -1539,10 +1549,10 @@ class TestArgumentsFromFileConverter(TempDirMixin, ParserTestCase): def setUp(self): super(TestArgumentsFromFileConverter, self).setUp() file_texts = [ - ('hello', 'hello world!\n'), + ('hello', b'hello world!\n'), ] for path, text in file_texts: - with open(path, 'w', encoding="utf-8") as file: + with open(path, 'wb') as file: file.write(text) class FromFileConverterArgumentParser(ErrorRaisingArgumentParser): From 2e2250aa3f70c52fcd705ed6fb2b1b1eca5ca9ad Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 23 Jun 2022 15:41:14 +0300 Subject: [PATCH 2/3] Fix tests on Windows. --- Lib/test/test_argparse.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 33ada515d4726b..ac3d60db241012 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -1505,7 +1505,7 @@ class TestArgumentsFromFile(TempDirMixin, ParserTestCase): def setUp(self): super(TestArgumentsFromFile, self).setUp() file_texts = [ - ('hello', os.fsencode(self.hello) + b'\n'), + ('hello', self.hello.encode(sys.getfilesystemencoding()) + b'\n'), ('recursive', b'-a\n' b'A\n' b'@hello'), @@ -1535,9 +1535,11 @@ def setUp(self): ] if os_helper.TESTFN_UNDECODABLE: undecodable = os_helper.TESTFN_UNDECODABLE.lstrip(b'@') + decoded_undecodable = undecodable.decode(sys.getfilesystemencoding(), + sys.getfilesystemencodeerrors()) successes += [ - ('@undecodable X', NS(a=None, x=os.fsdecode(undecodable), y=['X'])), - ('X @undecodable', NS(a=None, x='X', y=[os.fsdecode(undecodable)])), + ('@undecodable X', NS(a=None, x=decoded_undecodable, y=['X'])), + ('X @undecodable', NS(a=None, x='X', y=[decoded_undecodable])), ] else: undecodable = b'' From c47e76db4e73422e3879bbd57801eba6b4c98a8a Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 24 Jun 2022 16:21:53 +0300 Subject: [PATCH 3/3] Try to fix tests on Windows again. --- Lib/test/support/os_helper.py | 5 +++++ Lib/test/test_argparse.py | 5 ++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Lib/test/support/os_helper.py b/Lib/test/support/os_helper.py index ff2fc338cbf239..589ef19dd34cc2 100644 --- a/Lib/test/support/os_helper.py +++ b/Lib/test/support/os_helper.py @@ -141,6 +141,11 @@ try: name.decode(sys.getfilesystemencoding()) except UnicodeDecodeError: + try: + name.decode(sys.getfilesystemencoding(), + sys.getfilesystemencodeerrors()) + except UnicodeDecodeError: + continue TESTFN_UNDECODABLE = os.fsencode(TESTFN_ASCII) + name break diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index ac3d60db241012..673ef89b5e1cfe 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -1505,7 +1505,7 @@ class TestArgumentsFromFile(TempDirMixin, ParserTestCase): def setUp(self): super(TestArgumentsFromFile, self).setUp() file_texts = [ - ('hello', self.hello.encode(sys.getfilesystemencoding()) + b'\n'), + ('hello', os.fsencode(self.hello) + b'\n'), ('recursive', b'-a\n' b'A\n' b'@hello'), @@ -1535,8 +1535,7 @@ def setUp(self): ] if os_helper.TESTFN_UNDECODABLE: undecodable = os_helper.TESTFN_UNDECODABLE.lstrip(b'@') - decoded_undecodable = undecodable.decode(sys.getfilesystemencoding(), - sys.getfilesystemencodeerrors()) + decoded_undecodable = os.fsdecode(undecodable) successes += [ ('@undecodable X', NS(a=None, x=decoded_undecodable, y=['X'])), ('X @undecodable', NS(a=None, x='X', y=[decoded_undecodable])),