gh-133677: Fix tests when running in non-UTF-8 locale

python · serhiy-storchaka · May 12, 2025 · May 11, 2025 · May 11, 2025 · May 11, 2025
commit 842076d3b91e7f0d193485eb5227e3a7a2f6b01d
diff --git a/Lib/test/support/strace_helper.py b/Lib/test/support/strace_helper.py
@@ -38,7 +38,7 @@ def events(self):

        This assumes the program under inspection doesn't print any non-utf8
        strings which would mix into the strace output."""
-        decoded_events = self.event_bytes.decode('utf-8')
+        decoded_events = self.event_bytes.decode('utf-8', 'surrogateescape')
        matches = [
            _syscall_regex.match(event)
            for event in decoded_events.splitlines()

@@ -6972,8 +6972,8 @@ def make_zip_script(self, script_name, name_in_zip=None):
        return zip_name

    def check_usage(self, expected, *args, **kwargs):
-        res = script_helper.assert_python_ok('-Xutf8', *args, '-h', **kwargs)
-        self.assertEqual(res.out.splitlines()[0].decode(),
+        res = script_helper.assert_python_ok(*args, '-h', **kwargs)
+        self.assertEqual(os.fsdecode(res.out.splitlines()[0]),
                         f'usage: {expected} [-h]')

    def test_script(self, compiled=False):

@@ -791,21 +791,21 @@ def test_table_output_format(self):
 class TestAsyncioToolsEdgeCases(unittest.TestCase):

    def test_task_awaits_self(self):
-        """A task directly awaits itself – should raise a cycle."""
+        """A task directly awaits itself - should raise a cycle."""
        input_ = [(1, [(1, "Self-Awaiter", [[["loopback"], 1]])])]
        with self.assertRaises(tools.CycleFoundException) as ctx:
            tools.build_async_tree(input_)
        self.assertIn([1, 1], ctx.exception.cycles)

    def test_task_with_missing_awaiter_id(self):
-        """Awaiter ID not in task list – should not crash, just show 'Unknown'."""
+        """Awaiter ID not in task list - should not crash, just show 'Unknown'."""
        input_ = [(1, [(1, "Task-A", [[["coro"], 999]])])]  # 999 not defined
        table = tools.build_task_table(input_)
        self.assertEqual(len(table), 1)
        self.assertEqual(table[0][4], "Unknown")

    def test_duplicate_coroutine_frames(self):
-        """Same coroutine frame repeated under a parent – should deduplicate."""
+        """Same coroutine frame repeated under a parent - should deduplicate."""
        input_ = [
            (
                1,
@@ -829,7 +829,7 @@ def test_duplicate_coroutine_frames(self):
        self.assertIn("Task-1", flat)

    def test_task_with_no_name(self):
-        """Task with no name in id2name – should still render with fallback."""
+        """Task with no name in id2name - should still render with fallback."""
        input_ = [(1, [(1, "root", [[["f1"], 2]]), (2, None, [])])]
        # If name is None, fallback to string should not crash
        tree = tools.build_async_tree(input_)

@@ -20,7 +20,7 @@
 from test.support import is_emscripten, is_wasi
 from test.support import infinite_recursion
 from test.support import os_helper
-from test.support.os_helper import TESTFN, FakePath
+from test.support.os_helper import TESTFN, FS_NONASCII, FakePath
 try:
    import fcntl
 except ImportError:
@@ -770,12 +770,16 @@ def test_as_uri_windows(self):
        self.assertEqual(self.make_uri(P('c:/')), 'file:///c:/')
        self.assertEqual(self.make_uri(P('c:/a/b.c')), 'file:///c:/a/b.c')
        self.assertEqual(self.make_uri(P('c:/a/b%#c')), 'file:///c:/a/b%25%23c')
-        self.assertEqual(self.make_uri(P('c:/a/b\xe9')), 'file:///c:/a/b%C3%A9')
        self.assertEqual(self.make_uri(P('//some/share/')), 'file://some/share/')
        self.assertEqual(self.make_uri(P('//some/share/a/b.c')),
                         'file://some/share/a/b.c')
-        self.assertEqual(self.make_uri(P('//some/share/a/b%#c\xe9')),
-                         'file://some/share/a/b%25%23c%C3%A9')
+
+        from urllib.parse import quote_from_bytes
+        QUOTED_FS_NONASCII = quote_from_bytes(os.fsencode(FS_NONASCII))
+        self.assertEqual(self.make_uri(P('c:/a/b' + FS_NONASCII)),
+                         'file:///c:/a/b' + QUOTED_FS_NONASCII)
+        self.assertEqual(self.make_uri(P('//some/share/a/b%#c' + FS_NONASCII)),
+                         'file://some/share/a/b%25%23c' + QUOTED_FS_NONASCII)

    @needs_windows
    def test_ordering_windows(self):

diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py
@@ -109,7 +109,7 @@ def setUp(self):
        finally:
            f.close()
        self.pathname = os_helper.TESTFN
-        self.quoted_pathname = urllib.parse.quote(self.pathname)
+        self.quoted_pathname = urllib.parse.quote(os.fsencode(self.pathname))
        self.returned_obj = urllib.request.urlopen("file:%s" % self.quoted_pathname)

    def tearDown(self):

diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py
@@ -3642,7 +3642,7 @@ def test_cli_with_metadata_encoding_extract(self):
            except OSError:
                pass
            except UnicodeEncodeError:
-                self.skipTest(f'cannot encode file name {fn!r}')
+                self.skipTest(f'cannot encode file name {fn!a}')

        zipfile.main(["--metadata-encoding=shift_jis", "-e", TESTFN, TESTFN2])
        listing = os.listdir(TESTFN2)
-Original file line number
+Diff line change
@@ -791,21 +791,21 @@ def test_table_output_format(self):
     class TestAsyncioToolsEdgeCases(unittest.TestCase):
         def test_task_awaits_self(self):
-            """A task directly awaits itself – should raise a cycle."""
+            """A task directly awaits itself - should raise a cycle."""
             Copy link

  
      
    
  

  
      

  
    Contributor


      

  

  
    
      

      
            StanFromIreland
  

      

      

      


        May 11, 2025


      
    

  


        
      
  
  
  
    

    There was a problem hiding this comment.


  

 
  
    

    Choose a reason for hiding this comment

    
      The reason will be displayed to describe this comment to others. Learn more.
    

    
      
      


  


  
    
      These are all unrelated.
    
  
  


    

        
      
  
  
    
  
  
  
    
    Sorry, something went wrong.
  

  
    
  
    
      

              Uh oh!

              
There was an error while loading. Please reload this page.


  
  


          
      
  
    
    
      
        
            
    All reactions
  


          
          
        
      
    

    



    
        
  
    
        
    
  


      
          
  
      
            Copy link

  
      
    
  

  
      

  
  Member


      

  Author


  

  
    
      

      
            serhiy-storchaka
  

      

      

      


        May 11, 2025


      
    

  


        
      
  
  
  
    

    There was a problem hiding this comment.


  

 
  
    

    Choose a reason for hiding this comment

    
      The reason will be displayed to describe this comment to others. Learn more.
    

    
      
      


  


  
    
      This is very related. Error happens when non-ASCII "–" is written to stdout with a non-UTF-8 encoding.
    
  
  


    

        
      
  
  
    
  
  
  
    
    Sorry, something went wrong.
  

  
    
  
    
      

              Uh oh!

              
There was an error while loading. Please reload this page.


  
  


          
      
  
    
    
            AA-Turner reacted with thumbs up emoji
      
        
            
    All reactions
  


          
              
                👍
                  1 reaction
             input_ = [(1, [(1, "Self-Awaiter", [[["loopback"], 1]])])]
             with self.assertRaises(tools.CycleFoundException) as ctx:
                 tools.build_async_tree(input_)
             self.assertIn([1, 1], ctx.exception.cycles)
         def test_task_with_missing_awaiter_id(self):
-            """Awaiter ID not in task list – should not crash, just show 'Unknown'."""
+            """Awaiter ID not in task list - should not crash, just show 'Unknown'."""
             input_ = [(1, [(1, "Task-A", [[["coro"], 999]])])]  # 999 not defined
             table = tools.build_task_table(input_)
             self.assertEqual(len(table), 1)
             self.assertEqual(table[0][4], "Unknown")
         def test_duplicate_coroutine_frames(self):
-            """Same coroutine frame repeated under a parent – should deduplicate."""
+            """Same coroutine frame repeated under a parent - should deduplicate."""
             input_ = [
                 (
 ,
@@ -829,7 +829,7 @@ def test_duplicate_coroutine_frames(self):
             self.assertIn("Task-1", flat)
         def test_task_with_no_name(self):
-            """Task with no name in id2name – should still render with fallback."""
+            """Task with no name in id2name - should still render with fallback."""
             input_ = [(1, [(1, "root", [[["f1"], 2]]), (2, None, [])])]
             # If name is None, fallback to string should not crash
             tree = tools.build_async_tree(input_)
-          Expand Down