Allow unicode surrogates to be parsed

RustPython · garychia · Mar 19, 2023 · Mar 19, 2023 · a787bbf1cf91c7e199c3616d6010912edaad12d0
commit a787bbf1cf91c7e199c3616d6010912edaad12d0
diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py
@@ -541,8 +541,6 @@ def test_bytes_comparison(self):
            self.assertEqual('abc' == bytearray(b'abc'), False)
            self.assertEqual('abc' != bytearray(b'abc'), True)

-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
    def test_comparison(self):
        # Comparisons:
        self.assertEqual('abc', 'abc')
@@ -766,8 +764,6 @@ def test_isidentifier_legacy(self):
            warnings.simplefilter('ignore', DeprecationWarning)
            self.assertTrue(_testcapi.unicode_legacy_string(u).isidentifier())

-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
    def test_isprintable(self):
        self.assertTrue("".isprintable())
        self.assertTrue(" ".isprintable())
@@ -783,8 +779,6 @@ def test_isprintable(self):
        self.assertTrue('\U0001F46F'.isprintable())
        self.assertFalse('\U000E0020'.isprintable())

-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
    def test_surrogates(self):
        for s in ('a\uD800b\uDFFF', 'a\uDFFFb\uD800',
                  'a\uD800b\uDFFFa', 'a\uDFFFb\uD800a'):
@@ -1727,8 +1721,6 @@ def test_codecs_utf7(self):
                                    'ill-formed sequence'):
            b'+@'.decode('utf-7')

-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
    def test_codecs_utf8(self):
        self.assertEqual(''.encode('utf-8'), b'')
        self.assertEqual('\u20ac'.encode('utf-8'), b'\xe2\x82\xac')

diff --git a/compiler/parser/src/string.rs b/compiler/parser/src/string.rs
@@ -81,7 +81,7 @@ impl<'a> StringParser<'a> {
            }
        }
        match p {
-            0xD800..=0xDFFF => Ok(std::char::REPLACEMENT_CHARACTER),
+            0xD800..=0xDFFF => Ok(unsafe { std::char::from_u32_unchecked(p) }),
            _ => std::char::from_u32(p).ok_or(unicode_error),
        }
    }