From 67f8db8800c4a6a06d2f55e5c470b17badb7823a Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Mon, 3 Feb 2020 21:30:43 +0000 Subject: [PATCH 01/25] Add gitpod configuration --- .gitpod.Dockerfile | 13 +++++++++++++ .gitpod.yml | 2 ++ 2 files changed, 15 insertions(+) create mode 100644 .gitpod.Dockerfile create mode 100644 .gitpod.yml diff --git a/.gitpod.Dockerfile b/.gitpod.Dockerfile new file mode 100644 index 00000000000..fe57919463d --- /dev/null +++ b/.gitpod.Dockerfile @@ -0,0 +1,13 @@ +FROM gitpod/workspace-full + +USER gitpod + +# Update Rust to the latest version +RUN rm -rf ~/.rustup && ~/.cargo/bin/rustup update stable + +# Set up wasm-pack and wasm32-unknown-unknown for rustpython_wasm +RUN export PATH=$HOME/.cargo/bin:$PATH && \ + curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh && \ + rustup target add wasm32-unknown-unknown + +USER root diff --git a/.gitpod.yml b/.gitpod.yml new file mode 100644 index 00000000000..527c76d392d --- /dev/null +++ b/.gitpod.yml @@ -0,0 +1,2 @@ +image: + file: .gitpod.Dockerfile From 4228c766888e6358b678dd085d22f1c8066e7279 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Tue, 4 Feb 2020 00:06:17 +0000 Subject: [PATCH 02/25] Add Gitpod badge --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 9dcb647b48c..3ac524fe9aa 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ A Python-3 (CPython >= 3.5.0) Interpreter written in Rust :snake: :scream: [![Crates.io](https://img.shields.io/crates/v/rustpython)](https://crates.io/crates/rustpython) [![dependency status](https://deps.rs/crate/rustpython/0.1.1/status.svg)](https://deps.rs/crate/rustpython/0.1.1) [![WAPM package](https://wapm.io/package/rustpython/badge.svg?style=flat)](https://wapm.io/package/rustpython) +[![Open in Gitpod](https://gitpod.io/button/open-in-gitpod.svg)](https://gitpod.io#https://github.com/RustPython/RustPython) ## Usage From 0aac4f6387f491d70ce2929f89b30c288a045baf Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Tue, 4 Feb 2020 00:08:24 +0000 Subject: [PATCH 03/25] Add GH actions badge --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3ac524fe9aa..fc399b983de 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ A Python-3 (CPython >= 3.5.0) Interpreter written in Rust :snake: :scream: [![Build Status](https://travis-ci.org/RustPython/RustPython.svg?branch=master)](https://travis-ci.org/RustPython/RustPython) [![Build Status](https://dev.azure.com/ryan0463/ryan/_apis/build/status/RustPython.RustPython?branchName=master)](https://dev.azure.com/ryan0463/ryan/_build/latest?definitionId=1&branchName=master) +[![Build Status](https://github.com/RustPython/RustPython/workflows/CI/badge.svg)](https://github.com/RustPython/RustPython/actions?query=workflow%3ACI) [![codecov](https://codecov.io/gh/RustPython/RustPython/branch/master/graph/badge.svg)](https://codecov.io/gh/RustPython/RustPython) [![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](https://opensource.org/licenses/MIT) [![Contributors](https://img.shields.io/github/contributors/RustPython/RustPython.svg)](https://github.com/RustPython/RustPython/graphs/contributors) @@ -15,7 +16,7 @@ A Python-3 (CPython >= 3.5.0) Interpreter written in Rust :snake: :scream: [![Crates.io](https://img.shields.io/crates/v/rustpython)](https://crates.io/crates/rustpython) [![dependency status](https://deps.rs/crate/rustpython/0.1.1/status.svg)](https://deps.rs/crate/rustpython/0.1.1) [![WAPM package](https://wapm.io/package/rustpython/badge.svg?style=flat)](https://wapm.io/package/rustpython) -[![Open in Gitpod](https://gitpod.io/button/open-in-gitpod.svg)](https://gitpod.io#https://github.com/RustPython/RustPython) +[![Open in Gitpod](https://img.shields.io/static/v1?label=Open%20in&message=Gitpod&color=1aa6e4&logo=gitpod)](https://gitpod.io#https://github.com/RustPython/RustPython) ## Usage From adad962365f68b53dccc966ba7a26531ae915013 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Tue, 4 Feb 2020 16:47:48 -0600 Subject: [PATCH 04/25] Install rls in .gitpod.Dockerfile --- .gitpod.Dockerfile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.gitpod.Dockerfile b/.gitpod.Dockerfile index fe57919463d..546e02fd17a 100644 --- a/.gitpod.Dockerfile +++ b/.gitpod.Dockerfile @@ -3,10 +3,11 @@ FROM gitpod/workspace-full USER gitpod # Update Rust to the latest version -RUN rm -rf ~/.rustup && ~/.cargo/bin/rustup update stable - -# Set up wasm-pack and wasm32-unknown-unknown for rustpython_wasm -RUN export PATH=$HOME/.cargo/bin:$PATH && \ +RUN rm -rf ~/.rustup && \ + export PATH=$HOME/.cargo/bin:$PATH && \ + rustup update stable && \ + rustup component add rls && \ + # Set up wasm-pack and wasm32-unknown-unknown for rustpython_wasm curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh && \ rustup target add wasm32-unknown-unknown From c84ab037b8e85825f76864e3d869d1a4bcbd07c9 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Mon, 10 Feb 2020 18:20:08 +0200 Subject: [PATCH 05/25] Add test_baseexception from CPython 3.8 --- Lib/test/exception_hierarchy.txt | 65 +++++++++++ Lib/test/test_baseexception.py | 183 +++++++++++++++++++++++++++++++ 2 files changed, 248 insertions(+) create mode 100644 Lib/test/exception_hierarchy.txt create mode 100644 Lib/test/test_baseexception.py diff --git a/Lib/test/exception_hierarchy.txt b/Lib/test/exception_hierarchy.txt new file mode 100644 index 00000000000..15f4491cf23 --- /dev/null +++ b/Lib/test/exception_hierarchy.txt @@ -0,0 +1,65 @@ +BaseException + +-- SystemExit + +-- KeyboardInterrupt + +-- GeneratorExit + +-- Exception + +-- StopIteration + +-- StopAsyncIteration + +-- ArithmeticError + | +-- FloatingPointError + | +-- OverflowError + | +-- ZeroDivisionError + +-- AssertionError + +-- AttributeError + +-- BufferError + +-- EOFError + +-- ImportError + | +-- ModuleNotFoundError + +-- LookupError + | +-- IndexError + | +-- KeyError + +-- MemoryError + +-- NameError + | +-- UnboundLocalError + +-- OSError + | +-- BlockingIOError + | +-- ChildProcessError + | +-- ConnectionError + | | +-- BrokenPipeError + | | +-- ConnectionAbortedError + | | +-- ConnectionRefusedError + | | +-- ConnectionResetError + | +-- FileExistsError + | +-- FileNotFoundError + | +-- InterruptedError + | +-- IsADirectoryError + | +-- NotADirectoryError + | +-- PermissionError + | +-- ProcessLookupError + | +-- TimeoutError + +-- ReferenceError + +-- RuntimeError + | +-- NotImplementedError + | +-- RecursionError + +-- SyntaxError + | +-- TargetScopeError + | +-- IndentationError + | +-- TabError + +-- SystemError + +-- TypeError + +-- ValueError + | +-- UnicodeError + | +-- UnicodeDecodeError + | +-- UnicodeEncodeError + | +-- UnicodeTranslateError + +-- Warning + +-- DeprecationWarning + +-- PendingDeprecationWarning + +-- RuntimeWarning + +-- SyntaxWarning + +-- UserWarning + +-- FutureWarning + +-- ImportWarning + +-- UnicodeWarning + +-- BytesWarning + +-- ResourceWarning diff --git a/Lib/test/test_baseexception.py b/Lib/test/test_baseexception.py new file mode 100644 index 00000000000..c32468269a7 --- /dev/null +++ b/Lib/test/test_baseexception.py @@ -0,0 +1,183 @@ +import unittest +import builtins +import os +from platform import system as platform_system + + +class ExceptionClassTests(unittest.TestCase): + + """Tests for anything relating to exception objects themselves (e.g., + inheritance hierarchy)""" + + def test_builtins_new_style(self): + self.assertTrue(issubclass(Exception, object)) + + def verify_instance_interface(self, ins): + for attr in ("args", "__str__", "__repr__"): + self.assertTrue(hasattr(ins, attr), + "%s missing %s attribute" % + (ins.__class__.__name__, attr)) + + def test_inheritance(self): + # Make sure the inheritance hierarchy matches the documentation + exc_set = set() + for object_ in builtins.__dict__.values(): + try: + if issubclass(object_, BaseException): + exc_set.add(object_.__name__) + except TypeError: + pass + + inheritance_tree = open(os.path.join(os.path.split(__file__)[0], + 'exception_hierarchy.txt')) + try: + superclass_name = inheritance_tree.readline().rstrip() + try: + last_exc = getattr(builtins, superclass_name) + except AttributeError: + self.fail("base class %s not a built-in" % superclass_name) + self.assertIn(superclass_name, exc_set, + '%s not found' % superclass_name) + exc_set.discard(superclass_name) + superclasses = [] # Loop will insert base exception + last_depth = 0 + for exc_line in inheritance_tree: + exc_line = exc_line.rstrip() + depth = exc_line.rindex('-') + exc_name = exc_line[depth+2:] # Slice past space + if '(' in exc_name: + paren_index = exc_name.index('(') + platform_name = exc_name[paren_index+1:-1] + exc_name = exc_name[:paren_index-1] # Slice off space + if platform_system() != platform_name: + exc_set.discard(exc_name) + continue + if '[' in exc_name: + left_bracket = exc_name.index('[') + exc_name = exc_name[:left_bracket-1] # cover space + try: + exc = getattr(builtins, exc_name) + except AttributeError: + self.fail("%s not a built-in exception" % exc_name) + if last_depth < depth: + superclasses.append((last_depth, last_exc)) + elif last_depth > depth: + while superclasses[-1][0] >= depth: + superclasses.pop() + self.assertTrue(issubclass(exc, superclasses[-1][1]), + "%s is not a subclass of %s" % (exc.__name__, + superclasses[-1][1].__name__)) + try: # Some exceptions require arguments; just skip them + self.verify_instance_interface(exc()) + except TypeError: + pass + self.assertIn(exc_name, exc_set) + exc_set.discard(exc_name) + last_exc = exc + last_depth = depth + finally: + inheritance_tree.close() + self.assertEqual(len(exc_set), 0, "%s not accounted for" % exc_set) + + interface_tests = ("length", "args", "str", "repr") + + def interface_test_driver(self, results): + for test_name, (given, expected) in zip(self.interface_tests, results): + self.assertEqual(given, expected, "%s: %s != %s" % (test_name, + given, expected)) + + def test_interface_single_arg(self): + # Make sure interface works properly when given a single argument + arg = "spam" + exc = Exception(arg) + results = ([len(exc.args), 1], [exc.args[0], arg], + [str(exc), str(arg)], + [repr(exc), '%s(%r)' % (exc.__class__.__name__, arg)]) + self.interface_test_driver(results) + + def test_interface_multi_arg(self): + # Make sure interface correct when multiple arguments given + arg_count = 3 + args = tuple(range(arg_count)) + exc = Exception(*args) + results = ([len(exc.args), arg_count], [exc.args, args], + [str(exc), str(args)], + [repr(exc), exc.__class__.__name__ + repr(exc.args)]) + self.interface_test_driver(results) + + def test_interface_no_arg(self): + # Make sure that with no args that interface is correct + exc = Exception() + results = ([len(exc.args), 0], [exc.args, tuple()], + [str(exc), ''], + [repr(exc), exc.__class__.__name__ + '()']) + self.interface_test_driver(results) + +class UsageTests(unittest.TestCase): + + """Test usage of exceptions""" + + def raise_fails(self, object_): + """Make sure that raising 'object_' triggers a TypeError.""" + try: + raise object_ + except TypeError: + return # What is expected. + self.fail("TypeError expected for raising %s" % type(object_)) + + def catch_fails(self, object_): + """Catching 'object_' should raise a TypeError.""" + try: + try: + raise Exception + except object_: + pass + except TypeError: + pass + except Exception: + self.fail("TypeError expected when catching %s" % type(object_)) + + try: + try: + raise Exception + except (object_,): + pass + except TypeError: + return + except Exception: + self.fail("TypeError expected when catching %s as specified in a " + "tuple" % type(object_)) + + def test_raise_new_style_non_exception(self): + # You cannot raise a new-style class that does not inherit from + # BaseException; the ability was not possible until BaseException's + # introduction so no need to support new-style objects that do not + # inherit from it. + class NewStyleClass(object): + pass + self.raise_fails(NewStyleClass) + self.raise_fails(NewStyleClass()) + + def test_raise_string(self): + # Raising a string raises TypeError. + self.raise_fails("spam") + + def test_catch_non_BaseException(self): + # Trying to catch an object that does not inherit from BaseException + # is not allowed. + class NonBaseException(object): + pass + self.catch_fails(NonBaseException) + self.catch_fails(NonBaseException()) + + def test_catch_BaseException_instance(self): + # Catching an instance of a BaseException subclass won't work. + self.catch_fails(BaseException()) + + def test_catch_string(self): + # Catching a string is bad. + self.catch_fails("spam") + + +if __name__ == '__main__': + unittest.main() From 2bd932c22b43687497ff7d1f881f6e4d2422b452 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Mon, 10 Feb 2020 18:24:03 +0200 Subject: [PATCH 06/25] Mark unsupported tests --- Lib/test/test_baseexception.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Lib/test/test_baseexception.py b/Lib/test/test_baseexception.py index c32468269a7..06e9ab3bd0b 100644 --- a/Lib/test/test_baseexception.py +++ b/Lib/test/test_baseexception.py @@ -18,6 +18,8 @@ def verify_instance_interface(self, ins): "%s missing %s attribute" % (ins.__class__.__name__, attr)) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_inheritance(self): # Make sure the inheritance hierarchy matches the documentation exc_set = set() @@ -86,6 +88,8 @@ def interface_test_driver(self, results): self.assertEqual(given, expected, "%s: %s != %s" % (test_name, given, expected)) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_interface_single_arg(self): # Make sure interface works properly when given a single argument arg = "spam" @@ -162,6 +166,8 @@ def test_raise_string(self): # Raising a string raises TypeError. self.raise_fails("spam") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_catch_non_BaseException(self): # Trying to catch an object that does not inherit from BaseException # is not allowed. From 5f1c62bc5744bf5a623b5153fa9bc222fe1592bb Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Mon, 10 Feb 2020 18:33:21 +0200 Subject: [PATCH 07/25] Add test_int_literal from CPython 3.8 --- Lib/test/test_int_literal.py | 143 +++++++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 Lib/test/test_int_literal.py diff --git a/Lib/test/test_int_literal.py b/Lib/test/test_int_literal.py new file mode 100644 index 00000000000..bf725710d55 --- /dev/null +++ b/Lib/test/test_int_literal.py @@ -0,0 +1,143 @@ +"""Test correct treatment of hex/oct constants. + +This is complex because of changes due to PEP 237. +""" + +import unittest + +class TestHexOctBin(unittest.TestCase): + + def test_hex_baseline(self): + # A few upper/lowercase tests + self.assertEqual(0x0, 0X0) + self.assertEqual(0x1, 0X1) + self.assertEqual(0x123456789abcdef, 0X123456789abcdef) + # Baseline tests + self.assertEqual(0x0, 0) + self.assertEqual(0x10, 16) + self.assertEqual(0x7fffffff, 2147483647) + self.assertEqual(0x7fffffffffffffff, 9223372036854775807) + # Ditto with a minus sign and parentheses + self.assertEqual(-(0x0), 0) + self.assertEqual(-(0x10), -16) + self.assertEqual(-(0x7fffffff), -2147483647) + self.assertEqual(-(0x7fffffffffffffff), -9223372036854775807) + # Ditto with a minus sign and NO parentheses + self.assertEqual(-0x0, 0) + self.assertEqual(-0x10, -16) + self.assertEqual(-0x7fffffff, -2147483647) + self.assertEqual(-0x7fffffffffffffff, -9223372036854775807) + + def test_hex_unsigned(self): + # Positive constants + self.assertEqual(0x80000000, 2147483648) + self.assertEqual(0xffffffff, 4294967295) + # Ditto with a minus sign and parentheses + self.assertEqual(-(0x80000000), -2147483648) + self.assertEqual(-(0xffffffff), -4294967295) + # Ditto with a minus sign and NO parentheses + # This failed in Python 2.2 through 2.2.2 and in 2.3a1 + self.assertEqual(-0x80000000, -2147483648) + self.assertEqual(-0xffffffff, -4294967295) + + # Positive constants + self.assertEqual(0x8000000000000000, 9223372036854775808) + self.assertEqual(0xffffffffffffffff, 18446744073709551615) + # Ditto with a minus sign and parentheses + self.assertEqual(-(0x8000000000000000), -9223372036854775808) + self.assertEqual(-(0xffffffffffffffff), -18446744073709551615) + # Ditto with a minus sign and NO parentheses + # This failed in Python 2.2 through 2.2.2 and in 2.3a1 + self.assertEqual(-0x8000000000000000, -9223372036854775808) + self.assertEqual(-0xffffffffffffffff, -18446744073709551615) + + def test_oct_baseline(self): + # A few upper/lowercase tests + self.assertEqual(0o0, 0O0) + self.assertEqual(0o1, 0O1) + self.assertEqual(0o1234567, 0O1234567) + # Baseline tests + self.assertEqual(0o0, 0) + self.assertEqual(0o20, 16) + self.assertEqual(0o17777777777, 2147483647) + self.assertEqual(0o777777777777777777777, 9223372036854775807) + # Ditto with a minus sign and parentheses + self.assertEqual(-(0o0), 0) + self.assertEqual(-(0o20), -16) + self.assertEqual(-(0o17777777777), -2147483647) + self.assertEqual(-(0o777777777777777777777), -9223372036854775807) + # Ditto with a minus sign and NO parentheses + self.assertEqual(-0o0, 0) + self.assertEqual(-0o20, -16) + self.assertEqual(-0o17777777777, -2147483647) + self.assertEqual(-0o777777777777777777777, -9223372036854775807) + + def test_oct_unsigned(self): + # Positive constants + self.assertEqual(0o20000000000, 2147483648) + self.assertEqual(0o37777777777, 4294967295) + # Ditto with a minus sign and parentheses + self.assertEqual(-(0o20000000000), -2147483648) + self.assertEqual(-(0o37777777777), -4294967295) + # Ditto with a minus sign and NO parentheses + # This failed in Python 2.2 through 2.2.2 and in 2.3a1 + self.assertEqual(-0o20000000000, -2147483648) + self.assertEqual(-0o37777777777, -4294967295) + + # Positive constants + self.assertEqual(0o1000000000000000000000, 9223372036854775808) + self.assertEqual(0o1777777777777777777777, 18446744073709551615) + # Ditto with a minus sign and parentheses + self.assertEqual(-(0o1000000000000000000000), -9223372036854775808) + self.assertEqual(-(0o1777777777777777777777), -18446744073709551615) + # Ditto with a minus sign and NO parentheses + # This failed in Python 2.2 through 2.2.2 and in 2.3a1 + self.assertEqual(-0o1000000000000000000000, -9223372036854775808) + self.assertEqual(-0o1777777777777777777777, -18446744073709551615) + + def test_bin_baseline(self): + # A few upper/lowercase tests + self.assertEqual(0b0, 0B0) + self.assertEqual(0b1, 0B1) + self.assertEqual(0b10101010101, 0B10101010101) + # Baseline tests + self.assertEqual(0b0, 0) + self.assertEqual(0b10000, 16) + self.assertEqual(0b1111111111111111111111111111111, 2147483647) + self.assertEqual(0b111111111111111111111111111111111111111111111111111111111111111, 9223372036854775807) + # Ditto with a minus sign and parentheses + self.assertEqual(-(0b0), 0) + self.assertEqual(-(0b10000), -16) + self.assertEqual(-(0b1111111111111111111111111111111), -2147483647) + self.assertEqual(-(0b111111111111111111111111111111111111111111111111111111111111111), -9223372036854775807) + # Ditto with a minus sign and NO parentheses + self.assertEqual(-0b0, 0) + self.assertEqual(-0b10000, -16) + self.assertEqual(-0b1111111111111111111111111111111, -2147483647) + self.assertEqual(-0b111111111111111111111111111111111111111111111111111111111111111, -9223372036854775807) + + def test_bin_unsigned(self): + # Positive constants + self.assertEqual(0b10000000000000000000000000000000, 2147483648) + self.assertEqual(0b11111111111111111111111111111111, 4294967295) + # Ditto with a minus sign and parentheses + self.assertEqual(-(0b10000000000000000000000000000000), -2147483648) + self.assertEqual(-(0b11111111111111111111111111111111), -4294967295) + # Ditto with a minus sign and NO parentheses + # This failed in Python 2.2 through 2.2.2 and in 2.3a1 + self.assertEqual(-0b10000000000000000000000000000000, -2147483648) + self.assertEqual(-0b11111111111111111111111111111111, -4294967295) + + # Positive constants + self.assertEqual(0b1000000000000000000000000000000000000000000000000000000000000000, 9223372036854775808) + self.assertEqual(0b1111111111111111111111111111111111111111111111111111111111111111, 18446744073709551615) + # Ditto with a minus sign and parentheses + self.assertEqual(-(0b1000000000000000000000000000000000000000000000000000000000000000), -9223372036854775808) + self.assertEqual(-(0b1111111111111111111111111111111111111111111111111111111111111111), -18446744073709551615) + # Ditto with a minus sign and NO parentheses + # This failed in Python 2.2 through 2.2.2 and in 2.3a1 + self.assertEqual(-0b1000000000000000000000000000000000000000000000000000000000000000, -9223372036854775808) + self.assertEqual(-0b1111111111111111111111111111111111111111111111111111111111111111, -18446744073709551615) + +if __name__ == "__main__": + unittest.main() From 5c2b25c24a8c80cac08d67776bb10d09834a832b Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Mon, 10 Feb 2020 18:34:46 +0200 Subject: [PATCH 08/25] Add test_int from CPython 3.8 --- Lib/test/test_int.py | 521 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 521 insertions(+) create mode 100644 Lib/test/test_int.py diff --git a/Lib/test/test_int.py b/Lib/test/test_int.py new file mode 100644 index 00000000000..307ca36bb4f --- /dev/null +++ b/Lib/test/test_int.py @@ -0,0 +1,521 @@ +import sys + +import unittest +from test import support +from test.test_grammar import (VALID_UNDERSCORE_LITERALS, + INVALID_UNDERSCORE_LITERALS) + +L = [ + ('0', 0), + ('1', 1), + ('9', 9), + ('10', 10), + ('99', 99), + ('100', 100), + ('314', 314), + (' 314', 314), + ('314 ', 314), + (' \t\t 314 \t\t ', 314), + (repr(sys.maxsize), sys.maxsize), + (' 1x', ValueError), + (' 1 ', 1), + (' 1\02 ', ValueError), + ('', ValueError), + (' ', ValueError), + (' \t\t ', ValueError), + ("\u0200", ValueError) +] + +class IntSubclass(int): + pass + +class IntTestCases(unittest.TestCase): + + def test_basic(self): + self.assertEqual(int(314), 314) + self.assertEqual(int(3.14), 3) + # Check that conversion from float truncates towards zero + self.assertEqual(int(-3.14), -3) + self.assertEqual(int(3.9), 3) + self.assertEqual(int(-3.9), -3) + self.assertEqual(int(3.5), 3) + self.assertEqual(int(-3.5), -3) + self.assertEqual(int("-3"), -3) + self.assertEqual(int(" -3 "), -3) + self.assertEqual(int("\N{EM SPACE}-3\N{EN SPACE}"), -3) + # Different base: + self.assertEqual(int("10",16), 16) + # Test conversion from strings and various anomalies + for s, v in L: + for sign in "", "+", "-": + for prefix in "", " ", "\t", " \t\t ": + ss = prefix + sign + s + vv = v + if sign == "-" and v is not ValueError: + vv = -v + try: + self.assertEqual(int(ss), vv) + except ValueError: + pass + + s = repr(-1-sys.maxsize) + x = int(s) + self.assertEqual(x+1, -sys.maxsize) + self.assertIsInstance(x, int) + # should return int + self.assertEqual(int(s[1:]), sys.maxsize+1) + + # should return int + x = int(1e100) + self.assertIsInstance(x, int) + x = int(-1e100) + self.assertIsInstance(x, int) + + + # SF bug 434186: 0x80000000/2 != 0x80000000>>1. + # Worked by accident in Windows release build, but failed in debug build. + # Failed in all Linux builds. + x = -1-sys.maxsize + self.assertEqual(x >> 1, x//2) + + x = int('1' * 600) + self.assertIsInstance(x, int) + + + self.assertRaises(TypeError, int, 1, 12) + + self.assertEqual(int('0o123', 0), 83) + self.assertEqual(int('0x123', 16), 291) + + # Bug 1679: "0x" is not a valid hex literal + self.assertRaises(ValueError, int, "0x", 16) + self.assertRaises(ValueError, int, "0x", 0) + + self.assertRaises(ValueError, int, "0o", 8) + self.assertRaises(ValueError, int, "0o", 0) + + self.assertRaises(ValueError, int, "0b", 2) + self.assertRaises(ValueError, int, "0b", 0) + + # SF bug 1334662: int(string, base) wrong answers + # Various representations of 2**32 evaluated to 0 + # rather than 2**32 in previous versions + + self.assertEqual(int('100000000000000000000000000000000', 2), 4294967296) + self.assertEqual(int('102002022201221111211', 3), 4294967296) + self.assertEqual(int('10000000000000000', 4), 4294967296) + self.assertEqual(int('32244002423141', 5), 4294967296) + self.assertEqual(int('1550104015504', 6), 4294967296) + self.assertEqual(int('211301422354', 7), 4294967296) + self.assertEqual(int('40000000000', 8), 4294967296) + self.assertEqual(int('12068657454', 9), 4294967296) + self.assertEqual(int('4294967296', 10), 4294967296) + self.assertEqual(int('1904440554', 11), 4294967296) + self.assertEqual(int('9ba461594', 12), 4294967296) + self.assertEqual(int('535a79889', 13), 4294967296) + self.assertEqual(int('2ca5b7464', 14), 4294967296) + self.assertEqual(int('1a20dcd81', 15), 4294967296) + self.assertEqual(int('100000000', 16), 4294967296) + self.assertEqual(int('a7ffda91', 17), 4294967296) + self.assertEqual(int('704he7g4', 18), 4294967296) + self.assertEqual(int('4f5aff66', 19), 4294967296) + self.assertEqual(int('3723ai4g', 20), 4294967296) + self.assertEqual(int('281d55i4', 21), 4294967296) + self.assertEqual(int('1fj8b184', 22), 4294967296) + self.assertEqual(int('1606k7ic', 23), 4294967296) + self.assertEqual(int('mb994ag', 24), 4294967296) + self.assertEqual(int('hek2mgl', 25), 4294967296) + self.assertEqual(int('dnchbnm', 26), 4294967296) + self.assertEqual(int('b28jpdm', 27), 4294967296) + self.assertEqual(int('8pfgih4', 28), 4294967296) + self.assertEqual(int('76beigg', 29), 4294967296) + self.assertEqual(int('5qmcpqg', 30), 4294967296) + self.assertEqual(int('4q0jto4', 31), 4294967296) + self.assertEqual(int('4000000', 32), 4294967296) + self.assertEqual(int('3aokq94', 33), 4294967296) + self.assertEqual(int('2qhxjli', 34), 4294967296) + self.assertEqual(int('2br45qb', 35), 4294967296) + self.assertEqual(int('1z141z4', 36), 4294967296) + + # tests with base 0 + # this fails on 3.0, but in 2.x the old octal syntax is allowed + self.assertEqual(int(' 0o123 ', 0), 83) + self.assertEqual(int(' 0o123 ', 0), 83) + self.assertEqual(int('000', 0), 0) + self.assertEqual(int('0o123', 0), 83) + self.assertEqual(int('0x123', 0), 291) + self.assertEqual(int('0b100', 0), 4) + self.assertEqual(int(' 0O123 ', 0), 83) + self.assertEqual(int(' 0X123 ', 0), 291) + self.assertEqual(int(' 0B100 ', 0), 4) + + # without base still base 10 + self.assertEqual(int('0123'), 123) + self.assertEqual(int('0123', 10), 123) + + # tests with prefix and base != 0 + self.assertEqual(int('0x123', 16), 291) + self.assertEqual(int('0o123', 8), 83) + self.assertEqual(int('0b100', 2), 4) + self.assertEqual(int('0X123', 16), 291) + self.assertEqual(int('0O123', 8), 83) + self.assertEqual(int('0B100', 2), 4) + + # the code has special checks for the first character after the + # type prefix + self.assertRaises(ValueError, int, '0b2', 2) + self.assertRaises(ValueError, int, '0b02', 2) + self.assertRaises(ValueError, int, '0B2', 2) + self.assertRaises(ValueError, int, '0B02', 2) + self.assertRaises(ValueError, int, '0o8', 8) + self.assertRaises(ValueError, int, '0o08', 8) + self.assertRaises(ValueError, int, '0O8', 8) + self.assertRaises(ValueError, int, '0O08', 8) + self.assertRaises(ValueError, int, '0xg', 16) + self.assertRaises(ValueError, int, '0x0g', 16) + self.assertRaises(ValueError, int, '0Xg', 16) + self.assertRaises(ValueError, int, '0X0g', 16) + + # SF bug 1334662: int(string, base) wrong answers + # Checks for proper evaluation of 2**32 + 1 + self.assertEqual(int('100000000000000000000000000000001', 2), 4294967297) + self.assertEqual(int('102002022201221111212', 3), 4294967297) + self.assertEqual(int('10000000000000001', 4), 4294967297) + self.assertEqual(int('32244002423142', 5), 4294967297) + self.assertEqual(int('1550104015505', 6), 4294967297) + self.assertEqual(int('211301422355', 7), 4294967297) + self.assertEqual(int('40000000001', 8), 4294967297) + self.assertEqual(int('12068657455', 9), 4294967297) + self.assertEqual(int('4294967297', 10), 4294967297) + self.assertEqual(int('1904440555', 11), 4294967297) + self.assertEqual(int('9ba461595', 12), 4294967297) + self.assertEqual(int('535a7988a', 13), 4294967297) + self.assertEqual(int('2ca5b7465', 14), 4294967297) + self.assertEqual(int('1a20dcd82', 15), 4294967297) + self.assertEqual(int('100000001', 16), 4294967297) + self.assertEqual(int('a7ffda92', 17), 4294967297) + self.assertEqual(int('704he7g5', 18), 4294967297) + self.assertEqual(int('4f5aff67', 19), 4294967297) + self.assertEqual(int('3723ai4h', 20), 4294967297) + self.assertEqual(int('281d55i5', 21), 4294967297) + self.assertEqual(int('1fj8b185', 22), 4294967297) + self.assertEqual(int('1606k7id', 23), 4294967297) + self.assertEqual(int('mb994ah', 24), 4294967297) + self.assertEqual(int('hek2mgm', 25), 4294967297) + self.assertEqual(int('dnchbnn', 26), 4294967297) + self.assertEqual(int('b28jpdn', 27), 4294967297) + self.assertEqual(int('8pfgih5', 28), 4294967297) + self.assertEqual(int('76beigh', 29), 4294967297) + self.assertEqual(int('5qmcpqh', 30), 4294967297) + self.assertEqual(int('4q0jto5', 31), 4294967297) + self.assertEqual(int('4000001', 32), 4294967297) + self.assertEqual(int('3aokq95', 33), 4294967297) + self.assertEqual(int('2qhxjlj', 34), 4294967297) + self.assertEqual(int('2br45qc', 35), 4294967297) + self.assertEqual(int('1z141z5', 36), 4294967297) + + def test_underscores(self): + for lit in VALID_UNDERSCORE_LITERALS: + if any(ch in lit for ch in '.eEjJ'): + continue + self.assertEqual(int(lit, 0), eval(lit)) + self.assertEqual(int(lit, 0), int(lit.replace('_', ''), 0)) + for lit in INVALID_UNDERSCORE_LITERALS: + if any(ch in lit for ch in '.eEjJ'): + continue + self.assertRaises(ValueError, int, lit, 0) + # Additional test cases with bases != 0, only for the constructor: + self.assertEqual(int("1_00", 3), 9) + self.assertEqual(int("0_100"), 100) # not valid as a literal! + self.assertEqual(int(b"1_00"), 100) # byte underscore + self.assertRaises(ValueError, int, "_100") + self.assertRaises(ValueError, int, "+_100") + self.assertRaises(ValueError, int, "1__00") + self.assertRaises(ValueError, int, "100_") + + @support.cpython_only + def test_small_ints(self): + # Bug #3236: Return small longs from PyLong_FromString + self.assertIs(int('10'), 10) + self.assertIs(int('-1'), -1) + self.assertIs(int(b'10'), 10) + self.assertIs(int(b'-1'), -1) + + def test_no_args(self): + self.assertEqual(int(), 0) + + def test_keyword_args(self): + # Test invoking int() using keyword arguments. + self.assertEqual(int('100', base=2), 4) + with self.assertRaisesRegex(TypeError, 'keyword argument'): + int(x=1.2) + with self.assertRaisesRegex(TypeError, 'keyword argument'): + int(x='100', base=2) + self.assertRaises(TypeError, int, base=10) + self.assertRaises(TypeError, int, base=0) + + def test_int_base_limits(self): + """Testing the supported limits of the int() base parameter.""" + self.assertEqual(int('0', 5), 0) + with self.assertRaises(ValueError): + int('0', 1) + with self.assertRaises(ValueError): + int('0', 37) + with self.assertRaises(ValueError): + int('0', -909) # An old magic value base from Python 2. + with self.assertRaises(ValueError): + int('0', base=0-(2**234)) + with self.assertRaises(ValueError): + int('0', base=2**234) + # Bases 2 through 36 are supported. + for base in range(2,37): + self.assertEqual(int('0', base=base), 0) + + def test_int_base_bad_types(self): + """Not integer types are not valid bases; issue16772.""" + with self.assertRaises(TypeError): + int('0', 5.5) + with self.assertRaises(TypeError): + int('0', 5.0) + + def test_int_base_indexable(self): + class MyIndexable(object): + def __init__(self, value): + self.value = value + def __index__(self): + return self.value + + # Check out of range bases. + for base in 2**100, -2**100, 1, 37: + with self.assertRaises(ValueError): + int('43', base) + + # Check in-range bases. + self.assertEqual(int('101', base=MyIndexable(2)), 5) + self.assertEqual(int('101', base=MyIndexable(10)), 101) + self.assertEqual(int('101', base=MyIndexable(36)), 1 + 36**2) + + def test_non_numeric_input_types(self): + # Test possible non-numeric types for the argument x, including + # subclasses of the explicitly documented accepted types. + class CustomStr(str): pass + class CustomBytes(bytes): pass + class CustomByteArray(bytearray): pass + + factories = [ + bytes, + bytearray, + lambda b: CustomStr(b.decode()), + CustomBytes, + CustomByteArray, + memoryview, + ] + try: + from array import array + except ImportError: + pass + else: + factories.append(lambda b: array('B', b)) + + for f in factories: + x = f(b'100') + with self.subTest(type(x)): + self.assertEqual(int(x), 100) + if isinstance(x, (str, bytes, bytearray)): + self.assertEqual(int(x, 2), 4) + else: + msg = "can't convert non-string" + with self.assertRaisesRegex(TypeError, msg): + int(x, 2) + with self.assertRaisesRegex(ValueError, 'invalid literal'): + int(f(b'A' * 0x10)) + + def test_int_memoryview(self): + self.assertEqual(int(memoryview(b'123')[1:3]), 23) + self.assertEqual(int(memoryview(b'123\x00')[1:3]), 23) + self.assertEqual(int(memoryview(b'123 ')[1:3]), 23) + self.assertEqual(int(memoryview(b'123A')[1:3]), 23) + self.assertEqual(int(memoryview(b'1234')[1:3]), 23) + + def test_string_float(self): + self.assertRaises(ValueError, int, '1.2') + + def test_intconversion(self): + # Test __int__() + class ClassicMissingMethods: + pass + self.assertRaises(TypeError, int, ClassicMissingMethods()) + + class MissingMethods(object): + pass + self.assertRaises(TypeError, int, MissingMethods()) + + class Foo0: + def __int__(self): + return 42 + + self.assertEqual(int(Foo0()), 42) + + class Classic: + pass + for base in (object, Classic): + class IntOverridesTrunc(base): + def __int__(self): + return 42 + def __trunc__(self): + return -12 + self.assertEqual(int(IntOverridesTrunc()), 42) + + class JustTrunc(base): + def __trunc__(self): + return 42 + self.assertEqual(int(JustTrunc()), 42) + + class ExceptionalTrunc(base): + def __trunc__(self): + 1 / 0 + with self.assertRaises(ZeroDivisionError): + int(ExceptionalTrunc()) + + for trunc_result_base in (object, Classic): + class Integral(trunc_result_base): + def __int__(self): + return 42 + + class TruncReturnsNonInt(base): + def __trunc__(self): + return Integral() + with self.assertWarns(DeprecationWarning): + self.assertEqual(int(TruncReturnsNonInt()), 42) + + class NonIntegral(trunc_result_base): + def __trunc__(self): + # Check that we avoid infinite recursion. + return NonIntegral() + + class TruncReturnsNonIntegral(base): + def __trunc__(self): + return NonIntegral() + try: + int(TruncReturnsNonIntegral()) + except TypeError as e: + self.assertEqual(str(e), + "__trunc__ returned non-Integral" + " (type NonIntegral)") + else: + self.fail("Failed to raise TypeError with %s" % + ((base, trunc_result_base),)) + + # Regression test for bugs.python.org/issue16060. + class BadInt(trunc_result_base): + def __int__(self): + return 42.0 + + class TruncReturnsBadInt(base): + def __trunc__(self): + return BadInt() + + with self.assertRaises(TypeError): + int(TruncReturnsBadInt()) + + def test_int_subclass_with_int(self): + class MyInt(int): + def __int__(self): + return 42 + + class BadInt(int): + def __int__(self): + return 42.0 + + my_int = MyInt(7) + self.assertEqual(my_int, 7) + self.assertEqual(int(my_int), 42) + + self.assertRaises(TypeError, int, BadInt()) + + def test_int_returns_int_subclass(self): + class BadInt: + def __int__(self): + return True + + class BadInt2(int): + def __int__(self): + return True + + class TruncReturnsBadInt: + def __trunc__(self): + return BadInt() + + class TruncReturnsIntSubclass: + def __trunc__(self): + return True + + bad_int = BadInt() + with self.assertWarns(DeprecationWarning): + n = int(bad_int) + self.assertEqual(n, 1) + self.assertIs(type(n), int) + + bad_int = BadInt2() + with self.assertWarns(DeprecationWarning): + n = int(bad_int) + self.assertEqual(n, 1) + self.assertIs(type(n), int) + + bad_int = TruncReturnsBadInt() + with self.assertWarns(DeprecationWarning): + n = int(bad_int) + self.assertEqual(n, 1) + self.assertIs(type(n), int) + + good_int = TruncReturnsIntSubclass() + n = int(good_int) + self.assertEqual(n, 1) + self.assertIs(type(n), int) + n = IntSubclass(good_int) + self.assertEqual(n, 1) + self.assertIs(type(n), IntSubclass) + + def test_error_message(self): + def check(s, base=None): + with self.assertRaises(ValueError, + msg="int(%r, %r)" % (s, base)) as cm: + if base is None: + int(s) + else: + int(s, base) + self.assertEqual(cm.exception.args[0], + "invalid literal for int() with base %d: %r" % + (10 if base is None else base, s)) + + check('\xbd') + check('123\xbd') + check(' 123 456 ') + + check('123\x00') + # SF bug 1545497: embedded NULs were not detected with explicit base + check('123\x00', 10) + check('123\x00 245', 20) + check('123\x00 245', 16) + check('123\x00245', 20) + check('123\x00245', 16) + # byte string with embedded NUL + check(b'123\x00') + check(b'123\x00', 10) + # non-UTF-8 byte string + check(b'123\xbd') + check(b'123\xbd', 10) + # lone surrogate in Unicode string + check('123\ud800') + check('123\ud800', 10) + + def test_issue31619(self): + self.assertEqual(int('1_0_1_0_1_0_1_0_1_0_1_0_1_0_1_0_1_0_1_0_1_0_1_0_1_0_1_0_1_0_1', 2), + 0b1010101010101010101010101010101) + self.assertEqual(int('1_2_3_4_5_6_7_0_1_2_3', 8), 0o12345670123) + self.assertEqual(int('1_2_3_4_5_6_7_8_9', 16), 0x123456789) + self.assertEqual(int('1_2_3_4_5_6_7', 32), 1144132807) + + +if __name__ == "__main__": + unittest.main() From 2e0691366038f02ce89c5218910efad517788e43 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Mon, 10 Feb 2020 18:38:11 +0200 Subject: [PATCH 09/25] Mark unsupported tests --- Lib/test/test_int.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_int.py b/Lib/test/test_int.py index 307ca36bb4f..29dfe99203f 100644 --- a/Lib/test/test_int.py +++ b/Lib/test/test_int.py @@ -2,8 +2,8 @@ import unittest from test import support -from test.test_grammar import (VALID_UNDERSCORE_LITERALS, - INVALID_UNDERSCORE_LITERALS) +# from test.test_grammar import (VALID_UNDERSCORE_LITERALS, +# INVALID_UNDERSCORE_LITERALS) L = [ ('0', 0), @@ -31,6 +31,7 @@ class IntSubclass(int): class IntTestCases(unittest.TestCase): + @unittest.skip("TODO: RUSTPYTHON") def test_basic(self): self.assertEqual(int(314), 314) self.assertEqual(int(3.14), 3) @@ -214,6 +215,7 @@ def test_basic(self): self.assertEqual(int('2br45qc', 35), 4294967297) self.assertEqual(int('1z141z5', 36), 4294967297) + @unittest.skip("TODO: RUSTPYTHON") def test_underscores(self): for lit in VALID_UNDERSCORE_LITERALS: if any(ch in lit for ch in '.eEjJ'): @@ -278,6 +280,7 @@ def test_int_base_bad_types(self): with self.assertRaises(TypeError): int('0', 5.0) + @unittest.skip("TODO: RUSTPYTHON") def test_int_base_indexable(self): class MyIndexable(object): def __init__(self, value): @@ -295,6 +298,7 @@ def __index__(self): self.assertEqual(int('101', base=MyIndexable(10)), 101) self.assertEqual(int('101', base=MyIndexable(36)), 1 + 36**2) + @unittest.skip("TODO: RUSTPYTHON") def test_non_numeric_input_types(self): # Test possible non-numeric types for the argument x, including # subclasses of the explicitly documented accepted types. @@ -340,6 +344,7 @@ def test_int_memoryview(self): def test_string_float(self): self.assertRaises(ValueError, int, '1.2') + @unittest.skip("TODO: RUSTPYTHON") def test_intconversion(self): # Test __int__() class ClassicMissingMethods: @@ -433,6 +438,8 @@ def __int__(self): self.assertRaises(TypeError, int, BadInt()) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_int_returns_int_subclass(self): class BadInt: def __int__(self): @@ -476,6 +483,8 @@ def __trunc__(self): self.assertEqual(n, 1) self.assertIs(type(n), IntSubclass) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_error_message(self): def check(s, base=None): with self.assertRaises(ValueError, From 63f00f46ba04a9780c10940eb7a109f25ba88e8e Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Mon, 10 Feb 2020 18:43:46 +0200 Subject: [PATCH 10/25] Add test_opcodes from CPython 3.8 --- Lib/test/test_opcodes.py | 138 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 138 insertions(+) create mode 100644 Lib/test/test_opcodes.py diff --git a/Lib/test/test_opcodes.py b/Lib/test/test_opcodes.py new file mode 100644 index 00000000000..527aca664d3 --- /dev/null +++ b/Lib/test/test_opcodes.py @@ -0,0 +1,138 @@ +# Python test set -- part 2, opcodes + +import unittest +from test import ann_module, support + +class OpcodeTest(unittest.TestCase): + + def test_try_inside_for_loop(self): + n = 0 + for i in range(10): + n = n+i + try: 1/0 + except NameError: pass + except ZeroDivisionError: pass + except TypeError: pass + try: pass + except: pass + try: pass + finally: pass + n = n+i + if n != 90: + self.fail('try inside for') + + def test_setup_annotations_line(self): + # check that SETUP_ANNOTATIONS does not create spurious line numbers + try: + with open(ann_module.__file__) as f: + txt = f.read() + co = compile(txt, ann_module.__file__, 'exec') + self.assertEqual(co.co_firstlineno, 3) + except OSError: + pass + + def test_no_annotations_if_not_needed(self): + class C: pass + with self.assertRaises(AttributeError): + C.__annotations__ + + def test_use_existing_annotations(self): + ns = {'__annotations__': {1: 2}} + exec('x: int', ns) + self.assertEqual(ns['__annotations__'], {'x': int, 1: 2}) + + def test_do_not_recreate_annotations(self): + # Don't rely on the existence of the '__annotations__' global. + with support.swap_item(globals(), '__annotations__', {}): + del globals()['__annotations__'] + class C: + del __annotations__ + with self.assertRaises(NameError): + x: int + + def test_raise_class_exceptions(self): + + class AClass(Exception): pass + class BClass(AClass): pass + class CClass(Exception): pass + class DClass(AClass): + def __init__(self, ignore): + pass + + try: raise AClass() + except: pass + + try: raise AClass() + except AClass: pass + + try: raise BClass() + except AClass: pass + + try: raise BClass() + except CClass: self.fail() + except: pass + + a = AClass() + b = BClass() + + try: + raise b + except AClass as v: + self.assertEqual(v, b) + else: + self.fail("no exception") + + # not enough arguments + ##try: raise BClass, a + ##except TypeError: pass + ##else: self.fail("no exception") + + try: raise DClass(a) + except DClass as v: + self.assertIsInstance(v, DClass) + else: + self.fail("no exception") + + def test_compare_function_objects(self): + + f = eval('lambda: None') + g = eval('lambda: None') + self.assertNotEqual(f, g) + + f = eval('lambda a: a') + g = eval('lambda a: a') + self.assertNotEqual(f, g) + + f = eval('lambda a=1: a') + g = eval('lambda a=1: a') + self.assertNotEqual(f, g) + + f = eval('lambda: 0') + g = eval('lambda: 1') + self.assertNotEqual(f, g) + + f = eval('lambda: None') + g = eval('lambda a: None') + self.assertNotEqual(f, g) + + f = eval('lambda a: None') + g = eval('lambda b: None') + self.assertNotEqual(f, g) + + f = eval('lambda a: None') + g = eval('lambda a=None: None') + self.assertNotEqual(f, g) + + f = eval('lambda a=0: None') + g = eval('lambda a=1: None') + self.assertNotEqual(f, g) + + def test_modulo_of_string_subclasses(self): + class MyString(str): + def __mod__(self, value): + return 42 + self.assertEqual(MyString() % 3, 42) + + +if __name__ == '__main__': + unittest.main() From df99dd1ea49d3d0cc7cec1339165f7c2f2161209 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Mon, 10 Feb 2020 18:45:32 +0200 Subject: [PATCH 11/25] Mark unsupported tests --- Lib/test/test_opcodes.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_opcodes.py b/Lib/test/test_opcodes.py index 527aca664d3..d37dc6e88fb 100644 --- a/Lib/test/test_opcodes.py +++ b/Lib/test/test_opcodes.py @@ -1,7 +1,7 @@ # Python test set -- part 2, opcodes import unittest -from test import ann_module, support +from test import support #,ann_module class OpcodeTest(unittest.TestCase): @@ -21,6 +21,7 @@ def test_try_inside_for_loop(self): if n != 90: self.fail('try inside for') + @unittest.skip("TODO: RUSTPYTHON") def test_setup_annotations_line(self): # check that SETUP_ANNOTATIONS does not create spurious line numbers try: @@ -36,11 +37,13 @@ class C: pass with self.assertRaises(AttributeError): C.__annotations__ + @unittest.skip("TODO: RUSTPYTHON") def test_use_existing_annotations(self): ns = {'__annotations__': {1: 2}} exec('x: int', ns) self.assertEqual(ns['__annotations__'], {'x': int, 1: 2}) + @unittest.skip("TODO: RUSTPYTHON") def test_do_not_recreate_annotations(self): # Don't rely on the existence of the '__annotations__' global. with support.swap_item(globals(), '__annotations__', {}): From e5ea09457597175dd4a768053d49c81c483153fe Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Mon, 10 Feb 2020 18:51:33 +0200 Subject: [PATCH 12/25] Add test_pow from CPython 3.8 --- Lib/test/test_pow.py | 123 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 Lib/test/test_pow.py diff --git a/Lib/test/test_pow.py b/Lib/test/test_pow.py new file mode 100644 index 00000000000..cac1ae5ea2d --- /dev/null +++ b/Lib/test/test_pow.py @@ -0,0 +1,123 @@ +import unittest + +class PowTest(unittest.TestCase): + + def powtest(self, type): + if type != float: + for i in range(-1000, 1000): + self.assertEqual(pow(type(i), 0), 1) + self.assertEqual(pow(type(i), 1), type(i)) + self.assertEqual(pow(type(0), 1), type(0)) + self.assertEqual(pow(type(1), 1), type(1)) + + for i in range(-100, 100): + self.assertEqual(pow(type(i), 3), i*i*i) + + pow2 = 1 + for i in range(0, 31): + self.assertEqual(pow(2, i), pow2) + if i != 30 : pow2 = pow2*2 + + for othertype in (int,): + for i in list(range(-10, 0)) + list(range(1, 10)): + ii = type(i) + for j in range(1, 11): + jj = -othertype(j) + pow(ii, jj) + + for othertype in int, float: + for i in range(1, 100): + zero = type(0) + exp = -othertype(i/10.0) + if exp == 0: + continue + self.assertRaises(ZeroDivisionError, pow, zero, exp) + + il, ih = -20, 20 + jl, jh = -5, 5 + kl, kh = -10, 10 + asseq = self.assertEqual + if type == float: + il = 1 + asseq = self.assertAlmostEqual + elif type == int: + jl = 0 + elif type == int: + jl, jh = 0, 15 + for i in range(il, ih+1): + for j in range(jl, jh+1): + for k in range(kl, kh+1): + if k != 0: + if type == float or j < 0: + self.assertRaises(TypeError, pow, type(i), j, k) + continue + asseq( + pow(type(i),j,k), + pow(type(i),j)% type(k) + ) + + def test_powint(self): + self.powtest(int) + + def test_powfloat(self): + self.powtest(float) + + def test_other(self): + # Other tests-- not very systematic + self.assertEqual(pow(3,3) % 8, pow(3,3,8)) + self.assertEqual(pow(3,3) % -8, pow(3,3,-8)) + self.assertEqual(pow(3,2) % -2, pow(3,2,-2)) + self.assertEqual(pow(-3,3) % 8, pow(-3,3,8)) + self.assertEqual(pow(-3,3) % -8, pow(-3,3,-8)) + self.assertEqual(pow(5,2) % -8, pow(5,2,-8)) + + self.assertEqual(pow(3,3) % 8, pow(3,3,8)) + self.assertEqual(pow(3,3) % -8, pow(3,3,-8)) + self.assertEqual(pow(3,2) % -2, pow(3,2,-2)) + self.assertEqual(pow(-3,3) % 8, pow(-3,3,8)) + self.assertEqual(pow(-3,3) % -8, pow(-3,3,-8)) + self.assertEqual(pow(5,2) % -8, pow(5,2,-8)) + + for i in range(-10, 11): + for j in range(0, 6): + for k in range(-7, 11): + if j >= 0 and k != 0: + self.assertEqual( + pow(i,j) % k, + pow(i,j,k) + ) + if j >= 0 and k != 0: + self.assertEqual( + pow(int(i),j) % k, + pow(int(i),j,k) + ) + + def test_bug643260(self): + class TestRpow: + def __rpow__(self, other): + return None + None ** TestRpow() # Won't fail when __rpow__ invoked. SF bug #643260. + + def test_bug705231(self): + # -1.0 raised to an integer should never blow up. It did if the + # platform pow() was buggy, and Python didn't worm around it. + eq = self.assertEqual + a = -1.0 + # The next two tests can still fail if the platform floor() + # function doesn't treat all large inputs as integers + # test_math should also fail if that is happening + eq(pow(a, 1.23e167), 1.0) + eq(pow(a, -1.23e167), 1.0) + for b in range(-10, 11): + eq(pow(a, float(b)), b & 1 and -1.0 or 1.0) + for n in range(0, 100): + fiveto = float(5 ** n) + # For small n, fiveto will be odd. Eventually we run out of + # mantissa bits, though, and thereafer fiveto will be even. + expected = fiveto % 2.0 and -1.0 or 1.0 + eq(pow(a, fiveto), expected) + eq(pow(a, -fiveto), expected) + eq(expected, 1.0) # else we didn't push fiveto to evenness + +if __name__ == "__main__": + unittest.main() From def812f7fdd88afd519b22a1eefe8e3cfd9d2dfb Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Mon, 10 Feb 2020 18:56:13 +0200 Subject: [PATCH 13/25] Add test_raise from CPython 3.8 --- Lib/test/test_raise.py | 484 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 484 insertions(+) create mode 100644 Lib/test/test_raise.py diff --git a/Lib/test/test_raise.py b/Lib/test/test_raise.py new file mode 100644 index 00000000000..c1ef154a9a9 --- /dev/null +++ b/Lib/test/test_raise.py @@ -0,0 +1,484 @@ +# Copyright 2007 Google, Inc. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. + +"""Tests for the raise statement.""" + +from test import support +import sys +import types +import unittest + + +def get_tb(): + try: + raise OSError() + except: + return sys.exc_info()[2] + + +class Context: + def __enter__(self): + return self + def __exit__(self, exc_type, exc_value, exc_tb): + return True + + +class TestRaise(unittest.TestCase): + def test_invalid_reraise(self): + try: + raise + except RuntimeError as e: + self.assertIn("No active exception", str(e)) + else: + self.fail("No exception raised") + + def test_reraise(self): + try: + try: + raise IndexError() + except IndexError as e: + exc1 = e + raise + except IndexError as exc2: + self.assertIs(exc1, exc2) + else: + self.fail("No exception raised") + + def test_except_reraise(self): + def reraise(): + try: + raise TypeError("foo") + except: + try: + raise KeyError("caught") + except KeyError: + pass + raise + self.assertRaises(TypeError, reraise) + + def test_finally_reraise(self): + def reraise(): + try: + raise TypeError("foo") + except: + try: + raise KeyError("caught") + finally: + raise + self.assertRaises(KeyError, reraise) + + def test_nested_reraise(self): + def nested_reraise(): + raise + def reraise(): + try: + raise TypeError("foo") + except: + nested_reraise() + self.assertRaises(TypeError, reraise) + + def test_raise_from_None(self): + try: + try: + raise TypeError("foo") + except: + raise ValueError() from None + except ValueError as e: + self.assertIsInstance(e.__context__, TypeError) + self.assertIsNone(e.__cause__) + + def test_with_reraise1(self): + def reraise(): + try: + raise TypeError("foo") + except: + with Context(): + pass + raise + self.assertRaises(TypeError, reraise) + + def test_with_reraise2(self): + def reraise(): + try: + raise TypeError("foo") + except: + with Context(): + raise KeyError("caught") + raise + self.assertRaises(TypeError, reraise) + + def test_yield_reraise(self): + def reraise(): + try: + raise TypeError("foo") + except: + yield 1 + raise + g = reraise() + next(g) + self.assertRaises(TypeError, lambda: next(g)) + self.assertRaises(StopIteration, lambda: next(g)) + + def test_erroneous_exception(self): + class MyException(Exception): + def __init__(self): + raise RuntimeError() + + try: + raise MyException + except RuntimeError: + pass + else: + self.fail("No exception raised") + + def test_new_returns_invalid_instance(self): + # See issue #11627. + class MyException(Exception): + def __new__(cls, *args): + return object() + + with self.assertRaises(TypeError): + raise MyException + + def test_assert_with_tuple_arg(self): + try: + assert False, (3,) + except AssertionError as e: + self.assertEqual(str(e), "(3,)") + + + +class TestCause(unittest.TestCase): + + def testCauseSyntax(self): + try: + try: + try: + raise TypeError + except Exception: + raise ValueError from None + except ValueError as exc: + self.assertIsNone(exc.__cause__) + self.assertTrue(exc.__suppress_context__) + exc.__suppress_context__ = False + raise exc + except ValueError as exc: + e = exc + + self.assertIsNone(e.__cause__) + self.assertFalse(e.__suppress_context__) + self.assertIsInstance(e.__context__, TypeError) + + def test_invalid_cause(self): + try: + raise IndexError from 5 + except TypeError as e: + self.assertIn("exception cause", str(e)) + else: + self.fail("No exception raised") + + def test_class_cause(self): + try: + raise IndexError from KeyError + except IndexError as e: + self.assertIsInstance(e.__cause__, KeyError) + else: + self.fail("No exception raised") + + def test_instance_cause(self): + cause = KeyError() + try: + raise IndexError from cause + except IndexError as e: + self.assertIs(e.__cause__, cause) + else: + self.fail("No exception raised") + + def test_erroneous_cause(self): + class MyException(Exception): + def __init__(self): + raise RuntimeError() + + try: + raise IndexError from MyException + except RuntimeError: + pass + else: + self.fail("No exception raised") + + +class TestTraceback(unittest.TestCase): + + def test_sets_traceback(self): + try: + raise IndexError() + except IndexError as e: + self.assertIsInstance(e.__traceback__, types.TracebackType) + else: + self.fail("No exception raised") + + def test_accepts_traceback(self): + tb = get_tb() + try: + raise IndexError().with_traceback(tb) + except IndexError as e: + self.assertNotEqual(e.__traceback__, tb) + self.assertEqual(e.__traceback__.tb_next, tb) + else: + self.fail("No exception raised") + + +class TestTracebackType(unittest.TestCase): + + def raiser(self): + raise ValueError + + def test_attrs(self): + try: + self.raiser() + except Exception as exc: + tb = exc.__traceback__ + + self.assertIsInstance(tb.tb_next, types.TracebackType) + self.assertIs(tb.tb_frame, sys._getframe()) + self.assertIsInstance(tb.tb_lasti, int) + self.assertIsInstance(tb.tb_lineno, int) + + self.assertIs(tb.tb_next.tb_next, None) + + # Invalid assignments + with self.assertRaises(TypeError): + del tb.tb_next + + with self.assertRaises(TypeError): + tb.tb_next = "asdf" + + # Loops + with self.assertRaises(ValueError): + tb.tb_next = tb + + with self.assertRaises(ValueError): + tb.tb_next.tb_next = tb + + # Valid assignments + tb.tb_next = None + self.assertIs(tb.tb_next, None) + + new_tb = get_tb() + tb.tb_next = new_tb + self.assertIs(tb.tb_next, new_tb) + + def test_constructor(self): + other_tb = get_tb() + frame = sys._getframe() + + tb = types.TracebackType(other_tb, frame, 1, 2) + self.assertEqual(tb.tb_next, other_tb) + self.assertEqual(tb.tb_frame, frame) + self.assertEqual(tb.tb_lasti, 1) + self.assertEqual(tb.tb_lineno, 2) + + tb = types.TracebackType(None, frame, 1, 2) + self.assertEqual(tb.tb_next, None) + + with self.assertRaises(TypeError): + types.TracebackType("no", frame, 1, 2) + + with self.assertRaises(TypeError): + types.TracebackType(other_tb, "no", 1, 2) + + with self.assertRaises(TypeError): + types.TracebackType(other_tb, frame, "no", 2) + + with self.assertRaises(TypeError): + types.TracebackType(other_tb, frame, 1, "nuh-uh") + + +class TestContext(unittest.TestCase): + def test_instance_context_instance_raise(self): + context = IndexError() + try: + try: + raise context + except: + raise OSError() + except OSError as e: + self.assertEqual(e.__context__, context) + else: + self.fail("No exception raised") + + def test_class_context_instance_raise(self): + context = IndexError + try: + try: + raise context + except: + raise OSError() + except OSError as e: + self.assertNotEqual(e.__context__, context) + self.assertIsInstance(e.__context__, context) + else: + self.fail("No exception raised") + + def test_class_context_class_raise(self): + context = IndexError + try: + try: + raise context + except: + raise OSError + except OSError as e: + self.assertNotEqual(e.__context__, context) + self.assertIsInstance(e.__context__, context) + else: + self.fail("No exception raised") + + def test_c_exception_context(self): + try: + try: + 1/0 + except: + raise OSError + except OSError as e: + self.assertIsInstance(e.__context__, ZeroDivisionError) + else: + self.fail("No exception raised") + + def test_c_exception_raise(self): + try: + try: + 1/0 + except: + xyzzy + except NameError as e: + self.assertIsInstance(e.__context__, ZeroDivisionError) + else: + self.fail("No exception raised") + + def test_noraise_finally(self): + try: + try: + pass + finally: + raise OSError + except OSError as e: + self.assertIsNone(e.__context__) + else: + self.fail("No exception raised") + + def test_raise_finally(self): + try: + try: + 1/0 + finally: + raise OSError + except OSError as e: + self.assertIsInstance(e.__context__, ZeroDivisionError) + else: + self.fail("No exception raised") + + def test_context_manager(self): + class ContextManager: + def __enter__(self): + pass + def __exit__(self, t, v, tb): + xyzzy + try: + with ContextManager(): + 1/0 + except NameError as e: + self.assertIsInstance(e.__context__, ZeroDivisionError) + else: + self.fail("No exception raised") + + def test_cycle_broken(self): + # Self-cycles (when re-raising a caught exception) are broken + try: + try: + 1/0 + except ZeroDivisionError as e: + raise e + except ZeroDivisionError as e: + self.assertIsNone(e.__context__) + + def test_reraise_cycle_broken(self): + # Non-trivial context cycles (through re-raising a previous exception) + # are broken too. + try: + try: + xyzzy + except NameError as a: + try: + 1/0 + except ZeroDivisionError: + raise a + except NameError as e: + self.assertIsNone(e.__context__.__context__) + + def test_3118(self): + # deleting the generator caused the __context__ to be cleared + def gen(): + try: + yield 1 + finally: + pass + + def f(): + g = gen() + next(g) + try: + try: + raise ValueError + except: + del g + raise KeyError + except Exception as e: + self.assertIsInstance(e.__context__, ValueError) + + f() + + def test_3611(self): + # A re-raised exception in a __del__ caused the __context__ + # to be cleared + class C: + def __del__(self): + try: + 1/0 + except: + raise + + def f(): + x = C() + try: + try: + x.x + except AttributeError: + del x + raise TypeError + except Exception as e: + self.assertNotEqual(e.__context__, None) + self.assertIsInstance(e.__context__, AttributeError) + + with support.captured_output("stderr"): + f() + +class TestRemovedFunctionality(unittest.TestCase): + def test_tuples(self): + try: + raise (IndexError, KeyError) # This should be a tuple! + except TypeError: + pass + else: + self.fail("No exception raised") + + def test_strings(self): + try: + raise "foo" + except TypeError: + pass + else: + self.fail("No exception raised") + + +if __name__ == "__main__": + unittest.main() From eb767cd9a16bfdf9d2ab90982c7a896f4d10a1e0 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Mon, 10 Feb 2020 18:59:39 +0200 Subject: [PATCH 14/25] Mark unsupported tests --- Lib/test/test_raise.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Lib/test/test_raise.py b/Lib/test/test_raise.py index c1ef154a9a9..062b8fcc175 100644 --- a/Lib/test/test_raise.py +++ b/Lib/test/test_raise.py @@ -56,6 +56,7 @@ def reraise(): raise self.assertRaises(TypeError, reraise) + @unittest.skip("TODO: RUSTPYTHON") def test_finally_reraise(self): def reraise(): try: @@ -150,6 +151,8 @@ def test_assert_with_tuple_arg(self): class TestCause(unittest.TestCase): + # TODO: RUSTPYTHON + @unittest.expectedFailure def testCauseSyntax(self): try: try: @@ -169,6 +172,8 @@ def testCauseSyntax(self): self.assertFalse(e.__suppress_context__) self.assertIsInstance(e.__context__, TypeError) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_invalid_cause(self): try: raise IndexError from 5 @@ -209,6 +214,7 @@ def __init__(self): class TestTraceback(unittest.TestCase): + @unittest.skip("TODO: RUSTPYTHON") def test_sets_traceback(self): try: raise IndexError() @@ -233,6 +239,7 @@ class TestTracebackType(unittest.TestCase): def raiser(self): raise ValueError + @unittest.skip("TODO: RUSTPYTHON") def test_attrs(self): try: self.raiser() @@ -268,6 +275,7 @@ def test_attrs(self): tb.tb_next = new_tb self.assertIs(tb.tb_next, new_tb) + @unittest.skip("TODO: RUSTPYTHON") def test_constructor(self): other_tb = get_tb() frame = sys._getframe() @@ -344,6 +352,8 @@ def test_c_exception_context(self): else: self.fail("No exception raised") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_c_exception_raise(self): try: try: @@ -366,6 +376,8 @@ def test_noraise_finally(self): else: self.fail("No exception raised") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_raise_finally(self): try: try: @@ -377,6 +389,8 @@ def test_raise_finally(self): else: self.fail("No exception raised") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_context_manager(self): class ContextManager: def __enter__(self): @@ -391,6 +405,8 @@ def __exit__(self, t, v, tb): else: self.fail("No exception raised") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_cycle_broken(self): # Self-cycles (when re-raising a caught exception) are broken try: From 74b74d32ec57c8573d72834db9189ffc414ab06a Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Thu, 13 Feb 2020 22:12:27 +0000 Subject: [PATCH 15/25] Make _random work on wasm --- Cargo.lock | 1 + Lib/random.py | 8 +++++++- vm/Cargo.toml | 4 ++-- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index adda6f05886..cc9f4166f9f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -636,6 +636,7 @@ dependencies = [ "cfg-if", "libc", "wasi", + "wasm-bindgen", ] [[package]] diff --git a/Lib/random.py b/Lib/random.py index 61e881642cb..13f05f7388d 100644 --- a/Lib/random.py +++ b/Lib/random.py @@ -41,7 +41,13 @@ from types import MethodType as _MethodType, BuiltinMethodType as _BuiltinMethodType from math import log as _log, exp as _exp, pi as _pi, e as _e, ceil as _ceil from math import sqrt as _sqrt, acos as _acos, cos as _cos, sin as _sin -from os import urandom as _urandom +try: + from os import urandom as _urandom +except ImportError: + # On wasm, _random.Random.random() does give a proper random value, but + # we don't have the os module + def _urandom(*args, **kwargs): + raise NotImplementedError("urandom") from _collections_abc import Set as _Set, Sequence as _Sequence from hashlib import sha512 as _sha512 import itertools as _itertools diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 3b3dbf9013c..a026f9d4abf 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -30,9 +30,9 @@ num-traits = "0.2.8" num-integer = "0.1.41" num-rational = "0.2.2" num-iter = "0.1.39" -rand = "0.7" +rand = { version = "0.7", features = ["wasm-bindgen"] } rand_core = "0.5" -getrandom = "0.1" +getrandom = { version = "0.1", features = ["wasm-bindgen"] } log = "0.4" rustpython-derive = {path = "../derive", version = "0.1.1"} rustpython-parser = {path = "../parser", optional = true, version = "0.1.1"} From 2355ac7fb162e76c249ad613149c639bd9cd234d Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Tue, 18 Feb 2020 08:14:30 +0000 Subject: [PATCH 16/25] Remove test warnings --- vm/src/obj/objstr.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index a2c8acc0b4a..f4538eb7054 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -1698,8 +1698,6 @@ mod tests { #[test] fn str_title() { - let vm: VirtualMachine = Default::default(); - let tests = vec![ (" Hello ", " hello "), ("Hello ", "hello "), @@ -1717,8 +1715,6 @@ mod tests { #[test] fn str_istitle() { - let vm: VirtualMachine = Default::default(); - let pos = vec![ "A", "A Titlecased Line", From 749d643d3e17fe28ff69cb23a11860b8e0c0446d Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 16 Feb 2020 17:31:55 +0900 Subject: [PATCH 17/25] Allow star expression for list elements --- parser/src/python.lalrpop | 2 +- tests/snippets/list.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/parser/src/python.lalrpop b/parser/src/python.lalrpop index 58ea0024618..12dcf32234b 100644 --- a/parser/src/python.lalrpop +++ b/parser/src/python.lalrpop @@ -1008,7 +1008,7 @@ ExpressionList2: Vec = { // - a single expression // - a single expression followed by a trailing comma TestList: ast::Expression = { - > => { + > => { if elements.len() == 1 && trailing_comma.is_none() { elements.into_iter().next().unwrap() } else { diff --git a/tests/snippets/list.py b/tests/snippets/list.py index 999a5399ed0..e9ccc01cd36 100644 --- a/tests/snippets/list.py +++ b/tests/snippets/list.py @@ -632,3 +632,6 @@ def iadd_slice(): assert it.__length_hint__() == 3 assert list(it) == [3,2,1] assert it.__length_hint__() == 0 + +a = [*[1, 2], 3, *[4, 5]] +assert a == [1, 2, 3, 4, 5] From 1a3d5b2b0bdecc651d9eea3abda9f3b3487587e7 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Thu, 20 Feb 2020 19:44:45 -0600 Subject: [PATCH 18/25] Reorganize struct module, add struct.Struct --- Lib/struct.py | 15 +++ vm/src/function.rs | 6 ++ vm/src/stdlib/mod.rs | 2 +- vm/src/stdlib/pystruct.rs | 215 ++++++++++++++++++++++---------------- 4 files changed, 148 insertions(+), 90 deletions(-) create mode 100644 Lib/struct.py diff --git a/Lib/struct.py b/Lib/struct.py new file mode 100644 index 00000000000..d6bba588636 --- /dev/null +++ b/Lib/struct.py @@ -0,0 +1,15 @@ +__all__ = [ + # Functions + 'calcsize', 'pack', 'pack_into', 'unpack', 'unpack_from', + 'iter_unpack', + + # Classes + 'Struct', + + # Exceptions + 'error' + ] + +from _struct import * +from _struct import _clearcache +from _struct import __doc__ diff --git a/vm/src/function.rs b/vm/src/function.rs index f2f574fc6f2..b993e8ee31d 100644 --- a/vm/src/function.rs +++ b/vm/src/function.rs @@ -294,6 +294,12 @@ impl Args { } } +impl AsRef<[T]> for Args { + fn as_ref(&self) -> &[T] { + &self.0 + } +} + impl Args> { pub fn into_tuple(self, vm: &VirtualMachine) -> PyObjectRef { vm.ctx diff --git a/vm/src/stdlib/mod.rs b/vm/src/stdlib/mod.rs index 4289bcc9c95..c709de3e0c2 100644 --- a/vm/src/stdlib/mod.rs +++ b/vm/src/stdlib/mod.rs @@ -80,7 +80,7 @@ pub fn get_module_inits() -> HashMap { "regex_crate".to_owned() => Box::new(re::make_module), "_random".to_owned() => Box::new(random::make_module), "_string".to_owned() => Box::new(string::make_module), - "struct".to_owned() => Box::new(pystruct::make_module), + "_struct".to_owned() => Box::new(pystruct::make_module), "_thread".to_owned() => Box::new(thread::make_module), "time".to_owned() => Box::new(time_module::make_module), "_weakref".to_owned() => Box::new(weakref::make_module), diff --git a/vm/src/stdlib/pystruct.rs b/vm/src/stdlib/pystruct.rs index 2c61784da60..abba7398ba3 100644 --- a/vm/src/stdlib/pystruct.rs +++ b/vm/src/stdlib/pystruct.rs @@ -14,21 +14,13 @@ use std::iter::Peekable; use byteorder::{ReadBytesExt, WriteBytesExt}; -use crate::function::PyFuncArgs; +use crate::function::Args; use crate::obj::{ - objbytes::PyBytesRef, - objstr::{self, PyStringRef}, - objtype, + objbytes::PyBytesRef, objstr::PyStringRef, objtuple::PyTuple, objtype::PyClassRef, }; -use crate::pyobject::{PyObjectRef, PyResult, TryFromObject}; +use crate::pyobject::{PyClassImpl, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject}; use crate::VirtualMachine; -#[derive(Debug)] -struct FormatSpec { - endianness: Endianness, - codes: Vec, -} - #[derive(Debug)] enum Endianness { Native, @@ -56,16 +48,78 @@ impl FormatCode { } } -fn parse_format_string(fmt: String) -> Result { - let mut chars = fmt.chars().peekable(); +#[derive(Debug)] +struct FormatSpec { + endianness: Endianness, + codes: Vec, +} + +impl FormatSpec { + fn parse(fmt: &str) -> Result { + let mut chars = fmt.chars().peekable(); + + // First determine "<", ">","!" or "=" + let endianness = parse_endiannes(&mut chars); - // First determine "<", ">","!" or "=" - let endianness = parse_endiannes(&mut chars); + // Now, analyze struct string furter: + let codes = parse_format_codes(&mut chars)?; - // Now, analyze struct string furter: - let codes = parse_format_codes(&mut chars)?; + Ok(FormatSpec { endianness, codes }) + } + + fn pack(&self, args: &[PyObjectRef], vm: &VirtualMachine) -> PyResult> { + if self.codes.len() != args.len() { + return Err(vm.new_exception_msg( + vm.try_class("_struct", "error")?, + format!( + "pack expected {} items for packing (got {})", + self.codes.len(), + args.len() + ), + )); + } + + // Create data vector: + let mut data = Vec::::new(); + // Loop over all opcodes: + for (code, arg) in self.codes.iter().zip(args.iter()) { + debug!("code: {:?}", code); + match self.endianness { + Endianness::Little => { + pack_item::(vm, code, arg, &mut data)? + } + Endianness::Big => pack_item::(vm, code, arg, &mut data)?, + Endianness::Network => { + pack_item::(vm, code, arg, &mut data)? + } + Endianness::Native => { + pack_item::(vm, code, arg, &mut data)? + } + } + } + + Ok(data) + } + + fn unpack(&self, data: &[u8], vm: &VirtualMachine) -> PyResult { + let mut rdr = Cursor::new(data); + + let mut items = vec![]; + for code in &self.codes { + debug!("unpack code: {:?}", code); + let item = match self.endianness { + Endianness::Little => unpack_code::(vm, &code, &mut rdr)?, + Endianness::Big => unpack_code::(vm, &code, &mut rdr)?, + Endianness::Network => { + unpack_code::(vm, &code, &mut rdr)? + } + Endianness::Native => unpack_code::(vm, &code, &mut rdr)?, + }; + items.push(item); + } - Ok(FormatSpec { endianness, codes }) + Ok(PyTuple::from(items)) + } } /// Parse endianness @@ -222,53 +276,9 @@ where Ok(()) } -fn struct_pack(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - if args.args.is_empty() { - Err(vm.new_type_error(format!( - "Expected at least 1 argument (got: {})", - args.args.len() - ))) - } else { - let fmt_arg = args.args[0].clone(); - if objtype::isinstance(&fmt_arg, &vm.ctx.str_type()) { - let fmt_str = objstr::clone_value(&fmt_arg); - - let format_spec = parse_format_string(fmt_str).map_err(|e| vm.new_value_error(e))?; - - if format_spec.codes.len() + 1 == args.args.len() { - // Create data vector: - let mut data = Vec::::new(); - // Loop over all opcodes: - for (code, arg) in format_spec.codes.iter().zip(args.args.iter().skip(1)) { - debug!("code: {:?}", code); - match format_spec.endianness { - Endianness::Little => { - pack_item::(vm, code, arg, &mut data)? - } - Endianness::Big => { - pack_item::(vm, code, arg, &mut data)? - } - Endianness::Network => { - pack_item::(vm, code, arg, &mut data)? - } - Endianness::Native => { - pack_item::(vm, code, arg, &mut data)? - } - } - } - - Ok(vm.ctx.new_bytes(data)) - } else { - Err(vm.new_type_error(format!( - "Expected {} arguments (got: {})", - format_spec.codes.len() + 1, - args.args.len() - ))) - } - } else { - Err(vm.new_type_error("First argument must be of str type".to_owned())) - } - } +fn struct_pack(fmt: PyStringRef, args: Args, vm: &VirtualMachine) -> PyResult> { + let format_spec = FormatSpec::parse(fmt.as_str()).map_err(|e| vm.new_value_error(e))?; + format_spec.pack(args.as_ref(), vm) } fn unpack_i8(vm: &VirtualMachine, rdr: &mut dyn Read) -> PyResult { @@ -372,26 +382,10 @@ where } } -fn struct_unpack(fmt: PyStringRef, buffer: PyBytesRef, vm: &VirtualMachine) -> PyResult { - let fmt_str = fmt.as_str().to_owned(); - - let format_spec = parse_format_string(fmt_str).map_err(|e| vm.new_value_error(e))?; - let data = buffer.get_value().to_vec(); - let mut rdr = Cursor::new(data); - - let mut items = vec![]; - for code in format_spec.codes { - debug!("unpack code: {:?}", code); - let item = match format_spec.endianness { - Endianness::Little => unpack_code::(vm, &code, &mut rdr)?, - Endianness::Big => unpack_code::(vm, &code, &mut rdr)?, - Endianness::Network => unpack_code::(vm, &code, &mut rdr)?, - Endianness::Native => unpack_code::(vm, &code, &mut rdr)?, - }; - items.push(item); - } - - Ok(vm.ctx.new_tuple(items)) +fn struct_unpack(fmt: PyStringRef, buffer: PyBytesRef, vm: &VirtualMachine) -> PyResult { + let fmt_str = fmt.as_str(); + let format_spec = FormatSpec::parse(fmt_str).map_err(|e| vm.new_value_error(e))?; + format_spec.unpack(buffer.get_value(), vm) } fn unpack_code(vm: &VirtualMachine, code: &FormatCode, rdr: &mut dyn Read) -> PyResult @@ -417,20 +411,63 @@ where } fn struct_calcsize(fmt: PyStringRef, vm: &VirtualMachine) -> PyResult { - let fmt_str = fmt.as_str().to_owned(); - let format_spec = parse_format_string(fmt_str).map_err(|e| vm.new_value_error(e))?; + let fmt_str = fmt.as_str(); + let format_spec = FormatSpec::parse(fmt_str).map_err(|e| vm.new_value_error(e))?; Ok(format_spec.codes.iter().map(|code| code.size()).sum()) } +#[pyclass(name = "Struct")] +#[derive(Debug)] +struct PyStruct { + spec: FormatSpec, + fmt_str: PyStringRef, +} + +impl PyValue for PyStruct { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.class("_struct", "Struct") + } +} + +#[pyimpl] +impl PyStruct { + #[pyslot] + fn tp_new(cls: PyClassRef, fmt_str: PyStringRef, vm: &VirtualMachine) -> PyResult> { + let spec = FormatSpec::parse(fmt_str.as_str()).map_err(|e| vm.new_value_error(e))?; + + PyStruct { spec, fmt_str }.into_ref_with_type(vm, cls) + } + + #[pyproperty] + fn format(&self) -> PyStringRef { + self.fmt_str.clone() + } + + #[pymethod] + fn pack(&self, args: Args, vm: &VirtualMachine) -> PyResult> { + self.spec.pack(args.as_ref(), vm) + } + #[pymethod] + fn unpack(&self, data: PyBytesRef, vm: &VirtualMachine) -> PyResult { + self.spec.unpack(data.get_value(), vm) + } +} + +// seems weird that this is part of the "public" API, but whatever +// TODO: implement a format code->spec cache like CPython does? +fn clearcache() {} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; - let struct_error = ctx.new_class("struct.error", ctx.object()); + let struct_error = ctx.new_class("struct.error", ctx.exceptions.exception_type.clone()); - py_module!(vm, "struct", { + py_module!(vm, "_struct", { + "_clearcache" => ctx.new_function(clearcache), "pack" => ctx.new_function(struct_pack), "unpack" => ctx.new_function(struct_unpack), "calcsize" => ctx.new_function(struct_calcsize), "error" => struct_error, + "Struct" => PyStruct::make_class(ctx), }) } From 18d16eedc8201d9bb5eecc02fa7f5e0cb8476883 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Thu, 20 Feb 2020 22:33:33 -0600 Subject: [PATCH 19/25] Add struct.Struct test --- tests/snippets/stdlib_struct.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/snippets/stdlib_struct.py b/tests/snippets/stdlib_struct.py index 100c60bcf7a..fde58cb6818 100644 --- a/tests/snippets/stdlib_struct.py +++ b/tests/snippets/stdlib_struct.py @@ -44,3 +44,6 @@ assert struct.calcsize("B") == 1 assert struct.calcsize(" Date: Thu, 20 Feb 2020 23:08:37 -0600 Subject: [PATCH 20/25] Add struct.Struct.size property --- vm/src/stdlib/pystruct.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/vm/src/stdlib/pystruct.rs b/vm/src/stdlib/pystruct.rs index abba7398ba3..568d54c4b81 100644 --- a/vm/src/stdlib/pystruct.rs +++ b/vm/src/stdlib/pystruct.rs @@ -120,6 +120,10 @@ impl FormatSpec { Ok(PyTuple::from(items)) } + + fn size(&self) -> usize { + self.codes.iter().map(FormatCode::size).sum() + } } /// Parse endianness @@ -413,7 +417,7 @@ where fn struct_calcsize(fmt: PyStringRef, vm: &VirtualMachine) -> PyResult { let fmt_str = fmt.as_str(); let format_spec = FormatSpec::parse(fmt_str).map_err(|e| vm.new_value_error(e))?; - Ok(format_spec.codes.iter().map(|code| code.size()).sum()) + Ok(format_spec.size()) } #[pyclass(name = "Struct")] @@ -442,6 +446,10 @@ impl PyStruct { fn format(&self) -> PyStringRef { self.fmt_str.clone() } + #[pyproperty] + fn size(&self) -> usize { + self.spec.size() + } #[pymethod] fn pack(&self, args: Args, vm: &VirtualMachine) -> PyResult> { From aa26ee1b9e3528c380e0237264ef647b3f1f5166 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Thu, 20 Feb 2020 23:53:19 -0600 Subject: [PATCH 21/25] Misc changes --- vm/src/builtins.rs | 1 + vm/src/obj/objbuiltinfunc.rs | 6 +++++- vm/src/obj/objgetset.rs | 7 ++++++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index bfa1a01f4a8..d13dbbcb8c0 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -882,6 +882,7 @@ pub fn make_module(vm: &VirtualMachine, module: PyObjectRef) { "MemoryError" => ctx.exceptions.memory_error.clone(), "OSError" => ctx.exceptions.os_error.clone(), + "IOError" => ctx.exceptions.os_error.clone(), "FileNotFoundError" => ctx.exceptions.file_not_found_error.clone(), "PermissionError" => ctx.exceptions.permission_error.clone(), "FileExistsError" => ctx.exceptions.file_exists_error.clone(), diff --git a/vm/src/obj/objbuiltinfunc.rs b/vm/src/obj/objbuiltinfunc.rs index 767e6294eca..9b91ce0681b 100644 --- a/vm/src/obj/objbuiltinfunc.rs +++ b/vm/src/obj/objbuiltinfunc.rs @@ -99,7 +99,11 @@ impl SlotCall for PyBuiltinMethod { } #[pyimpl(with(SlotDescriptor, SlotCall))] -impl PyBuiltinMethod {} +impl PyBuiltinMethod { + // TODO: give builtin functions names + #[pyproperty(magic)] + fn name(&self) {} +} pub fn init(context: &PyContext) { PyBuiltinFunction::extend_class(context, &context.types.builtin_function_or_method_type); diff --git a/vm/src/obj/objgetset.rs b/vm/src/obj/objgetset.rs index 20e89e09f4d..c67ea6a82c8 100644 --- a/vm/src/obj/objgetset.rs +++ b/vm/src/obj/objgetset.rs @@ -5,6 +5,7 @@ use super::objtype::PyClassRef; use crate::function::{OptionalArg, OwnedParam, RefParam}; use crate::pyobject::{ IntoPyObject, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, + TypeProtocol, }; use crate::slots::SlotDescriptor; use crate::vm::VirtualMachine; @@ -238,10 +239,14 @@ impl PyGetSet { Err(vm.new_attribute_error(format!( "attribute '{}' of '{}' objects is not writable", self.name, - Self::class(vm).name + obj.class().name ))) } } + + // TODO: give getset_descriptors names + #[pyproperty(magic)] + fn name(&self) {} } pub(crate) fn init(context: &PyContext) { From e9beca4aaa7e31e697e0a03d97ade72f613cd08c Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 16 Feb 2020 19:09:30 +0200 Subject: [PATCH 22/25] Support Args in set/frozenset methods --- vm/src/function.rs | 4 ++ vm/src/obj/objset.rs | 144 ++++++++++++++++++++++++++----------------- 2 files changed, 91 insertions(+), 57 deletions(-) diff --git a/vm/src/function.rs b/vm/src/function.rs index b993e8ee31d..a13b600da6e 100644 --- a/vm/src/function.rs +++ b/vm/src/function.rs @@ -289,6 +289,10 @@ impl IntoIterator for KwArgs { pub struct Args(Vec); impl Args { + pub fn new(args: Vec) -> Self { + Args(args) + } + pub fn into_vec(self) -> Vec { self.0 } diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index 146d8511b12..1483a8a3dde 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -8,7 +8,7 @@ use std::fmt; use super::objlist::PyListIterator; use super::objtype::{self, PyClassRef}; use crate::dictdatatype; -use crate::function::OptionalArg; +use crate::function::{Args, OptionalArg}; use crate::pyobject::{ PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, @@ -280,39 +280,51 @@ impl PySetInner { } } - fn update(&mut self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult<()> { - for item in iterable.iter(vm)? { - self.add(&item?, vm)?; + fn update(&mut self, others: Args, vm: &VirtualMachine) -> PyResult<()> { + for iterable in others { + for item in iterable.iter(vm)? { + self.add(&item?, vm)?; + } } Ok(()) } - fn intersection_update(&mut self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult<()> { + fn intersection_update( + &mut self, + others: Args, + vm: &VirtualMachine, + ) -> PyResult<()> { let temp_inner = self.copy(); self.clear(); - for item in iterable.iter(vm)? { - let obj = item?; - if temp_inner.contains(&obj, vm)? { - self.add(&obj, vm)?; + for iterable in others { + for item in iterable.iter(vm)? { + let obj = item?; + if temp_inner.contains(&obj, vm)? { + self.add(&obj, vm)?; + } } } Ok(()) } - fn difference_update(&mut self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult<()> { - for item in iterable.iter(vm)? { - self.content.delete_if_exists(vm, &item?)?; + fn difference_update(&mut self, others: Args, vm: &VirtualMachine) -> PyResult<()> { + for iterable in others { + for item in iterable.iter(vm)? { + self.content.delete_if_exists(vm, &item?)?; + } } Ok(()) } fn symmetric_difference_update( &mut self, - iterable: PyIterable, + others: Args, vm: &VirtualMachine, ) -> PyResult<()> { - for item in iterable.iter(vm)? { - self.content.delete_or_insert(vm, &item?, ())?; + for iterable in others { + for item in iterable.iter(vm)? { + self.content.delete_or_insert(vm, &item?, ())?; + } } Ok(()) } @@ -328,6 +340,18 @@ macro_rules! try_set_cmp { }; } +macro_rules! multi_args_set { + ($vm:expr, $others:expr, $zelf:expr, $op:tt) => {{ + let mut res = $zelf.inner.borrow().copy(); + for other in $others { + res = res.$op(other, $vm)? + } + Ok(Self { + inner: RefCell::new(res), + }) + }}; +} + #[pyimpl(flags(BASETYPE))] impl PySet { #[pyslot] @@ -395,31 +419,27 @@ impl PySet { } #[pymethod] - fn union(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { - Ok(Self { - inner: RefCell::new(self.inner.borrow().union(other, vm)?), - }) + fn union(&self, others: Args, vm: &VirtualMachine) -> PyResult { + multi_args_set!(vm, others, self, union) } #[pymethod] - fn intersection(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { - Ok(Self { - inner: RefCell::new(self.inner.borrow().intersection(other, vm)?), - }) + fn intersection(&self, others: Args, vm: &VirtualMachine) -> PyResult { + multi_args_set!(vm, others, self, intersection) } #[pymethod] - fn difference(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { - Ok(Self { - inner: RefCell::new(self.inner.borrow().difference(other, vm)?), - }) + fn difference(&self, others: Args, vm: &VirtualMachine) -> PyResult { + multi_args_set!(vm, others, self, difference) } #[pymethod] - fn symmetric_difference(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { - Ok(Self { - inner: RefCell::new(self.inner.borrow().symmetric_difference(other, vm)?), - }) + fn symmetric_difference( + &self, + others: Args, + vm: &VirtualMachine, + ) -> PyResult { + multi_args_set!(vm, others, self, symmetric_difference) } #[pymethod] @@ -529,14 +549,14 @@ impl PySet { } #[pymethod] - fn update(&self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { - self.inner.borrow_mut().update(iterable, vm)?; + fn update(&self, others: Args, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().update(others, vm)?; Ok(vm.get_none()) } #[pymethod] - fn intersection_update(&self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { - self.inner.borrow_mut().intersection_update(iterable, vm)?; + fn intersection_update(&self, others: Args, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().intersection_update(others, vm)?; Ok(vm.get_none()) } @@ -549,8 +569,8 @@ impl PySet { } #[pymethod] - fn difference_update(&self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { - self.inner.borrow_mut().difference_update(iterable, vm)?; + fn difference_update(&self, others: Args, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().difference_update(others, vm)?; Ok(vm.get_none()) } @@ -563,10 +583,14 @@ impl PySet { } #[pymethod] - fn symmetric_difference_update(&self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + fn symmetric_difference_update( + &self, + others: Args, + vm: &VirtualMachine, + ) -> PyResult { self.inner .borrow_mut() - .symmetric_difference_update(iterable, vm)?; + .symmetric_difference_update(others, vm)?; Ok(vm.get_none()) } @@ -584,6 +608,16 @@ impl PySet { } } +macro_rules! multi_args_frozenset { + ($vm:expr, $others:expr, $zelf:expr, $op:tt) => {{ + let mut res = $zelf.inner.copy(); + for other in $others { + res = res.$op(other, $vm)? + } + Ok(Self { inner: res }) + }}; +} + #[pyimpl(flags(BASETYPE))] impl PyFrozenSet { #[pyslot] @@ -651,31 +685,27 @@ impl PyFrozenSet { } #[pymethod] - fn union(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { - Ok(Self { - inner: self.inner.union(other, vm)?, - }) + fn union(&self, others: Args, vm: &VirtualMachine) -> PyResult { + multi_args_frozenset!(vm, others, self, union) } #[pymethod] - fn intersection(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { - Ok(Self { - inner: self.inner.intersection(other, vm)?, - }) + fn intersection(&self, others: Args, vm: &VirtualMachine) -> PyResult { + multi_args_frozenset!(vm, others, self, intersection) } #[pymethod] - fn difference(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { - Ok(Self { - inner: self.inner.difference(other, vm)?, - }) + fn difference(&self, others: Args, vm: &VirtualMachine) -> PyResult { + multi_args_frozenset!(vm, others, self, difference) } #[pymethod] - fn symmetric_difference(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { - Ok(Self { - inner: self.inner.symmetric_difference(other, vm)?, - }) + fn symmetric_difference( + &self, + others: Args, + vm: &VirtualMachine, + ) -> PyResult { + multi_args_frozenset!(vm, others, self, symmetric_difference) } #[pymethod] @@ -753,7 +783,7 @@ impl PyFrozenSet { } struct SetIterable { - iterable: PyIterable, + iterable: Args, } impl TryFromObject for SetIterable { @@ -762,7 +792,7 @@ impl TryFromObject for SetIterable { || objtype::issubclass(&obj.class(), &vm.ctx.frozenset_type()) { Ok(SetIterable { - iterable: PyIterable::try_from_object(vm, obj)?, + iterable: Args::new(vec![PyIterable::try_from_object(vm, obj)?]), }) } else { Err(vm.new_type_error(format!( From 9a5a5fb93bd14c4041b414c52902c26fec3179b8 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 16 Feb 2020 19:20:50 +0200 Subject: [PATCH 23/25] Add test_set from CPython 3.8 --- Lib/test/test_set.py | 1902 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1902 insertions(+) create mode 100644 Lib/test/test_set.py diff --git a/Lib/test/test_set.py b/Lib/test/test_set.py new file mode 100644 index 00000000000..bb1081f034f --- /dev/null +++ b/Lib/test/test_set.py @@ -0,0 +1,1902 @@ +import unittest +from test import support +import gc +import weakref +import operator +import copy +import pickle +from random import randrange, shuffle +import warnings +import collections +import collections.abc +import itertools + +class PassThru(Exception): + pass + +def check_pass_thru(): + raise PassThru + yield 1 + +class BadCmp: + def __hash__(self): + return 1 + def __eq__(self, other): + raise RuntimeError + +class ReprWrapper: + 'Used to test self-referential repr() calls' + def __repr__(self): + return repr(self.value) + +class HashCountingInt(int): + 'int-like object that counts the number of times __hash__ is called' + def __init__(self, *args): + self.hash_count = 0 + def __hash__(self): + self.hash_count += 1 + return int.__hash__(self) + +class TestJointOps: + # Tests common to both set and frozenset + + def setUp(self): + self.word = word = 'simsalabim' + self.otherword = 'madagascar' + self.letters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' + self.s = self.thetype(word) + self.d = dict.fromkeys(word) + + def test_new_or_init(self): + self.assertRaises(TypeError, self.thetype, [], 2) + self.assertRaises(TypeError, set().__init__, a=1) + + def test_uniquification(self): + actual = sorted(self.s) + expected = sorted(self.d) + self.assertEqual(actual, expected) + self.assertRaises(PassThru, self.thetype, check_pass_thru()) + self.assertRaises(TypeError, self.thetype, [[]]) + + def test_len(self): + self.assertEqual(len(self.s), len(self.d)) + + def test_contains(self): + for c in self.letters: + self.assertEqual(c in self.s, c in self.d) + self.assertRaises(TypeError, self.s.__contains__, [[]]) + s = self.thetype([frozenset(self.letters)]) + self.assertIn(self.thetype(self.letters), s) + + def test_union(self): + u = self.s.union(self.otherword) + for c in self.letters: + self.assertEqual(c in u, c in self.d or c in self.otherword) + self.assertEqual(self.s, self.thetype(self.word)) + self.assertEqual(type(u), self.basetype) + self.assertRaises(PassThru, self.s.union, check_pass_thru()) + self.assertRaises(TypeError, self.s.union, [[]]) + for C in set, frozenset, dict.fromkeys, str, list, tuple: + self.assertEqual(self.thetype('abcba').union(C('cdc')), set('abcd')) + self.assertEqual(self.thetype('abcba').union(C('efgfe')), set('abcefg')) + self.assertEqual(self.thetype('abcba').union(C('ccb')), set('abc')) + self.assertEqual(self.thetype('abcba').union(C('ef')), set('abcef')) + self.assertEqual(self.thetype('abcba').union(C('ef'), C('fg')), set('abcefg')) + + # Issue #6573 + x = self.thetype() + self.assertEqual(x.union(set([1]), x, set([2])), self.thetype([1, 2])) + + def test_or(self): + i = self.s.union(self.otherword) + self.assertEqual(self.s | set(self.otherword), i) + self.assertEqual(self.s | frozenset(self.otherword), i) + try: + self.s | self.otherword + except TypeError: + pass + else: + self.fail("s|t did not screen-out general iterables") + + def test_intersection(self): + i = self.s.intersection(self.otherword) + for c in self.letters: + self.assertEqual(c in i, c in self.d and c in self.otherword) + self.assertEqual(self.s, self.thetype(self.word)) + self.assertEqual(type(i), self.basetype) + self.assertRaises(PassThru, self.s.intersection, check_pass_thru()) + for C in set, frozenset, dict.fromkeys, str, list, tuple: + self.assertEqual(self.thetype('abcba').intersection(C('cdc')), set('cc')) + self.assertEqual(self.thetype('abcba').intersection(C('efgfe')), set('')) + self.assertEqual(self.thetype('abcba').intersection(C('ccb')), set('bc')) + self.assertEqual(self.thetype('abcba').intersection(C('ef')), set('')) + self.assertEqual(self.thetype('abcba').intersection(C('cbcf'), C('bag')), set('b')) + s = self.thetype('abcba') + z = s.intersection() + if self.thetype == frozenset(): + self.assertEqual(id(s), id(z)) + else: + self.assertNotEqual(id(s), id(z)) + + def test_isdisjoint(self): + def f(s1, s2): + 'Pure python equivalent of isdisjoint()' + return not set(s1).intersection(s2) + for larg in '', 'a', 'ab', 'abc', 'ababac', 'cdc', 'cc', 'efgfe', 'ccb', 'ef': + s1 = self.thetype(larg) + for rarg in '', 'a', 'ab', 'abc', 'ababac', 'cdc', 'cc', 'efgfe', 'ccb', 'ef': + for C in set, frozenset, dict.fromkeys, str, list, tuple: + s2 = C(rarg) + actual = s1.isdisjoint(s2) + expected = f(s1, s2) + self.assertEqual(actual, expected) + self.assertTrue(actual is True or actual is False) + + def test_and(self): + i = self.s.intersection(self.otherword) + self.assertEqual(self.s & set(self.otherword), i) + self.assertEqual(self.s & frozenset(self.otherword), i) + try: + self.s & self.otherword + except TypeError: + pass + else: + self.fail("s&t did not screen-out general iterables") + + def test_difference(self): + i = self.s.difference(self.otherword) + for c in self.letters: + self.assertEqual(c in i, c in self.d and c not in self.otherword) + self.assertEqual(self.s, self.thetype(self.word)) + self.assertEqual(type(i), self.basetype) + self.assertRaises(PassThru, self.s.difference, check_pass_thru()) + self.assertRaises(TypeError, self.s.difference, [[]]) + for C in set, frozenset, dict.fromkeys, str, list, tuple: + self.assertEqual(self.thetype('abcba').difference(C('cdc')), set('ab')) + self.assertEqual(self.thetype('abcba').difference(C('efgfe')), set('abc')) + self.assertEqual(self.thetype('abcba').difference(C('ccb')), set('a')) + self.assertEqual(self.thetype('abcba').difference(C('ef')), set('abc')) + self.assertEqual(self.thetype('abcba').difference(), set('abc')) + self.assertEqual(self.thetype('abcba').difference(C('a'), C('b')), set('c')) + + def test_sub(self): + i = self.s.difference(self.otherword) + self.assertEqual(self.s - set(self.otherword), i) + self.assertEqual(self.s - frozenset(self.otherword), i) + try: + self.s - self.otherword + except TypeError: + pass + else: + self.fail("s-t did not screen-out general iterables") + + def test_symmetric_difference(self): + i = self.s.symmetric_difference(self.otherword) + for c in self.letters: + self.assertEqual(c in i, (c in self.d) ^ (c in self.otherword)) + self.assertEqual(self.s, self.thetype(self.word)) + self.assertEqual(type(i), self.basetype) + self.assertRaises(PassThru, self.s.symmetric_difference, check_pass_thru()) + self.assertRaises(TypeError, self.s.symmetric_difference, [[]]) + for C in set, frozenset, dict.fromkeys, str, list, tuple: + self.assertEqual(self.thetype('abcba').symmetric_difference(C('cdc')), set('abd')) + self.assertEqual(self.thetype('abcba').symmetric_difference(C('efgfe')), set('abcefg')) + self.assertEqual(self.thetype('abcba').symmetric_difference(C('ccb')), set('a')) + self.assertEqual(self.thetype('abcba').symmetric_difference(C('ef')), set('abcef')) + + def test_xor(self): + i = self.s.symmetric_difference(self.otherword) + self.assertEqual(self.s ^ set(self.otherword), i) + self.assertEqual(self.s ^ frozenset(self.otherword), i) + try: + self.s ^ self.otherword + except TypeError: + pass + else: + self.fail("s^t did not screen-out general iterables") + + def test_equality(self): + self.assertEqual(self.s, set(self.word)) + self.assertEqual(self.s, frozenset(self.word)) + self.assertEqual(self.s == self.word, False) + self.assertNotEqual(self.s, set(self.otherword)) + self.assertNotEqual(self.s, frozenset(self.otherword)) + self.assertEqual(self.s != self.word, True) + + def test_setOfFrozensets(self): + t = map(frozenset, ['abcdef', 'bcd', 'bdcb', 'fed', 'fedccba']) + s = self.thetype(t) + self.assertEqual(len(s), 3) + + def test_sub_and_super(self): + p, q, r = map(self.thetype, ['ab', 'abcde', 'def']) + self.assertTrue(p < q) + self.assertTrue(p <= q) + self.assertTrue(q <= q) + self.assertTrue(q > p) + self.assertTrue(q >= p) + self.assertFalse(q < r) + self.assertFalse(q <= r) + self.assertFalse(q > r) + self.assertFalse(q >= r) + self.assertTrue(set('a').issubset('abc')) + self.assertTrue(set('abc').issuperset('a')) + self.assertFalse(set('a').issubset('cbs')) + self.assertFalse(set('cbs').issuperset('a')) + + def test_pickling(self): + for i in range(pickle.HIGHEST_PROTOCOL + 1): + p = pickle.dumps(self.s, i) + dup = pickle.loads(p) + self.assertEqual(self.s, dup, "%s != %s" % (self.s, dup)) + if type(self.s) not in (set, frozenset): + self.s.x = 10 + p = pickle.dumps(self.s, i) + dup = pickle.loads(p) + self.assertEqual(self.s.x, dup.x) + + def test_iterator_pickling(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + itorg = iter(self.s) + data = self.thetype(self.s) + d = pickle.dumps(itorg, proto) + it = pickle.loads(d) + # Set iterators unpickle as list iterators due to the + # undefined order of set items. + # self.assertEqual(type(itorg), type(it)) + self.assertIsInstance(it, collections.abc.Iterator) + self.assertEqual(self.thetype(it), data) + + it = pickle.loads(d) + try: + drop = next(it) + except StopIteration: + continue + d = pickle.dumps(it, proto) + it = pickle.loads(d) + self.assertEqual(self.thetype(it), data - self.thetype((drop,))) + + def test_deepcopy(self): + class Tracer: + def __init__(self, value): + self.value = value + def __hash__(self): + return self.value + def __deepcopy__(self, memo=None): + return Tracer(self.value + 1) + t = Tracer(10) + s = self.thetype([t]) + dup = copy.deepcopy(s) + self.assertNotEqual(id(s), id(dup)) + for elem in dup: + newt = elem + self.assertNotEqual(id(t), id(newt)) + self.assertEqual(t.value + 1, newt.value) + + def test_gc(self): + # Create a nest of cycles to exercise overall ref count check + class A: + pass + s = set(A() for i in range(1000)) + for elem in s: + elem.cycle = s + elem.sub = elem + elem.set = set([elem]) + + def test_subclass_with_custom_hash(self): + # Bug #1257731 + class H(self.thetype): + def __hash__(self): + return int(id(self) & 0x7fffffff) + s=H() + f=set() + f.add(s) + self.assertIn(s, f) + f.remove(s) + f.add(s) + f.discard(s) + + def test_badcmp(self): + s = self.thetype([BadCmp()]) + # Detect comparison errors during insertion and lookup + self.assertRaises(RuntimeError, self.thetype, [BadCmp(), BadCmp()]) + self.assertRaises(RuntimeError, s.__contains__, BadCmp()) + # Detect errors during mutating operations + if hasattr(s, 'add'): + self.assertRaises(RuntimeError, s.add, BadCmp()) + self.assertRaises(RuntimeError, s.discard, BadCmp()) + self.assertRaises(RuntimeError, s.remove, BadCmp()) + + def test_cyclical_repr(self): + w = ReprWrapper() + s = self.thetype([w]) + w.value = s + if self.thetype == set: + self.assertEqual(repr(s), '{set(...)}') + else: + name = repr(s).partition('(')[0] # strip class name + self.assertEqual(repr(s), '%s({%s(...)})' % (name, name)) + + def test_cyclical_print(self): + w = ReprWrapper() + s = self.thetype([w]) + w.value = s + fo = open(support.TESTFN, "w") + try: + fo.write(str(s)) + fo.close() + fo = open(support.TESTFN, "r") + self.assertEqual(fo.read(), repr(s)) + finally: + fo.close() + support.unlink(support.TESTFN) + + def test_do_not_rehash_dict_keys(self): + n = 10 + d = dict.fromkeys(map(HashCountingInt, range(n))) + self.assertEqual(sum(elem.hash_count for elem in d), n) + s = self.thetype(d) + self.assertEqual(sum(elem.hash_count for elem in d), n) + s.difference(d) + self.assertEqual(sum(elem.hash_count for elem in d), n) + if hasattr(s, 'symmetric_difference_update'): + s.symmetric_difference_update(d) + self.assertEqual(sum(elem.hash_count for elem in d), n) + d2 = dict.fromkeys(set(d)) + self.assertEqual(sum(elem.hash_count for elem in d), n) + d3 = dict.fromkeys(frozenset(d)) + self.assertEqual(sum(elem.hash_count for elem in d), n) + d3 = dict.fromkeys(frozenset(d), 123) + self.assertEqual(sum(elem.hash_count for elem in d), n) + self.assertEqual(d3, dict.fromkeys(d, 123)) + + def test_container_iterator(self): + # Bug #3680: tp_traverse was not implemented for set iterator object + class C(object): + pass + obj = C() + ref = weakref.ref(obj) + container = set([obj, 1]) + obj.x = iter(container) + del obj, container + gc.collect() + self.assertTrue(ref() is None, "Cycle was not collected") + + def test_free_after_iterating(self): + support.check_free_after_iterating(self, iter, self.thetype) + +class TestSet(TestJointOps, unittest.TestCase): + thetype = set + basetype = set + + def test_init(self): + s = self.thetype() + s.__init__(self.word) + self.assertEqual(s, set(self.word)) + s.__init__(self.otherword) + self.assertEqual(s, set(self.otherword)) + self.assertRaises(TypeError, s.__init__, s, 2); + self.assertRaises(TypeError, s.__init__, 1); + + def test_constructor_identity(self): + s = self.thetype(range(3)) + t = self.thetype(s) + self.assertNotEqual(id(s), id(t)) + + def test_set_literal(self): + s = set([1,2,3]) + t = {1,2,3} + self.assertEqual(s, t) + + def test_set_literal_insertion_order(self): + # SF Issue #26020 -- Expect left to right insertion + s = {1, 1.0, True} + self.assertEqual(len(s), 1) + stored_value = s.pop() + self.assertEqual(type(stored_value), int) + + def test_set_literal_evaluation_order(self): + # Expect left to right expression evaluation + events = [] + def record(obj): + events.append(obj) + s = {record(1), record(2), record(3)} + self.assertEqual(events, [1, 2, 3]) + + def test_hash(self): + self.assertRaises(TypeError, hash, self.s) + + def test_clear(self): + self.s.clear() + self.assertEqual(self.s, set()) + self.assertEqual(len(self.s), 0) + + def test_copy(self): + dup = self.s.copy() + self.assertEqual(self.s, dup) + self.assertNotEqual(id(self.s), id(dup)) + self.assertEqual(type(dup), self.basetype) + + def test_add(self): + self.s.add('Q') + self.assertIn('Q', self.s) + dup = self.s.copy() + self.s.add('Q') + self.assertEqual(self.s, dup) + self.assertRaises(TypeError, self.s.add, []) + + def test_remove(self): + self.s.remove('a') + self.assertNotIn('a', self.s) + self.assertRaises(KeyError, self.s.remove, 'Q') + self.assertRaises(TypeError, self.s.remove, []) + s = self.thetype([frozenset(self.word)]) + self.assertIn(self.thetype(self.word), s) + s.remove(self.thetype(self.word)) + self.assertNotIn(self.thetype(self.word), s) + self.assertRaises(KeyError, self.s.remove, self.thetype(self.word)) + + def test_remove_keyerror_unpacking(self): + # bug: www.python.org/sf/1576657 + for v1 in ['Q', (1,)]: + try: + self.s.remove(v1) + except KeyError as e: + v2 = e.args[0] + self.assertEqual(v1, v2) + else: + self.fail() + + def test_remove_keyerror_set(self): + key = self.thetype([3, 4]) + try: + self.s.remove(key) + except KeyError as e: + self.assertTrue(e.args[0] is key, + "KeyError should be {0}, not {1}".format(key, + e.args[0])) + else: + self.fail() + + def test_discard(self): + self.s.discard('a') + self.assertNotIn('a', self.s) + self.s.discard('Q') + self.assertRaises(TypeError, self.s.discard, []) + s = self.thetype([frozenset(self.word)]) + self.assertIn(self.thetype(self.word), s) + s.discard(self.thetype(self.word)) + self.assertNotIn(self.thetype(self.word), s) + s.discard(self.thetype(self.word)) + + def test_pop(self): + for i in range(len(self.s)): + elem = self.s.pop() + self.assertNotIn(elem, self.s) + self.assertRaises(KeyError, self.s.pop) + + def test_update(self): + retval = self.s.update(self.otherword) + self.assertEqual(retval, None) + for c in (self.word + self.otherword): + self.assertIn(c, self.s) + self.assertRaises(PassThru, self.s.update, check_pass_thru()) + self.assertRaises(TypeError, self.s.update, [[]]) + for p, q in (('cdc', 'abcd'), ('efgfe', 'abcefg'), ('ccb', 'abc'), ('ef', 'abcef')): + for C in set, frozenset, dict.fromkeys, str, list, tuple: + s = self.thetype('abcba') + self.assertEqual(s.update(C(p)), None) + self.assertEqual(s, set(q)) + for p in ('cdc', 'efgfe', 'ccb', 'ef', 'abcda'): + q = 'ahi' + for C in set, frozenset, dict.fromkeys, str, list, tuple: + s = self.thetype('abcba') + self.assertEqual(s.update(C(p), C(q)), None) + self.assertEqual(s, set(s) | set(p) | set(q)) + + def test_ior(self): + self.s |= set(self.otherword) + for c in (self.word + self.otherword): + self.assertIn(c, self.s) + + def test_intersection_update(self): + retval = self.s.intersection_update(self.otherword) + self.assertEqual(retval, None) + for c in (self.word + self.otherword): + if c in self.otherword and c in self.word: + self.assertIn(c, self.s) + else: + self.assertNotIn(c, self.s) + self.assertRaises(PassThru, self.s.intersection_update, check_pass_thru()) + self.assertRaises(TypeError, self.s.intersection_update, [[]]) + for p, q in (('cdc', 'c'), ('efgfe', ''), ('ccb', 'bc'), ('ef', '')): + for C in set, frozenset, dict.fromkeys, str, list, tuple: + s = self.thetype('abcba') + self.assertEqual(s.intersection_update(C(p)), None) + self.assertEqual(s, set(q)) + ss = 'abcba' + s = self.thetype(ss) + t = 'cbc' + self.assertEqual(s.intersection_update(C(p), C(t)), None) + self.assertEqual(s, set('abcba')&set(p)&set(t)) + + def test_iand(self): + self.s &= set(self.otherword) + for c in (self.word + self.otherword): + if c in self.otherword and c in self.word: + self.assertIn(c, self.s) + else: + self.assertNotIn(c, self.s) + + def test_difference_update(self): + retval = self.s.difference_update(self.otherword) + self.assertEqual(retval, None) + for c in (self.word + self.otherword): + if c in self.word and c not in self.otherword: + self.assertIn(c, self.s) + else: + self.assertNotIn(c, self.s) + self.assertRaises(PassThru, self.s.difference_update, check_pass_thru()) + self.assertRaises(TypeError, self.s.difference_update, [[]]) + self.assertRaises(TypeError, self.s.symmetric_difference_update, [[]]) + for p, q in (('cdc', 'ab'), ('efgfe', 'abc'), ('ccb', 'a'), ('ef', 'abc')): + for C in set, frozenset, dict.fromkeys, str, list, tuple: + s = self.thetype('abcba') + self.assertEqual(s.difference_update(C(p)), None) + self.assertEqual(s, set(q)) + + s = self.thetype('abcdefghih') + s.difference_update() + self.assertEqual(s, self.thetype('abcdefghih')) + + s = self.thetype('abcdefghih') + s.difference_update(C('aba')) + self.assertEqual(s, self.thetype('cdefghih')) + + s = self.thetype('abcdefghih') + s.difference_update(C('cdc'), C('aba')) + self.assertEqual(s, self.thetype('efghih')) + + def test_isub(self): + self.s -= set(self.otherword) + for c in (self.word + self.otherword): + if c in self.word and c not in self.otherword: + self.assertIn(c, self.s) + else: + self.assertNotIn(c, self.s) + + def test_symmetric_difference_update(self): + retval = self.s.symmetric_difference_update(self.otherword) + self.assertEqual(retval, None) + for c in (self.word + self.otherword): + if (c in self.word) ^ (c in self.otherword): + self.assertIn(c, self.s) + else: + self.assertNotIn(c, self.s) + self.assertRaises(PassThru, self.s.symmetric_difference_update, check_pass_thru()) + self.assertRaises(TypeError, self.s.symmetric_difference_update, [[]]) + for p, q in (('cdc', 'abd'), ('efgfe', 'abcefg'), ('ccb', 'a'), ('ef', 'abcef')): + for C in set, frozenset, dict.fromkeys, str, list, tuple: + s = self.thetype('abcba') + self.assertEqual(s.symmetric_difference_update(C(p)), None) + self.assertEqual(s, set(q)) + + def test_ixor(self): + self.s ^= set(self.otherword) + for c in (self.word + self.otherword): + if (c in self.word) ^ (c in self.otherword): + self.assertIn(c, self.s) + else: + self.assertNotIn(c, self.s) + + def test_inplace_on_self(self): + t = self.s.copy() + t |= t + self.assertEqual(t, self.s) + t &= t + self.assertEqual(t, self.s) + t -= t + self.assertEqual(t, self.thetype()) + t = self.s.copy() + t ^= t + self.assertEqual(t, self.thetype()) + + def test_weakref(self): + s = self.thetype('gallahad') + p = weakref.proxy(s) + self.assertEqual(str(p), str(s)) + s = None + self.assertRaises(ReferenceError, str, p) + + def test_rich_compare(self): + class TestRichSetCompare: + def __gt__(self, some_set): + self.gt_called = True + return False + def __lt__(self, some_set): + self.lt_called = True + return False + def __ge__(self, some_set): + self.ge_called = True + return False + def __le__(self, some_set): + self.le_called = True + return False + + # This first tries the builtin rich set comparison, which doesn't know + # how to handle the custom object. Upon returning NotImplemented, the + # corresponding comparison on the right object is invoked. + myset = {1, 2, 3} + + myobj = TestRichSetCompare() + myset < myobj + self.assertTrue(myobj.gt_called) + + myobj = TestRichSetCompare() + myset > myobj + self.assertTrue(myobj.lt_called) + + myobj = TestRichSetCompare() + myset <= myobj + self.assertTrue(myobj.ge_called) + + myobj = TestRichSetCompare() + myset >= myobj + self.assertTrue(myobj.le_called) + + @unittest.skipUnless(hasattr(set, "test_c_api"), + 'C API test only available in a debug build') + def test_c_api(self): + self.assertEqual(set().test_c_api(), True) + +class SetSubclass(set): + pass + +class TestSetSubclass(TestSet): + thetype = SetSubclass + basetype = set + +class SetSubclassWithKeywordArgs(set): + def __init__(self, iterable=[], newarg=None): + set.__init__(self, iterable) + +class TestSetSubclassWithKeywordArgs(TestSet): + + def test_keywords_in_subclass(self): + 'SF bug #1486663 -- this used to erroneously raise a TypeError' + SetSubclassWithKeywordArgs(newarg=1) + +class TestFrozenSet(TestJointOps, unittest.TestCase): + thetype = frozenset + basetype = frozenset + + def test_init(self): + s = self.thetype(self.word) + s.__init__(self.otherword) + self.assertEqual(s, set(self.word)) + + def test_singleton_empty_frozenset(self): + f = frozenset() + efs = [frozenset(), frozenset([]), frozenset(()), frozenset(''), + frozenset(), frozenset([]), frozenset(()), frozenset(''), + frozenset(range(0)), frozenset(frozenset()), + frozenset(f), f] + # All of the empty frozensets should have just one id() + self.assertEqual(len(set(map(id, efs))), 1) + + def test_constructor_identity(self): + s = self.thetype(range(3)) + t = self.thetype(s) + self.assertEqual(id(s), id(t)) + + def test_hash(self): + self.assertEqual(hash(self.thetype('abcdeb')), + hash(self.thetype('ebecda'))) + + # make sure that all permutations give the same hash value + n = 100 + seq = [randrange(n) for i in range(n)] + results = set() + for i in range(200): + shuffle(seq) + results.add(hash(self.thetype(seq))) + self.assertEqual(len(results), 1) + + def test_copy(self): + dup = self.s.copy() + self.assertEqual(id(self.s), id(dup)) + + def test_frozen_as_dictkey(self): + seq = list(range(10)) + list('abcdefg') + ['apple'] + key1 = self.thetype(seq) + key2 = self.thetype(reversed(seq)) + self.assertEqual(key1, key2) + self.assertNotEqual(id(key1), id(key2)) + d = {} + d[key1] = 42 + self.assertEqual(d[key2], 42) + + def test_hash_caching(self): + f = self.thetype('abcdcda') + self.assertEqual(hash(f), hash(f)) + + def test_hash_effectiveness(self): + n = 13 + hashvalues = set() + addhashvalue = hashvalues.add + elemmasks = [(i+1, 1<=": "issuperset", + } + + reverse = {"==": "==", + "!=": "!=", + "<": ">", + ">": "<", + "<=": ">=", + ">=": "<=", + } + + def test_issubset(self): + x = self.left + y = self.right + for case in "!=", "==", "<", "<=", ">", ">=": + expected = case in self.cases + # Test the binary infix spelling. + result = eval("x" + case + "y", locals()) + self.assertEqual(result, expected) + # Test the "friendly" method-name spelling, if one exists. + if case in TestSubsets.case2method: + method = getattr(x, TestSubsets.case2method[case]) + result = method(y) + self.assertEqual(result, expected) + + # Now do the same for the operands reversed. + rcase = TestSubsets.reverse[case] + result = eval("y" + rcase + "x", locals()) + self.assertEqual(result, expected) + if rcase in TestSubsets.case2method: + method = getattr(y, TestSubsets.case2method[rcase]) + result = method(x) + self.assertEqual(result, expected) +#------------------------------------------------------------------------------ + +class TestSubsetEqualEmpty(TestSubsets, unittest.TestCase): + left = set() + right = set() + name = "both empty" + cases = "==", "<=", ">=" + +#------------------------------------------------------------------------------ + +class TestSubsetEqualNonEmpty(TestSubsets, unittest.TestCase): + left = set([1, 2]) + right = set([1, 2]) + name = "equal pair" + cases = "==", "<=", ">=" + +#------------------------------------------------------------------------------ + +class TestSubsetEmptyNonEmpty(TestSubsets, unittest.TestCase): + left = set() + right = set([1, 2]) + name = "one empty, one non-empty" + cases = "!=", "<", "<=" + +#------------------------------------------------------------------------------ + +class TestSubsetPartial(TestSubsets, unittest.TestCase): + left = set([1]) + right = set([1, 2]) + name = "one a non-empty proper subset of other" + cases = "!=", "<", "<=" + +#------------------------------------------------------------------------------ + +class TestSubsetNonOverlap(TestSubsets, unittest.TestCase): + left = set([1]) + right = set([2]) + name = "neither empty, neither contains" + cases = "!=" + +#============================================================================== + +class TestOnlySetsInBinaryOps: + + def test_eq_ne(self): + # Unlike the others, this is testing that == and != *are* allowed. + self.assertEqual(self.other == self.set, False) + self.assertEqual(self.set == self.other, False) + self.assertEqual(self.other != self.set, True) + self.assertEqual(self.set != self.other, True) + + def test_ge_gt_le_lt(self): + self.assertRaises(TypeError, lambda: self.set < self.other) + self.assertRaises(TypeError, lambda: self.set <= self.other) + self.assertRaises(TypeError, lambda: self.set > self.other) + self.assertRaises(TypeError, lambda: self.set >= self.other) + + self.assertRaises(TypeError, lambda: self.other < self.set) + self.assertRaises(TypeError, lambda: self.other <= self.set) + self.assertRaises(TypeError, lambda: self.other > self.set) + self.assertRaises(TypeError, lambda: self.other >= self.set) + + def test_update_operator(self): + try: + self.set |= self.other + except TypeError: + pass + else: + self.fail("expected TypeError") + + def test_update(self): + if self.otherIsIterable: + self.set.update(self.other) + else: + self.assertRaises(TypeError, self.set.update, self.other) + + def test_union(self): + self.assertRaises(TypeError, lambda: self.set | self.other) + self.assertRaises(TypeError, lambda: self.other | self.set) + if self.otherIsIterable: + self.set.union(self.other) + else: + self.assertRaises(TypeError, self.set.union, self.other) + + def test_intersection_update_operator(self): + try: + self.set &= self.other + except TypeError: + pass + else: + self.fail("expected TypeError") + + def test_intersection_update(self): + if self.otherIsIterable: + self.set.intersection_update(self.other) + else: + self.assertRaises(TypeError, + self.set.intersection_update, + self.other) + + def test_intersection(self): + self.assertRaises(TypeError, lambda: self.set & self.other) + self.assertRaises(TypeError, lambda: self.other & self.set) + if self.otherIsIterable: + self.set.intersection(self.other) + else: + self.assertRaises(TypeError, self.set.intersection, self.other) + + def test_sym_difference_update_operator(self): + try: + self.set ^= self.other + except TypeError: + pass + else: + self.fail("expected TypeError") + + def test_sym_difference_update(self): + if self.otherIsIterable: + self.set.symmetric_difference_update(self.other) + else: + self.assertRaises(TypeError, + self.set.symmetric_difference_update, + self.other) + + def test_sym_difference(self): + self.assertRaises(TypeError, lambda: self.set ^ self.other) + self.assertRaises(TypeError, lambda: self.other ^ self.set) + if self.otherIsIterable: + self.set.symmetric_difference(self.other) + else: + self.assertRaises(TypeError, self.set.symmetric_difference, self.other) + + def test_difference_update_operator(self): + try: + self.set -= self.other + except TypeError: + pass + else: + self.fail("expected TypeError") + + def test_difference_update(self): + if self.otherIsIterable: + self.set.difference_update(self.other) + else: + self.assertRaises(TypeError, + self.set.difference_update, + self.other) + + def test_difference(self): + self.assertRaises(TypeError, lambda: self.set - self.other) + self.assertRaises(TypeError, lambda: self.other - self.set) + if self.otherIsIterable: + self.set.difference(self.other) + else: + self.assertRaises(TypeError, self.set.difference, self.other) + +#------------------------------------------------------------------------------ + +class TestOnlySetsNumeric(TestOnlySetsInBinaryOps, unittest.TestCase): + def setUp(self): + self.set = set((1, 2, 3)) + self.other = 19 + self.otherIsIterable = False + +#------------------------------------------------------------------------------ + +class TestOnlySetsDict(TestOnlySetsInBinaryOps, unittest.TestCase): + def setUp(self): + self.set = set((1, 2, 3)) + self.other = {1:2, 3:4} + self.otherIsIterable = True + +#------------------------------------------------------------------------------ + +class TestOnlySetsOperator(TestOnlySetsInBinaryOps, unittest.TestCase): + def setUp(self): + self.set = set((1, 2, 3)) + self.other = operator.add + self.otherIsIterable = False + +#------------------------------------------------------------------------------ + +class TestOnlySetsTuple(TestOnlySetsInBinaryOps, unittest.TestCase): + def setUp(self): + self.set = set((1, 2, 3)) + self.other = (2, 4, 6) + self.otherIsIterable = True + +#------------------------------------------------------------------------------ + +class TestOnlySetsString(TestOnlySetsInBinaryOps, unittest.TestCase): + def setUp(self): + self.set = set((1, 2, 3)) + self.other = 'abc' + self.otherIsIterable = True + +#------------------------------------------------------------------------------ + +class TestOnlySetsGenerator(TestOnlySetsInBinaryOps, unittest.TestCase): + def setUp(self): + def gen(): + for i in range(0, 10, 2): + yield i + self.set = set((1, 2, 3)) + self.other = gen() + self.otherIsIterable = True + +#============================================================================== + +class TestCopying: + + def test_copy(self): + dup = self.set.copy() + dup_list = sorted(dup, key=repr) + set_list = sorted(self.set, key=repr) + self.assertEqual(len(dup_list), len(set_list)) + for i in range(len(dup_list)): + self.assertTrue(dup_list[i] is set_list[i]) + + def test_deep_copy(self): + dup = copy.deepcopy(self.set) + ##print type(dup), repr(dup) + dup_list = sorted(dup, key=repr) + set_list = sorted(self.set, key=repr) + self.assertEqual(len(dup_list), len(set_list)) + for i in range(len(dup_list)): + self.assertEqual(dup_list[i], set_list[i]) + +#------------------------------------------------------------------------------ + +class TestCopyingEmpty(TestCopying, unittest.TestCase): + def setUp(self): + self.set = set() + +#------------------------------------------------------------------------------ + +class TestCopyingSingleton(TestCopying, unittest.TestCase): + def setUp(self): + self.set = set(["hello"]) + +#------------------------------------------------------------------------------ + +class TestCopyingTriple(TestCopying, unittest.TestCase): + def setUp(self): + self.set = set(["zero", 0, None]) + +#------------------------------------------------------------------------------ + +class TestCopyingTuple(TestCopying, unittest.TestCase): + def setUp(self): + self.set = set([(1, 2)]) + +#------------------------------------------------------------------------------ + +class TestCopyingNested(TestCopying, unittest.TestCase): + def setUp(self): + self.set = set([((1, 2), (3, 4))]) + +#============================================================================== + +class TestIdentities(unittest.TestCase): + def setUp(self): + self.a = set('abracadabra') + self.b = set('alacazam') + + def test_binopsVsSubsets(self): + a, b = self.a, self.b + self.assertTrue(a - b < a) + self.assertTrue(b - a < b) + self.assertTrue(a & b < a) + self.assertTrue(a & b < b) + self.assertTrue(a | b > a) + self.assertTrue(a | b > b) + self.assertTrue(a ^ b < a | b) + + def test_commutativity(self): + a, b = self.a, self.b + self.assertEqual(a&b, b&a) + self.assertEqual(a|b, b|a) + self.assertEqual(a^b, b^a) + if a != b: + self.assertNotEqual(a-b, b-a) + + def test_summations(self): + # check that sums of parts equal the whole + a, b = self.a, self.b + self.assertEqual((a-b)|(a&b)|(b-a), a|b) + self.assertEqual((a&b)|(a^b), a|b) + self.assertEqual(a|(b-a), a|b) + self.assertEqual((a-b)|b, a|b) + self.assertEqual((a-b)|(a&b), a) + self.assertEqual((b-a)|(a&b), b) + self.assertEqual((a-b)|(b-a), a^b) + + def test_exclusion(self): + # check that inverse operations show non-overlap + a, b, zero = self.a, self.b, set() + self.assertEqual((a-b)&b, zero) + self.assertEqual((b-a)&a, zero) + self.assertEqual((a&b)&(a^b), zero) + +# Tests derived from test_itertools.py ======================================= + +def R(seqn): + 'Regular generator' + for i in seqn: + yield i + +class G: + 'Sequence using __getitem__' + def __init__(self, seqn): + self.seqn = seqn + def __getitem__(self, i): + return self.seqn[i] + +class I: + 'Sequence using iterator protocol' + def __init__(self, seqn): + self.seqn = seqn + self.i = 0 + def __iter__(self): + return self + def __next__(self): + if self.i >= len(self.seqn): raise StopIteration + v = self.seqn[self.i] + self.i += 1 + return v + +class Ig: + 'Sequence using iterator protocol defined with a generator' + def __init__(self, seqn): + self.seqn = seqn + self.i = 0 + def __iter__(self): + for val in self.seqn: + yield val + +class X: + 'Missing __getitem__ and __iter__' + def __init__(self, seqn): + self.seqn = seqn + self.i = 0 + def __next__(self): + if self.i >= len(self.seqn): raise StopIteration + v = self.seqn[self.i] + self.i += 1 + return v + +class N: + 'Iterator missing __next__()' + def __init__(self, seqn): + self.seqn = seqn + self.i = 0 + def __iter__(self): + return self + +class E: + 'Test propagation of exceptions' + def __init__(self, seqn): + self.seqn = seqn + self.i = 0 + def __iter__(self): + return self + def __next__(self): + 3 // 0 + +class S: + 'Test immediate stop' + def __init__(self, seqn): + pass + def __iter__(self): + return self + def __next__(self): + raise StopIteration + +from itertools import chain +def L(seqn): + 'Test multiple tiers of iterators' + return chain(map(lambda x:x, R(Ig(G(seqn))))) + +class TestVariousIteratorArgs(unittest.TestCase): + + def test_constructor(self): + for cons in (set, frozenset): + for s in ("123", "", range(1000), ('do', 1.2), range(2000,2200,5)): + for g in (G, I, Ig, S, L, R): + self.assertEqual(sorted(cons(g(s)), key=repr), sorted(g(s), key=repr)) + self.assertRaises(TypeError, cons , X(s)) + self.assertRaises(TypeError, cons , N(s)) + self.assertRaises(ZeroDivisionError, cons , E(s)) + + def test_inline_methods(self): + s = set('november') + for data in ("123", "", range(1000), ('do', 1.2), range(2000,2200,5), 'december'): + for meth in (s.union, s.intersection, s.difference, s.symmetric_difference, s.isdisjoint): + for g in (G, I, Ig, L, R): + expected = meth(data) + actual = meth(g(data)) + if isinstance(expected, bool): + self.assertEqual(actual, expected) + else: + self.assertEqual(sorted(actual, key=repr), sorted(expected, key=repr)) + self.assertRaises(TypeError, meth, X(s)) + self.assertRaises(TypeError, meth, N(s)) + self.assertRaises(ZeroDivisionError, meth, E(s)) + + def test_inplace_methods(self): + for data in ("123", "", range(1000), ('do', 1.2), range(2000,2200,5), 'december'): + for methname in ('update', 'intersection_update', + 'difference_update', 'symmetric_difference_update'): + for g in (G, I, Ig, S, L, R): + s = set('january') + t = s.copy() + getattr(s, methname)(list(g(data))) + getattr(t, methname)(g(data)) + self.assertEqual(sorted(s, key=repr), sorted(t, key=repr)) + + self.assertRaises(TypeError, getattr(set('january'), methname), X(data)) + self.assertRaises(TypeError, getattr(set('january'), methname), N(data)) + self.assertRaises(ZeroDivisionError, getattr(set('january'), methname), E(data)) + +class bad_eq: + def __eq__(self, other): + if be_bad: + set2.clear() + raise ZeroDivisionError + return self is other + def __hash__(self): + return 0 + +class bad_dict_clear: + def __eq__(self, other): + if be_bad: + dict2.clear() + return self is other + def __hash__(self): + return 0 + +class TestWeirdBugs(unittest.TestCase): + def test_8420_set_merge(self): + # This used to segfault + global be_bad, set2, dict2 + be_bad = False + set1 = {bad_eq()} + set2 = {bad_eq() for i in range(75)} + be_bad = True + self.assertRaises(ZeroDivisionError, set1.update, set2) + + be_bad = False + set1 = {bad_dict_clear()} + dict2 = {bad_dict_clear(): None} + be_bad = True + set1.symmetric_difference_update(dict2) + + def test_iter_and_mutate(self): + # Issue #24581 + s = set(range(100)) + s.clear() + s.update(range(100)) + si = iter(s) + s.clear() + a = list(range(100)) + s.update(range(100)) + list(si) + + def test_merge_and_mutate(self): + class X: + def __hash__(self): + return hash(0) + def __eq__(self, o): + other.clear() + return False + + other = set() + other = {X() for i in range(10)} + s = {0} + s.update(other) + +# Application tests (based on David Eppstein's graph recipes ==================================== + +def powerset(U): + """Generates all subsets of a set or sequence U.""" + U = iter(U) + try: + x = frozenset([next(U)]) + for S in powerset(U): + yield S + yield S | x + except StopIteration: + yield frozenset() + +def cube(n): + """Graph of n-dimensional hypercube.""" + singletons = [frozenset([x]) for x in range(n)] + return dict([(x, frozenset([x^s for s in singletons])) + for x in powerset(range(n))]) + +def linegraph(G): + """Graph, the vertices of which are edges of G, + with two vertices being adjacent iff the corresponding + edges share a vertex.""" + L = {} + for x in G: + for y in G[x]: + nx = [frozenset([x,z]) for z in G[x] if z != y] + ny = [frozenset([y,z]) for z in G[y] if z != x] + L[frozenset([x,y])] = frozenset(nx+ny) + return L + +def faces(G): + 'Return a set of faces in G. Where a face is a set of vertices on that face' + # currently limited to triangles,squares, and pentagons + f = set() + for v1, edges in G.items(): + for v2 in edges: + for v3 in G[v2]: + if v1 == v3: + continue + if v1 in G[v3]: + f.add(frozenset([v1, v2, v3])) + else: + for v4 in G[v3]: + if v4 == v2: + continue + if v1 in G[v4]: + f.add(frozenset([v1, v2, v3, v4])) + else: + for v5 in G[v4]: + if v5 == v3 or v5 == v2: + continue + if v1 in G[v5]: + f.add(frozenset([v1, v2, v3, v4, v5])) + return f + + +class TestGraphs(unittest.TestCase): + + def test_cube(self): + + g = cube(3) # vert --> {v1, v2, v3} + vertices1 = set(g) + self.assertEqual(len(vertices1), 8) # eight vertices + for edge in g.values(): + self.assertEqual(len(edge), 3) # each vertex connects to three edges + vertices2 = set(v for edges in g.values() for v in edges) + self.assertEqual(vertices1, vertices2) # edge vertices in original set + + cubefaces = faces(g) + self.assertEqual(len(cubefaces), 6) # six faces + for face in cubefaces: + self.assertEqual(len(face), 4) # each face is a square + + def test_cuboctahedron(self): + + # http://en.wikipedia.org/wiki/Cuboctahedron + # 8 triangular faces and 6 square faces + # 12 identical vertices each connecting a triangle and square + + g = cube(3) + cuboctahedron = linegraph(g) # V( --> {V1, V2, V3, V4} + self.assertEqual(len(cuboctahedron), 12)# twelve vertices + + vertices = set(cuboctahedron) + for edges in cuboctahedron.values(): + self.assertEqual(len(edges), 4) # each vertex connects to four other vertices + othervertices = set(edge for edges in cuboctahedron.values() for edge in edges) + self.assertEqual(vertices, othervertices) # edge vertices in original set + + cubofaces = faces(cuboctahedron) + facesizes = collections.defaultdict(int) + for face in cubofaces: + facesizes[len(face)] += 1 + self.assertEqual(facesizes[3], 8) # eight triangular faces + self.assertEqual(facesizes[4], 6) # six square faces + + for vertex in cuboctahedron: + edge = vertex # Cuboctahedron vertices are edges in Cube + self.assertEqual(len(edge), 2) # Two cube vertices define an edge + for cubevert in edge: + self.assertIn(cubevert, g) + + +#============================================================================== + +if __name__ == "__main__": + unittest.main() From 3affa0d161b7f39ae6a976ab3c2ea71e18e3e53c Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 16 Feb 2020 19:34:40 +0200 Subject: [PATCH 24/25] Mark unsupported tests --- Lib/test/test_set.py | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_set.py b/Lib/test/test_set.py index bb1081f034f..0d9ac1fc079 100644 --- a/Lib/test/test_set.py +++ b/Lib/test/test_set.py @@ -1,6 +1,6 @@ import unittest from test import support -import gc +# import gc import weakref import operator import copy @@ -47,6 +47,7 @@ def setUp(self): self.s = self.thetype(word) self.d = dict.fromkeys(word) + @unittest.skip("TODO: RUSTPYTHON") def test_new_or_init(self): self.assertRaises(TypeError, self.thetype, [], 2) self.assertRaises(TypeError, set().__init__, a=1) @@ -61,6 +62,7 @@ def test_uniquification(self): def test_len(self): self.assertEqual(len(self.s), len(self.d)) + @unittest.skip("TODO: RUSTPYTHON") def test_contains(self): for c in self.letters: self.assertEqual(c in self.s, c in self.d) @@ -170,6 +172,7 @@ def test_sub(self): else: self.fail("s-t did not screen-out general iterables") + @unittest.skip("TODO: RUSTPYTHON") def test_symmetric_difference(self): i = self.s.symmetric_difference(self.otherword) for c in self.letters: @@ -184,6 +187,7 @@ def test_symmetric_difference(self): self.assertEqual(self.thetype('abcba').symmetric_difference(C('ccb')), set('a')) self.assertEqual(self.thetype('abcba').symmetric_difference(C('ef')), set('abcef')) + @unittest.skip("TODO: RUSTPYTHON") def test_xor(self): i = self.s.symmetric_difference(self.otherword) self.assertEqual(self.s ^ set(self.otherword), i) @@ -203,6 +207,7 @@ def test_equality(self): self.assertNotEqual(self.s, frozenset(self.otherword)) self.assertEqual(self.s != self.word, True) + @unittest.skip("TODO: RUSTPYTHON") def test_setOfFrozensets(self): t = map(frozenset, ['abcdef', 'bcd', 'bdcb', 'fed', 'fedccba']) s = self.thetype(t) @@ -224,6 +229,7 @@ def test_sub_and_super(self): self.assertFalse(set('a').issubset('cbs')) self.assertFalse(set('cbs').issuperset('a')) + @unittest.skip("TODO: RUSTPYTHON") def test_pickling(self): for i in range(pickle.HIGHEST_PROTOCOL + 1): p = pickle.dumps(self.s, i) @@ -235,6 +241,7 @@ def test_pickling(self): dup = pickle.loads(p) self.assertEqual(self.s.x, dup.x) + @unittest.skip("TODO: RUSTPYTHON") def test_iterator_pickling(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): itorg = iter(self.s) @@ -256,6 +263,7 @@ def test_iterator_pickling(self): it = pickle.loads(d) self.assertEqual(self.thetype(it), data - self.thetype((drop,))) + @unittest.skip("TODO: RUSTPYTHON") def test_deepcopy(self): class Tracer: def __init__(self, value): @@ -273,6 +281,7 @@ def __deepcopy__(self, memo=None): self.assertNotEqual(id(t), id(newt)) self.assertEqual(t.value + 1, newt.value) + @unittest.skip("TODO: RUSTPYTHON") def test_gc(self): # Create a nest of cycles to exercise overall ref count check class A: @@ -331,6 +340,7 @@ def test_cyclical_print(self): fo.close() support.unlink(support.TESTFN) + @unittest.skip("TODO: RUSTPYTHON") def test_do_not_rehash_dict_keys(self): n = 10 d = dict.fromkeys(map(HashCountingInt, range(n))) @@ -350,6 +360,7 @@ def test_do_not_rehash_dict_keys(self): self.assertEqual(sum(elem.hash_count for elem in d), n) self.assertEqual(d3, dict.fromkeys(d, 123)) + @unittest.skip("TODO: RUSTPYTHON") def test_container_iterator(self): # Bug #3680: tp_traverse was not implemented for set iterator object class C(object): @@ -362,6 +373,7 @@ class C(object): gc.collect() self.assertTrue(ref() is None, "Cycle was not collected") + @unittest.skip("TODO: RUSTPYTHON") def test_free_after_iterating(self): support.check_free_after_iterating(self, iter, self.thetype) @@ -369,6 +381,7 @@ class TestSet(TestJointOps, unittest.TestCase): thetype = set basetype = set + @unittest.skip("TODO: RUSTPYTHON") def test_init(self): s = self.thetype() s.__init__(self.word) @@ -425,6 +438,7 @@ def test_add(self): self.assertEqual(self.s, dup) self.assertRaises(TypeError, self.s.add, []) + @unittest.skip("TODO: RUSTPYTHON") def test_remove(self): self.s.remove('a') self.assertNotIn('a', self.s) @@ -447,6 +461,7 @@ def test_remove_keyerror_unpacking(self): else: self.fail() + @unittest.skip("TODO: RUSTPYTHON") def test_remove_keyerror_set(self): key = self.thetype([3, 4]) try: @@ -458,6 +473,7 @@ def test_remove_keyerror_set(self): else: self.fail() + @unittest.skip("TODO: RUSTPYTHON") def test_discard(self): self.s.discard('a') self.assertNotIn('a', self.s) @@ -499,6 +515,7 @@ def test_ior(self): for c in (self.word + self.otherword): self.assertIn(c, self.s) + @unittest.skip("TODO: RUSTPYTHON") def test_intersection_update(self): retval = self.s.intersection_update(self.otherword) self.assertEqual(retval, None) @@ -565,6 +582,7 @@ def test_isub(self): else: self.assertNotIn(c, self.s) + @unittest.skip("TODO: RUSTPYTHON") def test_symmetric_difference_update(self): retval = self.s.symmetric_difference_update(self.otherword) self.assertEqual(retval, None) @@ -589,6 +607,7 @@ def test_ixor(self): else: self.assertNotIn(c, self.s) + @unittest.skip("TODO: RUSTPYTHON") def test_inplace_on_self(self): t = self.s.copy() t |= t @@ -601,6 +620,7 @@ def test_inplace_on_self(self): t ^= t self.assertEqual(t, self.thetype()) + @unittest.skip("TODO: RUSTPYTHON") def test_weakref(self): s = self.thetype('gallahad') p = weakref.proxy(s) @@ -652,6 +672,7 @@ def test_c_api(self): class SetSubclass(set): pass +@unittest.skip("TODO: RUSTPYTHON") class TestSetSubclass(TestSet): thetype = SetSubclass basetype = set @@ -660,6 +681,7 @@ class SetSubclassWithKeywordArgs(set): def __init__(self, iterable=[], newarg=None): set.__init__(self, iterable) +@unittest.skip("TODO: RUSTPYTHON") class TestSetSubclassWithKeywordArgs(TestSet): def test_keywords_in_subclass(self): @@ -675,6 +697,7 @@ def test_init(self): s.__init__(self.otherword) self.assertEqual(s, set(self.word)) + @unittest.skip("TODO: RUSTPYTHON") def test_singleton_empty_frozenset(self): f = frozenset() efs = [frozenset(), frozenset([]), frozenset(()), frozenset(''), @@ -684,11 +707,13 @@ def test_singleton_empty_frozenset(self): # All of the empty frozensets should have just one id() self.assertEqual(len(set(map(id, efs))), 1) + @unittest.skip("TODO: RUSTPYTHON") def test_constructor_identity(self): s = self.thetype(range(3)) t = self.thetype(s) self.assertEqual(id(s), id(t)) + @unittest.skip("TODO: RUSTPYTHON") def test_hash(self): self.assertEqual(hash(self.thetype('abcdeb')), hash(self.thetype('ebecda'))) @@ -702,10 +727,12 @@ def test_hash(self): results.add(hash(self.thetype(seq))) self.assertEqual(len(results), 1) + @unittest.skip("TODO: RUSTPYTHON") def test_copy(self): dup = self.s.copy() self.assertEqual(id(self.s), id(dup)) + @unittest.skip("TODO: RUSTPYTHON") def test_frozen_as_dictkey(self): seq = list(range(10)) + list('abcdefg') + ['apple'] key1 = self.thetype(seq) @@ -720,6 +747,7 @@ def test_hash_caching(self): f = self.thetype('abcdcda') self.assertEqual(hash(f), hash(f)) + @unittest.skip("TODO: RUSTPYTHON") def test_hash_effectiveness(self): n = 13 hashvalues = set() @@ -769,6 +797,7 @@ def test_nested_empty_constructor(self): t = self.thetype(s) self.assertEqual(s, t) + @unittest.skip("TODO: RUSTPYTHON") def test_singleton_empty_frozenset(self): Frozenset = self.thetype f = frozenset() @@ -888,6 +917,7 @@ def test_iteration(self): setiter = iter(self.set) self.assertEqual(setiter.__length_hint__(), len(self.set)) + @unittest.skip("TODO: RUSTPYTHON") def test_pickling(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): p = pickle.dumps(self.set, proto) @@ -1020,6 +1050,7 @@ def test_instancesWithoutException(self): set('abc') set(gooditer()) + @unittest.skip("TODO: RUSTPYTHON") def test_changingSizeWhileIterating(self): s = set([1,2,3]) try: @@ -1533,6 +1564,7 @@ def test_copy(self): for i in range(len(dup_list)): self.assertTrue(dup_list[i] is set_list[i]) + @unittest.skip("TODO: RUSTPYTHON") def test_deep_copy(self): dup = copy.deepcopy(self.set) ##print type(dup), repr(dup) @@ -1753,6 +1785,7 @@ def __hash__(self): return 0 class TestWeirdBugs(unittest.TestCase): + @unittest.skip("TODO: RUSTPYTHON") def test_8420_set_merge(self): # This used to segfault global be_bad, set2, dict2 @@ -1849,6 +1882,7 @@ def faces(G): return f +@unittest.skip("TODO: RUSTPYTHON") class TestGraphs(unittest.TestCase): def test_cube(self): From 4e057271ecae2c923eb671b0456f2c06a8449b60 Mon Sep 17 00:00:00 2001 From: philippeitis <33013301+philippeitis@users.noreply.github.com> Date: Wed, 26 Feb 2020 13:32:34 -0800 Subject: [PATCH 25/25] Throw errors for invalid f-strings, allow != in f-string braces, tests. --- parser/src/error.rs | 420 +++++++++++++-------------- parser/src/fstring.rs | 642 ++++++++++++++++++++++-------------------- 2 files changed, 549 insertions(+), 513 deletions(-) diff --git a/parser/src/error.rs b/parser/src/error.rs index 6afc4627b2d..158de8dc994 100644 --- a/parser/src/error.rs +++ b/parser/src/error.rs @@ -1,209 +1,211 @@ -//! Define internal parse error types -//! The goal is to provide a matching and a safe error API, maksing errors from LALR -use lalrpop_util::ParseError as LalrpopError; - -use crate::location::Location; -use crate::token::Tok; - -use std::error::Error; -use std::fmt; - -/// Represents an error during lexical scanning. -#[derive(Debug, PartialEq)] -pub struct LexicalError { - pub error: LexicalErrorType, - pub location: Location, -} - -#[derive(Debug, PartialEq)] -pub enum LexicalErrorType { - StringError, - UnicodeError, - NestingError, - IndentationError, - TabError, - DefaultArgumentError, - PositionalArgumentError, - DuplicateKeywordArgumentError, - UnrecognizedToken { tok: char }, - FStringError(FStringErrorType), - LineContinuationError, - OtherError(String), -} - -impl fmt::Display for LexicalErrorType { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - LexicalErrorType::StringError => write!(f, "Got unexpected string"), - LexicalErrorType::FStringError(error) => write!(f, "Got error in f-string: {}", error), - LexicalErrorType::UnicodeError => write!(f, "Got unexpected unicode"), - LexicalErrorType::NestingError => write!(f, "Got unexpected nesting"), - LexicalErrorType::IndentationError => { - write!(f, "unindent does not match any outer indentation level") - } - LexicalErrorType::TabError => { - write!(f, "inconsistent use of tabs and spaces in indentation") - } - LexicalErrorType::DefaultArgumentError => { - write!(f, "non-default argument follows default argument") - } - LexicalErrorType::DuplicateKeywordArgumentError => { - write!(f, "keyword argument repeated") - } - LexicalErrorType::PositionalArgumentError => { - write!(f, "positional argument follows keyword argument") - } - LexicalErrorType::UnrecognizedToken { tok } => { - write!(f, "Got unexpected token {}", tok) - } - LexicalErrorType::LineContinuationError => { - write!(f, "unexpected character after line continuation character") - } - LexicalErrorType::OtherError(msg) => write!(f, "{}", msg), - } - } -} - -impl From for LalrpopError { - fn from(err: LexicalError) -> Self { - lalrpop_util::ParseError::User { error: err } - } -} - -// TODO: consolidate these with ParseError -#[derive(Debug, PartialEq)] -pub struct FStringError { - pub error: FStringErrorType, - pub location: Location, -} - -#[derive(Debug, PartialEq)] -pub enum FStringErrorType { - UnclosedLbrace, - UnopenedRbrace, - InvalidExpression(Box), - InvalidConversionFlag, - EmptyExpression, - MismatchedDelimiter, - ExpressionNestedTooDeeply, -} - -impl fmt::Display for FStringErrorType { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - FStringErrorType::UnclosedLbrace => write!(f, "Unclosed '('"), - FStringErrorType::UnopenedRbrace => write!(f, "Unopened ')'"), - FStringErrorType::InvalidExpression(error) => { - write!(f, "Invalid expression: {}", error) - } - FStringErrorType::InvalidConversionFlag => write!(f, "Invalid conversion flag"), - FStringErrorType::EmptyExpression => write!(f, "Empty expression"), - FStringErrorType::MismatchedDelimiter => write!(f, "Mismatched delimiter"), - FStringErrorType::ExpressionNestedTooDeeply => { - write!(f, "expressions nested too deeply") - } - } - } -} - -impl From for LalrpopError { - fn from(err: FStringError) -> Self { - lalrpop_util::ParseError::User { - error: LexicalError { - error: LexicalErrorType::FStringError(err.error), - location: err.location, - }, - } - } -} - -/// Represents an error during parsing -#[derive(Debug, PartialEq)] -pub struct ParseError { - pub error: ParseErrorType, - pub location: Location, -} - -#[derive(Debug, PartialEq)] -pub enum ParseErrorType { - /// Parser encountered an unexpected end of input - EOF, - /// Parser encountered an extra token - ExtraToken(Tok), - /// Parser encountered an invalid token - InvalidToken, - /// Parser encountered an unexpected token - UnrecognizedToken(Tok, Option), - /// Maps to `User` type from `lalrpop-util` - Lexical(LexicalErrorType), -} - -/// Convert `lalrpop_util::ParseError` to our internal type -impl From> for ParseError { - fn from(err: LalrpopError) -> Self { - match err { - // TODO: Are there cases where this isn't an EOF? - LalrpopError::InvalidToken { location } => ParseError { - error: ParseErrorType::EOF, - location, - }, - LalrpopError::ExtraToken { token } => ParseError { - error: ParseErrorType::ExtraToken(token.1), - location: token.0, - }, - LalrpopError::User { error } => ParseError { - error: ParseErrorType::Lexical(error.error), - location: error.location, - }, - LalrpopError::UnrecognizedToken { token, expected } => { - // Hacky, but it's how CPython does it. See PyParser_AddToken, - // in particular "Only one possible expected token" comment. - let expected = if expected.len() == 1 { - Some(expected[0].clone()) - } else { - None - }; - ParseError { - error: ParseErrorType::UnrecognizedToken(token.1, expected), - location: token.0, - } - } - LalrpopError::UnrecognizedEOF { location, .. } => ParseError { - error: ParseErrorType::EOF, - location, - }, - } - } -} - -impl fmt::Display for ParseError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "{} at {}", self.error, self.location) - } -} - -impl fmt::Display for ParseErrorType { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match *self { - ParseErrorType::EOF => write!(f, "Got unexpected EOF"), - ParseErrorType::ExtraToken(ref tok) => write!(f, "Got extraneous token: {:?}", tok), - ParseErrorType::InvalidToken => write!(f, "Got invalid token"), - ParseErrorType::UnrecognizedToken(ref tok, ref expected) => { - if *tok == Tok::Indent { - write!(f, "unexpected indent") - } else if expected.clone() == Some("Indent".to_owned()) { - write!(f, "expected an indented block") - } else { - write!(f, "Got unexpected token {}", tok) - } - } - ParseErrorType::Lexical(ref error) => write!(f, "{}", error), - } - } -} - -impl Error for ParseError { - fn source(&self) -> Option<&(dyn Error + 'static)> { - None - } -} +//! Define internal parse error types +//! The goal is to provide a matching and a safe error API, maksing errors from LALR +use lalrpop_util::ParseError as LalrpopError; + +use crate::location::Location; +use crate::token::Tok; + +use std::error::Error; +use std::fmt; + +/// Represents an error during lexical scanning. +#[derive(Debug, PartialEq)] +pub struct LexicalError { + pub error: LexicalErrorType, + pub location: Location, +} + +#[derive(Debug, PartialEq)] +pub enum LexicalErrorType { + StringError, + UnicodeError, + NestingError, + IndentationError, + TabError, + DefaultArgumentError, + PositionalArgumentError, + DuplicateKeywordArgumentError, + UnrecognizedToken { tok: char }, + FStringError(FStringErrorType), + LineContinuationError, + OtherError(String), +} + +impl fmt::Display for LexicalErrorType { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + LexicalErrorType::StringError => write!(f, "Got unexpected string"), + LexicalErrorType::FStringError(error) => write!(f, "Got error in f-string: {}", error), + LexicalErrorType::UnicodeError => write!(f, "Got unexpected unicode"), + LexicalErrorType::NestingError => write!(f, "Got unexpected nesting"), + LexicalErrorType::IndentationError => { + write!(f, "unindent does not match any outer indentation level") + } + LexicalErrorType::TabError => { + write!(f, "inconsistent use of tabs and spaces in indentation") + } + LexicalErrorType::DefaultArgumentError => { + write!(f, "non-default argument follows default argument") + } + LexicalErrorType::DuplicateKeywordArgumentError => { + write!(f, "keyword argument repeated") + } + LexicalErrorType::PositionalArgumentError => { + write!(f, "positional argument follows keyword argument") + } + LexicalErrorType::UnrecognizedToken { tok } => { + write!(f, "Got unexpected token {}", tok) + } + LexicalErrorType::LineContinuationError => { + write!(f, "unexpected character after line continuation character") + } + LexicalErrorType::OtherError(msg) => write!(f, "{}", msg), + } + } +} + +impl From for LalrpopError { + fn from(err: LexicalError) -> Self { + lalrpop_util::ParseError::User { error: err } + } +} + +// TODO: consolidate these with ParseError +#[derive(Debug, PartialEq)] +pub struct FStringError { + pub error: FStringErrorType, + pub location: Location, +} + +#[derive(Debug, PartialEq)] +pub enum FStringErrorType { + UnclosedLbrace, + UnopenedRbrace, + ExpectedRbrace, + InvalidExpression(Box), + InvalidConversionFlag, + EmptyExpression, + MismatchedDelimiter, + ExpressionNestedTooDeeply, +} + +impl fmt::Display for FStringErrorType { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + FStringErrorType::UnclosedLbrace => write!(f, "Unclosed '{{'"), + FStringErrorType::UnopenedRbrace => write!(f, "Unopened '}}'"), + FStringErrorType::ExpectedRbrace => write!(f, "Expected '}}' after conversion flag."), + FStringErrorType::InvalidExpression(error) => { + write!(f, "Invalid expression: {}", error) + } + FStringErrorType::InvalidConversionFlag => write!(f, "Invalid conversion flag"), + FStringErrorType::EmptyExpression => write!(f, "Empty expression"), + FStringErrorType::MismatchedDelimiter => write!(f, "Mismatched delimiter"), + FStringErrorType::ExpressionNestedTooDeeply => { + write!(f, "expressions nested too deeply") + } + } + } +} + +impl From for LalrpopError { + fn from(err: FStringError) -> Self { + lalrpop_util::ParseError::User { + error: LexicalError { + error: LexicalErrorType::FStringError(err.error), + location: err.location, + }, + } + } +} + +/// Represents an error during parsing +#[derive(Debug, PartialEq)] +pub struct ParseError { + pub error: ParseErrorType, + pub location: Location, +} + +#[derive(Debug, PartialEq)] +pub enum ParseErrorType { + /// Parser encountered an unexpected end of input + EOF, + /// Parser encountered an extra token + ExtraToken(Tok), + /// Parser encountered an invalid token + InvalidToken, + /// Parser encountered an unexpected token + UnrecognizedToken(Tok, Option), + /// Maps to `User` type from `lalrpop-util` + Lexical(LexicalErrorType), +} + +/// Convert `lalrpop_util::ParseError` to our internal type +impl From> for ParseError { + fn from(err: LalrpopError) -> Self { + match err { + // TODO: Are there cases where this isn't an EOF? + LalrpopError::InvalidToken { location } => ParseError { + error: ParseErrorType::EOF, + location, + }, + LalrpopError::ExtraToken { token } => ParseError { + error: ParseErrorType::ExtraToken(token.1), + location: token.0, + }, + LalrpopError::User { error } => ParseError { + error: ParseErrorType::Lexical(error.error), + location: error.location, + }, + LalrpopError::UnrecognizedToken { token, expected } => { + // Hacky, but it's how CPython does it. See PyParser_AddToken, + // in particular "Only one possible expected token" comment. + let expected = if expected.len() == 1 { + Some(expected[0].clone()) + } else { + None + }; + ParseError { + error: ParseErrorType::UnrecognizedToken(token.1, expected), + location: token.0, + } + } + LalrpopError::UnrecognizedEOF { location, .. } => ParseError { + error: ParseErrorType::EOF, + location, + }, + } + } +} + +impl fmt::Display for ParseError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{} at {}", self.error, self.location) + } +} + +impl fmt::Display for ParseErrorType { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match *self { + ParseErrorType::EOF => write!(f, "Got unexpected EOF"), + ParseErrorType::ExtraToken(ref tok) => write!(f, "Got extraneous token: {:?}", tok), + ParseErrorType::InvalidToken => write!(f, "Got invalid token"), + ParseErrorType::UnrecognizedToken(ref tok, ref expected) => { + if *tok == Tok::Indent { + write!(f, "unexpected indent") + } else if expected.clone() == Some("Indent".to_owned()) { + write!(f, "expected an indented block") + } else { + write!(f, "Got unexpected token {}", tok) + } + } + ParseErrorType::Lexical(ref error) => write!(f, "{}", error), + } + } +} + +impl Error for ParseError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + None + } +} diff --git a/parser/src/fstring.rs b/parser/src/fstring.rs index 5ca92c484bd..7d023a89fa4 100644 --- a/parser/src/fstring.rs +++ b/parser/src/fstring.rs @@ -1,304 +1,338 @@ -use std::iter; -use std::mem; -use std::str; - -use crate::ast::{ConversionFlag, StringGroup}; -use crate::error::{FStringError, FStringErrorType}; -use crate::location::Location; -use crate::parser::parse_expression; - -use self::FStringErrorType::*; -use self::StringGroup::*; - -struct FStringParser<'a> { - chars: iter::Peekable>, -} - -impl<'a> FStringParser<'a> { - fn new(source: &'a str) -> Self { - Self { - chars: source.chars().peekable(), - } - } - - fn parse_formatted_value(&mut self) -> Result { - let mut expression = String::new(); - let mut spec = None; - let mut delims = Vec::new(); - let mut conversion = None; - - while let Some(ch) = self.chars.next() { - match ch { - '!' if delims.is_empty() => { - conversion = Some(match self.chars.next() { - Some('s') => ConversionFlag::Str, - Some('a') => ConversionFlag::Ascii, - Some('r') => ConversionFlag::Repr, - Some(_) => { - return Err(InvalidConversionFlag); - } - None => { - break; - } - }) - } - ':' if delims.is_empty() => { - let mut nested = false; - let mut in_nested = false; - let mut spec_expression = String::new(); - while let Some(&next) = self.chars.peek() { - match next { - '{' => { - if in_nested { - return Err(ExpressionNestedTooDeeply); - } - in_nested = true; - nested = true; - self.chars.next(); - continue; - } - '}' => { - if in_nested { - in_nested = false; - self.chars.next(); - } - break; - } - _ => (), - } - spec_expression.push(next); - self.chars.next(); - } - if in_nested { - return Err(UnclosedLbrace); - } - if nested { - spec = Some(Box::new(FormattedValue { - value: Box::new( - parse_expression(spec_expression.trim()) - .map_err(|e| InvalidExpression(Box::new(e.error)))?, - ), - conversion: None, - spec: None, - })) - } else { - spec = Some(Box::new(Constant { - value: spec_expression.to_owned(), - })) - } - } - '(' | '{' | '[' => { - expression.push(ch); - delims.push(ch); - } - ')' => { - if delims.pop() != Some('(') { - return Err(MismatchedDelimiter); - } - expression.push(ch); - } - ']' => { - if delims.pop() != Some('[') { - return Err(MismatchedDelimiter); - } - expression.push(ch); - } - '}' if !delims.is_empty() => { - if delims.pop() != Some('{') { - return Err(MismatchedDelimiter); - } - expression.push(ch); - } - '}' => { - if expression.is_empty() { - return Err(EmptyExpression); - } - return Ok(FormattedValue { - value: Box::new( - parse_expression(expression.trim()) - .map_err(|e| InvalidExpression(Box::new(e.error)))?, - ), - conversion, - spec, - }); - } - '"' | '\'' => { - expression.push(ch); - while let Some(next) = self.chars.next() { - expression.push(next); - if next == ch { - break; - } - } - } - _ => { - expression.push(ch); - } - } - } - - Err(UnclosedLbrace) - } - - fn parse(mut self) -> Result { - let mut content = String::new(); - let mut values = vec![]; - - while let Some(ch) = self.chars.next() { - match ch { - '{' => { - if let Some('{') = self.chars.peek() { - self.chars.next(); - content.push('{'); - } else { - if !content.is_empty() { - values.push(Constant { - value: mem::replace(&mut content, String::new()), - }); - } - - values.push(self.parse_formatted_value()?); - } - } - '}' => { - if let Some('}') = self.chars.peek() { - self.chars.next(); - content.push('}'); - } else { - return Err(UnopenedRbrace); - } - } - _ => { - content.push(ch); - } - } - } - - if !content.is_empty() { - values.push(Constant { value: content }) - } - - Ok(match values.len() { - 0 => Constant { - value: String::new(), - }, - 1 => values.into_iter().next().unwrap(), - _ => Joined { values }, - }) - } -} - -/// Parse an f-string into a string group. -fn parse_fstring(source: &str) -> Result { - FStringParser::new(source).parse() -} - -/// Parse an fstring from a string, located at a certain position in the sourcecode. -/// In case of errors, we will get the location and the error returned. -pub fn parse_located_fstring( - source: &str, - location: Location, -) -> Result { - parse_fstring(source).map_err(|error| FStringError { error, location }) -} - -#[cfg(test)] -mod tests { - use crate::ast; - - use super::*; - - fn mk_ident(name: &str, row: usize, col: usize) -> ast::Expression { - ast::Expression { - location: ast::Location::new(row, col), - node: ast::ExpressionType::Identifier { - name: name.to_owned(), - }, - } - } - - #[test] - fn test_parse_fstring() { - let source = String::from("{a}{ b }{{foo}}"); - let parse_ast = parse_fstring(&source).unwrap(); - - assert_eq!( - parse_ast, - Joined { - values: vec![ - FormattedValue { - value: Box::new(mk_ident("a", 1, 1)), - conversion: None, - spec: None, - }, - FormattedValue { - value: Box::new(mk_ident("b", 1, 1)), - conversion: None, - spec: None, - }, - Constant { - value: "{foo}".to_owned() - } - ] - } - ); - } - - #[test] - fn test_parse_fstring_nested_spec() { - let source = String::from("{foo:{spec}}"); - let parse_ast = parse_fstring(&source).unwrap(); - - assert_eq!( - parse_ast, - FormattedValue { - value: Box::new(mk_ident("foo", 1, 1)), - conversion: None, - spec: Some(Box::new(FormattedValue { - value: Box::new(mk_ident("spec", 1, 1)), - conversion: None, - spec: None, - })), - } - ); - } - - #[test] - fn test_parse_fstring_not_nested_spec() { - let source = String::from("{foo:spec}"); - let parse_ast = parse_fstring(&source).unwrap(); - - assert_eq!( - parse_ast, - FormattedValue { - value: Box::new(mk_ident("foo", 1, 1)), - conversion: None, - spec: Some(Box::new(Constant { - value: "spec".to_owned(), - })), - } - ); - } - - #[test] - fn test_parse_empty_fstring() { - assert_eq!( - parse_fstring(""), - Ok(Constant { - value: String::new(), - }), - ); - } - - #[test] - fn test_parse_invalid_fstring() { - assert_eq!(parse_fstring("{"), Err(UnclosedLbrace)); - assert_eq!(parse_fstring("}"), Err(UnopenedRbrace)); - assert_eq!(parse_fstring("{a:{a:{b}}"), Err(ExpressionNestedTooDeeply)); - assert_eq!(parse_fstring("{a:b}}"), Err(UnopenedRbrace)); - assert_eq!(parse_fstring("{a:{b}"), Err(UnclosedLbrace)); - - // TODO: check for InvalidExpression enum? - assert!(parse_fstring("{class}").is_err()); - } -} +use std::iter; +use std::mem; +use std::str; + +use crate::ast::{ConversionFlag, StringGroup}; +use crate::error::{FStringError, FStringErrorType}; +use crate::location::Location; +use crate::parser::parse_expression; + +use self::FStringErrorType::*; +use self::StringGroup::*; + +struct FStringParser<'a> { + chars: iter::Peekable>, +} + +impl<'a> FStringParser<'a> { + fn new(source: &'a str) -> Self { + Self { + chars: source.chars().peekable(), + } + } + + fn parse_formatted_value(&mut self) -> Result { + let mut expression = String::new(); + let mut spec = None; + let mut delims = Vec::new(); + let mut conversion = None; + + while let Some(ch) = self.chars.next() { + match ch { + '!' if delims.is_empty() => { + let x = self.chars.next(); + if let Some('=') = x { + expression.push(ch); + expression.push('='); + } else { + if expression.is_empty() { + return Err(EmptyExpression); + } + + conversion = Some(match x { + Some('s') => ConversionFlag::Str, + Some('a') => ConversionFlag::Ascii, + Some('r') => ConversionFlag::Repr, + Some(_) => { + return Err(InvalidConversionFlag); + } + None => { + break; + } + }); + + match self.chars.peek() { + Some('}') => { + continue; + } + Some(_) => { + return Err(ExpectedRbrace); + } + None => { + break; + } + } + } + } + ':' if delims.is_empty() => { + let mut nested = false; + let mut in_nested = false; + let mut spec_expression = String::new(); + while let Some(&next) = self.chars.peek() { + match next { + '{' => { + if in_nested { + return Err(ExpressionNestedTooDeeply); + } + in_nested = true; + nested = true; + self.chars.next(); + continue; + } + '}' => { + if in_nested { + in_nested = false; + self.chars.next(); + } + break; + } + _ => (), + } + spec_expression.push(next); + self.chars.next(); + } + if in_nested { + return Err(UnclosedLbrace); + } + if nested { + spec = Some(Box::new(FormattedValue { + value: Box::new( + parse_expression(spec_expression.trim()) + .map_err(|e| InvalidExpression(Box::new(e.error)))?, + ), + conversion: None, + spec: None, + })) + } else { + spec = Some(Box::new(Constant { + value: spec_expression.to_owned(), + })) + } + } + '(' | '{' | '[' => { + expression.push(ch); + delims.push(ch); + } + ')' => { + if delims.pop() != Some('(') { + return Err(MismatchedDelimiter); + } + expression.push(ch); + } + ']' => { + if delims.pop() != Some('[') { + return Err(MismatchedDelimiter); + } + expression.push(ch); + } + '}' if !delims.is_empty() => { + if delims.pop() != Some('{') { + return Err(MismatchedDelimiter); + } + expression.push(ch); + } + '}' => { + if expression.is_empty() { + return Err(EmptyExpression); + } + return Ok(FormattedValue { + value: Box::new( + parse_expression(expression.trim()) + .map_err(|e| InvalidExpression(Box::new(e.error)))?, + ), + conversion, + spec, + }); + } + '"' | '\'' => { + expression.push(ch); + while let Some(next) = self.chars.next() { + expression.push(next); + if next == ch { + break; + } + } + } + _ => { + expression.push(ch); + } + } + } + + Err(UnclosedLbrace) + } + + fn parse(mut self) -> Result { + let mut content = String::new(); + let mut values = vec![]; + + while let Some(ch) = self.chars.next() { + match ch { + '{' => { + if let Some('{') = self.chars.peek() { + self.chars.next(); + content.push('{'); + } else { + if !content.is_empty() { + values.push(Constant { + value: mem::replace(&mut content, String::new()), + }); + } + + values.push(self.parse_formatted_value()?); + } + } + '}' => { + if let Some('}') = self.chars.peek() { + self.chars.next(); + content.push('}'); + } else { + return Err(UnopenedRbrace); + } + } + _ => { + content.push(ch); + } + } + } + + if !content.is_empty() { + values.push(Constant { value: content }) + } + + Ok(match values.len() { + 0 => Constant { + value: String::new(), + }, + 1 => values.into_iter().next().unwrap(), + _ => Joined { values }, + }) + } +} + +/// Parse an f-string into a string group. +fn parse_fstring(source: &str) -> Result { + FStringParser::new(source).parse() +} + +/// Parse an fstring from a string, located at a certain position in the sourcecode. +/// In case of errors, we will get the location and the error returned. +pub fn parse_located_fstring( + source: &str, + location: Location, +) -> Result { + parse_fstring(source).map_err(|error| FStringError { error, location }) +} + +#[cfg(test)] +mod tests { + use crate::ast; + + use super::*; + + fn mk_ident(name: &str, row: usize, col: usize) -> ast::Expression { + ast::Expression { + location: ast::Location::new(row, col), + node: ast::ExpressionType::Identifier { + name: name.to_owned(), + }, + } + } + + #[test] + fn test_parse_fstring() { + let source = String::from("{a}{ b }{{foo}}"); + let parse_ast = parse_fstring(&source).unwrap(); + + assert_eq!( + parse_ast, + Joined { + values: vec![ + FormattedValue { + value: Box::new(mk_ident("a", 1, 1)), + conversion: None, + spec: None, + }, + FormattedValue { + value: Box::new(mk_ident("b", 1, 1)), + conversion: None, + spec: None, + }, + Constant { + value: "{foo}".to_owned() + } + ] + } + ); + } + + #[test] + fn test_parse_fstring_nested_spec() { + let source = String::from("{foo:{spec}}"); + let parse_ast = parse_fstring(&source).unwrap(); + + assert_eq!( + parse_ast, + FormattedValue { + value: Box::new(mk_ident("foo", 1, 1)), + conversion: None, + spec: Some(Box::new(FormattedValue { + value: Box::new(mk_ident("spec", 1, 1)), + conversion: None, + spec: None, + })), + } + ); + } + + #[test] + fn test_parse_fstring_not_nested_spec() { + let source = String::from("{foo:spec}"); + let parse_ast = parse_fstring(&source).unwrap(); + + assert_eq!( + parse_ast, + FormattedValue { + value: Box::new(mk_ident("foo", 1, 1)), + conversion: None, + spec: Some(Box::new(Constant { + value: "spec".to_owned(), + })), + } + ); + } + + #[test] + fn test_parse_empty_fstring() { + assert_eq!( + parse_fstring(""), + Ok(Constant { + value: String::new(), + }), + ); + } + + #[test] + fn test_parse_invalid_fstring() { + assert_eq!(parse_fstring("{"), Err(UnclosedLbrace)); + assert_eq!(parse_fstring("}"), Err(UnopenedRbrace)); + assert_eq!(parse_fstring("{5!a"), Err(UnclosedLbrace)); + assert_eq!(parse_fstring("{5!a1}"), Err(ExpectedRbrace)); + assert_eq!(parse_fstring("abc{!a 'cat'}"), Err(EmptyExpression)); + assert_eq!(parse_fstring("{!x}"), Err(EmptyExpression)); + + assert_eq!(parse_fstring("{a:{a:{b}}"), Err(ExpressionNestedTooDeeply)); + assert_eq!(parse_fstring("{a:b}}"), Err(UnopenedRbrace)); + assert_eq!(parse_fstring("{a:{b}"), Err(UnclosedLbrace)); + + // TODO: check for InvalidExpression enum? + assert!(parse_fstring("{class}").is_err()); + } + + #[test] + fn test_parse_fstring_not_equals() { + let source = String::from("{1 != 2}"); + let parse_ast = parse_fstring(&source); + assert_ne!(parse_ast, Err(InvalidConversionFlag)); + } +}