From 36049cf891f06f1c40a3c92639e44c501b3687ed Mon Sep 17 00:00:00 2001 From: Paul Ganssle Date: Tue, 25 Apr 2023 11:22:48 -0600 Subject: [PATCH 01/12] Add stubs for hypothesis tests These are stubs to be used for adding hypothesis (https://hypothesis.readthedocs.io/en/latest/) tests to the standard library. When the tests are run in an environment where `hypothesis` and its various dependencies are not installed, the stubs will turn any tests with examples into simple parameterized tests and any tests without examples are skipped. --- .../support/_hypothesis_stubs/__init__.py | 102 ++++++++++++++++++ .../support/_hypothesis_stubs/_helpers.py | 33 ++++++ .../support/_hypothesis_stubs/strategies.py | 85 +++++++++++++++ Lib/test/support/hypothesis_helper.py | 4 + 4 files changed, 224 insertions(+) create mode 100644 Lib/test/support/_hypothesis_stubs/__init__.py create mode 100644 Lib/test/support/_hypothesis_stubs/_helpers.py create mode 100644 Lib/test/support/_hypothesis_stubs/strategies.py create mode 100644 Lib/test/support/hypothesis_helper.py diff --git a/Lib/test/support/_hypothesis_stubs/__init__.py b/Lib/test/support/_hypothesis_stubs/__init__.py new file mode 100644 index 00000000000000..edf9e2b4935453 --- /dev/null +++ b/Lib/test/support/_hypothesis_stubs/__init__.py @@ -0,0 +1,102 @@ +from enum import Enum +import functools +import unittest + +__all__ = [ + "given", + "example", + "assume", + "reject", + "register_random", + "strategies", + "HealthCheck", + "settings", + "Verbosity", +] + +from . import strategies + + +def given(*_args, **_kwargs): + def decorator(f): + if examples := getattr(f, "_examples", []): + + @functools.wraps(f) + def test_function(self): + for example_args, example_kwargs in examples: + with self.subTest(*example_args, **example_kwargs): + f(self, *example_args, **example_kwargs) + + else: + # If we have found no examples, we must skip the test. If @example + # is applied after @given, it will re-wrap the test to remove the + # skip decorator. + test_function = unittest.skip( + "Hypothesis required for property test with no " + + "specified examples" + )(f) + + test_function._given = True + return test_function + + return decorator + + +def example(*args, **kwargs): + if bool(args) == bool(kwargs): + raise ValueError("Must specify exactly one of *args or **kwargs") + + def decorator(f): + base_func = getattr(f, "__wrapped__", f) + if not hasattr(base_func, "_examples"): + base_func._examples = [] + + base_func._examples.append((args, kwargs)) + + if getattr(f, "_given", False): + # If the given decorator is below all the example decorators, + # it would be erroneously skipped, so we need to re-wrap the new + # base function. + f = given()(base_func) + + return f + + return decorator + + +def assume(condition): + if not condition: + raise unittest.SkipTest("Unsatisfied assumption") + return True + + +def reject(): + assume(False) + + +def register_random(*args, **kwargs): + pass # pragma: no cover + + +def settings(*args, **kwargs): + pass # pragma: nocover + + +class HealthCheck(Enum): + data_too_large = 1 + filter_too_much = 2 + too_slow = 3 + return_value = 5 + large_base_example = 7 + not_a_test_method = 8 + + @classmethod + def all(cls): + return list(cls) + + +class Verbosity(Enum): + quiet = 0 + normal = 1 + verbose = 2 + debug = 3 diff --git a/Lib/test/support/_hypothesis_stubs/_helpers.py b/Lib/test/support/_hypothesis_stubs/_helpers.py new file mode 100644 index 00000000000000..84c74725068d58 --- /dev/null +++ b/Lib/test/support/_hypothesis_stubs/_helpers.py @@ -0,0 +1,33 @@ +# Stub out only the subset of the interface that we actually use in our tests. +class StubClass: + def __init__(self, *args, **kwargs): + self.__stub_args = args + self.__stub_kwargs = kwargs + + def __repr__(self): + argstr = ", ".join(self.__stub_args) + kwargstr = ", ".join( + f"{kw}={val}" for kw, val in self.__stub_kwargs.items() + ) + + in_parens = argstr + if kwargstr: + in_parens += ", " + kwargstr + + return f"{self.__qualname__}({in_parens})" + + +def stub_factory(klass, name, _seen={}): + if (klass, name) not in _seen: + + class Stub(klass): + def __init__(self, *args, **kwargs): + super().__init__() + self.__stub_args = args + self.__stub_kwargs = kwargs + + Stub.__name__ = name + Stub.__qualname__ = name + _seen.setdefault((klass, name), Stub) + + return _seen[(klass, name)] diff --git a/Lib/test/support/_hypothesis_stubs/strategies.py b/Lib/test/support/_hypothesis_stubs/strategies.py new file mode 100644 index 00000000000000..76f759d22eb507 --- /dev/null +++ b/Lib/test/support/_hypothesis_stubs/strategies.py @@ -0,0 +1,85 @@ +import functools + +from ._helpers import StubClass, stub_factory + + +class StubStrategy(StubClass): + def map(self, pack): + return self + + def flatmap(self, expand): + return self + + def filter(self, condition): + return self + + def __or__(self, other): + return self + + +_STRATEGIES = { + "binary", + "booleans", + "builds", + "characters", + "complex_numbers", + "composite", + "data", + "dates", + "datetimes", + "decimals", + "deferred", + "dictionaries", + "emails", + "fixed_dictionaries", + "floats", + "fractions", + "from_regex", + "from_type", + "frozensets", + "functions" "integers", + "iterables", + "just", + "lists", + "none", + "nothing", + "one_of", + "permutations", + "random_module", + "randoms", + "recursive", + "register_type_strategy", + "runner", + "sampled_from", + "sets", + "shared", + "slices", + "timedeltas", + "times", + "text", + "tuples", + "uuids", +} + +__all__ = sorted(_STRATEGIES) + + +def composite(f): + strategy = stub_factory(StubStrategy, f.__name__) + + @functools.wraps(f) + def inner(*args, **kwargs): + return strategy(*args, **kwargs) + + return inner + + +def __getattr__(name): + if name not in _STRATEGIES: + raise AttributeError(f"Unknown attribute {name}") + + return stub_factory(StubStrategy, f"hypothesis.strategies.{name}") + + +def __dir__(): + return __all__ diff --git a/Lib/test/support/hypothesis_helper.py b/Lib/test/support/hypothesis_helper.py new file mode 100644 index 00000000000000..76bd2490fe6e3d --- /dev/null +++ b/Lib/test/support/hypothesis_helper.py @@ -0,0 +1,4 @@ +try: + import hypothesis +except ImportError: + from . import _hypothesis_stubs as hypothesis From 9bb7f5c02c55c6b0e86b2ea44c45ff07e4de059e Mon Sep 17 00:00:00 2001 From: Paul Ganssle Date: Tue, 25 Apr 2023 11:22:48 -0600 Subject: [PATCH 02/12] Add property tests for the zoneinfo module This migrates the tests from https://github.com/Zac-HD/stdlib-property-tests into the standard library, using the hypothesis stubs. --- Lib/test/test_zoneinfo/__init__.py | 1 + .../test_zoneinfo/test_zoneinfo_property.py | 318 ++++++++++++++++++ 2 files changed, 319 insertions(+) create mode 100644 Lib/test/test_zoneinfo/test_zoneinfo_property.py diff --git a/Lib/test/test_zoneinfo/__init__.py b/Lib/test/test_zoneinfo/__init__.py index 98cc4412ae16c2..c3ea567103275d 100644 --- a/Lib/test/test_zoneinfo/__init__.py +++ b/Lib/test/test_zoneinfo/__init__.py @@ -1 +1,2 @@ from .test_zoneinfo import * +from .test_zoneinfo_property import * diff --git a/Lib/test/test_zoneinfo/test_zoneinfo_property.py b/Lib/test/test_zoneinfo/test_zoneinfo_property.py new file mode 100644 index 00000000000000..c0b270085a2f58 --- /dev/null +++ b/Lib/test/test_zoneinfo/test_zoneinfo_property.py @@ -0,0 +1,318 @@ +import contextlib +import datetime +import os +import pickle +import unittest +import zoneinfo + +from test.support.hypothesis_helper import hypothesis + +import test.test_zoneinfo._support as test_support + +ZoneInfoTestBase = test_support.ZoneInfoTestBase + +py_zoneinfo, c_zoneinfo = test_support.get_modules() + +UTC = datetime.timezone.utc +MIN_UTC = datetime.datetime.min.replace(tzinfo=UTC) +MAX_UTC = datetime.datetime.max.replace(tzinfo=UTC) +ZERO = datetime.timedelta(0) + + +def _valid_keys(): + """Get available time zones, including posix/ and right/ directories.""" + from importlib import resources + + available_zones = sorted(zoneinfo.available_timezones()) + TZPATH = zoneinfo.TZPATH + + def valid_key(key): + for root in TZPATH: + key_file = os.path.join(root, key) + if os.path.exists(key_file): + return True + + components = key.split("/") + package_name = ".".join(["tzdata.zoneinfo"] + components[:-1]) + resource_name = components[-1] + + try: + return resources.files(package_name).joinpath(resource_name).is_file() + except ModuleNotFoundError: + return False + + # This relies on the fact that dictionaries maintain insertion order — for + # shrinking purposes, it is preferable to start with the standard version, + # then move to the posix/ version, then to the right/ version. + out_zones = {"": available_zones} + for prefix in ["posix", "right"]: + prefix_out = [] + for key in available_zones: + prefix_key = f"{prefix}/{key}" + if valid_key(prefix_key): + prefix_out.append(prefix_key) + + out_zones[prefix] = prefix_out + + output = [] + for keys in out_zones.values(): + output.extend(keys) + + return output + + +VALID_KEYS = _valid_keys() +if not VALID_KEYS: + raise unittest.SkipTest("No time zone data available") + + +def valid_keys(): + return hypothesis.strategies.sampled_from(VALID_KEYS) + + +class ZoneInfoTest(ZoneInfoTestBase): + module = py_zoneinfo + + @hypothesis.given(key=valid_keys()) + def test_str(self, key): + zi = self.klass(key) + self.assertEqual(str(zi), key) + + @hypothesis.given(key=valid_keys()) + def test_key(self, key): + zi = self.klass(key) + + self.assertEqual(zi.key, key) + + @hypothesis.given( + dt=hypothesis.strategies.one_of( + hypothesis.strategies.datetimes(), hypothesis.strategies.times() + ) + ) + def test_utc(self, dt): + zi = self.klass("UTC") + dt_zi = dt.replace(tzinfo=zi) + + self.assertEqual(dt_zi.utcoffset(), ZERO) + self.assertEqual(dt_zi.dst(), ZERO) + self.assertEqual(dt_zi.tzname(), "UTC") + + +class CZoneInfoTest(ZoneInfoTest): + module = c_zoneinfo + + +class ZoneInfoPickleTest(ZoneInfoTestBase): + module = py_zoneinfo + + def setUp(self): + with contextlib.ExitStack() as stack: + stack.enter_context(test_support.set_zoneinfo_module(self.module)) + self.addCleanup(stack.pop_all().close) + + super().setUp() + + @hypothesis.given(key=valid_keys()) + def test_pickle_unpickle_cache(self, key): + zi = self.klass(key) + pkl_str = pickle.dumps(zi) + zi_rt = pickle.loads(pkl_str) + + self.assertIs(zi, zi_rt) + + @hypothesis.given(key=valid_keys()) + def test_pickle_unpickle_no_cache(self, key): + zi = self.klass.no_cache(key) + pkl_str = pickle.dumps(zi) + zi_rt = pickle.loads(pkl_str) + + self.assertIsNot(zi, zi_rt) + self.assertEqual(str(zi), str(zi_rt)) + + @hypothesis.given(key=valid_keys()) + def test_pickle_unpickle_cache_multiple_rounds(self, key): + """Test that pickle/unpickle is idempotent.""" + zi_0 = self.klass(key) + pkl_str_0 = pickle.dumps(zi_0) + zi_1 = pickle.loads(pkl_str_0) + pkl_str_1 = pickle.dumps(zi_1) + zi_2 = pickle.loads(pkl_str_1) + pkl_str_2 = pickle.dumps(zi_2) + + self.assertEqual(pkl_str_0, pkl_str_1) + self.assertEqual(pkl_str_1, pkl_str_2) + + self.assertIs(zi_0, zi_1) + self.assertIs(zi_0, zi_2) + self.assertIs(zi_1, zi_2) + + @hypothesis.given(key=valid_keys()) + def test_pickle_unpickle_no_cache_multiple_rounds(self, key): + """Test that pickle/unpickle is idempotent.""" + zi_cache = self.klass(key) + + zi_0 = self.klass.no_cache(key) + pkl_str_0 = pickle.dumps(zi_0) + zi_1 = pickle.loads(pkl_str_0) + pkl_str_1 = pickle.dumps(zi_1) + zi_2 = pickle.loads(pkl_str_1) + pkl_str_2 = pickle.dumps(zi_2) + + self.assertEqual(pkl_str_0, pkl_str_1) + self.assertEqual(pkl_str_1, pkl_str_2) + + self.assertIsNot(zi_0, zi_1) + self.assertIsNot(zi_0, zi_2) + self.assertIsNot(zi_1, zi_2) + + self.assertIsNot(zi_0, zi_cache) + self.assertIsNot(zi_1, zi_cache) + self.assertIsNot(zi_2, zi_cache) + + +class CZoneInfoPickleTest(ZoneInfoPickleTest): + module = c_zoneinfo + + +class ZoneInfoCacheTest(ZoneInfoTestBase): + module = py_zoneinfo + + @hypothesis.given(key=valid_keys()) + def test_cache(self, key): + zi_0 = self.klass(key) + zi_1 = self.klass(key) + + self.assertIs(zi_0, zi_1) + + @hypothesis.given(key=valid_keys()) + def test_no_cache(self, key): + zi_0 = self.klass.no_cache(key) + zi_1 = self.klass.no_cache(key) + + self.assertIsNot(zi_0, zi_1) + + +class CZoneInfoCacheTest(ZoneInfoCacheTest): + klass = c_zoneinfo.ZoneInfo + + +class PythonCConsistencyTest(unittest.TestCase): + """Tests that the C and Python versions do the same thing.""" + + def _is_ambiguous(self, dt): + return dt.replace(fold=not dt.fold).utcoffset() == dt.utcoffset() + + @hypothesis.given(dt=hypothesis.strategies.datetimes(), key=valid_keys()) + def test_same_str(self, dt, key): + py_dt = dt.replace(tzinfo=py_zoneinfo.ZoneInfo(key)) + c_dt = dt.replace(tzinfo=c_zoneinfo.ZoneInfo(key)) + + self.assertEqual(str(py_dt), str(c_dt)) + + @hypothesis.given(dt=hypothesis.strategies.datetimes(), key=valid_keys()) + def test_same_offsets_and_names(self, dt, key): + py_dt = dt.replace(tzinfo=py_zoneinfo.ZoneInfo(key)) + c_dt = dt.replace(tzinfo=c_zoneinfo.ZoneInfo(key)) + + self.assertEqual(py_dt.tzname(), c_dt.tzname()) + self.assertEqual(py_dt.utcoffset(), c_dt.utcoffset()) + self.assertEqual(py_dt.dst(), c_dt.dst()) + + @hypothesis.given( + dt=hypothesis.strategies.datetimes(timezones=hypothesis.strategies.just(UTC)), + key=valid_keys(), + ) + @hypothesis.example(dt=MIN_UTC, key="Asia/Tokyo") + @hypothesis.example(dt=MAX_UTC, key="Asia/Tokyo") + @hypothesis.example(dt=MIN_UTC, key="America/New_York") + @hypothesis.example(dt=MAX_UTC, key="America/New_York") + @hypothesis.example( + dt=datetime.datetime(2006, 10, 29, 5, 15, tzinfo=UTC), + key="America/New_York", + ) + def test_same_from_utc(self, dt, key): + py_zi = py_zoneinfo.ZoneInfo(key) + c_zi = c_zoneinfo.ZoneInfo(key) + + # Convert to UTC: This can overflow, but we just care about consistency + py_overflow_exc = None + c_overflow_exc = None + try: + py_dt = dt.astimezone(py_zi) + except OverflowError as e: + py_overflow_exc = e + + try: + c_dt = dt.astimezone(c_zi) + except OverflowError as e: + c_overflow_exc = e + + if (py_overflow_exc is not None) != (c_overflow_exc is not None): + raise py_overflow_exc or c_overflow_exc # pragma: nocover + + if py_overflow_exc is not None: + return # Consistently raises the same exception + + # PEP 495 says that an inter-zone comparison between ambiguous + # datetimes is always False. + if py_dt != c_dt: + self.assertEqual( + self._is_ambiguous(py_dt), + self._is_ambiguous(c_dt), + (py_dt, c_dt), + ) + + self.assertEqual(py_dt.tzname(), c_dt.tzname()) + self.assertEqual(py_dt.utcoffset(), c_dt.utcoffset()) + self.assertEqual(py_dt.dst(), c_dt.dst()) + + @hypothesis.given(dt=hypothesis.strategies.datetimes(), key=valid_keys()) + @hypothesis.example(dt=datetime.datetime.max, key="America/New_York") + @hypothesis.example(dt=datetime.datetime.min, key="America/New_York") + @hypothesis.example(dt=datetime.datetime.min, key="Asia/Tokyo") + @hypothesis.example(dt=datetime.datetime.max, key="Asia/Tokyo") + def test_same_to_utc(self, dt, key): + py_dt = dt.replace(tzinfo=py_zoneinfo.ZoneInfo(key)) + c_dt = dt.replace(tzinfo=c_zoneinfo.ZoneInfo(key)) + + # Convert from UTC: Overflow OK if it happens in both implementations + py_overflow_exc = None + c_overflow_exc = None + try: + py_utc = py_dt.astimezone(UTC) + except OverflowError as e: + py_overflow_exc = e + + try: + c_utc = c_dt.astimezone(UTC) + except OverflowError as e: + c_overflow_exc = e + + if (py_overflow_exc is not None) != (c_overflow_exc is not None): + raise py_overflow_exc or c_overflow_exc # pragma: nocover + + if py_overflow_exc is not None: + return # Consistently raises the same exception + + self.assertEqual(py_utc, c_utc) + + @hypothesis.given(key=valid_keys()) + def test_cross_module_pickle(self, key): + py_zi = py_zoneinfo.ZoneInfo(key) + c_zi = c_zoneinfo.ZoneInfo(key) + + with test_support.set_zoneinfo_module(py_zoneinfo): + py_pkl = pickle.dumps(py_zi) + + with test_support.set_zoneinfo_module(c_zoneinfo): + c_pkl = pickle.dumps(c_zi) + + with test_support.set_zoneinfo_module(c_zoneinfo): + # Python → C + py_to_c_zi = pickle.loads(py_pkl) + self.assertIs(py_to_c_zi, c_zi) + + with test_support.set_zoneinfo_module(py_zoneinfo): + # C → Python + c_to_py_zi = pickle.loads(c_pkl) + self.assertIs(c_to_py_zi, py_zi) From 840aea1ec172da225c5b9da16e5ae0f593449d90 Mon Sep 17 00:00:00 2001 From: Paul Ganssle Date: Tue, 25 Apr 2023 11:22:48 -0600 Subject: [PATCH 03/12] Add examples to zoneinfo hypothesis tests --- .../test_zoneinfo/test_zoneinfo_property.py | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/Lib/test/test_zoneinfo/test_zoneinfo_property.py b/Lib/test/test_zoneinfo/test_zoneinfo_property.py index c0b270085a2f58..feaa77f3e7f0b9 100644 --- a/Lib/test/test_zoneinfo/test_zoneinfo_property.py +++ b/Lib/test/test_zoneinfo/test_zoneinfo_property.py @@ -70,15 +70,38 @@ def valid_keys(): return hypothesis.strategies.sampled_from(VALID_KEYS) +KEY_EXAMPLES = [ + "Africa/Abidjan", + "Africa/Casablanca", + "America/Los_Angeles", + "America/Santiago", + "Asia/Tokyo", + "Australia/Sydney", + "Europe/Dublin", + "Europe/Lisbon", + "Europe/London", + "Pacific/Kiritimati", + "UTC", +] + + +def add_key_examples(f): + for key in KEY_EXAMPLES: + f = hypothesis.example(key)(f) + return f + + class ZoneInfoTest(ZoneInfoTestBase): module = py_zoneinfo @hypothesis.given(key=valid_keys()) + @add_key_examples def test_str(self, key): zi = self.klass(key) self.assertEqual(str(zi), key) @hypothesis.given(key=valid_keys()) + @add_key_examples def test_key(self, key): zi = self.klass(key) @@ -89,6 +112,13 @@ def test_key(self, key): hypothesis.strategies.datetimes(), hypothesis.strategies.times() ) ) + @hypothesis.example(dt=datetime.datetime.min) + @hypothesis.example(dt=datetime.datetime.max) + @hypothesis.example(dt=datetime.datetime(1970, 1, 1)) + @hypothesis.example(dt=datetime.datetime(2039, 1, 1)) + @hypothesis.example(dt=datetime.time(0)) + @hypothesis.example(dt=datetime.time(12, 0)) + @hypothesis.example(dt=datetime.time(23, 59, 59, 999999)) def test_utc(self, dt): zi = self.klass("UTC") dt_zi = dt.replace(tzinfo=zi) @@ -113,6 +143,7 @@ def setUp(self): super().setUp() @hypothesis.given(key=valid_keys()) + @add_key_examples def test_pickle_unpickle_cache(self, key): zi = self.klass(key) pkl_str = pickle.dumps(zi) @@ -121,6 +152,7 @@ def test_pickle_unpickle_cache(self, key): self.assertIs(zi, zi_rt) @hypothesis.given(key=valid_keys()) + @add_key_examples def test_pickle_unpickle_no_cache(self, key): zi = self.klass.no_cache(key) pkl_str = pickle.dumps(zi) @@ -130,6 +162,7 @@ def test_pickle_unpickle_no_cache(self, key): self.assertEqual(str(zi), str(zi_rt)) @hypothesis.given(key=valid_keys()) + @add_key_examples def test_pickle_unpickle_cache_multiple_rounds(self, key): """Test that pickle/unpickle is idempotent.""" zi_0 = self.klass(key) @@ -147,6 +180,7 @@ def test_pickle_unpickle_cache_multiple_rounds(self, key): self.assertIs(zi_1, zi_2) @hypothesis.given(key=valid_keys()) + @add_key_examples def test_pickle_unpickle_no_cache_multiple_rounds(self, key): """Test that pickle/unpickle is idempotent.""" zi_cache = self.klass(key) @@ -178,6 +212,7 @@ class ZoneInfoCacheTest(ZoneInfoTestBase): module = py_zoneinfo @hypothesis.given(key=valid_keys()) + @add_key_examples def test_cache(self, key): zi_0 = self.klass(key) zi_1 = self.klass(key) @@ -185,6 +220,7 @@ def test_cache(self, key): self.assertIs(zi_0, zi_1) @hypothesis.given(key=valid_keys()) + @add_key_examples def test_no_cache(self, key): zi_0 = self.klass.no_cache(key) zi_1 = self.klass.no_cache(key) @@ -203,6 +239,11 @@ def _is_ambiguous(self, dt): return dt.replace(fold=not dt.fold).utcoffset() == dt.utcoffset() @hypothesis.given(dt=hypothesis.strategies.datetimes(), key=valid_keys()) + @hypothesis.example(dt=datetime.datetime.min, key="America/New_York") + @hypothesis.example(dt=datetime.datetime.max, key="America/New_York") + @hypothesis.example(dt=datetime.datetime(1970, 1, 1), key="America/New_York") + @hypothesis.example(dt=datetime.datetime(2020, 1, 1), key="Europe/Paris") + @hypothesis.example(dt=datetime.datetime(2020, 6, 1), key="Europe/Paris") def test_same_str(self, dt, key): py_dt = dt.replace(tzinfo=py_zoneinfo.ZoneInfo(key)) c_dt = dt.replace(tzinfo=c_zoneinfo.ZoneInfo(key)) @@ -210,6 +251,14 @@ def test_same_str(self, dt, key): self.assertEqual(str(py_dt), str(c_dt)) @hypothesis.given(dt=hypothesis.strategies.datetimes(), key=valid_keys()) + @hypothesis.example(dt=datetime.datetime(1970, 1, 1), key="America/New_York") + @hypothesis.example(dt=datetime.datetime(2020, 2, 5), key="America/New_York") + @hypothesis.example(dt=datetime.datetime(2020, 8, 12), key="America/New_York") + @hypothesis.example(dt=datetime.datetime(2040, 1, 1), key="Africa/Casablanca") + @hypothesis.example(dt=datetime.datetime(1970, 1, 1), key="Europe/Paris") + @hypothesis.example(dt=datetime.datetime(2040, 1, 1), key="Europe/Paris") + @hypothesis.example(dt=datetime.datetime.min, key="Asia/Tokyo") + @hypothesis.example(dt=datetime.datetime.max, key="Asia/Tokyo") def test_same_offsets_and_names(self, dt, key): py_dt = dt.replace(tzinfo=py_zoneinfo.ZoneInfo(key)) c_dt = dt.replace(tzinfo=c_zoneinfo.ZoneInfo(key)) @@ -297,6 +346,7 @@ def test_same_to_utc(self, dt, key): self.assertEqual(py_utc, c_utc) @hypothesis.given(key=valid_keys()) + @add_key_examples def test_cross_module_pickle(self, key): py_zi = py_zoneinfo.ZoneInfo(key) c_zi = c_zoneinfo.ZoneInfo(key) From 87c6bdbe4c10897a6514e663037ee29eef4574c6 Mon Sep 17 00:00:00 2001 From: Paul Ganssle Date: Tue, 25 Apr 2023 11:25:45 -0600 Subject: [PATCH 04/12] Enable settings to operate as a decorator Co-authored-by: Zac Hatfield-Dodds --- Lib/test/support/_hypothesis_stubs/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/support/_hypothesis_stubs/__init__.py b/Lib/test/support/_hypothesis_stubs/__init__.py index edf9e2b4935453..d2a0c5466a9b13 100644 --- a/Lib/test/support/_hypothesis_stubs/__init__.py +++ b/Lib/test/support/_hypothesis_stubs/__init__.py @@ -79,7 +79,7 @@ def register_random(*args, **kwargs): def settings(*args, **kwargs): - pass # pragma: nocover + return lambda f: f # pragma: nocover class HealthCheck(Enum): From c97ec97eb56ade742eb1d186b54cc1c77d12ca0b Mon Sep 17 00:00:00 2001 From: Paul Ganssle Date: Tue, 25 Apr 2023 11:28:35 -0600 Subject: [PATCH 05/12] Add Phase enum --- Lib/test/support/_hypothesis_stubs/__init__.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Lib/test/support/_hypothesis_stubs/__init__.py b/Lib/test/support/_hypothesis_stubs/__init__.py index d2a0c5466a9b13..6ba5bb814b92f7 100644 --- a/Lib/test/support/_hypothesis_stubs/__init__.py +++ b/Lib/test/support/_hypothesis_stubs/__init__.py @@ -100,3 +100,12 @@ class Verbosity(Enum): normal = 1 verbose = 2 debug = 3 + + +class Phase(Enum): + explicit = 0 + reuse = 1 + generate = 2 + target = 3 + shrink = 4 + explain = 5 From cd3ddd728848bf44ab4b3858ff1c8c8ff5daf92b Mon Sep 17 00:00:00 2001 From: Paul Ganssle Date: Tue, 25 Apr 2023 12:10:48 -0600 Subject: [PATCH 06/12] Make reprs more accurate --- .../support/_hypothesis_stubs/_helpers.py | 24 +++++++++++++------ .../support/_hypothesis_stubs/strategies.py | 16 +++++++++---- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/Lib/test/support/_hypothesis_stubs/_helpers.py b/Lib/test/support/_hypothesis_stubs/_helpers.py index 84c74725068d58..3f6244e4dbc0b1 100644 --- a/Lib/test/support/_hypothesis_stubs/_helpers.py +++ b/Lib/test/support/_hypothesis_stubs/_helpers.py @@ -3,21 +3,28 @@ class StubClass: def __init__(self, *args, **kwargs): self.__stub_args = args self.__stub_kwargs = kwargs + self.__repr = None + + def _with_repr(self, new_repr): + new_obj = self.__class__(*self.__stub_args, **self.__stub_kwargs) + new_obj.__repr = new_repr + return new_obj def __repr__(self): + if self.__repr is not None: + return self.__repr + argstr = ", ".join(self.__stub_args) - kwargstr = ", ".join( - f"{kw}={val}" for kw, val in self.__stub_kwargs.items() - ) + kwargstr = ", ".join(f"{kw}={val}" for kw, val in self.__stub_kwargs.items()) in_parens = argstr if kwargstr: in_parens += ", " + kwargstr - return f"{self.__qualname__}({in_parens})" + return f"{self.__class__.__qualname__}({in_parens})" -def stub_factory(klass, name, _seen={}): +def stub_factory(klass, name, *, with_repr=None, _seen={}): if (klass, name) not in _seen: class Stub(klass): @@ -28,6 +35,9 @@ def __init__(self, *args, **kwargs): Stub.__name__ = name Stub.__qualname__ = name - _seen.setdefault((klass, name), Stub) + if with_repr is not None: + Stub._repr = None + + _seen.setdefault((klass, name, with_repr), Stub) - return _seen[(klass, name)] + return _seen[(klass, name, with_repr)] diff --git a/Lib/test/support/_hypothesis_stubs/strategies.py b/Lib/test/support/_hypothesis_stubs/strategies.py index 76f759d22eb507..d2b885d41e16d0 100644 --- a/Lib/test/support/_hypothesis_stubs/strategies.py +++ b/Lib/test/support/_hypothesis_stubs/strategies.py @@ -4,17 +4,22 @@ class StubStrategy(StubClass): + def __make_trailing_repr(self, transformation_name, func): + func_name = func.__name__ or repr(func) + return f"{self!r}.{transformation_name}({func_name})" + def map(self, pack): - return self + return self._with_repr(self.__make_trailing_repr("map", pack)) def flatmap(self, expand): - return self + return self._with_repr(self.__make_trailing_repr("flatmap", expand)) def filter(self, condition): - return self + return self._with_repr(self.__make_trailing_repr("filter", condition)) def __or__(self, other): - return self + new_repr = f"one_of({self!r}, {other!r})" + return self._with_repr(new_repr) _STRATEGIES = { @@ -37,7 +42,8 @@ def __or__(self, other): "from_regex", "from_type", "frozensets", - "functions" "integers", + "functions", + "integers", "iterables", "just", "lists", From 57a357b1510666cb4e27e4363ac6b4deaa4bbbc2 Mon Sep 17 00:00:00 2001 From: Paul Ganssle Date: Thu, 27 Apr 2023 13:07:16 -0600 Subject: [PATCH 07/12] Hard-code ignoring hypothesis files in libregrtest --- Lib/test/libregrtest/save_env.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/libregrtest/save_env.py b/Lib/test/libregrtest/save_env.py index cc5870ab2b833e..c7801b767c590c 100644 --- a/Lib/test/libregrtest/save_env.py +++ b/Lib/test/libregrtest/save_env.py @@ -257,8 +257,10 @@ def restore_sysconfig__INSTALL_SCHEMES(self, saved): sysconfig._INSTALL_SCHEMES.update(saved[2]) def get_files(self): + # XXX: Maybe add an allow-list here? return sorted(fn + ('/' if os.path.isdir(fn) else '') - for fn in os.listdir()) + for fn in os.listdir() + if not fn.startswith(".hypothesis")) def restore_files(self, saved_value): fn = os_helper.TESTFN if fn not in saved_value and (fn + '/') not in saved_value: From 176cc6b4d5fdb70eeb321c12823d51bb3c93ba95 Mon Sep 17 00:00:00 2001 From: Paul Ganssle Date: Tue, 25 Apr 2023 12:19:45 -0600 Subject: [PATCH 08/12] Add news entry --- .../next/Tests/2023-04-25-12-19-37.gh-issue-86275.-RoLIt.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Tests/2023-04-25-12-19-37.gh-issue-86275.-RoLIt.rst diff --git a/Misc/NEWS.d/next/Tests/2023-04-25-12-19-37.gh-issue-86275.-RoLIt.rst b/Misc/NEWS.d/next/Tests/2023-04-25-12-19-37.gh-issue-86275.-RoLIt.rst new file mode 100644 index 00000000000000..37ab74ffc441ed --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-04-25-12-19-37.gh-issue-86275.-RoLIt.rst @@ -0,0 +1,2 @@ +Added property-based tests to the :mod:`zoneinfo` tests, along with stubs +for the ``hypothesis`` interface. (Patch by Paul Ganssle) From fd4391c45c5afa6daa6f9f65eb6e9e51d735fb66 Mon Sep 17 00:00:00 2001 From: Paul Ganssle Date: Wed, 26 Apr 2023 11:08:44 -0600 Subject: [PATCH 09/12] Add Azure Pipelines CI and PR jobs for hypothesis --- .azure-pipelines/ci.yml | 25 ++++++++++++++++++++++++ .azure-pipelines/posix-steps.yml | 33 +++++++++++++++++++++++++++++++- .azure-pipelines/pr.yml | 28 +++++++++++++++++++++++++++ 3 files changed, 85 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml index 6302b547982118..fe63c2c496710a 100644 --- a/.azure-pipelines/ci.yml +++ b/.azure-pipelines/ci.yml @@ -1,5 +1,6 @@ variables: coverage: false + hypothesis: true trigger: ['main', '3.11', '3.10', '3.9', '3.8', '3.7'] @@ -91,6 +92,30 @@ jobs: dependencies: apt coverage: true +- job: Ubuntu_Hypothesis_CI_Tests + displayName: Ubuntu CI Tests (Hypothesis) + dependsOn: Prebuild + condition: | + and( + succeeded(), + eq(variables['hypothesis'], 'true'), + eq(dependencies.Prebuild.outputs['tests.run'], 'true'), + notIn(variables['Build.SourceBranchName'], '3.8', '3.9', '3.10', '3.11') + ) + + pool: + vmImage: ubuntu-22.04 + + variables: + testRunTitle: '$(Build.SourceBranchName)-linux-hypothesis' + testRunPlatform: linux-hypothesis + openssl_version: 1.1.1t + + steps: + - template: ./posix-steps.yml + parameters: + dependencies: apt + hypothesis: true - job: Windows_CI_Tests displayName: Windows CI Tests diff --git a/.azure-pipelines/posix-steps.yml b/.azure-pipelines/posix-steps.yml index 9d7c5e1279f46d..8394389bd32d21 100644 --- a/.azure-pipelines/posix-steps.yml +++ b/.azure-pipelines/posix-steps.yml @@ -1,4 +1,5 @@ parameters: + hypothesis: false coverage: false sudo_dependencies: sudo dependencies: apt @@ -53,7 +54,7 @@ steps: displayName: 'Publish code coverage results' -- ${{ if ne(parameters.coverage, 'true') }}: +- ${{ if and(ne(parameters.coverage, 'true'), ne(parameters.hypothesis, 'true')) }}: - script: make pythoninfo displayName: 'Display build info' @@ -72,6 +73,36 @@ steps: displayName: 'Run patchcheck.py' condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest')) +- ${{ if eq(parameters.hypothesis, 'true') }}: + - script: ./python -m venv hypovenv && ./hypovenv/bin/python -m pip install -U hypothesis + displayName: 'Set up virtual environment' + + - script: ./hypovenv/bin/python -m test.pythoninfo + displayName: 'Display build info' + + - script: | + # Most of the excluded tests are slow test suites with no property tests + # + # (GH-104097) test_sysconfig is skipped because it has tests that are + # failing when executed from inside a virtual environment. + $COMMAND -m test \ + -W \ + -x test_asyncio \ + -x test_multiprocessing_fork \ + -x test_multiprocessing_forkserver \ + -x test_multiprocessing_spawn \ + -x test_concurrent_futures \ + -x test_socket \ + -x test_subprocess \ + -x test_signal \ + -x test_sysconfig + displayName: 'Run tests with hypothesis installed' + env: + ${{ if eq(parameters.xvfb, 'true') }}: + COMMAND: xvfb-run ./hypovenv/bin/python + ${{ if ne(parameters.xvfb, 'true') }}: + COMMAND: ./hypovenv/bin/python + - task: PublishTestResults@2 displayName: 'Publish Test Results' diff --git a/.azure-pipelines/pr.yml b/.azure-pipelines/pr.yml index 5f7218768c18af..2fb39b725e97ea 100644 --- a/.azure-pipelines/pr.yml +++ b/.azure-pipelines/pr.yml @@ -1,5 +1,6 @@ variables: coverage: false + hypothesis: true pr: ['main', '3.11', '3.10', '3.9', '3.8', '3.7'] @@ -92,6 +93,33 @@ jobs: coverage: true +- job: Ubuntu_Hypothesis_PR_Tests + displayName: Ubuntu PR Tests (Hypothesis) + dependsOn: Prebuild + condition: | + and( + succeeded(), + eq(variables['hypothesis'], 'true'), + eq(dependencies.Prebuild.outputs['tests.run'], 'true'), + notIn(variables['System.PullRequest.targetBranchName'], + '3.8', '3.9', '3.10', '3.11') + ) + + pool: + vmImage: ubuntu-22.04 + + variables: + testRunTitle: '$(Build.SourceBranchName)-linux-hypothesis' + testRunPlatform: linux-hypothesis + openssl_version: 1.1.1t + + steps: + - template: ./posix-steps.yml + parameters: + dependencies: apt + hypothesis: true + + - job: Windows_PR_Tests displayName: Windows PR Tests dependsOn: Prebuild From 46cbf7ddcbdc8102f2f3bc3bbbd9ca50915c9406 Mon Sep 17 00:00:00 2001 From: Paul Ganssle Date: Wed, 26 Apr 2023 11:38:38 -0600 Subject: [PATCH 10/12] Add GHA job to run Hypothesis tests This version of the GHA tries to build CPython once, package it in a tarball, then re-use it for the normal tests and the hypothesis tests. It doesn't seem to be working. It excludes many of the slowest tests manually, plus one test that is broken for unrelated reasons. --- .github/workflows/build.yml | 96 ++++++++++++++++++++++++++++++++++++- 1 file changed, 94 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index df0f107a541614..193f334002ba48 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -36,6 +36,7 @@ jobs: timeout-minutes: 10 outputs: run_tests: ${{ steps.check.outputs.run_tests }} + run_hypothesis: ${{ steps.check.outputs.run_hypothesis }} steps: - uses: actions/checkout@v3 - name: Check for source changes @@ -61,6 +62,17 @@ jobs: git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qvE '(\.rst$|^Doc|^Misc)' && echo "run_tests=true" >> $GITHUB_OUTPUT || true fi + # Check if we should run hypothesis tests + GIT_BRANCH=${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}} + echo $GIT_BRANCH + if $(echo "$GIT_BRANCH" | grep -q -w '3\.\(8\|9\|10\|11\)'); then + echo "Branch too old for hypothesis tests" + echo "run_hypothesis=false" >> $GITHUB_OUTPUT + else + echo "Run hypothesis tests" + echo "run_hypothesis=true" >> $GITHUB_OUTPUT + fi + check_generated_files: name: 'Check if generated files are up to date' runs-on: ubuntu-latest @@ -193,7 +205,10 @@ jobs: env: OPENSSL_VER: 1.1.1t PYTHONSTRICTEXTENSIONBUILD: 1 + RUN_HYPOTHESIS: ${{needs.check_source.outputs.run_hypothesis}} steps: + - name: Print run_hypothesis + run: echo $RUN_HYPOTHESIS - uses: actions/checkout@v3 - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" @@ -238,9 +253,18 @@ jobs: - name: Remount sources writable for tests # some tests write to srcdir, lack of pyc files slows down testing run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw - - name: Tests + - name: "Create archive of build for other jobs" working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu" + run: | + # This complicated invocation is required to get a tarball with the + # build directory at its root, including all hidden files. + find . -printf "%P\n" | \ + tar -czf ${GITHUB_WORKSPACE}/ubuntu-build.tar.gz --no-recursion . -T - + - name: "Upload artifact for other jobs" + uses: actions/upload-artifact@v3 + with: + name: ubuntu-latest-build + path: "ubuntu-build.tar.gz" build_ubuntu_ssltests: name: 'Ubuntu SSL tests with OpenSSL' @@ -291,6 +315,74 @@ jobs: - name: SSL tests run: ./python Lib/test/ssltests.py + test_ubuntu: + name: "Tests on Ubuntu" + runs-on: ubuntu-20.04 + needs: build_ubuntu + if: needs.check_source.outputs.run_tests == 'true' + steps: + - name: Setup directory envs for out-of-tree builds + run: | + echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV + - name: "Download build artifact" + uses: actions/download-artifact@v3 + with: + name: ubuntu-latest-build + - name: "Unpack build" + run: | + mkdir -p ${{ env.CPYTHON_BUILDDIR }} + tar -xf ubuntu-build.tar.gz -C ${{ env.CPYTHON_BUILDDIR }} + - name: Tests + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu" + + test_hypothesis: + name: "Hypothesis Tests on Ubuntu" + runs-on: ubuntu-20.04 + needs: build_ubuntu + if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_hypothesis == 'true' + env: + OPENSSL_VER: 1.1.1t + PYTHONSTRICTEXTENSIONBUILD: 1 + steps: + - name: Setup directory envs for out-of-tree builds + run: | + echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV + - name: "Download build artifact" + uses: actions/download-artifact@v3 + with: + name: ubuntu-latest-build + - name: "Unpack build" + run: | + mkdir -p ${{ env.CPYTHON_BUILDDIR }} + tar -xf ubuntu-build.tar.gz -C ${{ env.CPYTHON_BUILDDIR }} + - name: "Create hypothesis venv" + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: | + VENV_LOC=$(realpath -m .)/hypovenv + VENV_PYTHON=$VENV_LOC/bin/python + echo "HYPOVENV=${VENV_LOC}" >> $GITHUB_ENV + echo "VENV_PYTHON=${VENV_PYTHON}" >> $GITHUB_ENV + ./python -m venv $VENV_LOC && $VENV_PYTHON -m pip install -U hypothesis + - name: "Run tests" + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: | + # Most of the excluded tests are slow test suites with no property tests + # + # (GH-104097) test_sysconfig is skipped because it has tests that are + # failing when executed from inside a virtual environment. + ${{ env.VENV_PYTHON }} -m test \ + -W \ + -x test_asyncio \ + -x test_multiprocessing_fork \ + -x test_multiprocessing_forkserver \ + -x test_multiprocessing_spawn \ + -x test_concurrent_futures \ + -x test_socket \ + -x test_subprocess \ + -x test_signal \ + -x test_sysconfig + build_asan: name: 'Address sanitizer' From 728f5e04e497b6c95c0c94c9ecf230bb0db1d436 Mon Sep 17 00:00:00 2001 From: Paul Ganssle Date: Sun, 30 Apr 2023 15:14:39 -0400 Subject: [PATCH 11/12] Use independent build stages for hypothesis GHA job --- .github/workflows/build.yml | 90 +++++++++++++++++++------------------ 1 file changed, 47 insertions(+), 43 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 193f334002ba48..f0a2dd5694c078 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -205,10 +205,7 @@ jobs: env: OPENSSL_VER: 1.1.1t PYTHONSTRICTEXTENSIONBUILD: 1 - RUN_HYPOTHESIS: ${{needs.check_source.outputs.run_hypothesis}} steps: - - name: Print run_hypothesis - run: echo $RUN_HYPOTHESIS - uses: actions/checkout@v3 - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" @@ -253,18 +250,9 @@ jobs: - name: Remount sources writable for tests # some tests write to srcdir, lack of pyc files slows down testing run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw - - name: "Create archive of build for other jobs" + - name: Tests working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: | - # This complicated invocation is required to get a tarball with the - # build directory at its root, including all hidden files. - find . -printf "%P\n" | \ - tar -czf ${GITHUB_WORKSPACE}/ubuntu-build.tar.gz --no-recursion . -T - - - name: "Upload artifact for other jobs" - uses: actions/upload-artifact@v3 - with: - name: ubuntu-latest-build - path: "ubuntu-build.tar.gz" + run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu" build_ubuntu_ssltests: name: 'Ubuntu SSL tests with OpenSSL' @@ -315,47 +303,63 @@ jobs: - name: SSL tests run: ./python Lib/test/ssltests.py - test_ubuntu: - name: "Tests on Ubuntu" - runs-on: ubuntu-20.04 - needs: build_ubuntu - if: needs.check_source.outputs.run_tests == 'true' - steps: - - name: Setup directory envs for out-of-tree builds - run: | - echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV - - name: "Download build artifact" - uses: actions/download-artifact@v3 - with: - name: ubuntu-latest-build - - name: "Unpack build" - run: | - mkdir -p ${{ env.CPYTHON_BUILDDIR }} - tar -xf ubuntu-build.tar.gz -C ${{ env.CPYTHON_BUILDDIR }} - - name: Tests - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu" - test_hypothesis: name: "Hypothesis Tests on Ubuntu" runs-on: ubuntu-20.04 - needs: build_ubuntu + timeout-minutes: 60 + needs: check_source if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_hypothesis == 'true' env: OPENSSL_VER: 1.1.1t PYTHONSTRICTEXTENSIONBUILD: 1 steps: + - uses: actions/checkout@v3 + - name: Register gcc problem matcher + run: echo "::add-matcher::.github/problem-matchers/gcc.json" + - name: Install Dependencies + run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Configure OpenSSL env vars + run: | + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV + - name: 'Restore OpenSSL build' + id: cache-openssl + uses: actions/cache@v3 + with: + path: ./multissl/openssl/${{ env.OPENSSL_VER }} + key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + - name: Install OpenSSL + if: steps.cache-openssl.outputs.cache-hit != 'true' + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux + - name: Add ccache to PATH + run: | + echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + - name: Configure ccache action + uses: hendrikmuhs/ccache-action@v1.2 - name: Setup directory envs for out-of-tree builds run: | + echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV - - name: "Download build artifact" - uses: actions/download-artifact@v3 - with: - name: ubuntu-latest-build - - name: "Unpack build" + - name: Create directories for read-only out-of-tree builds + run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR + - name: Bind mount sources read-only + run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR + - name: Configure CPython out-of-tree + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: ../cpython-ro-srcdir/configure --with-pydebug --with-openssl=$OPENSSL_DIR + - name: Build CPython out-of-tree + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: make -j4 + - name: Display build info + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: make pythoninfo + - name: Remount sources writable for tests + # some tests write to srcdir, lack of pyc files slows down testing + run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw + - name: Setup directory envs for out-of-tree builds run: | - mkdir -p ${{ env.CPYTHON_BUILDDIR }} - tar -xf ubuntu-build.tar.gz -C ${{ env.CPYTHON_BUILDDIR }} + echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV - name: "Create hypothesis venv" working-directory: ${{ env.CPYTHON_BUILDDIR }} run: | From a305d29c8f0907d9a1de88e98b6f80e1e3949074 Mon Sep 17 00:00:00 2001 From: Paul Ganssle Date: Wed, 3 May 2023 11:09:17 -0400 Subject: [PATCH 12/12] Revert "Add Azure Pipelines CI and PR jobs for hypothesis" This reverts commit 5ba1a6a5a701c333278bce5554d93529177b9984. --- .azure-pipelines/ci.yml | 25 ------------------------ .azure-pipelines/posix-steps.yml | 33 +------------------------------- .azure-pipelines/pr.yml | 28 --------------------------- 3 files changed, 1 insertion(+), 85 deletions(-) diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml index fe63c2c496710a..6302b547982118 100644 --- a/.azure-pipelines/ci.yml +++ b/.azure-pipelines/ci.yml @@ -1,6 +1,5 @@ variables: coverage: false - hypothesis: true trigger: ['main', '3.11', '3.10', '3.9', '3.8', '3.7'] @@ -92,30 +91,6 @@ jobs: dependencies: apt coverage: true -- job: Ubuntu_Hypothesis_CI_Tests - displayName: Ubuntu CI Tests (Hypothesis) - dependsOn: Prebuild - condition: | - and( - succeeded(), - eq(variables['hypothesis'], 'true'), - eq(dependencies.Prebuild.outputs['tests.run'], 'true'), - notIn(variables['Build.SourceBranchName'], '3.8', '3.9', '3.10', '3.11') - ) - - pool: - vmImage: ubuntu-22.04 - - variables: - testRunTitle: '$(Build.SourceBranchName)-linux-hypothesis' - testRunPlatform: linux-hypothesis - openssl_version: 1.1.1t - - steps: - - template: ./posix-steps.yml - parameters: - dependencies: apt - hypothesis: true - job: Windows_CI_Tests displayName: Windows CI Tests diff --git a/.azure-pipelines/posix-steps.yml b/.azure-pipelines/posix-steps.yml index 8394389bd32d21..9d7c5e1279f46d 100644 --- a/.azure-pipelines/posix-steps.yml +++ b/.azure-pipelines/posix-steps.yml @@ -1,5 +1,4 @@ parameters: - hypothesis: false coverage: false sudo_dependencies: sudo dependencies: apt @@ -54,7 +53,7 @@ steps: displayName: 'Publish code coverage results' -- ${{ if and(ne(parameters.coverage, 'true'), ne(parameters.hypothesis, 'true')) }}: +- ${{ if ne(parameters.coverage, 'true') }}: - script: make pythoninfo displayName: 'Display build info' @@ -73,36 +72,6 @@ steps: displayName: 'Run patchcheck.py' condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest')) -- ${{ if eq(parameters.hypothesis, 'true') }}: - - script: ./python -m venv hypovenv && ./hypovenv/bin/python -m pip install -U hypothesis - displayName: 'Set up virtual environment' - - - script: ./hypovenv/bin/python -m test.pythoninfo - displayName: 'Display build info' - - - script: | - # Most of the excluded tests are slow test suites with no property tests - # - # (GH-104097) test_sysconfig is skipped because it has tests that are - # failing when executed from inside a virtual environment. - $COMMAND -m test \ - -W \ - -x test_asyncio \ - -x test_multiprocessing_fork \ - -x test_multiprocessing_forkserver \ - -x test_multiprocessing_spawn \ - -x test_concurrent_futures \ - -x test_socket \ - -x test_subprocess \ - -x test_signal \ - -x test_sysconfig - displayName: 'Run tests with hypothesis installed' - env: - ${{ if eq(parameters.xvfb, 'true') }}: - COMMAND: xvfb-run ./hypovenv/bin/python - ${{ if ne(parameters.xvfb, 'true') }}: - COMMAND: ./hypovenv/bin/python - - task: PublishTestResults@2 displayName: 'Publish Test Results' diff --git a/.azure-pipelines/pr.yml b/.azure-pipelines/pr.yml index 2fb39b725e97ea..5f7218768c18af 100644 --- a/.azure-pipelines/pr.yml +++ b/.azure-pipelines/pr.yml @@ -1,6 +1,5 @@ variables: coverage: false - hypothesis: true pr: ['main', '3.11', '3.10', '3.9', '3.8', '3.7'] @@ -93,33 +92,6 @@ jobs: coverage: true -- job: Ubuntu_Hypothesis_PR_Tests - displayName: Ubuntu PR Tests (Hypothesis) - dependsOn: Prebuild - condition: | - and( - succeeded(), - eq(variables['hypothesis'], 'true'), - eq(dependencies.Prebuild.outputs['tests.run'], 'true'), - notIn(variables['System.PullRequest.targetBranchName'], - '3.8', '3.9', '3.10', '3.11') - ) - - pool: - vmImage: ubuntu-22.04 - - variables: - testRunTitle: '$(Build.SourceBranchName)-linux-hypothesis' - testRunPlatform: linux-hypothesis - openssl_version: 1.1.1t - - steps: - - template: ./posix-steps.yml - parameters: - dependencies: apt - hypothesis: true - - - job: Windows_PR_Tests displayName: Windows PR Tests dependsOn: Prebuild