Closed
Description
Bug report
Testing on tip of 3.12 which includes the fix for #105013 (thanks!), I get the following difference in behaviour with Python 3.11 vs tip of Python 3.12:
import inspect
from hypothesis.internal.reflection import extract_lambda_source
from hypothesis.strategies import just
#from hypothesis.strategies import just, one_of
# This variant doesn't trigger a TypeError mid-callback, but both variants get the same final inspect error
#one_of_nested_strategy_with_filter = one_of(
# just(0),
# just(1),
# one_of(just(2), just(3), one_of(just(4), just(5), one_of(just(6), just(7)))),
#).filter(lambda x: x % 2 == 0)
#x = get_pretty_function_description(one_of_nested_strategy_with_filter)
#print(inspect.getsource(x))
one_of_nested_strategy_with_filter = (
just(0)
).filter(lambda x: x % 2 == 0)
x = extract_lambda_source(one_of_nested_strategy_with_filter)With Python 3.12, I get:
Traceback (most recent call last):
File "/usr/lib/python3.12/inspect.py", line 1241, in getblock
for _token in tokens:
File "/usr/lib/python3.12/tokenize.py", line 450, in _tokenize
for token in _generate_tokens_from_c_tokenizer(source, extra_tokens=True):
File "/usr/lib/python3.12/tokenize.py", line 537, in _generate_tokens_from_c_tokenizer
for info in c_tokenizer.TokenizerIter(source, extra_tokens=extra_tokens):
File "<string>", line 1
).filter(lambda x: x % 2 == 0)
^
SyntaxError: unmatched ')'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/tmp/foo2.py", line 10, in <module>
x = extract_lambda_source(one_of_nested_strategy_with_filter)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/site-packages/hypothesis/internal/reflection.py", line 305, in extract_lambda_source
sig = inspect.signature(f)
^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/inspect.py", line 3326, in signature
return Signature.from_callable(obj, follow_wrapped=follow_wrapped,
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/inspect.py", line 3070, in from_callable
return _signature_from_callable(obj, sigcls=cls,
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/inspect.py", line 2484, in _signature_from_callable
raise TypeError('{!r} is not a callable object'.format(obj))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/site-packages/hypothesis/strategies/_internal/misc.py", line 39, in __repr__
suffix = "".join(
^^^^^^^^
File "/usr/lib/python3.12/site-packages/hypothesis/strategies/_internal/misc.py", line 40, in <genexpr>
f".{name}({get_pretty_function_description(f)})"
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/site-packages/hypothesis/internal/reflection.py", line 432, in get_pretty_function_description
return extract_lambda_source(f)
^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/site-packages/hypothesis/internal/reflection.py", line 312, in extract_lambda_source
source = inspect.getsource(f)
^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/inspect.py", line 1282, in getsource
lines, lnum = getsourcelines(object)
^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/inspect.py", line 1274, in getsourcelines
return getblock(lines[lnum:]), lnum + 1
^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/inspect.py", line 1248, in getblock
_, *_token_info = _token
^^^^^^
UnboundLocalError: cannot access local variable '_token' where it is not associated with a value
But with Python 3.11, I get:
Traceback (most recent call last):
File "/tmp/foo2.py", line 10, in <module>
x = extract_lambda_source(one_of_nested_strategy_with_filter)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.11/site-packages/hypothesis/internal/reflection.py", line 305, in extract_lambda_source
sig = inspect.signature(f)
^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.11/inspect.py", line 3279, in signature
return Signature.from_callable(obj, follow_wrapped=follow_wrapped,
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.11/inspect.py", line 3027, in from_callable
return _signature_from_callable(obj, sigcls=cls,
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.11/inspect.py", line 2447, in _signature_from_callable
raise TypeError('{!r} is not a callable object'.format(obj))
TypeError: just(0).filter(lambda x: <unknown>) is not a callable object
If I change the program to drop the whitespace, it works in 3.12 too:
import inspect
from hypothesis.internal.reflection import extract_lambda_source
from hypothesis.strategies import just
one_of_nested_strategy_with_filter = (just(0)).filter(lambda x: x % 2 == 0)
x = extract_lambda_source(one_of_nested_strategy_with_filter)
I noticed this w/ a test failure in priority (the output is huge, so just a snippet here)
ERROR collecting test/test_priority.py ____________________________________________________________________________________
/usr/lib/python3.12/inspect.py:1241: in getblock
for _token in tokens:
[...]
/usr/lib/python3.12/tokenize.py:537: in _generate_tokens_from_c_tokenizer
for info in c_tokenizer.TokenizerIter(source, extra_tokens=extra_tokens):
E File "<string>", line 1
E ).map(lambda blocked: (blocked, active_readme_streams_from_filter(blocked)))
E ^
E SyntaxError: unmatched ')'
c_tokenizer = <module '_tokenize' (built-in)>
extra_tokens = True
source = (').map(lambda blocked: (blocked, '
'active_readme_streams_from_filter(blocked)))\n'
[...]
ERROR test/test_priority.py - UnboundLocalError: cannot access local variable '_token' where it is not associated with a value
and a perhaps more useful test failure in hypothesis, which priority uses:
test_one_of_flattens_filter_branches_2 ____________________________________________________________________________________
[gw14] linux -- Python 3.12.0 /var/tmp/portage/dev-python/hypothesis-6.75.6/work/hypothesis-hypothesis-python-6.75.6/hypothesis-python-python3_12/install/usr/bin/python3.12
Traceback (most recent call last):
File "/usr/lib/python3.12/inspect.py", line 1241, in getblock
for _token in tokens:
File "/usr/lib/python3.12/tokenize.py", line 450, in _tokenize
for token in _generate_tokens_from_c_tokenizer(source, extra_tokens=True):
File "/usr/lib/python3.12/tokenize.py", line 537, in _generate_tokens_from_c_tokenizer
for info in c_tokenizer.TokenizerIter(source, extra_tokens=extra_tokens):
File "<string>", line 1
).filter(lambda x: x % 2 == 0)
^
SyntaxError: unmatched ')'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/var/tmp/portage/dev-python/hypothesis-6.75.6/work/hypothesis-hypothesis-python-6.75.6/hypothesis-python/tests/quality/test_discovery_ability.py", line 101, in run_test
runner.run()
File "/var/tmp/portage/dev-python/hypothesis-6.75.6/work/hypothesis-hypothesis-python-6.75.6/hypothesis-python-python3_12/install/usr/lib/python3.12/site-packages/hypothesis/internal/conjecture/engine.py", line 474, in run
self._run()
File "/var/tmp/portage/dev-python/hypothesis-6.75.6/work/hypothesis-hypothesis-python-6.75.6/hypothesis-python-python3_12/install/usr/lib/python3.12/site-packages/hypothesis/internal/conjecture/engine.py", line 880, in _run
self.generate_new_examples()
File "/var/tmp/portage/dev-python/hypothesis-6.75.6/work/hypothesis-hypothesis-python-6.75.6/hypothesis-python-python3_12/install/usr/lib/python3.12/site-packages/hypothesis/internal/conjecture/engine.py", line 684, in generate_new_examples
minimal_example = self.cached_test_function(
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/var/tmp/portage/dev-python/hypothesis-6.75.6/work/hypothesis-hypothesis-python-6.75.6/hypothesis-python-python3_12/install/usr/lib/python3.12/site-packages/hypothesis/internal/conjecture/engine.py", line 1065, in cached_test_function
self.test_function(data)
File "/var/tmp/portage/dev-python/hypothesis-6.75.6/work/hypothesis-hypothesis-python-6.75.6/hypothesis-python-python3_12/install/usr/lib/python3.12/site-packages/hypothesis/internal/conjecture/engine.py", line 209, in test_function
self.__stoppable_test_function(data)
File "/var/tmp/portage/dev-python/hypothesis-6.75.6/work/hypothesis-hypothesis-python-6.75.6/hypothesis-python-python3_12/install/usr/lib/python3.12/site-packages/hypothesis/internal/conjecture/engine.py", line 185, in __stoppable_test_function
self._test_function(data)
File "/var/tmp/portage/dev-python/hypothesis-6.75.6/work/hypothesis-hypothesis-python-6.75.6/hypothesis-python/tests/quality/test_discovery_ability.py", line 79, in test_function
value = data.draw(specifier)
^^^^^^^^^^^^^^^^^^^^
File "/var/tmp/portage/dev-python/hypothesis-6.75.6/work/hypothesis-hypothesis-python-6.75.6/hypothesis-python-python3_12/install/usr/lib/python3.12/site-packages/hypothesis/internal/conjecture/data.py", line 956, in draw
return strategy.do_draw(self)
^^^^^^^^^^^^^^^^^^^^^^
File "/var/tmp/portage/dev-python/hypothesis-6.75.6/work/hypothesis-hypothesis-python-6.75.6/hypothesis-python-python3_12/install/usr/lib/python3.12/site-packages/hypothesis/strategies/_internal/strategies.py", line 942, in do_draw
result = self.do_filtered_draw(data)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/var/tmp/portage/dev-python/hypothesis-6.75.6/work/hypothesis-hypothesis-python-6.75.6/hypothesis-python-python3_12/install/usr/lib/python3.12/site-packages/hypothesis/strategies/_internal/strategies.py", line 956, in do_filtered_draw
value = data.draw(self.filtered_strategy)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/var/tmp/portage/dev-python/hypothesis-6.75.6/work/hypothesis-hypothesis-python-6.75.6/hypothesis-python-python3_12/install/usr/lib/python3.12/site-packages/hypothesis/internal/conjecture/data.py", line 951, in draw
return strategy.do_draw(self)
^^^^^^^^^^^^^^^^^^^^^^
File "/var/tmp/portage/dev-python/hypothesis-6.75.6/work/hypothesis-hypothesis-python-6.75.6/hypothesis-python-python3_12/install/usr/lib/python3.12/site-packages/hypothesis/strategies/_internal/strategies.py", line 666, in do_draw
return data.draw(strategy)
^^^^^^^^^^^^^^^^^^^
File "/var/tmp/portage/dev-python/hypothesis-6.75.6/work/hypothesis-hypothesis-python-6.75.6/hypothesis-python-python3_12/install/usr/lib/python3.12/site-packages/hypothesis/internal/conjecture/data.py", line 951, in draw
return strategy.do_draw(self)
^^^^^^^^^^^^^^^^^^^^^^
File "/var/tmp/portage/dev-python/hypothesis-6.75.6/work/hypothesis-hypothesis-python-6.75.6/hypothesis-python-python3_12/install/usr/lib/python3.12/site-packages/hypothesis/strategies/_internal/strategies.py", line 532, in do_draw
data.mark_invalid(f"Aborted test because unable to satisfy {self!r}")
^^^^^^^^
File "/var/tmp/portage/dev-python/hypothesis-6.75.6/work/hypothesis-hypothesis-python-6.75.6/hypothesis-python-python3_12/install/usr/lib/python3.12/site-packages/hypothesis/strategies/_internal/misc.py", line 39, in __repr__
suffix = "".join(
^^^^^^^^
File "/var/tmp/portage/dev-python/hypothesis-6.75.6/work/hypothesis-hypothesis-python-6.75.6/hypothesis-python-python3_12/install/usr/lib/python3.12/site-packages/hypothesis/strategies/_internal/misc.py", line 40, in <genexpr>
f".{name}({get_pretty_function_description(f)})"
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/var/tmp/portage/dev-python/hypothesis-6.75.6/work/hypothesis-hypothesis-python-6.75.6/hypothesis-python-python3_12/install/usr/lib/python3.12/site-packages/hypothesis/internal/reflection.py", line 432, in get_pretty_function_description
return extract_lambda_source(f)
^^^^^^^^^^^^^^^^^^^^^^^^
File "/var/tmp/portage/dev-python/hypothesis-6.75.6/work/hypothesis-hypothesis-python-6.75.6/hypothesis-python-python3_12/install/usr/lib/python3.12/site-packages/hypothesis/internal/reflection.py", line 312, in extract_lambda_source
source = inspect.getsource(f)
^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/inspect.py", line 1282, in getsource
lines, lnum = getsourcelines(object)
^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/inspect.py", line 1274, in getsourcelines
return getblock(lines[lnum:]), lnum + 1
^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/inspect.py", line 1248, in getblock
_, *_token_info = _token
^^^^^^
UnboundLocalError: cannot access local variable '_token' where it is not associated with a value
See also #105013.
Your environment
- CPython versions tested on: 3.11.3, tip of 3.12
- Operating system and architecture: Gentoo Linux, amd64
Linked PRs
Metadata
Metadata
Assignees
Labels
An unexpected behavior, bug, or errorAn unexpected behavior, bug, or error