Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit 094a256

Browse filesBrowse files
aignasaignas
authored
feat(pypi): support freethreaded in experimental_index_url (#2460)
With this we: * Fix the previous behaviour where `abi3` wheels would be selected when freethreaded builds are selected. Whilst this may work in practise sometimes, I am not sure it has been supported by reading PEP703. * Start selecting `cp313t` wheels when we scan what is available on PyPI. * Ensure that the `whl_library` repository rule handles `cp313t` wheel extraction. * Generate `cp313t` config_settings so that we can use them in `pkg_aliases`. * Generate `cp313t` references in `pkg_aliases` macro. * Add the 3.13 deps to dev_pip for testing. Also tested by manually running: ``` $ bazel cquery --//python/config_settings:python_version=3.13 --//python/config_settings:py_freethreaded=yes 'kind("py_library rule", deps(@dev_pip//markupsafe))' INFO: Analyzed target @@_main~pip~dev_pip//markupsafe:markupsafe (3 packages loaded, 4091 targets configured). INFO: Found 1 target... @@_main~pip~dev_pip_313_markupsafe_cp313_cp313t_manylinux_2_17_x86_64_c0ef13ea//:pkg (008c5a5) $bazel build --//python/config_settings:python_version=3.13 --//python/config_settings:py_freethreaded=yes @dev_pip//markupsafe ``` Fixes #2386
1 parent ca98773 commit 094a256
Copy full SHA for 094a256

File tree

Expand file treeCollapse file tree

9 files changed

+171
-33
lines changed
Filter options
Expand file treeCollapse file tree

9 files changed

+171
-33
lines changed

‎MODULE.bazel

Copy file name to clipboardExpand all lines: MODULE.bazel
+7Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,13 @@ dev_pip.parse(
121121
python_version = "3.11",
122122
requirements_lock = "//docs:requirements.txt",
123123
)
124+
dev_pip.parse(
125+
download_only = True,
126+
experimental_index_url = "https://pypi.org/simple",
127+
hub_name = "dev_pip",
128+
python_version = "3.13.0",
129+
requirements_lock = "//docs:requirements.txt",
130+
)
124131
dev_pip.parse(
125132
download_only = True,
126133
experimental_index_url = "https://pypi.org/simple",

‎python/config_settings/BUILD.bazel

Copy file name to clipboardExpand all lines: python/config_settings/BUILD.bazel
+6Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,12 @@ config_setting(
106106
visibility = ["//visibility:public"],
107107
)
108108

109+
config_setting(
110+
name = "is_py_non_freethreaded",
111+
flag_values = {":py_freethreaded": FreeThreadedFlag.NO},
112+
visibility = ["//visibility:public"],
113+
)
114+
109115
# pip.parse related flags
110116

111117
string_flag(

‎python/private/pypi/config_settings.bzl

Copy file name to clipboardExpand all lines: python/private/pypi/config_settings.bzl
+70-31Lines changed: 70 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -20,20 +20,21 @@ that matches the target platform. We can leverage this fact to ensure that the
2020
most specialized wheels are used by default with the users being able to
2121
configure string_flag values to select the less specialized ones.
2222
23-
The list of specialization of the dists goes like follows:
23+
The list of specialization of the dists goes like follows (cpxyt stands for freethreaded
24+
environments):
2425
* sdist
2526
* py*-none-any.whl
2627
* py*-abi3-any.whl
27-
* py*-cpxy-any.whl
28+
* py*-cpxy-any.whl or py*-cpxyt-any.whl
2829
* cp*-none-any.whl
2930
* cp*-abi3-any.whl
30-
* cp*-cpxy-plat.whl
31+
* cp*-cpxy-any.whl or cp*-cpxyt-any.whl
3132
* py*-none-plat.whl
3233
* py*-abi3-plat.whl
33-
* py*-cpxy-plat.whl
34+
* py*-cpxy-plat.whl or py*-cpxyt-plat.whl
3435
* cp*-none-plat.whl
3536
* cp*-abi3-plat.whl
36-
* cp*-cpxy-plat.whl
37+
* cp*-cpxy-plat.whl or cp*-cpxyt-plat.whl
3738
3839
Note, that here the specialization of musl vs manylinux wheels is the same in
3940
order to ensure that the matching fails if the user requests for `musl` and we don't have it or vice versa.
@@ -46,19 +47,24 @@ FLAGS = struct(
4647
**{
4748
f: str(Label("//python/config_settings:" + f))
4849
for f in [
49-
"python_version",
50+
"is_pip_whl_auto",
51+
"is_pip_whl_no",
52+
"is_pip_whl_only",
53+
"is_py_freethreaded",
54+
"is_py_non_freethreaded",
5055
"pip_whl_glibc_version",
5156
"pip_whl_muslc_version",
5257
"pip_whl_osx_arch",
5358
"pip_whl_osx_version",
5459
"py_linux_libc",
55-
"is_pip_whl_no",
56-
"is_pip_whl_only",
57-
"is_pip_whl_auto",
60+
"python_version",
5861
]
5962
}
6063
)
6164

65+
_DEFAULT = "//conditions:default"
66+
_INCOMPATIBLE = "@platforms//:incompatible"
67+
6268
# Here we create extra string flags that are just to work with the select
6369
# selecting the most specialized match. We don't allow the user to change
6470
# them.
@@ -170,52 +176,70 @@ def _dist_config_settings(*, suffix, plat_flag_values, **kwargs):
170176
**kwargs
171177
)
172178

173-
for name, f in [
174-
("py_none", _flags.whl_py2_py3),
175-
("py3_none", _flags.whl_py3),
176-
("py3_abi3", _flags.whl_py3_abi3),
177-
("cp3x_none", _flags.whl_pycp3x),
178-
("cp3x_abi3", _flags.whl_pycp3x_abi3),
179-
("cp3x_cp", _flags.whl_pycp3x_abicp),
179+
used_flags = {}
180+
181+
# NOTE @aignas 2024-12-01: the abi3 is not compatible with freethreaded
182+
# builds as per PEP703 (https://peps.python.org/pep-0703/#backwards-compatibility)
183+
#
184+
# The discussion here also reinforces this notion:
185+
# https://discuss.python.org/t/pep-703-making-the-global-interpreter-lock-optional-3-12-updates/26503/99
186+
187+
for name, f, abi in [
188+
("py_none", _flags.whl_py2_py3, None),
189+
("py3_none", _flags.whl_py3, None),
190+
("py3_abi3", _flags.whl_py3_abi3, (FLAGS.is_py_non_freethreaded,)),
191+
("cp3x_none", _flags.whl_pycp3x, None),
192+
("cp3x_abi3", _flags.whl_pycp3x_abi3, (FLAGS.is_py_non_freethreaded,)),
193+
# The below are not specializations of one another, they are variants
194+
("cp3x_cp", _flags.whl_pycp3x_abicp, (FLAGS.is_py_non_freethreaded,)),
195+
("cp3x_cpt", _flags.whl_pycp3x_abicp, (FLAGS.is_py_freethreaded,)),
180196
]:
181-
if f in flag_values:
197+
if (f, abi) in used_flags:
182198
# This should never happen as all of the different whls should have
183-
# unique flag values.
199+
# unique flag values
184200
fail("BUG: the flag {} is attempted to be added twice to the list".format(f))
185201
else:
186202
flag_values[f] = ""
203+
used_flags[(f, abi)] = True
187204

188205
_dist_config_setting(
189206
name = "{}_any{}".format(name, suffix),
190207
flag_values = flag_values,
191208
is_pip_whl = FLAGS.is_pip_whl_only,
209+
abi = abi,
192210
**kwargs
193211
)
194212

195213
generic_flag_values = flag_values
214+
generic_used_flags = used_flags
196215

197216
for (suffix, flag_values) in plat_flag_values:
217+
used_flags = {(f, None): True for f in flag_values} | generic_used_flags
198218
flag_values = flag_values | generic_flag_values
199219

200-
for name, f in [
201-
("py_none", _flags.whl_plat),
202-
("py3_none", _flags.whl_plat_py3),
203-
("py3_abi3", _flags.whl_plat_py3_abi3),
204-
("cp3x_none", _flags.whl_plat_pycp3x),
205-
("cp3x_abi3", _flags.whl_plat_pycp3x_abi3),
206-
("cp3x_cp", _flags.whl_plat_pycp3x_abicp),
220+
for name, f, abi in [
221+
("py_none", _flags.whl_plat, None),
222+
("py3_none", _flags.whl_plat_py3, None),
223+
("py3_abi3", _flags.whl_plat_py3_abi3, (FLAGS.is_py_non_freethreaded,)),
224+
("cp3x_none", _flags.whl_plat_pycp3x, None),
225+
("cp3x_abi3", _flags.whl_plat_pycp3x_abi3, (FLAGS.is_py_non_freethreaded,)),
226+
# The below are not specializations of one another, they are variants
227+
("cp3x_cp", _flags.whl_plat_pycp3x_abicp, (FLAGS.is_py_non_freethreaded,)),
228+
("cp3x_cpt", _flags.whl_plat_pycp3x_abicp, (FLAGS.is_py_freethreaded,)),
207229
]:
208-
if f in flag_values:
230+
if (f, abi) in used_flags:
209231
# This should never happen as all of the different whls should have
210232
# unique flag values.
211233
fail("BUG: the flag {} is attempted to be added twice to the list".format(f))
212234
else:
213235
flag_values[f] = ""
236+
used_flags[(f, abi)] = True
214237

215238
_dist_config_setting(
216239
name = "{}_{}".format(name, suffix),
217240
flag_values = flag_values,
218241
is_pip_whl = FLAGS.is_pip_whl_only,
242+
abi = abi,
219243
**kwargs
220244
)
221245

@@ -285,7 +309,7 @@ def _plat_flag_values(os, cpu, osx_versions, glibc_versions, muslc_versions):
285309

286310
return ret
287311

288-
def _dist_config_setting(*, name, is_python, python_version, is_pip_whl = None, native = native, **kwargs):
312+
def _dist_config_setting(*, name, is_python, python_version, is_pip_whl = None, abi = None, native = native, **kwargs):
289313
"""A macro to create a target that matches is_pip_whl_auto and one more value.
290314
291315
Args:
@@ -294,6 +318,10 @@ def _dist_config_setting(*, name, is_python, python_version, is_pip_whl = None,
294318
`is_pip_whl_auto` when evaluating the config setting.
295319
is_python: The python version config_setting to match.
296320
python_version: The python version name.
321+
abi: {type}`tuple[Label]` A collection of ABI config settings that are
322+
compatible with the given dist config setting. For example, if only
323+
non-freethreaded python builds are allowed, add
324+
FLAGS.is_py_non_freethreaded here.
297325
native (struct): The struct containing alias and config_setting rules
298326
to use for creating the objects. Can be overridden for unit tests
299327
reasons.
@@ -306,9 +334,9 @@ def _dist_config_setting(*, name, is_python, python_version, is_pip_whl = None,
306334
native.alias(
307335
name = "is_cp{}_{}".format(python_version, name) if python_version else "is_{}".format(name),
308336
actual = select({
309-
# First match by the python version
310-
is_python: _name,
311-
"//conditions:default": is_python,
337+
# First match by the python version and then by ABI
338+
is_python: _name + ("_abi" if abi else ""),
339+
_DEFAULT: _INCOMPATIBLE,
312340
}),
313341
visibility = visibility,
314342
)
@@ -325,12 +353,23 @@ def _dist_config_setting(*, name, is_python, python_version, is_pip_whl = None,
325353
config_setting_name = _name + "_setting"
326354
native.config_setting(name = config_setting_name, **kwargs)
327355

356+
if abi:
357+
native.alias(
358+
name = _name + "_abi",
359+
actual = select(
360+
{k: _name for k in abi} | {
361+
_DEFAULT: _INCOMPATIBLE,
362+
},
363+
),
364+
visibility = visibility,
365+
)
366+
328367
# Next match by the `pip_whl` flag value and then match by the flags that
329368
# are intrinsic to the distribution.
330369
native.alias(
331370
name = _name,
332371
actual = select({
333-
"//conditions:default": FLAGS.is_pip_whl_auto,
372+
_DEFAULT: _INCOMPATIBLE,
334373
FLAGS.is_pip_whl_auto: config_setting_name,
335374
is_pip_whl: config_setting_name,
336375
}),

‎python/private/pypi/pkg_aliases.bzl

Copy file name to clipboardExpand all lines: python/private/pypi/pkg_aliases.bzl
+3-1Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -308,7 +308,9 @@ def get_filename_config_settings(
308308
else:
309309
py = "py3"
310310

311-
if parsed.abi_tag.startswith("cp"):
311+
if parsed.abi_tag.startswith("cp") and parsed.abi_tag.endswith("t"):
312+
abi = "cpt"
313+
elif parsed.abi_tag.startswith("cp"):
312314
abi = "cp"
313315
else:
314316
abi = parsed.abi_tag

‎python/private/pypi/whl_library.bzl

Copy file name to clipboardExpand all lines: python/private/pypi/whl_library.bzl
+1-1Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -287,7 +287,7 @@ def _whl_library_impl(rctx):
287287
p.target_platform
288288
for p in whl_target_platforms(
289289
platform_tag = parsed_whl.platform_tag,
290-
abi_tag = parsed_whl.abi_tag,
290+
abi_tag = parsed_whl.abi_tag.strip("tm"),
291291
)
292292
]
293293

‎python/private/pypi/whl_target_platforms.bzl

Copy file name to clipboardExpand all lines: python/private/pypi/whl_target_platforms.bzl
+4Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,10 @@ def select_whls(*, whls, want_platforms = [], logger = None):
8989
want_abis[abi] = None
9090
want_abis[abi + "m"] = None
9191

92+
# Also add freethreaded wheels if we find them since we started supporting them
93+
_want_platforms["{}t_{}".format(abi, os_cpu)] = None
94+
want_abis[abi + "t"] = None
95+
9296
want_platforms = sorted(_want_platforms)
9397

9498
candidates = {}

‎tests/pypi/config_settings/config_settings_tests.bzl

Copy file name to clipboardExpand all lines: tests/pypi/config_settings/config_settings_tests.bzl
+50Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ _flag = struct(
3939
pip_whl_osx_arch = lambda x: (str(Label("//python/config_settings:pip_whl_osx_arch")), str(x)),
4040
py_linux_libc = lambda x: (str(Label("//python/config_settings:py_linux_libc")), str(x)),
4141
python_version = lambda x: (str(Label("//python/config_settings:python_version")), str(x)),
42+
py_freethreaded = lambda x: (str(Label("//python/config_settings:py_freethreaded")), str(x)),
4243
)
4344

4445
def _analysis_test(*, name, dist, want, config_settings = [_flag.platform("linux_aarch64")]):
@@ -286,6 +287,38 @@ def _test_py_none_any_versioned(name):
286287

287288
_tests.append(_test_py_none_any_versioned)
288289

290+
def _test_cp_whl_is_not_prefered_over_py3_non_freethreaded(name):
291+
_analysis_test(
292+
name = name,
293+
dist = {
294+
"is_cp3.7_cp3x_abi3_any": "py3_abi3",
295+
"is_cp3.7_cp3x_cpt_any": "cp",
296+
"is_cp3.7_cp3x_none_any": "py3",
297+
},
298+
want = "py3_abi3",
299+
config_settings = [
300+
_flag.py_freethreaded("no"),
301+
],
302+
)
303+
304+
_tests.append(_test_cp_whl_is_not_prefered_over_py3_non_freethreaded)
305+
306+
def _test_cp_whl_is_not_prefered_over_py3_freethreaded(name):
307+
_analysis_test(
308+
name = name,
309+
dist = {
310+
"is_cp3.7_cp3x_abi3_any": "py3_abi3",
311+
"is_cp3.7_cp3x_cp_any": "cp",
312+
"is_cp3.7_cp3x_none_any": "py3",
313+
},
314+
want = "py3",
315+
config_settings = [
316+
_flag.py_freethreaded("yes"),
317+
],
318+
)
319+
320+
_tests.append(_test_cp_whl_is_not_prefered_over_py3_freethreaded)
321+
289322
def _test_cp_cp_whl(name):
290323
_analysis_test(
291324
name = name,
@@ -412,6 +445,7 @@ def _test_windows(name):
412445
name = name,
413446
dist = {
414447
"is_cp3.7_cp3x_cp_windows_x86_64": "whl",
448+
"is_cp3.7_cp3x_cpt_windows_x86_64": "whl_freethreaded",
415449
},
416450
want = "whl",
417451
config_settings = [
@@ -421,6 +455,22 @@ def _test_windows(name):
421455

422456
_tests.append(_test_windows)
423457

458+
def _test_windows_freethreaded(name):
459+
_analysis_test(
460+
name = name,
461+
dist = {
462+
"is_cp3.7_cp3x_cp_windows_x86_64": "whl",
463+
"is_cp3.7_cp3x_cpt_windows_x86_64": "whl_freethreaded",
464+
},
465+
want = "whl_freethreaded",
466+
config_settings = [
467+
_flag.platform("windows_x86_64"),
468+
_flag.py_freethreaded("yes"),
469+
],
470+
)
471+
472+
_tests.append(_test_windows_freethreaded)
473+
424474
def _test_osx(name):
425475
_analysis_test(
426476
name = name,

‎tests/pypi/pkg_aliases/pkg_aliases_test.bzl

Copy file name to clipboardExpand all lines: tests/pypi/pkg_aliases/pkg_aliases_test.bzl
+10Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -287,6 +287,14 @@ def _test_multiplatform_whl_aliases_filename(env):
287287
filename = "foo-0.0.1-py3-none-any.whl",
288288
version = "3.1",
289289
): "foo-py3-0.0.1",
290+
whl_config_setting(
291+
filename = "foo-0.0.1-cp313-cp313-any.whl",
292+
version = "3.1",
293+
): "foo-cp-0.0.1",
294+
whl_config_setting(
295+
filename = "foo-0.0.1-cp313-cp313t-any.whl",
296+
version = "3.1",
297+
): "foo-cpt-0.0.1",
290298
whl_config_setting(
291299
filename = "foo-0.0.2-py3-none-any.whl",
292300
version = "3.1",
@@ -303,6 +311,8 @@ def _test_multiplatform_whl_aliases_filename(env):
303311
osx_versions = [],
304312
)
305313
want = {
314+
"//_config:is_cp3.1_cp3x_cp_any": "foo-cp-0.0.1",
315+
"//_config:is_cp3.1_cp3x_cpt_any": "foo-cpt-0.0.1",
306316
"//_config:is_cp3.1_py3_none_any": "foo-py3-0.0.1",
307317
"//_config:is_cp3.1_py3_none_any_linux_aarch64": "foo-0.0.2",
308318
"//_config:is_cp3.1_py3_none_any_linux_x86_64": "foo-0.0.2",

‎tests/pypi/whl_target_platforms/select_whl_tests.bzl

Copy file name to clipboardExpand all lines: tests/pypi/whl_target_platforms/select_whl_tests.bzl
+20Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,10 @@ WHL_LIST = [
2727
"pkg-0.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl",
2828
"pkg-0.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
2929
"pkg-0.0.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl",
30+
"pkg-0.0.1-cp313-cp313t-musllinux_1_1_x86_64.whl",
31+
"pkg-0.0.1-cp313-cp313-musllinux_1_1_x86_64.whl",
32+
"pkg-0.0.1-cp313-abi3-musllinux_1_1_x86_64.whl",
33+
"pkg-0.0.1-cp313-none-musllinux_1_1_x86_64.whl",
3034
"pkg-0.0.1-cp311-cp311-musllinux_1_1_aarch64.whl",
3135
"pkg-0.0.1-cp311-cp311-musllinux_1_1_i686.whl",
3236
"pkg-0.0.1-cp311-cp311-musllinux_1_1_ppc64le.whl",
@@ -269,6 +273,22 @@ def _test_prefer_manylinux_wheels(env):
269273

270274
_tests.append(_test_prefer_manylinux_wheels)
271275

276+
def _test_freethreaded_wheels(env):
277+
# Check we prefer platform specific wheels
278+
got = _select_whls(whls = WHL_LIST, want_platforms = ["cp313_linux_x86_64"])
279+
_match(
280+
env,
281+
got,
282+
"pkg-0.0.1-cp313-cp313t-musllinux_1_1_x86_64.whl",
283+
"pkg-0.0.1-cp313-cp313-musllinux_1_1_x86_64.whl",
284+
"pkg-0.0.1-cp313-abi3-musllinux_1_1_x86_64.whl",
285+
"pkg-0.0.1-cp313-none-musllinux_1_1_x86_64.whl",
286+
"pkg-0.0.1-cp39-abi3-any.whl",
287+
"pkg-0.0.1-py3-none-any.whl",
288+
)
289+
290+
_tests.append(_test_freethreaded_wheels)
291+
272292
def select_whl_test_suite(name):
273293
"""Create the test suite.
274294

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.