Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit e8039db

Browse filesBrowse files
aignasaignas
authored
feat(twine): support 'bzlmod' users out of the box (bazel-contrib#1572)
Implements a test that starts a [`pypiserver`] and checks that the publishing with the new machinery still works. Fixes bazel-contrib#1369 [pypiserver]: https://github.com/pypiserver/pypiserver
1 parent c5c03b2 commit e8039db
Copy full SHA for e8039db

File tree

Expand file treeCollapse file tree

16 files changed

+305
-37
lines changed
Filter options
Expand file treeCollapse file tree

16 files changed

+305
-37
lines changed

‎CHANGELOG.md

Copy file name to clipboardExpand all lines: CHANGELOG.md
+3-1Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,9 @@ A brief description of the categories of changes:
4343
* (wheel) Add support for `data_files` attributes in py_wheel rule
4444
([#1777](https://github.com/bazelbuild/rules_python/issues/1777))
4545

46+
* (py_wheel) `bzlmod` installations now provide a `twine` setup for the default
47+
Python toolchain in `rules_python` for version 3.11.
48+
4649
[0.XX.0]: https://github.com/bazelbuild/rules_python/releases/tag/0.XX.0
4750
[python_default_visibility]: gazelle/README.md#directive-python_default_visibility
4851

@@ -269,7 +272,6 @@ A brief description of the categories of changes:
269272
attribute for every target in the package. This is enabled through a separate
270273
directive `python_generation_mode_per_file_include_init`.
271274

272-
273275
## [0.27.0] - 2023-11-16
274276

275277
[0.27.0]: https://github.com/bazelbuild/rules_python/releases/tag/0.27.0

‎MODULE.bazel

Copy file name to clipboardExpand all lines: MODULE.bazel
+17-16Lines changed: 17 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -48,11 +48,24 @@ python.toolchain(
4848
is_default = True,
4949
python_version = "3.11",
5050
)
51-
use_repo(python, "pythons_hub")
51+
use_repo(python, "python_versions", "pythons_hub")
5252

5353
# This call registers the Python toolchains.
5454
register_toolchains("@pythons_hub//:all")
5555

56+
#####################
57+
# Install twine for our own runfiles wheel publishing and allow bzlmod users to use it.
58+
59+
pip = use_extension("//python/extensions:pip.bzl", "pip")
60+
pip.parse(
61+
hub_name = "rules_python_publish_deps",
62+
python_version = "3.11",
63+
requirements_darwin = "//tools/publish:requirements_darwin.txt",
64+
requirements_lock = "//tools/publish:requirements.txt",
65+
requirements_windows = "//tools/publish:requirements_windows.txt",
66+
)
67+
use_repo(pip, "rules_python_publish_deps")
68+
5669
# ===== DEV ONLY DEPS AND SETUP BELOW HERE =====
5770
bazel_dep(name = "stardoc", version = "0.6.2", dev_dependency = True, repo_name = "io_bazel_stardoc")
5871
bazel_dep(name = "rules_bazel_integration_test", version = "0.20.0", dev_dependency = True)
@@ -84,24 +97,12 @@ dev_pip.parse(
8497
python_version = "3.11",
8598
requirements_lock = "//docs/sphinx:requirements.txt",
8699
)
87-
88-
#####################
89-
# Install twine for our own runfiles wheel publishing.
90-
# Eventually we might want to install twine automatically for users too, see:
91-
# https://github.com/bazelbuild/rules_python/issues/1016.
92-
93100
dev_pip.parse(
94-
hub_name = "publish_deps",
101+
hub_name = "pypiserver",
95102
python_version = "3.11",
96-
requirements_darwin = "//tools/publish:requirements_darwin.txt",
97-
requirements_lock = "//tools/publish:requirements.txt",
98-
requirements_windows = "//tools/publish:requirements_windows.txt",
99-
)
100-
use_repo(
101-
dev_pip,
102-
"dev_pip",
103-
publish_deps_twine = "publish_deps_311_twine",
103+
requirements_lock = "//examples/wheel:requirements_server.txt",
104104
)
105+
use_repo(dev_pip, "dev_pip", "pypiserver")
105106

106107
# Bazel integration test setup below
107108

‎WORKSPACE

Copy file name to clipboardExpand all lines: WORKSPACE
+12-2Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -94,17 +94,27 @@ load("@python//3.11.8:defs.bzl", "interpreter")
9494
load("@rules_python//python:pip.bzl", "pip_parse")
9595

9696
pip_parse(
97-
name = "publish_deps",
97+
name = "rules_python_publish_deps",
9898
python_interpreter_target = interpreter,
9999
requirements_darwin = "//tools/publish:requirements_darwin.txt",
100100
requirements_lock = "//tools/publish:requirements.txt",
101101
requirements_windows = "//tools/publish:requirements_windows.txt",
102102
)
103103

104-
load("@publish_deps//:requirements.bzl", "install_deps")
104+
load("@rules_python_publish_deps//:requirements.bzl", "install_deps")
105105

106106
install_deps()
107107

108+
pip_parse(
109+
name = "pypiserver",
110+
python_interpreter_target = interpreter,
111+
requirements_lock = "//examples/wheel:requirements_server.txt",
112+
)
113+
114+
load("@pypiserver//:requirements.bzl", install_pypiserver = "install_deps")
115+
116+
install_pypiserver()
117+
108118
#####################
109119
# Install sphinx for doc generation.
110120

‎examples/wheel/BUILD.bazel

Copy file name to clipboardExpand all lines: examples/wheel/BUILD.bazel
+43Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,10 @@ load("@bazel_skylib//rules:write_file.bzl", "write_file")
1717
load("//examples/wheel/private:wheel_utils.bzl", "directory_writer", "make_variable_tags")
1818
load("//python:defs.bzl", "py_library", "py_test")
1919
load("//python:packaging.bzl", "py_package", "py_wheel")
20+
load("//python:pip.bzl", "compile_pip_requirements")
2021
load("//python:versions.bzl", "gen_python_config_settings")
22+
load("//python/entry_points:py_console_script_binary.bzl", "py_console_script_binary")
23+
load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") # buildifier: disable=bzl-visibility
2124

2225
package(default_visibility = ["//visibility:public"])
2326

@@ -56,6 +59,10 @@ py_wheel(
5659
# Package data. We're building "example_minimal_library-0.0.1-py3-none-any.whl"
5760
distribution = "example_minimal_library",
5861
python_tag = "py3",
62+
# NOTE: twine_binary = "//tools/publish:twine" does not work on non-bzlmod
63+
# setups because the `//tools/publish:twine` produces multiple files and is
64+
# unsuitable as the `src` to the underlying native_binary rule.
65+
twine = None if BZLMOD_ENABLED else "@rules_python_publish_deps_twine//:pkg",
5966
version = "0.0.1",
6067
deps = [
6168
"//examples/wheel/lib:module_with_data",
@@ -348,3 +355,39 @@ py_test(
348355
"//python/runfiles",
349356
],
350357
)
358+
359+
# Test wheel publishing
360+
361+
compile_pip_requirements(
362+
name = "requirements_server",
363+
src = "requirements_server.in",
364+
)
365+
366+
py_test(
367+
name = "test_publish",
368+
srcs = ["test_publish.py"],
369+
data = [
370+
":minimal_with_py_library",
371+
":minimal_with_py_library.publish",
372+
":pypiserver",
373+
],
374+
env = {
375+
"PUBLISH_PATH": "$(location :minimal_with_py_library.publish)",
376+
"SERVER_PATH": "$(location :pypiserver)",
377+
"WHEEL_PATH": "$(rootpath :minimal_with_py_library)",
378+
},
379+
target_compatible_with = select({
380+
"@platforms//os:linux": [],
381+
"@platforms//os:macos": [],
382+
"//conditions:default": ["@platforms//:incompatible"],
383+
}),
384+
deps = [
385+
"@pypiserver//pypiserver",
386+
],
387+
)
388+
389+
py_console_script_binary(
390+
name = "pypiserver",
391+
pkg = "@pypiserver//pypiserver",
392+
script = "pypi-server",
393+
)

‎examples/wheel/requirements_server.in

Copy file name to clipboard
+2Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# This is for running publishing tests
2+
pypiserver

‎examples/wheel/requirements_server.txt

Copy file name to clipboard
+16Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#
2+
# This file is autogenerated by pip-compile with Python 3.11
3+
# by the following command:
4+
#
5+
# bazel run //examples/wheel:requirements_server.update
6+
#
7+
pypiserver==2.0.1 \
8+
--hash=sha256:1dd98fb99d2da4199fb44c7284e57d69a9f7fda2c6c8dc01975c151c592677bf \
9+
--hash=sha256:7b58fbd54468235f79e4de07c4f7a9ff829e7ac6869bef47ec11e0710138e162
10+
# via -r examples/wheel/requirements_server.in
11+
12+
# The following packages are considered to be unsafe in a requirements file:
13+
pip==24.0 \
14+
--hash=sha256:ba0d021a166865d2265246961bec0152ff124de910c5cc39f1156ce3fa7c69dc \
15+
--hash=sha256:ea9bd1a847e8c5774a5777bb398c19e80bcd4e2aa16a4b301b718fe6f593aba2
16+
# via pypiserver

‎examples/wheel/test_publish.py

Copy file name to clipboard
+117Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
import os
2+
import socket
3+
import subprocess
4+
import textwrap
5+
import time
6+
import unittest
7+
from contextlib import closing
8+
from pathlib import Path
9+
from urllib.request import urlopen
10+
11+
12+
def find_free_port():
13+
with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
14+
s.bind(("", 0))
15+
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
16+
return s.getsockname()[1]
17+
18+
19+
class TestTwineUpload(unittest.TestCase):
20+
def setUp(self):
21+
self.maxDiff = 1000
22+
self.port = find_free_port()
23+
self.url = f"http://localhost:{self.port}"
24+
self.dir = Path(os.environ["TEST_TMPDIR"])
25+
26+
self.log_file = self.dir / "pypiserver-log.txt"
27+
self.log_file.touch()
28+
_storage_dir = self.dir / "data"
29+
for d in [_storage_dir]:
30+
d.mkdir(exist_ok=True)
31+
32+
print("Starting PyPI server...")
33+
self._server = subprocess.Popen(
34+
[
35+
str(Path(os.environ["SERVER_PATH"])),
36+
"run",
37+
"--verbose",
38+
"--log-file",
39+
str(self.log_file),
40+
"--host",
41+
"localhost",
42+
"--port",
43+
str(self.port),
44+
# Allow unauthenticated access
45+
"--authenticate",
46+
".",
47+
"--passwords",
48+
".",
49+
str(_storage_dir),
50+
],
51+
)
52+
53+
line = "Hit Ctrl-C to quit"
54+
interval = 0.1
55+
wait_seconds = 40
56+
for _ in range(int(wait_seconds / interval)): # 40 second timeout
57+
current_logs = self.log_file.read_text()
58+
if line in current_logs:
59+
print(current_logs.strip())
60+
print("...")
61+
break
62+
63+
time.sleep(0.1)
64+
else:
65+
raise RuntimeError(
66+
f"Could not get the server running fast enough, waited for {wait_seconds}s"
67+
)
68+
69+
def tearDown(self):
70+
self._server.terminate()
71+
print(f"Stopped PyPI server, all logs:\n{self.log_file.read_text()}")
72+
73+
def test_upload_and_query_simple_api(self):
74+
# Given
75+
script_path = Path(os.environ["PUBLISH_PATH"])
76+
whl = Path(os.environ["WHEEL_PATH"])
77+
78+
# When I publish a whl to a package registry
79+
subprocess.check_output(
80+
[
81+
str(script_path),
82+
"--no-color",
83+
"upload",
84+
str(whl),
85+
"--verbose",
86+
"--non-interactive",
87+
"--disable-progress-bar",
88+
],
89+
env={
90+
"TWINE_REPOSITORY_URL": self.url,
91+
"TWINE_USERNAME": "dummy",
92+
"TWINE_PASSWORD": "dummy",
93+
},
94+
)
95+
96+
# Then I should be able to get its contents
97+
with urlopen(self.url + "/example-minimal-library/") as response:
98+
got_content = response.read().decode("utf-8")
99+
want_content = """
100+
<!DOCTYPE html>
101+
<html>
102+
<head>
103+
<title>Links for example-minimal-library</title>
104+
</head>
105+
<body>
106+
<h1>Links for example-minimal-library</h1>
107+
<a href="/packages/example_minimal_library-0.0.1-py3-none-any.whl#sha256=79a4e9c1838c0631d5d8fa49a26efd6e9a364f6b38d9597c0f6df112271a0e28">example_minimal_library-0.0.1-py3-none-any.whl</a><br>
108+
</body>
109+
</html>"""
110+
self.assertEqual(
111+
textwrap.dedent(want_content).strip(),
112+
textwrap.dedent(got_content).strip(),
113+
)
114+
115+
116+
if __name__ == "__main__":
117+
unittest.main()

‎python/BUILD.bazel

Copy file name to clipboardExpand all lines: python/BUILD.bazel
+2Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,11 +76,13 @@ bzl_library(
7676
srcs = ["packaging.bzl"],
7777
deps = [
7878
":py_binary_bzl",
79+
"//python/private:bzlmod_enabled_bzl",
7980
"//python/private:py_package.bzl",
8081
"//python/private:py_wheel_bzl",
8182
"//python/private:py_wheel_normalize_pep440.bzl",
8283
"//python/private:stamp_bzl",
8384
"//python/private:util_bzl",
85+
"@bazel_skylib//rules:native_binary",
8486
],
8587
)
8688

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.