Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit 915d7a0

Browse filesBrowse files
aignasaignas
authored
refactor(whl_library): split wheel downloading and extraction into separate executions (bazel-contrib#1487)
Before the PR the downloading/building of the wheel and the extraction would be done as a single step, which meant that for patching of the wheel to happen, we would need to do it within the python script. In order to have more flexibility in the approach, this PR splits the process to two separate invocations of the wheel_installer, which incidentally also helps in a case where the downloading of the wheel file can happen separately via http_file. Related issues bazel-contrib#1076, bazel-contrib#1357
1 parent a94deb8 commit 915d7a0
Copy full SHA for 915d7a0

File tree

Expand file treeCollapse file tree

5 files changed

+50
-14
lines changed
Filter options
Expand file treeCollapse file tree

5 files changed

+50
-14
lines changed

‎python/pip_install/pip_repository.bzl

Copy file name to clipboardExpand all lines: python/pip_install/pip_repository.bzl
+18-2Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -526,10 +526,25 @@ def _whl_library_impl(rctx):
526526

527527
args = _parse_optional_attrs(rctx, args)
528528

529+
# Manually construct the PYTHONPATH since we cannot use the toolchain here
530+
environment = _create_repository_execution_environment(rctx, python_interpreter)
531+
529532
result = rctx.execute(
530533
args,
531-
# Manually construct the PYTHONPATH since we cannot use the toolchain here
532-
environment = _create_repository_execution_environment(rctx, python_interpreter),
534+
environment = environment,
535+
quiet = rctx.attr.quiet,
536+
timeout = rctx.attr.timeout,
537+
)
538+
if result.return_code:
539+
fail("whl_library %s failed: %s (%s) error code: '%s'" % (rctx.attr.name, result.stdout, result.stderr, result.return_code))
540+
541+
whl_path = rctx.path(json.decode(rctx.read("whl_file.json"))["whl_file"])
542+
if not rctx.delete("whl_file.json"):
543+
fail("failed to delete the whl_file.json file")
544+
545+
result = rctx.execute(
546+
args + ["--whl-file", whl_path],
547+
environment = environment,
533548
quiet = rctx.attr.quiet,
534549
timeout = rctx.attr.timeout,
535550
)
@@ -562,6 +577,7 @@ def _whl_library_impl(rctx):
562577

563578
build_file_contents = generate_whl_library_build_bazel(
564579
repo_prefix = rctx.attr.repo_prefix,
580+
whl_name = whl_path.basename,
565581
dependencies = metadata["deps"],
566582
data_exclude = rctx.attr.pip_data_exclude,
567583
tags = [

‎python/pip_install/private/generate_whl_library_build_bazel.bzl

Copy file name to clipboardExpand all lines: python/pip_install/private/generate_whl_library_build_bazel.bzl
+5-1Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ filegroup(
6060
6161
filegroup(
6262
name = "{whl_file_label}",
63-
srcs = glob(["*.whl"], allow_empty = True),
63+
srcs = ["{whl_name}"],
6464
data = {whl_file_deps},
6565
)
6666
@@ -86,7 +86,9 @@ py_library(
8686
"""
8787

8888
def generate_whl_library_build_bazel(
89+
*,
8990
repo_prefix,
91+
whl_name,
9092
dependencies,
9193
data_exclude,
9294
tags,
@@ -96,6 +98,7 @@ def generate_whl_library_build_bazel(
9698
9799
Args:
98100
repo_prefix: the repo prefix that should be used for dependency lists.
101+
whl_name: the whl_name that this is generated for.
99102
dependencies: a list of PyPI packages that are dependencies to the py_library.
100103
data_exclude: more patterns to exclude from the data attribute of generated py_library rules.
101104
tags: list of tags to apply to generated py_library rules.
@@ -166,6 +169,7 @@ def generate_whl_library_build_bazel(
166169
name = _PY_LIBRARY_LABEL,
167170
dependencies = repr(lib_dependencies),
168171
data_exclude = repr(_data_exclude),
172+
whl_name = whl_name,
169173
whl_file_label = _WHEEL_FILE_LABEL,
170174
whl_file_deps = repr(whl_file_deps),
171175
tags = repr(tags),

‎python/pip_install/tools/wheel_installer/arguments.py

Copy file name to clipboardExpand all lines: python/pip_install/tools/wheel_installer/arguments.py
+6Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414

1515
import argparse
1616
import json
17+
import pathlib
1718
from typing import Any
1819

1920

@@ -59,6 +60,11 @@ def parser(**kwargs: Any) -> argparse.ArgumentParser:
5960
help="Use 'pip download' instead of 'pip wheel'. Disables building wheels from source, but allows use of "
6061
"--platform, --python-version, --implementation, and --abi in --extra_pip_args.",
6162
)
63+
parser.add_argument(
64+
"--whl-file",
65+
type=pathlib.Path,
66+
help="Extract a whl file to be used within Bazel.",
67+
)
6268
return parser
6369

6470

‎python/pip_install/tools/wheel_installer/wheel_installer.py

Copy file name to clipboardExpand all lines: python/pip_install/tools/wheel_installer/wheel_installer.py
+15-8Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,18 @@ def main() -> None:
155155

156156
_configure_reproducible_wheels()
157157

158+
if args.whl_file:
159+
whl = Path(args.whl_file)
160+
161+
name, extras_for_pkg = _parse_requirement_for_extra(args.requirement)
162+
extras = {name: extras_for_pkg} if extras_for_pkg and name else dict()
163+
_extract_wheel(
164+
wheel_file=whl,
165+
extras=extras,
166+
enable_implicit_namespace_pkgs=args.enable_implicit_namespace_pkgs,
167+
)
168+
return
169+
158170
pip_args = (
159171
[sys.executable, "-m", "pip"]
160172
+ (["--isolated"] if args.isolated else [])
@@ -185,15 +197,10 @@ def main() -> None:
185197
if e.errno != errno.ENOENT:
186198
raise
187199

188-
name, extras_for_pkg = _parse_requirement_for_extra(args.requirement)
189-
extras = {name: extras_for_pkg} if extras_for_pkg and name else dict()
200+
whl = Path(next(iter(glob.glob("*.whl"))))
190201

191-
whl = next(iter(glob.glob("*.whl")))
192-
_extract_wheel(
193-
wheel_file=whl,
194-
extras=extras,
195-
enable_implicit_namespace_pkgs=args.enable_implicit_namespace_pkgs,
196-
)
202+
with open("whl_file.json", "w") as f:
203+
json.dump({"whl_file": f"{whl.resolve()}"}, f)
197204

198205

199206
if __name__ == "__main__":

‎tests/pip_install/whl_library/generate_build_bazel_tests.bzl

Copy file name to clipboardExpand all lines: tests/pip_install/whl_library/generate_build_bazel_tests.bzl
+6-3Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ filegroup(
3838
3939
filegroup(
4040
name = "whl",
41-
srcs = glob(["*.whl"], allow_empty = True),
41+
srcs = ["foo.whl"],
4242
data = ["@pypi_bar_baz//:whl", "@pypi_foo//:whl"],
4343
)
4444
@@ -64,6 +64,7 @@ py_library(
6464
"""
6565
actual = generate_whl_library_build_bazel(
6666
repo_prefix = "pypi_",
67+
whl_name = "foo.whl",
6768
dependencies = ["foo", "bar-baz"],
6869
data_exclude = [],
6970
tags = ["tag1", "tag2"],
@@ -93,7 +94,7 @@ filegroup(
9394
9495
filegroup(
9596
name = "whl",
96-
srcs = glob(["*.whl"], allow_empty = True),
97+
srcs = ["foo.whl"],
9798
data = ["@pypi_bar_baz//:whl", "@pypi_foo//:whl"],
9899
)
99100
@@ -135,6 +136,7 @@ copy_file(
135136
"""
136137
actual = generate_whl_library_build_bazel(
137138
repo_prefix = "pypi_",
139+
whl_name = "foo.whl",
138140
dependencies = ["foo", "bar-baz"],
139141
data_exclude = [],
140142
tags = ["tag1", "tag2"],
@@ -171,7 +173,7 @@ filegroup(
171173
172174
filegroup(
173175
name = "whl",
174-
srcs = glob(["*.whl"], allow_empty = True),
176+
srcs = ["foo.whl"],
175177
data = ["@pypi_bar_baz//:whl", "@pypi_foo//:whl"],
176178
)
177179
@@ -206,6 +208,7 @@ py_binary(
206208
"""
207209
actual = generate_whl_library_build_bazel(
208210
repo_prefix = "pypi_",
211+
whl_name = "foo.whl",
209212
dependencies = ["foo", "bar-baz"],
210213
data_exclude = [],
211214
tags = ["tag1", "tag2"],

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.