Skip to content

Navigation Menu

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

Appearance settings

Commit beed9a2

Browse filesBrowse files
authored
Allow controlling the prefix added to repos/packages (bazel-contrib#459)
This consists of a breaking change for users who were directly depending on the prior pip_parse labels.
1 parent a876fca commit beed9a2
Copy full SHA for beed9a2

File tree

14 files changed

+89
-72
lines changed
Filter options

14 files changed

+89
-72
lines changed

‎docs/pip.md

Copy file name to clipboardExpand all lines: docs/pip.md
+1-1Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -216,7 +216,7 @@ alias(
216216
| Name | Description | Default Value |
217217
| :-------------: | :-------------: | :-------------: |
218218
| requirements_lock | A fully resolved 'requirements.txt' pip requirement file containing the transitive set of your dependencies. If this file is passed instead of 'requirements' no resolve will take place and pip_repository will create individual repositories for each of your dependencies so that wheels are fetched/built only for the targets specified by 'build/run/test'. | none |
219-
| name | The name of the generated repository. | <code>"pip_parsed_deps"</code> |
219+
| name | The name of the generated repository. The generated repositories containing each requirement will be of the form &lt;name&gt;_&lt;requirement-name&gt;. | <code>"pip_parsed_deps"</code> |
220220
| kwargs | Additional keyword arguments for the underlying <code>pip_repository</code> rule. | none |
221221

222222

‎examples/pip_parse/BUILD

Copy file name to clipboardExpand all lines: examples/pip_parse/BUILD
+2-3Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,8 @@
11
load(
2-
"@pip_parsed_deps//:requirements.bzl",
2+
"@pypi//:requirements.bzl",
33
"data_requirement",
44
"dist_info_requirement",
55
"entry_point",
6-
"requirement",
76
)
87
load("@rules_python//python:defs.bzl", "py_binary", "py_test")
98
load("@rules_python//python:pip.bzl", "compile_pip_requirements")
@@ -38,7 +37,7 @@ py_binary(
3837
name = "main",
3938
srcs = ["main.py"],
4039
deps = [
41-
requirement("requests"),
40+
"@pypi_requests//:pkg",
4241
],
4342
)
4443

‎examples/pip_parse/WORKSPACE

Copy file name to clipboardExpand all lines: examples/pip_parse/WORKSPACE
+2-3Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,12 +34,11 @@ pip_parse(
3434
# style env vars are read, but env vars that control requests and urllib3
3535
# can be passed
3636
# environment = {"HTTPS_PROXY": "http://my.proxy.fun/"},
37-
38-
# Uses the default repository name "pip_parsed_deps"
37+
name = "pypi",
3938
requirements_lock = "//:requirements_lock.txt",
4039
)
4140

42-
load("@pip_parsed_deps//:requirements.bzl", "install_deps")
41+
load("@pypi//:requirements.bzl", "install_deps")
4342

4443
# Initialize repositories for all packages in requirements_lock.txt.
4544
install_deps()

‎examples/pip_parse/pip_parse_test.py

Copy file name to clipboardExpand all lines: examples/pip_parse/pip_parse_test.py
+11-11Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -46,12 +46,12 @@ def test_data(self):
4646
self.assertListEqual(
4747
env.split(" "),
4848
[
49-
"external/pip_parsed_deps_pypi__s3cmd/s3cmd-2.1.0.data/data/share/doc/packages/s3cmd/INSTALL.md",
50-
"external/pip_parsed_deps_pypi__s3cmd/s3cmd-2.1.0.data/data/share/doc/packages/s3cmd/LICENSE",
51-
"external/pip_parsed_deps_pypi__s3cmd/s3cmd-2.1.0.data/data/share/doc/packages/s3cmd/NEWS",
52-
"external/pip_parsed_deps_pypi__s3cmd/s3cmd-2.1.0.data/data/share/doc/packages/s3cmd/README.md",
53-
"external/pip_parsed_deps_pypi__s3cmd/s3cmd-2.1.0.data/data/share/man/man1/s3cmd.1",
54-
"external/pip_parsed_deps_pypi__s3cmd/s3cmd-2.1.0.data/scripts/s3cmd",
49+
"external/pypi_s3cmd/s3cmd-2.1.0.data/data/share/doc/packages/s3cmd/INSTALL.md",
50+
"external/pypi_s3cmd/s3cmd-2.1.0.data/data/share/doc/packages/s3cmd/LICENSE",
51+
"external/pypi_s3cmd/s3cmd-2.1.0.data/data/share/doc/packages/s3cmd/NEWS",
52+
"external/pypi_s3cmd/s3cmd-2.1.0.data/data/share/doc/packages/s3cmd/README.md",
53+
"external/pypi_s3cmd/s3cmd-2.1.0.data/data/share/man/man1/s3cmd.1",
54+
"external/pypi_s3cmd/s3cmd-2.1.0.data/scripts/s3cmd",
5555
],
5656
)
5757

@@ -61,11 +61,11 @@ def test_dist_info(self):
6161
self.assertListEqual(
6262
env.split(" "),
6363
[
64-
"external/pip_parsed_deps_pypi__requests/requests-2.25.1.dist-info/LICENSE",
65-
"external/pip_parsed_deps_pypi__requests/requests-2.25.1.dist-info/METADATA",
66-
"external/pip_parsed_deps_pypi__requests/requests-2.25.1.dist-info/RECORD",
67-
"external/pip_parsed_deps_pypi__requests/requests-2.25.1.dist-info/WHEEL",
68-
"external/pip_parsed_deps_pypi__requests/requests-2.25.1.dist-info/top_level.txt",
64+
"external/pypi_requests/requests-2.25.1.dist-info/LICENSE",
65+
"external/pypi_requests/requests-2.25.1.dist-info/METADATA",
66+
"external/pypi_requests/requests-2.25.1.dist-info/RECORD",
67+
"external/pypi_requests/requests-2.25.1.dist-info/WHEEL",
68+
"external/pypi_requests/requests-2.25.1.dist-info/top_level.txt",
6969
],
7070
)
7171

‎python/pip.bzl

Copy file name to clipboardExpand all lines: python/pip.bzl
+4-1Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@ def pip_install(requirements, name = "pip", **kwargs):
9696
pip_repository(
9797
name = name,
9898
requirements = requirements,
99+
repo_prefix = "pypi__",
99100
**kwargs
100101
)
101102

@@ -171,7 +172,8 @@ def pip_parse(requirements_lock, name = "pip_parsed_deps", **kwargs):
171172
of 'requirements' no resolve will take place and pip_repository will create
172173
individual repositories for each of your dependencies so that wheels are
173174
fetched/built only for the targets specified by 'build/run/test'.
174-
name (str, optional): The name of the generated repository.
175+
name (str, optional): The name of the generated repository. The generated repositories
176+
containing each requirement will be of the form <name>_<requirement-name>.
175177
**kwargs (dict): Additional keyword arguments for the underlying
176178
`pip_repository` rule.
177179
"""
@@ -182,6 +184,7 @@ def pip_parse(requirements_lock, name = "pip_parsed_deps", **kwargs):
182184
pip_repository(
183185
name = name,
184186
requirements_lock = requirements_lock,
187+
repo_prefix = "{}_".format(name),
185188
incremental = True,
186189
**kwargs
187190
)

‎python/pip_install/extract_wheels/__init__.py

Copy file name to clipboardExpand all lines: python/pip_install/extract_wheels/__init__.py
+8-5Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -68,8 +68,8 @@ def main() -> None:
6868
# relative requirements to be correctly resolved. The --wheel-dir is therefore required to be repointed back to the
6969
# current calling working directory (the repo root in .../external/name), where the wheel files should be written to
7070
pip_args = (
71-
[sys.executable, "-m", "pip"] +
72-
(["--isolated"] if args.isolated else []) +
71+
[sys.executable, "-m", "pip"] +
72+
(["--isolated"] if args.isolated else []) +
7373
["wheel", "-r", args.requirements] +
7474
["--wheel-dir", os.getcwd()] +
7575
deserialized_args["extra_pip_args"]
@@ -86,11 +86,14 @@ def main() -> None:
8686
repo_label = "@%s" % args.repo
8787

8888
targets = [
89-
'"%s%s"'
90-
% (
89+
'"{}{}"'.format(
9190
repo_label,
9291
bazel.extract_wheel(
93-
whl, extras, deserialized_args["pip_data_exclude"], args.enable_implicit_namespace_pkgs
92+
whl,
93+
extras,
94+
deserialized_args["pip_data_exclude"],
95+
args.enable_implicit_namespace_pkgs,
96+
args.repo_prefix,
9497
),
9598
)
9699
for whl in glob.glob("*.whl")

‎python/pip_install/extract_wheels/lib/arguments.py

Copy file name to clipboardExpand all lines: python/pip_install/extract_wheels/lib/arguments.py
+5-1Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,11 @@ def parse_common_args(parser: ArgumentParser) -> ArgumentParser:
3030
action="store",
3131
help="Extra environment variables to set on the pip environment.",
3232
)
33+
parser.add_argument(
34+
"--repo-prefix",
35+
required=True,
36+
help="Prefix to prepend to packages",
37+
)
3338
return parser
3439

3540

@@ -45,4 +50,3 @@ def deserialize_structured_args(args):
4550
else:
4651
args[arg_name] = []
4752
return args
48-

‎python/pip_install/extract_wheels/lib/arguments_test.py

Copy file name to clipboardExpand all lines: python/pip_install/extract_wheels/lib/arguments_test.py
+10-1Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,16 +10,25 @@ def test_arguments(self) -> None:
1010
parser = argparse.ArgumentParser()
1111
parser = arguments.parse_common_args(parser)
1212
repo_name = "foo"
13+
repo_prefix = "pypi_"
1314
index_url = "--index_url=pypi.org/simple"
1415
extra_pip_args = [index_url]
1516
args_dict = vars(parser.parse_args(
16-
args=["--repo", repo_name, f"--extra_pip_args={json.dumps({'arg': extra_pip_args})}"]))
17+
args=[
18+
"--repo",
19+
repo_name,
20+
f"--extra_pip_args={json.dumps({'arg': extra_pip_args})}",
21+
"--repo-prefix",
22+
repo_prefix,
23+
]
24+
))
1725
args_dict = arguments.deserialize_structured_args(args_dict)
1826
self.assertIn("repo", args_dict)
1927
self.assertIn("extra_pip_args", args_dict)
2028
self.assertEqual(args_dict["pip_data_exclude"], [])
2129
self.assertEqual(args_dict["enable_implicit_namespace_pkgs"], False)
2230
self.assertEqual(args_dict["repo"], repo_name)
31+
self.assertEqual(args_dict["repo_prefix"], repo_prefix)
2332
self.assertEqual(args_dict["extra_pip_args"], extra_pip_args)
2433

2534
def test_deserialize_structured_args(self) -> None:

‎python/pip_install/extract_wheels/lib/bazel.py

Copy file name to clipboardExpand all lines: python/pip_install/extract_wheels/lib/bazel.py
+13-29Lines changed: 13 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -215,17 +215,7 @@ def install_deps():
215215
)
216216

217217

218-
DEFAULT_PACKAGE_PREFIX = "pypi__"
219-
220-
221-
def whl_library_repo_prefix(parent_repo: str) -> str:
222-
return "{parent}_{default_package_prefix}".format(
223-
parent=parent_repo,
224-
default_package_prefix=DEFAULT_PACKAGE_PREFIX
225-
)
226-
227-
228-
def sanitise_name(name: str, prefix: str = DEFAULT_PACKAGE_PREFIX) -> str:
218+
def sanitise_name(name: str, prefix: str) -> str:
229219
"""Sanitises the name to be compatible with Bazel labels.
230220
231221
There are certain requirements around Bazel labels that we need to consider. From the Bazel docs:
@@ -268,12 +258,12 @@ def setup_namespace_pkg_compatibility(wheel_dir: str) -> None:
268258
namespace_pkgs.add_pkgutil_style_namespace_pkg_init(ns_pkg_dir)
269259

270260

271-
def sanitised_library_label(whl_name: str) -> str:
272-
return '"//%s"' % sanitise_name(whl_name)
261+
def sanitised_library_label(whl_name: str, prefix: str) -> str:
262+
return '"//%s"' % sanitise_name(whl_name, prefix)
273263

274264

275-
def sanitised_file_label(whl_name: str) -> str:
276-
return '"//%s:%s"' % (sanitise_name(whl_name), WHEEL_FILE_LABEL)
265+
def sanitised_file_label(whl_name: str, prefix: str) -> str:
266+
return '"//%s:%s"' % (sanitise_name(whl_name, prefix), WHEEL_FILE_LABEL)
277267

278268

279269
def _whl_name_to_repo_root(whl_name: str, repo_prefix: str) -> str:
@@ -293,8 +283,8 @@ def extract_wheel(
293283
extras: Dict[str, Set[str]],
294284
pip_data_exclude: List[str],
295285
enable_implicit_namespace_pkgs: bool,
286+
repo_prefix: str,
296287
incremental: bool = False,
297-
incremental_repo_prefix: Optional[str] = None,
298288
) -> Optional[str]:
299289
"""Extracts wheel into given directory and creates py_library and filegroup targets.
300290
@@ -305,8 +295,6 @@ def extract_wheel(
305295
enable_implicit_namespace_pkgs: if true, disables conversion of implicit namespace packages and will unzip as-is
306296
incremental: If true the extract the wheel in a format suitable for an external repository. This
307297
effects the names of libraries and their dependencies, which point to other external repositories.
308-
incremental_repo_prefix: If incremental is true, use this prefix when creating labels from wheel
309-
names instead of the default.
310298
311299
Returns:
312300
The Bazel label for the extracted wheel, in the form '//path/to/wheel'.
@@ -316,7 +304,7 @@ def extract_wheel(
316304
if incremental:
317305
directory = "."
318306
else:
319-
directory = sanitise_name(whl.name)
307+
directory = sanitise_name(whl.name, prefix=repo_prefix)
320308

321309
os.mkdir(directory)
322310
# copy the original wheel
@@ -333,25 +321,21 @@ def extract_wheel(
333321
whl_deps = sorted(whl.dependencies(extras_requested))
334322

335323
if incremental:
336-
# check for mypy Optional validity
337-
if incremental_repo_prefix is None:
338-
raise TypeError(
339-
"incremental_repo_prefix arguement cannot be None if incremental == True")
340324
sanitised_dependencies = [
341-
sanitised_repo_library_label(d, repo_prefix=incremental_repo_prefix) for d in whl_deps
325+
sanitised_repo_library_label(d, repo_prefix=repo_prefix) for d in whl_deps
342326
]
343327
sanitised_wheel_file_dependencies = [
344-
sanitised_repo_file_label(d, repo_prefix=incremental_repo_prefix) for d in whl_deps
328+
sanitised_repo_file_label(d, repo_prefix=repo_prefix) for d in whl_deps
345329
]
346330
else:
347331
sanitised_dependencies = [
348-
sanitised_library_label(d) for d in whl_deps
332+
sanitised_library_label(d, prefix=repo_prefix) for d in whl_deps
349333
]
350334
sanitised_wheel_file_dependencies = [
351-
sanitised_file_label(d) for d in whl_deps
335+
sanitised_file_label(d, prefix=repo_prefix) for d in whl_deps
352336
]
353337

354-
library_name = PY_LIBRARY_LABEL if incremental else sanitise_name(whl.name)
338+
library_name = PY_LIBRARY_LABEL if incremental else sanitise_name(whl.name, repo_prefix)
355339

356340
directory_path = Path(directory)
357341
entry_points = []
@@ -365,7 +349,7 @@ def extract_wheel(
365349

366350
with open(os.path.join(directory, "BUILD.bazel"), "w") as build_file:
367351
contents = generate_build_file_contents(
368-
library_name,
352+
PY_LIBRARY_LABEL if incremental else sanitise_name(whl.name, repo_prefix),
369353
sanitised_dependencies,
370354
sanitised_wheel_file_dependencies,
371355
pip_data_exclude,

‎python/pip_install/extract_wheels/lib/whl_filegroup_test.py

Copy file name to clipboardExpand all lines: python/pip_install/extract_wheels/lib/whl_filegroup_test.py
+4-4Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,16 +24,16 @@ def tearDown(self):
2424

2525
def _run(
2626
self,
27+
repo_prefix: str,
2728
incremental: bool = False,
28-
incremental_repo_prefix: Optional[str] = None,
2929
) -> None:
3030
generated_bazel_dir = bazel.extract_wheel(
3131
self.wheel_path,
3232
extras={},
3333
pip_data_exclude=[],
3434
enable_implicit_namespace_pkgs=False,
3535
incremental=incremental,
36-
incremental_repo_prefix=incremental_repo_prefix
36+
repo_prefix=repo_prefix
3737
)
3838
# Take off the leading // from the returned label.
3939
# Assert that the raw wheel ends up in the package.
@@ -44,10 +44,10 @@ def _run(
4444
self.assertIn('filegroup', build_file_content)
4545

4646
def test_nonincremental(self) -> None:
47-
self._run()
47+
self._run(repo_prefix="prefix_")
4848

4949
def test_incremental(self) -> None:
50-
self._run(incremental=True, incremental_repo_prefix="test")
50+
self._run(incremental=True, repo_prefix="prefix_")
5151

5252

5353
if __name__ == "__main__":

‎python/pip_install/parse_requirements_to_bzl/__init__.py

Copy file name to clipboardExpand all lines: python/pip_install/parse_requirements_to_bzl/__init__.py
+11-8Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -60,16 +60,19 @@ def generate_parsed_requirements_contents(all_args: argparse.Namespace) -> str:
6060
args.setdefault("python_interpreter", sys.executable)
6161
# Pop this off because it wont be used as a config argument to the whl_library rule.
6262
requirements_lock = args.pop("requirements_lock")
63-
repo_prefix = bazel.whl_library_repo_prefix(args["repo"])
6463

6564
install_req_and_lines = parse_install_requirements(requirements_lock, args["extra_pip_args"])
66-
repo_names_and_reqs = repo_names_and_requirements(install_req_and_lines, repo_prefix)
67-
all_requirements = ", ".join(
68-
[bazel.sanitised_repo_library_label(ir.name, repo_prefix=repo_prefix) for ir, _ in install_req_and_lines]
69-
)
70-
all_whl_requirements = ", ".join(
71-
[bazel.sanitised_repo_file_label(ir.name, repo_prefix=repo_prefix) for ir, _ in install_req_and_lines]
65+
repo_names_and_reqs = repo_names_and_requirements(
66+
install_req_and_lines, args["repo_prefix"]
7267
)
68+
all_requirements = ", ".join([
69+
bazel.sanitised_repo_library_label(ir.name, repo_prefix=args["repo_prefix"])
70+
for ir, _ in install_req_and_lines
71+
])
72+
all_whl_requirements = ", ".join([
73+
bazel.sanitised_repo_file_label(ir.name, repo_prefix=args["repo_prefix"])
74+
for ir, _ in install_req_and_lines
75+
])
7376
return textwrap.dedent("""\
7477
load("@rules_python//python/pip_install:pip_repository.bzl", "whl_library")
7578
@@ -112,7 +115,7 @@ def install_deps():
112115
all_whl_requirements=all_whl_requirements,
113116
repo_names_and_reqs=repo_names_and_reqs,
114117
args=args,
115-
repo_prefix=repo_prefix,
118+
repo_prefix=args["repo_prefix"],
116119
py_library_label=bazel.PY_LIBRARY_LABEL,
117120
wheel_file_label=bazel.WHEEL_FILE_LABEL,
118121
data_label=bazel.DATA_LABEL,

‎python/pip_install/parse_requirements_to_bzl/extract_single_wheel/__init__.py

Copy file name to clipboardExpand all lines: python/pip_install/parse_requirements_to_bzl/extract_single_wheel/__init__.py
+2-2Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ def main() -> None:
3030

3131
pip_args = (
3232
[sys.executable, "-m", "pip"] +
33-
(["--isolated"] if args.isolated else []) +
33+
(["--isolated"] if args.isolated else []) +
3434
["wheel", "--no-deps"] +
3535
deserialized_args["extra_pip_args"]
3636
)
@@ -67,5 +67,5 @@ def main() -> None:
6767
deserialized_args["pip_data_exclude"],
6868
args.enable_implicit_namespace_pkgs,
6969
incremental=True,
70-
incremental_repo_prefix=bazel.whl_library_repo_prefix(args.repo)
70+
repo_prefix=args.repo_prefix,
7171
)

‎python/pip_install/parse_requirements_to_bzl/parse_requirements_to_bzl_test.py

Copy file name to clipboardExpand all lines: python/pip_install/parse_requirements_to_bzl/parse_requirements_to_bzl_test.py
+1-2Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
from python.pip_install.parse_requirements_to_bzl import generate_parsed_requirements_contents
77
from python.pip_install.extract_wheels.lib.bazel import (
88
sanitised_repo_library_label,
9-
whl_library_repo_prefix,
109
sanitised_repo_file_label
1110
)
1211

@@ -21,7 +20,7 @@ def test_generated_requirements_bzl(self) -> None:
2120
requirements_lock.flush()
2221
args = argparse.Namespace()
2322
args.requirements_lock = requirements_lock.name
24-
args.repo = "pip_parsed_deps"
23+
args.repo_prefix = "pip_parsed_deps_pypi__"
2524
extra_pip_args = ["--index-url=pypi.org/simple"]
2625
pip_data_exclude = ["**.foo"]
2726
args.extra_pip_args = json.dumps({"arg": extra_pip_args})

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.