Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit 8fa6212

Browse filesBrowse files
vonschultzvonschultz
authored
feat: Add support for envsubst in extra_pip_args (bazel-contrib#1673)
Since `MODULE.bazel` files don't support `load()`, it is more difficult to grab the value of an environment variable such as `PIP_INDEX_URL` or `PIP_RETRIES` and feed it to `pip.parse()`. Some of these `PIP_*` variables are useful in `extra_pip_args`. To enable this use case, also for the bzlmod scenario, add the `envsubst` attribute, which is a list of environment variables to substitute in `extra_pip_args`. If `$VARNAME`, `${VARNAME}` or `${VARNAME:-default}` is found in `extra_pip_args`, substitute the value of the variable.
1 parent 8fba377 commit 8fa6212
Copy full SHA for 8fa6212

File tree

10 files changed

+267
-4
lines changed
Filter options

10 files changed

+267
-4
lines changed

‎CHANGELOG.md

Copy file name to clipboardExpand all lines: CHANGELOG.md
+6Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,12 @@ A brief description of the categories of changes:
2727
This fixes issues due to pyc files being created at runtime and affecting the
2828
definition of what files were considered part of the runtime.
2929

30+
* (pip_parse) Added the `envsubst` parameter, which enables environment variable
31+
substitutions in the `extra_pip_args` attribute.
32+
33+
* (pip_repository) Added the `envsubst` parameter, which enables environment
34+
variable substitutions in the `extra_pip_args` attribute.
35+
3036
### Fixed
3137

3238
* (bzlmod) pip.parse now does not fail with an empty `requirements.txt`.

‎examples/pip_parse_vendored/WORKSPACE

Copy file name to clipboardExpand all lines: examples/pip_parse_vendored/WORKSPACE
+6Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,14 @@ load("@rules_python//python:pip.bzl", "pip_parse")
1818

1919
# This repository isn't referenced, except by our test that asserts the requirements.bzl is updated.
2020
# It also wouldn't be needed by users of this ruleset.
21+
# If you're using envsubst with extra_pip_args, as we do below, the value of the environment
22+
# variables at the time we generate requirements.bzl don't make it into the file, as you may
23+
# verify by inspection; the environment variables at a later time, when we download the
24+
# packages, will be the ones that take effect.
2125
pip_parse(
2226
name = "pip",
27+
envsubst = ["PIP_RETRIES"],
28+
extra_pip_args = ["--retries=${PIP_RETRIES:-5}"],
2329
python_interpreter_target = "@python39_host//:python",
2430
requirements_lock = "//:requirements.txt",
2531
)

‎examples/pip_parse_vendored/requirements.bzl

Copy file name to clipboardExpand all lines: examples/pip_parse_vendored/requirements.bzl
+1-1Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ all_whl_requirements = all_whl_requirements_by_package.values()
1616
all_data_requirements = ["@pip//certifi:data", "@pip//charset_normalizer:data", "@pip//idna:data", "@pip//requests:data", "@pip//urllib3:data"]
1717

1818
_packages = [("pip_certifi", "certifi==2023.7.22 --hash=sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082 --hash=sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"), ("pip_charset_normalizer", "charset-normalizer==2.1.1 --hash=sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845 --hash=sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"), ("pip_idna", "idna==3.4 --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4 --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"), ("pip_requests", "requests==2.28.1 --hash=sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983 --hash=sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"), ("pip_urllib3", "urllib3==1.26.13 --hash=sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc --hash=sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8")]
19-
_config = {"download_only": False, "enable_implicit_namespace_pkgs": False, "environment": {}, "extra_pip_args": [], "isolated": True, "pip_data_exclude": [], "python_interpreter": "python3", "python_interpreter_target": "@python39_host//:python", "quiet": True, "repo": "pip", "repo_prefix": "pip_", "timeout": 600}
19+
_config = {"download_only": False, "enable_implicit_namespace_pkgs": False, "environment": {}, "envsubst": ["PIP_RETRIES"], "extra_pip_args": ["--retries=${PIP_RETRIES:-5}"], "isolated": True, "pip_data_exclude": [], "python_interpreter": "python3", "python_interpreter_target": "@python39_host//:python", "quiet": True, "repo": "pip", "repo_prefix": "pip_", "timeout": 600}
2020
_annotations = {}
2121

2222
def requirement(name):

‎python/pip_install/BUILD.bazel

Copy file name to clipboardExpand all lines: python/pip_install/BUILD.bazel
+1Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ bzl_library(
3030
"//python/pip_install/private:generate_whl_library_build_bazel_bzl",
3131
"//python/pip_install/private:srcs_bzl",
3232
"//python/private:bzlmod_enabled_bzl",
33+
"//python/private:envsubst_bzl",
3334
"//python/private:normalize_name_bzl",
3435
"//python/private:parse_whl_name_bzl",
3536
"//python/private:patch_whl_bzl",

‎python/pip_install/pip_repository.bzl

Copy file name to clipboardExpand all lines: python/pip_install/pip_repository.bzl
+37-3Lines changed: 37 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ load("//python/pip_install:requirements_parser.bzl", parse_requirements = "parse
2222
load("//python/pip_install/private:generate_group_library_build_bazel.bzl", "generate_group_library_build_bazel")
2323
load("//python/pip_install/private:generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel")
2424
load("//python/pip_install/private:srcs.bzl", "PIP_INSTALL_PY_SRCS")
25+
load("//python/private:envsubst.bzl", "envsubst")
2526
load("//python/private:normalize_name.bzl", "normalize_name")
2627
load("//python/private:parse_whl_name.bzl", "parse_whl_name")
2728
load("//python/private:patch_whl.bzl", "patch_whl")
@@ -195,12 +196,24 @@ def _parse_optional_attrs(rctx, args):
195196
if use_isolated(rctx, rctx.attr):
196197
args.append("--isolated")
197198

199+
# At the time of writing, the very latest Bazel, as in `USE_BAZEL_VERSION=last_green bazelisk`
200+
# supports rctx.getenv(name, default): When building incrementally, any change to the value of
201+
# the variable named by name will cause this repository to be re-fetched. That hasn't yet made
202+
# its way into the official releases, though.
203+
if "getenv" in dir(rctx):
204+
getenv = rctx.getenv
205+
else:
206+
getenv = rctx.os.environ.get
207+
198208
# Check for None so we use empty default types from our attrs.
199209
# Some args want to be list, and some want to be dict.
200210
if rctx.attr.extra_pip_args != None:
201211
args += [
202212
"--extra_pip_args",
203-
json.encode(struct(arg = rctx.attr.extra_pip_args)),
213+
json.encode(struct(arg = [
214+
envsubst(pip_arg, rctx.attr.envsubst, getenv)
215+
for pip_arg in rctx.attr.extra_pip_args
216+
])),
204217
]
205218

206219
if rctx.attr.download_only:
@@ -338,6 +351,7 @@ def _pip_repository_impl(rctx):
338351
"download_only": rctx.attr.download_only,
339352
"enable_implicit_namespace_pkgs": rctx.attr.enable_implicit_namespace_pkgs,
340353
"environment": rctx.attr.environment,
354+
"envsubst": rctx.attr.envsubst,
341355
"extra_pip_args": options,
342356
"isolated": use_isolated(rctx, rctx.attr),
343357
"pip_data_exclude": rctx.attr.pip_data_exclude,
@@ -418,10 +432,23 @@ Environment variables to set in the pip subprocess.
418432
Can be used to set common variables such as `http_proxy`, `https_proxy` and `no_proxy`
419433
Note that pip is run with "--isolated" on the CLI so `PIP_<VAR>_<NAME>`
420434
style env vars are ignored, but env vars that control requests and urllib3
421-
can be passed.
435+
can be passed. If you need `PIP_<VAR>_<NAME>`, take a look at `extra_pip_args`
436+
and `envsubst`.
422437
""",
423438
default = {},
424439
),
440+
"envsubst": attr.string_list(
441+
mandatory = False,
442+
doc = """\
443+
A list of environment variables to substitute (e.g. `["PIP_INDEX_URL",
444+
"PIP_RETRIES"]`). The corresponding variables are expanded in `extra_pip_args`
445+
using the syntax `$VARNAME` or `${VARNAME}` (expanding to empty string if unset)
446+
or `${VARNAME:-default}` (expanding to default if the variable is unset or empty
447+
in the environment). Note: On Bazel 6 and Bazel 7 changes to the variables named
448+
here do not cause packages to be re-fetched. Don't fetch different things based
449+
on the value of these variables.
450+
""",
451+
),
425452
"experimental_requirement_cycles": attr.string_list_dict(
426453
default = {},
427454
doc = """\
@@ -509,7 +536,14 @@ NOTE: this is not for cross-compiling Python wheels but rather for parsing the `
509536
""",
510537
),
511538
"extra_pip_args": attr.string_list(
512-
doc = "Extra arguments to pass on to pip. Must not contain spaces.",
539+
doc = """Extra arguments to pass on to pip. Must not contain spaces.
540+
541+
Supports environment variables using the syntax `$VARNAME` or
542+
`${VARNAME}` (expanding to empty string if unset) or
543+
`${VARNAME:-default}` (expanding to default if the variable is unset
544+
or empty in the environment), if `"VARNAME"` is listed in the
545+
`envsubst` attribute. See also `envsubst`.
546+
""",
513547
),
514548
"isolated": attr.bool(
515549
doc = """\

‎python/private/BUILD.bazel

Copy file name to clipboardExpand all lines: python/private/BUILD.bazel
+5Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,11 @@ bzl_library(
7777
],
7878
)
7979

80+
bzl_library(
81+
name = "envsubst_bzl",
82+
srcs = ["envsubst.bzl"],
83+
)
84+
8085
bzl_library(
8186
name = "full_version_bzl",
8287
srcs = ["full_version.bzl"],

‎python/private/bzlmod/pip.bzl

Copy file name to clipboardExpand all lines: python/private/bzlmod/pip.bzl
+1Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -200,6 +200,7 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides):
200200
pip_data_exclude = pip_attr.pip_data_exclude,
201201
enable_implicit_namespace_pkgs = pip_attr.enable_implicit_namespace_pkgs,
202202
environment = pip_attr.environment,
203+
envsubst = pip_attr.envsubst,
203204
group_name = group_name,
204205
group_deps = group_deps,
205206
)

‎python/private/envsubst.bzl

Copy file name to clipboard
+65Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
# Copyright 2024 The Bazel Authors. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"""Substitute environment variables in shell format strings."""
16+
17+
def envsubst(template_string, varnames, getenv):
18+
"""Helper function to substitute environment variables.
19+
20+
Supports `$VARNAME`, `${VARNAME}` and `${VARNAME:-default}`
21+
syntaxes in the `template_string`, looking up each `VARNAME`
22+
listed in the `varnames` list in the environment defined by the
23+
`getenv` function. Typically called with `getenv = rctx.getenv`
24+
(if it is available) or `getenv = rctx.os.environ.get` (on e.g.
25+
Bazel 6 or Bazel 7, which don't have `rctx.getenv` yet).
26+
27+
Limitations: Unlike the shell, we don't support `${VARNAME}` and
28+
`${VARNAME:-default}` in the default expression for a different
29+
environment variable expansion. We do support the braceless syntax
30+
in the default, so an expression such as `${HOME:-/home/$USER}` is
31+
valid.
32+
33+
Args:
34+
template_string: String that may contain variables to be expanded.
35+
varnames: List of variable names of variables to expand in
36+
`template_string`.
37+
getenv: Callable mapping variable names (in the first argument)
38+
to their values, or returns the default (provided in the
39+
second argument to `getenv`) if a value wasn't found.
40+
41+
Returns:
42+
`template_string` with environment variables expanded according
43+
to their values as determined by `getenv`.
44+
"""
45+
46+
if not varnames:
47+
return template_string
48+
49+
for varname in varnames:
50+
value = getenv(varname, "")
51+
template_string = template_string.replace("$%s" % varname, value)
52+
template_string = template_string.replace("${%s}" % varname, value)
53+
segments = template_string.split("${%s:-" % varname)
54+
template_string = segments.pop(0)
55+
for segment in segments:
56+
default_value, separator, rest = segment.partition("}")
57+
if "{" in default_value:
58+
fail("Environment substitution expression " +
59+
"\"${%s:-\" has an opening \"{\" " % varname +
60+
"in default value \"%s\"." % default_value)
61+
if not separator:
62+
fail("Environment substitution expression " +
63+
"\"${%s:-\" is missing the final \"}\"" % varname)
64+
template_string += (value if value else default_value) + rest
65+
return template_string

‎tests/private/envsubst/BUILD.bazel

Copy file name to clipboard
+19Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# Copyright 2024 The Bazel Authors. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"""Tests for envsubsts."""
16+
17+
load(":envsubst_tests.bzl", "envsubst_test_suite")
18+
19+
envsubst_test_suite(name = "envsubst_tests")

‎tests/private/envsubst/envsubst_tests.bzl

Copy file name to clipboard
+126Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
# Copyright 2024 The Bazel Authors. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
"""Test for py_wheel."""
15+
16+
load("@rules_testing//lib:analysis_test.bzl", "test_suite")
17+
load("//python/private:envsubst.bzl", "envsubst") # buildifier: disable=bzl-visibility
18+
19+
_basic_tests = []
20+
21+
def _test_envsubst_braceless(env):
22+
env.expect.that_str(
23+
envsubst("--retries=$PIP_RETRIES", ["PIP_RETRIES"], {"PIP_RETRIES": "5"}.get),
24+
).equals("--retries=5")
25+
26+
env.expect.that_str(
27+
envsubst("--retries=$PIP_RETRIES", [], {"PIP_RETRIES": "5"}.get),
28+
).equals("--retries=$PIP_RETRIES")
29+
30+
env.expect.that_str(
31+
envsubst("--retries=$PIP_RETRIES", ["PIP_RETRIES"], {}.get),
32+
).equals("--retries=")
33+
34+
_basic_tests.append(_test_envsubst_braceless)
35+
36+
def _test_envsubst_braces_without_default(env):
37+
env.expect.that_str(
38+
envsubst("--retries=${PIP_RETRIES}", ["PIP_RETRIES"], {"PIP_RETRIES": "5"}.get),
39+
).equals("--retries=5")
40+
41+
env.expect.that_str(
42+
envsubst("--retries=${PIP_RETRIES}", [], {"PIP_RETRIES": "5"}.get),
43+
).equals("--retries=${PIP_RETRIES}")
44+
45+
env.expect.that_str(
46+
envsubst("--retries=${PIP_RETRIES}", ["PIP_RETRIES"], {}.get),
47+
).equals("--retries=")
48+
49+
_basic_tests.append(_test_envsubst_braces_without_default)
50+
51+
def _test_envsubst_braces_with_default(env):
52+
env.expect.that_str(
53+
envsubst("--retries=${PIP_RETRIES:-6}", ["PIP_RETRIES"], {"PIP_RETRIES": "5"}.get),
54+
).equals("--retries=5")
55+
56+
env.expect.that_str(
57+
envsubst("--retries=${PIP_RETRIES:-6}", [], {"PIP_RETRIES": "5"}.get),
58+
).equals("--retries=${PIP_RETRIES:-6}")
59+
60+
env.expect.that_str(
61+
envsubst("--retries=${PIP_RETRIES:-6}", ["PIP_RETRIES"], {}.get),
62+
).equals("--retries=6")
63+
64+
_basic_tests.append(_test_envsubst_braces_with_default)
65+
66+
def _test_envsubst_nested_both_vars(env):
67+
env.expect.that_str(
68+
envsubst(
69+
"${HOME:-/home/$USER}",
70+
["HOME", "USER"],
71+
{"HOME": "/home/testuser", "USER": "mockuser"}.get,
72+
),
73+
).equals("/home/testuser")
74+
75+
_basic_tests.append(_test_envsubst_nested_both_vars)
76+
77+
def _test_envsubst_nested_outer_var(env):
78+
env.expect.that_str(
79+
envsubst(
80+
"${HOME:-/home/$USER}",
81+
["HOME"],
82+
{"HOME": "/home/testuser", "USER": "mockuser"}.get,
83+
),
84+
).equals("/home/testuser")
85+
86+
_basic_tests.append(_test_envsubst_nested_outer_var)
87+
88+
def _test_envsubst_nested_no_vars(env):
89+
env.expect.that_str(
90+
envsubst(
91+
"${HOME:-/home/$USER}",
92+
[],
93+
{"HOME": "/home/testuser", "USER": "mockuser"}.get,
94+
),
95+
).equals("${HOME:-/home/$USER}")
96+
97+
env.expect.that_str(
98+
envsubst("${HOME:-/home/$USER}", ["HOME", "USER"], {}.get),
99+
).equals("/home/")
100+
101+
_basic_tests.append(_test_envsubst_nested_no_vars)
102+
103+
def _test_envsubst_nested_braces_inner_var(env):
104+
env.expect.that_str(
105+
envsubst(
106+
"Home directory is ${HOME:-/home/$USER}.",
107+
["HOME", "USER"],
108+
{"USER": "mockuser"}.get,
109+
),
110+
).equals("Home directory is /home/mockuser.")
111+
112+
env.expect.that_str(
113+
envsubst(
114+
"Home directory is ${HOME:-/home/$USER}.",
115+
["USER"],
116+
{"USER": "mockuser"}.get,
117+
),
118+
).equals("Home directory is ${HOME:-/home/mockuser}.")
119+
120+
_basic_tests.append(_test_envsubst_nested_braces_inner_var)
121+
122+
def envsubst_test_suite(name):
123+
test_suite(
124+
name = name,
125+
basic_tests = _basic_tests,
126+
)

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.