Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit 4910961

Browse filesBrowse files
aignasrickeylev
aignas
and
rickeylev
authored
fix(pypi): use python -B for repo-phase invocations (bazel-contrib#2641)
Before this change we would just invoke the Python interpreter. This means that in the `rules_python` directory there would be `__pycache__` folders created in the source tree and the same `__pycache__` folders would be created in the python interpreter repository rules if the directories were writable. This change ensures that we are executing `python` with `-B` in those contexts and reduces any likelihood of us doing the wrong thing. Work towards bazel-contrib#1169. --------- Co-authored-by: Richard Levasseur <richardlev@gmail.com>
1 parent 0fa6667 commit 4910961
Copy full SHA for 4910961

File tree

5 files changed

+61
-25
lines changed
Filter options

5 files changed

+61
-25
lines changed

‎CHANGELOG.md

Copy file name to clipboardExpand all lines: CHANGELOG.md
+3Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,9 @@ Unreleased changes template.
6767
([#1169](https://github.com/bazelbuild/rules_python/issues/1169)).
6868
* (gazelle) Don't collapse depsets to a list or into args when generating the modules mapping file.
6969
Support spilling modules mapping args into a params file.
70+
* (pypi) From now on `python` invocations in repository and module extension
71+
evaluation contexts will invoke Python interpreter with `-B` to avoid
72+
creating `.pyc` files.
7073

7174
{#v0-0-0-added}
7275
### Added

‎python/private/pypi/evaluate_markers.bzl

Copy file name to clipboardExpand all lines: python/private/pypi/evaluate_markers.bzl
+5-5Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -55,12 +55,12 @@ def evaluate_markers(mrctx, *, requirements, python_interpreter, python_interpre
5555
pypi_repo_utils.execute_checked(
5656
mrctx,
5757
op = "ResolveRequirementEnvMarkers({})".format(in_file),
58+
python = pypi_repo_utils.resolve_python_interpreter(
59+
mrctx,
60+
python_interpreter = python_interpreter,
61+
python_interpreter_target = python_interpreter_target,
62+
),
5863
arguments = [
59-
pypi_repo_utils.resolve_python_interpreter(
60-
mrctx,
61-
python_interpreter = python_interpreter,
62-
python_interpreter_target = python_interpreter_target,
63-
),
6464
"-m",
6565
"python.private.pypi.requirements_parser.resolve_target_platforms",
6666
in_file,

‎python/private/pypi/patch_whl.bzl

Copy file name to clipboardExpand all lines: python/private/pypi/patch_whl.bzl
+7-3Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,8 @@ other patches ensures that the users have overview on exactly what has changed
2727
within the wheel.
2828
"""
2929

30-
load("//python/private:repo_utils.bzl", "repo_utils")
3130
load(":parse_whl_name.bzl", "parse_whl_name")
31+
load(":pypi_repo_utils.bzl", "pypi_repo_utils")
3232

3333
_rules_python_root = Label("//:BUILD.bazel")
3434

@@ -102,10 +102,14 @@ def patch_whl(rctx, *, python_interpreter, whl_path, patches, **kwargs):
102102
record_patch = rctx.path("RECORD.patch")
103103
whl_patched = patched_whl_name(whl_input.basename)
104104

105-
repo_utils.execute_checked(
105+
pypi_repo_utils.execute_checked(
106106
rctx,
107+
python = python_interpreter,
108+
srcs = [
109+
Label("//python/private/pypi:repack_whl.py"),
110+
Label("//tools:wheelmaker.py"),
111+
],
107112
arguments = [
108-
python_interpreter,
109113
"-m",
110114
"python.private.pypi.repack_whl",
111115
"--record-patch",

‎python/private/pypi/pypi_repo_utils.bzl

Copy file name to clipboardExpand all lines: python/private/pypi/pypi_repo_utils.bzl
+41-14Lines changed: 41 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -104,11 +104,30 @@ def _construct_pypath(mrctx, *, entries):
104104
])
105105
return pypath
106106

107-
def _execute_checked(mrctx, *, srcs, **kwargs):
107+
def _execute_prep(mrctx, *, python, srcs, **kwargs):
108+
for src in srcs:
109+
# This will ensure that we will re-evaluate the bzlmod extension or
110+
# refetch the repository_rule when the srcs change. This should work on
111+
# Bazel versions without `mrctx.watch` as well.
112+
repo_utils.watch(mrctx, mrctx.path(src))
113+
114+
environment = kwargs.pop("environment", {})
115+
pythonpath = environment.get("PYTHONPATH", "")
116+
if pythonpath and not types.is_string(pythonpath):
117+
environment["PYTHONPATH"] = _construct_pypath(mrctx, entries = pythonpath)
118+
kwargs["environment"] = environment
119+
120+
# -B is added to prevent the repo-phase invocation from creating timestamp
121+
# based pyc files, which contributes to race conditions and non-determinism
122+
kwargs["arguments"] = [python, "-B"] + kwargs.get("arguments", [])
123+
return kwargs
124+
125+
def _execute_checked(mrctx, *, python, srcs, **kwargs):
108126
"""Helper function to run a python script and modify the PYTHONPATH to include external deps.
109127
110128
Args:
111129
mrctx: Handle to the module_ctx or repository_ctx.
130+
python: The python interpreter to use.
112131
srcs: The src files that the script depends on. This is important to
113132
ensure that the Bazel repository cache or the bzlmod lock file gets
114133
invalidated when any one file changes. It is advisable to use
@@ -118,26 +137,34 @@ def _execute_checked(mrctx, *, srcs, **kwargs):
118137
the `environment` has a value `PYTHONPATH` and it is a list, then
119138
it will be passed to `construct_pythonpath` function.
120139
"""
140+
return repo_utils.execute_checked(
141+
mrctx,
142+
**_execute_prep(mrctx, python = python, srcs = srcs, **kwargs)
143+
)
121144

122-
for src in srcs:
123-
# This will ensure that we will re-evaluate the bzlmod extension or
124-
# refetch the repository_rule when the srcs change. This should work on
125-
# Bazel versions without `mrctx.watch` as well.
126-
repo_utils.watch(mrctx, mrctx.path(src))
127-
128-
env = kwargs.pop("environment", {})
129-
pythonpath = env.get("PYTHONPATH", "")
130-
if pythonpath and not types.is_string(pythonpath):
131-
env["PYTHONPATH"] = _construct_pypath(mrctx, entries = pythonpath)
145+
def _execute_checked_stdout(mrctx, *, python, srcs, **kwargs):
146+
"""Helper function to run a python script and modify the PYTHONPATH to include external deps.
132147
133-
return repo_utils.execute_checked(
148+
Args:
149+
mrctx: Handle to the module_ctx or repository_ctx.
150+
python: The python interpreter to use.
151+
srcs: The src files that the script depends on. This is important to
152+
ensure that the Bazel repository cache or the bzlmod lock file gets
153+
invalidated when any one file changes. It is advisable to use
154+
`RECORD` files for external deps and the list of srcs from the
155+
rules_python repo for any scripts.
156+
**kwargs: Arguments forwarded to `repo_utils.execute_checked`. If
157+
the `environment` has a value `PYTHONPATH` and it is a list, then
158+
it will be passed to `construct_pythonpath` function.
159+
"""
160+
return repo_utils.execute_checked_stdout(
134161
mrctx,
135-
environment = env,
136-
**kwargs
162+
**_execute_prep(mrctx, python = python, srcs = srcs, **kwargs)
137163
)
138164

139165
pypi_repo_utils = struct(
140166
construct_pythonpath = _construct_pypath,
141167
execute_checked = _execute_checked,
168+
execute_checked_stdout = _execute_checked_stdout,
142169
resolve_python_interpreter = _resolve_python_interpreter,
143170
)

‎python/private/pypi/whl_library.bzl

Copy file name to clipboardExpand all lines: python/private/pypi/whl_library.bzl
+5-3Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -75,14 +75,15 @@ def _get_toolchain_unix_cflags(rctx, python_interpreter, logger = None):
7575
if not is_standalone_interpreter(rctx, python_interpreter, logger = logger):
7676
return []
7777

78-
stdout = repo_utils.execute_checked_stdout(
78+
stdout = pypi_repo_utils.execute_checked_stdout(
7979
rctx,
8080
op = "GetPythonVersionForUnixCflags",
81+
python = python_interpreter,
8182
arguments = [
82-
python_interpreter,
8383
"-c",
8484
"import sys; print(f'{sys.version_info[0]}.{sys.version_info[1]}', end='')",
8585
],
86+
srcs = [],
8687
)
8788
_python_version = stdout
8889
include_path = "{}/include/python{}".format(
@@ -181,7 +182,6 @@ def _whl_library_impl(rctx):
181182
python_interpreter_target = rctx.attr.python_interpreter_target,
182183
)
183184
args = [
184-
python_interpreter,
185185
"-m",
186186
"python.private.pypi.whl_installer.wheel_installer",
187187
"--requirement",
@@ -247,6 +247,7 @@ def _whl_library_impl(rctx):
247247
# truncate the requirement value when logging it / reporting
248248
# progress since it may contain several ' --hash=sha256:...
249249
# --hash=sha256:...' substrings that fill up the console
250+
python = python_interpreter,
250251
op = op_tmpl.format(name = rctx.attr.name, requirement = rctx.attr.requirement.split(" ", 1)[0]),
251252
arguments = args,
252253
environment = environment,
@@ -295,6 +296,7 @@ def _whl_library_impl(rctx):
295296
pypi_repo_utils.execute_checked(
296297
rctx,
297298
op = "whl_library.ExtractWheel({}, {})".format(rctx.attr.name, whl_path),
299+
python = python_interpreter,
298300
arguments = args + [
299301
"--whl-file",
300302
whl_path,

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.