Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit cdc0305

Browse filesBrowse files
dillon-giacoppodillon-giacoppo
authored
Fix unnecessary namespace directory creation (bazel-contrib#54)
1 parent bd1f4be commit cdc0305
Copy full SHA for cdc0305

File tree

9 files changed

+117
-187
lines changed
Filter options

9 files changed

+117
-187
lines changed

‎example/BUILD

Copy file name to clipboardExpand all lines: example/BUILD
+22-20Lines changed: 22 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,29 @@
11
load("@pip//:requirements.bzl", "requirement")
2-
load("@rules_python//python:defs.bzl", "py_runtime_pair")
32

43
# Toolchain setup, this is optional.
54
# Demonstrate that we can use the same python interpreter for the toolchain and executing pip in pip install (see WORKSPACE).
6-
py_runtime(
7-
name = "python3_runtime",
8-
files = ["@python_interpreter//:files"],
9-
interpreter = "@python_interpreter//:python_bin",
10-
python_version = "PY3",
11-
visibility = ["//visibility:public"],
12-
)
13-
14-
py_runtime_pair(
15-
name = "my_py_runtime_pair",
16-
py2_runtime = None,
17-
py3_runtime = ":python3_runtime",
18-
)
19-
20-
toolchain(
21-
name = "my_py_toolchain",
22-
toolchain = ":my_py_runtime_pair",
23-
toolchain_type = "@bazel_tools//tools/python:toolchain_type",
24-
)
5+
#
6+
#load("@rules_python//python:defs.bzl", "py_runtime_pair")
7+
#
8+
#py_runtime(
9+
# name = "python3_runtime",
10+
# files = ["@python_interpreter//:files"],
11+
# interpreter = "@python_interpreter//:python_bin",
12+
# python_version = "PY3",
13+
# visibility = ["//visibility:public"],
14+
#)
15+
#
16+
#py_runtime_pair(
17+
# name = "my_py_runtime_pair",
18+
# py2_runtime = None,
19+
# py3_runtime = ":python3_runtime",
20+
#)
21+
#
22+
#toolchain(
23+
# name = "my_py_toolchain",
24+
# toolchain = ":my_py_runtime_pair",
25+
# toolchain_type = "@bazel_tools//tools/python:toolchain_type",
26+
#)
2527
# End of toolchain setup.
2628

2729
py_binary(

‎example/WORKSPACE

Copy file name to clipboardExpand all lines: example/WORKSPACE
+43-42Lines changed: 43 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -4,52 +4,15 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
44

55
http_archive(
66
name = "rules_python",
7-
sha256 = "aa96a691d3a8177f3215b14b0edc9641787abaaa30363a080165d06ab65e1161",
8-
url = "https://github.com/bazelbuild/rules_python/releases/download/0.0.1/rules_python-0.0.1.tar.gz",
7+
url = "https://github.com/bazelbuild/rules_python/releases/download/0.0.2/rules_python-0.0.2.tar.gz",
8+
strip_prefix = "rules_python-0.0.2",
9+
sha256 = "b5668cde8bb6e3515057ef465a35ad712214962f0b3a314e551204266c7be90c",
910
)
1011

1112
load("@rules_python//python:repositories.bzl", "py_repositories")
1213

1314
py_repositories()
1415

15-
# You could optionally use an in-build, compiled python interpreter as a toolchain,
16-
# and also use it to execute pip.
17-
18-
# Special logic for building python interpreter with OpenSSL from homebrew.
19-
# See https://devguide.python.org/setup/#macos-and-os-x
20-
_py_configure = """
21-
if [[ "$OSTYPE" == "darwin"* ]]; then
22-
./configure --prefix=$(pwd)/bazel_install --with-openssl=$(brew --prefix openssl)
23-
else
24-
./configure --prefix=$(pwd)/bazel_install
25-
fi
26-
"""
27-
28-
# NOTE: you need to have the SSL headers installed to build with openssl support (and use HTTPS).
29-
# E.g. on Ubuntu: `sudo apt install libssl-dev`
30-
http_archive(
31-
name = "python_interpreter",
32-
build_file_content = """
33-
exports_files(["python_bin"])
34-
filegroup(
35-
name = "files",
36-
srcs = glob(["bazel_install/**"], exclude = ["**/* *"]),
37-
visibility = ["//visibility:public"],
38-
)
39-
""",
40-
patch_cmds = [
41-
"mkdir $(pwd)/bazel_install",
42-
_py_configure,
43-
"make",
44-
"make install",
45-
"ln -s bazel_install/bin/python3 python_bin",
46-
],
47-
sha256 = "dfab5ec723c218082fe3d5d7ae17ecbdebffa9a1aea4d64aa3a2ecdd2e795864",
48-
strip_prefix = "Python-3.8.3",
49-
urls = ["https://www.python.org/ftp/python/3.8.3/Python-3.8.3.tar.xz"],
50-
)
51-
# End of in-build Python interpreter setup.
52-
5316
local_repository(
5417
name = "rules_python_external",
5518
path = "../",
@@ -75,7 +38,7 @@ pip_install(
7538
# 1. Python interpreter that you compile in the build file (as above in @python_interpreter).
7639
# 2. Pre-compiled python interpreter included with http_archive
7740
# 3. Wrapper script, like in the autodetecting python toolchain.
78-
python_interpreter_target = "@python_interpreter//:python_bin",
41+
#python_interpreter_target = "@python_interpreter//:python_bin",
7942

8043
# (Optional) You can set quiet to False if you want to see pip output.
8144
#quiet = False,
@@ -84,6 +47,44 @@ pip_install(
8447
requirements = "//:requirements.txt",
8548
)
8649

50+
# You could optionally use an in-build, compiled python interpreter as a toolchain,
51+
# and also use it to execute pip.
52+
#
53+
# Special logic for building python interpreter with OpenSSL from homebrew.
54+
# See https://devguide.python.org/setup/#macos-and-os-x
55+
#_py_configure = """
56+
#if [[ "$OSTYPE" == "darwin"* ]]; then
57+
# ./configure --prefix=$(pwd)/bazel_install --with-openssl=$(brew --prefix openssl)
58+
#else
59+
# ./configure --prefix=$(pwd)/bazel_install
60+
#fi
61+
#"""
62+
#
63+
# NOTE: you need to have the SSL headers installed to build with openssl support (and use HTTPS).
64+
# E.g. on Ubuntu: `sudo apt install libssl-dev`
65+
#http_archive(
66+
# name = "python_interpreter",
67+
# build_file_content = """
68+
#exports_files(["python_bin"])
69+
#filegroup(
70+
# name = "files",
71+
# srcs = glob(["bazel_install/**"], exclude = ["**/* *"]),
72+
# visibility = ["//visibility:public"],
73+
#)
74+
#""",
75+
# patch_cmds = [
76+
# "mkdir $(pwd)/bazel_install",
77+
# _py_configure,
78+
# "make",
79+
# "make install",
80+
# "ln -s bazel_install/bin/python3 python_bin",
81+
# ],
82+
# sha256 = "dfab5ec723c218082fe3d5d7ae17ecbdebffa9a1aea4d64aa3a2ecdd2e795864",
83+
# strip_prefix = "Python-3.8.3",
84+
# urls = ["https://www.python.org/ftp/python/3.8.3/Python-3.8.3.tar.xz"],
85+
#)
86+
8787
# Optional:
8888
# Register the toolchain with the same python interpreter we used for pip in pip_install().
89-
register_toolchains("//:my_py_toolchain")
89+
#register_toolchains("//:my_py_toolchain")
90+
# End of in-build Python interpreter setup.

‎example/main.py

Copy file name to clipboard
+2-1Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
import boto3
22

3-
print("Hello World")
3+
if __name__ == "__main__":
4+
pass

‎extract_wheels/__init__.py

Copy file name to clipboardExpand all lines: extract_wheels/__init__.py
+8-4Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -64,18 +64,22 @@ def main() -> None:
6464
required=True,
6565
help="The external repo name to install dependencies. In the format '@{REPO_NAME}'",
6666
)
67-
parser.add_argument('--extra_pip_args', action='store',
68-
help=('Extra arguments to pass down to pip.'))
67+
parser.add_argument(
68+
"--extra_pip_args",
69+
action="store",
70+
help=("Extra arguments to pass down to pip."),
71+
)
6972
parser.add_argument(
7073
"--pip_data_exclude",
71-
action='store',
72-
help='Additional data exclusion parameters to add to the pip packages BUILD file.'
74+
action="store",
75+
help="Additional data exclusion parameters to add to the pip packages BUILD file.",
7376
)
7477
args = parser.parse_args()
7578

7679
pip_args = [sys.executable, "-m", "pip", "wheel", "-r", args.requirements]
7780
if args.extra_pip_args:
7881
pip_args += json.loads(args.extra_pip_args)["args"]
82+
7983
# Assumes any errors are logged by pip so do nothing. This command will fail if pip fails
8084
subprocess.run(pip_args, check=True)
8185

‎extract_wheels/lib/bazel.py

Copy file name to clipboardExpand all lines: extract_wheels/lib/bazel.py
+15-13Lines changed: 15 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,9 @@
77
from extract_wheels.lib import namespace_pkgs, wheel, purelib
88

99

10-
def generate_build_file_contents(name: str, dependencies: List[str], pip_data_exclude: List[str]) -> str:
10+
def generate_build_file_contents(
11+
name: str, dependencies: List[str], pip_data_exclude: List[str]
12+
) -> str:
1113
"""Generate a BUILD file for an unzipped Wheel
1214
1315
Args:
@@ -39,7 +41,9 @@ def generate_build_file_contents(name: str, dependencies: List[str], pip_data_ex
3941
deps = [{dependencies}],
4042
)
4143
""".format(
42-
name=name, dependencies=",".join(dependencies), data_exclude=json.dumps(data_exclude)
44+
name=name,
45+
dependencies=",".join(dependencies),
46+
data_exclude=json.dumps(data_exclude),
4347
)
4448
)
4549

@@ -93,33 +97,31 @@ def sanitise_name(name: str) -> str:
9397

9498

9599
def setup_namespace_pkg_compatibility(wheel_dir: str) -> None:
96-
"""Converts native namespace packages and pkg_resource-style packages to pkgutil-style packages
100+
"""Converts native namespace packages to pkgutil-style packages
97101
98102
Namespace packages can be created in one of three ways. They are detailed here:
99103
https://packaging.python.org/guides/packaging-namespace-packages/#creating-a-namespace-package
100104
101-
'pkgutil-style namespace packages' (2) works in Bazel, but 'native namespace packages' (1) and
102-
'pkg_resources-style namespace packages' (3) do not.
105+
'pkgutil-style namespace packages' (2) and 'pkg_resources-style namespace packages' (3) works in Bazel, but
106+
'native namespace packages' (1) do not.
103107
104-
We ensure compatibility with Bazel of methods 1 and 3 by converting them into method 2.
108+
We ensure compatibility with Bazel of method 1 by converting them into method 2.
105109
106110
Args:
107111
wheel_dir: the directory of the wheel to convert
108112
"""
109113

110-
namespace_pkg_dirs = namespace_pkgs.pkg_resources_style_namespace_packages(
111-
wheel_dir
114+
namespace_pkg_dirs = namespace_pkgs.implicit_namespace_packages(
115+
wheel_dir, ignored_dirnames=["%s/bin" % wheel_dir,],
112116
)
113-
if not namespace_pkg_dirs and namespace_pkgs.native_namespace_packages_supported():
114-
namespace_pkg_dirs = namespace_pkgs.implicit_namespace_packages(
115-
wheel_dir, ignored_dirnames=["%s/bin" % wheel_dir,],
116-
)
117117

118118
for ns_pkg_dir in namespace_pkg_dirs:
119119
namespace_pkgs.add_pkgutil_style_namespace_pkg_init(ns_pkg_dir)
120120

121121

122-
def extract_wheel(wheel_file: str, extras: Dict[str, Set[str]], pip_data_exclude: List[str]) -> str:
122+
def extract_wheel(
123+
wheel_file: str, extras: Dict[str, Set[str]], pip_data_exclude: List[str]
124+
) -> str:
123125
"""Extracts wheel into given directory and creates a py_library target.
124126
125127
Args:

‎extract_wheels/lib/namespace_pkgs.py

Copy file name to clipboardExpand all lines: extract_wheels/lib/namespace_pkgs.py
+1-27Lines changed: 1 addition & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,36 +1,11 @@
11
"""Utility functions to discover python package types"""
22
import os
3-
import sys
43
import textwrap
54
from typing import Set, List, Optional
65

76
from extract_wheels.lib import wheel
87

98

10-
def pkg_resources_style_namespace_packages(wheel_dir: str) -> Set[str]:
11-
"""Discovers namespace packages implemented using the 'pkg_resources-style namespace packages' method.
12-
13-
"While this approach is no longer recommended, it is widely present in most existing namespace packages." - PyPA
14-
See https://packaging.python.org/guides/packaging-namespace-packages/#pkg-resources-style-namespace-packages
15-
"""
16-
namespace_pkg_dirs = set()
17-
18-
dist_info = wheel.get_dist_info(wheel_dir)
19-
namespace_packages_record_file = os.path.join(dist_info, "namespace_packages.txt")
20-
if os.path.exists(namespace_packages_record_file):
21-
with open(namespace_packages_record_file) as nspkg:
22-
for line in nspkg.readlines():
23-
namespace = line.strip().replace(".", os.sep)
24-
if namespace:
25-
namespace_pkg_dirs.add(os.path.join(wheel_dir, namespace))
26-
return namespace_pkg_dirs
27-
28-
29-
def native_namespace_packages_supported() -> bool:
30-
"""Returns true if this version of Python supports native namespace packages."""
31-
return (sys.version_info.major, sys.version_info.minor) >= (3, 3)
32-
33-
349
def implicit_namespace_packages(
3510
directory: str, ignored_dirnames: Optional[List[str]] = None
3611
) -> Set[str]:
@@ -84,8 +59,7 @@ def add_pkgutil_style_namespace_pkg_init(dir_path: str) -> None:
8459

8560
if os.path.isfile(ns_pkg_init_filepath):
8661
raise ValueError("%s already contains an __init__.py file." % dir_path)
87-
if not os.path.exists(dir_path):
88-
os.makedirs(dir_path)
62+
8963
with open(ns_pkg_init_filepath, "w") as ns_pkg_init_f:
9064
# See https://packaging.python.org/guides/packaging-namespace-packages/#pkgutil-style-namespace-packages
9165
ns_pkg_init_f.write(

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.