Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit 2b6c185

Browse filesBrowse files
mattmoormattmoor
committed
This adds support for "extras".
"Extras" are additional dependencies of a given library, which are consumed by passing the "extra" name in brackets after the distribution name, for example: ``` mock[docs]==1.0.1 ``` We see this in the dependencies of several Google Cloud libraries, which depend on: `googleapis_common_protos[grpc]` I've added a simple test that the dependency structure we synthesize for this kind of thing is correct via an "extras" test that has a `requirements.txt` of: ``` google-cloud-language==0.27.0 ``` Fixes: bazel-contrib#12
1 parent 72456c9 commit 2b6c185
Copy full SHA for 2b6c185

File tree

Expand file treeCollapse file tree

12 files changed

+260
-57
lines changed
Filter options
Expand file treeCollapse file tree

12 files changed

+260
-57
lines changed

‎README.md

Copy file name to clipboardExpand all lines: README.md
+4Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,10 @@ for dependencies, however, it is recommended that folks stick with the
106106
`requirement` pattern in case the need arises for us to make changes to this
107107
format in the future.
108108

109+
["Extras"](
110+
https://packaging.python.org/tutorials/installing-packages/#installing-setuptools-extras)
111+
will have a target of the extra name (in place of `pkg` above).
112+
109113
## Updating `docs/`
110114

111115
All of the content (except `BUILD`) under `docs/` is generated. To update the

‎WORKSPACE

Copy file name to clipboardExpand all lines: WORKSPACE
+22-1Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,8 @@ sass_repositories()
2626

2727
git_repository(
2828
name = "io_bazel_skydoc",
29-
remote = "https://github.com/bazelbuild/skydoc.git",
3029
commit = "e9be81cf5be41e4200749f5d8aa2db7955f8aacc",
30+
remote = "https://github.com/bazelbuild/skydoc.git",
3131
)
3232

3333
load("@io_bazel_skydoc//skylark:skylark.bzl", "skydoc_repositories")
@@ -92,6 +92,15 @@ http_file(
9292
"mock-2.0.0-py2.py3-none-any.whl"),
9393
)
9494

95+
http_file(
96+
name = "google_cloud_language_whl",
97+
sha256 = "a2dd34f0a0ebf5705dcbe34bd41199b1d0a55c4597d38ed045bd183361a561e9",
98+
# From https://pypi.python.org/pypi/google-cloud-language
99+
url = ("https://pypi.python.org/packages/6e/86/" +
100+
"cae57e4802e72d9e626ee5828ed5a646cf4016b473a4a022f1038dba3460/" +
101+
"google_cloud_language-0.29.0-py2.py3-none-any.whl"),
102+
)
103+
95104
# Imports for examples
96105
pip_import(
97106
name = "examples_helloworld",
@@ -128,3 +137,15 @@ load(
128137
)
129138

130139
_boto_install()
140+
141+
pip_import(
142+
name = "examples_extras",
143+
requirements = "//examples/extras:requirements.txt",
144+
)
145+
146+
load(
147+
"@examples_extras//:requirements.bzl",
148+
_extras_install = "pip_install",
149+
)
150+
151+
_extras_install()

‎examples/extras/BUILD

Copy file name to clipboard
+29Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
# Copyright 2017 The Bazel Authors. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
package(default_visibility = ["//visibility:public"])
15+
16+
licenses(["notice"]) # Apache 2.0
17+
18+
load("@examples_extras//:requirements.bzl", "requirement")
19+
load("//python:python.bzl", "py_test")
20+
21+
py_test(
22+
name = "extras_test",
23+
srcs = ["extras_test.py"],
24+
deps = [
25+
requirement("google-cloud-language"),
26+
# Make sure that we can resolve the "extra" dependency
27+
requirement("googleapis-common-protos[grpc]"),
28+
],
29+
)

‎examples/extras/extras_test.py

Copy file name to clipboard
+25Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
# Copyright 2017 The Bazel Authors. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import unittest
16+
17+
18+
# The test is the build itself, which should not work if extras are missing.
19+
class ExtrasTest(unittest.TestCase):
20+
def test_nothing(self):
21+
pass
22+
23+
24+
if __name__ == '__main__':
25+
unittest.main()

‎examples/extras/requirements.txt

Copy file name to clipboard
+1Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
google-cloud-language==0.27.0

‎examples/extras/version_test.py

Copy file name to clipboard
+26Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
# Copyright 2017 The Bazel Authors. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import pip
16+
import unittest
17+
18+
19+
class VersionTest(unittest.TestCase):
20+
21+
def test_version(self):
22+
self.assertEqual(pip.__version__, '9.0.1')
23+
24+
25+
if __name__ == '__main__':
26+
unittest.main()

‎python/whl.bzl

Copy file name to clipboardExpand all lines: python/whl.bzl
+11-2Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,17 @@
1616
def _whl_impl(repository_ctx):
1717
"""Core implementation of whl_library."""
1818

19-
result = repository_ctx.execute([
19+
args = [
2020
"python",
2121
repository_ctx.path(repository_ctx.attr._script),
2222
"--whl", repository_ctx.path(repository_ctx.attr.whl),
2323
"--requirements", repository_ctx.attr.requirements,
24-
])
24+
]
25+
26+
if repository_ctx.attr.extras:
27+
args += ["--extras", ",".join(repository_ctx.attr.extras)]
28+
29+
result = repository_ctx.execute(args)
2530
if result.return_code:
2631
fail("whl_library failed: %s (%s)" % (result.stdout, result.stderr))
2732

@@ -33,6 +38,7 @@ whl_library = repository_rule(
3338
single_file = True,
3439
),
3540
"requirements": attr.string(),
41+
"extras": attr.string_list(),
3642
"_script": attr.label(
3743
executable = True,
3844
default = Label("//rules_python:whl.py"),
@@ -64,4 +70,7 @@ Args:
6470
6571
requirements: The name of the pip_import repository rule from which to
6672
load this .whl's dependencies.
73+
74+
extras: A subset of the "extras" available from this <code>.whl</code> for which
75+
<code>requirements</code> has the dependencies.
6776
"""

‎rules_python/BUILD

Copy file name to clipboardExpand all lines: rules_python/BUILD
+1Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ py_test(
2828
data = [
2929
"@futures_3_1_1_whl//file",
3030
"@futures_2_2_0_whl//file",
31+
"@google_cloud_language_whl//file",
3132
"@grpc_whl//file",
3233
"@mock_whl//file",
3334
],

‎rules_python/piptool.py

Copy file name to clipboardExpand all lines: rules_python/piptool.py
+77-37Lines changed: 77 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import json
1919
import os
2020
import pkgutil
21+
import pkg_resources
2122
import re
2223
import shutil
2324
import sys
@@ -78,36 +79,7 @@ def pip_main(argv):
7879
argv = ["--disable-pip-version-check", "--cert", cert_path] + argv
7980
return pip.main(argv)
8081

81-
82-
# TODO(mattmoor): We can't easily depend on other libraries when
83-
# being invoked as a raw .py file. Once bundled, we should be able
84-
# to remove this fallback on a stub implementation of Wheel.
85-
try:
86-
from rules_python.whl import Wheel
87-
except:
88-
class Wheel(object):
89-
90-
def __init__(self, path):
91-
self._path = path
92-
93-
def basename(self):
94-
return os.path.basename(self._path)
95-
96-
def distribution(self):
97-
# See https://www.python.org/dev/peps/pep-0427/#file-name-convention
98-
parts = self.basename().split('-')
99-
return parts[0]
100-
101-
def version(self):
102-
# See https://www.python.org/dev/peps/pep-0427/#file-name-convention
103-
parts = self.basename().split('-')
104-
return parts[1]
105-
106-
def repository_name(self):
107-
# Returns the canonical name of the Bazel repository for this package.
108-
canonical = 'pypi__{}_{}'.format(self.distribution(), self.version())
109-
# Escape any illegal characters with underscore.
110-
return re.sub('[-.]', '_', canonical)
82+
from rules_python.whl import Wheel
11183

11284
parser = argparse.ArgumentParser(
11385
description='Import Python dependencies into Bazel.')
@@ -124,6 +96,59 @@ def repository_name(self):
12496
parser.add_argument('--directory', action='store',
12597
help=('The directory into which to put .whl files.'))
12698

99+
def determine_possible_extras(whls):
100+
"""Determines the list of possible "extras" for each .whl
101+
102+
The possibility of an extra is determined by looking at its
103+
additional requirements, and determinine whether they are
104+
satisfied by the complete list of available wheels.
105+
106+
Args:
107+
whls: a list of Wheel objects
108+
109+
Returns:
110+
a dict that is keyed by the Wheel objects in whls, and whose
111+
values are lists of possible extras.
112+
"""
113+
whl_map = {
114+
whl.distribution(): whl
115+
for whl in whls
116+
}
117+
118+
# TODO(mattmoor): Consider memoizing if this recursion ever becomes
119+
# expensive enough to warrant it.
120+
def is_possible(distro, extra):
121+
distro = distro.replace("-", "_")
122+
# If we don't have the .whl at all, then this isn't possible.
123+
if distro not in whl_map:
124+
return False
125+
whl = whl_map[distro]
126+
# If we have the .whl, and we don't need anything extra then
127+
# we can satisfy this dependency.
128+
if not extra:
129+
return True
130+
# If we do need something extra, then check the extra's
131+
# dependencies to make sure they are fully satisfied.
132+
for extra_dep in whl.dependencies(extra=extra):
133+
req = pkg_resources.Requirement.parse(extra_dep)
134+
# Check that the dep and any extras are all possible.
135+
if not is_possible(req.project_name, None):
136+
return False
137+
for e in req.extras:
138+
if not is_possible(req.project_name, e):
139+
return False
140+
# If all of the dependencies of the extra are satisfiable then
141+
# it is possible to construct this dependency.
142+
return True
143+
144+
return {
145+
whl: [
146+
extra
147+
for extra in whl.extras()
148+
if is_possible(whl.distribution(), extra)
149+
]
150+
for whl in whls
151+
}
127152

128153
def main():
129154
args = parser.parse_args()
@@ -140,6 +165,9 @@ def list_whls():
140165
if fname.endswith('.whl'):
141166
yield os.path.join(root, fname)
142167

168+
whls = [Wheel(path) for path in list_whls()]
169+
possible_extras = determine_possible_extras(whls)
170+
143171
def whl_library(wheel):
144172
# Indentation here matters. whl_library must be within the scope
145173
# of the function below. We also avoid reimporting an existing WHL.
@@ -149,10 +177,25 @@ def whl_library(wheel):
149177
name = "{repo_name}",
150178
whl = "@{name}//:{path}",
151179
requirements = "@{name}//:requirements.bzl",
180+
extras = [{extras}]
152181
)""".format(name=args.name, repo_name=wheel.repository_name(),
153-
path=wheel.basename())
154-
155-
whls = [Wheel(path) for path in list_whls()]
182+
path=wheel.basename(),
183+
extras=','.join([
184+
'"%s"' % extra
185+
for extra in possible_extras.get(wheel, [])
186+
]))
187+
188+
whl_targets = ','.join([
189+
','.join([
190+
'"%s": "@%s//:pkg"' % (whl.distribution().lower(), whl.repository_name())
191+
] + [
192+
# For every extra that is possible from this requirements.txt
193+
'"%s[%s]": "@%s//:%s"' % (whl.distribution().lower(), extra.lower(),
194+
whl.repository_name(), extra)
195+
for extra in possible_extras.get(whl, [])
196+
])
197+
for whl in whls
198+
])
156199

157200
with open(args.output, 'w') as f:
158201
f.write("""\
@@ -178,10 +221,7 @@ def requirement(name):
178221
return _requirements[name_key]
179222
""".format(input=args.input,
180223
whl_libraries='\n'.join(map(whl_library, whls)) if whls else "pass",
181-
mappings=','.join([
182-
'"%s": "@%s//:pkg"' % (wheel.distribution().lower(), wheel.repository_name())
183-
for wheel in whls
184-
])))
224+
mappings=whl_targets))
185225

186226
if __name__ == '__main__':
187227
main()

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.