diff --git a/README.md b/README.md index 0a0540a17e..03bcf0b3ee 100644 --- a/README.md +++ b/README.md @@ -97,8 +97,16 @@ load("@rules_python//python:pip.bzl", "pip_import") # This rule translates the specified requirements.txt into # @my_deps//:requirements.bzl, which itself exposes a pip_install method. pip_import( - name = "my_deps", - requirements = "//path/to:requirements.txt", + name = "my_deps", + requirements = "//path/to:requirements.txt", +) + +# Python interpreter can be specify optionally to support customized version, +# such as "python3" or "python3.7" +pip_import( + name = "my_deps", + requirements = "//path/to:requirements.txt", + python_interpreter = 'python3.7', ) # Load the pip_install symbol for my_deps, and create the dependencies' diff --git a/packaging/piptool.py b/packaging/piptool.py index 2544032ee8..9fef4c7ece 100644 --- a/packaging/piptool.py +++ b/packaging/piptool.py @@ -86,6 +86,9 @@ def pip_main(argv): parser = argparse.ArgumentParser( description='Import Python dependencies into Bazel.') +parser.add_argument('--python_interpreter', action='store', + help=('The python python interpreter to use ')) + parser.add_argument('--name', action='store', help=('The namespace of the import.')) @@ -98,6 +101,12 @@ def pip_main(argv): parser.add_argument('--directory', action='store', help=('The directory into which to put .whl files.')) + +def sort_wheels(whls): + """Sorts a list of wheels deterministically.""" + return sorted(whls, key=lambda w: w.distribution() + '_' + w.version()) + + def determine_possible_extras(whls): """Determines the list of possible "extras" for each .whl @@ -146,7 +155,7 @@ def is_possible(distro, extra): return { whl: [ extra - for extra in whl.extras() + for extra in sorted(whl.extras()) if is_possible(whl.distribution(), extra) ] for whl in whls @@ -167,7 +176,7 @@ def list_whls(): if fname.endswith('.whl'): yield os.path.join(root, fname) - whls = [Wheel(path) for path in list_whls()] + whls = sort_wheels(Wheel(path) for path in list_whls()) possible_extras = determine_possible_extras(whls) def whl_library(wheel): @@ -177,10 +186,12 @@ def whl_library(wheel): if "{repo_name}" not in native.existing_rules(): whl_library( name = "{repo_name}", + python_interpreter = "{python_interpreter}", whl = "@{name}//:{path}", requirements = "@{name}//:requirements.bzl", extras = [{extras}] )""".format(name=args.name, repo_name=wheel.repository_name(), + python_interpreter=args.python_interpreter, path=wheel.basename(), extras=','.join([ '"%s"' % extra diff --git a/packaging/whl.py b/packaging/whl.py index 63a6d62b8f..7dd8845ea7 100644 --- a/packaging/whl.py +++ b/packaging/whl.py @@ -14,11 +14,17 @@ """The whl modules defines classes for interacting with Python packages.""" import argparse +import collections +import email.parser import json import os import pkg_resources import re import zipfile +import sys + +EXTRA_RE = re.compile("""^(?P.*?)(;\s*(?P.*?)(extra == '(?P.*?)')?)$""") +MayRequiresKey = collections.namedtuple('MayRequiresKey', ('condition', 'extra')) class Wheel(object): @@ -44,7 +50,7 @@ def version(self): def repository_name(self): # Returns the canonical name of the Bazel repository for this package. - canonical = 'pypi__{}_{}'.format(self.distribution(), self.version()) + canonical = 'pypi__py{}__{}_{}'.format(sys.version_info.major, self.distribution(), self.version()) # Escape any illegal characters with underscore. return re.sub('[-.+]', '_', canonical) @@ -66,7 +72,7 @@ def metadata(self): pass # fall back to METADATA file (https://www.python.org/dev/peps/pep-0427/) with whl.open(self._dist_info() + '/METADATA') as f: - return self._parse_metadata(f.read().decode("utf-8")) + return self._parse_metadata(f) def name(self): return self.metadata().get('name') @@ -111,9 +117,48 @@ def expand(self, directory): # _parse_metadata parses METADATA files according to https://www.python.org/dev/peps/pep-0314/ def _parse_metadata(self, content): - # TODO: handle fields other than just name - name_pattern = re.compile('Name: (.*)') - return { 'name': name_pattern.search(content).group(1) } + metadata = {} + content_str = content.read() + try: + pkg_info = email.parser.Parser().parsestr(content_str) + except: + pkg_info = email.parser.BytesParser().parsebytes(content_str) + + metadata['name'] = pkg_info.get('Name') + extras = pkg_info.get_all('Provides-Extra') + if extras: + metadata['extras'] = list(set(extras)) + + reqs_dist = pkg_info.get_all('Requires-Dist') or [] + requires = collections.defaultdict(set) + for value in sorted(reqs_dist): + extra_match = EXTRA_RE.search(value) + if extra_match: + groupdict = extra_match.groupdict() + condition = groupdict['condition'] + extra = groupdict['extra'] + package = groupdict['package'] + if condition.endswith(' and '): + condition = condition[:-5] + else: + condition, extra = None, None + package = value + key = MayRequiresKey(condition, extra) + requires[key].add(package) + + if requires: + metadata['run_requires'] = [] + for key, value in requires.items(): + requirement = { + 'requires': list(value) + } + if key.extra: + requirement['extra'] = key.extra + if key.condition: + requirement['environment'] = key.condition + metadata['run_requires'].append(requirement) + + return metadata parser = argparse.ArgumentParser( diff --git a/python/pip.bzl b/python/pip.bzl index b69eb9366e..8554386997 100644 --- a/python/pip.bzl +++ b/python/pip.bzl @@ -24,8 +24,10 @@ def _pip_import_impl(repository_ctx): # To see the output, pass: quiet=False result = repository_ctx.execute([ - "python", + repository_ctx.attr.python_interpreter, repository_ctx.path(repository_ctx.attr._script), + "--python_interpreter", + repository_ctx.attr.python_interpreter, "--name", repository_ctx.attr.name, "--input", @@ -34,13 +36,14 @@ def _pip_import_impl(repository_ctx): repository_ctx.path("requirements.bzl"), "--directory", repository_ctx.path(""), - ]) + ], quiet=False) if result.return_code: fail("pip_import failed: %s (%s)" % (result.stdout, result.stderr)) pip_import = repository_rule( attrs = { + "python_interpreter": attr.string(default="python"), "requirements": attr.label( mandatory = True, allow_single_file = True, diff --git a/python/whl.bzl b/python/whl.bzl index 3f869c29ec..2eebfee49f 100644 --- a/python/whl.bzl +++ b/python/whl.bzl @@ -17,7 +17,7 @@ def _whl_impl(repository_ctx): """Core implementation of whl_library.""" args = [ - "python", + repository_ctx.attr.python_interpreter, repository_ctx.path(repository_ctx.attr._script), "--whl", repository_ctx.path(repository_ctx.attr.whl), @@ -37,6 +37,7 @@ def _whl_impl(repository_ctx): whl_library = repository_rule( attrs = { + "python_interpreter": attr.string(default="python"), "whl": attr.label( mandatory = True, allow_single_file = True, diff --git a/tools/piptool.par b/tools/piptool.par index f0e0f224e5..85dff20a2b 100755 Binary files a/tools/piptool.par and b/tools/piptool.par differ diff --git a/tools/whltool.par b/tools/whltool.par index c7c0212a1b..3d255ae6f1 100755 Binary files a/tools/whltool.par and b/tools/whltool.par differ