From 1cc9ab58563444c6f8519ec1c8d4810d9dfd2d87 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius Date: Thu, 18 May 2023 10:44:46 +0900 Subject: [PATCH] feat(gazelle): allow passing multiple files to the manifest generator For certain workflows it is useful to calculate the integrity hash of the manifest file based on a number of requirements files. The requirements locking is usually done by executing a script on each platform and having gazelle manifest generator be aware that more than one requirements file may affect the outcome (e.g. the wheels that get passed to modules map may come from `multi_pip_parse` rule) is generally useful. This change modifies the `generator` to support multiple `--requirements` flag specification and uses all files that got passed to generate the final integrity hash. --- examples/build_file_generation/BUILD.bazel | 5 ++- .../build_file_generation/gazelle_python.yaml | 2 +- .../bzlmod_build_file_generation/BUILD.bazel | 5 ++- .../gazelle_python.yaml | 3 +- gazelle/manifest/defs.bzl | 28 +++++++++---- gazelle/manifest/generate/generate.go | 42 +++++++++++++------ gazelle/manifest/manifest.go | 13 +++--- 7 files changed, 67 insertions(+), 31 deletions(-) diff --git a/examples/build_file_generation/BUILD.bazel b/examples/build_file_generation/BUILD.bazel index 7c88d9203d..69cdd0615b 100644 --- a/examples/build_file_generation/BUILD.bazel +++ b/examples/build_file_generation/BUILD.bazel @@ -42,7 +42,10 @@ gazelle_python_manifest( name = "gazelle_python_manifest", modules_mapping = ":modules_map", pip_repository_name = "pip", - requirements = "//:requirements_lock.txt", + requirements_files = [ + "//:requirements_lock.txt", + "//:requirements_windows.txt", + ], # NOTE: we can use this flag in order to make our setup compatible with # bzlmod. use_pip_repository_aliases = True, diff --git a/examples/build_file_generation/gazelle_python.yaml b/examples/build_file_generation/gazelle_python.yaml index 1000757ea5..3ab318f7fe 100644 --- a/examples/build_file_generation/gazelle_python.yaml +++ b/examples/build_file_generation/gazelle_python.yaml @@ -115,4 +115,4 @@ manifest: pip_repository: name: pip use_pip_repository_aliases: true -integrity: 030d6d99b56c32d6577e616b617260d0a93588af791269162e43391a5a4fa576 +integrity: 35d9c488d8dc6e6c8f315dd45e3c942e98ac5b28a2d29ba9430513290b0ea063 diff --git a/examples/bzlmod_build_file_generation/BUILD.bazel b/examples/bzlmod_build_file_generation/BUILD.bazel index c667f1e49b..4de2ae807d 100644 --- a/examples/bzlmod_build_file_generation/BUILD.bazel +++ b/examples/bzlmod_build_file_generation/BUILD.bazel @@ -50,7 +50,10 @@ gazelle_python_manifest( name = "gazelle_python_manifest", modules_mapping = ":modules_map", pip_repository_name = "pip", - requirements = "//:requirements_lock.txt", + requirements_files = [ + "//:requirements_lock.txt", + "//:requirements_windows.txt", + ], # NOTE: we can use this flag in order to make our setup compatible with # bzlmod. use_pip_repository_aliases = True, diff --git a/examples/bzlmod_build_file_generation/gazelle_python.yaml b/examples/bzlmod_build_file_generation/gazelle_python.yaml index 12096e5837..e33021b9c8 100644 --- a/examples/bzlmod_build_file_generation/gazelle_python.yaml +++ b/examples/bzlmod_build_file_generation/gazelle_python.yaml @@ -232,6 +232,7 @@ manifest: isort.wrap: isort isort.wrap_modes: isort lazy_object_proxy: lazy_object_proxy + lazy_object_proxy.cext: lazy_object_proxy lazy_object_proxy.compat: lazy_object_proxy lazy_object_proxy.simple: lazy_object_proxy lazy_object_proxy.slots: lazy_object_proxy @@ -587,4 +588,4 @@ manifest: pip_repository: name: pip use_pip_repository_aliases: true -integrity: d979738b10adbbaff0884837e4414688990491c6c40f6a25d58b9bb564411477 +integrity: cee7684391c4a8a1ff219cd354deae61cdcdee70f2076789aabd5249f3c4eca9 diff --git a/gazelle/manifest/defs.bzl b/gazelle/manifest/defs.bzl index 05562a1583..e1dd59aea7 100644 --- a/gazelle/manifest/defs.bzl +++ b/gazelle/manifest/defs.bzl @@ -20,8 +20,9 @@ load("@io_bazel_rules_go//go:def.bzl", "GoSource", "go_binary", "go_test") def gazelle_python_manifest( name, - requirements, modules_mapping, + requirements = None, + requirements_files = None, pip_repository_name = "", pip_deps_repository_name = "", manifest = ":gazelle_python.yaml", @@ -31,6 +32,7 @@ def gazelle_python_manifest( Args: name: the name used as a base for the targets. requirements: the target for the requirements.txt file. + requirements_files: the list of requirements files to include in the hashing algorithm. pip_repository_name: the name of the pip_install or pip_repository target. use_pip_repository_aliases: boolean flag to enable using user-friendly python package aliases. @@ -38,6 +40,12 @@ def gazelle_python_manifest( modules_mapping: the target for the generated modules_mapping.json file. manifest: the target for the Gazelle manifest file. """ + if not requirements and not requirements_files: + fail("at least one of 'requirements' and 'requirements_files' must be defined") + + if requirements and requirements_files: + fail("only one of 'requirements' and 'requirements_files' can be defined at a time") + if pip_deps_repository_name != "": # buildifier: disable=print print("DEPRECATED pip_deps_repository_name in //{}:{}. Please use pip_repository_name instead.".format( @@ -58,8 +66,6 @@ def gazelle_python_manifest( update_args = [ "--manifest-generator-hash", "$(rootpath {})".format(manifest_generator_hash), - "--requirements", - "$(rootpath {})".format(requirements), "--pip-repository-name", pip_repository_name, "--modules-mapping", @@ -70,6 +76,12 @@ def gazelle_python_manifest( update_target_label, ] + requirements_files = requirements_files or [requirements] + + for requirements in sorted(requirements_files): + update_args.append("--requirements") + update_args.append("$(rootpath {})".format(requirements)) + if use_pip_repository_aliases: update_args += [ "--use-pip-repository-aliases", @@ -81,10 +93,9 @@ def gazelle_python_manifest( embed = [Label("//manifest/generate:generate_lib")], data = [ manifest, - modules_mapping, - requirements, manifest_generator_hash, - ], + modules_mapping, + ] + requirements_files, args = update_args, visibility = ["//visibility:private"], tags = ["manual"], @@ -95,13 +106,12 @@ def gazelle_python_manifest( srcs = [Label("//manifest/test:test.go")], data = [ manifest, - requirements, manifest_generator_hash, - ], + ] + requirements_files, env = { "_TEST_MANIFEST": "$(rootpath {})".format(manifest), "_TEST_MANIFEST_GENERATOR_HASH": "$(rootpath {})".format(manifest_generator_hash), - "_TEST_REQUIREMENTS": "$(rootpath {})".format(requirements), + "_TEST_REQUIREMENTS": "$(rootpaths {})".format(requirements_files), }, rundir = ".", deps = [Label("//manifest")], diff --git a/gazelle/manifest/generate/generate.go b/gazelle/manifest/generate/generate.go index 1f56e630cc..7fc1ff4d4a 100644 --- a/gazelle/manifest/generate/generate.go +++ b/gazelle/manifest/generate/generate.go @@ -24,6 +24,7 @@ import ( "encoding/json" "flag" "fmt" + "io" "log" "os" "strings" @@ -37,10 +38,21 @@ func init() { } } +type arrayFlags []string + +func (i arrayFlags) String() string { + return fmt.Sprintf("%q", []string(i)) +} + +func (i *arrayFlags) Set(value string) error { + *i = append(*i, value) + return nil +} + func main() { var ( manifestGeneratorHashPath string - requirementsPath string + requirementsPaths arrayFlags pipRepositoryName string usePipRepositoryAliases bool modulesMappingPath string @@ -53,11 +65,10 @@ func main() { "", "The file containing the hash for the source code of the manifest generator."+ "This is important to force manifest updates when the generator logic changes.") - flag.StringVar( - &requirementsPath, + flag.Var( + &requirementsPaths, "requirements", - "", - "The requirements.txt file.") + "The requirements.txt files. The argument can be supplied multiple times.") flag.StringVar( &pipRepositoryName, "pip-repository-name", @@ -85,7 +96,7 @@ func main() { "The Bazel target to update the YAML manifest file.") flag.Parse() - if requirementsPath == "" { + if len(requirementsPaths) == 0 { log.Fatalln("ERROR: --requirements must be set") } @@ -120,7 +131,7 @@ func main() { header, manifestFile, manifestGeneratorHashPath, - requirementsPath, + []string(requirementsPaths), ); err != nil { log.Fatalf("ERROR: %v\n", err) } @@ -159,7 +170,7 @@ func writeOutput( header string, manifestFile *manifest.File, manifestGeneratorHashPath string, - requirementsPath string, + requirementsPaths []string, ) error { stat, err := os.Stat(outputPath) if err != nil { @@ -182,13 +193,18 @@ func writeOutput( } defer manifestGeneratorHash.Close() - requirements, err := os.Open(requirementsPath) - if err != nil { - return fmt.Errorf("failed to write output: %w", err) + var allRequirements []io.Reader + for _, requirementsPath := range(requirementsPaths) { + requirements, err := os.Open(requirementsPath) + if err != nil { + return fmt.Errorf("failed to open %q: %w", requirementsPath, err) + } + defer requirements.Close() + + allRequirements = append(allRequirements, requirements) } - defer requirements.Close() - if err := manifestFile.Encode(outputFile, manifestGeneratorHash, requirements); err != nil { + if err := manifestFile.Encode(outputFile, manifestGeneratorHash, allRequirements...); err != nil { return fmt.Errorf("failed to write output: %w", err) } diff --git a/gazelle/manifest/manifest.go b/gazelle/manifest/manifest.go index c49951dd3e..99e8c3459d 100644 --- a/gazelle/manifest/manifest.go +++ b/gazelle/manifest/manifest.go @@ -40,11 +40,12 @@ func NewFile(manifest *Manifest) *File { } // Encode encodes the manifest file to the given writer. -func (f *File) Encode(w io.Writer, manifestGeneratorHashFile, requirements io.Reader) error { - integrityBytes, err := f.calculateIntegrity(manifestGeneratorHashFile, requirements) +func (f *File) Encode(w io.Writer, manifestGeneratorHashFile io.Reader, requirements ...io.Reader) error { + integrityBytes, err := f.calculateIntegrity(manifestGeneratorHashFile, requirements...) if err != nil { return fmt.Errorf("failed to encode manifest file: %w", err) } + f.Integrity = fmt.Sprintf("%x", integrityBytes) encoder := yaml.NewEncoder(w) defer encoder.Close() @@ -69,7 +70,7 @@ func (f *File) VerifyIntegrity(manifestGeneratorHashFile, requirements io.Reader // mapping, plus the manifest structure in the manifest file. This integrity // calculation ensures the manifest files are kept up-to-date. func (f *File) calculateIntegrity( - manifestGeneratorHash, requirements io.Reader, + manifestGeneratorHash io.Reader, requirements ...io.Reader, ) ([]byte, error) { hash := sha256.New() @@ -86,8 +87,10 @@ func (f *File) calculateIntegrity( } // Sum the requirements.txt checksum bytes. - if _, err := io.Copy(hash, requirements); err != nil { - return nil, fmt.Errorf("failed to calculate integrity: %w", err) + for _, r := range(requirements) { + if _, err := io.Copy(hash, r); err != nil { + return nil, fmt.Errorf("failed to calculate integrity: %w", err) + } } return hash.Sum(nil), nil