diff --git a/examples/build_file_generation/BUILD.bazel b/examples/build_file_generation/BUILD.bazel index 7c88d9203d..69cdd0615b 100644 --- a/examples/build_file_generation/BUILD.bazel +++ b/examples/build_file_generation/BUILD.bazel @@ -42,7 +42,10 @@ gazelle_python_manifest( name = "gazelle_python_manifest", modules_mapping = ":modules_map", pip_repository_name = "pip", - requirements = "//:requirements_lock.txt", + requirements_files = [ + "//:requirements_lock.txt", + "//:requirements_windows.txt", + ], # NOTE: we can use this flag in order to make our setup compatible with # bzlmod. use_pip_repository_aliases = True, diff --git a/examples/build_file_generation/gazelle_python.yaml b/examples/build_file_generation/gazelle_python.yaml index 1000757ea5..3ab318f7fe 100644 --- a/examples/build_file_generation/gazelle_python.yaml +++ b/examples/build_file_generation/gazelle_python.yaml @@ -115,4 +115,4 @@ manifest: pip_repository: name: pip use_pip_repository_aliases: true -integrity: 030d6d99b56c32d6577e616b617260d0a93588af791269162e43391a5a4fa576 +integrity: 35d9c488d8dc6e6c8f315dd45e3c942e98ac5b28a2d29ba9430513290b0ea063 diff --git a/examples/bzlmod_build_file_generation/BUILD.bazel b/examples/bzlmod_build_file_generation/BUILD.bazel index c667f1e49b..4de2ae807d 100644 --- a/examples/bzlmod_build_file_generation/BUILD.bazel +++ b/examples/bzlmod_build_file_generation/BUILD.bazel @@ -50,7 +50,10 @@ gazelle_python_manifest( name = "gazelle_python_manifest", modules_mapping = ":modules_map", pip_repository_name = "pip", - requirements = "//:requirements_lock.txt", + requirements_files = [ + "//:requirements_lock.txt", + "//:requirements_windows.txt", + ], # NOTE: we can use this flag in order to make our setup compatible with # bzlmod. use_pip_repository_aliases = True, diff --git a/examples/bzlmod_build_file_generation/gazelle_python.yaml b/examples/bzlmod_build_file_generation/gazelle_python.yaml index 12096e5837..e33021b9c8 100644 --- a/examples/bzlmod_build_file_generation/gazelle_python.yaml +++ b/examples/bzlmod_build_file_generation/gazelle_python.yaml @@ -232,6 +232,7 @@ manifest: isort.wrap: isort isort.wrap_modes: isort lazy_object_proxy: lazy_object_proxy + lazy_object_proxy.cext: lazy_object_proxy lazy_object_proxy.compat: lazy_object_proxy lazy_object_proxy.simple: lazy_object_proxy lazy_object_proxy.slots: lazy_object_proxy @@ -587,4 +588,4 @@ manifest: pip_repository: name: pip use_pip_repository_aliases: true -integrity: d979738b10adbbaff0884837e4414688990491c6c40f6a25d58b9bb564411477 +integrity: cee7684391c4a8a1ff219cd354deae61cdcdee70f2076789aabd5249f3c4eca9 diff --git a/gazelle/manifest/defs.bzl b/gazelle/manifest/defs.bzl index 05562a1583..e1dd59aea7 100644 --- a/gazelle/manifest/defs.bzl +++ b/gazelle/manifest/defs.bzl @@ -20,8 +20,9 @@ load("@io_bazel_rules_go//go:def.bzl", "GoSource", "go_binary", "go_test") def gazelle_python_manifest( name, - requirements, modules_mapping, + requirements = None, + requirements_files = None, pip_repository_name = "", pip_deps_repository_name = "", manifest = ":gazelle_python.yaml", @@ -31,6 +32,7 @@ def gazelle_python_manifest( Args: name: the name used as a base for the targets. requirements: the target for the requirements.txt file. + requirements_files: the list of requirements files to include in the hashing algorithm. pip_repository_name: the name of the pip_install or pip_repository target. use_pip_repository_aliases: boolean flag to enable using user-friendly python package aliases. @@ -38,6 +40,12 @@ def gazelle_python_manifest( modules_mapping: the target for the generated modules_mapping.json file. manifest: the target for the Gazelle manifest file. """ + if not requirements and not requirements_files: + fail("at least one of 'requirements' and 'requirements_files' must be defined") + + if requirements and requirements_files: + fail("only one of 'requirements' and 'requirements_files' can be defined at a time") + if pip_deps_repository_name != "": # buildifier: disable=print print("DEPRECATED pip_deps_repository_name in //{}:{}. Please use pip_repository_name instead.".format( @@ -58,8 +66,6 @@ def gazelle_python_manifest( update_args = [ "--manifest-generator-hash", "$(rootpath {})".format(manifest_generator_hash), - "--requirements", - "$(rootpath {})".format(requirements), "--pip-repository-name", pip_repository_name, "--modules-mapping", @@ -70,6 +76,12 @@ def gazelle_python_manifest( update_target_label, ] + requirements_files = requirements_files or [requirements] + + for requirements in sorted(requirements_files): + update_args.append("--requirements") + update_args.append("$(rootpath {})".format(requirements)) + if use_pip_repository_aliases: update_args += [ "--use-pip-repository-aliases", @@ -81,10 +93,9 @@ def gazelle_python_manifest( embed = [Label("//manifest/generate:generate_lib")], data = [ manifest, - modules_mapping, - requirements, manifest_generator_hash, - ], + modules_mapping, + ] + requirements_files, args = update_args, visibility = ["//visibility:private"], tags = ["manual"], @@ -95,13 +106,12 @@ def gazelle_python_manifest( srcs = [Label("//manifest/test:test.go")], data = [ manifest, - requirements, manifest_generator_hash, - ], + ] + requirements_files, env = { "_TEST_MANIFEST": "$(rootpath {})".format(manifest), "_TEST_MANIFEST_GENERATOR_HASH": "$(rootpath {})".format(manifest_generator_hash), - "_TEST_REQUIREMENTS": "$(rootpath {})".format(requirements), + "_TEST_REQUIREMENTS": "$(rootpaths {})".format(requirements_files), }, rundir = ".", deps = [Label("//manifest")], diff --git a/gazelle/manifest/generate/generate.go b/gazelle/manifest/generate/generate.go index 1f56e630cc..7fc1ff4d4a 100644 --- a/gazelle/manifest/generate/generate.go +++ b/gazelle/manifest/generate/generate.go @@ -24,6 +24,7 @@ import ( "encoding/json" "flag" "fmt" + "io" "log" "os" "strings" @@ -37,10 +38,21 @@ func init() { } } +type arrayFlags []string + +func (i arrayFlags) String() string { + return fmt.Sprintf("%q", []string(i)) +} + +func (i *arrayFlags) Set(value string) error { + *i = append(*i, value) + return nil +} + func main() { var ( manifestGeneratorHashPath string - requirementsPath string + requirementsPaths arrayFlags pipRepositoryName string usePipRepositoryAliases bool modulesMappingPath string @@ -53,11 +65,10 @@ func main() { "", "The file containing the hash for the source code of the manifest generator."+ "This is important to force manifest updates when the generator logic changes.") - flag.StringVar( - &requirementsPath, + flag.Var( + &requirementsPaths, "requirements", - "", - "The requirements.txt file.") + "The requirements.txt files. The argument can be supplied multiple times.") flag.StringVar( &pipRepositoryName, "pip-repository-name", @@ -85,7 +96,7 @@ func main() { "The Bazel target to update the YAML manifest file.") flag.Parse() - if requirementsPath == "" { + if len(requirementsPaths) == 0 { log.Fatalln("ERROR: --requirements must be set") } @@ -120,7 +131,7 @@ func main() { header, manifestFile, manifestGeneratorHashPath, - requirementsPath, + []string(requirementsPaths), ); err != nil { log.Fatalf("ERROR: %v\n", err) } @@ -159,7 +170,7 @@ func writeOutput( header string, manifestFile *manifest.File, manifestGeneratorHashPath string, - requirementsPath string, + requirementsPaths []string, ) error { stat, err := os.Stat(outputPath) if err != nil { @@ -182,13 +193,18 @@ func writeOutput( } defer manifestGeneratorHash.Close() - requirements, err := os.Open(requirementsPath) - if err != nil { - return fmt.Errorf("failed to write output: %w", err) + var allRequirements []io.Reader + for _, requirementsPath := range(requirementsPaths) { + requirements, err := os.Open(requirementsPath) + if err != nil { + return fmt.Errorf("failed to open %q: %w", requirementsPath, err) + } + defer requirements.Close() + + allRequirements = append(allRequirements, requirements) } - defer requirements.Close() - if err := manifestFile.Encode(outputFile, manifestGeneratorHash, requirements); err != nil { + if err := manifestFile.Encode(outputFile, manifestGeneratorHash, allRequirements...); err != nil { return fmt.Errorf("failed to write output: %w", err) } diff --git a/gazelle/manifest/manifest.go b/gazelle/manifest/manifest.go index c49951dd3e..99e8c3459d 100644 --- a/gazelle/manifest/manifest.go +++ b/gazelle/manifest/manifest.go @@ -40,11 +40,12 @@ func NewFile(manifest *Manifest) *File { } // Encode encodes the manifest file to the given writer. -func (f *File) Encode(w io.Writer, manifestGeneratorHashFile, requirements io.Reader) error { - integrityBytes, err := f.calculateIntegrity(manifestGeneratorHashFile, requirements) +func (f *File) Encode(w io.Writer, manifestGeneratorHashFile io.Reader, requirements ...io.Reader) error { + integrityBytes, err := f.calculateIntegrity(manifestGeneratorHashFile, requirements...) if err != nil { return fmt.Errorf("failed to encode manifest file: %w", err) } + f.Integrity = fmt.Sprintf("%x", integrityBytes) encoder := yaml.NewEncoder(w) defer encoder.Close() @@ -69,7 +70,7 @@ func (f *File) VerifyIntegrity(manifestGeneratorHashFile, requirements io.Reader // mapping, plus the manifest structure in the manifest file. This integrity // calculation ensures the manifest files are kept up-to-date. func (f *File) calculateIntegrity( - manifestGeneratorHash, requirements io.Reader, + manifestGeneratorHash io.Reader, requirements ...io.Reader, ) ([]byte, error) { hash := sha256.New() @@ -86,8 +87,10 @@ func (f *File) calculateIntegrity( } // Sum the requirements.txt checksum bytes. - if _, err := io.Copy(hash, requirements); err != nil { - return nil, fmt.Errorf("failed to calculate integrity: %w", err) + for _, r := range(requirements) { + if _, err := io.Copy(hash, r); err != nil { + return nil, fmt.Errorf("failed to calculate integrity: %w", err) + } } return hash.Sum(nil), nil