Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit 5ca8803

Browse filesBrowse files
ohmayrohmayr
authored
fix(templates): resolve core dependencies locally and batch pip installs (#17032)
fix(templates): resolve core dependencies locally and batch pip installs Previously, the `core_deps_from_source` and `prerelease_deps` nox sessions installed core sibling packages sequentially. In a monorepo environment, this caused two major issues: 1. Correctness: It frequently fetched remote upstream code (or PyPI releases) instead of testing the actual local code modified in the active PR. 2. Performance: Sequential `pip install` commands triggered strict dependency resolution repeatedly, leading to severe CI timeouts. This updates the noxfile template to: * Dynamically resolve first-party dependencies from the local `packages/` directory to guarantee the presubmit tests the active PR's code. * Batch the `pip install` commands using `--no-deps` and `--ignore-installed` to bypass the resolver overhead and eliminate the sequential network bottleneck. * Dynamically sort packages in `prerelease_deps` into local vs. PyPI deployments using a regex parser, ensuring safe handling of complex version bounds (e.g., `grpcio>=1.75.1`) without hardcoding multiple lists.
1 parent 43dfc16 commit 5ca8803
Copy full SHA for 5ca8803

9 files changed

+523-226Lines changed: 523 additions & 226 deletions

File tree

Expand file treeCollapse file tree
Open diff view settings
Filter options
Expand file treeCollapse file tree
Open diff view settings
Collapse file

‎packages/gapic-generator/gapic/templates/noxfile.py.j2‎

Copy file name to clipboardExpand all lines: packages/gapic-generator/gapic/templates/noxfile.py.j2
+59-26Lines changed: 59 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -509,24 +509,51 @@ def prerelease_deps(session, protobuf_implementation):
509509
"proto-plus",
510510
]
511511

512-
for dep in prerel_deps:
513-
session.install("--pre", "--no-deps", "--ignore-installed", dep)
514-
# TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status``
515-
# to the dictionary below once this bug is fixed.
516-
# TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add
517-
# `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below
518-
# once this bug is fixed.
519-
package_namespaces = {
520-
"google-api-core": "google.api_core",
521-
"google-auth": "google.auth",
522-
"grpcio": "grpc",
523-
"protobuf": "google.protobuf",
524-
"proto-plus": "proto",
525-
}
526-
527-
version_namespace = package_namespaces.get(dep)
528-
512+
deps_dir = CURRENT_DIRECTORY.parent
513+
while deps_dir.name != "packages" and deps_dir.parent != deps_dir:
514+
deps_dir = deps_dir.parent
515+
516+
# Extract the base package name, safely ignoring version bounds and spaces
517+
# (e.g., "grpcio>=1.75.1" becomes "grpcio")
518+
parsed_deps = {
519+
dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1)
520+
for dep in prerel_deps
521+
}
522+
523+
# Dynamically sort local packages vs PyPI dependencies
524+
local_paths = []
525+
pypi_deps = []
526+
527+
for dep, pkg_name in parsed_deps.items():
528+
if (deps_dir / pkg_name).exists():
529+
local_paths.append(str(deps_dir / pkg_name))
530+
else:
531+
pypi_deps.append(dep)
532+
533+
# Batch pip installations to avoid sequential overhead
534+
if local_paths:
535+
session.install(*local_paths, "--no-deps", "--ignore-installed")
536+
if pypi_deps:
537+
session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed")
538+
539+
# TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status``
540+
# to the dictionary below once this bug is fixed.
541+
# TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add
542+
# `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below
543+
# once this bug is fixed.
544+
package_namespaces = {
545+
"google-api-core": "google.api_core",
546+
"google-auth": "google.auth",
547+
"grpcio": "grpc",
548+
"protobuf": "google.protobuf",
549+
"proto-plus": "proto",
550+
}
551+
552+
# Reuse the parsed names for logging and version verification
553+
for dep, pkg_name in parsed_deps.items():
529554
print(f"Installed {dep}")
555+
version_namespace = package_namespaces.get(pkg_name)
556+
530557
if version_namespace:
531558
session.run(
532559
"python",
@@ -595,17 +622,23 @@ def core_deps_from_source(session, protobuf_implementation):
595622
# added to the list below so that it is installed from source, rather than PyPI
596623
# Note: If a dependency is added to the `core_dependencies_from_source` list,
597624
# the `prerel_deps` list in the `prerelease_deps` nox session should also be updated.
598-
core_dependencies_from_source = [
599-
"googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos",
600-
"google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core",
601-
"google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth",
602-
"grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1",
603-
"proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus",
625+
core_dependencies_from_source = [
626+
"googleapis-common-protos",
627+
"google-api-core",
628+
"google-auth",
629+
"grpc-google-iam-v1",
630+
"proto-plus",
604631
]
605632

606-
for dep in core_dependencies_from_source:
607-
session.install(dep, "--no-deps", "--ignore-installed")
608-
print(f"Installed {dep}")
633+
deps_dir = CURRENT_DIRECTORY.parent
634+
while deps_dir.name != "packages" and deps_dir.parent != deps_dir:
635+
deps_dir = deps_dir.parent
636+
637+
# Batch the pip installation to avoid sequential overhead
638+
dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source]
639+
640+
session.install(*dep_paths, "--no-deps", "--ignore-installed")
641+
print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}")
609642

610643
session.run(
611644
"py.test",
Collapse file

‎packages/gapic-generator/tests/integration/goldens/asset/noxfile.py‎

Copy file name to clipboardExpand all lines: packages/gapic-generator/tests/integration/goldens/asset/noxfile.py
+58-25Lines changed: 58 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -501,24 +501,51 @@ def prerelease_deps(session, protobuf_implementation):
501501
"proto-plus",
502502
]
503503

504-
for dep in prerel_deps:
505-
session.install("--pre", "--no-deps", "--ignore-installed", dep)
506-
# TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status``
507-
# to the dictionary below once this bug is fixed.
508-
# TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add
509-
# `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below
510-
# once this bug is fixed.
511-
package_namespaces = {
512-
"google-api-core": "google.api_core",
513-
"google-auth": "google.auth",
514-
"grpcio": "grpc",
515-
"protobuf": "google.protobuf",
516-
"proto-plus": "proto",
517-
}
518-
519-
version_namespace = package_namespaces.get(dep)
520-
504+
deps_dir = CURRENT_DIRECTORY.parent
505+
while deps_dir.name != "packages" and deps_dir.parent != deps_dir:
506+
deps_dir = deps_dir.parent
507+
508+
# Extract the base package name, safely ignoring version bounds and spaces
509+
# (e.g., "grpcio>=1.75.1" becomes "grpcio")
510+
parsed_deps = {
511+
dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1)
512+
for dep in prerel_deps
513+
}
514+
515+
# Dynamically sort local packages vs PyPI dependencies
516+
local_paths = []
517+
pypi_deps = []
518+
519+
for dep, pkg_name in parsed_deps.items():
520+
if (deps_dir / pkg_name).exists():
521+
local_paths.append(str(deps_dir / pkg_name))
522+
else:
523+
pypi_deps.append(dep)
524+
525+
# Batch pip installations to avoid sequential overhead
526+
if local_paths:
527+
session.install(*local_paths, "--no-deps", "--ignore-installed")
528+
if pypi_deps:
529+
session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed")
530+
531+
# TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status``
532+
# to the dictionary below once this bug is fixed.
533+
# TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add
534+
# `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below
535+
# once this bug is fixed.
536+
package_namespaces = {
537+
"google-api-core": "google.api_core",
538+
"google-auth": "google.auth",
539+
"grpcio": "grpc",
540+
"protobuf": "google.protobuf",
541+
"proto-plus": "proto",
542+
}
543+
544+
# Reuse the parsed names for logging and version verification
545+
for dep, pkg_name in parsed_deps.items():
521546
print(f"Installed {dep}")
547+
version_namespace = package_namespaces.get(pkg_name)
548+
522549
if version_namespace:
523550
session.run(
524551
"python",
@@ -588,16 +615,22 @@ def core_deps_from_source(session, protobuf_implementation):
588615
# Note: If a dependency is added to the `core_dependencies_from_source` list,
589616
# the `prerel_deps` list in the `prerelease_deps` nox session should also be updated.
590617
core_dependencies_from_source = [
591-
"googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos",
592-
"google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core",
593-
"google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth",
594-
"grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1",
595-
"proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus",
618+
"googleapis-common-protos",
619+
"google-api-core",
620+
"google-auth",
621+
"grpc-google-iam-v1",
622+
"proto-plus",
596623
]
597624

598-
for dep in core_dependencies_from_source:
599-
session.install(dep, "--no-deps", "--ignore-installed")
600-
print(f"Installed {dep}")
625+
deps_dir = CURRENT_DIRECTORY.parent
626+
while deps_dir.name != "packages" and deps_dir.parent != deps_dir:
627+
deps_dir = deps_dir.parent
628+
629+
# Batch the pip installation to avoid sequential overhead
630+
dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source]
631+
632+
session.install(*dep_paths, "--no-deps", "--ignore-installed")
633+
print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}")
601634

602635
session.run(
603636
"py.test",
Collapse file

‎packages/gapic-generator/tests/integration/goldens/credentials/noxfile.py‎

Copy file name to clipboardExpand all lines: packages/gapic-generator/tests/integration/goldens/credentials/noxfile.py
+58-25Lines changed: 58 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -501,24 +501,51 @@ def prerelease_deps(session, protobuf_implementation):
501501
"proto-plus",
502502
]
503503

504-
for dep in prerel_deps:
505-
session.install("--pre", "--no-deps", "--ignore-installed", dep)
506-
# TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status``
507-
# to the dictionary below once this bug is fixed.
508-
# TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add
509-
# `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below
510-
# once this bug is fixed.
511-
package_namespaces = {
512-
"google-api-core": "google.api_core",
513-
"google-auth": "google.auth",
514-
"grpcio": "grpc",
515-
"protobuf": "google.protobuf",
516-
"proto-plus": "proto",
517-
}
518-
519-
version_namespace = package_namespaces.get(dep)
520-
504+
deps_dir = CURRENT_DIRECTORY.parent
505+
while deps_dir.name != "packages" and deps_dir.parent != deps_dir:
506+
deps_dir = deps_dir.parent
507+
508+
# Extract the base package name, safely ignoring version bounds and spaces
509+
# (e.g., "grpcio>=1.75.1" becomes "grpcio")
510+
parsed_deps = {
511+
dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1)
512+
for dep in prerel_deps
513+
}
514+
515+
# Dynamically sort local packages vs PyPI dependencies
516+
local_paths = []
517+
pypi_deps = []
518+
519+
for dep, pkg_name in parsed_deps.items():
520+
if (deps_dir / pkg_name).exists():
521+
local_paths.append(str(deps_dir / pkg_name))
522+
else:
523+
pypi_deps.append(dep)
524+
525+
# Batch pip installations to avoid sequential overhead
526+
if local_paths:
527+
session.install(*local_paths, "--no-deps", "--ignore-installed")
528+
if pypi_deps:
529+
session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed")
530+
531+
# TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status``
532+
# to the dictionary below once this bug is fixed.
533+
# TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add
534+
# `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below
535+
# once this bug is fixed.
536+
package_namespaces = {
537+
"google-api-core": "google.api_core",
538+
"google-auth": "google.auth",
539+
"grpcio": "grpc",
540+
"protobuf": "google.protobuf",
541+
"proto-plus": "proto",
542+
}
543+
544+
# Reuse the parsed names for logging and version verification
545+
for dep, pkg_name in parsed_deps.items():
521546
print(f"Installed {dep}")
547+
version_namespace = package_namespaces.get(pkg_name)
548+
522549
if version_namespace:
523550
session.run(
524551
"python",
@@ -588,16 +615,22 @@ def core_deps_from_source(session, protobuf_implementation):
588615
# Note: If a dependency is added to the `core_dependencies_from_source` list,
589616
# the `prerel_deps` list in the `prerelease_deps` nox session should also be updated.
590617
core_dependencies_from_source = [
591-
"googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos",
592-
"google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core",
593-
"google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth",
594-
"grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1",
595-
"proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus",
618+
"googleapis-common-protos",
619+
"google-api-core",
620+
"google-auth",
621+
"grpc-google-iam-v1",
622+
"proto-plus",
596623
]
597624

598-
for dep in core_dependencies_from_source:
599-
session.install(dep, "--no-deps", "--ignore-installed")
600-
print(f"Installed {dep}")
625+
deps_dir = CURRENT_DIRECTORY.parent
626+
while deps_dir.name != "packages" and deps_dir.parent != deps_dir:
627+
deps_dir = deps_dir.parent
628+
629+
# Batch the pip installation to avoid sequential overhead
630+
dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source]
631+
632+
session.install(*dep_paths, "--no-deps", "--ignore-installed")
633+
print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}")
601634

602635
session.run(
603636
"py.test",

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.