diff --git a/.bazelrc b/.bazelrc index 3da0a4c..333e7bb 100644 --- a/.bazelrc +++ b/.bazelrc @@ -2,6 +2,10 @@ # This allows Bazel to automatically pick up the `windows` config on Windows. common --enable_platform_specific_config +# Prevent rules_android (pulled in transitively) from attempting to configure an Android SDK repository +# using the preinstalled Android SDK on CI machines (e.g. Kokoro Windows), which fails build tools version checks. +common --repo_env=ANDROID_HOME= + # Disable automatic creation of `__init__.py` files, which prevent multiple # workspaces from providing files with the same package prefix (e.g., "cel"). # See https://github.com/bazelbuild/rules_python/issues/330. @@ -24,11 +28,19 @@ common:windows --experimental_repository_downloader_retries=10 build --verbose_failures test --test_output=errors -# GCS remote caching config (Windows-only, active by default on Windows!) -build:windows --remote_cache=https://storage.googleapis.com/windows-cel-python-remote-cache -build:windows --google_default_credentials=true +# GCS remote caching config (Linux-only, opt-in) +build:remote-cache-linux --remote_cache=https://storage.googleapis.com/linux-cel-python-remote-cache +build:remote-cache-linux --google_default_credentials=true + +# GCS remote caching config (Windows-only, opt-in) +build:remote-cache-windows --remote_cache=https://storage.googleapis.com/windows-cel-python-remote-cache +build:remote-cache-windows --google_default_credentials=true + +# GCS remote caching config (macOS-only, opt-in) +build:remote-cache-macos --remote_cache=https://storage.googleapis.com/macos-cel-python-remote-cache +build:remote-cache-macos --google_default_credentials=true -# GCS remote caching config (macOS-only, active by default on macOS!) -build:macos --remote_cache=https://storage.googleapis.com/macos-cel-python-remote-cache -build:macos --google_default_credentials=true +# Silence deprecation warnings from external dependencies (Linux and macOS) +build:linux --cxxopt=-Wno-deprecated-declarations +build:macos --cxxopt=-Wno-deprecated-declarations diff --git a/MODULE.bazel b/MODULE.bazel index 1ed9a28..089d1b1 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -15,8 +15,8 @@ bazel_dep(name = "bazel_skylib", version = "1.9.0") bazel_dep(name = "cel-cpp", version = "0.15.0", repo_name = "com_google_cel_cpp") git_override( module_name = "cel-cpp", - commit = "2e6e9ff4493bfbe0baf883107f3fb7ce6f675d88", - remote = "https://github.com/google/cel-cpp", + commit = "76ae0b3c1768d93a10270f904101de338867bdb1", + remote = "https://github.com/cel-expr/cel-cpp", ) # https://registry.bazel.build/modules/cel-spec @@ -46,13 +46,6 @@ bazel_dep(name = "rules_proto", version = "7.1.0") # https://registry.bazel.build/modules/rules_python bazel_dep(name = "rules_python", version = "1.9.0") -# On Windows the file system is case-insensitive, which creates a collision between -# antlr4-cpp-runtime/VERSION and the system `#include ` -single_version_override( - module_name = "antlr4-cpp-runtime", - patches = ["//bazel:antlr.patch"], -) - # Configure rules_python's hermetic toolchains to resolve external # dependencies natively. Under Windows, compiling pybind C++ extensions # against Python 3.11 headers requires that the execution environment diff --git a/bazel/BUILD b/bazel/BUILD deleted file mode 100644 index 18c905f..0000000 --- a/bazel/BUILD +++ /dev/null @@ -1,10 +0,0 @@ -load("@pybind11_bazel//:build_defs.bzl", "pybind_extension") -load("@rules_python//python:py_library.bzl", "py_library") -load("@rules_python//python:py_test.bzl", "py_test") - -exports_files( - srcs = [ - "antlr.patch", - ], - visibility = ["//visibility:public"], -) diff --git a/bazel/antlr.patch b/bazel/antlr.patch deleted file mode 100644 index afe9cb4..0000000 --- a/bazel/antlr.patch +++ /dev/null @@ -1,30 +0,0 @@ ---- BUILD.bazel -+++ BUILD.bazel -@@ -17,21 +17,21 @@ - cc_library( - name = "antlr4-cpp-runtime", - srcs = glob(["runtime/src/**/*.cpp"]), - hdrs = ["runtime/src/antlr4-runtime.h"], - copts = ["-fexceptions"], -- defines = ["ANTLR4CPP_USING_ABSEIL"], -+ defines = ["ANTLR4CPP_USING_ABSEIL", "ANTLR4CPP_STATIC"], - features = ["-use_header_modules"], - includes = ["runtime/src"], - textual_hdrs = glob( - ["runtime/src/**/*.h"], - exclude = ["runtime/src/antlr4-runtime.h"], - ), - visibility = ["//visibility:public"], - deps = [ - "@com_google_absl//absl/base", - "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/container:flat_hash_map", - "@com_google_absl//absl/container:flat_hash_set", - "@com_google_absl//absl/synchronization", - ], - ) - ---- VERSION -+++ /dev/null -@@ -1,1 +1,0 @@ --4.13.2 diff --git a/cel_expr_python/BUILD b/cel_expr_python/BUILD index adbee6c..190c18a 100644 --- a/cel_expr_python/BUILD +++ b/cel_expr_python/BUILD @@ -9,46 +9,42 @@ licenses(["notice"]) exports_files(["LICENSE"]) # For Python programs using CEL. -pybind_extension( - name = "cel", +pybind_library( + name = "cel_pybind_lib", srcs = [ "py_cel_activation.cc", - "py_cel_activation.h", "py_cel_arena.cc", - "py_cel_arena.h", "py_cel_env.cc", - "py_cel_env.h", "py_cel_env_config.cc", - "py_cel_env_config.h", "py_cel_env_internal.cc", - "py_cel_env_internal.h", "py_cel_expression.cc", - "py_cel_expression.h", "py_cel_function.cc", - "py_cel_function.h", "py_cel_function_decl.cc", - "py_cel_function_decl.h", - "py_cel_module.cc", "py_cel_overload.cc", - "py_cel_overload.h", "py_cel_python_extension.cc", - "py_cel_python_extension.h", "py_cel_type.cc", - "py_cel_type.h", "py_cel_value.cc", + "py_descriptor_database.cc", + "py_message_factory.cc", + ], + hdrs = [ + "py_cel_activation.h", + "py_cel_arena.h", + "py_cel_env.h", + "py_cel_env_config.h", + "py_cel_env_internal.h", + "py_cel_expression.h", + "py_cel_function.h", + "py_cel_function_decl.h", + "py_cel_overload.h", + "py_cel_python_extension.h", + "py_cel_type.h", "py_cel_value.h", "py_cel_value_provider.h", - "py_descriptor_database.cc", "py_descriptor_database.h", - "py_error_status.cc", - "py_error_status.h", - "py_message_factory.cc", "py_message_factory.h", ], - data = ["cel.pyi"], - visibility = [ - "//visibility:public", - ], + visibility = [":__subpackages__"], deps = [ ":cel_extension", ":status_macros", @@ -74,6 +70,7 @@ pybind_extension( "@com_google_cel_cpp//common:function_descriptor", "@com_google_cel_cpp//common:kind", "@com_google_cel_cpp//common:minimal_descriptor_pool", + "@com_google_cel_cpp//common:signature", "@com_google_cel_cpp//common:source", "@com_google_cel_cpp//common:type", "@com_google_cel_cpp//common:type_kind", @@ -103,6 +100,20 @@ pybind_extension( ], ) +pybind_extension( + name = "cel", + srcs = [ + "py_cel_module.cc", + ], + data = ["cel.pyi"], + visibility = [ + "//visibility:public", + ], + deps = [ + ":cel_pybind_lib", + ], +) + # For pybind11-based CEL extensions. pybind_library( name = "cel_extension", @@ -177,3 +188,15 @@ py_test( ], }), ) + +# Platform definition for macOS x86_64. +# Used for cross-compiling from arm64 macOS hosts (Apple Silicon) to x86_64. +# This is passed to Bazel via --platforms in setup.py when building x86_64 wheels. +platform( + name = "macos_x86_64", + constraint_values = [ + "@platforms//os:osx", + "@platforms//cpu:x86_64", + ], + visibility = ["//visibility:public"], +) diff --git a/cel_expr_python/cel.pyi b/cel_expr_python/cel.pyi index 549e7bd..f5cc7c8 100644 --- a/cel_expr_python/cel.pyi +++ b/cel_expr_python/cel.pyi @@ -13,6 +13,8 @@ class CelExtensionBase: def __init__(self, name: str) -> None: ... class EnvConfig: + @property + def context_type(self) -> str: ... def to_yaml(self) -> str: ... class ExpressionContainer: @@ -37,7 +39,7 @@ class FunctionDecl: def __init__(self, name: str, overloads: Sequence[Overload]) -> None: ... class Overload: - def __init__(self, overload_id: str, return_type: Type = ..., parameters: Sequence[Type] = ..., is_member: bool = ..., impl: Callable[..., Any] = ...) -> None: ... + def __init__(self, id: str | None = ..., return_type: Type = ..., parameters: Sequence[Type] = ..., is_member: bool = ..., impl: Callable[..., Any] = ..., signature: str | None = ...) -> None: ... class Type: BOOL: ClassVar[Type] = ... @@ -55,7 +57,7 @@ class Type: TYPE: ClassVar[Type] = ... UINT: ClassVar[Type] = ... UNKNOWN: ClassVar[Type] = ... - def __init__(self, name: str) -> None: ... + def __init__(self, signature: str) -> None: ... @staticmethod def AbstractType(name: str, params: Sequence[Type] = ...) -> Type: ... @staticmethod diff --git a/cel_expr_python/cel_env_test.py b/cel_expr_python/cel_env_test.py index 92a64ff..eacfc9f 100644 --- a/cel_expr_python/cel_env_test.py +++ b/cel_expr_python/cel_env_test.py @@ -46,17 +46,13 @@ def test_env_config_from_and_to_yaml(self): - name: math variables: - name: one - type_name: int + type: int value: 1 functions: - name: add overloads: - - id: "add_int_int" - args: - - type_name: int - - type_name: int - return: - type_name: int + - signature: "add(int,int)" + return: int """) yaml: str = config.to_yaml() self.assertEqual( @@ -74,17 +70,13 @@ def test_env_config_from_and_to_yaml(self): - name: "_+_" variables: - name: "one" - type_name: "int" + type: "int" value: 1 functions: - name: "add" overloads: - - id: "add_int_int" - args: - - type_name: "int" - - type_name: "int" - return: - type_name: "int" + - signature: "add(int,int)" + return: "int" """), ) @@ -98,6 +90,52 @@ def test_invalid_yaml(self): str(e.exception), ) + def test_parse_context_variable_config(self): + config = cel.NewEnvConfigFromYaml(""" + context_variable: + type_name: "cel.expr.conformance.proto2.TestAllTypes" + """) + self.assertEqual( + config.context_type, "cel.expr.conformance.proto2.TestAllTypes" + ) + + def test_parse_context_variable_config_alternative_syntax(self): + config = cel.NewEnvConfigFromYaml(""" + context_variable: + type: "cel.expr.conformance.proto2.TestAllTypes" + """) + self.assertEqual( + config.context_type, "cel.expr.conformance.proto2.TestAllTypes" + ) + + def test_parse_context_variable_malformed(self): + with self.assertRaisesRegex( + Exception, "Node 'context_variable' is not a map" + ): + cel.NewEnvConfigFromYaml("context_variable: 123") + + def test_parse_context_variable_malformed2(self): + with self.assertRaisesRegex( + Exception, "Node 'context_variable' does not have a valid type" + ): + cel.NewEnvConfigFromYaml(""" + context_variable: + type: + foo: bar + """) + + def test_context_variable_basic(self): + config = cel.NewEnvConfigFromYaml(""" + context_variable: + type_name: "cel.expr.conformance.proto2.TestAllTypes" + """) + env = cel.NewEnv(config=config) + ast = env.compile("single_int32 > 10") + self.assertIsNotNone(ast) + + with self.assertRaises(Exception): + env.compile("non_existent_field > 10") + def test_config_export_container(self): env: cel.Env = cel.NewEnv(container="test.container") yaml: str = env.config().to_yaml() @@ -139,9 +177,9 @@ def test_expression_container_abbreviations_and_aliases(self): qualified_name: "x.y.bar" variables: - name: "x.y.bar" - type_name: "string" + type: "string" - name: "x.y.foo" - type_name: "int" + type: "int" """), ) @@ -156,9 +194,9 @@ def test_abbreviations_and_aliases_from_yaml(self): qualified_name: "x.y.bar" variables: - name: "x.y.bar" - type_name: "string" + type: "string" - name: "x.y.foo" - type_name: "int" + type: "int" """)) res = env.compile("foo").eval(data={"x.y.foo": 42}) @@ -178,13 +216,13 @@ def test_abbreviations_and_aliases_combined(self): qualified_name: "x.y.bar" variables: - name: "x.y.bar" - type_name: "string" + type: "string" - name: "x.y.foo" - type_name: "int" + type: "int" - name: "a.b.qux" - type_name: "string" + type: "string" - name: "a.b.baz" - type_name: "int" + type: "int" """), container=cel.ExpressionContainer( "test.container", @@ -219,13 +257,13 @@ def test_abbreviations_and_aliases_combined(self): qualified_name: "a.b.qux" variables: - name: "a.b.baz" - type_name: "int" + type: "int" - name: "a.b.qux" - type_name: "string" + type: "string" - name: "x.y.bar" - type_name: "string" + type: "string" - name: "x.y.foo" - type_name: "int" + type: "int" """), ) @@ -275,48 +313,35 @@ def test_config_export_variables(self): normalize_yaml(""" variables: - name: "var_bool" - type_name: "bool" + type: "bool" - name: "var_bytes" - type_name: "bytes" + type: "bytes" - name: "var_double" - type_name: "double" + type: "double" - name: "var_duration" - type_name: "duration" + type: "duration" - name: "var_dyn" - type_name: "dyn" + type: "dyn" - name: "var_dyn_list" - type_name: "list" - params: - - type_name: "dyn" + type: "list" - name: "var_dyn_map" - type_name: "map" - params: - - type_name: "dyn" - - type_name: "dyn" + type: "map" - name: "var_int" - type_name: "int" + type: "int" - name: "var_int_map" - type_name: "map" - params: - - type_name: "int" - - type_name: "string" + type: "map" - name: "var_msg" - type_name: "cel.expr.conformance.proto2.TestAllTypes" + type: "cel.expr.conformance.proto2.TestAllTypes" - name: "var_str" - type_name: "string" + type: "string" - name: "var_string_list" - type_name: "list" - params: - - type_name: "string" + type: "list" - name: "var_string_map" - type_name: "map" - params: - - type_name: "string" - - type_name: "bool" + type: "map" - name: "var_timestamp" - type_name: "timestamp" + type: "timestamp" - name: "var_uint" - type_name: "uint" + type: "uint" """), ) @@ -324,7 +349,7 @@ def test_config_augmented_variables(self): config = cel.NewEnvConfigFromYaml(""" variables: - name: "var_bool" - type_name: "bool" + type: "bool" """) env: cel.Env = cel.NewEnv( config=config, @@ -338,9 +363,9 @@ def test_config_augmented_variables(self): normalize_yaml(""" variables: - name: "var_bool" - type_name: "bool" + type: "bool" - name: "var_msg" - type_name: "cel.expr.conformance.proto2.TestAllTypes" + type: "cel.expr.conformance.proto2.TestAllTypes" """), ) @@ -348,7 +373,7 @@ def test_config_variable_override(self): config: cel.EnvConfig = cel.NewEnvConfigFromYaml(""" variables: - name: "var_bool" - type_name: "bool" + type: "bool" """) with self.assertRaises(Exception) as e: @@ -367,9 +392,9 @@ def test_config_variable_types(self): config: cel.EnvConfig = cel.NewEnvConfigFromYaml(""" variables: - name: "var_bool" - type_name: "bool" + type: "bool" - name: "var_int" - type_name: "int" + type: "int" value: 42 """) env: cel.Env = cel.NewEnv( @@ -514,11 +539,8 @@ def test_config_functions(self): functions: - name: is_ok overloads: - - id: "is_ok_string" - target: - type_name: string - return: - type_name: bool + - signature: "string.is_ok()" + return: "bool" """) env: cel.Env = cel.NewEnv( config=config, @@ -527,12 +549,8 @@ def test_config_functions(self): "hello", [ cel.Overload( - "good_time_of_day", + signature="hello(string,string)", return_type=cel.Type.STRING, - parameters=[ - cel.Type.STRING, - cel.Type.STRING, - ], impl=lambda ampm, arg: ( "Good" f" {'morning' if ampm == 'am' else 'afternoon'}," @@ -543,7 +561,7 @@ def test_config_functions(self): ) ], function_impls={ - "is_ok_string": lambda arg: arg in ["excellent", "good", "fair"], + "string.is_ok()": lambda arg: arg in ["excellent", "good", "fair"], }, ) yaml = env.config().to_yaml() @@ -553,19 +571,12 @@ def test_config_functions(self): functions: - name: "hello" overloads: - - id: "good_time_of_day" - args: - - type_name: "string" - - type_name: "string" - return: - type_name: "string" + - signature: "hello(string,string)" + return: "string" - name: "is_ok" overloads: - - id: "is_ok_string" - target: - type_name: "string" - return: - type_name: "bool" + - signature: "string.is_ok()" + return: "bool" """), ) res: cel.Value = env.compile("hello('am', 'Sunshine')").eval() @@ -582,32 +593,109 @@ def test_config_function_override(self): functions: - name: foo overloads: - - id: "unique_id" + - signature: "foo()" """) with self.assertRaises(Exception) as e: cel.NewEnv( config=config, functions=[ cel.FunctionDecl( - "bar", + "foo", [ cel.Overload( - "unique_id", + signature="foo()", impl=lambda: "hello", ) ], ) ], function_impls={ - "unique_id": lambda: "goodbye", + "foo()": lambda: "goodbye", }, ) self.assertIn( - "An implementation for function overload id 'unique_id' already" - " exists.", + "An implementation for function overload 'foo()' already exists.", str(e.exception), ) + def test_overload_signature_errors(self): + with self.assertRaises(ValueError) as e2: + cel.Overload(signature="greet(string)", parameters=[cel.Type.STRING]) + self.assertIn( + "If 'signature' is specified, 'parameters' should not be specified", + str(e2.exception), + ) + + with self.assertRaises(ValueError) as e3: + cel.Overload() + self.assertIn( + "Either 'id' or 'signature' must be specified", str(e3.exception) + ) + + def test_config_functions_deprecated_syntax(self): + """Test that the deprecated function syntax is still supported.""" + config: cel.EnvConfig = cel.NewEnvConfigFromYaml(""" + functions: + - name: is_ok + overloads: + - id: "is_ok_string" + target: + type_name: string + return: + type_name: bool + """) + env: cel.Env = cel.NewEnv( + config=config, + functions=[ + cel.FunctionDecl( + "hello", + [ + cel.Overload( + "good_time_of_day", + return_type=cel.Type.STRING, + parameters=[ + cel.Type.STRING, + cel.Type.STRING, + ], + impl=lambda ampm, arg: ( + "Good" + f" {'morning' if ampm == 'am' else 'afternoon'}," + f" {arg}!" + ), + ) + ], + ) + ], + function_impls={ + "is_ok_string": lambda arg: arg in ["excellent", "good", "fair"], + }, + ) + yaml = env.config().to_yaml() + self.assertEqual( + normalize_yaml(yaml), + normalize_yaml(""" + functions: + - name: "hello" + overloads: + - id: "good_time_of_day" + signature: "hello(string,string)" + return: "string" + - name: "is_ok" + overloads: + - id: "is_ok_string" + signature: "string.is_ok()" + return: "bool" + """), + ) + res: cel.Value = env.compile("hello('am', 'Sunshine')").eval() + self.assertEqual(res.value(), "Good morning, Sunshine!") + res = env.compile("hello('pm', 'tea is served')").eval() + self.assertEqual(res.value(), "Good afternoon, tea is served!") + res = env.compile("'good'.is_ok()").eval() + self.assertTrue(res.value()) + res = env.compile("'bad'.is_ok()").eval() + self.assertFalse(res.value()) + class TestCelExtension(cel.CelExtension): """An example CEL extension for testing.""" diff --git a/cel_expr_python/cel_test.py b/cel_expr_python/cel_test.py index f812fe4..938db60 100644 --- a/cel_expr_python/cel_test.py +++ b/cel_expr_python/cel_test.py @@ -668,6 +668,18 @@ def testTypeType(self): res.value(), cel.Type.INT ) # This behavior is counterintuitive but works as implemented. + def testTypeInitSignature(self): + self.assertEqual(cel.Type("int"), cel.Type.INT) + self.assertEqual(cel.Type("list"), cel.Type.List(cel.Type.INT)) + self.assertEqual( + cel.Type("map"), + cel.Type.Map(cel.Type.STRING, cel.Type.DYN), + ) + self.assertEqual( + cel.Type("cel.expr.conformance.proto2.TestAllTypes"), + cel.Type("cel.expr.conformance.proto2.TestAllTypes"), + ) + def testCelExpressionPersistence_checkedExpr(self): expr: cel.Expression = self.env.compile("var_msg.single_string") as_bytes: bytes = expr.serialize() diff --git a/cel_expr_python/py_cel_env_config.cc b/cel_expr_python/py_cel_env_config.cc index 1f4014f..8ea330c 100644 --- a/cel_expr_python/py_cel_env_config.cc +++ b/cel_expr_python/py_cel_env_config.cc @@ -33,9 +33,11 @@ void PyCelEnvConfig::DefinePythonBindings(pybind11::module& m) { m.def("NewEnvConfigFromYaml", &PyCelEnvConfig::FromYaml, py::arg("yaml")); cel_class.def("to_yaml", &PyCelEnvConfig::ToYaml); + cel_class.def_property_readonly("context_type", + &PyCelEnvConfig::GetContextType); } -PyCelEnvConfig PyCelEnvConfig::FromYaml(std::string yaml) { +PyCelEnvConfig PyCelEnvConfig::FromYaml(const std::string& yaml) { PyCelEnvConfig config; config.config_ = ThrowIfError(cel::EnvConfigFromYaml(yaml)); return config; @@ -43,7 +45,7 @@ PyCelEnvConfig PyCelEnvConfig::FromYaml(std::string yaml) { std::string PyCelEnvConfig::ToYaml() const { std::stringstream ss; - cel::EnvConfigToYaml(config_, ss); + cel::EnvConfigToYaml(config_, ss, {.use_type_signatures = true}); return ss.str(); } diff --git a/cel_expr_python/py_cel_env_config.h b/cel_expr_python/py_cel_env_config.h index 1742413..ed30ea2 100644 --- a/cel_expr_python/py_cel_env_config.h +++ b/cel_expr_python/py_cel_env_config.h @@ -30,10 +30,11 @@ class PyCelEnvConfig { PyCelEnvConfig() = default; explicit PyCelEnvConfig(const cel::Config& config) : config_(config) {} - static PyCelEnvConfig FromYaml(std::string yaml); + static PyCelEnvConfig FromYaml(const std::string& yaml); std::string ToYaml() const; const cel::Config& GetConfig() const { return config_; } + std::string GetContextType() const { return config_.GetContextType(); } private: cel::Config config_; diff --git a/cel_expr_python/py_cel_env_internal.cc b/cel_expr_python/py_cel_env_internal.cc index a591580..af0391b 100644 --- a/cel_expr_python/py_cel_env_internal.cc +++ b/cel_expr_python/py_cel_env_internal.cc @@ -202,11 +202,11 @@ PyCelEnvInternal::NewCelEnvInternal( if (overload.py_function().is_none()) { continue; } - if (!impls.insert({overload.overload_id(), overload.py_function()}) - .second) { + std::string overload_id = overload.overload_id(); + if (!impls.insert({overload_id, overload.py_function()}).second) { return absl::AlreadyExistsError( - absl::StrCat("An implementation for function overload id '", - overload.overload_id(), "' already exists.")); + absl::StrCat("An implementation for function overload '", + overload_id, "' already exists.")); } } } @@ -214,8 +214,8 @@ PyCelEnvInternal::NewCelEnvInternal( for (const auto& [overload_id, py_function] : function_impls) { if (!impls.insert({overload_id, py_function}).second) { return absl::AlreadyExistsError( - absl::StrCat("An implementation for function overload id '", - overload_id, "' already exists.")); + absl::StrCat("An implementation for function overload '", overload_id, + "' already exists.")); } } return std::shared_ptr( diff --git a/cel_expr_python/py_cel_function_decl.cc b/cel_expr_python/py_cel_function_decl.cc index b5f084b..382fdae 100644 --- a/cel_expr_python/py_cel_function_decl.cc +++ b/cel_expr_python/py_cel_function_decl.cc @@ -16,10 +16,13 @@ #include #include +#include #include #include "env/config.h" +#include "env/type_info.h" #include "cel_expr_python/py_cel_overload.h" +#include "cel_expr_python/py_cel_type.h" #include #include diff --git a/cel_expr_python/py_cel_overload.cc b/cel_expr_python/py_cel_overload.cc index 5311f46..321af6c 100644 --- a/cel_expr_python/py_cel_overload.cc +++ b/cel_expr_python/py_cel_overload.cc @@ -20,8 +20,11 @@ #include #include +#include "common/signature.h" #include "env/config.h" +#include "env/type_info.h" #include "cel_expr_python/py_cel_type.h" +#include "cel_expr_python/py_error_status.h" #include #include @@ -31,16 +34,47 @@ namespace py = ::pybind11; void PyCelOverload::DefinePythonBindings(py::module_& m) { py::class_>(m, "Overload") - .def(py::init([](const std::string& overload_id, + .def(py::init([](std::optional id, const PyCelType& return_type, const std::vector& parameters, bool is_member, - py::object impl) { - return PyCelOverload(overload_id, return_type, parameters, - is_member, std::move(impl)); + py::object impl, std::optional signature) { + if (signature.has_value()) { + if (!parameters.empty()) { + throw py::value_error( + "If 'signature' is specified, 'parameters' " + "should not be specified"); + } + cel::ParsedFunctionOverload parsed = + ThrowIfError(cel::ParseFunctionSignature(*signature)); + std::string overload_id = + id.has_value() ? std::move(*id) : *signature; + std::vector parsed_parameters; + if (parsed.signature_type.has_function()) { + const auto& function_type_spec = + parsed.signature_type.function(); + parsed_parameters.reserve( + function_type_spec.arg_types().size()); + for (const auto& arg : function_type_spec.arg_types()) { + parsed_parameters.push_back(PyCelType::FromTypeInfo( + ThrowIfError(cel::TypeSpecToTypeInfo(arg)))); + } + } + return PyCelOverload(std::move(overload_id), return_type, + std::move(parsed_parameters), + parsed.is_member, std::move(impl)); + } + if (id.has_value()) { + return PyCelOverload(std::move(*id), return_type, parameters, + is_member, std::move(impl)); + } + throw py::value_error( + "Either 'id' or 'signature' must be specified"); }), - py::arg("overload_id"), py::arg("return_type") = PyCelType::Dyn(), + py::arg("id") = std::nullopt, + py::arg("return_type") = PyCelType::Dyn(), py::arg("parameters") = std::vector{}, - py::arg("is_member") = false, py::arg("impl") = py::none()); + py::arg("is_member") = false, py::arg("impl") = py::none(), + py::arg("signature") = std::nullopt); } PyCelOverload::PyCelOverload(std::string overload_id, diff --git a/cel_expr_python/py_cel_type.cc b/cel_expr_python/py_cel_type.cc index 0cf7b54..be49498 100644 --- a/cel_expr_python/py_cel_type.cc +++ b/cel_expr_python/py_cel_type.cc @@ -30,6 +30,7 @@ #include "absl/strings/str_format.h" #include "absl/strings/str_join.h" #include "common/kind.h" +#include "common/signature.h" #include "common/type.h" #include "common/type_kind.h" #include "common/types/list_type.h" @@ -37,6 +38,7 @@ #include "common/value.h" #include "common/value_kind.h" #include "env/config.h" +#include "env/type_info.h" #include "cel_expr_python/py_error_status.h" #include "cel_expr_python/status_macros.h" #include "google/protobuf/arena.h" @@ -50,7 +52,14 @@ namespace py = ::pybind11; void PyCelType::DefinePythonBindings(py::module& m) { py::class_ type(m, "Type"); - type.def(py::init(), py::arg("name")) + type.def(py::init([](const std::string& signature) { + cel::TypeSpec type_spec = + ThrowIfError(cel::ParseTypeSpec(signature)); + cel::Config::TypeInfo type_info = + ThrowIfError(cel::TypeSpecToTypeInfo(type_spec)); + return PyCelType::FromTypeInfo(type_info); + }), + py::arg("signature")) .def("name", &PyCelType::GetName) .def("is_message", &PyCelType::IsMessage) .def("is_assignable_from", &PyCelType::IsAssignableFrom) @@ -526,6 +535,47 @@ PyCelType PyCelType::FromTypeProto(const cel::expr::Type& type) { return PyCelType::Error(); } +PyCelType PyCelType::FromTypeInfo(const cel::Config::TypeInfo& type_info) { + if (type_info.name == "null") return PyCelType::Null(); + if (type_info.name == "bool") return PyCelType::Bool(); + if (type_info.name == "int") return PyCelType::Int(); + if (type_info.name == "uint") return PyCelType::Uint(); + if (type_info.name == "double") return PyCelType::Double(); + if (type_info.name == "string") return PyCelType::String(); + if (type_info.name == "bytes") return PyCelType::Bytes(); + if (type_info.name == "timestamp") return PyCelType::Timestamp(); + if (type_info.name == "duration") return PyCelType::Duration(); + if (type_info.name == "dyn") return PyCelType::Dyn(); + if (type_info.name == "list") { + if (type_info.params.empty()) { + return PyCelType::List(); + } + return PyCelType::ListType(FromTypeInfo(type_info.params[0])); + } + if (type_info.name == "map") { + if (type_info.params.size() < 2) { + return PyCelType::Map(); + } + return PyCelType::MapType(FromTypeInfo(type_info.params[0]), + FromTypeInfo(type_info.params[1])); + } + if (type_info.name == "type") { + if (type_info.params.empty()) { + return PyCelType::Type(); + } + return PyCelType::TypeType(FromTypeInfo(type_info.params[0])); + } + if (type_info.is_type_param || !type_info.params.empty()) { + std::vector params; + params.reserve(type_info.params.size()); + for (const auto& param : type_info.params) { + params.push_back(FromTypeInfo(param)); + } + return PyCelType::AbstractType(type_info.name, params); + } + return PyCelType(type_info.name); +} + absl::StatusOr PyCelType::ToCelType( const PyCelType& type, google::protobuf::Arena* arena, const google::protobuf::DescriptorPool& descriptor_pool) { diff --git a/cel_expr_python/py_cel_type.h b/cel_expr_python/py_cel_type.h index 892c488..19cac0e 100644 --- a/cel_expr_python/py_cel_type.h +++ b/cel_expr_python/py_cel_type.h @@ -77,6 +77,7 @@ class PyCelType { static PyCelType ForCelValue(const cel::Value& cel_value); static PyCelType FromCelType(const cel::Type& cel_type); static PyCelType FromTypeProto(const cel::expr::Type& type); + static PyCelType FromTypeInfo(const cel::Config::TypeInfo& type_info); static absl::StatusOr ToCelType( const PyCelType& type, google::protobuf::Arena* arena, const google::protobuf::DescriptorPool& descriptor_pool); diff --git a/cel_expr_python/py_descriptor_database.h b/cel_expr_python/py_descriptor_database.h index 610f8b2..5f621f4 100644 --- a/cel_expr_python/py_descriptor_database.h +++ b/cel_expr_python/py_descriptor_database.h @@ -19,6 +19,8 @@ #include // IWYU pragma: keep - Needed for PyObject +#include // IWYU pragma: keep - Needed for string_view in OSS + #include "google/protobuf/descriptor.pb.h" #include "google/protobuf/descriptor_database.h" @@ -27,7 +29,7 @@ namespace cel_python { // A DescriptorDatabase that uses a Python DescriptorPool to find descriptors. class PyDescriptorDatabase : public google::protobuf::DescriptorDatabase { private: - using StringViewArg = const std::string&; + using StringViewArg = std::string_view; public: explicit PyDescriptorDatabase(PyObject* py_descriptor_pool); ~PyDescriptorDatabase() override; diff --git a/codelab/index.lab.md b/codelab/index.lab.md index eb5ef2f..5ca5fba 100644 --- a/codelab/index.lab.md +++ b/codelab/index.lab.md @@ -846,14 +846,8 @@ def exercise5(): "contains", [ cel.Overload( - "contains_key_value", + signature="map.contains(string, dyn)", return_type=cel.Type.BOOL, - parameters=[ - cel.Type.Map(cel.Type.STRING, cel.Type.DYN), - cel.Type.STRING, - cel.Type.DYN, - ], - is_member=True, # Provide the implementation as a Python function ) ], @@ -922,14 +916,8 @@ def exercise5(): "contains", [ cel.Overload( - "contains_string_any", + signature="map.contains(string, dyn)", return_type=cel.Type.BOOL, - parameters=[ - cel.Type.Map(cel.Type.STRING, cel.Type.DYN), - cel.Type.STRING, - cel.Type.DYN, - ], - is_member=True, # Reference a Python function impl=contains_key_value, ) @@ -1319,7 +1307,7 @@ def exercise7(): # Add variable definitions for 'jwt' as a map(string, Dyn) type # and for 'now' as a timestamp. variables={ - "jwt": cel.Type.Map(cel.Type.STRING, cel.Type.DYN), + "jwt": cel.Type("map"), "now": cel.Type.TIMESTAMP, }, ) diff --git a/codelab/solution/codelab.py b/codelab/solution/codelab.py index 15d12c0..861f0ef 100644 --- a/codelab/solution/codelab.py +++ b/codelab/solution/codelab.py @@ -188,14 +188,11 @@ def exercise5(): "containsKeyValue", [ cel.Overload( - "contains_key_value", + signature=( + "map.containsKeyValue(string," + " dyn)" + ), return_type=cel.Type.BOOL, - parameters=[ - cel.Type.Map(cel.Type.STRING, cel.Type.DYN), - cel.Type.STRING, - cel.Type.DYN, - ], - is_member=True, impl=contains_key_value, ) ], @@ -323,7 +320,7 @@ def exercise7(): # Add variable definitions for 'jwt' as a map(string, Dyn) type # and for 'now' as a timestamp. variables={ - "jwt": cel.Type.Map(cel.Type.STRING, cel.Type.DYN), + "jwt": cel.Type("map"), "now": cel.Type.TIMESTAMP, }, ) diff --git a/release/kokoro/presubmit_windows.bat b/release/kokoro/presubmit_windows.bat index 30136f1..5e9e8ea 100644 --- a/release/kokoro/presubmit_windows.bat +++ b/release/kokoro/presubmit_windows.bat @@ -56,7 +56,7 @@ for %%V in (%PYTHON_VERSIONS%) do ( :fetch_loop set /a ATTEMPTS+=1 echo Fetch attempt !ATTEMPTS! of !FETCH_RETRIES!... - bazel %STARTUP_FLAGS% fetch //... > fetch.log 2>&1 + bazel !STARTUP_FLAGS! fetch //... > fetch.log 2>&1 set FETCH_STATUS=!ERRORLEVEL! type fetch.log if !FETCH_STATUS! NEQ 0 ( @@ -78,7 +78,7 @@ for %%V in (%PYTHON_VERSIONS%) do ( if exist fetch.log del fetch.log echo --- Getting Output Base --- - for /f "tokens=*" %%i in ('bazel %STARTUP_FLAGS% info output_base') do set "OUTPUT_BASE=%%i" + for /f "tokens=*" %%i in ('bazel !STARTUP_FLAGS! info output_base') do set "OUTPUT_BASE=%%i" set "OUTPUT_BASE=!OUTPUT_BASE:/=\!" echo Output Base: !OUTPUT_BASE! @@ -99,7 +99,7 @@ for %%V in (%PYTHON_VERSIONS%) do ( ) echo --- Bazel Build --- - bazel %STARTUP_FLAGS% build %LINK_FLAGS% //... + bazel !STARTUP_FLAGS! build !LINK_FLAGS! //... if !ERRORLEVEL! NEQ 0 ( echo Build failed! set "PRESUBMIT_STATUS=1" @@ -107,7 +107,7 @@ for %%V in (%PYTHON_VERSIONS%) do ( ) echo --- Bazel Test Python %%V --- - bazel %STARTUP_FLAGS% test %LINK_FLAGS% --test_output=errors //... + bazel !STARTUP_FLAGS! test !LINK_FLAGS! --test_output=errors //... if !ERRORLEVEL! NEQ 0 ( echo Tests failed for Python %%V! set "PRESUBMIT_STATUS=1" diff --git a/release/kokoro/release_linux.cfg b/release/kokoro/release_linux.cfg index 6a91c7b..018c022 100644 --- a/release/kokoro/release_linux.cfg +++ b/release/kokoro/release_linux.cfg @@ -3,3 +3,8 @@ build_file: "cel-python/release/kokoro/release_linux.sh" timeout_mins: 120 + +container_properties { + docker_image: "mirror.gcr.io/library/ubuntu:24.04" + docker_sibling_containers: true +} diff --git a/release/kokoro/release_linux.sh b/release/kokoro/release_linux.sh index 3d92d51..6375ed2 100755 --- a/release/kokoro/release_linux.sh +++ b/release/kokoro/release_linux.sh @@ -1,6 +1,59 @@ #!/bin/bash +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + set -e +if ! command -v pip3 &> /dev/null || ! command -v curl &> /dev/null || ! command -v docker &> /dev/null || ! command -v git &> /dev/null; then + echo "Installing basic dependencies..." + apt-get update && apt-get install -y python3-pip curl git + + if ! command -v docker &> /dev/null; then + echo "Installing docker CLI..." + ARCH=$(uname -m) + if [ "$ARCH" = "x86_64" ]; then + DOCKER_ARCH="x86_64" + elif [ "$ARCH" = "aarch64" ]; then + DOCKER_ARCH="aarch64" + else + echo "Unsupported arch: $ARCH" + exit 1 + fi + curl -fsSL "https://download.docker.com/linux/static/stable/${DOCKER_ARCH}/docker-24.0.7.tgz" -o docker.tgz + tar xzvf docker.tgz --strip-components=1 docker/docker + mv docker /usr/local/bin/ + rm -f docker.tgz + fi +fi + +# Avoid virtualenv/pip trying to download/upgrade tools from PyPI on host +export VIRTUALENV_NO_DOWNLOAD=1 +export PIP_DISABLE_PIP_VERSION_CHECK=1 +export PIP_BREAK_SYSTEM_PACKAGES=1 +export PIP_DEFAULT_TIMEOUT=60 + +if [ "$(uname -m)" = "aarch64" ]; then + export CIBW_ARCHS="aarch64" +else + export CIBW_ARCHS="x86_64" +fi + +# Pass these environment variables to the cibuildwheel Docker container +export CIBW_ENVIRONMENT="VIRTUALENV_NO_DOWNLOAD=1 PIP_DISABLE_PIP_VERSION_CHECK=1 PIP_DEFAULT_TIMEOUT=120 CEL_BAZEL_FLAGS=--config=remote-cache-linux" +export CIBW_DEPENDENCY_VERSIONS="latest" +export CIBW_CONTAINER_ENGINE_EXTRA_ARGS="--network=host" + # If running locally (not on Kokoro), authenticate with gcloud. if [ -z "${KOKORO_BUILD_ID}" ]; then if ! gcloud auth application-default print-access-token --quiet > /dev/null; then @@ -8,14 +61,140 @@ if [ -z "${KOKORO_BUILD_ID}" ]; then fi fi -pip install -U keyring keyrings.google-artifactregistry-auth twine cibuildwheel +# We use --no-cache-dir to force pip to download packages fresh and bypass the local +# cache. In a sandboxed build environment, writing to the default cache directory +# (~/.cache/pip) can encounter permission/sandbox restrictions or lead to stale +# dependency resolution. Disabling the cache ensures a reliable, reproducible install. +pip install --no-cache-dir -U keyring keyrings.google-artifactregistry-auth twine +curl -fsSL https://github.com/pypa/cibuildwheel/archive/refs/tags/v4.1.0.tar.gz -o cibuildwheel-4.1.0.tar.gz +pip install --no-cache-dir cibuildwheel-4.1.0.tar.gz +rm -f cibuildwheel-4.1.0.tar.gz + +# ============================================================================== +# FUTURE-PROOF RUNTIME PATCHING OF CIBUILDWHEEL +# ============================================================================== +# To run cibuildwheel on Google's sandboxed RBE/Kokoro infrastructure, we must: +# 1. Bypass RBE's stdout proxy buffering deadlock (requires 32KB padding). +# 2. Bypass RBE's stdin EOF deadlock during copy-in (requires 'docker cp' +# since we use disable_host_mount: True in pyproject.toml). +# +# Since cibuildwheel is installed fresh from PyPI on every build (ensuring we get +# the latest security and feature updates), we apply these patches at runtime. +# +# Why this patching strategy is future-proof and safe: +# - Strict Validation: The Python patcher strictly validates that all target +# code blocks exist before applying replacements. If cibuildwheel's internal +# code changes in a future release, the patcher will FAIL LOUDLY and exit the +# build immediately (sys.exit(1)) rather than silently running a broken, +# hanging build. +# - Stable Boundaries: The copy_into patch uses a robust regular expression +# anchored to class method boundaries (def copy_into -> def copy_out). These +# are stable, long-standing internal APIs of cibuildwheel's OCIContainer. +# - Core Protocol Stability: The buffering patches target the core protocol +# used to communicate with the container's persistent bash shell. This +# protocol is fundamental to cibuildwheel and highly unlikely to change. +# ============================================================================== +OCI_PATH=$(python3 -c "import cibuildwheel.oci_container; print(cibuildwheel.oci_container.__file__)") +echo "Patching cibuildwheel at $OCI_PATH..." + +cat << 'EOF' > patch_oci.py +import sys +import re + +path = sys.argv[1] +with open(path, 'r') as f: + content = f.read() + +# 1. Force a 32KB flush at the end of every command execution +target_write = 'printf "%04d%s\\n" $? {end_of_message}' +replacement_write = 'printf "%04d%s\\n%32768s\\n" $? {end_of_message} " "' +if target_write in content: + content = content.replace(target_write, replacement_write) + print("Patched write loop.") +else: + print("ERROR: Could not find write loop target in oci_container.py! The cibuildwheel version might have changed.") + sys.exit(1) + +# 2. Read and discard the 32KB padding to keep the stream clean +target_read = """ # add the last line to output, without the footer + output_io.write(line[0:footer_offset]) + output_io.flush() + break""" + +replacement_read = """ # add the last line to output, without the footer + output_io.write(line[0:footer_offset]) + output_io.flush() + # Read and discard the 32KB padding line to clear the stream! + self.bash_stdout.readline() + break""" + +if target_read in content: + content = content.replace(target_read, replacement_read) + print("Patched read loop.") +else: + print("ERROR: Could not find read loop target in oci_container.py! The cibuildwheel version might have changed.") + sys.exit(1) + +# 3. Patch the entire copy_into method using a unique regex to use native 'docker cp'. +# This bypasses the RBE stdin EOF deadlock when copying the project into the container. +pattern = re.compile(r' def copy_into\(self,.*?\).*?:.*? def copy_out', re.DOTALL) + +replacement_copy = """ def copy_into(self, from_path: Path, to_path: PurePath) -> None: + if from_path.is_dir(): + self.call(["mkdir", "-p", to_path]) + subprocess.run( + f"tar -c {self.host_tar_format} -f - . | {self.engine.name} exec -i {self.name} tar --no-same-owner -xC {shell_quote(to_path)} -f -", + shell=True, + check=True, + cwd=from_path, + ) + else: + self.call(["mkdir", "-p", to_path.parent]) + # Use native docker cp to copy the file, avoiding stdin EOF deadlocks in RBE + subprocess.run( + [ + self.engine.name, + "cp", + str(from_path), + f"{self.name}:{to_path}", + ], + check=True, + ) + + def copy_out""" + +if pattern.search(content): + content = pattern.sub(replacement_copy, content) + print("Patched copy_into method using unique regex.") +else: + print("ERROR: Could not find copy_into method boundary in oci_container.py! The cibuildwheel version might have changed.") + sys.exit(1) + +with open(path, 'w') as f: + f.write(content) + +print("Successfully patched oci_container.py!") +EOF + +python3 patch_oci.py "$OCI_PATH" +rm patch_oci.py + +# Verify that the patched file is syntactically valid Python +echo "Verifying patched oci_container.py syntax..." +python3 -m py_compile "$OCI_PATH" || { echo "ERROR: Patched oci_container.py is corrupted!"; exit 1; } + +REPO_DIR="" +TMP_DIR="" +cleanup() { + echo "Cleaning up temporary directories..." + [ -n "${REPO_DIR}" ] && rm -rf "${REPO_DIR}" + [ -n "${TMP_DIR}" ] && rm -rf "${TMP_DIR}" +} +trap cleanup EXIT REPO_DIR=$(mktemp -d) echo "Created temporary directory: ${REPO_DIR}" -# Ensure the temporary directory is removed on script exit -trap 'echo "Cleaning up temporary directory: ${REPO_DIR}"; rm -rf "${REPO_DIR}"' EXIT - if [ "${DRY_RUN}" = "true" ]; then echo "[DRY RUN] Using local Kokoro clone instead of cloning main." SRC_DIR="$(cd "$(dirname "$0")/../.." && pwd)" @@ -40,11 +219,14 @@ fi VERSION=${VERSION#v} echo "Building release for version: ${VERSION}" -TMP_DIR=$(mktemp -d) +# Create the build directory inside the workspace volume (SRC_DIR) +# instead of the ephemeral /tmp, so that the sibling container can +# access it natively via volume propagation +TMP_DIR="${SRC_DIR}/build_area" +mkdir -p "${TMP_DIR}" echo "Build directory: ${TMP_DIR}" - -# Add trap cleanup for TMP_DIR as well -trap 'echo "Cleaning up temporary directories: ${REPO_DIR} ${TMP_DIR}"; rm -rf "${REPO_DIR}" "${TMP_DIR}"' EXIT +export TMPDIR="${TMP_DIR}/tmp" +mkdir -p "${TMPDIR}" pushd "${TMP_DIR}" @@ -52,12 +234,26 @@ cp -r "${SRC_DIR}"/{*,.*} . 2>/dev/null || true cp -r "${SRC_DIR}"/release/* . 2>/dev/null || true rm -rf cel_expr_python/*_test.py -# Check if pyproject.toml exists before running sed +echo "Downloading bazelisk on host..." +curl -LO https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazelisk-linux-amd64 +curl -LO https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazelisk-linux-arm64 +chmod +x bazelisk-linux-amd64 bazelisk-linux-arm64 + +echo "Downloading build dependencies on host..." +mkdir -p build_deps +pip download --no-cache-dir --only-binary=:all: --dest build_deps "setuptools>=40.8.0" "wheel" +if [ "$(uname -m)" = "aarch64" ]; then + PLATFORM_SUFFIX="aarch64" +else + PLATFORM_SUFFIX="x86_64" +fi +pip download --no-cache-dir --only-binary=:all: --dest build_deps --python-version 3.9 --platform "manylinux2014_${PLATFORM_SUFFIX}" "virtualenv" "typing-extensions>=4.13.2" + if [ -f pyproject.toml ]; then sed -i "" "s/\$VERSION/${VERSION}/g" pyproject.toml || sed -i "s/\$VERSION/${VERSION}/g" pyproject.toml fi -echo "Running cibuildwheel: ${CIBWHEEL_BIN}" +echo "Running cibuildwheel..." # Default CIBWHEEL_BIN if not set if [ -z "${CIBWHEEL_BIN}" ]; then CIBWHEEL_BIN="python3 -m cibuildwheel" diff --git a/release/kokoro/release_macos.sh b/release/kokoro/release_macos.sh index 71d4805..54b92fa 100644 --- a/release/kokoro/release_macos.sh +++ b/release/kokoro/release_macos.sh @@ -65,6 +65,8 @@ if [ -f pyproject.toml ]; then sed -i "" "s/\$VERSION/${VERSION}/g" pyproject.toml || sed -i "s/\$VERSION/${VERSION}/g" pyproject.toml fi +export CEL_BAZEL_FLAGS="--config=remote-cache-macos" + echo "Running cibuildwheel: ${CIBWHEEL_BIN}" # Default CIBWHEEL_BIN if not set if [ -z "${CIBWHEEL_BIN}" ]; then diff --git a/release/kokoro/release_windows.bat b/release/kokoro/release_windows.bat index 9f95c7e..8b6f003 100644 --- a/release/kokoro/release_windows.bat +++ b/release/kokoro/release_windows.bat @@ -16,6 +16,11 @@ setlocal enabledelayedexpansion set "RELEASE_STATUS=0" set "FETCH_RETRIES=10" set "FETCH_RETRY_DELAY_S=10" +echo --- Installing Python 3.11 via Chocolatey --- +choco install python311 -y --no-progress +if !ERRORLEVEL! NEQ 0 ( + echo WARNING: Failed to install Python 3.11 via Chocolatey. +) echo === Loading Environment Configuration === call "%~dp0set_env_windows.bat" @@ -46,42 +51,51 @@ echo Created temporary directories: %REPO_DIR%, %TMP_DIR% mkdir "%TMP_DIR%" echo --- Resolving Repository Source --- -if "%DRY_RUN%" == "true" ( - echo [DRY RUN] Using local Kokoro clone instead of cloning main. - set "SRC_DIR=%~dp0..\.." - pushd "!SRC_DIR!" - for /f "tokens=*" %%i in ('git tag --sort=-v:refname 2^>nul') do ( - set "VERSION=%%i" - goto :got_local_tag - ) - set "VERSION=0.1.2" - :got_local_tag +if "%DRY_RUN%" == "true" goto resolution_dry +goto resolution_real + +:resolution_dry +echo [DRY RUN] Using local Kokoro clone instead of cloning main. +set "SRC_DIR=%~dp0..\.." +pushd "!SRC_DIR!" +set "VERSION=" +for /f "tokens=*" %%i in ('git tag --sort=-v:refname 2^>nul') do ( + set "VERSION=%%i" + goto got_local_tag +) +:got_local_tag +if "%VERSION%" == "" set "VERSION=0.1.2" +popd +goto resolution_done + +:resolution_real +mkdir "%REPO_DIR%" +pushd "%REPO_DIR%" +git clone https://github.com/cel-expr/cel-python.git +if !ERRORLEVEL! NEQ 0 ( + echo Failed to clone repository! + set "RELEASE_STATUS=1" popd -) else ( - mkdir "%REPO_DIR%" - pushd "%REPO_DIR%" - git clone https://github.com/cel-expr/cel-python.git - if !ERRORLEVEL! NEQ 0 ( - echo Failed to clone repository! - set "RELEASE_STATUS=1" - popd - goto cleanup - ) - cd cel-python - for /f "tokens=*" %%i in ('git tag --sort=-v:refname') do ( - set "VERSION=%%i" - goto :got_tag - ) - :got_tag - if "%VERSION%" == "" ( - echo Failed to get version tag! - set "RELEASE_STATUS=1" - popd - goto cleanup - ) - set "SRC_DIR=%REPO_DIR%\cel-python" + goto cleanup +) +cd cel-python +set "VERSION=" +for /f "tokens=*" %%i in ('git tag --sort=-v:refname') do ( + set "VERSION=%%i" + goto got_tag +) +:got_tag +if "%VERSION%" == "" ( + echo Failed to get version tag! + set "RELEASE_STATUS=1" popd + goto cleanup ) +set "SRC_DIR=%REPO_DIR%\cel-python" +popd +goto resolution_done + +:resolution_done if "%VERSION:~0,1%" == "v" ( set "VERSION=%VERSION:~1%" @@ -145,6 +159,7 @@ if !FETCH_STATUS! NEQ 0 ( if exist fetch.log del fetch.log echo --- Running cibuildwheel --- +set "CEL_BAZEL_FLAGS=--config=remote-cache-windows" if "%CIBWHEEL_BIN%" == "" ( set "CIBWHEEL_BIN=!PYTHON_EXE! -m cibuildwheel" ) diff --git a/release/pyproject.toml b/release/pyproject.toml index 3022d61..d7aa211 100644 --- a/release/pyproject.toml +++ b/release/pyproject.toml @@ -39,15 +39,28 @@ exclude = ["codelab*", "conformance*", "custom_ext*", "release*", "testing*", "w [tool.cibuildwheel] build = "cp311-* cp312-* cp313-* cp314-*" -skip = "*musllinux* *win32*" +skip = "*musllinux* *win32* *i686*" test-command = "python {project}/cel_basic_test.py" build-verbosity = 1 [tool.cibuildwheel.linux] -before-all = "echo 'Installing bazelisk'; curl -LO https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazelisk-linux-amd64 && chmod +x bazelisk-linux-amd64 && mv bazelisk-linux-amd64 /usr/local/bin/bazel" +build-frontend = { name = "pip", args = ["--no-build-isolation"] } +before-build = "pip install --no-index --find-links={project}/build_deps setuptools wheel virtualenv" +archs = ["x86_64", "aarch64"] +manylinux-x86_64-image = "manylinux_2_28" +manylinux-aarch64-image = "manylinux_2_28" +container-engine = "docker; disable_host_mount: True" +# Google's internal Kokoro/RBE network uses a secure MITM proxy that resigns HTTPS +# traffic with an internal Google CA. Since the public manylinux container does not +# trust this CA, git fetches for external dependencies (like @cel-cpp) will fail +# with SSL certificate errors. We disable http.sslVerify inside the container to +# bypass this and allow Bazel to fetch SCM dependencies through the proxy. +before-all = "git config --global http.sslVerify false && echo 'Installing bazelisk' && if [ $(uname -m) = 'aarch64' ]; then cp {project}/bazelisk-linux-arm64 /usr/local/bin/bazel; else cp {project}/bazelisk-linux-amd64 /usr/local/bin/bazel; fi && python3 -m pip install --no-index --find-links={project}/build_deps virtualenv" [tool.cibuildwheel.macos] +archs = ["x86_64", "arm64"] before-all = "echo 'Installing bazelisk'; brew install bazelisk" +environment = { MACOSX_DEPLOYMENT_TARGET = "10.13" } [tool.cibuildwheel.windows] # Bazel is expected to be already installed and in the PATH on Windows. diff --git a/release/setup.py b/release/setup.py index 1fec997..e2d0962 100644 --- a/release/setup.py +++ b/release/setup.py @@ -16,6 +16,7 @@ import glob import os +import platform import re import shutil import subprocess @@ -71,6 +72,9 @@ def build_extension(self, ext): # Build with bazel # Use --compilation_mode=opt for release builds cmd = ['bazel', 'build', ext.target, '--compilation_mode=opt'] + extra_flags = os.environ.get('CEL_BAZEL_FLAGS') + if extra_flags: + cmd.extend(extra_flags.split()) if sys.platform == 'win32': self.platform_config_windows(cmd, python_version) if sys.platform == 'darwin': @@ -161,7 +165,31 @@ def platform_config_windows(self, cmd, python_version): def platform_config_macos(self, cmd): """Applies macOS-specific Bazel configurations.""" - cmd.extend(['--macos_minimum_os=10.13', '--cxxopt=-faligned-allocation']) + deployment_target = os.environ.get('MACOSX_DEPLOYMENT_TARGET', '10.13') + cmd.extend([ + f'--macos_minimum_os={deployment_target}', + '--cxxopt=-faligned-allocation', + ]) + + archflags = os.environ.get('ARCHFLAGS', '') + if 'x86_64' in archflags: + target_arch = 'x86_64' + elif 'arm64' in archflags: + target_arch = 'arm64' + else: + machine = platform.machine() + if machine in ('AMD64', 'x86_64'): + target_arch = 'x86_64' + elif machine in ('arm64', 'aarch64'): + target_arch = 'arm64' + else: + target_arch = machine + + print(f'Target architecture for macOS: {target_arch}', flush=True) + cmd.append(f'--macos_cpus={target_arch}') + cmd.append(f'--cpu=darwin_{target_arch}') + if target_arch == 'x86_64': + cmd.append('--platforms=//cel_expr_python:macos_x86_64') setuptools.setup(