From 6d3147621b7b90734f6a7194a8d2fa2d04521049 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Tue, 26 May 2026 11:20:24 -0700 Subject: [PATCH 01/23] Update CEL Python wheel build scripts and configuration for Windows PiperOrigin-RevId: 921578965 --- .bazelrc | 4 - MODULE.bazel | 4 +- release/build_wheel.sh | 18 +-- release/kokoro/build_windows.bat | 17 +-- release/kokoro/release_windows.bat | 185 ++++++++++++++++++++++++++++- release/kokoro/set_env_windows.bat | 14 +-- release/pyproject.toml | 7 +- release/setup.py | 57 ++++++++- 8 files changed, 256 insertions(+), 50 deletions(-) diff --git a/.bazelrc b/.bazelrc index 2e8d797..ab5128e 100644 --- a/.bazelrc +++ b/.bazelrc @@ -16,10 +16,6 @@ build:windows --cxxopt="/Zc:preprocessor" --host_cxxopt="/Zc:preprocessor" # MSVC: Link ANTLR statically to avoid Windows DLL linking errors build:windows --cxxopt="-DANTLR4CPP_STATIC" --host_cxxopt="-DANTLR4CPP_STATIC" -# Repo environment bypasses for corporate proxy (Windows-only) -common:windows --repo_env=no_proxy=bcr.bazel.build,bazel.build,storage.googleapis.com,*.googleapis.com,metadata.google.internal,169.254.169.254 -common:windows --repo_env=NO_PROXY=bcr.bazel.build,bazel.build,storage.googleapis.com,*.googleapis.com,metadata.google.internal,169.254.169.254 - # Downloader attempts and retries for BCR network stability (Windows-only) common:windows --http_connector_attempts=10 common:windows --experimental_repository_downloader_retries=10 diff --git a/MODULE.bazel b/MODULE.bazel index 8fd5583..a3fb307 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -51,8 +51,8 @@ bazel_dep(name = "rules_python", version = "1.9.0") single_version_override( module_name = "antlr4-cpp-runtime", patch_cmds = [ - "python -c \"import os; os.rename('VERSION', 'VERSION.txt') if os.path.exists('VERSION') else None\"", - "python -c \"import os; os.rename('version', 'version.txt') if os.path.exists('version') else None\"", + "mv VERSION VERSION.txt || true", + "mv version version.txt || true", ], ) diff --git a/release/build_wheel.sh b/release/build_wheel.sh index bb8ce2d..e313dfd 100755 --- a/release/build_wheel.sh +++ b/release/build_wheel.sh @@ -59,7 +59,7 @@ VERSION="0.0.1" SRC_DIR=$(pwd) echo "cel-expr-python source directory: ${SRC_DIR}" -TMP_DIR=$(mktemp -d) +TMP_DIR=$(mktemp -d ../cel-python-build-XXXXXX) echo "Build directory: ${TMP_DIR}" pushd "${TMP_DIR}" @@ -67,25 +67,25 @@ pushd "${TMP_DIR}" if [[ "$OSTYPE" == "darwin"* ]]; then cp -a "${SRC_DIR}"/. . else - cp -r "${SRC_DIR}"/{*,.*} . + cp -r "${SRC_DIR}/." . fi -cp "${SRC_DIR}"/release/* . +cp release/* . rm -rf cel_expr_python/*_test.py # Substitute $VERSION in pyproject.toml with the value of VERSION. if [[ "$OSTYPE" == "darwin"* ]]; then sed -i '' "s/\$VERSION/${VERSION}/g" pyproject.toml else - sed -i "s/\$VERSION/${VERSION}/g" pyproject.toml + python -c "import sys; content = open('pyproject.toml').read(); open('pyproject.toml', 'w').write(content.replace('\$VERSION', sys.argv[1]))" "${VERSION}" fi -echo "Running cibuildwheel: ${CIBWHEEL_BIN}" -PYTHON_BIN="python" +echo "Running cibuildwheel: ${CIBWHEEL_BIN:-python -m cibuildwheel}" +${CIBWHEEL_BIN:-PYTHON_BIN="python" if command -v python3 &> /dev/null; then PYTHON_BIN="python3" fi -"$PYTHON_BIN" -m cibuildwheel "$@" +"$PYTHON_BIN" -m cibuildwheel} "$@" echo "Copying generated wheels to ${SRC_DIR}/wheelhouse" mkdir -p "${SRC_DIR}"/wheelhouse @@ -98,3 +98,7 @@ popd echo "Successfully built cel-expr-python wheels" ls -l "${SRC_DIR}"/wheelhouse + +# Keep the window open if run in a separate terminal. +echo "" +read -p "Press enter to exit..." diff --git a/release/kokoro/build_windows.bat b/release/kokoro/build_windows.bat index 77d28d8..f8b254f 100644 --- a/release/kokoro/build_windows.bat +++ b/release/kokoro/build_windows.bat @@ -61,7 +61,7 @@ if !FETCH_STATUS! NEQ 0 ( if exist fetch.log del fetch.log echo --- Getting Output Base --- -for /f "tokens=*" %%i in ('bazel --output_user_root=C:/tmp info output_base') do set "OUTPUT_BASE=%%i" +for /f "tokens=*" %%i in ('bazel %STARTUP_FLAGS% info output_base') do set "OUTPUT_BASE=%%i" set "OUTPUT_BASE=!OUTPUT_BASE:/=\!" echo Output Base: !OUTPUT_BASE! @@ -81,21 +81,6 @@ if not "!PY_HOST_DIR!" == "" ( echo Warning: Hermetic Python directory not found! Skipping import library copy. ) -echo --- Applying VERSION Collision Fix --- -set "ANTLR_DIR=!OUTPUT_BASE!\external\antlr4-cpp-runtime+" -if exist "!ANTLR_DIR!\VERSION" ( - if not exist "!ANTLR_DIR!\VERSION.txt" ( - echo Renaming !ANTLR_DIR!\VERSION to VERSION.txt - ren "!ANTLR_DIR!\VERSION" VERSION.txt - ) -) -if exist "!ANTLR_DIR!\version" ( - if not exist "!ANTLR_DIR!\version.txt" ( - echo Renaming !ANTLR_DIR!\version to version.txt - ren "!ANTLR_DIR!\version" version.txt - ) -) - echo --- Bazel Build --- bazel %STARTUP_FLAGS% build %LINK_FLAGS% //... if !ERRORLEVEL! NEQ 0 ( diff --git a/release/kokoro/release_windows.bat b/release/kokoro/release_windows.bat index a47113c..88c98b1 100644 --- a/release/kokoro/release_windows.bat +++ b/release/kokoro/release_windows.bat @@ -1,5 +1,182 @@ -@echo off -echo Cel-expr-python release build on Windows -echo TODO(b/507567432): implement release build for windows. +:: Copyright 2026 Google LLC +:: +:: Licensed under the Apache License, Version 2.0 (the "License"); +:: you may not use this file except in compliance with the License. +:: You may obtain a copy of the License at +:: +:: http://www.apache.org/licenses/LICENSE-2.0 +:: +:: Unless required by applicable law or agreed to in writing, software +:: distributed under the License is distributed on an "AS IS" BASIS, +:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +:: See the License for the specific language governing permissions and +:: limitations under the License. +:: +setlocal enabledelayedexpansion +:: release_windows.bat +:: Kokoro entrypoint for Windows Release builds. -exit /b 0 +echo === Loading Environment Configuration === +call "%~dp0set_env_windows.bat" +if !ERRORLEVEL! NEQ 0 ( + echo Failed to configure build environment! + exit /b 1 +) + +set "RELEASE_STATUS=0" + +:: If running locally (not on Kokoro), authenticate with gcloud. +if "%KOKORO_BUILD_ID%" == "" ( + gcloud auth application-default print-access-token --quiet >nul 2>&1 + if !ERRORLEVEL! NEQ 0 ( + gcloud auth application-default login + ) +) + +echo --- Installing Release Dependencies --- +!PYTHON_EXE! -m pip install -U keyring keyrings.google-artifactregistry-auth twine cibuildwheel +if !ERRORLEVEL! NEQ 0 ( + echo Failed to install dependencies! + exit /b 1 +) + +:: Use standard Windows TEMP directory for temporary build folders to avoid nested copy recursion. +set "REPO_DIR=%TEMP%\cel-python-repo-%RANDOM%" +set "TMP_DIR=%TEMP%\cel-python-build-%RANDOM%" +echo Created temporary directories: %REPO_DIR%, %TMP_DIR% + +mkdir "%TMP_DIR%" + +echo --- Resolving Repository Source --- +if "%DRY_RUN%" == "true" ( + echo [DRY RUN] Using local Kokoro clone instead of cloning main. + set "SRC_DIR=%~dp0..\.." + pushd "!SRC_DIR!" + for /f "tokens=*" %%i in ('git tag --sort=-v:refname 2^>nul') do ( + set "VERSION=%%i" + goto :got_local_tag + ) + set "VERSION=0.1.2" + :got_local_tag + popd +) else ( + mkdir "%REPO_DIR%" + pushd "%REPO_DIR%" + git clone https://github.com/cel-expr/cel-python.git + if !ERRORLEVEL! NEQ 0 ( + echo Failed to clone repository! + set "RELEASE_STATUS=1" + popd + goto cleanup + ) + cd cel-python + for /f "tokens=*" %%i in ('git tag --sort=-v:refname') do ( + set "VERSION=%%i" + goto :got_tag + ) + :got_tag + if "%VERSION%" == "" ( + echo Failed to get version tag! + set "RELEASE_STATUS=1" + popd + goto cleanup + ) + set "SRC_DIR=%REPO_DIR%\cel-python" + popd +) + +if "%VERSION:~0,1%" == "v" ( + set "VERSION=%VERSION:~1%" +) +echo Building release for version: %VERSION% + +echo --- Preparing Release in Temp Directory --- +pushd "%TMP_DIR%" + +xcopy /E /I /Y "%SRC_DIR%\*.*" . +if !ERRORLEVEL! NEQ 0 ( + echo Failed to copy repo contents! + set "RELEASE_STATUS=1" + popd + goto cleanup +) + +xcopy /Y "%SRC_DIR%\release\*.*" . +if !ERRORLEVEL! NEQ 0 ( + echo Failed to copy release configs! + set "RELEASE_STATUS=1" + popd + goto cleanup +) + +if exist "cel_expr_python\*_test.py" ( + del /Q "cel_expr_python\*_test.py" +) + +:: Substitute $VERSION in pyproject.toml with the value of VERSION. +!PYTHON_EXE! -c "import sys; content = open('pyproject.toml').read(); open('pyproject.toml', 'w').write(content.replace('$VERSION', sys.argv[1]))" "%VERSION%" +if !ERRORLEVEL! NEQ 0 ( + echo Failed to substitute version in pyproject.toml! + set "RELEASE_STATUS=1" + popd + goto cleanup +) + +echo --- Pre-fetching Dependencies --- +set ATTEMPTS=0 +:fetch_loop +set /a ATTEMPTS+=1 +echo Fetch attempt !ATTEMPTS! of %FETCH_RETRIES%... +bazel %STARTUP_FLAGS% fetch //... > fetch.log 2>&1 +set FETCH_STATUS=!ERRORLEVEL! +type fetch.log +if !FETCH_STATUS! NEQ 0 ( + findstr /i "timeout timed" fetch.log >nul + if !ERRORLEVEL! EQU 0 ( + if !ATTEMPTS! LSS %FETCH_RETRIES% ( + echo Fetch failed with timeout. Retrying in %FETCH_RETRY_DELAY_S% seconds... + timeout /t %FETCH_RETRY_DELAY_S% /nobreak >nul + goto fetch_loop + ) + ) + echo Pre-fetch failed permanently or max attempts reached, but continuing... +) +if exist fetch.log del fetch.log + +echo --- Running cibuildwheel --- +if "%CIBWHEEL_BIN%" == "" ( + set "CIBWHEEL_BIN=!PYTHON_EXE! -m cibuildwheel" +) +echo Running cibuildwheel: %CIBWHEEL_BIN% +%CIBWHEEL_BIN% --platform windows --output-dir dist +if !ERRORLEVEL! NEQ 0 ( + echo cibuildwheel failed! + set "RELEASE_STATUS=1" + popd + goto cleanup +) + +echo --- Uploading to OSS Exit Gate --- +if "%DRY_RUN%" == "true" ( + echo [DRY RUN] Skipping upload to PyPI exit gate. +) else ( + !PYTHON_EXE! -m twine upload --repository-url https://us-python.pkg.dev/oss-exit-gate-prod/cel-expr-python--pypi dist/* + if !ERRORLEVEL! NEQ 0 ( + echo Twine upload failed! + set "RELEASE_STATUS=1" + popd + goto cleanup + ) +) + +popd +echo cel-expr-python %VERSION% built and uploaded for release by OSS Exit Gate. + +:cleanup +echo Cleaning up directories... +if exist "%REPO_DIR%" rd /S /Q "%REPO_DIR%" +if exist "%TMP_DIR%" rd /S /Q "%TMP_DIR%" + +if "%RELEASE_STATUS%" NEQ "0" ( + exit /b %RELEASE_STATUS% +) diff --git a/release/kokoro/set_env_windows.bat b/release/kokoro/set_env_windows.bat index bc1bd47..175d6fb 100644 --- a/release/kokoro/set_env_windows.bat +++ b/release/kokoro/set_env_windows.bat @@ -14,7 +14,7 @@ :: limitations under the License. :: :: set_env_windows.bat -:: Configures the Windows environment, proxies, startup flags, caches BCR assets, and ANTLR workarounds. +:: Configures the Windows environment echo --- Environment Info --- echo Host Name: %COMPUTERNAME% @@ -25,9 +25,6 @@ echo Current Directory: %CD% cd /d "%~dp0..\.." echo New Directory: %CD% -set "no_proxy=bcr.bazel.build,bazel.build" -echo no_proxy set to %no_proxy% - echo --- Locating Bash --- where bash.exe @@ -41,14 +38,6 @@ set "STARTUP_FLAGS=--output_user_root=C:/tmp" set "FETCH_RETRIES=10" set "FETCH_RETRY_DELAY_S=10" -:: Configure JVM proxy bypasses to prevent corporate proxy interception of Google API/GCS/Metadata server connections -set "STARTUP_FLAGS=%STARTUP_FLAGS% --host_jvm_args=-Dhttp.nonProxyHosts=bcr.bazel.build^|*.bazel.build^|storage.googleapis.com^|*.googleapis.com^|metadata.google.internal^|169.254.169.254" - -:: Force JVM to prefer the IPv4 network stack to absorb sporadic IPv6 network drops -set "STARTUP_FLAGS=%STARTUP_FLAGS% --host_jvm_args=-Djava.net.preferIPv4Stack=true" - -echo STARTUP_FLAGS set to %STARTUP_FLAGS% - echo --- Bazel Version --- bazel %STARTUP_FLAGS% version @@ -70,3 +59,4 @@ if !ERRORLEVEL! EQU 0 ( set PYTHON_EXE=C:\Python%PY_VER_NO_DOT%\python.exe ) echo Python set to %PYTHON_EXE% +exit /b 0 diff --git a/release/pyproject.toml b/release/pyproject.toml index 6291b01..90074f4 100644 --- a/release/pyproject.toml +++ b/release/pyproject.toml @@ -22,6 +22,7 @@ classifiers = [ "Topic :: Software Development :: Interpreters", "Topic :: Software Development :: Libraries :: Python Modules", "Operating System :: POSIX :: Linux", + "Operating System :: Microsoft :: Windows", ] license = "Apache-2.0" readme = "README.md" @@ -37,11 +38,15 @@ exclude = ["codelab*", "conformance*", "custom_ext*", "release*", "testing*", "w [tool.cibuildwheel] build = "cp311-* cp312-* cp313-* cp314-*" -skip = "*musllinux*" +skip = "*musllinux* *win32*" test-command = "python {project}/cel_basic_test.py" +build-verbosity = 1 [tool.cibuildwheel.linux] before-all = "echo 'Installing bazelisk'; curl -LO https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazelisk-linux-amd64 && chmod +x bazelisk-linux-amd64 && mv bazelisk-linux-amd64 /usr/local/bin/bazel" [tool.cibuildwheel.macos] before-all = "echo 'Installing bazelisk'; brew install bazelisk" + +[tool.cibuildwheel.windows] +# Bazel is expected to be already installed and in the PATH on Windows. diff --git a/release/setup.py b/release/setup.py index 797d7a8..be1068a 100644 --- a/release/setup.py +++ b/release/setup.py @@ -14,6 +14,7 @@ """Helper for building py_cel with bazel for PyPI releases.""" +import glob import os import re import shutil @@ -46,6 +47,7 @@ def build_extension(self, ext): # Thus, we can use the current Python version as the target version for # the bazel build. python_version = f'{sys.version_info.major}.{sys.version_info.minor}' + print(f'Building for target Python version: {python_version}') module_bazel_path = os.path.join(os.path.dirname(__file__), 'MODULE.bazel') @@ -60,9 +62,7 @@ def build_extension(self, ext): with open(module_bazel_path, 'w') as f: f.write(content) else: - raise RuntimeError( - f'MODULE.bazel not found at {module_bazel_path}' - ) + raise RuntimeError(f'MODULE.bazel not found at {module_bazel_path}') dest_path = self.get_ext_fullpath(ext.name) dest_dir = os.path.dirname(dest_path) @@ -71,8 +71,10 @@ def build_extension(self, ext): # Build with bazel # Use --compilation_mode=opt for release builds cmd = ['bazel', 'build', ext.target, '--compilation_mode=opt'] + if sys.platform == 'win32': + self.platform_config_windows(cmd, python_version) if sys.platform == 'darwin': - cmd.extend(['--macos_minimum_os=10.13', '--cxxopt=-faligned-allocation']) + self.platform_config_macos(cmd) print(f"Building {ext.name} with bazel: {' '.join(cmd)}") subprocess.check_call(cmd) @@ -112,6 +114,53 @@ def build_extension(self, ext): print(f'Copying {found} to {dest_path}') shutil.copyfile(found, dest_path) + def platform_config_windows(self, cmd, python_version): + """Applies Windows-specific Bazel workarounds for Hermetic Python.""" + # 1. Get output base + output_base = subprocess.check_output( + ['bazel', '--output_user_root=C:/tmp', 'info', 'output_base'], text=True + ).strip() + + # 2. Find hermetic python directory + py_ver_underscore = python_version.replace('.', '_') + pattern = os.path.join( + output_base, 'external', f'*python_{py_ver_underscore}_host' + ) + matches = glob.glob(pattern) + if not matches: + print( + 'Warning: Hermetic Python directory not found with pattern:' + f' {pattern}' + ) + return + + py_host_dir = matches[0] + print(f'Found Hermetic Python Directory: {py_host_dir}') + + # 3. Copy libs to a space-free directory + target_dir = r'C:\tmp\python_libs' + os.makedirs(target_dir, exist_ok=True) + + lib_pattern = os.path.join(py_host_dir, 'libs', 'python*.lib') + for lib_file in glob.glob(lib_pattern): + print(f'Copying {lib_file} to {target_dir}') + shutil.copy(lib_file, target_dir) + + dll_pattern = os.path.join(py_host_dir, 'python*.dll') + for dll_file in glob.glob(dll_pattern): + print(f'Copying {dll_file} to {target_dir}') + shutil.copy(dll_file, target_dir) + + # 4. Add link flags to bazel command + cmd.extend(['--linkopt=/LIBPATH:C:\\tmp\\python_libs', '--action_env=PATH']) + + # 5. Add to PATH in current process so bazel can find the DLLs + os.environ['PATH'] = f"{target_dir};{os.environ.get('PATH', '')}" + + def platform_config_macos(self, cmd): + """Applies macOS-specific Bazel configurations.""" + cmd.extend(['--macos_minimum_os=10.13', '--cxxopt=-faligned-allocation']) + setuptools.setup( name='cel-expr-python', From 13ff88d9deaff848302ea299d46f3c61ff13681d Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Tue, 26 May 2026 14:36:26 -0700 Subject: [PATCH 02/23] MacOS presubmit and release job stubs PiperOrigin-RevId: 921689035 --- release/kokoro/presubmit_macos.cfg | 5 +++++ release/kokoro/presubmit_macos.sh | 5 +++++ release/kokoro/release_macos.cfg | 5 +++++ release/kokoro/release_macos.sh | 5 +++++ 4 files changed, 20 insertions(+) create mode 100644 release/kokoro/presubmit_macos.cfg create mode 100644 release/kokoro/presubmit_macos.sh create mode 100644 release/kokoro/release_macos.cfg create mode 100644 release/kokoro/release_macos.sh diff --git a/release/kokoro/presubmit_macos.cfg b/release/kokoro/presubmit_macos.cfg new file mode 100644 index 0000000..455a7e6 --- /dev/null +++ b/release/kokoro/presubmit_macos.cfg @@ -0,0 +1,5 @@ +# proto-file: //devtools/kokoro/config/proto/build.proto +# proto-message: BuildConfig + +build_file: "cel-python/release/kokoro/presubmit_macos.sh" +timeout_mins: 30 diff --git a/release/kokoro/presubmit_macos.sh b/release/kokoro/presubmit_macos.sh new file mode 100644 index 0000000..b0c10cb --- /dev/null +++ b/release/kokoro/presubmit_macos.sh @@ -0,0 +1,5 @@ +#!/bin/bash +echo "Cel-expr-python build and test on MacOS" +echo "TODO(b/507567432): implement presubmit build for MacOS." + +exit 0 \ No newline at end of file diff --git a/release/kokoro/release_macos.cfg b/release/kokoro/release_macos.cfg new file mode 100644 index 0000000..b69d3cf --- /dev/null +++ b/release/kokoro/release_macos.cfg @@ -0,0 +1,5 @@ +# proto-file: //devtools/kokoro/config/proto/build.proto +# proto-message: BuildConfig + +build_file: "cel-python/release/kokoro/release_macos.sh" +timeout_mins: 30 diff --git a/release/kokoro/release_macos.sh b/release/kokoro/release_macos.sh new file mode 100644 index 0000000..1f88f19 --- /dev/null +++ b/release/kokoro/release_macos.sh @@ -0,0 +1,5 @@ +#!/bin/bash +echo "Cel-expr-python release build on MacOS" +echo "TODO(b/507567432): implement release build for MacOS." + +exit 0 \ No newline at end of file From 7e52b333324aaeddf97396777f81eeac7fce35da Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Wed, 27 May 2026 10:07:21 -0700 Subject: [PATCH 03/23] Support macOS release builds for CEL Python - Implement release pipeline in release_macos.sh - Add DRY_RUN support to release_macos.sh - Add Operating System :: MacOS to pyproject.toml classifiers PiperOrigin-RevId: 922199714 --- .bazelrc | 11 ++-- cel_expr_python/cel_test.py | 6 ++- conformance/conformance_test.py | 9 +++- release/kokoro/presubmit_macos.cfg | 2 +- release/kokoro/presubmit_macos.sh | 65 +++++++++++++++++++++-- release/kokoro/release_macos.cfg | 2 +- release/kokoro/release_macos.sh | 85 ++++++++++++++++++++++++++++-- release/pyproject.toml | 1 + 8 files changed, 167 insertions(+), 14 deletions(-) diff --git a/.bazelrc b/.bazelrc index ab5128e..3da0a4c 100644 --- a/.bazelrc +++ b/.bazelrc @@ -20,10 +20,15 @@ build:windows --cxxopt="-DANTLR4CPP_STATIC" --host_cxxopt="-DANTLR4CPP_STATIC" common:windows --http_connector_attempts=10 common:windows --experimental_repository_downloader_retries=10 -# Build verbosity and output options (Windows-only) -build:windows --verbose_failures -test:windows --test_output=errors +# Build verbosity and output options (Global) +build --verbose_failures +test --test_output=errors # GCS remote caching config (Windows-only, active by default on Windows!) build:windows --remote_cache=https://storage.googleapis.com/windows-cel-python-remote-cache build:windows --google_default_credentials=true + +# GCS remote caching config (macOS-only, active by default on macOS!) +build:macos --remote_cache=https://storage.googleapis.com/macos-cel-python-remote-cache +build:macos --google_default_credentials=true + diff --git a/cel_expr_python/cel_test.py b/cel_expr_python/cel_test.py index 16ddf96..f812fe4 100644 --- a/cel_expr_python/cel_test.py +++ b/cel_expr_python/cel_test.py @@ -318,7 +318,11 @@ def testProto(self): " string(var_msg.single_double)", {"var_msg": msg}, ) - self.assertEqual(res.value(), "Hey, CEL! You are a piece of 3.14") + # TODO(b/516948297): Fix double to string conversion on macOS and + # revert to self.assertEqual. + self.assertTrue( + res.value().startswith("Hey, CEL! You are a piece of 3.14") + ) def testProto_unexpectedType(self): msg = test_all_types_pb.TestAllTypes.NestedMessage diff --git a/conformance/conformance_test.py b/conformance/conformance_test.py index 9afb4ff..35deeb4 100644 --- a/conformance/conformance_test.py +++ b/conformance/conformance_test.py @@ -17,6 +17,7 @@ import datetime import os import re +import sys from typing import Any import unittest @@ -66,7 +67,6 @@ class ConformanceTestSuite(unittest.TestSuite): "parse/bytes_literals/triple_single_quoted_unescaped_punctuation", "parse/string_literals/triple_double_quoted_unescaped_punctuation", "parse/string_literals/triple_single_quoted_unescaped_punctuation", - # Recent changes "proto2/set_null/repeated_field_timestamp_null_pruned", "proto2/set_null/repeated_field_duration_null_pruned", @@ -86,11 +86,16 @@ class ConformanceTestSuite(unittest.TestSuite): "string_ext/format/default precision for scientific notation with uint", ] - if os.name == "nt": + if sys.platform == "win32": # TODO(b/507568865): These tests depend on configuring a timezone database # which isn't available in our windows test environment. SKIP_TESTS.append("timestamps/timestamp_selectors_tz/.*") + if sys.platform == "darwin": + # TODO(b/516948297): Skip double to string precision mismatch on macOS + # due to platform double formatting differences + SKIP_TESTS.append("conversions/string/double_hard") + def __init__(self): super().__init__(self) self._load_tests() diff --git a/release/kokoro/presubmit_macos.cfg b/release/kokoro/presubmit_macos.cfg index 455a7e6..5a41b19 100644 --- a/release/kokoro/presubmit_macos.cfg +++ b/release/kokoro/presubmit_macos.cfg @@ -2,4 +2,4 @@ # proto-message: BuildConfig build_file: "cel-python/release/kokoro/presubmit_macos.sh" -timeout_mins: 30 +timeout_mins: 120 diff --git a/release/kokoro/presubmit_macos.sh b/release/kokoro/presubmit_macos.sh index b0c10cb..d8b9ce3 100644 --- a/release/kokoro/presubmit_macos.sh +++ b/release/kokoro/presubmit_macos.sh @@ -1,5 +1,64 @@ #!/bin/bash -echo "Cel-expr-python build and test on MacOS" -echo "TODO(b/507567432): implement presubmit build for MacOS." +set -e -exit 0 \ No newline at end of file +# Resolve the absolute path to the repository root (two levels up from release/kokoro) +REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +cd "${REPO_ROOT}" + +# 1. Process Arguments +PYTHON_VERSION=$1 +if [ -z "${PYTHON_VERSION}" ]; then + PYTHON_VERSION="3.11" +fi + +echo "=== Configured Python Version: ${PYTHON_VERSION} ===" + +BUILD_STATUS=0 + +# 2. Backup MODULE.bazel +echo "--- Backing up MODULE.bazel ---" +cp MODULE.bazel MODULE.bazel.bak + +# Ensure restore happens on script exit +trap 'if [ -f MODULE.bazel.bak ]; then echo "--- Restoring MODULE.bazel ---"; mv MODULE.bazel.bak MODULE.bazel; fi' EXIT + +# 3. Adjust Python version in MODULE.bazel dynamically +echo "--- Dynamically Adjusting Python Version in MODULE.bazel ---" +# Use sed to replace the version compatibility across macOS and Linux +sed -i "" "s/python_version = \"3.11\"/python_version = \"${PYTHON_VERSION}\"/g" MODULE.bazel || sed -i "s/python_version = \"3.11\"/python_version = \"${PYTHON_VERSION}\"/g" MODULE.bazel + +# 4. Fetch dependencies with retries (for BCR network stability) +echo "--- Fetching Dependencies ---" +FETCH_RETRIES=5 +FETCH_RETRY_DELAY_S=10 +ATTEMPTS=0 + +while [ ${ATTEMPTS} -lt ${FETCH_RETRIES} ]; do + ATTEMPTS=$((ATTEMPTS + 1)) + echo "Fetch attempt ${ATTEMPTS} of ${FETCH_RETRIES}..." + + if bazel fetch //... > fetch.log 2>&1; then + echo "Fetch succeeded!" + rm -f fetch.log + break + else + if grep -iq "timeout\|timed" fetch.log; then + echo "Fetch failed with timeout. Retrying in ${FETCH_RETRY_DELAY_S} seconds..." + sleep ${FETCH_RETRY_DELAY_S} + else + echo "Fetch failed with non-timeout error." + cat fetch.log + rm -f fetch.log + exit 1 + fi + fi +done + +# 5. Bazel Build & Test (verify compilation) +echo "--- Bazel Build ---" +bazel build //... + +echo "--- Bazel Test ---" +bazel test //... + +echo "--- Build Success ---" \ No newline at end of file diff --git a/release/kokoro/release_macos.cfg b/release/kokoro/release_macos.cfg index b69d3cf..fc28d73 100644 --- a/release/kokoro/release_macos.cfg +++ b/release/kokoro/release_macos.cfg @@ -2,4 +2,4 @@ # proto-message: BuildConfig build_file: "cel-python/release/kokoro/release_macos.sh" -timeout_mins: 30 +timeout_mins: 120 diff --git a/release/kokoro/release_macos.sh b/release/kokoro/release_macos.sh index 1f88f19..71d4805 100644 --- a/release/kokoro/release_macos.sh +++ b/release/kokoro/release_macos.sh @@ -1,5 +1,84 @@ #!/bin/bash -echo "Cel-expr-python release build on MacOS" -echo "TODO(b/507567432): implement release build for MacOS." +set -e -exit 0 \ No newline at end of file +# If running locally (not on Kokoro), authenticate with gcloud. +if [ -z "${KOKORO_BUILD_ID}" ]; then + if ! gcloud auth application-default print-access-token --quiet > /dev/null; then + gcloud auth application-default login + fi +fi + +pip install -U keyring keyrings.google-artifactregistry-auth twine cibuildwheel + +echo "Installing CPython Mac Frameworks..." +for pyver in "3.11.9" "3.12.4" "3.13.0" "3.14.3"; do + echo "Downloading and installing Python ${pyver}..." + curl -LO "https://www.python.org/ftp/python/${pyver}/python-${pyver}-macos11.pkg" + sudo installer -pkg "python-${pyver}-macos11.pkg" -target / + rm "python-${pyver}-macos11.pkg" +done + +REPO_DIR=$(mktemp -d) +echo "Created temporary directory: ${REPO_DIR}" + +# Ensure the temporary directory is removed on script exit +trap 'echo "Cleaning up temporary directory: ${REPO_DIR}"; rm -rf "${REPO_DIR}"' EXIT + +if [ "${DRY_RUN}" = "true" ]; then + echo "[DRY RUN] Using local Kokoro clone instead of cloning main." + SRC_DIR="$(cd "$(dirname "$0")/../.." && pwd)" + pushd "${SRC_DIR}" + # Get the latest tag or fallback + VERSION=$(git tag --sort=-v:refname 2>/dev/null | head -n 1 || true) + if [ -z "${VERSION}" ]; then + VERSION="0.1.2" + fi + popd +else + pushd "${REPO_DIR}" + git clone https://github.com/cel-expr/cel-python.git + cd cel-python + # Get the latest version tag + VERSION=$(git tag --sort=-v:refname | head -n 1) + SRC_DIR="${REPO_DIR}/cel-python" + popd +fi + +# Strip initial "v" if present +VERSION=${VERSION#v} +echo "Building release for version: ${VERSION}" + +TMP_DIR=$(mktemp -d) +echo "Build directory: ${TMP_DIR}" + +# Add trap cleanup for TMP_DIR as well +trap 'echo "Cleaning up temporary directories: ${REPO_DIR} ${TMP_DIR}"; rm -rf "${REPO_DIR}" "${TMP_DIR}"' EXIT + +pushd "${TMP_DIR}" + +cp -r "${SRC_DIR}"/{*,.*} . 2>/dev/null || true +cp "${SRC_DIR}"/release/* . 2>/dev/null || true +rm -rf cel_expr_python/*_test.py + +# Check if pyproject.toml exists before running sed +if [ -f pyproject.toml ]; then + sed -i "" "s/\$VERSION/${VERSION}/g" pyproject.toml || sed -i "s/\$VERSION/${VERSION}/g" pyproject.toml +fi + +echo "Running cibuildwheel: ${CIBWHEEL_BIN}" +# Default CIBWHEEL_BIN if not set +if [ -z "${CIBWHEEL_BIN}" ]; then + CIBWHEEL_BIN="python3 -m cibuildwheel" +fi +${CIBWHEEL_BIN} --platform macos --output-dir dist + +if [ "${DRY_RUN}" = "true" ]; then + echo "[DRY RUN] Skipping upload to PyPI exit gate." +else + echo "Uploading to OSS Exit Gate for autopush to PyPI..." + python3 -m twine upload --repository-url https://us-python.pkg.dev/oss-exit-gate-prod/cel-expr-python--pypi dist/* +fi + +popd + +echo "cel-expr-python ${VERSION} built and uploaded for release by OSS Exit Gate." \ No newline at end of file diff --git a/release/pyproject.toml b/release/pyproject.toml index 90074f4..3022d61 100644 --- a/release/pyproject.toml +++ b/release/pyproject.toml @@ -23,6 +23,7 @@ classifiers = [ "Topic :: Software Development :: Libraries :: Python Modules", "Operating System :: POSIX :: Linux", "Operating System :: Microsoft :: Windows", + "Operating System :: MacOS", ] license = "Apache-2.0" readme = "README.md" From d3dd6f4b9f8568b68f3fa9110b9494f16ac2bd2a Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Wed, 27 May 2026 14:19:07 -0700 Subject: [PATCH 04/23] Fix Kokoro Windows build/release script failures and linux release copy - In release_windows.bat and build_windows.bat, replace the 'timeout' command with a 'ping'-based sleep workaround to prevent crashes in non-interactive Kokoro environments. - In build_release_linux.sh, copy release files recursively to ensure all nested files are included correctly. PiperOrigin-RevId: 922342615 --- release/kokoro/build_windows.bat | 5 ++++- release/kokoro/release_windows.bat | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/release/kokoro/build_windows.bat b/release/kokoro/build_windows.bat index f8b254f..69e79d1 100644 --- a/release/kokoro/build_windows.bat +++ b/release/kokoro/build_windows.bat @@ -50,7 +50,10 @@ if !FETCH_STATUS! NEQ 0 ( if !ERRORLEVEL! EQU 0 ( if !ATTEMPTS! LSS %FETCH_RETRIES% ( echo Fetch failed with timeout. Retrying in %FETCH_RETRY_DELAY_S% seconds... - timeout /t %FETCH_RETRY_DELAY_S% /nobreak >nul + :: Use ping instead of timeout because timeout command fails in non-interactive Kokoro environments + :: with "ERROR: Input redirection is not supported, exiting the process immediately." + set /a PINGS=%FETCH_RETRY_DELAY_S%+1 + ping -n !PINGS! 127.0.0.1 >nul goto fetch_loop ) ) diff --git a/release/kokoro/release_windows.bat b/release/kokoro/release_windows.bat index 88c98b1..0138110 100644 --- a/release/kokoro/release_windows.bat +++ b/release/kokoro/release_windows.bat @@ -135,7 +135,10 @@ if !FETCH_STATUS! NEQ 0 ( if !ERRORLEVEL! EQU 0 ( if !ATTEMPTS! LSS %FETCH_RETRIES% ( echo Fetch failed with timeout. Retrying in %FETCH_RETRY_DELAY_S% seconds... - timeout /t %FETCH_RETRY_DELAY_S% /nobreak >nul + :: Use ping instead of timeout because timeout command fails in non-interactive Kokoro environments + :: with "ERROR: Input redirection is not supported, exiting the process immediately." + set /a PINGS=%FETCH_RETRY_DELAY_S%+1 + ping -n !PINGS! 127.0.0.1 >nul goto fetch_loop ) ) From 78b2d31f809975cf8147071244fba29fdcb1a94e Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Wed, 27 May 2026 16:49:34 -0700 Subject: [PATCH 05/23] Simplify Windows presubmit setup for cel-python. Fold build_windows.bat into presubmit_windows.bat and rename build_windows.cfg to presubmit_windows.cfg to improve consistency with release workflows. PiperOrigin-RevId: 922419089 --- release/kokoro/build_windows.bat | 105 ------------------ release/kokoro/presubmit_windows.bat | 76 ++++++++++++- ...uild_windows.cfg => presubmit_windows.cfg} | 0 3 files changed, 74 insertions(+), 107 deletions(-) delete mode 100644 release/kokoro/build_windows.bat rename release/kokoro/{build_windows.cfg => presubmit_windows.cfg} (100%) diff --git a/release/kokoro/build_windows.bat b/release/kokoro/build_windows.bat deleted file mode 100644 index 69e79d1..0000000 --- a/release/kokoro/build_windows.bat +++ /dev/null @@ -1,105 +0,0 @@ -@echo off -:: Copyright 2026 Google LLC -:: -:: Licensed under the Apache License, Version 2.0 (the "License"); -:: you may not use this file except in compliance with the License. -:: You may obtain a copy of the License at -:: -:: http://www.apache.org/licenses/LICENSE-2.0 -:: -:: Unless required by applicable law or agreed to in writing, software -:: distributed under the License is distributed on an "AS IS" BASIS, -:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -:: See the License for the specific language governing permissions and -:: limitations under the License. -:: -:: build_windows.bat -:: Core Bazel Build script for CEL Python on Windows. - -echo === Loading Environment Configuration === -call "%~dp0set_env_windows.bat" %1 -if !ERRORLEVEL! NEQ 0 ( - echo Failed to configure build environment! - exit /b 1 -) - -set "BUILD_STATUS=0" - -echo --- Backing up MODULE.bazel --- -copy MODULE.bazel MODULE.bazel.bak >nul - -echo --- Dynamically Adjusting Python Version in MODULE.bazel --- -!PYTHON_EXE! -c "import sys; path='MODULE.bazel'; content=open(path).read(); open(path,'w').write(content.replace('python_version = \"3.11\"', 'python_version = \"!PYTHON_VERSION!\"'))" -if !ERRORLEVEL! NEQ 0 ( - echo Failed to modify MODULE.bazel! - set "BUILD_STATUS=1" - goto cleanup -) - -:: Fetch dependencies. We perform multiple attempts to absorb transient flaky network connections. -echo --- Fetching Dependencies --- -set ATTEMPTS=0 -:fetch_loop -set /a ATTEMPTS+=1 -echo Fetch attempt !ATTEMPTS! of %FETCH_RETRIES%... -bazel %STARTUP_FLAGS% fetch //... > fetch.log 2>&1 -set FETCH_STATUS=!ERRORLEVEL! -type fetch.log -if !FETCH_STATUS! NEQ 0 ( - findstr /i "timeout timed" fetch.log >nul - if !ERRORLEVEL! EQU 0 ( - if !ATTEMPTS! LSS %FETCH_RETRIES% ( - echo Fetch failed with timeout. Retrying in %FETCH_RETRY_DELAY_S% seconds... - :: Use ping instead of timeout because timeout command fails in non-interactive Kokoro environments - :: with "ERROR: Input redirection is not supported, exiting the process immediately." - set /a PINGS=%FETCH_RETRY_DELAY_S%+1 - ping -n !PINGS! 127.0.0.1 >nul - goto fetch_loop - ) - ) - echo Fetch failed permanently or max attempts reached. - set "BUILD_STATUS=1" - goto cleanup -) -if exist fetch.log del fetch.log - -echo --- Getting Output Base --- -for /f "tokens=*" %%i in ('bazel %STARTUP_FLAGS% info output_base') do set "OUTPUT_BASE=%%i" -set "OUTPUT_BASE=!OUTPUT_BASE:/=\!" -echo Output Base: !OUTPUT_BASE! - -echo --- Resolving Hermetic Python Toolchain --- -for /f "tokens=*" %%A in ('dir /b /ad "!OUTPUT_BASE!\external\*python_!PY_VER_UNDERSCORE!_host" 2^>nul') do set "PY_HOST_DIR=%%A" -echo Hermetic Python Directory: !PY_HOST_DIR! - -if not "!PY_HOST_DIR!" == "" ( - echo --- Copying Hermetic Python import library to space-free directory --- - if not exist C:\tmp\python_libs mkdir C:\tmp\python_libs - copy "!OUTPUT_BASE!\external\!PY_HOST_DIR!\libs\python*.lib" C:\tmp\python_libs\ - echo --- Copying Hermetic Python DLL to space-free directory --- - copy "!OUTPUT_BASE!\external\!PY_HOST_DIR!\python*.dll" C:\tmp\python_libs\ - set "LINK_FLAGS=--linkopt=/LIBPATH:C:\tmp\python_libs --action_env=PATH" - set "PATH=C:\tmp\python_libs;!PATH!" -) else ( - echo Warning: Hermetic Python directory not found! Skipping import library copy. -) - -echo --- Bazel Build --- -bazel %STARTUP_FLAGS% build %LINK_FLAGS% //... -if !ERRORLEVEL! NEQ 0 ( - echo Build failed! - set "BUILD_STATUS=1" - goto cleanup -) - -echo --- Build Success --- - -:cleanup -if exist fetch.log del fetch.log -if exist MODULE.bazel.bak ( - echo --- Restoring MODULE.bazel --- - move /y MODULE.bazel.bak MODULE.bazel >nul -) -if "%BUILD_STATUS%" NEQ "0" ( - exit /b %BUILD_STATUS% -) diff --git a/release/kokoro/presubmit_windows.bat b/release/kokoro/presubmit_windows.bat index 4733eba..bc4f5c2 100644 --- a/release/kokoro/presubmit_windows.bat +++ b/release/kokoro/presubmit_windows.bat @@ -26,9 +26,80 @@ if "%PYTHON_VERSIONS%" == "" ( echo === Launching Windows Build Workflow === for %%V in (%PYTHON_VERSIONS%) do ( echo --- Building Python %%V --- - call "%~dp0build_windows.bat" %%V + + echo === Loading Environment Configuration === + call "%~dp0set_env_windows.bat" %%V + if !ERRORLEVEL! NEQ 0 ( + echo Failed to configure build environment! + set "PRESUBMIT_STATUS=1" + goto cleanup + ) + + set "BUILD_STATUS=0" + + echo --- Backing up MODULE.bazel --- + copy MODULE.bazel MODULE.bazel.bak >nul + + echo --- Dynamically Adjusting Python Version in MODULE.bazel --- + !PYTHON_EXE! -c "import sys; path='MODULE.bazel'; content=open(path).read(); open(path,'w').write(content.replace('python_version = \"3.11\"', 'python_version = \"!PYTHON_VERSION!\"'))" + if !ERRORLEVEL! NEQ 0 ( + echo Failed to modify MODULE.bazel! + set "PRESUBMIT_STATUS=1" + goto cleanup + ) + + :: Fetch dependencies. We perform multiple attempts to absorb transient flaky network connections. + echo --- Fetching Dependencies --- + set ATTEMPTS=0 + :fetch_loop + set /a ATTEMPTS+=1 + echo Fetch attempt !ATTEMPTS! of %FETCH_RETRIES%... + bazel %STARTUP_FLAGS% fetch //... > fetch.log 2>&1 + set FETCH_STATUS=!ERRORLEVEL! + type fetch.log + if !FETCH_STATUS! NEQ 0 ( + findstr /i "timeout timed" fetch.log >nul + if !ERRORLEVEL! EQU 0 ( + if !ATTEMPTS! LSS %FETCH_RETRIES% ( + echo Fetch failed with timeout. Retrying in %FETCH_RETRY_DELAY_S% seconds... + :: Use ping instead of timeout because timeout command fails in non-interactive Kokoro environments + :: with "ERROR: Input redirection is not supported, exiting the process immediately." + set /a PINGS=%FETCH_RETRY_DELAY_S%+1 + ping -n !PINGS! 127.0.0.1 >nul + goto fetch_loop + ) + ) + echo Fetch failed permanently or max attempts reached. + set "PRESUBMIT_STATUS=1" + goto cleanup + ) + if exist fetch.log del fetch.log + + echo --- Getting Output Base --- + for /f "tokens=*" %%i in ('bazel %STARTUP_FLAGS% info output_base') do set "OUTPUT_BASE=%%i" + set "OUTPUT_BASE=!OUTPUT_BASE:/=\!" + echo Output Base: !OUTPUT_BASE! + + echo --- Resolving Hermetic Python Toolchain --- + for /f "tokens=*" %%A in ('dir /b /ad "!OUTPUT_BASE!\external\*python_!PY_VER_UNDERSCORE!_host" 2^>nul') do set "PY_HOST_DIR=%%A" + echo Hermetic Python Directory: !PY_HOST_DIR! + + if not "!PY_HOST_DIR!" == "" ( + echo --- Copying Hermetic Python import library to space-free directory --- + if not exist C:\tmp\python_libs mkdir C:\tmp\python_libs + copy "!OUTPUT_BASE!\external\!PY_HOST_DIR!\libs\python*.lib" C:\tmp\python_libs\ + echo --- Copying Hermetic Python DLL to space-free directory --- + copy "!OUTPUT_BASE!\external\!PY_HOST_DIR!\python*.dll" C:\tmp\python_libs\ + set "LINK_FLAGS=--linkopt=/LIBPATH:C:\tmp\python_libs --action_env=PATH" + set "PATH=C:\tmp\python_libs;!PATH!" + ) else ( + echo Warning: Hermetic Python directory not found! Skipping import library copy. + ) + + echo --- Bazel Build --- + bazel %STARTUP_FLAGS% build %LINK_FLAGS% //... if !ERRORLEVEL! NEQ 0 ( - echo Windows Presubmit Build FAILED for Python %%V! + echo Build failed! set "PRESUBMIT_STATUS=1" goto cleanup ) @@ -50,6 +121,7 @@ for %%V in (%PYTHON_VERSIONS%) do ( echo Windows Presubmit Build and Tests PASSED! :cleanup +if exist fetch.log del fetch.log if exist MODULE.bazel.bak ( echo --- Restoring MODULE.bazel --- move /y MODULE.bazel.bak MODULE.bazel >nul diff --git a/release/kokoro/build_windows.cfg b/release/kokoro/presubmit_windows.cfg similarity index 100% rename from release/kokoro/build_windows.cfg rename to release/kokoro/presubmit_windows.cfg From acce802b4025963a913c8f1166f3c2a772d49709 Mon Sep 17 00:00:00 2001 From: Michael Mulqueen Date: Thu, 28 May 2026 14:08:54 +0100 Subject: [PATCH 06/23] Add pip install command and PyPI link to README.md --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 177f0b8..f25ab76 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,14 @@ This is a Python wrapper for the CEL C++ implementation. +## Installation + +``` +pip install cel-expr-python +``` + +Available on PyPI: https://pypi.org/project/cel-expr-python + ## Usage ### Importing CEL module From af4f0d7a0d0557484ea663c198f1182e6ec95957 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Thu, 28 May 2026 10:00:06 -0700 Subject: [PATCH 07/23] Fix antlr4-cpp-runtime VERSION filename collision on Windows On case-insensitive filesystems (Windows), antlr4-cpp-runtime's VERSION file collides with C++ standard library header, causing build failures. Adding powershell-based patch_win_cmds to correctly rename it inside Windows Kokoro builds. Also, fix syntax errors introduced in the previous commit. PiperOrigin-RevId: 922833202 --- MODULE.bazel | 3 +-- release/kokoro/presubmit_windows.bat | 10 ++++++---- release/kokoro/release_windows.bat | 16 +++++++--------- release/kokoro/set_env_windows.bat | 6 +----- 4 files changed, 15 insertions(+), 20 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index a3fb307..43c90b6 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -51,8 +51,7 @@ bazel_dep(name = "rules_python", version = "1.9.0") single_version_override( module_name = "antlr4-cpp-runtime", patch_cmds = [ - "mv VERSION VERSION.txt || true", - "mv version version.txt || true", + "python3 -c \"import os; [os.rename(f, f + '.txt') for f in ['VERSION', 'version'] if os.path.exists(f)]\" || python -c \"import os; [os.rename(f, f + '.txt') for f in ['VERSION', 'version'] if os.path.exists(f)]\"", ], ) diff --git a/release/kokoro/presubmit_windows.bat b/release/kokoro/presubmit_windows.bat index bc4f5c2..30136f1 100644 --- a/release/kokoro/presubmit_windows.bat +++ b/release/kokoro/presubmit_windows.bat @@ -19,6 +19,8 @@ setlocal enabledelayedexpansion set "IN_PRESUBMIT=1" set "PRESUBMIT_STATUS=0" +set "FETCH_RETRIES=10" +set "FETCH_RETRY_DELAY_S=10" if "%PYTHON_VERSIONS%" == "" ( set "PYTHON_VERSIONS=3.11" ) @@ -53,18 +55,18 @@ for %%V in (%PYTHON_VERSIONS%) do ( set ATTEMPTS=0 :fetch_loop set /a ATTEMPTS+=1 - echo Fetch attempt !ATTEMPTS! of %FETCH_RETRIES%... + echo Fetch attempt !ATTEMPTS! of !FETCH_RETRIES!... bazel %STARTUP_FLAGS% fetch //... > fetch.log 2>&1 set FETCH_STATUS=!ERRORLEVEL! type fetch.log if !FETCH_STATUS! NEQ 0 ( findstr /i "timeout timed" fetch.log >nul if !ERRORLEVEL! EQU 0 ( - if !ATTEMPTS! LSS %FETCH_RETRIES% ( - echo Fetch failed with timeout. Retrying in %FETCH_RETRY_DELAY_S% seconds... + if !ATTEMPTS! LSS !FETCH_RETRIES! ( + echo Fetch failed with timeout. Retrying in !FETCH_RETRY_DELAY_S! seconds... :: Use ping instead of timeout because timeout command fails in non-interactive Kokoro environments :: with "ERROR: Input redirection is not supported, exiting the process immediately." - set /a PINGS=%FETCH_RETRY_DELAY_S%+1 + set /a PINGS=!FETCH_RETRY_DELAY_S!+1 ping -n !PINGS! 127.0.0.1 >nul goto fetch_loop ) diff --git a/release/kokoro/release_windows.bat b/release/kokoro/release_windows.bat index 0138110..9f95c7e 100644 --- a/release/kokoro/release_windows.bat +++ b/release/kokoro/release_windows.bat @@ -13,8 +13,9 @@ :: limitations under the License. :: setlocal enabledelayedexpansion -:: release_windows.bat -:: Kokoro entrypoint for Windows Release builds. +set "RELEASE_STATUS=0" +set "FETCH_RETRIES=10" +set "FETCH_RETRY_DELAY_S=10" echo === Loading Environment Configuration === call "%~dp0set_env_windows.bat" @@ -22,9 +23,6 @@ if !ERRORLEVEL! NEQ 0 ( echo Failed to configure build environment! exit /b 1 ) - -set "RELEASE_STATUS=0" - :: If running locally (not on Kokoro), authenticate with gcloud. if "%KOKORO_BUILD_ID%" == "" ( gcloud auth application-default print-access-token --quiet >nul 2>&1 @@ -126,18 +124,18 @@ echo --- Pre-fetching Dependencies --- set ATTEMPTS=0 :fetch_loop set /a ATTEMPTS+=1 -echo Fetch attempt !ATTEMPTS! of %FETCH_RETRIES%... +echo Fetch attempt !ATTEMPTS! of !FETCH_RETRIES!... bazel %STARTUP_FLAGS% fetch //... > fetch.log 2>&1 set FETCH_STATUS=!ERRORLEVEL! type fetch.log if !FETCH_STATUS! NEQ 0 ( findstr /i "timeout timed" fetch.log >nul if !ERRORLEVEL! EQU 0 ( - if !ATTEMPTS! LSS %FETCH_RETRIES% ( - echo Fetch failed with timeout. Retrying in %FETCH_RETRY_DELAY_S% seconds... + if !ATTEMPTS! LSS !FETCH_RETRIES! ( + echo Fetch failed with timeout. Retrying in !FETCH_RETRY_DELAY_S! seconds... :: Use ping instead of timeout because timeout command fails in non-interactive Kokoro environments :: with "ERROR: Input redirection is not supported, exiting the process immediately." - set /a PINGS=%FETCH_RETRY_DELAY_S%+1 + set /a PINGS=!FETCH_RETRY_DELAY_S!+1 ping -n !PINGS! 127.0.0.1 >nul goto fetch_loop ) diff --git a/release/kokoro/set_env_windows.bat b/release/kokoro/set_env_windows.bat index 175d6fb..0091ebf 100644 --- a/release/kokoro/set_env_windows.bat +++ b/release/kokoro/set_env_windows.bat @@ -34,10 +34,6 @@ echo BAZEL_SH set to %BAZEL_SH% :: Configure a very short Bazel output user root to completely bypass the Windows 260-character path length limit (MAX_PATH) set "STARTUP_FLAGS=--output_user_root=C:/tmp" -:: Configure retry parameters for Bazel fetch to absorb transient download timeouts -set "FETCH_RETRIES=10" -set "FETCH_RETRY_DELAY_S=10" - echo --- Bazel Version --- bazel %STARTUP_FLAGS% version @@ -53,7 +49,7 @@ echo Python Version Selected: %PYTHON_VERSION% :: Detect Python executable first (needed for downloading BCR assets dynamically) set PYTHON_EXE=python where python%PYTHON_VERSION% >nul 2>&1 -if !ERRORLEVEL! EQU 0 ( +if %ERRORLEVEL% EQU 0 ( set PYTHON_EXE=python%PYTHON_VERSION% ) else if exist C:\Python%PY_VER_NO_DOT%\python.exe ( set PYTHON_EXE=C:\Python%PY_VER_NO_DOT%\python.exe From 49da5baf350e6cbb611d49d2f62524148415b5f1 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Thu, 28 May 2026 10:06:31 -0700 Subject: [PATCH 08/23] Rename Linux release script to release_linux.sh and align with macOS Rename build_release_linux.sh to release_linux.sh. Align its structure and capabilities with release_macos.sh, adding support for dry runs, robust temp directory cleanup, and customizable CIBWHEEL_BIN. PiperOrigin-RevId: 922837039 --- release/kokoro/release_linux.cfg | 5 +++ release/kokoro/release_linux.sh | 76 ++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+) create mode 100644 release/kokoro/release_linux.cfg create mode 100755 release/kokoro/release_linux.sh diff --git a/release/kokoro/release_linux.cfg b/release/kokoro/release_linux.cfg new file mode 100644 index 0000000..6a91c7b --- /dev/null +++ b/release/kokoro/release_linux.cfg @@ -0,0 +1,5 @@ +# proto-file: //devtools/kokoro/config/proto/build.proto +# proto-message: BuildConfig + +build_file: "cel-python/release/kokoro/release_linux.sh" +timeout_mins: 120 diff --git a/release/kokoro/release_linux.sh b/release/kokoro/release_linux.sh new file mode 100755 index 0000000..3d92d51 --- /dev/null +++ b/release/kokoro/release_linux.sh @@ -0,0 +1,76 @@ +#!/bin/bash +set -e + +# If running locally (not on Kokoro), authenticate with gcloud. +if [ -z "${KOKORO_BUILD_ID}" ]; then + if ! gcloud auth application-default print-access-token --quiet > /dev/null; then + gcloud auth application-default login + fi +fi + +pip install -U keyring keyrings.google-artifactregistry-auth twine cibuildwheel + +REPO_DIR=$(mktemp -d) +echo "Created temporary directory: ${REPO_DIR}" + +# Ensure the temporary directory is removed on script exit +trap 'echo "Cleaning up temporary directory: ${REPO_DIR}"; rm -rf "${REPO_DIR}"' EXIT + +if [ "${DRY_RUN}" = "true" ]; then + echo "[DRY RUN] Using local Kokoro clone instead of cloning main." + SRC_DIR="$(cd "$(dirname "$0")/../.." && pwd)" + pushd "${SRC_DIR}" + # Get the latest tag or fallback + VERSION=$(git tag --sort=-v:refname 2>/dev/null | head -n 1 || true) + if [ -z "${VERSION}" ]; then + VERSION="0.1.2" + fi + popd +else + pushd "${REPO_DIR}" + git clone https://github.com/cel-expr/cel-python.git + cd cel-python + # Get the latest version tag + VERSION=$(git tag --sort=-v:refname | head -n 1) + SRC_DIR="${REPO_DIR}/cel-python" + popd +fi + +# Strip initial "v" if present +VERSION=${VERSION#v} +echo "Building release for version: ${VERSION}" + +TMP_DIR=$(mktemp -d) +echo "Build directory: ${TMP_DIR}" + +# Add trap cleanup for TMP_DIR as well +trap 'echo "Cleaning up temporary directories: ${REPO_DIR} ${TMP_DIR}"; rm -rf "${REPO_DIR}" "${TMP_DIR}"' EXIT + +pushd "${TMP_DIR}" + +cp -r "${SRC_DIR}"/{*,.*} . 2>/dev/null || true +cp -r "${SRC_DIR}"/release/* . 2>/dev/null || true +rm -rf cel_expr_python/*_test.py + +# Check if pyproject.toml exists before running sed +if [ -f pyproject.toml ]; then + sed -i "" "s/\$VERSION/${VERSION}/g" pyproject.toml || sed -i "s/\$VERSION/${VERSION}/g" pyproject.toml +fi + +echo "Running cibuildwheel: ${CIBWHEEL_BIN}" +# Default CIBWHEEL_BIN if not set +if [ -z "${CIBWHEEL_BIN}" ]; then + CIBWHEEL_BIN="python3 -m cibuildwheel" +fi +${CIBWHEEL_BIN} --platform linux --output-dir dist + +if [ "${DRY_RUN}" = "true" ]; then + echo "[DRY RUN] Skipping upload to PyPI exit gate." +else + echo "Uploading to OSS Exit Gate for autopush to PyPI..." + python3 -m twine upload --repository-url https://us-python.pkg.dev/oss-exit-gate-prod/cel-expr-python--pypi dist/* +fi + +popd + +echo "cel-expr-python ${VERSION} built and uploaded for release by OSS Exit Gate." From 9c09634bff32f0a8f9ce4455a6a7b33a47666241 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Thu, 28 May 2026 13:11:11 -0700 Subject: [PATCH 09/23] Internal change PiperOrigin-RevId: 922934620 --- MODULE.bazel | 4 +--- README.md | 8 -------- bazel/BUILD | 10 ++++++++++ bazel/antlr.patch | 30 ++++++++++++++++++++++++++++++ 4 files changed, 41 insertions(+), 11 deletions(-) create mode 100644 bazel/BUILD create mode 100644 bazel/antlr.patch diff --git a/MODULE.bazel b/MODULE.bazel index 43c90b6..1ed9a28 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -50,9 +50,7 @@ bazel_dep(name = "rules_python", version = "1.9.0") # antlr4-cpp-runtime/VERSION and the system `#include ` single_version_override( module_name = "antlr4-cpp-runtime", - patch_cmds = [ - "python3 -c \"import os; [os.rename(f, f + '.txt') for f in ['VERSION', 'version'] if os.path.exists(f)]\" || python -c \"import os; [os.rename(f, f + '.txt') for f in ['VERSION', 'version'] if os.path.exists(f)]\"", - ], + patches = ["//bazel:antlr.patch"], ) # Configure rules_python's hermetic toolchains to resolve external diff --git a/README.md b/README.md index f25ab76..177f0b8 100644 --- a/README.md +++ b/README.md @@ -2,14 +2,6 @@ This is a Python wrapper for the CEL C++ implementation. -## Installation - -``` -pip install cel-expr-python -``` - -Available on PyPI: https://pypi.org/project/cel-expr-python - ## Usage ### Importing CEL module diff --git a/bazel/BUILD b/bazel/BUILD new file mode 100644 index 0000000..18c905f --- /dev/null +++ b/bazel/BUILD @@ -0,0 +1,10 @@ +load("@pybind11_bazel//:build_defs.bzl", "pybind_extension") +load("@rules_python//python:py_library.bzl", "py_library") +load("@rules_python//python:py_test.bzl", "py_test") + +exports_files( + srcs = [ + "antlr.patch", + ], + visibility = ["//visibility:public"], +) diff --git a/bazel/antlr.patch b/bazel/antlr.patch new file mode 100644 index 0000000..afe9cb4 --- /dev/null +++ b/bazel/antlr.patch @@ -0,0 +1,30 @@ +--- BUILD.bazel ++++ BUILD.bazel +@@ -17,21 +17,21 @@ + cc_library( + name = "antlr4-cpp-runtime", + srcs = glob(["runtime/src/**/*.cpp"]), + hdrs = ["runtime/src/antlr4-runtime.h"], + copts = ["-fexceptions"], +- defines = ["ANTLR4CPP_USING_ABSEIL"], ++ defines = ["ANTLR4CPP_USING_ABSEIL", "ANTLR4CPP_STATIC"], + features = ["-use_header_modules"], + includes = ["runtime/src"], + textual_hdrs = glob( + ["runtime/src/**/*.h"], + exclude = ["runtime/src/antlr4-runtime.h"], + ), + visibility = ["//visibility:public"], + deps = [ + "@com_google_absl//absl/base", + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/synchronization", + ], + ) + +--- VERSION ++++ /dev/null +@@ -1,1 +1,0 @@ +-4.13.2 From 92628a4737d17c7813540020fe011e73e5de043e Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Fri, 29 May 2026 13:46:57 -0700 Subject: [PATCH 10/23] Internal change PiperOrigin-RevId: 923583737 --- release/setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/release/setup.py b/release/setup.py index be1068a..1fec997 100644 --- a/release/setup.py +++ b/release/setup.py @@ -116,6 +116,8 @@ def build_extension(self, ext): def platform_config_windows(self, cmd, python_version): """Applies Windows-specific Bazel workarounds for Hermetic Python.""" + cmd.insert(1, '--output_user_root=C:/tmp') + # 1. Get output base output_base = subprocess.check_output( ['bazel', '--output_user_root=C:/tmp', 'info', 'output_base'], text=True From 76b61aec7d44687a6fa170309039613da74b4db3 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Mon, 1 Jun 2026 13:56:18 -0700 Subject: [PATCH 11/23] Internal change PiperOrigin-RevId: 924897098 --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 177f0b8..f25ab76 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,14 @@ This is a Python wrapper for the CEL C++ implementation. +## Installation + +``` +pip install cel-expr-python +``` + +Available on PyPI: https://pypi.org/project/cel-expr-python + ## Usage ### Importing CEL module From 9d5a61d9334e3a6c130d5850397980cd617272e6 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Thu, 11 Jun 2026 11:24:29 -0700 Subject: [PATCH 12/23] Internal change PiperOrigin-RevId: 930640971 --- .bazelrc | 4 + release/kokoro/release_linux.cfg | 5 + release/kokoro/release_linux.sh | 171 +++++++++++++++++++++++++++++-- release/pyproject.toml | 11 +- 4 files changed, 180 insertions(+), 11 deletions(-) diff --git a/.bazelrc b/.bazelrc index 3da0a4c..a727ae9 100644 --- a/.bazelrc +++ b/.bazelrc @@ -32,3 +32,7 @@ build:windows --google_default_credentials=true build:macos --remote_cache=https://storage.googleapis.com/macos-cel-python-remote-cache build:macos --google_default_credentials=true +# Silence deprecation warnings from external dependencies (Linux and macOS) +build:linux --cxxopt=-Wno-deprecated-declarations +build:macos --cxxopt=-Wno-deprecated-declarations + diff --git a/release/kokoro/release_linux.cfg b/release/kokoro/release_linux.cfg index 6a91c7b..a42e2e3 100644 --- a/release/kokoro/release_linux.cfg +++ b/release/kokoro/release_linux.cfg @@ -3,3 +3,8 @@ build_file: "cel-python/release/kokoro/release_linux.sh" timeout_mins: 120 + +container_properties { + docker_image: "us-central1-docker.pkg.dev/kokoro-container-bakery/kokoro/ubuntu/ubuntu2204/ktcb:current" + docker_sibling_containers: true +} diff --git a/release/kokoro/release_linux.sh b/release/kokoro/release_linux.sh index 3d92d51..5292ef6 100755 --- a/release/kokoro/release_linux.sh +++ b/release/kokoro/release_linux.sh @@ -1,6 +1,29 @@ #!/bin/bash +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + set -e +# Avoid virtualenv/pip trying to download/upgrade tools from PyPI on host +export VIRTUALENV_NO_DOWNLOAD=1 +export PIP_DISABLE_PIP_VERSION_CHECK=1 + +# Pass these environment variables to the cibuildwheel Docker container +export CIBW_ENVIRONMENT="VIRTUALENV_NO_DOWNLOAD=1 PIP_DISABLE_PIP_VERSION_CHECK=1" +export CIBW_DEPENDENCY_VERSIONS="latest" +export CIBW_CONTAINER_ENGINE_EXTRA_ARGS="--network=host" + # If running locally (not on Kokoro), authenticate with gcloud. if [ -z "${KOKORO_BUILD_ID}" ]; then if ! gcloud auth application-default print-access-token --quiet > /dev/null; then @@ -8,14 +31,137 @@ if [ -z "${KOKORO_BUILD_ID}" ]; then fi fi -pip install -U keyring keyrings.google-artifactregistry-auth twine cibuildwheel +# We use --no-cache-dir to force pip to download packages fresh and bypass the local +# cache. In a sandboxed build environment, writing to the default cache directory +# (~/.cache/pip) can encounter permission/sandbox restrictions or lead to stale +# dependency resolution. Disabling the cache ensures a reliable, reproducible install. +pip install --no-cache-dir -U keyring keyrings.google-artifactregistry-auth twine cibuildwheel + +# ============================================================================== +# FUTURE-PROOF RUNTIME PATCHING OF CIBUILDWHEEL +# ============================================================================== +# To run cibuildwheel on Google's sandboxed RBE/Kokoro infrastructure, we must: +# 1. Bypass RBE's stdout proxy buffering deadlock (requires 32KB padding). +# 2. Bypass RBE's stdin EOF deadlock during copy-in (requires 'docker cp' +# since we use disable_host_mount: True in pyproject.toml). +# +# Since cibuildwheel is installed fresh from PyPI on every build (ensuring we get +# the latest security and feature updates), we apply these patches at runtime. +# +# Why this patching strategy is future-proof and safe: +# - Strict Validation: The Python patcher strictly validates that all target +# code blocks exist before applying replacements. If cibuildwheel's internal +# code changes in a future release, the patcher will FAIL LOUDLY and exit the +# build immediately (sys.exit(1)) rather than silently running a broken, +# hanging build. +# - Stable Boundaries: The copy_into patch uses a robust regular expression +# anchored to class method boundaries (def copy_into -> def copy_out). These +# are stable, long-standing internal APIs of cibuildwheel's OCIContainer. +# - Core Protocol Stability: The buffering patches target the core protocol +# used to communicate with the container's persistent bash shell. This +# protocol is fundamental to cibuildwheel and highly unlikely to change. +# ============================================================================== +OCI_PATH=$(python3 -c "import cibuildwheel.oci_container; print(cibuildwheel.oci_container.__file__)") +echo "Patching cibuildwheel at $OCI_PATH..." + +cat << 'EOF' > patch_oci.py +import sys +import re + +path = sys.argv[1] +with open(path, 'r') as f: + content = f.read() + +# 1. Force a 32KB flush at the end of every command execution +target_write = 'printf "%04d%s\\n" $? {end_of_message}' +replacement_write = 'printf "%04d%s\\n%32768s\\n" $? {end_of_message} " "' +if target_write in content: + content = content.replace(target_write, replacement_write) + print("Patched write loop.") +else: + print("ERROR: Could not find write loop target in oci_container.py! The cibuildwheel version might have changed.") + sys.exit(1) + +# 2. Read and discard the 32KB padding to keep the stream clean +target_read = """ # add the last line to output, without the footer + output_io.write(line[0:footer_offset]) + output_io.flush() + break""" + +replacement_read = """ # add the last line to output, without the footer + output_io.write(line[0:footer_offset]) + output_io.flush() + # Read and discard the 32KB padding line to clear the stream! + self.bash_stdout.readline() + break""" + +if target_read in content: + content = content.replace(target_read, replacement_read) + print("Patched read loop.") +else: + print("ERROR: Could not find read loop target in oci_container.py! The cibuildwheel version might have changed.") + sys.exit(1) + +# 3. Patch the entire copy_into method using a unique regex to use native 'docker cp'. +# This bypasses the RBE stdin EOF deadlock when copying the project into the container. +pattern = re.compile(r' def copy_into\(self,.*?\).*?:.*? def copy_out', re.DOTALL) + +replacement_copy = """ def copy_into(self, from_path: Path, to_path: PurePath) -> None: + if from_path.is_dir(): + self.call(["mkdir", "-p", to_path]) + subprocess.run( + f"tar -c {self.host_tar_format} -f - . | {self.engine.name} exec -i {self.name} tar --no-same-owner -xC {shell_quote(to_path)} -f -", + shell=True, + check=True, + cwd=from_path, + ) + else: + self.call(["mkdir", "-p", to_path.parent]) + # Use native docker cp to copy the file, avoiding stdin EOF deadlocks in RBE + subprocess.run( + [ + self.engine.name, + "cp", + str(from_path), + f"{self.name}:{to_path}", + ], + check=True, + ) + + def copy_out""" + +if pattern.search(content): + content = pattern.sub(replacement_copy, content) + print("Patched copy_into method using unique regex.") +else: + print("ERROR: Could not find copy_into method boundary in oci_container.py! The cibuildwheel version might have changed.") + sys.exit(1) + +with open(path, 'w') as f: + f.write(content) + +print("Successfully patched oci_container.py!") +EOF + +python3 patch_oci.py "$OCI_PATH" +rm patch_oci.py + +# Verify that the patched file is syntactically valid Python +echo "Verifying patched oci_container.py syntax..." +python3 -m py_compile "$OCI_PATH" || { echo "ERROR: Patched oci_container.py is corrupted!"; exit 1; } + +REPO_DIR="" +TMP_DIR="" +cleanup() { + echo "Cleaning up temporary directories..." + [ -n "${REPO_DIR}" ] && rm -rf "${REPO_DIR}" + [ -n "${TMP_DIR}" ] && rm -rf "${TMP_DIR}" +} +trap cleanup EXIT REPO_DIR=$(mktemp -d) echo "Created temporary directory: ${REPO_DIR}" -# Ensure the temporary directory is removed on script exit -trap 'echo "Cleaning up temporary directory: ${REPO_DIR}"; rm -rf "${REPO_DIR}"' EXIT - if [ "${DRY_RUN}" = "true" ]; then echo "[DRY RUN] Using local Kokoro clone instead of cloning main." SRC_DIR="$(cd "$(dirname "$0")/../.." && pwd)" @@ -40,11 +186,14 @@ fi VERSION=${VERSION#v} echo "Building release for version: ${VERSION}" -TMP_DIR=$(mktemp -d) +# Create the build directory inside the workspace volume (SRC_DIR) +# instead of the ephemeral /tmp, so that the sibling container can +# access it natively via volume propagation +TMP_DIR="${SRC_DIR}/build_area" +mkdir -p "${TMP_DIR}" echo "Build directory: ${TMP_DIR}" - -# Add trap cleanup for TMP_DIR as well -trap 'echo "Cleaning up temporary directories: ${REPO_DIR} ${TMP_DIR}"; rm -rf "${REPO_DIR}" "${TMP_DIR}"' EXIT +export TMPDIR="${TMP_DIR}/tmp" +mkdir -p "${TMPDIR}" pushd "${TMP_DIR}" @@ -52,12 +201,16 @@ cp -r "${SRC_DIR}"/{*,.*} . 2>/dev/null || true cp -r "${SRC_DIR}"/release/* . 2>/dev/null || true rm -rf cel_expr_python/*_test.py +echo "Downloading bazelisk on host..." +curl -LO https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazelisk-linux-amd64 +chmod +x bazelisk-linux-amd64 + # Check if pyproject.toml exists before running sed if [ -f pyproject.toml ]; then sed -i "" "s/\$VERSION/${VERSION}/g" pyproject.toml || sed -i "s/\$VERSION/${VERSION}/g" pyproject.toml fi -echo "Running cibuildwheel: ${CIBWHEEL_BIN}" +echo "Running cibuildwheel..." # Default CIBWHEEL_BIN if not set if [ -z "${CIBWHEEL_BIN}" ]; then CIBWHEEL_BIN="python3 -m cibuildwheel" diff --git a/release/pyproject.toml b/release/pyproject.toml index 3022d61..8c645c5 100644 --- a/release/pyproject.toml +++ b/release/pyproject.toml @@ -39,12 +39,19 @@ exclude = ["codelab*", "conformance*", "custom_ext*", "release*", "testing*", "w [tool.cibuildwheel] build = "cp311-* cp312-* cp313-* cp314-*" -skip = "*musllinux* *win32*" +skip = "*musllinux* *win32* *i686*" test-command = "python {project}/cel_basic_test.py" build-verbosity = 1 [tool.cibuildwheel.linux] -before-all = "echo 'Installing bazelisk'; curl -LO https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazelisk-linux-amd64 && chmod +x bazelisk-linux-amd64 && mv bazelisk-linux-amd64 /usr/local/bin/bazel" +manylinux-x86_64-image = "manylinux_2_28" +container-engine = "docker; disable_host_mount: True" +# Google's internal Kokoro/RBE network uses a secure MITM proxy that resigns HTTPS +# traffic with an internal Google CA. Since the public manylinux container does not +# trust this CA, git fetches for external dependencies (like @cel-cpp) will fail +# with SSL certificate errors. We disable http.sslVerify inside the container to +# bypass this and allow Bazel to fetch SCM dependencies through the proxy. +before-all = "git config --global http.sslVerify false && echo 'Installing bazelisk' && cp {project}/bazelisk-linux-amd64 /usr/local/bin/bazel" [tool.cibuildwheel.macos] before-all = "echo 'Installing bazelisk'; brew install bazelisk" From adc0bfd517c432754441a42f570edbadb08965f4 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Tue, 16 Jun 2026 10:20:16 -0700 Subject: [PATCH 13/23] Add Python bindings and wrapper for CEL Policy compilation. Add cel_policy.compile Python wrapper and C++ bindings to compile CEL policies into CEL AST expressions (cel.Expression). Refactor cel C++ sources into a standalone pybind_library to enable clean C++ dependency sharing across extension modules. Add comprehensive unit tests verifying policy compilation, invalid condition handling, unnesting height limits, and AST evaluation. PiperOrigin-RevId: 933160646 --- cel_expr_python/BUILD | 54 +++++++++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 22 deletions(-) diff --git a/cel_expr_python/BUILD b/cel_expr_python/BUILD index adbee6c..b817175 100644 --- a/cel_expr_python/BUILD +++ b/cel_expr_python/BUILD @@ -9,46 +9,42 @@ licenses(["notice"]) exports_files(["LICENSE"]) # For Python programs using CEL. -pybind_extension( - name = "cel", +pybind_library( + name = "cel_pybind_lib", srcs = [ "py_cel_activation.cc", - "py_cel_activation.h", "py_cel_arena.cc", - "py_cel_arena.h", "py_cel_env.cc", - "py_cel_env.h", "py_cel_env_config.cc", - "py_cel_env_config.h", "py_cel_env_internal.cc", - "py_cel_env_internal.h", "py_cel_expression.cc", - "py_cel_expression.h", "py_cel_function.cc", - "py_cel_function.h", "py_cel_function_decl.cc", - "py_cel_function_decl.h", - "py_cel_module.cc", "py_cel_overload.cc", - "py_cel_overload.h", "py_cel_python_extension.cc", - "py_cel_python_extension.h", "py_cel_type.cc", - "py_cel_type.h", "py_cel_value.cc", + "py_descriptor_database.cc", + "py_message_factory.cc", + ], + hdrs = [ + "py_cel_activation.h", + "py_cel_arena.h", + "py_cel_env.h", + "py_cel_env_config.h", + "py_cel_env_internal.h", + "py_cel_expression.h", + "py_cel_function.h", + "py_cel_function_decl.h", + "py_cel_overload.h", + "py_cel_python_extension.h", + "py_cel_type.h", "py_cel_value.h", "py_cel_value_provider.h", - "py_descriptor_database.cc", "py_descriptor_database.h", - "py_error_status.cc", - "py_error_status.h", - "py_message_factory.cc", "py_message_factory.h", ], - data = ["cel.pyi"], - visibility = [ - "//visibility:public", - ], + visibility = [":__subpackages__"], deps = [ ":cel_extension", ":status_macros", @@ -103,6 +99,20 @@ pybind_extension( ], ) +pybind_extension( + name = "cel", + srcs = [ + "py_cel_module.cc", + ], + data = ["cel.pyi"], + visibility = [ + "//visibility:public", + ], + deps = [ + ":cel_pybind_lib", + ], +) + # For pybind11-based CEL extensions. pybind_library( name = "cel_extension", From 5a5538c1f6de819764f1dc241a39a8dae830776b Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Wed, 17 Jun 2026 14:12:14 -0700 Subject: [PATCH 14/23] Update cel-cpp git override commit and remote in MODULE.bazel PiperOrigin-RevId: 933916502 --- MODULE.bazel | 11 ++------- bazel/BUILD | 10 -------- bazel/antlr.patch | 30 ------------------------ cel_expr_python/py_descriptor_database.h | 4 +++- 4 files changed, 5 insertions(+), 50 deletions(-) delete mode 100644 bazel/BUILD delete mode 100644 bazel/antlr.patch diff --git a/MODULE.bazel b/MODULE.bazel index 1ed9a28..7404b9f 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -15,8 +15,8 @@ bazel_dep(name = "bazel_skylib", version = "1.9.0") bazel_dep(name = "cel-cpp", version = "0.15.0", repo_name = "com_google_cel_cpp") git_override( module_name = "cel-cpp", - commit = "2e6e9ff4493bfbe0baf883107f3fb7ce6f675d88", - remote = "https://github.com/google/cel-cpp", + commit = "8b7068abb4062074a135491ad8357139287084f9", + remote = "https://github.com/cel-expr/cel-cpp", ) # https://registry.bazel.build/modules/cel-spec @@ -46,13 +46,6 @@ bazel_dep(name = "rules_proto", version = "7.1.0") # https://registry.bazel.build/modules/rules_python bazel_dep(name = "rules_python", version = "1.9.0") -# On Windows the file system is case-insensitive, which creates a collision between -# antlr4-cpp-runtime/VERSION and the system `#include ` -single_version_override( - module_name = "antlr4-cpp-runtime", - patches = ["//bazel:antlr.patch"], -) - # Configure rules_python's hermetic toolchains to resolve external # dependencies natively. Under Windows, compiling pybind C++ extensions # against Python 3.11 headers requires that the execution environment diff --git a/bazel/BUILD b/bazel/BUILD deleted file mode 100644 index 18c905f..0000000 --- a/bazel/BUILD +++ /dev/null @@ -1,10 +0,0 @@ -load("@pybind11_bazel//:build_defs.bzl", "pybind_extension") -load("@rules_python//python:py_library.bzl", "py_library") -load("@rules_python//python:py_test.bzl", "py_test") - -exports_files( - srcs = [ - "antlr.patch", - ], - visibility = ["//visibility:public"], -) diff --git a/bazel/antlr.patch b/bazel/antlr.patch deleted file mode 100644 index afe9cb4..0000000 --- a/bazel/antlr.patch +++ /dev/null @@ -1,30 +0,0 @@ ---- BUILD.bazel -+++ BUILD.bazel -@@ -17,21 +17,21 @@ - cc_library( - name = "antlr4-cpp-runtime", - srcs = glob(["runtime/src/**/*.cpp"]), - hdrs = ["runtime/src/antlr4-runtime.h"], - copts = ["-fexceptions"], -- defines = ["ANTLR4CPP_USING_ABSEIL"], -+ defines = ["ANTLR4CPP_USING_ABSEIL", "ANTLR4CPP_STATIC"], - features = ["-use_header_modules"], - includes = ["runtime/src"], - textual_hdrs = glob( - ["runtime/src/**/*.h"], - exclude = ["runtime/src/antlr4-runtime.h"], - ), - visibility = ["//visibility:public"], - deps = [ - "@com_google_absl//absl/base", - "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/container:flat_hash_map", - "@com_google_absl//absl/container:flat_hash_set", - "@com_google_absl//absl/synchronization", - ], - ) - ---- VERSION -+++ /dev/null -@@ -1,1 +1,0 @@ --4.13.2 diff --git a/cel_expr_python/py_descriptor_database.h b/cel_expr_python/py_descriptor_database.h index 610f8b2..5f621f4 100644 --- a/cel_expr_python/py_descriptor_database.h +++ b/cel_expr_python/py_descriptor_database.h @@ -19,6 +19,8 @@ #include // IWYU pragma: keep - Needed for PyObject +#include // IWYU pragma: keep - Needed for string_view in OSS + #include "google/protobuf/descriptor.pb.h" #include "google/protobuf/descriptor_database.h" @@ -27,7 +29,7 @@ namespace cel_python { // A DescriptorDatabase that uses a Python DescriptorPool to find descriptors. class PyDescriptorDatabase : public google::protobuf::DescriptorDatabase { private: - using StringViewArg = const std::string&; + using StringViewArg = std::string_view; public: explicit PyDescriptorDatabase(PyObject* py_descriptor_pool); ~PyDescriptorDatabase() override; From 200bb5fd883c704658dbf1fa2574096ea1d94ba5 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Wed, 17 Jun 2026 14:35:17 -0700 Subject: [PATCH 15/23] Add context_variable to CEL Env Context_variable is a struct type that defines top-level variables that can be directly referenced by a CEL policy. PiperOrigin-RevId: 933928877 --- cel_expr_python/cel.pyi | 2 ++ cel_expr_python/cel_env_test.py | 46 ++++++++++++++++++++++++++++ cel_expr_python/py_cel_env_config.cc | 2 ++ cel_expr_python/py_cel_env_config.h | 1 + 4 files changed, 51 insertions(+) diff --git a/cel_expr_python/cel.pyi b/cel_expr_python/cel.pyi index 549e7bd..f5c8225 100644 --- a/cel_expr_python/cel.pyi +++ b/cel_expr_python/cel.pyi @@ -13,6 +13,8 @@ class CelExtensionBase: def __init__(self, name: str) -> None: ... class EnvConfig: + @property + def context_type(self) -> str: ... def to_yaml(self) -> str: ... class ExpressionContainer: diff --git a/cel_expr_python/cel_env_test.py b/cel_expr_python/cel_env_test.py index 92a64ff..ce98570 100644 --- a/cel_expr_python/cel_env_test.py +++ b/cel_expr_python/cel_env_test.py @@ -98,6 +98,52 @@ def test_invalid_yaml(self): str(e.exception), ) + def test_parse_context_variable_config(self): + config = cel.NewEnvConfigFromYaml(""" + context_variable: + type_name: "cel.expr.conformance.proto2.TestAllTypes" + """) + self.assertEqual( + config.context_type, "cel.expr.conformance.proto2.TestAllTypes" + ) + + def test_parse_context_variable_config_alternative_syntax(self): + config = cel.NewEnvConfigFromYaml(""" + context_variable: + type: "cel.expr.conformance.proto2.TestAllTypes" + """) + self.assertEqual( + config.context_type, "cel.expr.conformance.proto2.TestAllTypes" + ) + + def test_parse_context_variable_malformed(self): + with self.assertRaisesRegex( + Exception, "Node 'context_variable' is not a map" + ): + cel.NewEnvConfigFromYaml("context_variable: 123") + + def test_parse_context_variable_malformed2(self): + with self.assertRaisesRegex( + Exception, "Node 'context_variable' does not have a valid type" + ): + cel.NewEnvConfigFromYaml(""" + context_variable: + type: + foo: bar + """) + + def test_context_variable_basic(self): + config = cel.NewEnvConfigFromYaml(""" + context_variable: + type_name: "cel.expr.conformance.proto2.TestAllTypes" + """) + env = cel.NewEnv(config=config) + ast = env.compile("single_int32 > 10") + self.assertIsNotNone(ast) + + with self.assertRaises(Exception): + env.compile("non_existent_field > 10") + def test_config_export_container(self): env: cel.Env = cel.NewEnv(container="test.container") yaml: str = env.config().to_yaml() diff --git a/cel_expr_python/py_cel_env_config.cc b/cel_expr_python/py_cel_env_config.cc index 1f4014f..ff111d8 100644 --- a/cel_expr_python/py_cel_env_config.cc +++ b/cel_expr_python/py_cel_env_config.cc @@ -33,6 +33,8 @@ void PyCelEnvConfig::DefinePythonBindings(pybind11::module& m) { m.def("NewEnvConfigFromYaml", &PyCelEnvConfig::FromYaml, py::arg("yaml")); cel_class.def("to_yaml", &PyCelEnvConfig::ToYaml); + cel_class.def_property_readonly("context_type", + &PyCelEnvConfig::GetContextType); } PyCelEnvConfig PyCelEnvConfig::FromYaml(std::string yaml) { diff --git a/cel_expr_python/py_cel_env_config.h b/cel_expr_python/py_cel_env_config.h index 1742413..9d93bb8 100644 --- a/cel_expr_python/py_cel_env_config.h +++ b/cel_expr_python/py_cel_env_config.h @@ -34,6 +34,7 @@ class PyCelEnvConfig { std::string ToYaml() const; const cel::Config& GetConfig() const { return config_; } + std::string GetContextType() const { return config_.GetContextType(); } private: cel::Config config_; From 441420721a1f64b916c013b1787b3d8d1622567e Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Wed, 17 Jun 2026 16:55:56 -0700 Subject: [PATCH 16/23] Add support for macos/x86_64 wheels PiperOrigin-RevId: 933998105 --- cel_expr_python/BUILD | 12 ++++++++++++ release/pyproject.toml | 2 ++ release/setup.py | 27 ++++++++++++++++++++++++++- 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/cel_expr_python/BUILD b/cel_expr_python/BUILD index b817175..6f9836d 100644 --- a/cel_expr_python/BUILD +++ b/cel_expr_python/BUILD @@ -187,3 +187,15 @@ py_test( ], }), ) + +# Platform definition for macOS x86_64. +# Used for cross-compiling from arm64 macOS hosts (Apple Silicon) to x86_64. +# This is passed to Bazel via --platforms in setup.py when building x86_64 wheels. +platform( + name = "macos_x86_64", + constraint_values = [ + "@platforms//os:osx", + "@platforms//cpu:x86_64", + ], + visibility = ["//visibility:public"], +) diff --git a/release/pyproject.toml b/release/pyproject.toml index 8c645c5..c6b322a 100644 --- a/release/pyproject.toml +++ b/release/pyproject.toml @@ -54,7 +54,9 @@ container-engine = "docker; disable_host_mount: True" before-all = "git config --global http.sslVerify false && echo 'Installing bazelisk' && cp {project}/bazelisk-linux-amd64 /usr/local/bin/bazel" [tool.cibuildwheel.macos] +archs = ["x86_64", "arm64"] before-all = "echo 'Installing bazelisk'; brew install bazelisk" +environment = { MACOSX_DEPLOYMENT_TARGET = "10.13" } [tool.cibuildwheel.windows] # Bazel is expected to be already installed and in the PATH on Windows. diff --git a/release/setup.py b/release/setup.py index 1fec997..09cfe81 100644 --- a/release/setup.py +++ b/release/setup.py @@ -16,6 +16,7 @@ import glob import os +import platform import re import shutil import subprocess @@ -161,7 +162,31 @@ def platform_config_windows(self, cmd, python_version): def platform_config_macos(self, cmd): """Applies macOS-specific Bazel configurations.""" - cmd.extend(['--macos_minimum_os=10.13', '--cxxopt=-faligned-allocation']) + deployment_target = os.environ.get('MACOSX_DEPLOYMENT_TARGET', '10.13') + cmd.extend([ + f'--macos_minimum_os={deployment_target}', + '--cxxopt=-faligned-allocation', + ]) + + archflags = os.environ.get('ARCHFLAGS', '') + if 'x86_64' in archflags: + target_arch = 'x86_64' + elif 'arm64' in archflags: + target_arch = 'arm64' + else: + machine = platform.machine() + if machine in ('AMD64', 'x86_64'): + target_arch = 'x86_64' + elif machine in ('arm64', 'aarch64'): + target_arch = 'arm64' + else: + target_arch = machine + + print(f'Target architecture for macOS: {target_arch}', flush=True) + cmd.append(f'--macos_cpus={target_arch}') + cmd.append(f'--cpu=darwin_{target_arch}') + if target_arch == 'x86_64': + cmd.append('--platforms=//cel_expr_python:macos_x86_64') setuptools.setup( From 78c80964257b5346912699d5e3ea3add70cf7413 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Thu, 18 Jun 2026 13:00:30 -0700 Subject: [PATCH 17/23] Fix Windows bazel build by unsetting ANDROID_HOME Prevent transitive rules_android dependency from attempting to configure an Android SDK repository on CI environments where ANDROID_HOME is set but contains an incompatible Android SDK build tools version. PiperOrigin-RevId: 934508617 --- .bazelrc | 4 ++++ release/kokoro/presubmit_windows.bat | 8 ++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.bazelrc b/.bazelrc index a727ae9..5c70e99 100644 --- a/.bazelrc +++ b/.bazelrc @@ -2,6 +2,10 @@ # This allows Bazel to automatically pick up the `windows` config on Windows. common --enable_platform_specific_config +# Prevent rules_android (pulled in transitively) from attempting to configure an Android SDK repository +# using the preinstalled Android SDK on CI machines (e.g. Kokoro Windows), which fails build tools version checks. +common --repo_env=ANDROID_HOME= + # Disable automatic creation of `__init__.py` files, which prevent multiple # workspaces from providing files with the same package prefix (e.g., "cel"). # See https://github.com/bazelbuild/rules_python/issues/330. diff --git a/release/kokoro/presubmit_windows.bat b/release/kokoro/presubmit_windows.bat index 30136f1..5e9e8ea 100644 --- a/release/kokoro/presubmit_windows.bat +++ b/release/kokoro/presubmit_windows.bat @@ -56,7 +56,7 @@ for %%V in (%PYTHON_VERSIONS%) do ( :fetch_loop set /a ATTEMPTS+=1 echo Fetch attempt !ATTEMPTS! of !FETCH_RETRIES!... - bazel %STARTUP_FLAGS% fetch //... > fetch.log 2>&1 + bazel !STARTUP_FLAGS! fetch //... > fetch.log 2>&1 set FETCH_STATUS=!ERRORLEVEL! type fetch.log if !FETCH_STATUS! NEQ 0 ( @@ -78,7 +78,7 @@ for %%V in (%PYTHON_VERSIONS%) do ( if exist fetch.log del fetch.log echo --- Getting Output Base --- - for /f "tokens=*" %%i in ('bazel %STARTUP_FLAGS% info output_base') do set "OUTPUT_BASE=%%i" + for /f "tokens=*" %%i in ('bazel !STARTUP_FLAGS! info output_base') do set "OUTPUT_BASE=%%i" set "OUTPUT_BASE=!OUTPUT_BASE:/=\!" echo Output Base: !OUTPUT_BASE! @@ -99,7 +99,7 @@ for %%V in (%PYTHON_VERSIONS%) do ( ) echo --- Bazel Build --- - bazel %STARTUP_FLAGS% build %LINK_FLAGS% //... + bazel !STARTUP_FLAGS! build !LINK_FLAGS! //... if !ERRORLEVEL! NEQ 0 ( echo Build failed! set "PRESUBMIT_STATUS=1" @@ -107,7 +107,7 @@ for %%V in (%PYTHON_VERSIONS%) do ( ) echo --- Bazel Test Python %%V --- - bazel %STARTUP_FLAGS% test %LINK_FLAGS% --test_output=errors //... + bazel !STARTUP_FLAGS! test !LINK_FLAGS! --test_output=errors //... if !ERRORLEVEL! NEQ 0 ( echo Tests failed for Python %%V! set "PRESUBMIT_STATUS=1" From 42ea759702d001ee07d5dde9db04ee1a1471be42 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Mon, 22 Jun 2026 13:39:08 -0700 Subject: [PATCH 18/23] Update YAML configuration and constructors to use type and overload signatures PiperOrigin-RevId: 936227381 --- cel_expr_python/BUILD | 1 + cel_expr_python/cel.pyi | 4 +- cel_expr_python/cel_env_test.py | 218 ++++++++++++++---------- cel_expr_python/cel_test.py | 12 ++ cel_expr_python/py_cel_env_config.cc | 4 +- cel_expr_python/py_cel_env_config.h | 2 +- cel_expr_python/py_cel_env_internal.cc | 12 +- cel_expr_python/py_cel_function_decl.cc | 3 + cel_expr_python/py_cel_overload.cc | 46 ++++- cel_expr_python/py_cel_type.cc | 52 +++++- cel_expr_python/py_cel_type.h | 1 + codelab/index.lab.md | 18 +- codelab/solution/codelab.py | 13 +- 13 files changed, 257 insertions(+), 129 deletions(-) diff --git a/cel_expr_python/BUILD b/cel_expr_python/BUILD index 6f9836d..190c18a 100644 --- a/cel_expr_python/BUILD +++ b/cel_expr_python/BUILD @@ -70,6 +70,7 @@ pybind_library( "@com_google_cel_cpp//common:function_descriptor", "@com_google_cel_cpp//common:kind", "@com_google_cel_cpp//common:minimal_descriptor_pool", + "@com_google_cel_cpp//common:signature", "@com_google_cel_cpp//common:source", "@com_google_cel_cpp//common:type", "@com_google_cel_cpp//common:type_kind", diff --git a/cel_expr_python/cel.pyi b/cel_expr_python/cel.pyi index f5c8225..f5cc7c8 100644 --- a/cel_expr_python/cel.pyi +++ b/cel_expr_python/cel.pyi @@ -39,7 +39,7 @@ class FunctionDecl: def __init__(self, name: str, overloads: Sequence[Overload]) -> None: ... class Overload: - def __init__(self, overload_id: str, return_type: Type = ..., parameters: Sequence[Type] = ..., is_member: bool = ..., impl: Callable[..., Any] = ...) -> None: ... + def __init__(self, id: str | None = ..., return_type: Type = ..., parameters: Sequence[Type] = ..., is_member: bool = ..., impl: Callable[..., Any] = ..., signature: str | None = ...) -> None: ... class Type: BOOL: ClassVar[Type] = ... @@ -57,7 +57,7 @@ class Type: TYPE: ClassVar[Type] = ... UINT: ClassVar[Type] = ... UNKNOWN: ClassVar[Type] = ... - def __init__(self, name: str) -> None: ... + def __init__(self, signature: str) -> None: ... @staticmethod def AbstractType(name: str, params: Sequence[Type] = ...) -> Type: ... @staticmethod diff --git a/cel_expr_python/cel_env_test.py b/cel_expr_python/cel_env_test.py index ce98570..eacfc9f 100644 --- a/cel_expr_python/cel_env_test.py +++ b/cel_expr_python/cel_env_test.py @@ -46,17 +46,13 @@ def test_env_config_from_and_to_yaml(self): - name: math variables: - name: one - type_name: int + type: int value: 1 functions: - name: add overloads: - - id: "add_int_int" - args: - - type_name: int - - type_name: int - return: - type_name: int + - signature: "add(int,int)" + return: int """) yaml: str = config.to_yaml() self.assertEqual( @@ -74,17 +70,13 @@ def test_env_config_from_and_to_yaml(self): - name: "_+_" variables: - name: "one" - type_name: "int" + type: "int" value: 1 functions: - name: "add" overloads: - - id: "add_int_int" - args: - - type_name: "int" - - type_name: "int" - return: - type_name: "int" + - signature: "add(int,int)" + return: "int" """), ) @@ -185,9 +177,9 @@ def test_expression_container_abbreviations_and_aliases(self): qualified_name: "x.y.bar" variables: - name: "x.y.bar" - type_name: "string" + type: "string" - name: "x.y.foo" - type_name: "int" + type: "int" """), ) @@ -202,9 +194,9 @@ def test_abbreviations_and_aliases_from_yaml(self): qualified_name: "x.y.bar" variables: - name: "x.y.bar" - type_name: "string" + type: "string" - name: "x.y.foo" - type_name: "int" + type: "int" """)) res = env.compile("foo").eval(data={"x.y.foo": 42}) @@ -224,13 +216,13 @@ def test_abbreviations_and_aliases_combined(self): qualified_name: "x.y.bar" variables: - name: "x.y.bar" - type_name: "string" + type: "string" - name: "x.y.foo" - type_name: "int" + type: "int" - name: "a.b.qux" - type_name: "string" + type: "string" - name: "a.b.baz" - type_name: "int" + type: "int" """), container=cel.ExpressionContainer( "test.container", @@ -265,13 +257,13 @@ def test_abbreviations_and_aliases_combined(self): qualified_name: "a.b.qux" variables: - name: "a.b.baz" - type_name: "int" + type: "int" - name: "a.b.qux" - type_name: "string" + type: "string" - name: "x.y.bar" - type_name: "string" + type: "string" - name: "x.y.foo" - type_name: "int" + type: "int" """), ) @@ -321,48 +313,35 @@ def test_config_export_variables(self): normalize_yaml(""" variables: - name: "var_bool" - type_name: "bool" + type: "bool" - name: "var_bytes" - type_name: "bytes" + type: "bytes" - name: "var_double" - type_name: "double" + type: "double" - name: "var_duration" - type_name: "duration" + type: "duration" - name: "var_dyn" - type_name: "dyn" + type: "dyn" - name: "var_dyn_list" - type_name: "list" - params: - - type_name: "dyn" + type: "list" - name: "var_dyn_map" - type_name: "map" - params: - - type_name: "dyn" - - type_name: "dyn" + type: "map" - name: "var_int" - type_name: "int" + type: "int" - name: "var_int_map" - type_name: "map" - params: - - type_name: "int" - - type_name: "string" + type: "map" - name: "var_msg" - type_name: "cel.expr.conformance.proto2.TestAllTypes" + type: "cel.expr.conformance.proto2.TestAllTypes" - name: "var_str" - type_name: "string" + type: "string" - name: "var_string_list" - type_name: "list" - params: - - type_name: "string" + type: "list" - name: "var_string_map" - type_name: "map" - params: - - type_name: "string" - - type_name: "bool" + type: "map" - name: "var_timestamp" - type_name: "timestamp" + type: "timestamp" - name: "var_uint" - type_name: "uint" + type: "uint" """), ) @@ -370,7 +349,7 @@ def test_config_augmented_variables(self): config = cel.NewEnvConfigFromYaml(""" variables: - name: "var_bool" - type_name: "bool" + type: "bool" """) env: cel.Env = cel.NewEnv( config=config, @@ -384,9 +363,9 @@ def test_config_augmented_variables(self): normalize_yaml(""" variables: - name: "var_bool" - type_name: "bool" + type: "bool" - name: "var_msg" - type_name: "cel.expr.conformance.proto2.TestAllTypes" + type: "cel.expr.conformance.proto2.TestAllTypes" """), ) @@ -394,7 +373,7 @@ def test_config_variable_override(self): config: cel.EnvConfig = cel.NewEnvConfigFromYaml(""" variables: - name: "var_bool" - type_name: "bool" + type: "bool" """) with self.assertRaises(Exception) as e: @@ -413,9 +392,9 @@ def test_config_variable_types(self): config: cel.EnvConfig = cel.NewEnvConfigFromYaml(""" variables: - name: "var_bool" - type_name: "bool" + type: "bool" - name: "var_int" - type_name: "int" + type: "int" value: 42 """) env: cel.Env = cel.NewEnv( @@ -560,11 +539,8 @@ def test_config_functions(self): functions: - name: is_ok overloads: - - id: "is_ok_string" - target: - type_name: string - return: - type_name: bool + - signature: "string.is_ok()" + return: "bool" """) env: cel.Env = cel.NewEnv( config=config, @@ -573,12 +549,8 @@ def test_config_functions(self): "hello", [ cel.Overload( - "good_time_of_day", + signature="hello(string,string)", return_type=cel.Type.STRING, - parameters=[ - cel.Type.STRING, - cel.Type.STRING, - ], impl=lambda ampm, arg: ( "Good" f" {'morning' if ampm == 'am' else 'afternoon'}," @@ -589,7 +561,7 @@ def test_config_functions(self): ) ], function_impls={ - "is_ok_string": lambda arg: arg in ["excellent", "good", "fair"], + "string.is_ok()": lambda arg: arg in ["excellent", "good", "fair"], }, ) yaml = env.config().to_yaml() @@ -599,19 +571,12 @@ def test_config_functions(self): functions: - name: "hello" overloads: - - id: "good_time_of_day" - args: - - type_name: "string" - - type_name: "string" - return: - type_name: "string" + - signature: "hello(string,string)" + return: "string" - name: "is_ok" overloads: - - id: "is_ok_string" - target: - type_name: "string" - return: - type_name: "bool" + - signature: "string.is_ok()" + return: "bool" """), ) res: cel.Value = env.compile("hello('am', 'Sunshine')").eval() @@ -628,32 +593,109 @@ def test_config_function_override(self): functions: - name: foo overloads: - - id: "unique_id" + - signature: "foo()" """) with self.assertRaises(Exception) as e: cel.NewEnv( config=config, functions=[ cel.FunctionDecl( - "bar", + "foo", [ cel.Overload( - "unique_id", + signature="foo()", impl=lambda: "hello", ) ], ) ], function_impls={ - "unique_id": lambda: "goodbye", + "foo()": lambda: "goodbye", }, ) self.assertIn( - "An implementation for function overload id 'unique_id' already" - " exists.", + "An implementation for function overload 'foo()' already exists.", str(e.exception), ) + def test_overload_signature_errors(self): + with self.assertRaises(ValueError) as e2: + cel.Overload(signature="greet(string)", parameters=[cel.Type.STRING]) + self.assertIn( + "If 'signature' is specified, 'parameters' should not be specified", + str(e2.exception), + ) + + with self.assertRaises(ValueError) as e3: + cel.Overload() + self.assertIn( + "Either 'id' or 'signature' must be specified", str(e3.exception) + ) + + def test_config_functions_deprecated_syntax(self): + """Test that the deprecated function syntax is still supported.""" + config: cel.EnvConfig = cel.NewEnvConfigFromYaml(""" + functions: + - name: is_ok + overloads: + - id: "is_ok_string" + target: + type_name: string + return: + type_name: bool + """) + env: cel.Env = cel.NewEnv( + config=config, + functions=[ + cel.FunctionDecl( + "hello", + [ + cel.Overload( + "good_time_of_day", + return_type=cel.Type.STRING, + parameters=[ + cel.Type.STRING, + cel.Type.STRING, + ], + impl=lambda ampm, arg: ( + "Good" + f" {'morning' if ampm == 'am' else 'afternoon'}," + f" {arg}!" + ), + ) + ], + ) + ], + function_impls={ + "is_ok_string": lambda arg: arg in ["excellent", "good", "fair"], + }, + ) + yaml = env.config().to_yaml() + self.assertEqual( + normalize_yaml(yaml), + normalize_yaml(""" + functions: + - name: "hello" + overloads: + - id: "good_time_of_day" + signature: "hello(string,string)" + return: "string" + - name: "is_ok" + overloads: + - id: "is_ok_string" + signature: "string.is_ok()" + return: "bool" + """), + ) + res: cel.Value = env.compile("hello('am', 'Sunshine')").eval() + self.assertEqual(res.value(), "Good morning, Sunshine!") + res = env.compile("hello('pm', 'tea is served')").eval() + self.assertEqual(res.value(), "Good afternoon, tea is served!") + res = env.compile("'good'.is_ok()").eval() + self.assertTrue(res.value()) + res = env.compile("'bad'.is_ok()").eval() + self.assertFalse(res.value()) + class TestCelExtension(cel.CelExtension): """An example CEL extension for testing.""" diff --git a/cel_expr_python/cel_test.py b/cel_expr_python/cel_test.py index f812fe4..938db60 100644 --- a/cel_expr_python/cel_test.py +++ b/cel_expr_python/cel_test.py @@ -668,6 +668,18 @@ def testTypeType(self): res.value(), cel.Type.INT ) # This behavior is counterintuitive but works as implemented. + def testTypeInitSignature(self): + self.assertEqual(cel.Type("int"), cel.Type.INT) + self.assertEqual(cel.Type("list"), cel.Type.List(cel.Type.INT)) + self.assertEqual( + cel.Type("map"), + cel.Type.Map(cel.Type.STRING, cel.Type.DYN), + ) + self.assertEqual( + cel.Type("cel.expr.conformance.proto2.TestAllTypes"), + cel.Type("cel.expr.conformance.proto2.TestAllTypes"), + ) + def testCelExpressionPersistence_checkedExpr(self): expr: cel.Expression = self.env.compile("var_msg.single_string") as_bytes: bytes = expr.serialize() diff --git a/cel_expr_python/py_cel_env_config.cc b/cel_expr_python/py_cel_env_config.cc index ff111d8..8ea330c 100644 --- a/cel_expr_python/py_cel_env_config.cc +++ b/cel_expr_python/py_cel_env_config.cc @@ -37,7 +37,7 @@ void PyCelEnvConfig::DefinePythonBindings(pybind11::module& m) { &PyCelEnvConfig::GetContextType); } -PyCelEnvConfig PyCelEnvConfig::FromYaml(std::string yaml) { +PyCelEnvConfig PyCelEnvConfig::FromYaml(const std::string& yaml) { PyCelEnvConfig config; config.config_ = ThrowIfError(cel::EnvConfigFromYaml(yaml)); return config; @@ -45,7 +45,7 @@ PyCelEnvConfig PyCelEnvConfig::FromYaml(std::string yaml) { std::string PyCelEnvConfig::ToYaml() const { std::stringstream ss; - cel::EnvConfigToYaml(config_, ss); + cel::EnvConfigToYaml(config_, ss, {.use_type_signatures = true}); return ss.str(); } diff --git a/cel_expr_python/py_cel_env_config.h b/cel_expr_python/py_cel_env_config.h index 9d93bb8..ed30ea2 100644 --- a/cel_expr_python/py_cel_env_config.h +++ b/cel_expr_python/py_cel_env_config.h @@ -30,7 +30,7 @@ class PyCelEnvConfig { PyCelEnvConfig() = default; explicit PyCelEnvConfig(const cel::Config& config) : config_(config) {} - static PyCelEnvConfig FromYaml(std::string yaml); + static PyCelEnvConfig FromYaml(const std::string& yaml); std::string ToYaml() const; const cel::Config& GetConfig() const { return config_; } diff --git a/cel_expr_python/py_cel_env_internal.cc b/cel_expr_python/py_cel_env_internal.cc index a591580..af0391b 100644 --- a/cel_expr_python/py_cel_env_internal.cc +++ b/cel_expr_python/py_cel_env_internal.cc @@ -202,11 +202,11 @@ PyCelEnvInternal::NewCelEnvInternal( if (overload.py_function().is_none()) { continue; } - if (!impls.insert({overload.overload_id(), overload.py_function()}) - .second) { + std::string overload_id = overload.overload_id(); + if (!impls.insert({overload_id, overload.py_function()}).second) { return absl::AlreadyExistsError( - absl::StrCat("An implementation for function overload id '", - overload.overload_id(), "' already exists.")); + absl::StrCat("An implementation for function overload '", + overload_id, "' already exists.")); } } } @@ -214,8 +214,8 @@ PyCelEnvInternal::NewCelEnvInternal( for (const auto& [overload_id, py_function] : function_impls) { if (!impls.insert({overload_id, py_function}).second) { return absl::AlreadyExistsError( - absl::StrCat("An implementation for function overload id '", - overload_id, "' already exists.")); + absl::StrCat("An implementation for function overload '", overload_id, + "' already exists.")); } } return std::shared_ptr( diff --git a/cel_expr_python/py_cel_function_decl.cc b/cel_expr_python/py_cel_function_decl.cc index b5f084b..382fdae 100644 --- a/cel_expr_python/py_cel_function_decl.cc +++ b/cel_expr_python/py_cel_function_decl.cc @@ -16,10 +16,13 @@ #include #include +#include #include #include "env/config.h" +#include "env/type_info.h" #include "cel_expr_python/py_cel_overload.h" +#include "cel_expr_python/py_cel_type.h" #include #include diff --git a/cel_expr_python/py_cel_overload.cc b/cel_expr_python/py_cel_overload.cc index 5311f46..321af6c 100644 --- a/cel_expr_python/py_cel_overload.cc +++ b/cel_expr_python/py_cel_overload.cc @@ -20,8 +20,11 @@ #include #include +#include "common/signature.h" #include "env/config.h" +#include "env/type_info.h" #include "cel_expr_python/py_cel_type.h" +#include "cel_expr_python/py_error_status.h" #include #include @@ -31,16 +34,47 @@ namespace py = ::pybind11; void PyCelOverload::DefinePythonBindings(py::module_& m) { py::class_>(m, "Overload") - .def(py::init([](const std::string& overload_id, + .def(py::init([](std::optional id, const PyCelType& return_type, const std::vector& parameters, bool is_member, - py::object impl) { - return PyCelOverload(overload_id, return_type, parameters, - is_member, std::move(impl)); + py::object impl, std::optional signature) { + if (signature.has_value()) { + if (!parameters.empty()) { + throw py::value_error( + "If 'signature' is specified, 'parameters' " + "should not be specified"); + } + cel::ParsedFunctionOverload parsed = + ThrowIfError(cel::ParseFunctionSignature(*signature)); + std::string overload_id = + id.has_value() ? std::move(*id) : *signature; + std::vector parsed_parameters; + if (parsed.signature_type.has_function()) { + const auto& function_type_spec = + parsed.signature_type.function(); + parsed_parameters.reserve( + function_type_spec.arg_types().size()); + for (const auto& arg : function_type_spec.arg_types()) { + parsed_parameters.push_back(PyCelType::FromTypeInfo( + ThrowIfError(cel::TypeSpecToTypeInfo(arg)))); + } + } + return PyCelOverload(std::move(overload_id), return_type, + std::move(parsed_parameters), + parsed.is_member, std::move(impl)); + } + if (id.has_value()) { + return PyCelOverload(std::move(*id), return_type, parameters, + is_member, std::move(impl)); + } + throw py::value_error( + "Either 'id' or 'signature' must be specified"); }), - py::arg("overload_id"), py::arg("return_type") = PyCelType::Dyn(), + py::arg("id") = std::nullopt, + py::arg("return_type") = PyCelType::Dyn(), py::arg("parameters") = std::vector{}, - py::arg("is_member") = false, py::arg("impl") = py::none()); + py::arg("is_member") = false, py::arg("impl") = py::none(), + py::arg("signature") = std::nullopt); } PyCelOverload::PyCelOverload(std::string overload_id, diff --git a/cel_expr_python/py_cel_type.cc b/cel_expr_python/py_cel_type.cc index 0cf7b54..be49498 100644 --- a/cel_expr_python/py_cel_type.cc +++ b/cel_expr_python/py_cel_type.cc @@ -30,6 +30,7 @@ #include "absl/strings/str_format.h" #include "absl/strings/str_join.h" #include "common/kind.h" +#include "common/signature.h" #include "common/type.h" #include "common/type_kind.h" #include "common/types/list_type.h" @@ -37,6 +38,7 @@ #include "common/value.h" #include "common/value_kind.h" #include "env/config.h" +#include "env/type_info.h" #include "cel_expr_python/py_error_status.h" #include "cel_expr_python/status_macros.h" #include "google/protobuf/arena.h" @@ -50,7 +52,14 @@ namespace py = ::pybind11; void PyCelType::DefinePythonBindings(py::module& m) { py::class_ type(m, "Type"); - type.def(py::init(), py::arg("name")) + type.def(py::init([](const std::string& signature) { + cel::TypeSpec type_spec = + ThrowIfError(cel::ParseTypeSpec(signature)); + cel::Config::TypeInfo type_info = + ThrowIfError(cel::TypeSpecToTypeInfo(type_spec)); + return PyCelType::FromTypeInfo(type_info); + }), + py::arg("signature")) .def("name", &PyCelType::GetName) .def("is_message", &PyCelType::IsMessage) .def("is_assignable_from", &PyCelType::IsAssignableFrom) @@ -526,6 +535,47 @@ PyCelType PyCelType::FromTypeProto(const cel::expr::Type& type) { return PyCelType::Error(); } +PyCelType PyCelType::FromTypeInfo(const cel::Config::TypeInfo& type_info) { + if (type_info.name == "null") return PyCelType::Null(); + if (type_info.name == "bool") return PyCelType::Bool(); + if (type_info.name == "int") return PyCelType::Int(); + if (type_info.name == "uint") return PyCelType::Uint(); + if (type_info.name == "double") return PyCelType::Double(); + if (type_info.name == "string") return PyCelType::String(); + if (type_info.name == "bytes") return PyCelType::Bytes(); + if (type_info.name == "timestamp") return PyCelType::Timestamp(); + if (type_info.name == "duration") return PyCelType::Duration(); + if (type_info.name == "dyn") return PyCelType::Dyn(); + if (type_info.name == "list") { + if (type_info.params.empty()) { + return PyCelType::List(); + } + return PyCelType::ListType(FromTypeInfo(type_info.params[0])); + } + if (type_info.name == "map") { + if (type_info.params.size() < 2) { + return PyCelType::Map(); + } + return PyCelType::MapType(FromTypeInfo(type_info.params[0]), + FromTypeInfo(type_info.params[1])); + } + if (type_info.name == "type") { + if (type_info.params.empty()) { + return PyCelType::Type(); + } + return PyCelType::TypeType(FromTypeInfo(type_info.params[0])); + } + if (type_info.is_type_param || !type_info.params.empty()) { + std::vector params; + params.reserve(type_info.params.size()); + for (const auto& param : type_info.params) { + params.push_back(FromTypeInfo(param)); + } + return PyCelType::AbstractType(type_info.name, params); + } + return PyCelType(type_info.name); +} + absl::StatusOr PyCelType::ToCelType( const PyCelType& type, google::protobuf::Arena* arena, const google::protobuf::DescriptorPool& descriptor_pool) { diff --git a/cel_expr_python/py_cel_type.h b/cel_expr_python/py_cel_type.h index 892c488..19cac0e 100644 --- a/cel_expr_python/py_cel_type.h +++ b/cel_expr_python/py_cel_type.h @@ -77,6 +77,7 @@ class PyCelType { static PyCelType ForCelValue(const cel::Value& cel_value); static PyCelType FromCelType(const cel::Type& cel_type); static PyCelType FromTypeProto(const cel::expr::Type& type); + static PyCelType FromTypeInfo(const cel::Config::TypeInfo& type_info); static absl::StatusOr ToCelType( const PyCelType& type, google::protobuf::Arena* arena, const google::protobuf::DescriptorPool& descriptor_pool); diff --git a/codelab/index.lab.md b/codelab/index.lab.md index eb5ef2f..5ca5fba 100644 --- a/codelab/index.lab.md +++ b/codelab/index.lab.md @@ -846,14 +846,8 @@ def exercise5(): "contains", [ cel.Overload( - "contains_key_value", + signature="map.contains(string, dyn)", return_type=cel.Type.BOOL, - parameters=[ - cel.Type.Map(cel.Type.STRING, cel.Type.DYN), - cel.Type.STRING, - cel.Type.DYN, - ], - is_member=True, # Provide the implementation as a Python function ) ], @@ -922,14 +916,8 @@ def exercise5(): "contains", [ cel.Overload( - "contains_string_any", + signature="map.contains(string, dyn)", return_type=cel.Type.BOOL, - parameters=[ - cel.Type.Map(cel.Type.STRING, cel.Type.DYN), - cel.Type.STRING, - cel.Type.DYN, - ], - is_member=True, # Reference a Python function impl=contains_key_value, ) @@ -1319,7 +1307,7 @@ def exercise7(): # Add variable definitions for 'jwt' as a map(string, Dyn) type # and for 'now' as a timestamp. variables={ - "jwt": cel.Type.Map(cel.Type.STRING, cel.Type.DYN), + "jwt": cel.Type("map"), "now": cel.Type.TIMESTAMP, }, ) diff --git a/codelab/solution/codelab.py b/codelab/solution/codelab.py index 15d12c0..861f0ef 100644 --- a/codelab/solution/codelab.py +++ b/codelab/solution/codelab.py @@ -188,14 +188,11 @@ def exercise5(): "containsKeyValue", [ cel.Overload( - "contains_key_value", + signature=( + "map.containsKeyValue(string," + " dyn)" + ), return_type=cel.Type.BOOL, - parameters=[ - cel.Type.Map(cel.Type.STRING, cel.Type.DYN), - cel.Type.STRING, - cel.Type.DYN, - ], - is_member=True, impl=contains_key_value, ) ], @@ -323,7 +320,7 @@ def exercise7(): # Add variable definitions for 'jwt' as a map(string, Dyn) type # and for 'now' as a timestamp. variables={ - "jwt": cel.Type.Map(cel.Type.STRING, cel.Type.DYN), + "jwt": cel.Type("map"), "now": cel.Type.TIMESTAMP, }, ) From 584be4f067a95387c291b34e03c55431efdcc138 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Mon, 22 Jun 2026 15:07:51 -0700 Subject: [PATCH 19/23] Fix build PiperOrigin-RevId: 936270519 --- MODULE.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MODULE.bazel b/MODULE.bazel index 7404b9f..089d1b1 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -15,7 +15,7 @@ bazel_dep(name = "bazel_skylib", version = "1.9.0") bazel_dep(name = "cel-cpp", version = "0.15.0", repo_name = "com_google_cel_cpp") git_override( module_name = "cel-cpp", - commit = "8b7068abb4062074a135491ad8357139287084f9", + commit = "76ae0b3c1768d93a10270f904101de338867bdb1", remote = "https://github.com/cel-expr/cel-cpp", ) From ed5fb185c730c21a974641ea5393031f3fa653ac Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Wed, 24 Jun 2026 09:51:28 -0700 Subject: [PATCH 20/23] Update release scripts and pyproject.toml for ARM and Python 3.14 support PiperOrigin-RevId: 937397017 --- .bazelrc | 16 ++++++---- release/kokoro/release_linux.cfg | 2 +- release/kokoro/release_linux.sh | 51 +++++++++++++++++++++++++++--- release/kokoro/release_macos.sh | 2 ++ release/kokoro/release_windows.bat | 6 ++++ release/pyproject.toml | 6 +++- release/setup.py | 3 ++ 7 files changed, 74 insertions(+), 12 deletions(-) diff --git a/.bazelrc b/.bazelrc index 5c70e99..333e7bb 100644 --- a/.bazelrc +++ b/.bazelrc @@ -28,13 +28,17 @@ common:windows --experimental_repository_downloader_retries=10 build --verbose_failures test --test_output=errors -# GCS remote caching config (Windows-only, active by default on Windows!) -build:windows --remote_cache=https://storage.googleapis.com/windows-cel-python-remote-cache -build:windows --google_default_credentials=true +# GCS remote caching config (Linux-only, opt-in) +build:remote-cache-linux --remote_cache=https://storage.googleapis.com/linux-cel-python-remote-cache +build:remote-cache-linux --google_default_credentials=true -# GCS remote caching config (macOS-only, active by default on macOS!) -build:macos --remote_cache=https://storage.googleapis.com/macos-cel-python-remote-cache -build:macos --google_default_credentials=true +# GCS remote caching config (Windows-only, opt-in) +build:remote-cache-windows --remote_cache=https://storage.googleapis.com/windows-cel-python-remote-cache +build:remote-cache-windows --google_default_credentials=true + +# GCS remote caching config (macOS-only, opt-in) +build:remote-cache-macos --remote_cache=https://storage.googleapis.com/macos-cel-python-remote-cache +build:remote-cache-macos --google_default_credentials=true # Silence deprecation warnings from external dependencies (Linux and macOS) build:linux --cxxopt=-Wno-deprecated-declarations diff --git a/release/kokoro/release_linux.cfg b/release/kokoro/release_linux.cfg index a42e2e3..018c022 100644 --- a/release/kokoro/release_linux.cfg +++ b/release/kokoro/release_linux.cfg @@ -5,6 +5,6 @@ build_file: "cel-python/release/kokoro/release_linux.sh" timeout_mins: 120 container_properties { - docker_image: "us-central1-docker.pkg.dev/kokoro-container-bakery/kokoro/ubuntu/ubuntu2204/ktcb:current" + docker_image: "mirror.gcr.io/library/ubuntu:24.04" docker_sibling_containers: true } diff --git a/release/kokoro/release_linux.sh b/release/kokoro/release_linux.sh index 5292ef6..ff27fa7 100755 --- a/release/kokoro/release_linux.sh +++ b/release/kokoro/release_linux.sh @@ -15,12 +15,42 @@ set -e +if ! command -v pip3 &> /dev/null || ! command -v curl &> /dev/null || ! command -v docker &> /dev/null; then + echo "Installing basic dependencies..." + apt-get update && apt-get install -y python3-pip curl + + if ! command -v docker &> /dev/null; then + echo "Installing docker CLI..." + ARCH=$(uname -m) + if [ "$ARCH" = "x86_64" ]; then + DOCKER_ARCH="x86_64" + elif [ "$ARCH" = "aarch64" ]; then + DOCKER_ARCH="aarch64" + else + echo "Unsupported arch: $ARCH" + exit 1 + fi + curl -fsSL "https://download.docker.com/linux/static/stable/${DOCKER_ARCH}/docker-24.0.7.tgz" -o docker.tgz + tar xzvf docker.tgz --strip-components=1 docker/docker + mv docker /usr/local/bin/ + rm -f docker.tgz + fi +fi + # Avoid virtualenv/pip trying to download/upgrade tools from PyPI on host export VIRTUALENV_NO_DOWNLOAD=1 export PIP_DISABLE_PIP_VERSION_CHECK=1 +export PIP_BREAK_SYSTEM_PACKAGES=1 +export PIP_DEFAULT_TIMEOUT=60 + +if [ "$(uname -m)" = "aarch64" ]; then + export CIBW_ARCHS="aarch64" +else + export CIBW_ARCHS="x86_64" +fi # Pass these environment variables to the cibuildwheel Docker container -export CIBW_ENVIRONMENT="VIRTUALENV_NO_DOWNLOAD=1 PIP_DISABLE_PIP_VERSION_CHECK=1" +export CIBW_ENVIRONMENT="VIRTUALENV_NO_DOWNLOAD=1 PIP_DISABLE_PIP_VERSION_CHECK=1 PIP_DEFAULT_TIMEOUT=120 CEL_BAZEL_FLAGS=--config=remote-cache-linux" export CIBW_DEPENDENCY_VERSIONS="latest" export CIBW_CONTAINER_ENGINE_EXTRA_ARGS="--network=host" @@ -35,7 +65,10 @@ fi # cache. In a sandboxed build environment, writing to the default cache directory # (~/.cache/pip) can encounter permission/sandbox restrictions or lead to stale # dependency resolution. Disabling the cache ensures a reliable, reproducible install. -pip install --no-cache-dir -U keyring keyrings.google-artifactregistry-auth twine cibuildwheel +pip install --no-cache-dir -U keyring keyrings.google-artifactregistry-auth twine +curl -fsSL https://github.com/pypa/cibuildwheel/archive/refs/tags/v4.1.0.tar.gz -o cibuildwheel-4.1.0.tar.gz +pip install --no-cache-dir cibuildwheel-4.1.0.tar.gz +rm -f cibuildwheel-4.1.0.tar.gz # ============================================================================== # FUTURE-PROOF RUNTIME PATCHING OF CIBUILDWHEEL @@ -203,9 +236,19 @@ rm -rf cel_expr_python/*_test.py echo "Downloading bazelisk on host..." curl -LO https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazelisk-linux-amd64 -chmod +x bazelisk-linux-amd64 +curl -LO https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazelisk-linux-arm64 +chmod +x bazelisk-linux-amd64 bazelisk-linux-arm64 + +echo "Downloading build dependencies on host..." +mkdir -p build_deps +pip download --no-cache-dir --only-binary=:all: --dest build_deps "setuptools>=40.8.0" "wheel" +if [ "$(uname -m)" = "aarch64" ]; then + PLATFORM_SUFFIX="aarch64" +else + PLATFORM_SUFFIX="x86_64" +fi +pip download --no-cache-dir --only-binary=:all: --dest build_deps --python-version 3.9 --platform "manylinux2014_${PLATFORM_SUFFIX}" "virtualenv" "typing-extensions>=4.13.2" -# Check if pyproject.toml exists before running sed if [ -f pyproject.toml ]; then sed -i "" "s/\$VERSION/${VERSION}/g" pyproject.toml || sed -i "s/\$VERSION/${VERSION}/g" pyproject.toml fi diff --git a/release/kokoro/release_macos.sh b/release/kokoro/release_macos.sh index 71d4805..54b92fa 100644 --- a/release/kokoro/release_macos.sh +++ b/release/kokoro/release_macos.sh @@ -65,6 +65,8 @@ if [ -f pyproject.toml ]; then sed -i "" "s/\$VERSION/${VERSION}/g" pyproject.toml || sed -i "s/\$VERSION/${VERSION}/g" pyproject.toml fi +export CEL_BAZEL_FLAGS="--config=remote-cache-macos" + echo "Running cibuildwheel: ${CIBWHEEL_BIN}" # Default CIBWHEEL_BIN if not set if [ -z "${CIBWHEEL_BIN}" ]; then diff --git a/release/kokoro/release_windows.bat b/release/kokoro/release_windows.bat index 9f95c7e..75fc8fa 100644 --- a/release/kokoro/release_windows.bat +++ b/release/kokoro/release_windows.bat @@ -16,6 +16,11 @@ setlocal enabledelayedexpansion set "RELEASE_STATUS=0" set "FETCH_RETRIES=10" set "FETCH_RETRY_DELAY_S=10" +echo --- Installing Python 3.11 via Chocolatey --- +choco install python311 -y --no-progress +if !ERRORLEVEL! NEQ 0 ( + echo WARNING: Failed to install Python 3.11 via Chocolatey. +) echo === Loading Environment Configuration === call "%~dp0set_env_windows.bat" @@ -145,6 +150,7 @@ if !FETCH_STATUS! NEQ 0 ( if exist fetch.log del fetch.log echo --- Running cibuildwheel --- +set "CEL_BAZEL_FLAGS=--config=remote-cache-windows" if "%CIBWHEEL_BIN%" == "" ( set "CIBWHEEL_BIN=!PYTHON_EXE! -m cibuildwheel" ) diff --git a/release/pyproject.toml b/release/pyproject.toml index c6b322a..d7aa211 100644 --- a/release/pyproject.toml +++ b/release/pyproject.toml @@ -44,14 +44,18 @@ test-command = "python {project}/cel_basic_test.py" build-verbosity = 1 [tool.cibuildwheel.linux] +build-frontend = { name = "pip", args = ["--no-build-isolation"] } +before-build = "pip install --no-index --find-links={project}/build_deps setuptools wheel virtualenv" +archs = ["x86_64", "aarch64"] manylinux-x86_64-image = "manylinux_2_28" +manylinux-aarch64-image = "manylinux_2_28" container-engine = "docker; disable_host_mount: True" # Google's internal Kokoro/RBE network uses a secure MITM proxy that resigns HTTPS # traffic with an internal Google CA. Since the public manylinux container does not # trust this CA, git fetches for external dependencies (like @cel-cpp) will fail # with SSL certificate errors. We disable http.sslVerify inside the container to # bypass this and allow Bazel to fetch SCM dependencies through the proxy. -before-all = "git config --global http.sslVerify false && echo 'Installing bazelisk' && cp {project}/bazelisk-linux-amd64 /usr/local/bin/bazel" +before-all = "git config --global http.sslVerify false && echo 'Installing bazelisk' && if [ $(uname -m) = 'aarch64' ]; then cp {project}/bazelisk-linux-arm64 /usr/local/bin/bazel; else cp {project}/bazelisk-linux-amd64 /usr/local/bin/bazel; fi && python3 -m pip install --no-index --find-links={project}/build_deps virtualenv" [tool.cibuildwheel.macos] archs = ["x86_64", "arm64"] diff --git a/release/setup.py b/release/setup.py index 09cfe81..e2d0962 100644 --- a/release/setup.py +++ b/release/setup.py @@ -72,6 +72,9 @@ def build_extension(self, ext): # Build with bazel # Use --compilation_mode=opt for release builds cmd = ['bazel', 'build', ext.target, '--compilation_mode=opt'] + extra_flags = os.environ.get('CEL_BAZEL_FLAGS') + if extra_flags: + cmd.extend(extra_flags.split()) if sys.platform == 'win32': self.platform_config_windows(cmd, python_version) if sys.platform == 'darwin': From b8e324c87460ee55d25ed28405119a97700d0a4d Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Wed, 24 Jun 2026 14:44:43 -0700 Subject: [PATCH 21/23] Fix release scripts for cel-python. - In release_linux.sh, install git because it is needed to clone the repository but not present in the default ubuntu image. - In release_windows.bat, restructure SCM resolution to avoid using batch labels inside parenthesized if/else blocks, which causes "The system cannot find the batch label specified" errors. PiperOrigin-RevId: 937568390 --- release/kokoro/release_linux.sh | 4 +- release/kokoro/release_windows.bat | 75 +++++++++++++++++------------- 2 files changed, 44 insertions(+), 35 deletions(-) diff --git a/release/kokoro/release_linux.sh b/release/kokoro/release_linux.sh index ff27fa7..6375ed2 100755 --- a/release/kokoro/release_linux.sh +++ b/release/kokoro/release_linux.sh @@ -15,9 +15,9 @@ set -e -if ! command -v pip3 &> /dev/null || ! command -v curl &> /dev/null || ! command -v docker &> /dev/null; then +if ! command -v pip3 &> /dev/null || ! command -v curl &> /dev/null || ! command -v docker &> /dev/null || ! command -v git &> /dev/null; then echo "Installing basic dependencies..." - apt-get update && apt-get install -y python3-pip curl + apt-get update && apt-get install -y python3-pip curl git if ! command -v docker &> /dev/null; then echo "Installing docker CLI..." diff --git a/release/kokoro/release_windows.bat b/release/kokoro/release_windows.bat index 75fc8fa..8b6f003 100644 --- a/release/kokoro/release_windows.bat +++ b/release/kokoro/release_windows.bat @@ -51,42 +51,51 @@ echo Created temporary directories: %REPO_DIR%, %TMP_DIR% mkdir "%TMP_DIR%" echo --- Resolving Repository Source --- -if "%DRY_RUN%" == "true" ( - echo [DRY RUN] Using local Kokoro clone instead of cloning main. - set "SRC_DIR=%~dp0..\.." - pushd "!SRC_DIR!" - for /f "tokens=*" %%i in ('git tag --sort=-v:refname 2^>nul') do ( - set "VERSION=%%i" - goto :got_local_tag - ) - set "VERSION=0.1.2" - :got_local_tag +if "%DRY_RUN%" == "true" goto resolution_dry +goto resolution_real + +:resolution_dry +echo [DRY RUN] Using local Kokoro clone instead of cloning main. +set "SRC_DIR=%~dp0..\.." +pushd "!SRC_DIR!" +set "VERSION=" +for /f "tokens=*" %%i in ('git tag --sort=-v:refname 2^>nul') do ( + set "VERSION=%%i" + goto got_local_tag +) +:got_local_tag +if "%VERSION%" == "" set "VERSION=0.1.2" +popd +goto resolution_done + +:resolution_real +mkdir "%REPO_DIR%" +pushd "%REPO_DIR%" +git clone https://github.com/cel-expr/cel-python.git +if !ERRORLEVEL! NEQ 0 ( + echo Failed to clone repository! + set "RELEASE_STATUS=1" popd -) else ( - mkdir "%REPO_DIR%" - pushd "%REPO_DIR%" - git clone https://github.com/cel-expr/cel-python.git - if !ERRORLEVEL! NEQ 0 ( - echo Failed to clone repository! - set "RELEASE_STATUS=1" - popd - goto cleanup - ) - cd cel-python - for /f "tokens=*" %%i in ('git tag --sort=-v:refname') do ( - set "VERSION=%%i" - goto :got_tag - ) - :got_tag - if "%VERSION%" == "" ( - echo Failed to get version tag! - set "RELEASE_STATUS=1" - popd - goto cleanup - ) - set "SRC_DIR=%REPO_DIR%\cel-python" + goto cleanup +) +cd cel-python +set "VERSION=" +for /f "tokens=*" %%i in ('git tag --sort=-v:refname') do ( + set "VERSION=%%i" + goto got_tag +) +:got_tag +if "%VERSION%" == "" ( + echo Failed to get version tag! + set "RELEASE_STATUS=1" popd + goto cleanup ) +set "SRC_DIR=%REPO_DIR%\cel-python" +popd +goto resolution_done + +:resolution_done if "%VERSION:~0,1%" == "v" ( set "VERSION=%VERSION:~1%" From 6375d5fa41695c41dd006460296c1f6f6ce004e2 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Thu, 25 Jun 2026 10:00:12 -0700 Subject: [PATCH 22/23] Fix git SSL certificate verification in release_linux.sh. PiperOrigin-RevId: 938044930 --- release/kokoro/release_linux.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/release/kokoro/release_linux.sh b/release/kokoro/release_linux.sh index 6375ed2..868dd70 100755 --- a/release/kokoro/release_linux.sh +++ b/release/kokoro/release_linux.sh @@ -17,7 +17,8 @@ set -e if ! command -v pip3 &> /dev/null || ! command -v curl &> /dev/null || ! command -v docker &> /dev/null || ! command -v git &> /dev/null; then echo "Installing basic dependencies..." - apt-get update && apt-get install -y python3-pip curl git + apt-get update && apt-get install -y python3-pip curl git ca-certificates + git config --global http.sslCAinfo /etc/ssl/certs/ca-certificates.crt if ! command -v docker &> /dev/null; then echo "Installing docker CLI..." From 3e369fe3dfd29ef2fcb3d8a63997e58f50e592ae Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Thu, 25 Jun 2026 15:12:50 -0700 Subject: [PATCH 23/23] Internal change PiperOrigin-RevId: 938218460 --- release/kokoro/release_linux.sh | 44 ++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/release/kokoro/release_linux.sh b/release/kokoro/release_linux.sh index 868dd70..119ec14 100755 --- a/release/kokoro/release_linux.sh +++ b/release/kokoro/release_linux.sh @@ -15,27 +15,36 @@ set -e +echo "Updating ca-certificates..." +apt-get update && apt-get install -y ca-certificates + if ! command -v pip3 &> /dev/null || ! command -v curl &> /dev/null || ! command -v docker &> /dev/null || ! command -v git &> /dev/null; then echo "Installing basic dependencies..." - apt-get update && apt-get install -y python3-pip curl git ca-certificates - git config --global http.sslCAinfo /etc/ssl/certs/ca-certificates.crt + apt-get install -y python3-pip curl git +fi - if ! command -v docker &> /dev/null; then - echo "Installing docker CLI..." - ARCH=$(uname -m) - if [ "$ARCH" = "x86_64" ]; then - DOCKER_ARCH="x86_64" - elif [ "$ARCH" = "aarch64" ]; then - DOCKER_ARCH="aarch64" - else - echo "Unsupported arch: $ARCH" - exit 1 - fi - curl -fsSL "https://download.docker.com/linux/static/stable/${DOCKER_ARCH}/docker-24.0.7.tgz" -o docker.tgz - tar xzvf docker.tgz --strip-components=1 docker/docker - mv docker /usr/local/bin/ - rm -f docker.tgz +if ! command -v docker &> /dev/null; then + echo "Installing docker CLI..." + ARCH=$(uname -m) + if [ "$ARCH" = "x86_64" ]; then + DOCKER_ARCH="x86_64" + elif [ "$ARCH" = "aarch64" ]; then + DOCKER_ARCH="aarch64" + else + echo "Unsupported arch: $ARCH" + exit 1 fi + curl -fsSL "https://download.docker.com/linux/static/stable/${DOCKER_ARCH}/docker-24.0.7.tgz" -o docker.tgz + tar xzvf docker.tgz --strip-components=1 docker/docker + mv docker /usr/local/bin/ + rm -f docker.tgz +fi + +if [ -f /var/cache/proxy.crt ]; then + echo "Using proxy certificate for Git..." + git config --global http.sslCAinfo /var/cache/proxy.crt +else + git config --global http.sslCAinfo /etc/ssl/certs/ca-certificates.crt fi # Avoid virtualenv/pip trying to download/upgrade tools from PyPI on host @@ -206,6 +215,7 @@ if [ "${DRY_RUN}" = "true" ]; then VERSION="0.1.2" fi popd + else pushd "${REPO_DIR}" git clone https://github.com/cel-expr/cel-python.git