update build matrix

postgresml · montanalow · Jan 29, 2025 · Jan 29, 2025 · Jan 29, 2025 · Jan 29, 2025
commit fd379ad50084e0b11dd3785e752180f864efede9
diff --git a/.github/workflows/ubuntu-packages-and-docker-image.yml b/.github/workflows/ubuntu-packages-and-docker-image.yml
@@ -13,7 +13,28 @@ jobs:
    strategy:
      fail-fast: false # Let the other job finish
      matrix:
-        os: ["buildjet-4vcpu-ubuntu-2204", "buildjet-8vcpu-ubuntu-2204-arm"]
+        include:
+          # Ubuntu 24.04 LTS (Noble)
+          - os: "buildjet-4vcpu-ubuntu-2404"
+            ubuntu_version: "noble"
+            arch: "amd64"
+          - os: "buildjet-8vcpu-ubuntu-2404-arm"
+            ubuntu_version: "noble"
+            arch: "arm64"
+          # Ubuntu 22.04 LTS (Jammy)
+          - os: "buildjet-4vcpu-ubuntu-2204"
+            ubuntu_version: "jammy"
+            arch: "amd64"
+          - os: "buildjet-8vcpu-ubuntu-2204-arm"
+            ubuntu_version: "jammy"
+            arch: "arm64"
+          # Ubuntu 20.04 LTS (Focal) - Consider removing after April 2025
+          - os: "buildjet-4vcpu-ubuntu-2004"
+            ubuntu_version: "focal"
+            arch: "amd64"
+          - os: "buildjet-8vcpu-ubuntu-2004-arm"
+            ubuntu_version: "focal"
+            arch: "arm64"
    runs-on: ${{ matrix.os }}
    steps:
    - uses: actions/checkout@v3
@@ -78,12 +99,14 @@ jobs:
          postgresql-14 \
          postgresql-13 \
          postgresql-12 \
+          postgresql-11 \
          postgresql-server-dev-17 \
          postgresql-server-dev-16 \
          postgresql-server-dev-15 \
          postgresql-server-dev-14 \
          postgresql-server-dev-13 \
          postgresql-server-dev-12 \
+          postgresql-server-dev-11 \
          lsb-release \
          python3.10 \
          python3-pip \
@@ -106,7 +129,13 @@ jobs:
      with:
        working-directory: pgml-extension
        command: pgrx
-        args: init --pg12=/usr/lib/postgresql/12/bin/pg_config --pg13=/usr/lib/postgresql/13/bin/pg_config --pg14=/usr/lib/postgresql/14/bin/pg_config --pg15=/usr/lib/postgresql/15/bin/pg_config --pg16=/usr/lib/postgresql/16/bin/pg_config --pg17=/usr/lib/postgresql/17/bin/pg_config
+        args: init --pg11/usr/lib/postgresql/11/bin/pg_config --pg12=/usr/lib/postgresql/12/bin/pg_config --pg13=/usr/lib/postgresql/13/bin/pg_config --pg14=/usr/lib/postgresql/14/bin/pg_config --pg15=/usr/lib/postgresql/15/bin/pg_config --pg16=/usr/lib/postgresql/16/bin/pg_config --pg17=/usr/lib/postgresql/17/bin/pg_config
+    - name: Build Postgres 11
+      uses: postgresml/gh-actions-cargo@master
+      with:
+        working-directory: pgml-extension
+        command: pgrx
+        args: package --pg-config /usr/lib/postgresql/11/bin/pg_config
    - name: Build Postgres 12
      uses: postgresml/gh-actions-cargo@master
      with:
@@ -161,9 +190,30 @@ jobs:
  postgresml:
    needs: postgresml-pgml
    strategy:
-      fail-fast: false # Let the other job finish
+      fail-fast: false
      matrix:
-        os: ["ubuntu-22.04"]
+        include:
+          # Ubuntu 24.04 LTS (Noble)
+          - os: "buildjet-4vcpu-ubuntu-2404"
+            ubuntu_version: "noble"
+            arch: "amd64"
+          - os: "buildjet-8vcpu-ubuntu-2404-arm"
+            ubuntu_version: "noble"
+            arch: "arm64"
+          # Ubuntu 22.04 LTS (Jammy)
+          - os: "buildjet-4vcpu-ubuntu-2204"
+            ubuntu_version: "jammy"
+            arch: "amd64"
+          - os: "buildjet-8vcpu-ubuntu-2204-arm"
+            ubuntu_version: "jammy"
+            arch: "arm64"
+          # Ubuntu 20.04 LTS (Focal) - Consider removing after April 2025
+          - os: "buildjet-4vcpu-ubuntu-2004"
+            ubuntu_version: "focal"
+            arch: "amd64"
+          - os: "buildjet-8vcpu-ubuntu-2004-arm"
+            ubuntu_version: "focal"
+            arch: "arm64"
    runs-on: ${{ matrix.os }}
    steps:
    - uses: actions/checkout@v3
@@ -180,9 +230,30 @@ jobs:
  #
  postgresml-dashboard:
    strategy:
-      fail-fast: false # Let the other job finish
+      fail-fast: false
      matrix:
-        os: ["ubuntu-22.04", "buildjet-4vcpu-ubuntu-2204-arm"]
+        include:
+          # Ubuntu 24.04 LTS (Noble)
+          - os: "buildjet-4vcpu-ubuntu-2404"
+            ubuntu_version: "noble"
+            arch: "amd64"
+          - os: "buildjet-8vcpu-ubuntu-2404-arm"
+            ubuntu_version: "noble"
+            arch: "arm64"
+          # Ubuntu 22.04 LTS (Jammy)
+          - os: "buildjet-4vcpu-ubuntu-2204"
+            ubuntu_version: "jammy"
+            arch: "amd64"
+          - os: "buildjet-8vcpu-ubuntu-2204-arm"
+            ubuntu_version: "jammy"
+            arch: "arm64"
+          # Ubuntu 20.04 LTS (Focal) - Consider removing after April 2025
+          - os: "buildjet-4vcpu-ubuntu-2004"
+            ubuntu_version: "focal"
+            arch: "amd64"
+          - os: "buildjet-8vcpu-ubuntu-2004-arm"
+            ubuntu_version: "focal"
+            arch: "arm64"
    runs-on: ${{ matrix.os }}
    steps:
    - uses: actions/checkout@v3
@@ -208,7 +279,9 @@ jobs:
    strategy:
      fail-fast: false # Let the other job finish
      matrix:
-        os: ["buildjet-4vcpu-ubuntu-2204"]
+        include:
+          - os: "buildjet-4vcpu-ubuntu-2404"
+            ubuntu_version: "noble"
    runs-on: ${{ matrix.os }}
    defaults:
      run:

diff --git a/.github/workflows/ubuntu-postgresml-python-package.yaml b/.github/workflows/ubuntu-postgresml-python-package.yaml
@@ -4,14 +4,35 @@ on:
  workflow_dispatch:
    inputs:
      packageVersion:
-        default: "2.8.4"
+        default: "2.10.0"

 jobs:
  postgresml-python:
    strategy:
      fail-fast: false # Let the other job finish
      matrix:
-        os: ["buildjet-4vcpu-ubuntu-2204", "buildjet-4vcpu-ubuntu-2204-arm", "ubuntu-24.04"]
+        include:
+          # Ubuntu 24.04 LTS (Noble)
+          - os: "buildjet-4vcpu-ubuntu-2404"
+            ubuntu_version: "noble"
+            arch: "amd64"
+          - os: "buildjet-8vcpu-ubuntu-2404-arm"
+            ubuntu_version: "noble"
+            arch: "arm64"
+          # Ubuntu 22.04 LTS (Jammy)
+          - os: "buildjet-4vcpu-ubuntu-2204"
+            ubuntu_version: "jammy"
+            arch: "amd64"
+          - os: "buildjet-8vcpu-ubuntu-2204-arm"
+            ubuntu_version: "jammy"
+            arch: "arm64"
+          # Ubuntu 20.04 LTS (Focal) - Consider removing after April 2025
+          - os: "buildjet-4vcpu-ubuntu-2004"
+            ubuntu_version: "focal"
+            arch: "amd64"
+          - os: "buildjet-8vcpu-ubuntu-2004-arm"
+            ubuntu_version: "focal"
+            arch: "arm64"
    runs-on: ${{ matrix.os }}
    steps:
    - uses: actions/checkout@v3

diff --git a/packages/postgresml-dashboard/build.sh b/packages/postgresml-dashboard/build.sh
@@ -1,14 +1,15 @@
 #!/bin/bash
 set -e

-# Parse arguments
-PACKAGE_VERSION=${1:-"2.10.0"}
-UBUNTU_VERSION=${2:-"22.04"}
+# Parse arguments with environment variable fallbacks
+PACKAGE_VERSION=${1:-${PACKAGE_VERSION:-"2.10.0"}}
+UBUNTU_VERSION=${2:-${ubuntu_version:-$(lsb_release -rs)}}
+ARCH=${3:-${ARCH:-$(arch | sed 's/x86_64/amd64/; s/aarch64/arm64/')}}

 if [[ -z "$PACKAGE_VERSION" ]]; then
  echo "postgresml dashboard build script"
-  echo "Usage: $0 <package version> [ubuntu version]"
-  echo "Example: $0 2.10.0 22.04"
+  echo "Usage: $0 <package version> [ubuntu version] [arch]"
+  echo "Example: $0 2.10.0 22.04 amd64"
  exit 1
 fi

@@ -19,12 +20,16 @@ source_dir="$dir/source"

 export PACKAGE_VERSION
 export UBUNTU_VERSION
-export GITHUB_STARS=$(curl -s "https://api.github.com/repos/postgresml/postgresml" | grep stargazers_count | cut -d : -f 2 | tr -d " " | tr -d ",")
-if [[ $(arch) == "x86_64" ]]; then
-  export ARCH=amd64
-else
-  export ARCH=arm64
-fi
+export ARCH
+
+# Fetch GitHub stars count with error handling
+GITHUB_STARS=$(curl -s "https://api.github.com/repos/postgresml/postgresml" | grep stargazers_count | cut -d : -f 2 | tr -d " " | tr -d "," || echo "0")
+export GITHUB_STARS
+
+echo "Building dashboard package:"
+echo "- Version: ${PACKAGE_VERSION}"
+echo "- Ubuntu: ${UBUNTU_VERSION}"
+echo "- Architecture: ${ARCH}"

 rm -rf "$dir"
 mkdir -p "$deb_dir"
@@ -33,7 +38,6 @@ cp -R ${SCRIPT_DIR}/* "$deb_dir"
 rm "$deb_dir/build.sh"
 rm "$deb_dir/release.sh"

-
 ( cd ${SCRIPT_DIR}/../../pgml-dashboard && \
  cargo build --release && \
  cp target/release/pgml-dashboard "$deb_dir/usr/bin/pgml-dashboard" && \

diff --git a/packages/postgresml-dashboard/release.sh b/packages/postgresml-dashboard/release.sh
@@ -10,55 +10,64 @@ if [[ -z "$package_version" ]]; then
  exit 1
 fi

-# Active LTS Ubuntu versions and their codenames
-declare -A ubuntu_versions=(
-  ["20.04"]="focal"
-  ["22.04"]="jammy"
-  ["24.04"]="noble"
-)
+# Set architecture based on system unless overridden by environment
+if [[ -z "${ARCH}" ]]; then
+  if [[ $(arch) == "x86_64" ]]; then
+    export ARCH=amd64
+  else
+    export ARCH=arm64
+  fi
+fi
+
+# Get Ubuntu version from environment or try to detect it
+if [[ -z "${ubuntu_version}" ]]; then
+  ubuntu_version=$(lsb_release -rs)
+  echo "No ubuntu_version specified, detected: ${ubuntu_version}"
+fi

-# Supported architectures
-declare -a architectures=("amd64" "arm64")
+# Map version number to codename
+case "${ubuntu_version}" in
+  "20.04")
+    export CODENAME="focal"
+    ;;
+  "22.04")
+    export CODENAME="jammy"
+    ;;
+  "24.04")
+    export CODENAME="noble"
+    ;;
+  *)
+    echo "Error: Unsupported Ubuntu version: ${ubuntu_version}"
+    exit 1
+    ;;
+esac

-# Install deb-s3 if not present
 if ! which deb-s3; then
  curl -sLO https://github.com/deb-s3/deb-s3/releases/download/0.11.4/deb-s3-0.11.4.gem
  sudo gem install deb-s3-0.11.4.gem
  deb-s3
 fi

 function package_name() {
-  local ubuntu_version=$1
-  local arch=$2
-  echo "postgresml-dashboard-${package_version}-ubuntu${ubuntu_version}-${arch}.deb"
+  echo "postgresml-dashboard-${package_version}-ubuntu${ubuntu_version}-${ARCH}.deb"
 }

-# Loop through Ubuntu versions
-for ubuntu_version in "${!ubuntu_versions[@]}"; do
-  codename=${ubuntu_versions[$ubuntu_version]}
-  echo "Building packages for Ubuntu ${ubuntu_version} (${codename})"
+echo "Building package for Ubuntu ${ubuntu_version} (${CODENAME}) ${ARCH}"

-  # Loop through architectures
-  for arch in "${architectures[@]}"; do
-    echo "Building for architecture: ${arch}"
-    export ARCH=${arch}
+# Build the package
+bash ${SCRIPT_DIR}/build.sh "$package_version" "$ubuntu_version" "$ARCH"

-    # Build the dashboard package
-    bash ${SCRIPT_DIR}/build.sh "$package_version" "$ubuntu_version"
-
-    if [[ ! -f $(package_name ${ubuntu_version} ${arch}) ]]; then
-      echo "File $(package_name ${ubuntu_version} ${arch}) doesn't exist"
-      exit 1
-    fi
+if [[ ! -f $(package_name) ]]; then
+  echo "File $(package_name) doesn't exist"
+  exit 1
+fi

-    # Upload to S3
-    deb-s3 upload \
-      --lock \
-      --bucket apt.postgresml.org \
-      $(package_name ${ubuntu_version} ${arch}) \
-      --codename ${codename}
+# Upload to S3
+deb-s3 upload \
+  --lock \
+  --bucket apt.postgresml.org \
+  $(package_name) \
+  --codename ${CODENAME}

-    # Clean up the package file
-    rm $(package_name ${ubuntu_version} ${arch})
-  done
-done
+# Clean up the package file
+rm $(package_name)
diff --git a/packages/postgresml-python/build.sh b/packages/postgresml-python/build.sh
@@ -4,29 +4,25 @@ set -e
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 deb_dir="/tmp/postgresml-python/deb-build"

-# Parse arguments with defaults
-export PACKAGE_VERSION=${1:-"2.10.0"}
-export UBUNTU_VERSION=${2:-"22.04"}
-export PYTHON_VERSION=${3:-"3.10"}
+# Parse arguments with environment variable fallbacks
+export PACKAGE_VERSION=${1:-${PACKAGE_VERSION:-"2.10.0"}}
+export UBUNTU_VERSION=${2:-${ubuntu_version:-$(lsb_release -rs)}}
+export PYTHON_VERSION=${3:-${PYTHON_VERSION:-"3.10"}}

-# Handle architecture
-if [[ $(arch) == "x86_64" ]]; then
-  export ARCH=amd64
-else
-  export ARCH=arm64
+# Set architecture from environment or detect it
+if [[ -z "${ARCH}" ]]; then
+  if [[ $(arch) == "x86_64" ]]; then
+    export ARCH=amd64
+  else
+    export ARCH=arm64
+  fi
 fi

-# Map Ubuntu versions to Python versions if needed
-# For example: Ubuntu 20.04 uses Python 3.8 by default
-declare -A ubuntu_python_versions=(
-  ["20.04"]="3.8"
-  ["22.04"]="3.10"
-  ["24.04"]="3.11"
-)
-
-if [[ -z "$3" ]]; then
-  PYTHON_VERSION=${ubuntu_python_versions[$UBUNTU_VERSION]:-"3.10"}
-fi
+echo "Building package:"
+echo "- Package Version: ${PACKAGE_VERSION}"
+echo "- Ubuntu Version: ${UBUNTU_VERSION}"
+echo "- Python Version: ${PYTHON_VERSION}"
+echo "- Architecture: ${ARCH}"

 rm -rf "$deb_dir"
 mkdir -p "$deb_dir"
@@ -35,17 +31,21 @@ cp -R ${SCRIPT_DIR}/* "$deb_dir"
 rm "$deb_dir/build.sh"
 rm "$deb_dir/release.sh"

+# Process template files
 (cat ${SCRIPT_DIR}/DEBIAN/control | envsubst '${PACKAGE_VERSION} ${UBUNTU_VERSION} ${ARCH} ${PYTHON_VERSION}') > "$deb_dir/DEBIAN/control"
 (cat ${SCRIPT_DIR}/DEBIAN/postinst | envsubst '${PGVERSION} ${PYTHON_VERSION}') > "$deb_dir/DEBIAN/postinst"
 (cat ${SCRIPT_DIR}/DEBIAN/prerm | envsubst '${PGVERSION} ${PYTHON_VERSION}') > "$deb_dir/DEBIAN/prerm"
 (cat ${SCRIPT_DIR}/DEBIAN/postrm | envsubst '${PGVERSION} ${PYTHON_VERSION}') > "$deb_dir/DEBIAN/postrm"

+# Copy appropriate requirements file based on architecture
 if [[ "$ARCH" == "amd64" ]]; then
  cp ${SCRIPT_DIR}/../../pgml-extension/requirements.linux.txt "$deb_dir/etc/postgresml-python/requirements.txt"
 else
  cp ${SCRIPT_DIR}/../../pgml-extension/requirements.macos.txt "$deb_dir/etc/postgresml-python/requirements.txt"
 fi

+# Create and populate virtualenv
+echo "Creating Python virtual environment with Python ${PYTHON_VERSION}"
 virtualenv --python="python${PYTHON_VERSION}" "$deb_dir/var/lib/postgresml-python/pgml-venv"
 source "$deb_dir/var/lib/postgresml-python/pgml-venv/bin/activate"