.yaml
+```
+
+
---
diff --git a/pyproject.toml b/pyproject.toml
index 544dd42..17a98c1 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -10,9 +10,9 @@ readme = "README.md"
requires-python = ">=3.7"
license = {text = "MIT"}
authors = [
- {name = "TensorFuse", email = "team@tensorfuse.ai"}
+ {name = "TensorFuse", email = "saurabh@tensorfuse.io"}
]
-keywords = ["containers", "docker", "nydus", "snapshotter", "ml", "ai"]
+keywords = ["containers", "docker", "fastpull", "snapshotter", "ml", "ai"]
classifiers = [
"Development Status :: 4 - Beta",
"Intended Audience :: Developers",
diff --git a/scripts/builder/Dockerfile b/scripts/builder/Dockerfile
new file mode 100644
index 0000000..f90e9d5
--- /dev/null
+++ b/scripts/builder/Dockerfile
@@ -0,0 +1,56 @@
+# Build stage: Compile buildkit with Nydus support
+FROM golang:1.21-alpine AS buildkit-builder
+
+# Install build dependencies
+RUN apk add --no-cache git make
+
+# Clone nydusaccelerator/buildkit fork
+ARG BUILDKIT_VERSION=nydus-compression-type-enhance
+RUN git clone --depth 1 --branch ${BUILDKIT_VERSION} \
+ https://github.com/nydusaccelerator/buildkit.git /buildkit
+
+WORKDIR /buildkit
+
+# Build buildkitd and buildctl with Nydus support
+RUN go build -tags=nydus -o ./bin/buildkitd ./cmd/buildkitd && \
+ go build -o ./bin/buildctl ./cmd/buildctl
+
+# Runtime stage
+FROM alpine:latest
+
+# Copy buildkit binaries with Nydus support
+COPY --from=buildkit-builder /buildkit/bin/buildctl /usr/bin/buildctl
+COPY --from=buildkit-builder /buildkit/bin/buildkitd /usr/bin/buildkitd
+
+# Copy buildctl-daemonless.sh wrapper from moby/buildkit repo
+ADD https://raw.githubusercontent.com/moby/buildkit/master/examples/buildctl-daemonless/buildctl-daemonless.sh /usr/bin/buildctl-daemonless.sh
+RUN chmod +x /usr/bin/buildctl-daemonless.sh
+
+# Install runtime dependencies
+RUN apk add --no-cache \
+ ca-certificates \
+ curl \
+ wget \
+ iptables \
+ fuse-overlayfs \
+ containerd
+
+# Install nydus-image binary (v2.3.6)
+ARG NYDUS_VERSION=v2.3.6
+RUN wget -O /tmp/nydus.tgz \
+ "https://github.com/dragonflyoss/nydus/releases/download/${NYDUS_VERSION}/nydus-static-${NYDUS_VERSION}-linux-amd64.tgz" \
+ && tar -xzf /tmp/nydus.tgz -C /tmp \
+ && mv /tmp/nydus-static/nydus-image /usr/bin/nydus-image \
+ && chmod +x /usr/bin/nydus-image \
+ && rm -rf /tmp/nydus.tgz /tmp/nydus-static
+
+# Set NYDUS_BUILDER environment variable (required for buildkit)
+ENV NYDUS_BUILDER=/usr/bin/nydus-image
+
+# Copy build script
+COPY build.sh /usr/local/bin/build.sh
+RUN chmod +x /usr/local/bin/build.sh
+
+WORKDIR /workspace
+
+ENTRYPOINT ["/usr/local/bin/build.sh"]
diff --git a/scripts/builder/README.md b/scripts/builder/README.md
new file mode 100644
index 0000000..450d4f0
--- /dev/null
+++ b/scripts/builder/README.md
@@ -0,0 +1,156 @@
+# Container-Based Image Builder
+
+Builds container images using `buildctl` in a containerized environment. Produces both normal OCI and Nydus-optimized images.
+
+## Features
+
+- **Registry-agnostic**: Works with AWS ECR, Google Artifact Registry, Docker Hub, or any OCI registry
+- **No local dependencies**: All build tools run inside a container
+- **Two image formats**: Builds both normal OCI and Nydus images in one go
+- **Direct push**: Images pushed directly to registry via buildctl
+
+## Architecture
+
+```
+Host (authenticated) → Builder Container (buildctl + nydus-image) → Registry
+```
+
+- **Host**: Authenticates to registry, mounts build context and docker config
+- **Builder Container**: Runs buildctl to build and push images
+- **No Docker daemon dependency**: buildctl pushes directly to registries
+
+## Prerequisites
+
+1. **Docker** installed on host (no other dependencies needed!)
+2. **Authenticated to your registry** before running:
+
+```bash
+# AWS ECR
+aws ecr get-login-password --region us-east-1 | \
+ docker login --username AWS --password-stdin 123456789.dkr.ecr.us-east-1.amazonaws.com
+
+# Google Artifact Registry
+gcloud auth configure-docker us-central1-docker.pkg.dev
+
+# Docker Hub
+docker login
+```
+
+## Usage
+
+```bash
+docker run --rm --privileged \
+ -v /path/to/build-context:/workspace:ro \
+ -v ~/.docker/config.json:/root/.docker/config.json:ro \
+ tensorfuse/fastpull-builder:latest \
+
+```
+
+### Examples
+
+**AWS ECR:**
+```bash
+docker run --rm --privileged \
+ -v ./my-app:/workspace:ro \
+ -v ~/.docker/config.json:/root/.docker/config.json:ro \
+ tensorfuse/fastpull-builder:latest \
+ 123456789.dkr.ecr.us-east-1.amazonaws.com/my-app:latest
+```
+
+**Google Artifact Registry:**
+```bash
+docker run --rm --privileged \
+ -v ./my-app:/workspace:ro \
+ -v ~/.docker/config.json:/root/.docker/config.json:ro \
+ tensorfuse/fastpull-builder:latest \
+ us-central1-docker.pkg.dev/my-project/my-repo/my-app:v1.0
+```
+
+**Docker Hub:**
+```bash
+docker run --rm --privileged \
+ -v ./my-app:/workspace:ro \
+ -v ~/.docker/config.json:/root/.docker/config.json:ro \
+ tensorfuse/fastpull-builder:latest \
+ docker.io/username/my-app:latest
+```
+
+**No tag (defaults to :latest):**
+```bash
+docker run --rm --privileged \
+ -v ./my-app:/workspace:ro \
+ -v ~/.docker/config.json:/root/.docker/config.json:ro \
+ tensorfuse/fastpull-builder:latest \
+ my-registry.com/my-app
+```
+
+**Custom Dockerfile:**
+```bash
+docker run --rm --privileged \
+ -v ./my-app:/workspace:ro \
+ -v ~/.docker/config.json:/root/.docker/config.json:ro \
+ -e DOCKERFILE=Dockerfile.custom \
+ tensorfuse/fastpull-builder:latest \
+ my-registry.com/my-app:latest
+```
+
+## Output
+
+The script builds and pushes two images:
+- `:` - Normal OCI image
+- `:-fastpull` - Fastpull-optimized image
+
+## Files
+
+- `Dockerfile` - Builder container definition (builds from nydusaccelerator/buildkit fork)
+- `build.sh` - Build script that runs inside container (entrypoint)
+- `README.md` - This file
+
+## Technical Details
+
+### Buildkit with Nydus Support
+The Dockerfile builds `buildkitd` and `buildctl` from the [nydusaccelerator/buildkit](https://github.com/nydusaccelerator/buildkit) fork with the `-tags=nydus` flag, which enables Nydus compression support. The standard moby/buildkit does not include this functionality.
+
+### Components
+- **buildkitd/buildctl**: Compiled from nydusaccelerator/buildkit fork
+- **nydus-image**: v2.3.6 binary (set via `NYDUS_BUILDER` env var)
+- **buildctl-daemonless.sh**: Wrapper that runs buildkitd in rootless mode
+
+## How It Works
+
+1. **Pull builder image**: Downloads `tensorfuse/fastpull-builder:latest` from Docker Hub
+2. **Mount context**: Your build context is mounted read-only into `/workspace`
+3. **Mount auth**: `~/.docker/config.json` is mounted for registry authentication
+4. **Run buildctl**: Builds normal OCI image with `buildctl-daemonless.sh`
+5. **Run buildctl again**: Builds Fastpull image with Nydus compression
+6. **Direct push**: Both images pushed directly to registry
+
+## Troubleshooting
+
+**"Error: Docker config not found"**
+- Run registry authentication command first (see Prerequisites)
+
+**"Error: Build context path does not exist"**
+- Check that `--context` points to a valid directory
+
+**"Error: Dockerfile not found"**
+- Ensure Dockerfile exists in context directory
+- Or specify custom name with `--dockerfile`
+
+**Build fails with authentication error:**
+- Re-authenticate to your registry
+- Check that `~/.docker/config.json` contains valid credentials
+
+**"permission denied" errors:**
+- Builder container runs with `--privileged` flag (required for buildkit)
+- Ensure Docker is running with appropriate permissions
+
+## Comparison with Original build_push.py
+
+| Feature | Original | Container-Based |
+|---------|----------|-----------------|
+| Dependencies | Requires nerdctl, nydusify, soci, stargz locally | All tools in container |
+| Registry | AWS ECR or GAR | Any OCI registry |
+| Formats | normal, nydus, soci, estargz | normal, nydus |
+| Push method | nerdctl/docker | buildctl (direct) |
+| Portability | Requires snapshotter setup | Runs anywhere Docker runs |
diff --git a/scripts/builder/build.sh b/scripts/builder/build.sh
new file mode 100644
index 0000000..8858ccf
--- /dev/null
+++ b/scripts/builder/build.sh
@@ -0,0 +1,72 @@
+#!/bin/sh
+set -e
+
+# Usage: build.sh
+# Example: build.sh my-registry.com/my-app:latest
+# Example: build.sh my-registry.com/my-app (defaults to :latest)
+
+if [ $# -lt 1 ]; then
+ echo "Usage: $0 "
+ echo "Example: $0 123456789.dkr.ecr.us-east-1.amazonaws.com/my-app:v1.0"
+ echo "Example: $0 123456789.dkr.ecr.us-east-1.amazonaws.com/my-app (defaults to :latest)"
+ exit 1
+fi
+
+IMAGE_WITH_TAG="$1"
+DOCKERFILE="${DOCKERFILE:-Dockerfile}"
+CONTEXT_PATH="${CONTEXT_PATH:-/workspace}"
+
+# Parse image and tag (default to :latest if no tag provided)
+if echo "$IMAGE_WITH_TAG" | grep -q ":"; then
+ IMAGE_NAME="${IMAGE_WITH_TAG%:*}"
+ TAG="${IMAGE_WITH_TAG##*:}"
+else
+ IMAGE_NAME="$IMAGE_WITH_TAG"
+ TAG="latest"
+fi
+
+FULL_IMAGE="${IMAGE_NAME}:${TAG}"
+FULL_IMAGE_FASTPULL="${IMAGE_NAME}:${TAG}-fastpull"
+
+echo "=========================================="
+echo "Building images for: ${IMAGE_NAME}"
+echo "Tag: ${TAG}"
+echo "Context: ${CONTEXT_PATH}"
+echo "Dockerfile: ${DOCKERFILE}"
+echo "=========================================="
+
+# Build normal OCI image
+echo ""
+echo ">>> Building normal OCI image: ${FULL_IMAGE}"
+echo ""
+time buildctl-daemonless.sh build \
+ --frontend dockerfile.v0 \
+ --local context="${CONTEXT_PATH}" \
+ --local dockerfile="${CONTEXT_PATH}" \
+ --opt filename="${DOCKERFILE}" \
+ --output type=image,name="${FULL_IMAGE}",push=true
+
+echo ""
+echo "✓ Normal OCI image built and pushed: ${FULL_IMAGE}"
+echo ""
+
+# Build Fastpull image
+echo ""
+echo ">>> Building Fastpull image: ${FULL_IMAGE_FASTPULL}"
+echo ""
+time buildctl-daemonless.sh build \
+ --frontend dockerfile.v0 \
+ --local context="${CONTEXT_PATH}" \
+ --local dockerfile="${CONTEXT_PATH}" \
+ --opt filename="${DOCKERFILE}" \
+ --output type=image,name="${FULL_IMAGE_FASTPULL}",push=true,compression=nydus,force-compression=true,oci-mediatypes=true
+
+echo ""
+echo "✓ Fastpull image built and pushed: ${FULL_IMAGE_FASTPULL}"
+echo ""
+
+echo "=========================================="
+echo "✓ Build complete!"
+echo " Normal: ${FULL_IMAGE}"
+echo " Fastpull: ${FULL_IMAGE_FASTPULL}"
+echo "=========================================="
diff --git a/scripts/fastpull/benchmark.py b/scripts/fastpull/benchmark.py
index 2979423..f79d228 100644
--- a/scripts/fastpull/benchmark.py
+++ b/scripts/fastpull/benchmark.py
@@ -89,12 +89,17 @@ def wait_for_readiness(self, timeout: int = 600, poll_interval: int = 2):
if self.benchmark_mode != 'readiness' or not self.readiness_endpoint:
return True
- print(f"Polling {self.readiness_endpoint} for readiness...")
+ # Ensure endpoint has protocol prefix
+ endpoint = self.readiness_endpoint
+ if not endpoint.startswith(('http://', 'https://')):
+ endpoint = f'http://{endpoint}'
+
+ print(f"Polling {endpoint} for readiness...")
end_time = time.time() + timeout
while time.time() < end_time:
try:
- response = urlopen(self.readiness_endpoint, timeout=5)
+ response = urlopen(endpoint, timeout=5)
if response.getcode() == 200:
elapsed = time.time() - self.start_time
self.metrics['readiness_time'] = elapsed
diff --git a/scripts/fastpull/build.py b/scripts/fastpull/build.py
index 4a2d675..7418d17 100644
--- a/scripts/fastpull/build.py
+++ b/scripts/fastpull/build.py
@@ -125,12 +125,6 @@ def build_command(args):
print(f"Error: Invalid format '{fmt}'. Valid: {', '.join(valid_formats)}")
sys.exit(1)
- # Authenticate with registry
- print(f"\nAuthenticating with {args.registry}...")
- if not authenticate_registry(args):
- print("Error: Authentication failed")
- sys.exit(1)
-
# Determine build mode
if args.dockerfile_path:
# Mode 1: Build from Dockerfile
@@ -270,7 +264,7 @@ def build_from_dockerfile(args, formats: List[str]):
# Convert to other formats
if 'nydus' in formats:
- nydus_image = f"{args.repository_url.rsplit(':', 1)[0]}:{args.repository_url.rsplit(':', 1)[1]}-nydus"
+ nydus_image = f"{args.repository_url.rsplit(':', 1)[0]}:{args.repository_url.rsplit(':', 1)[1]}-fastpull"
if convert_to_nydus(args.repository_url, nydus_image):
built_images.append(nydus_image)
@@ -298,7 +292,7 @@ def convert_existing_image(args, formats: List[str]):
# Convert to requested formats
if 'nydus' in formats:
- nydus_image = f"{args.repository_url.rsplit(':', 1)[0]}:{args.repository_url.rsplit(':', 1)[1]}-nydus"
+ nydus_image = f"{args.repository_url.rsplit(':', 1)[0]}:{args.repository_url.rsplit(':', 1)[1]}-fastpull"
if convert_to_nydus(args.repository_url, nydus_image):
built_images.append(nydus_image)
@@ -322,7 +316,7 @@ def build_and_push_docker(args) -> bool:
# Build
cmd = [
- 'docker', 'build',
+ 'sudo', 'docker', 'build',
'-t', args.repository_url,
'-f', os.path.join(args.dockerfile_path, args.dockerfile)
]
@@ -346,7 +340,7 @@ def build_and_push_docker(args) -> bool:
# Push
print(f"[Docker] Pushing {args.repository_url}...")
try:
- subprocess.run(['docker', 'push', args.repository_url], check=True)
+ subprocess.run(['sudo', 'docker', 'push', args.repository_url], check=True)
print(f"[Docker] ✓ Pushed {args.repository_url}")
return True
except subprocess.CalledProcessError:
diff --git a/scripts/fastpull/run.py b/scripts/fastpull/run.py
index 71a8f67..3cfb0cb 100644
--- a/scripts/fastpull/run.py
+++ b/scripts/fastpull/run.py
@@ -26,7 +26,7 @@ def add_parser(subparsers):
'--mode',
choices=['nydus', 'normal'],
default='nydus',
- help='Run mode: nydus (default, adds -nydus suffix) or normal (overlayfs, no suffix)'
+ help='Run mode: nydus (default, adds -fastpull suffix) or normal (overlayfs, no suffix)'
)
# Benchmarking arguments
@@ -81,13 +81,13 @@ def run_command(args):
# Determine snapshotter and modify image tag based on mode
if args.mode == 'nydus':
args.snapshotter = 'nydus'
- # Add -nydus suffix to image tag if not already present
+ # Add -fastpull suffix to image tag if not already present
if ':' in args.image:
base, tag = args.image.rsplit(':', 1)
- if not tag.endswith('-nydus'):
- args.image = f"{base}:{tag}-nydus"
+ if not tag.endswith('-fastpull'):
+ args.image = f"{base}:{tag}-fastpull"
else:
- args.image = f"{args.image}:latest-nydus"
+ args.image = f"{args.image}:latest-fastpull"
else: # normal mode
args.snapshotter = 'overlayfs'
# Use image as-is for normal mode