From ad1b91757a9a1429ac78c2ed616d909422ead709 Mon Sep 17 00:00:00 2001
From: Rajat Ahuja <rahuja445@gmail.com>
Date: Tue, 9 Dec 2025 16:55:30 -0500
Subject: [PATCH] feat: run unified tests in CI

---
 .github/workflows/start-fly-runner.yml | 163 +++++++++++++++++++
 .github/workflows/unified-tests.yml    | 207 +++++++++++++++++++++++++
 pyproject.toml                         |   1 +
 tests/unified/runner.py                | 125 ++++++++++++++-
 uv.lock                                |  72 +++++++++
 5 files changed, 567 insertions(+), 1 deletion(-)
 create mode 100644 .github/workflows/start-fly-runner.yml
 create mode 100644 .github/workflows/unified-tests.yml

diff --git a/.github/workflows/start-fly-runner.yml b/.github/workflows/start-fly-runner.yml
new file mode 100644
index 00000000..7f20e18a
--- /dev/null
+++ b/.github/workflows/start-fly-runner.yml
@@ -0,0 +1,163 @@
+name: Start Fly Runner
+
+on:
+  workflow_call:
+    outputs:
+      runner-ready:
+        description: "Whether the runner is ready"
+        value: ${{ jobs.start-runner.outputs.runner-ready }}
+      machine-id:
+        description: "The Fly machine ID that was started"
+        value: ${{ jobs.start-runner.outputs.machine-id }}
+      runner-labels:
+        description: "Labels to target the self-hosted runner"
+        value: ${{ jobs.start-runner.outputs.runner-labels }}
+      runner-name:
+        description: "Resolved GitHub runner name"
+        value: ${{ jobs.start-runner.outputs.runner-name }}
+
+env:
+  FLY_RUNNER_APP: ivysaur
+  FLY_RUNNER_REGION: iad
+  FLY_RUNNER_IMAGE: registry.fly.io/ivysaur:latest
+
+jobs:
+  start-runner:
+    name: Start Fly Runner
+    runs-on: ubuntu-latest
+    outputs:
+      runner-ready: ${{ steps.wait-for-runner.outputs.ready }}
+      machine-id: ${{ steps.machine-management.outputs.machine-id }}
+      runner-labels: ${{ steps.generate-labels.outputs.labels }}
+      runner-name: ${{ steps.wait-for-runner.outputs.runner-name }}
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Generate unique runner labels
+        id: generate-labels
+        run: |
+          UNIQUE_LABELS='"self-hosted","${{ github.run_id }}"'
+          echo "Generated unique labels: $UNIQUE_LABELS"
+          echo "labels=$UNIQUE_LABELS" >> "$GITHUB_OUTPUT"
+
+      - name: Setup Fly CLI
+        uses: superfly/flyctl-actions/setup-flyctl@master
+
+      - name: Get Fly app info
+        id: get-app-info
+        env:
+          FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN_TESTING }}
+        run: |
+          echo "Getting app info for ${FLY_RUNNER_APP}..."
+          flyctl status -a "${FLY_RUNNER_APP}"
+
+      - name: Fly machine management
+        id: machine-management
+        env:
+          FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN_TESTING }}
+        run: |
+          set -euo pipefail
+
+          # Get list of machines and their status
+          MACHINES_JSON=$(flyctl machines list -a "${FLY_RUNNER_APP}" --json)
+
+          # Count online machines
+          ONLINE_COUNT=$(echo "$MACHINES_JSON" | jq '[.[] | select(.state == "started")] | length')
+          echo "📊 Online machines: $ONLINE_COUNT"
+
+          if [ "$ONLINE_COUNT" -ge 2 ]; then
+            echo "✅ Found $ONLINE_COUNT online machines (>=2), will reuse existing machine"
+
+            # Pick the first online machine
+            MACHINE_ID=$(echo "$MACHINES_JSON" | jq -r '[.[] | select(.state == "started")][0].id')
+
+            if [ -z "$MACHINE_ID" ] || [ "$MACHINE_ID" = "null" ]; then
+              echo "❌ Failed to find online machine ID"
+              exit 1
+            fi
+
+            echo "🔄 Reusing machine ID: $MACHINE_ID"
+            echo "machine-id=$MACHINE_ID" >> "$GITHUB_OUTPUT"
+
+          else
+            echo "📦 Need to create new machine (only $ONLINE_COUNT online machines)"
+
+            # Get available volumes
+            VOLUMES_JSON=$(flyctl volumes list -a "${FLY_RUNNER_APP}" --json)
+
+            # Find a volume with null attached_machine_id
+            AVAILABLE_VOLUME=$(echo "$VOLUMES_JSON" | jq -r '[.[] | select(.attached_machine_id == null)][0].id')
+
+            if [ -z "$AVAILABLE_VOLUME" ] || [ "$AVAILABLE_VOLUME" = "null" ]; then
+              echo "❌ No available volumes found"
+              echo "Available volumes: $(echo "$VOLUMES_JSON" | jq -r '.[] | select(.attached_machine_id == null) | .id')"
+              exit 1
+            fi
+
+            echo "📁 Using available volume: $AVAILABLE_VOLUME"
+
+            # Create new machine with the available volume
+            FULL_OUTPUT=$(flyctl machines run "${FLY_RUNNER_IMAGE}" \
+              -a "${FLY_RUNNER_APP}" \
+              --region "${FLY_RUNNER_REGION}" \
+              --env GH_TOKEN=${{ secrets.GH_TOKEN_ACTIONS }} \
+              --env RUN_ID=${{ github.run_id }} \
+              --env TEST_TYPE="honcho-unified-runner" \
+              --vm-size shared-cpu-8x \
+              --vm-memory 8192 \
+              --volume "$AVAILABLE_VOLUME":/mnt/vol )
+
+            MACHINE_ID=$(echo "$FULL_OUTPUT" | grep "Machine ID:" | awk '{print $3}')
+            echo "✅ Created new machine ID: $MACHINE_ID"
+            echo "machine-id=$MACHINE_ID" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Wait for runner to be online
+        id: wait-for-runner
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_TOKEN_ACTIONS }}
+          MAX_WAIT: 420
+        run: |
+          set -euo pipefail
+          if [ -z "${GITHUB_TOKEN}" ]; then
+            echo "GH_TOKEN secret is required to poll the Actions runner API."
+            exit 1
+          fi
+
+          EXPECTED_RUNNER_NAME="honcho-unified-runner-${{ github.run_id }}"
+          echo "Waiting for runner named ${EXPECTED_RUNNER_NAME} to come online..."
+
+          WAITED=0
+          RUNNER_NAME=""
+          while [ $WAITED -lt $MAX_WAIT ]; do
+            RESPONSE=$(curl -s \
+              -H "Authorization: Bearer ${GITHUB_TOKEN}" \
+              -H "Accept: application/vnd.github.v3+json" \
+              "https://api.github.com/repos/${{ github.repository }}/actions/runners")
+
+            if echo "$RESPONSE" | grep -q '"message"'; then
+              echo "API Error: $(echo "$RESPONSE" | jq -r '.message')"
+              exit 1
+            fi
+
+            # Find runner with exact name and is online and not busy
+            RUNNER_LINE=$(echo "$RESPONSE" | jq -r --arg runner_name "$EXPECTED_RUNNER_NAME" '.runners[]? | select(.name == $runner_name) | select(.status == "online") | select(.busy == false) | "\(.name)|\(.id)"' | head -n 1)
+
+            if [ -n "$RUNNER_LINE" ]; then
+              RUNNER_NAME=$(echo "$RUNNER_LINE" | cut -d'|' -f1)
+              echo "✅ Found runner: ${RUNNER_NAME}"
+              echo "ready=true" >> "$GITHUB_OUTPUT"
+              echo "runner-name=${RUNNER_NAME}" >> "$GITHUB_OUTPUT"
+              exit 0
+            fi
+
+            echo "⏳ Waiting for runner... (${WAITED}s elapsed)"
+            sleep 15
+            WAITED=$((WAITED + 15))
+          done
+
+          echo "Runner failed to come online within ${MAX_WAIT} seconds"
+          echo "ready=false" >> "$GITHUB_OUTPUT"
+          exit 1
diff --git a/.github/workflows/unified-tests.yml b/.github/workflows/unified-tests.yml
new file mode 100644
index 00000000..85df171f
--- /dev/null
+++ b/.github/workflows/unified-tests.yml
@@ -0,0 +1,207 @@
+name: Unified Tests (Fly Runner)
+
+on:
+  push:
+    branches: [main]
+
+permissions:
+  contents: read
+
+jobs:
+  start-runner:
+    name: Start Fly Runner
+    uses: ./.github/workflows/start-fly-runner.yml
+    secrets: inherit
+
+  wait-for-runner:
+    name: Wait for Runner
+    runs-on: ubuntu-latest
+    needs: start-runner
+    if: needs.start-runner.outputs.runner-ready == 'true'
+    steps:
+      - run: |
+          echo "✅ Runner is ready. Waiting a few seconds before scheduling tests..."
+          sleep 5
+
+  unified-tests:
+    name: Run Unified Tests
+    runs-on: ${{ fromJSON(format('[{0}]', needs.start-runner.outputs.runner-labels)) }}
+    needs: [start-runner, wait-for-runner]
+    if: needs.start-runner.outputs.runner-ready == 'true'
+    timeout-minutes: 90
+    environment: unified-tests
+    permissions:
+      id-token: write  # Required for OIDC authentication with AWS
+      contents: read
+
+    env:
+      PYTHONUNBUFFERED: "1"
+
+      # Session and embedding configuration
+      SESSION_OBSERVERS_LIMIT: ${{ vars.SESSION_OBSERVERS_LIMIT }}
+      EMBED_MESSAGES: ${{ vars.EMBED_MESSAGES }}
+      MAX_EMBEDDING_TOKENS: ${{ vars.MAX_EMBEDDING_TOKENS }}
+      MAX_EMBEDDING_TOKENS_PER_REQUEST: ${{ vars.MAX_EMBEDDING_TOKENS_PER_REQUEST }}
+
+      # Auth configuration
+      AUTH_USE_AUTH: ${{ vars.AUTH_USE_AUTH }}
+      SENTRY_ENABLED: ${{ vars.SENTRY_ENABLED }}
+      VECTOR_STORE_TYPE: ${{ vars.VECTOR_STORE_TYPE }}
+
+      # LLM API Keys (secrets)
+      LLM_ANTHROPIC_API_KEY: ${{ secrets.LLM_ANTHROPIC_API_KEY }}
+      LLM_GEMINI_API_KEY: ${{ secrets.LLM_GEMINI_API_KEY }}
+      LLM_GROQ_API_KEY: ${{ secrets.LLM_GROQ_API_KEY }}
+      LLM_OPENAI_API_KEY: ${{ secrets.LLM_OPENAI_API_KEY }}
+      LLM_OPENAI_COMPATIBLE_API_KEY: ${{ secrets.LLM_OPENAI_COMPATIBLE_API_KEY }}
+      LLM_OPENAI_COMPATIBLE_BASE_URL: ${{ secrets.LLM_OPENAI_COMPATIBLE_BASE_URL }}
+      LLM_DEFAULT_MAX_TOKENS: ${{ vars.LLM_DEFAULT_MAX_TOKENS }}
+
+      # Deriver configuration
+      DERIVER_WORKERS: ${{ vars.DERIVER_WORKERS }}
+      DERIVER_POLLING_SLEEP_INTERVAL_SECONDS: ${{ vars.DERIVER_POLLING_SLEEP_INTERVAL_SECONDS }}
+      DERIVER_STALE_SESSION_TIMEOUT_MINUTES: ${{ vars.DERIVER_STALE_SESSION_TIMEOUT_MINUTES }}
+      DERIVER_PROVIDER: ${{ vars.DERIVER_PROVIDER }}
+      DERIVER_MODEL: ${{ vars.DERIVER_MODEL }}
+      DERIVER_MAX_OUTPUT_TOKENS: ${{ vars.DERIVER_MAX_OUTPUT_TOKENS }}
+      DERIVER_THINKING_BUDGET_TOKENS: ${{ vars.DERIVER_THINKING_BUDGET_TOKENS }}
+      DERIVER_WORKING_REPRESENTATION_MAX_OBSERVATIONS: ${{ vars.DERIVER_WORKING_REPRESENTATION_MAX_OBSERVATIONS }}
+      DERIVER_MAX_INPUT_TOKENS: ${{ vars.DERIVER_MAX_INPUT_TOKENS }}
+      DERIVER_REPRESENTATION_BATCH_MAX_TOKENS: ${{ vars.DERIVER_REPRESENTATION_BATCH_MAX_TOKENS }}
+      DERIVER_BACKUP_PROVIDER: ${{ vars.DERIVER_BACKUP_PROVIDER }}
+      DERIVER_BACKUP_MODEL: ${{ vars.DERIVER_BACKUP_MODEL }}
+
+      # Peer Card configuration
+      PEER_CARD_ENABLED: ${{ vars.PEER_CARD_ENABLED }}
+      PEER_CARD_PROVIDER: ${{ vars.PEER_CARD_PROVIDER }}
+      PEER_CARD_MODEL: ${{ vars.PEER_CARD_MODEL }}
+      PEER_CARD_MAX_OUTPUT_TOKENS: ${{ vars.PEER_CARD_MAX_OUTPUT_TOKENS }}
+      PEER_CARD_BACKUP_PROVIDER: ${{ vars.PEER_CARD_BACKUP_PROVIDER }}
+      PEER_CARD_BACKUP_MODEL: ${{ vars.PEER_CARD_BACKUP_MODEL }}
+
+      # Dialectic configuration
+      DIALECTIC_PROVIDER: ${{ vars.DIALECTIC_PROVIDER }}
+      DIALECTIC_MODEL: ${{ vars.DIALECTIC_MODEL }}
+      DIALECTIC_PERFORM_QUERY_GENERATION: ${{ vars.DIALECTIC_PERFORM_QUERY_GENERATION }}
+      DIALECTIC_MAX_OUTPUT_TOKENS: ${{ vars.DIALECTIC_MAX_OUTPUT_TOKENS }}
+      DIALECTIC_SEMANTIC_SEARCH_TOP_K: ${{ vars.DIALECTIC_SEMANTIC_SEARCH_TOP_K }}
+      DIALECTIC_SEMANTIC_SEARCH_MAX_DISTANCE: ${{ vars.DIALECTIC_SEMANTIC_SEARCH_MAX_DISTANCE }}
+      DIALECTIC_THINKING_BUDGET_TOKENS: ${{ vars.DIALECTIC_THINKING_BUDGET_TOKENS }}
+      DIALECTIC_BACKUP_PROVIDER: ${{ vars.DIALECTIC_BACKUP_PROVIDER }}
+      DIALECTIC_BACKUP_MODEL: ${{ vars.DIALECTIC_BACKUP_MODEL }}
+
+      # Summary configuration
+      SUMMARY_MESSAGES_PER_SHORT_SUMMARY: ${{ vars.SUMMARY_MESSAGES_PER_SHORT_SUMMARY }}
+      SUMMARY_MESSAGES_PER_LONG_SUMMARY: ${{ vars.SUMMARY_MESSAGES_PER_LONG_SUMMARY }}
+      SUMMARY_PROVIDER: ${{ vars.SUMMARY_PROVIDER }}
+      SUMMARY_MODEL: ${{ vars.SUMMARY_MODEL }}
+      SUMMARY_MAX_TOKENS_SHORT: ${{ vars.SUMMARY_MAX_TOKENS_SHORT }}
+      SUMMARY_MAX_TOKENS_LONG: ${{ vars.SUMMARY_MAX_TOKENS_LONG }}
+      SUMMARY_BACKUP_PROVIDER: ${{ vars.SUMMARY_BACKUP_PROVIDER }}
+      SUMMARY_BACKUP_MODEL: ${{ vars.SUMMARY_BACKUP_MODEL }}
+
+      # Dream configuration
+      DREAM_ENABLED: ${{ vars.DREAM_ENABLED }}
+
+      # Cache configuration
+      CACHE_ENABLED: ${{ vars.CACHE_ENABLED }}
+
+      TEST_DISCORD_WEBHOOK_URL: ${{ secrets.TEST_DISCORD_WEBHOOK_URL }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.sha }}
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: arn:aws:iam::444554165670:role/GitHubActionsS3Role
+          aws-region: us-east-1
+
+      - name: Verify Docker is available
+        run: docker info
+
+      - name: Verify uv and Python
+        run: |
+          uv --version
+          python3.12 --version
+          which python3.12
+
+      - name: Install the project
+        run: uv sync --all-extras --dev
+
+      - name: Run unified tests
+        run: uv run python -m tests.unified.run
+
+  cleanup-machine:
+    name: Cleanup Fly Machine and Runner
+    runs-on: ubuntu-latest
+    needs: [start-runner, unified-tests]
+    if: always() && needs.start-runner.outputs.machine-id != ''
+    env:
+      FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN_TESTING }}
+      GITHUB_TOKEN: ${{ secrets.GH_TOKEN_ACTIONS }}
+      FLY_RUNNER_APP: ivysaur
+    steps:
+      - name: Setup Fly CLI
+        uses: superfly/flyctl-actions/setup-flyctl@1.5
+
+      - name: Cleanup fly machine
+        run: |
+          set -euo pipefail
+          MACHINE_ID="${{ needs.start-runner.outputs.machine-id }}"
+          if [ -z "$MACHINE_ID" ]; then
+            echo "No machine ID provided, skipping Fly cleanup."
+            exit 0
+          fi
+
+          echo "🧹 Cleaning up machine: $MACHINE_ID"
+          flyctl machines stop "$MACHINE_ID" -a "$FLY_RUNNER_APP" || echo "Machine may already be stopped"
+          flyctl machines destroy "$MACHINE_ID" -a "$FLY_RUNNER_APP" --force || echo "Failed to destroy machine"
+
+      - name: Cleanup GitHub runner
+        run: |
+          set -euo pipefail
+          RUNNER_NAME="${{ needs.start-runner.outputs.runner-name }}"
+          FALLBACK_LABEL="${{ github.run_id }}"
+
+          echo "🗑️ Cleaning up GitHub runner (name: ${RUNNER_NAME:-unknown}, label: ${FALLBACK_LABEL})"
+
+          RUNNERS_RESPONSE=$(curl -s \
+            -H "Authorization: Bearer $GITHUB_TOKEN" \
+            -H "Accept: application/vnd.github+json" \
+            "https://api.github.com/repos/${{ github.repository }}/actions/runners")
+
+          if echo "$RUNNERS_RESPONSE" | grep -q '"message"'; then
+            echo "⚠️ Failed to fetch runners: $(echo "$RUNNERS_RESPONSE" | jq -r '.message')"
+            exit 0
+          fi
+
+          RUNNER_ID="" 
+          if [ -n "$RUNNER_NAME" ]; then
+            RUNNER_ID=$(echo "$RUNNERS_RESPONSE" | jq -r --arg name "$RUNNER_NAME" '.runners[]? | select(.name == $name) | .id')
+          fi
+
+          if [ -z "$RUNNER_ID" ]; then
+            RUNNER_ID=$(echo "$RUNNERS_RESPONSE" | jq -r --arg label "$FALLBACK_LABEL" '.runners[]? | select([.labels[].name] | index($label)) | .id' | head -n 1)
+          fi
+
+          if [ -z "$RUNNER_ID" ] || [ "$RUNNER_ID" = "null" ]; then
+            echo "⚠️ Runner not found, nothing to delete."
+            exit 0
+          fi
+
+          DELETE_RESPONSE=$(curl -s -w "%{http_code}" \
+            -X DELETE \
+            -H "Accept: application/vnd.github+json" \
+            -H "Authorization: Bearer $GITHUB_TOKEN" \
+            "https://api.github.com/repos/${{ github.repository }}/actions/runners/$RUNNER_ID")
+
+          HTTP_CODE="${DELETE_RESPONSE: -3}"
+          if [ "$HTTP_CODE" = "204" ]; then
+            echo "✅ Successfully deleted runner."
+          else
+            echo "⚠️ Failed to delete runner. HTTP code: $HTTP_CODE"
+            echo "Response: ${DELETE_RESPONSE%???}"
+          fi
diff --git a/pyproject.toml b/pyproject.toml
index 3cce118f..e37bcbd3 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -35,6 +35,7 @@ dependencies = [
     "json-repair>=0.49.0",
     "redis>=6.0.0",
     "cashews[redis]==7.4.1",
+    "boto3>=1.42.5",
 ]
 [tool.uv]
 dev-dependencies = [
diff --git a/tests/unified/runner.py b/tests/unified/runner.py
index e4badf12..1bd0538f 100644
--- a/tests/unified/runner.py
+++ b/tests/unified/runner.py
@@ -5,9 +5,11 @@ import os
 import sys
 import threading
 import time
+from datetime import datetime, timezone
 from pathlib import Path
 from typing import Any, cast
 
+import httpx
 from anthropic import AsyncAnthropic
 from honcho.async_client.session import AsyncSession
 from honcho.session_context import SessionContext
@@ -66,6 +68,102 @@ class TestExecutionError(Exception):
     pass
 
 
+async def send_discord_message(webhook_url: str, message: str) -> None:
+    """Send a message to Discord via webhook."""
+    try:
+        async with httpx.AsyncClient() as client:
+            response = await client.post(webhook_url, json={"content": message})
+            response.raise_for_status()
+            logger.info("Discord notification sent successfully")
+    except Exception as e:
+        logger.error(f"Failed to send Discord notification: {e}", exc_info=True)
+
+
+async def save_results_to_s3(
+    results: dict[str, tuple[str, float]],
+    failed_count: int,
+    total_count: int,
+    execution_time: float,
+) -> str | None:
+    """Save comprehensive test results to S3."""
+    try:
+        import boto3
+
+        s3_bucket = "honcho-unified-tests"
+        s3_prefix = "unified-test-results"
+        aws_region = "us-east-1"
+
+        # AWS credentials are configured via OIDC in GitHub Actions
+        # Check if boto3 can access credentials (either from environment or OIDC)
+        try:
+            import boto3
+
+            session = boto3.Session()
+            credentials = session.get_credentials()  # pyright: ignore
+            if not credentials:
+                logger.warning("No AWS credentials available, skipping S3 upload")
+                return
+        except Exception as e:
+            logger.warning(f"Could not verify AWS credentials: {e}, skipping S3 upload")
+            return
+
+        # Create comprehensive results object
+        timestamp = datetime.now(timezone.utc).isoformat()
+        github_run_id = os.getenv("GITHUB_RUN_ID", "local")
+        github_sha = os.getenv("GITHUB_SHA", "unknown")
+        github_ref = os.getenv("GITHUB_REF_NAME", "unknown")
+
+        comprehensive_results = {
+            "timestamp": timestamp,
+            "summary": {
+                "total": total_count,
+                "passed": total_count - failed_count,
+                "failed": failed_count,
+                "execution_time": execution_time,
+            },
+            "metadata": {
+                "github_run_id": github_run_id,
+                "github_sha": github_sha,
+                "github_ref": github_ref,
+            },
+            "tests": [
+                {
+                    "name": name,
+                    "status": status,
+                    "duration": duration,
+                }
+                for name, (status, duration) in results.items()
+            ],
+        }
+
+        # Upload to S3
+        s3_client = boto3.client("s3", region_name=aws_region)  # pyright: ignore
+        key = f"{s3_prefix}/{github_run_id}-{timestamp}.json"
+
+        s3_client.put_object(  # pyright: ignore
+            Bucket=s3_bucket,
+            Key=key,
+            Body=json.dumps(comprehensive_results, indent=2).encode("utf-8"),
+            ContentType="application/json",
+        )
+
+        try:
+            url: str = s3_client.generate_presigned_url(  # pyright: ignore
+                "get_object",
+                Params={"Bucket": s3_bucket, "Key": key},
+                ExpiresIn=259200,  # 3 days
+            )
+            logger.info(f"Saved test results to s3://{s3_bucket}/{key}")
+            return url  # pyright: ignore
+        except Exception as e:
+            logger.warning(f"Could not generate S3 presigned URL: {e}")
+            logger.info(f"Saved test results to s3://{s3_bucket}/{key}")
+            return None
+
+    except Exception as e:
+        logger.error(f"Failed to save results to S3: {e}", exc_info=True)
+
+
 class UnifiedTestExecutor:
     def __init__(
         self, honcho_client: AsyncHoncho, anthropic_client: AsyncAnthropic | None
@@ -511,8 +609,33 @@ class UnifiedTestRunner:
             print(f"Total execution time: {total_suite_time:.2f}s")
             print("=" * 60)
 
+            # 5. Save results and send notifications
+            # Always attempt S3 upload - save_results_to_s3 will check for credentials
+            url: str | None = await save_results_to_s3(
+                results, failed_count, total_count, total_suite_time
+            )
+
+            # 6. Send Discord notification
+            discord_webhook_url = os.getenv("TEST_DISCORD_WEBHOOK_URL")
+            if discord_webhook_url:
+                passed_count = total_count - failed_count
+                status_emoji = "✅" if failed_count == 0 else "⚠️"
+                github_run_id = os.getenv("GITHUB_RUN_ID", "local")
+
+                message_lines = [
+                    f"{status_emoji} **Unified Test Results**",
+                    f"Results: {passed_count}/{total_count} passed, {failed_count}/{total_count} failed",
+                    f"Execution time: {total_suite_time:.2f}s",
+                    f"Run ID: {github_run_id}",
+                ]
+                if url:
+                    message_lines.append(f"[View Complete Results]({url})")
+                message = "\n".join(message_lines)
+
+                await send_discord_message(discord_webhook_url, message)
+
         finally:
-            # 5. Cleanup
+            # 7. Cleanup
             logger.info("Cleaning up harness...")
             await self.harness.cleanup()
 
diff --git a/uv.lock b/uv.lock
index f3dcab0e..454d741b 100644
--- a/uv.lock
+++ b/uv.lock
@@ -118,6 +118,34 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/1b/cc/8bca3b3a48d6a03a4b857a297fb1473ed1b9fa111be2d20c01f11112e75c/basedpyright-1.31.1-py3-none-any.whl", hash = "sha256:8b647bf07fff929892db4be83a116e6e1e59c13462ecb141214eb271f6785ee5", size = 11540576, upload-time = "2025-08-03T13:41:11.571Z" },
 ]
 
+[[package]]
+name = "boto3"
+version = "1.42.5"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "botocore" },
+    { name = "jmespath" },
+    { name = "s3transfer" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/8b/91/c00b45b5ca95184f7ab6140f586ba7d23074168ee3feae3eaf6954cc11c3/boto3-1.42.5.tar.gz", hash = "sha256:e3b7be255e5e29272b6424af4417005384f5a3f1caf6ca3352258ee1d9b8551a", size = 112754, upload-time = "2025-12-08T20:28:37.546Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/a1/3c/e70f47afdaf9172f90e80615f923fbb09f7fb4e5ea89e2d95562ec7f95c2/boto3-1.42.5-py3-none-any.whl", hash = "sha256:7d22cd102c77c37d552783308eeb01a088c0e3f6e707157dd6d1842b205ffce7", size = 140572, upload-time = "2025-12-08T20:28:36.076Z" },
+]
+
+[[package]]
+name = "botocore"
+version = "1.42.5"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "jmespath" },
+    { name = "python-dateutil" },
+    { name = "urllib3" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/8c/46/5b40b1deb780869ca9f0c1de47062a78a0494b53d6f9d6bad10fc38eef9d/botocore-1.42.5.tar.gz", hash = "sha256:37bfc487f14286d9795920807fcb8318b940835b18fff6bec5253449f377136f", size = 14851117, upload-time = "2025-12-08T20:28:26.876Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/8b/9a/da5e6cabf4da855d182fcdacf3573b69f30899e0e6c3e0d91ce6ad92ce74/botocore-1.42.5-py3-none-any.whl", hash = "sha256:6aa487f1876c881e2143f6a186b7d8faaf042fc05e0ba7421d821f145356a0c9", size = 14525346, upload-time = "2025-12-08T20:28:24.06Z" },
+]
+
 [[package]]
 name = "cachetools"
 version = "5.5.2"
@@ -714,6 +742,7 @@ version = "2.5.0"
 source = { virtual = "." }
 dependencies = [
     { name = "alembic" },
+    { name = "boto3" },
     { name = "cashews", extra = ["redis"] },
     { name = "fastapi", extra = ["standard"] },
     { name = "fastapi-pagination" },
@@ -763,6 +792,7 @@ dev = [
 [package.metadata]
 requires-dist = [
     { name = "alembic", specifier = ">=1.14.0" },
+    { name = "boto3", specifier = ">=1.42.5" },
     { name = "cashews", extras = ["redis"], specifier = "==7.4.1" },
     { name = "fastapi", extras = ["standard"], specifier = ">=0.111.0" },
     { name = "fastapi-pagination", specifier = ">=0.12.24" },
@@ -1056,6 +1086,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/b3/4a/4175a563579e884192ba6e81725fc0448b042024419be8d83aa8a80a3f44/jiter-0.10.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3aa96f2abba33dc77f79b4cf791840230375f9534e5fac927ccceb58c5e604a5", size = 354213, upload-time = "2025-05-18T19:04:41.894Z" },
 ]
 
+[[package]]
+name = "jmespath"
+version = "1.0.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/00/2a/e867e8531cf3e36b41201936b7fa7ba7b5702dbef42922193f05c8976cd6/jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe", size = 25843, upload-time = "2022-06-17T18:00:12.224Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/31/b4/b9b800c45527aadd64d5b442f9b932b00648617eb5d63d2c7a6587b7cafc/jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", size = 20256, upload-time = "2022-06-17T18:00:10.251Z" },
+]
+
 [[package]]
 name = "json-repair"
 version = "0.51.0"
@@ -1951,6 +1990,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/bc/16/4ea354101abb1287856baa4af2732be351c7bee728065aed451b678153fd/pytest_cov-6.2.1-py3-none-any.whl", hash = "sha256:f5bc4c23f42f1cdd23c70b1dab1bbaef4fc505ba950d53e0081d0730dd7e86d5", size = 24644, upload-time = "2025-06-12T10:47:45.932Z" },
 ]
 
+[[package]]
+name = "python-dateutil"
+version = "2.9.0.post0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "six" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" },
+]
+
 [[package]]
 name = "python-dotenv"
 version = "1.1.1"
@@ -2277,6 +2328,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/4c/9b/0b8aa09817b63e78d94b4977f18b1fcaead3165a5ee49251c5d5c245bb2d/ruff-0.12.7-py3-none-win_arm64.whl", hash = "sha256:dfce05101dbd11833a0776716d5d1578641b7fddb537fe7fa956ab85d1769b69", size = 11982083, upload-time = "2025-07-29T22:32:33.881Z" },
 ]
 
+[[package]]
+name = "s3transfer"
+version = "0.16.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "botocore" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/05/04/74127fc843314818edfa81b5540e26dd537353b123a4edc563109d8f17dd/s3transfer-0.16.0.tar.gz", hash = "sha256:8e990f13268025792229cd52fa10cb7163744bf56e719e0b9cb925ab79abf920", size = 153827, upload-time = "2025-12-01T02:30:59.114Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/fc/51/727abb13f44c1fcf6d145979e1535a35794db0f6e450a0cb46aa24732fe2/s3transfer-0.16.0-py3-none-any.whl", hash = "sha256:18e25d66fed509e3868dc1572b3f427ff947dd2c56f844a5bf09481ad3f3b2fe", size = 86830, upload-time = "2025-12-01T02:30:57.729Z" },
+]
+
 [[package]]
 name = "scipy"
 version = "1.15.3"
@@ -2444,6 +2507,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" },
 ]
 
+[[package]]
+name = "six"
+version = "1.17.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" },
+]
+
 [[package]]
 name = "sniffio"
 version = "1.3.1"