feat: run unified tests in CI
This commit is contained in:
Rajat Ahuja
parent
42a28dfda1
commit
ad1b91757a
|
|
@ -0,0 +1,163 @@
|
|||
name: Start Fly Runner
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
outputs:
|
||||
runner-ready:
|
||||
description: "Whether the runner is ready"
|
||||
value: ${{ jobs.start-runner.outputs.runner-ready }}
|
||||
machine-id:
|
||||
description: "The Fly machine ID that was started"
|
||||
value: ${{ jobs.start-runner.outputs.machine-id }}
|
||||
runner-labels:
|
||||
description: "Labels to target the self-hosted runner"
|
||||
value: ${{ jobs.start-runner.outputs.runner-labels }}
|
||||
runner-name:
|
||||
description: "Resolved GitHub runner name"
|
||||
value: ${{ jobs.start-runner.outputs.runner-name }}
|
||||
|
||||
env:
|
||||
FLY_RUNNER_APP: ivysaur
|
||||
FLY_RUNNER_REGION: iad
|
||||
FLY_RUNNER_IMAGE: registry.fly.io/ivysaur:latest
|
||||
|
||||
jobs:
|
||||
start-runner:
|
||||
name: Start Fly Runner
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
runner-ready: ${{ steps.wait-for-runner.outputs.ready }}
|
||||
machine-id: ${{ steps.machine-management.outputs.machine-id }}
|
||||
runner-labels: ${{ steps.generate-labels.outputs.labels }}
|
||||
runner-name: ${{ steps.wait-for-runner.outputs.runner-name }}
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Generate unique runner labels
|
||||
id: generate-labels
|
||||
run: |
|
||||
UNIQUE_LABELS='"self-hosted","${{ github.run_id }}"'
|
||||
echo "Generated unique labels: $UNIQUE_LABELS"
|
||||
echo "labels=$UNIQUE_LABELS" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Setup Fly CLI
|
||||
uses: superfly/flyctl-actions/setup-flyctl@master
|
||||
|
||||
- name: Get Fly app info
|
||||
id: get-app-info
|
||||
env:
|
||||
FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN_TESTING }}
|
||||
run: |
|
||||
echo "Getting app info for ${FLY_RUNNER_APP}..."
|
||||
flyctl status -a "${FLY_RUNNER_APP}"
|
||||
|
||||
- name: Fly machine management
|
||||
id: machine-management
|
||||
env:
|
||||
FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN_TESTING }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
# Get list of machines and their status
|
||||
MACHINES_JSON=$(flyctl machines list -a "${FLY_RUNNER_APP}" --json)
|
||||
|
||||
# Count online machines
|
||||
ONLINE_COUNT=$(echo "$MACHINES_JSON" | jq '[.[] | select(.state == "started")] | length')
|
||||
echo "📊 Online machines: $ONLINE_COUNT"
|
||||
|
||||
if [ "$ONLINE_COUNT" -ge 2 ]; then
|
||||
echo "✅ Found $ONLINE_COUNT online machines (>=2), will reuse existing machine"
|
||||
|
||||
# Pick the first online machine
|
||||
MACHINE_ID=$(echo "$MACHINES_JSON" | jq -r '[.[] | select(.state == "started")][0].id')
|
||||
|
||||
if [ -z "$MACHINE_ID" ] || [ "$MACHINE_ID" = "null" ]; then
|
||||
echo "❌ Failed to find online machine ID"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "🔄 Reusing machine ID: $MACHINE_ID"
|
||||
echo "machine-id=$MACHINE_ID" >> "$GITHUB_OUTPUT"
|
||||
|
||||
else
|
||||
echo "📦 Need to create new machine (only $ONLINE_COUNT online machines)"
|
||||
|
||||
# Get available volumes
|
||||
VOLUMES_JSON=$(flyctl volumes list -a "${FLY_RUNNER_APP}" --json)
|
||||
|
||||
# Find a volume with null attached_machine_id
|
||||
AVAILABLE_VOLUME=$(echo "$VOLUMES_JSON" | jq -r '[.[] | select(.attached_machine_id == null)][0].id')
|
||||
|
||||
if [ -z "$AVAILABLE_VOLUME" ] || [ "$AVAILABLE_VOLUME" = "null" ]; then
|
||||
echo "❌ No available volumes found"
|
||||
echo "Available volumes: $(echo "$VOLUMES_JSON" | jq -r '.[] | select(.attached_machine_id == null) | .id')"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "📁 Using available volume: $AVAILABLE_VOLUME"
|
||||
|
||||
# Create new machine with the available volume
|
||||
FULL_OUTPUT=$(flyctl machines run "${FLY_RUNNER_IMAGE}" \
|
||||
-a "${FLY_RUNNER_APP}" \
|
||||
--region "${FLY_RUNNER_REGION}" \
|
||||
--env GH_TOKEN=${{ secrets.GH_TOKEN_ACTIONS }} \
|
||||
--env RUN_ID=${{ github.run_id }} \
|
||||
--env TEST_TYPE="honcho-unified-runner" \
|
||||
--vm-size shared-cpu-8x \
|
||||
--vm-memory 8192 \
|
||||
--volume "$AVAILABLE_VOLUME":/mnt/vol )
|
||||
|
||||
MACHINE_ID=$(echo "$FULL_OUTPUT" | grep "Machine ID:" | awk '{print $3}')
|
||||
echo "✅ Created new machine ID: $MACHINE_ID"
|
||||
echo "machine-id=$MACHINE_ID" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Wait for runner to be online
|
||||
id: wait-for-runner
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GH_TOKEN_ACTIONS }}
|
||||
MAX_WAIT: 420
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GITHUB_TOKEN}" ]; then
|
||||
echo "GH_TOKEN secret is required to poll the Actions runner API."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
EXPECTED_RUNNER_NAME="honcho-unified-runner-${{ github.run_id }}"
|
||||
echo "Waiting for runner named ${EXPECTED_RUNNER_NAME} to come online..."
|
||||
|
||||
WAITED=0
|
||||
RUNNER_NAME=""
|
||||
while [ $WAITED -lt $MAX_WAIT ]; do
|
||||
RESPONSE=$(curl -s \
|
||||
-H "Authorization: Bearer ${GITHUB_TOKEN}" \
|
||||
-H "Accept: application/vnd.github.v3+json" \
|
||||
"https://api.github.com/repos/${{ github.repository }}/actions/runners")
|
||||
|
||||
if echo "$RESPONSE" | grep -q '"message"'; then
|
||||
echo "API Error: $(echo "$RESPONSE" | jq -r '.message')"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Find runner with exact name and is online and not busy
|
||||
RUNNER_LINE=$(echo "$RESPONSE" | jq -r --arg runner_name "$EXPECTED_RUNNER_NAME" '.runners[]? | select(.name == $runner_name) | select(.status == "online") | select(.busy == false) | "\(.name)|\(.id)"' | head -n 1)
|
||||
|
||||
if [ -n "$RUNNER_LINE" ]; then
|
||||
RUNNER_NAME=$(echo "$RUNNER_LINE" | cut -d'|' -f1)
|
||||
echo "✅ Found runner: ${RUNNER_NAME}"
|
||||
echo "ready=true" >> "$GITHUB_OUTPUT"
|
||||
echo "runner-name=${RUNNER_NAME}" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "⏳ Waiting for runner... (${WAITED}s elapsed)"
|
||||
sleep 15
|
||||
WAITED=$((WAITED + 15))
|
||||
done
|
||||
|
||||
echo "Runner failed to come online within ${MAX_WAIT} seconds"
|
||||
echo "ready=false" >> "$GITHUB_OUTPUT"
|
||||
exit 1
|
||||
|
|
@ -0,0 +1,207 @@
|
|||
name: Unified Tests (Fly Runner)
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
start-runner:
|
||||
name: Start Fly Runner
|
||||
uses: ./.github/workflows/start-fly-runner.yml
|
||||
secrets: inherit
|
||||
|
||||
wait-for-runner:
|
||||
name: Wait for Runner
|
||||
runs-on: ubuntu-latest
|
||||
needs: start-runner
|
||||
if: needs.start-runner.outputs.runner-ready == 'true'
|
||||
steps:
|
||||
- run: |
|
||||
echo "✅ Runner is ready. Waiting a few seconds before scheduling tests..."
|
||||
sleep 5
|
||||
|
||||
unified-tests:
|
||||
name: Run Unified Tests
|
||||
runs-on: ${{ fromJSON(format('[{0}]', needs.start-runner.outputs.runner-labels)) }}
|
||||
needs: [start-runner, wait-for-runner]
|
||||
if: needs.start-runner.outputs.runner-ready == 'true'
|
||||
timeout-minutes: 90
|
||||
environment: unified-tests
|
||||
permissions:
|
||||
id-token: write # Required for OIDC authentication with AWS
|
||||
contents: read
|
||||
|
||||
env:
|
||||
PYTHONUNBUFFERED: "1"
|
||||
|
||||
# Session and embedding configuration
|
||||
SESSION_OBSERVERS_LIMIT: ${{ vars.SESSION_OBSERVERS_LIMIT }}
|
||||
EMBED_MESSAGES: ${{ vars.EMBED_MESSAGES }}
|
||||
MAX_EMBEDDING_TOKENS: ${{ vars.MAX_EMBEDDING_TOKENS }}
|
||||
MAX_EMBEDDING_TOKENS_PER_REQUEST: ${{ vars.MAX_EMBEDDING_TOKENS_PER_REQUEST }}
|
||||
|
||||
# Auth configuration
|
||||
AUTH_USE_AUTH: ${{ vars.AUTH_USE_AUTH }}
|
||||
SENTRY_ENABLED: ${{ vars.SENTRY_ENABLED }}
|
||||
VECTOR_STORE_TYPE: ${{ vars.VECTOR_STORE_TYPE }}
|
||||
|
||||
# LLM API Keys (secrets)
|
||||
LLM_ANTHROPIC_API_KEY: ${{ secrets.LLM_ANTHROPIC_API_KEY }}
|
||||
LLM_GEMINI_API_KEY: ${{ secrets.LLM_GEMINI_API_KEY }}
|
||||
LLM_GROQ_API_KEY: ${{ secrets.LLM_GROQ_API_KEY }}
|
||||
LLM_OPENAI_API_KEY: ${{ secrets.LLM_OPENAI_API_KEY }}
|
||||
LLM_OPENAI_COMPATIBLE_API_KEY: ${{ secrets.LLM_OPENAI_COMPATIBLE_API_KEY }}
|
||||
LLM_OPENAI_COMPATIBLE_BASE_URL: ${{ secrets.LLM_OPENAI_COMPATIBLE_BASE_URL }}
|
||||
LLM_DEFAULT_MAX_TOKENS: ${{ vars.LLM_DEFAULT_MAX_TOKENS }}
|
||||
|
||||
# Deriver configuration
|
||||
DERIVER_WORKERS: ${{ vars.DERIVER_WORKERS }}
|
||||
DERIVER_POLLING_SLEEP_INTERVAL_SECONDS: ${{ vars.DERIVER_POLLING_SLEEP_INTERVAL_SECONDS }}
|
||||
DERIVER_STALE_SESSION_TIMEOUT_MINUTES: ${{ vars.DERIVER_STALE_SESSION_TIMEOUT_MINUTES }}
|
||||
DERIVER_PROVIDER: ${{ vars.DERIVER_PROVIDER }}
|
||||
DERIVER_MODEL: ${{ vars.DERIVER_MODEL }}
|
||||
DERIVER_MAX_OUTPUT_TOKENS: ${{ vars.DERIVER_MAX_OUTPUT_TOKENS }}
|
||||
DERIVER_THINKING_BUDGET_TOKENS: ${{ vars.DERIVER_THINKING_BUDGET_TOKENS }}
|
||||
DERIVER_WORKING_REPRESENTATION_MAX_OBSERVATIONS: ${{ vars.DERIVER_WORKING_REPRESENTATION_MAX_OBSERVATIONS }}
|
||||
DERIVER_MAX_INPUT_TOKENS: ${{ vars.DERIVER_MAX_INPUT_TOKENS }}
|
||||
DERIVER_REPRESENTATION_BATCH_MAX_TOKENS: ${{ vars.DERIVER_REPRESENTATION_BATCH_MAX_TOKENS }}
|
||||
DERIVER_BACKUP_PROVIDER: ${{ vars.DERIVER_BACKUP_PROVIDER }}
|
||||
DERIVER_BACKUP_MODEL: ${{ vars.DERIVER_BACKUP_MODEL }}
|
||||
|
||||
# Peer Card configuration
|
||||
PEER_CARD_ENABLED: ${{ vars.PEER_CARD_ENABLED }}
|
||||
PEER_CARD_PROVIDER: ${{ vars.PEER_CARD_PROVIDER }}
|
||||
PEER_CARD_MODEL: ${{ vars.PEER_CARD_MODEL }}
|
||||
PEER_CARD_MAX_OUTPUT_TOKENS: ${{ vars.PEER_CARD_MAX_OUTPUT_TOKENS }}
|
||||
PEER_CARD_BACKUP_PROVIDER: ${{ vars.PEER_CARD_BACKUP_PROVIDER }}
|
||||
PEER_CARD_BACKUP_MODEL: ${{ vars.PEER_CARD_BACKUP_MODEL }}
|
||||
|
||||
# Dialectic configuration
|
||||
DIALECTIC_PROVIDER: ${{ vars.DIALECTIC_PROVIDER }}
|
||||
DIALECTIC_MODEL: ${{ vars.DIALECTIC_MODEL }}
|
||||
DIALECTIC_PERFORM_QUERY_GENERATION: ${{ vars.DIALECTIC_PERFORM_QUERY_GENERATION }}
|
||||
DIALECTIC_MAX_OUTPUT_TOKENS: ${{ vars.DIALECTIC_MAX_OUTPUT_TOKENS }}
|
||||
DIALECTIC_SEMANTIC_SEARCH_TOP_K: ${{ vars.DIALECTIC_SEMANTIC_SEARCH_TOP_K }}
|
||||
DIALECTIC_SEMANTIC_SEARCH_MAX_DISTANCE: ${{ vars.DIALECTIC_SEMANTIC_SEARCH_MAX_DISTANCE }}
|
||||
DIALECTIC_THINKING_BUDGET_TOKENS: ${{ vars.DIALECTIC_THINKING_BUDGET_TOKENS }}
|
||||
DIALECTIC_BACKUP_PROVIDER: ${{ vars.DIALECTIC_BACKUP_PROVIDER }}
|
||||
DIALECTIC_BACKUP_MODEL: ${{ vars.DIALECTIC_BACKUP_MODEL }}
|
||||
|
||||
# Summary configuration
|
||||
SUMMARY_MESSAGES_PER_SHORT_SUMMARY: ${{ vars.SUMMARY_MESSAGES_PER_SHORT_SUMMARY }}
|
||||
SUMMARY_MESSAGES_PER_LONG_SUMMARY: ${{ vars.SUMMARY_MESSAGES_PER_LONG_SUMMARY }}
|
||||
SUMMARY_PROVIDER: ${{ vars.SUMMARY_PROVIDER }}
|
||||
SUMMARY_MODEL: ${{ vars.SUMMARY_MODEL }}
|
||||
SUMMARY_MAX_TOKENS_SHORT: ${{ vars.SUMMARY_MAX_TOKENS_SHORT }}
|
||||
SUMMARY_MAX_TOKENS_LONG: ${{ vars.SUMMARY_MAX_TOKENS_LONG }}
|
||||
SUMMARY_BACKUP_PROVIDER: ${{ vars.SUMMARY_BACKUP_PROVIDER }}
|
||||
SUMMARY_BACKUP_MODEL: ${{ vars.SUMMARY_BACKUP_MODEL }}
|
||||
|
||||
# Dream configuration
|
||||
DREAM_ENABLED: ${{ vars.DREAM_ENABLED }}
|
||||
|
||||
# Cache configuration
|
||||
CACHE_ENABLED: ${{ vars.CACHE_ENABLED }}
|
||||
|
||||
TEST_DISCORD_WEBHOOK_URL: ${{ secrets.TEST_DISCORD_WEBHOOK_URL }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.sha }}
|
||||
|
||||
- name: Configure AWS credentials
|
||||
uses: aws-actions/configure-aws-credentials@v4
|
||||
with:
|
||||
role-to-assume: arn:aws:iam::444554165670:role/GitHubActionsS3Role
|
||||
aws-region: us-east-1
|
||||
|
||||
- name: Verify Docker is available
|
||||
run: docker info
|
||||
|
||||
- name: Verify uv and Python
|
||||
run: |
|
||||
uv --version
|
||||
python3.12 --version
|
||||
which python3.12
|
||||
|
||||
- name: Install the project
|
||||
run: uv sync --all-extras --dev
|
||||
|
||||
- name: Run unified tests
|
||||
run: uv run python -m tests.unified.run
|
||||
|
||||
cleanup-machine:
|
||||
name: Cleanup Fly Machine and Runner
|
||||
runs-on: ubuntu-latest
|
||||
needs: [start-runner, unified-tests]
|
||||
if: always() && needs.start-runner.outputs.machine-id != ''
|
||||
env:
|
||||
FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN_TESTING }}
|
||||
GITHUB_TOKEN: ${{ secrets.GH_TOKEN_ACTIONS }}
|
||||
FLY_RUNNER_APP: ivysaur
|
||||
steps:
|
||||
- name: Setup Fly CLI
|
||||
uses: superfly/flyctl-actions/setup-flyctl@1.5
|
||||
|
||||
- name: Cleanup fly machine
|
||||
run: |
|
||||
set -euo pipefail
|
||||
MACHINE_ID="${{ needs.start-runner.outputs.machine-id }}"
|
||||
if [ -z "$MACHINE_ID" ]; then
|
||||
echo "No machine ID provided, skipping Fly cleanup."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "🧹 Cleaning up machine: $MACHINE_ID"
|
||||
flyctl machines stop "$MACHINE_ID" -a "$FLY_RUNNER_APP" || echo "Machine may already be stopped"
|
||||
flyctl machines destroy "$MACHINE_ID" -a "$FLY_RUNNER_APP" --force || echo "Failed to destroy machine"
|
||||
|
||||
- name: Cleanup GitHub runner
|
||||
run: |
|
||||
set -euo pipefail
|
||||
RUNNER_NAME="${{ needs.start-runner.outputs.runner-name }}"
|
||||
FALLBACK_LABEL="${{ github.run_id }}"
|
||||
|
||||
echo "🗑️ Cleaning up GitHub runner (name: ${RUNNER_NAME:-unknown}, label: ${FALLBACK_LABEL})"
|
||||
|
||||
RUNNERS_RESPONSE=$(curl -s \
|
||||
-H "Authorization: Bearer $GITHUB_TOKEN" \
|
||||
-H "Accept: application/vnd.github+json" \
|
||||
"https://api.github.com/repos/${{ github.repository }}/actions/runners")
|
||||
|
||||
if echo "$RUNNERS_RESPONSE" | grep -q '"message"'; then
|
||||
echo "⚠️ Failed to fetch runners: $(echo "$RUNNERS_RESPONSE" | jq -r '.message')"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
RUNNER_ID=""
|
||||
if [ -n "$RUNNER_NAME" ]; then
|
||||
RUNNER_ID=$(echo "$RUNNERS_RESPONSE" | jq -r --arg name "$RUNNER_NAME" '.runners[]? | select(.name == $name) | .id')
|
||||
fi
|
||||
|
||||
if [ -z "$RUNNER_ID" ]; then
|
||||
RUNNER_ID=$(echo "$RUNNERS_RESPONSE" | jq -r --arg label "$FALLBACK_LABEL" '.runners[]? | select([.labels[].name] | index($label)) | .id' | head -n 1)
|
||||
fi
|
||||
|
||||
if [ -z "$RUNNER_ID" ] || [ "$RUNNER_ID" = "null" ]; then
|
||||
echo "⚠️ Runner not found, nothing to delete."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
DELETE_RESPONSE=$(curl -s -w "%{http_code}" \
|
||||
-X DELETE \
|
||||
-H "Accept: application/vnd.github+json" \
|
||||
-H "Authorization: Bearer $GITHUB_TOKEN" \
|
||||
"https://api.github.com/repos/${{ github.repository }}/actions/runners/$RUNNER_ID")
|
||||
|
||||
HTTP_CODE="${DELETE_RESPONSE: -3}"
|
||||
if [ "$HTTP_CODE" = "204" ]; then
|
||||
echo "✅ Successfully deleted runner."
|
||||
else
|
||||
echo "⚠️ Failed to delete runner. HTTP code: $HTTP_CODE"
|
||||
echo "Response: ${DELETE_RESPONSE%???}"
|
||||
fi
|
||||
|
|
@ -35,6 +35,7 @@ dependencies = [
|
|||
"json-repair>=0.49.0",
|
||||
"redis>=6.0.0",
|
||||
"cashews[redis]==7.4.1",
|
||||
"boto3>=1.42.5",
|
||||
]
|
||||
[tool.uv]
|
||||
dev-dependencies = [
|
||||
|
|
|
|||
|
|
@ -5,9 +5,11 @@ import os
|
|||
import sys
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, cast
|
||||
|
||||
import httpx
|
||||
from anthropic import AsyncAnthropic
|
||||
from honcho.async_client.session import AsyncSession
|
||||
from honcho.session_context import SessionContext
|
||||
|
|
@ -66,6 +68,102 @@ class TestExecutionError(Exception):
|
|||
pass
|
||||
|
||||
|
||||
async def send_discord_message(webhook_url: str, message: str) -> None:
|
||||
"""Send a message to Discord via webhook."""
|
||||
try:
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.post(webhook_url, json={"content": message})
|
||||
response.raise_for_status()
|
||||
logger.info("Discord notification sent successfully")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to send Discord notification: {e}", exc_info=True)
|
||||
|
||||
|
||||
async def save_results_to_s3(
|
||||
results: dict[str, tuple[str, float]],
|
||||
failed_count: int,
|
||||
total_count: int,
|
||||
execution_time: float,
|
||||
) -> str | None:
|
||||
"""Save comprehensive test results to S3."""
|
||||
try:
|
||||
import boto3
|
||||
|
||||
s3_bucket = "honcho-unified-tests"
|
||||
s3_prefix = "unified-test-results"
|
||||
aws_region = "us-east-1"
|
||||
|
||||
# AWS credentials are configured via OIDC in GitHub Actions
|
||||
# Check if boto3 can access credentials (either from environment or OIDC)
|
||||
try:
|
||||
import boto3
|
||||
|
||||
session = boto3.Session()
|
||||
credentials = session.get_credentials() # pyright: ignore
|
||||
if not credentials:
|
||||
logger.warning("No AWS credentials available, skipping S3 upload")
|
||||
return
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not verify AWS credentials: {e}, skipping S3 upload")
|
||||
return
|
||||
|
||||
# Create comprehensive results object
|
||||
timestamp = datetime.now(timezone.utc).isoformat()
|
||||
github_run_id = os.getenv("GITHUB_RUN_ID", "local")
|
||||
github_sha = os.getenv("GITHUB_SHA", "unknown")
|
||||
github_ref = os.getenv("GITHUB_REF_NAME", "unknown")
|
||||
|
||||
comprehensive_results = {
|
||||
"timestamp": timestamp,
|
||||
"summary": {
|
||||
"total": total_count,
|
||||
"passed": total_count - failed_count,
|
||||
"failed": failed_count,
|
||||
"execution_time": execution_time,
|
||||
},
|
||||
"metadata": {
|
||||
"github_run_id": github_run_id,
|
||||
"github_sha": github_sha,
|
||||
"github_ref": github_ref,
|
||||
},
|
||||
"tests": [
|
||||
{
|
||||
"name": name,
|
||||
"status": status,
|
||||
"duration": duration,
|
||||
}
|
||||
for name, (status, duration) in results.items()
|
||||
],
|
||||
}
|
||||
|
||||
# Upload to S3
|
||||
s3_client = boto3.client("s3", region_name=aws_region) # pyright: ignore
|
||||
key = f"{s3_prefix}/{github_run_id}-{timestamp}.json"
|
||||
|
||||
s3_client.put_object( # pyright: ignore
|
||||
Bucket=s3_bucket,
|
||||
Key=key,
|
||||
Body=json.dumps(comprehensive_results, indent=2).encode("utf-8"),
|
||||
ContentType="application/json",
|
||||
)
|
||||
|
||||
try:
|
||||
url: str = s3_client.generate_presigned_url( # pyright: ignore
|
||||
"get_object",
|
||||
Params={"Bucket": s3_bucket, "Key": key},
|
||||
ExpiresIn=259200, # 3 days
|
||||
)
|
||||
logger.info(f"Saved test results to s3://{s3_bucket}/{key}")
|
||||
return url # pyright: ignore
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not generate S3 presigned URL: {e}")
|
||||
logger.info(f"Saved test results to s3://{s3_bucket}/{key}")
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to save results to S3: {e}", exc_info=True)
|
||||
|
||||
|
||||
class UnifiedTestExecutor:
|
||||
def __init__(
|
||||
self, honcho_client: AsyncHoncho, anthropic_client: AsyncAnthropic | None
|
||||
|
|
@ -511,8 +609,33 @@ class UnifiedTestRunner:
|
|||
print(f"Total execution time: {total_suite_time:.2f}s")
|
||||
print("=" * 60)
|
||||
|
||||
# 5. Save results and send notifications
|
||||
# Always attempt S3 upload - save_results_to_s3 will check for credentials
|
||||
url: str | None = await save_results_to_s3(
|
||||
results, failed_count, total_count, total_suite_time
|
||||
)
|
||||
|
||||
# 6. Send Discord notification
|
||||
discord_webhook_url = os.getenv("TEST_DISCORD_WEBHOOK_URL")
|
||||
if discord_webhook_url:
|
||||
passed_count = total_count - failed_count
|
||||
status_emoji = "✅" if failed_count == 0 else "⚠️"
|
||||
github_run_id = os.getenv("GITHUB_RUN_ID", "local")
|
||||
|
||||
message_lines = [
|
||||
f"{status_emoji} **Unified Test Results**",
|
||||
f"Results: {passed_count}/{total_count} passed, {failed_count}/{total_count} failed",
|
||||
f"Execution time: {total_suite_time:.2f}s",
|
||||
f"Run ID: {github_run_id}",
|
||||
]
|
||||
if url:
|
||||
message_lines.append(f"[View Complete Results]({url})")
|
||||
message = "\n".join(message_lines)
|
||||
|
||||
await send_discord_message(discord_webhook_url, message)
|
||||
|
||||
finally:
|
||||
# 5. Cleanup
|
||||
# 7. Cleanup
|
||||
logger.info("Cleaning up harness...")
|
||||
await self.harness.cleanup()
|
||||
|
||||
|
|
|
|||
72
uv.lock
72
uv.lock
|
|
@ -118,6 +118,34 @@ wheels = [
|
|||
{ url = "https://files.pythonhosted.org/packages/1b/cc/8bca3b3a48d6a03a4b857a297fb1473ed1b9fa111be2d20c01f11112e75c/basedpyright-1.31.1-py3-none-any.whl", hash = "sha256:8b647bf07fff929892db4be83a116e6e1e59c13462ecb141214eb271f6785ee5", size = 11540576, upload-time = "2025-08-03T13:41:11.571Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "boto3"
|
||||
version = "1.42.5"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "botocore" },
|
||||
{ name = "jmespath" },
|
||||
{ name = "s3transfer" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/8b/91/c00b45b5ca95184f7ab6140f586ba7d23074168ee3feae3eaf6954cc11c3/boto3-1.42.5.tar.gz", hash = "sha256:e3b7be255e5e29272b6424af4417005384f5a3f1caf6ca3352258ee1d9b8551a", size = 112754, upload-time = "2025-12-08T20:28:37.546Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/a1/3c/e70f47afdaf9172f90e80615f923fbb09f7fb4e5ea89e2d95562ec7f95c2/boto3-1.42.5-py3-none-any.whl", hash = "sha256:7d22cd102c77c37d552783308eeb01a088c0e3f6e707157dd6d1842b205ffce7", size = 140572, upload-time = "2025-12-08T20:28:36.076Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "botocore"
|
||||
version = "1.42.5"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "jmespath" },
|
||||
{ name = "python-dateutil" },
|
||||
{ name = "urllib3" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/8c/46/5b40b1deb780869ca9f0c1de47062a78a0494b53d6f9d6bad10fc38eef9d/botocore-1.42.5.tar.gz", hash = "sha256:37bfc487f14286d9795920807fcb8318b940835b18fff6bec5253449f377136f", size = 14851117, upload-time = "2025-12-08T20:28:26.876Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/8b/9a/da5e6cabf4da855d182fcdacf3573b69f30899e0e6c3e0d91ce6ad92ce74/botocore-1.42.5-py3-none-any.whl", hash = "sha256:6aa487f1876c881e2143f6a186b7d8faaf042fc05e0ba7421d821f145356a0c9", size = 14525346, upload-time = "2025-12-08T20:28:24.06Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cachetools"
|
||||
version = "5.5.2"
|
||||
|
|
@ -714,6 +742,7 @@ version = "2.5.0"
|
|||
source = { virtual = "." }
|
||||
dependencies = [
|
||||
{ name = "alembic" },
|
||||
{ name = "boto3" },
|
||||
{ name = "cashews", extra = ["redis"] },
|
||||
{ name = "fastapi", extra = ["standard"] },
|
||||
{ name = "fastapi-pagination" },
|
||||
|
|
@ -763,6 +792,7 @@ dev = [
|
|||
[package.metadata]
|
||||
requires-dist = [
|
||||
{ name = "alembic", specifier = ">=1.14.0" },
|
||||
{ name = "boto3", specifier = ">=1.42.5" },
|
||||
{ name = "cashews", extras = ["redis"], specifier = "==7.4.1" },
|
||||
{ name = "fastapi", extras = ["standard"], specifier = ">=0.111.0" },
|
||||
{ name = "fastapi-pagination", specifier = ">=0.12.24" },
|
||||
|
|
@ -1056,6 +1086,15 @@ wheels = [
|
|||
{ url = "https://files.pythonhosted.org/packages/b3/4a/4175a563579e884192ba6e81725fc0448b042024419be8d83aa8a80a3f44/jiter-0.10.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3aa96f2abba33dc77f79b4cf791840230375f9534e5fac927ccceb58c5e604a5", size = 354213, upload-time = "2025-05-18T19:04:41.894Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jmespath"
|
||||
version = "1.0.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/00/2a/e867e8531cf3e36b41201936b7fa7ba7b5702dbef42922193f05c8976cd6/jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe", size = 25843, upload-time = "2022-06-17T18:00:12.224Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/31/b4/b9b800c45527aadd64d5b442f9b932b00648617eb5d63d2c7a6587b7cafc/jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", size = 20256, upload-time = "2022-06-17T18:00:10.251Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "json-repair"
|
||||
version = "0.51.0"
|
||||
|
|
@ -1951,6 +1990,18 @@ wheels = [
|
|||
{ url = "https://files.pythonhosted.org/packages/bc/16/4ea354101abb1287856baa4af2732be351c7bee728065aed451b678153fd/pytest_cov-6.2.1-py3-none-any.whl", hash = "sha256:f5bc4c23f42f1cdd23c70b1dab1bbaef4fc505ba950d53e0081d0730dd7e86d5", size = 24644, upload-time = "2025-06-12T10:47:45.932Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "python-dateutil"
|
||||
version = "2.9.0.post0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "six" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "python-dotenv"
|
||||
version = "1.1.1"
|
||||
|
|
@ -2277,6 +2328,18 @@ wheels = [
|
|||
{ url = "https://files.pythonhosted.org/packages/4c/9b/0b8aa09817b63e78d94b4977f18b1fcaead3165a5ee49251c5d5c245bb2d/ruff-0.12.7-py3-none-win_arm64.whl", hash = "sha256:dfce05101dbd11833a0776716d5d1578641b7fddb537fe7fa956ab85d1769b69", size = 11982083, upload-time = "2025-07-29T22:32:33.881Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "s3transfer"
|
||||
version = "0.16.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "botocore" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/05/04/74127fc843314818edfa81b5540e26dd537353b123a4edc563109d8f17dd/s3transfer-0.16.0.tar.gz", hash = "sha256:8e990f13268025792229cd52fa10cb7163744bf56e719e0b9cb925ab79abf920", size = 153827, upload-time = "2025-12-01T02:30:59.114Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/fc/51/727abb13f44c1fcf6d145979e1535a35794db0f6e450a0cb46aa24732fe2/s3transfer-0.16.0-py3-none-any.whl", hash = "sha256:18e25d66fed509e3868dc1572b3f427ff947dd2c56f844a5bf09481ad3f3b2fe", size = 86830, upload-time = "2025-12-01T02:30:57.729Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "scipy"
|
||||
version = "1.15.3"
|
||||
|
|
@ -2444,6 +2507,15 @@ wheels = [
|
|||
{ url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "six"
|
||||
version = "1.17.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sniffio"
|
||||
version = "1.3.1"
|
||||
|
|
|
|||
Loading…
Reference in New Issue