paperclip/scripts/smoke/hermes-gateway-e2e.sh

1008 lines
36 KiB
Bash
Executable File

#!/usr/bin/env bash
set -euo pipefail
log() {
echo "[hermes-gateway-e2e] $*"
}
warn() {
echo "[hermes-gateway-e2e] WARN: $*" >&2
}
fail() {
echo "[hermes-gateway-e2e] ERROR: $*" >&2
if [[ -n "${HERMES_SMOKE_DIAG_DIR:-}" ]]; then
mkdir -p "$HERMES_SMOKE_DIAG_DIR" 2>/dev/null || true
printf "%s\n" "$*" > "${HERMES_SMOKE_DIAG_DIR}/failure.txt" 2>/dev/null || true
fi
exit 1
}
require_cmd() {
local cmd="$1"
command -v "$cmd" >/dev/null 2>&1 || fail "missing required command: ${cmd}"
}
PAPERCLIP_API_URL="${PAPERCLIP_API_URL:-http://127.0.0.1:3100}"
API_BASE="${PAPERCLIP_API_URL%/}/api"
COMPANY_ID="${COMPANY_ID:-${PAPERCLIP_COMPANY_ID:-}}"
COMPANY_SELECTOR="${COMPANY_SELECTOR:-}"
RUN_SUFFIX="${HERMES_SMOKE_RUN_SUFFIX:-$(date +%Y%m%d-%H%M%S)-$$}"
HERMES_IMAGE="${HERMES_IMAGE:-paperclip-hermes-gateway-smoke:local}"
HERMES_VERSION="${HERMES_VERSION:-0.17.0}"
HERMES_BUILD="${HERMES_BUILD:-1}"
HERMES_DOCKER_CONTEXT="${HERMES_DOCKER_CONTEXT:-docker/hermes-gateway-smoke}"
HERMES_CONTAINER_NAME="${HERMES_CONTAINER_NAME:-paperclip-hermes-gateway-smoke-${RUN_SUFFIX}}"
HERMES_GATEWAY_PORT="${HERMES_GATEWAY_PORT:-8642}"
HERMES_GATEWAY_API_BASE_URL="${HERMES_GATEWAY_API_BASE_URL:-http://127.0.0.1:${HERMES_GATEWAY_PORT}}"
HERMES_GATEWAY_PROBE_URL="${HERMES_GATEWAY_PROBE_URL:-http://127.0.0.1:${HERMES_GATEWAY_PORT}}"
HERMES_GATEWAY_API_KEY="${HERMES_GATEWAY_API_KEY:-${API_SERVER_KEY:-}}"
HERMES_GATEWAY_ALLOW_INSECURE_HTTP="${HERMES_GATEWAY_ALLOW_INSECURE_HTTP:-0}"
HERMES_GATEWAY_SESSION_KEY_STRATEGY="${HERMES_GATEWAY_SESSION_KEY_STRATEGY:-issue}"
HERMES_ADAPTER_TIMEOUT_SEC="${HERMES_ADAPTER_TIMEOUT_SEC:-180}"
HERMES_DIRECT_RUN_TIMEOUT_SEC="${HERMES_DIRECT_RUN_TIMEOUT_SEC:-180}"
HERMES_DIRECT_RUN_EVENTS_TIMEOUT_SEC="${HERMES_DIRECT_RUN_EVENTS_TIMEOUT_SEC:-20}"
HERMES_STOP_ASSERT="${HERMES_STOP_ASSERT:-auto}"
HERMES_SMOKE_KEEP="${HERMES_SMOKE_KEEP:-0}"
HERMES_SMOKE_NETWORK="${HERMES_SMOKE_NETWORK:-}"
HERMES_DOCKER_ADD_HOST="${HERMES_DOCKER_ADD_HOST:-1}"
HERMES_SMOKE_STATE_DIR="${HERMES_SMOKE_STATE_DIR:-${TMPDIR:-/tmp}/paperclip-hermes-gateway-smoke-${RUN_SUFFIX}}"
HERMES_SMOKE_DIAG_DIR="${HERMES_SMOKE_DIAG_DIR:-${TMPDIR:-/tmp}/paperclip-hermes-gateway-e2e-diag-${RUN_SUFFIX}}"
HERMES_SMOKE_MODEL_PROVIDER="${HERMES_SMOKE_MODEL_PROVIDER:-}"
HERMES_SMOKE_MODEL_DEFAULT="${HERMES_SMOKE_MODEL_DEFAULT:-}"
HERMES_SMOKE_MODEL_BASE_URL="${HERMES_SMOKE_MODEL_BASE_URL:-}"
HERMES_AGENT_NAME="${HERMES_AGENT_NAME:-Hermes Gateway Smoke Agent ${RUN_SUFFIX}}"
PAPERCLIP_API_URL_FOR_HERMES="${PAPERCLIP_API_URL_FOR_HERMES:-http://host.docker.internal:3100}"
RUN_TIMEOUT_SEC="${RUN_TIMEOUT_SEC:-420}"
CASE_TIMEOUT_SEC="${CASE_TIMEOUT_SEC:-420}"
GATEWAY_READY_TIMEOUT_SEC="${GATEWAY_READY_TIMEOUT_SEC:-90}"
STRICT_CASES="${STRICT_CASES:-1}"
HERMES_PROVIDER_ENV_KEYS=(
OPENROUTER_API_KEY
OPENAI_API_KEY
ANTHROPIC_API_KEY
GEMINI_API_KEY
GOOGLE_API_KEY
MISTRAL_API_KEY
)
print_usage() {
cat <<'EOF'
Hermes gateway Docker E2E smoke
Builds a fresh Hermes gateway container, verifies the gateway API directly,
joins it to Paperclip as a hermes_gateway agent, wakes that agent on a smoke
issue, verifies the issue result, captures redacted diagnostics, and cleans up
Paperclip and Docker state unless HERMES_SMOKE_KEEP=1.
Required:
PAPERCLIP_API_URL=http://127.0.0.1:3100
PAPERCLIP_AUTH_HEADER='Bearer <board-token>' # or PAPERCLIP_COOKIE
Common flags:
COMPANY_ID=<uuid> or COMPANY_SELECTOR=<prefix|name|uuid>
HERMES_VERSION=0.17.0
HERMES_IMAGE=paperclip-hermes-gateway-smoke:local
HERMES_GATEWAY_PORT=8642
HERMES_GATEWAY_API_BASE_URL=http://127.0.0.1:8642
HERMES_GATEWAY_PROBE_URL=http://127.0.0.1:8642
PAPERCLIP_API_URL_FOR_HERMES=http://host.docker.internal:3100
HERMES_GATEWAY_ALLOW_INSECURE_HTTP=1 # dev-only non-loopback HTTP
HERMES_SMOKE_NETWORK=<docker-network>
HERMES_DOCKER_ADD_HOST=0|1
HERMES_SMOKE_KEEP=1 # keep diagnostics/container
HERMES_SMOKE_DIAG_DIR=/tmp/hermes-gateway-diag
HERMES_SMOKE_MODEL_PROVIDER=openrouter
HERMES_SMOKE_MODEL_DEFAULT=z-ai/glm-5.2
HERMES_SMOKE_MODEL_BASE_URL=https://openrouter.ai/api/v1
Mode notes:
HERMES_GATEWAY_API_BASE_URL is the URL stored on the Paperclip adapter and
must be reachable by the Paperclip server. HERMES_GATEWAY_PROBE_URL is the URL
this operator shell uses for direct gateway checks. They can differ for Docker
network and reverse-proxy smoke runs.
Raw Hermes and Paperclip API keys are redacted from logs and diagnostic files.
The E2E helper seeds a minimal non-secret Hermes config in the fresh container
state, including command_allowlist: execute_code so gateway/API runs do not
pause on an interactive approval prompt.
The generated/claimed key material is kept only in the per-run state directory,
which is deleted on success unless HERMES_SMOKE_KEEP=1.
See doc/HERMES_GATEWAY_SMOKE.md for Docker Desktop, Linux, same-network,
LAN/private-network, and reverse-proxy/TLS examples.
EOF
}
case "${1:-}" in
-h|--help)
print_usage
exit 0
;;
esac
AUTH_HEADERS=()
if [[ -n "${PAPERCLIP_AUTH_HEADER:-}" ]]; then
AUTH_HEADERS+=(-H "Authorization: ${PAPERCLIP_AUTH_HEADER}")
elif [[ -n "${PAPERCLIP_API_KEY:-}" ]]; then
AUTH_HEADERS+=(-H "Authorization: Bearer ${PAPERCLIP_API_KEY}")
fi
if [[ -n "${PAPERCLIP_COOKIE:-}" ]]; then
AUTH_HEADERS+=(-H "Cookie: ${PAPERCLIP_COOKIE}")
PAPERCLIP_BROWSER_ORIGIN="${PAPERCLIP_BROWSER_ORIGIN:-${PAPERCLIP_API_URL%/}}"
AUTH_HEADERS+=(-H "Origin: ${PAPERCLIP_BROWSER_ORIGIN}" -H "Referer: ${PAPERCLIP_BROWSER_ORIGIN}/")
fi
RESPONSE_CODE=""
RESPONSE_BODY=""
AGENT_ID=""
AGENT_API_KEY=""
INVITE_ID=""
JOIN_REQUEST_ID=""
KEY_ID=""
SMOKE_ISSUE_ID=""
SMOKE_ISSUE_IDENTIFIER=""
RUN_ID=""
DIRECT_RUN_ID=""
STOP_RUN_ID=""
JOIN_OUTPUT_FILE="${HERMES_SMOKE_DIAG_DIR}/join-output.json"
KEEP_ON_EXIT="$HERMES_SMOKE_KEEP"
hash_prefix() {
local value="$1"
if command -v sha256sum >/dev/null 2>&1; then
printf "%s" "$value" | sha256sum | awk '{print substr($1,1,12)}'
elif command -v shasum >/dev/null 2>&1; then
printf "%s" "$value" | shasum -a 256 | awk '{print substr($1,1,12)}'
else
printf "unavailable"
fi
}
generate_key() {
if command -v openssl >/dev/null 2>&1; then
openssl rand -hex 32
elif command -v node >/dev/null 2>&1; then
node -e 'process.stdout.write(require("node:crypto").randomBytes(32).toString("hex"))'
else
fail "missing openssl or node for API_SERVER_KEY generation"
fi
}
redact_text() {
local text="$1"
local secret
for secret in \
"${HERMES_GATEWAY_API_KEY:-}" \
"${AGENT_API_KEY:-}" \
"${PAPERCLIP_API_KEY:-}" \
"${PAPERCLIP_AUTH_HEADER:-}" \
"${PAPERCLIP_COOKIE:-}"; do
if [[ -n "$secret" ]]; then
text="${text//$secret/[redacted len=${#secret}]}"
fi
done
local key
for key in "${HERMES_PROVIDER_ENV_KEYS[@]}"; do
secret="${!key-}"
if [[ -n "$secret" ]]; then
text="${text//$secret/[redacted len=${#secret}]}"
fi
done
printf "%s" "$text"
}
url_host() {
local url="$1"
local rest host_port host
rest="${url#http://}"
rest="${rest#https://}"
if [[ "$rest" == \[*\]* ]]; then
host="${rest#\[}"
host="${host%%\]*}"
else
host_port="${rest%%/*}"
host="${host_port%%:*}"
fi
printf "%s" "$host"
}
is_loopback_http_host() {
local host
host="$(printf "%s" "$1" | tr '[:upper:]' '[:lower:]')"
case "$host" in
localhost|0.0.0.0|::1|0:0:0:0:0:0:0:1) return 0 ;;
esac
[[ "$host" =~ ^127\.([0-9]{1,3}\.){2}[0-9]{1,3}$ ]]
}
is_remote_plain_http() {
local url="$1"
[[ "$url" == http://* ]] || return 1
! is_loopback_http_host "$(url_host "$url")"
}
assert_gateway_api_base_url_allowed() {
if is_remote_plain_http "$HERMES_GATEWAY_API_BASE_URL" && [[ "$HERMES_GATEWAY_ALLOW_INSECURE_HTTP" != "1" ]]; then
fail "HERMES_GATEWAY_API_BASE_URL uses non-loopback http. Set HERMES_GATEWAY_ALLOW_INSECURE_HTTP=1 for local-only unsafe HTTP, or use HTTPS."
fi
}
api_request() {
local method="$1"
local path="$2"
local data="${3-}"
local tmp
tmp="$(mktemp)"
local url
if [[ "$path" == http://* || "$path" == https://* ]]; then
url="$path"
elif [[ "$path" == /api/* ]]; then
url="${PAPERCLIP_API_URL%/}${path}"
else
url="${API_BASE}${path}"
fi
if [[ -n "$data" ]]; then
RESPONSE_CODE="$(curl -sS -o "$tmp" -w "%{http_code}" -X "$method" "${AUTH_HEADERS[@]}" -H "Content-Type: application/json" "$url" --data "$data")"
else
RESPONSE_CODE="$(curl -sS -o "$tmp" -w "%{http_code}" -X "$method" "${AUTH_HEADERS[@]}" "$url")"
fi
RESPONSE_BODY="$(cat "$tmp")"
rm -f "$tmp"
}
assert_status() {
local expected="$1"
if [[ "$RESPONSE_CODE" != "$expected" ]]; then
redact_text "$RESPONSE_BODY" >&2
echo >&2
fail "expected HTTP ${expected}, got HTTP ${RESPONSE_CODE}"
fi
}
gateway_request() {
local method="$1"
local path="$2"
local data="${3-}"
local output_file="${4-}"
local tmp
tmp="$(mktemp)"
local url="${HERMES_GATEWAY_PROBE_URL%/}${path}"
if [[ -n "$data" ]]; then
RESPONSE_CODE="$(curl -s -o "$tmp" -w "%{http_code}" -X "$method" -H "Authorization: Bearer ${HERMES_GATEWAY_API_KEY}" -H "Content-Type: application/json" "$url" --data "$data" || true)"
else
RESPONSE_CODE="$(curl -s -o "$tmp" -w "%{http_code}" -X "$method" -H "Authorization: Bearer ${HERMES_GATEWAY_API_KEY}" "$url" || true)"
fi
RESPONSE_BODY="$(cat "$tmp")"
if [[ -n "$output_file" ]]; then
redact_text "$RESPONSE_BODY" > "$output_file"
fi
rm -f "$tmp"
}
wait_http_ready() {
local url="$1"
local timeout_sec="$2"
local started now code
started="$(date +%s)"
while true; do
code="$(curl -sS -o /dev/null -w "%{http_code}" -H "Authorization: Bearer ${HERMES_GATEWAY_API_KEY}" "$url" || true)"
if [[ "$code" == "200" ]]; then
return 0
fi
now="$(date +%s)"
if (( now - started >= timeout_sec )); then
return 1
fi
sleep 1
done
}
require_board_auth() {
if [[ ${#AUTH_HEADERS[@]} -eq 0 ]]; then
fail "board/operator auth required. Set PAPERCLIP_COOKIE, PAPERCLIP_AUTH_HEADER, or a board-capable PAPERCLIP_API_KEY."
fi
api_request "GET" "/companies"
if [[ "$RESPONSE_CODE" != "200" ]]; then
redact_text "$RESPONSE_BODY" >&2
echo >&2
fail "board/operator auth invalid for /api/companies (HTTP ${RESPONSE_CODE})"
fi
}
resolve_company_id() {
if [[ -n "$COMPANY_ID" ]]; then
log "using company ${COMPANY_ID}"
return
fi
api_request "GET" "/companies"
assert_status "200"
if [[ -n "$COMPANY_SELECTOR" ]]; then
COMPANY_ID="$(jq -r --arg selector "$COMPANY_SELECTOR" '
map(select(
(.id == $selector)
or ((.issuePrefix // "") == $selector)
or ((.name // "") == $selector)
)) | .[0].id // empty
' <<<"$RESPONSE_BODY")"
[[ -n "$COMPANY_ID" ]] || fail "no company matched COMPANY_SELECTOR=${COMPANY_SELECTOR}"
else
COMPANY_ID="$(jq -r '.[0].id // empty' <<<"$RESPONSE_BODY")"
[[ -n "$COMPANY_ID" ]] || fail "no companies found"
fi
log "resolved company ${COMPANY_ID}"
}
capture_container_logs() {
mkdir -p "$HERMES_SMOKE_DIAG_DIR"
docker logs --tail=2000 "$HERMES_CONTAINER_NAME" > "${HERMES_SMOKE_DIAG_DIR}/hermes-container.log" 2>&1 || true
if [[ -s "${HERMES_SMOKE_DIAG_DIR}/hermes-container.log" ]]; then
local redacted_tmp
redacted_tmp="$(mktemp)"
redact_text "$(cat "${HERMES_SMOKE_DIAG_DIR}/hermes-container.log")" > "$redacted_tmp"
mv "$redacted_tmp" "${HERMES_SMOKE_DIAG_DIR}/hermes-container.log"
fi
}
capture_run_diagnostics() {
local run_id="$1"
local label="${2:-run}"
[[ -n "$run_id" ]] || return 0
mkdir -p "$HERMES_SMOKE_DIAG_DIR"
api_request "GET" "/heartbeat-runs/${run_id}/events?limit=1000"
if [[ "$RESPONSE_CODE" == "200" ]]; then
redact_text "$RESPONSE_BODY" > "${HERMES_SMOKE_DIAG_DIR}/${label}-${run_id}-events.json"
fi
api_request "GET" "/heartbeat-runs/${run_id}/log?limitBytes=524288"
if [[ "$RESPONSE_CODE" == "200" ]]; then
redact_text "$RESPONSE_BODY" > "${HERMES_SMOKE_DIAG_DIR}/${label}-${run_id}-log.json"
jq -r '.content // ""' <<<"$RESPONSE_BODY" | while IFS= read -r line; do redact_text "$line"; echo; done > "${HERMES_SMOKE_DIAG_DIR}/${label}-${run_id}-log.txt" 2>/dev/null || true
fi
}
capture_issue_diagnostics() {
local issue_id="$1"
local label="${2:-issue}"
[[ -n "$issue_id" ]] || return 0
mkdir -p "$HERMES_SMOKE_DIAG_DIR"
api_request "GET" "/issues/${issue_id}"
if [[ "$RESPONSE_CODE" == "200" ]]; then
redact_text "$RESPONSE_BODY" > "${HERMES_SMOKE_DIAG_DIR}/${label}-${issue_id}.json"
fi
api_request "GET" "/issues/${issue_id}/comments"
if [[ "$RESPONSE_CODE" == "200" ]]; then
redact_text "$RESPONSE_BODY" > "${HERMES_SMOKE_DIAG_DIR}/${label}-${issue_id}-comments.json"
fi
}
capture_diagnostics() {
mkdir -p "$HERMES_SMOKE_DIAG_DIR"
{
echo "runSuffix=${RUN_SUFFIX}"
echo "companyId=${COMPANY_ID:-}"
echo "agentId=${AGENT_ID:-}"
echo "inviteId=${INVITE_ID:-}"
echo "joinRequestId=${JOIN_REQUEST_ID:-}"
echo "container=${HERMES_CONTAINER_NAME}"
echo "image=${HERMES_IMAGE}"
echo "gateway=${HERMES_GATEWAY_API_BASE_URL}"
echo "gatewayProbe=${HERMES_GATEWAY_PROBE_URL}"
echo "paperclipApiUrl=${PAPERCLIP_API_URL}"
echo "paperclipApiUrlForHermes=${PAPERCLIP_API_URL_FOR_HERMES}"
echo "apiServerKeySha256=$(hash_prefix "${HERMES_GATEWAY_API_KEY:-}") len=${#HERMES_GATEWAY_API_KEY}"
echo "agentApiKeySha256=$(hash_prefix "${AGENT_API_KEY:-}") len=${#AGENT_API_KEY}"
} > "${HERMES_SMOKE_DIAG_DIR}/summary.env"
gateway_request "GET" "/health" "" "${HERMES_SMOKE_DIAG_DIR}/gateway-health.json" || true
gateway_request "GET" "/v1/capabilities" "" "${HERMES_SMOKE_DIAG_DIR}/gateway-capabilities.json" || true
capture_container_logs
capture_issue_diagnostics "$SMOKE_ISSUE_ID" "paperclip-smoke"
capture_run_diagnostics "$RUN_ID" "paperclip-smoke"
}
cleanup_paperclip_state() {
[[ "$KEEP_ON_EXIT" != "1" ]] || return 0
if [[ -n "$SMOKE_ISSUE_ID" ]]; then
log "deleting smoke issue ${SMOKE_ISSUE_ID}"
api_request "DELETE" "/issues/${SMOKE_ISSUE_ID}"
if [[ "$RESPONSE_CODE" != "200" && "$RESPONSE_CODE" != "404" && "$RESPONSE_CODE" != "204" ]]; then
warn "delete issue returned HTTP ${RESPONSE_CODE}"
fi
fi
if [[ -n "$AGENT_ID" ]]; then
log "terminating/deleting smoke agent ${AGENT_ID}"
api_request "POST" "/agents/${AGENT_ID}/terminate" "{}"
if [[ "$RESPONSE_CODE" != "200" && "$RESPONSE_CODE" != "404" ]]; then
warn "terminate agent returned HTTP ${RESPONSE_CODE}"
fi
api_request "DELETE" "/agents/${AGENT_ID}"
if [[ "$RESPONSE_CODE" != "200" && "$RESPONSE_CODE" != "404" && "$RESPONSE_CODE" != "204" ]]; then
warn "delete agent returned HTTP ${RESPONSE_CODE}"
fi
fi
if [[ -n "$JOIN_REQUEST_ID" && -n "$COMPANY_ID" ]]; then
api_request "POST" "/companies/${COMPANY_ID}/join-requests/${JOIN_REQUEST_ID}/reject" "{}"
if [[ "$RESPONSE_CODE" != "200" && "$RESPONSE_CODE" != "404" && "$RESPONSE_CODE" != "409" ]]; then
warn "reject join request returned HTTP ${RESPONSE_CODE}"
fi
fi
}
cleanup_local_state() {
[[ "$KEEP_ON_EXIT" != "1" ]] || return 0
docker rm -f "$HERMES_CONTAINER_NAME" >/dev/null 2>&1 || true
rm -rf "$HERMES_SMOKE_STATE_DIR"
rm -f "$JOIN_OUTPUT_FILE"
}
on_exit() {
local status=$?
if [[ "$status" -ne 0 ]]; then
KEEP_ON_EXIT=1
warn "smoke failed; preserving diagnostics/state"
capture_diagnostics || true
fi
cleanup_paperclip_state || true
cleanup_local_state || true
if [[ "$KEEP_ON_EXIT" == "1" ]]; then
warn "retained diagnostics: ${HERMES_SMOKE_DIAG_DIR}"
warn "retained state dir: ${HERMES_SMOKE_STATE_DIR}"
warn "retained container: ${HERMES_CONTAINER_NAME}"
fi
exit "$status"
}
trap on_exit EXIT
build_image() {
if [[ "$HERMES_BUILD" != "1" ]]; then
log "HERMES_BUILD=${HERMES_BUILD}; reusing image ${HERMES_IMAGE}"
return
fi
log "building Hermes gateway image ${HERMES_IMAGE} (HERMES_VERSION=${HERMES_VERSION})"
docker build --build-arg "HERMES_VERSION=${HERMES_VERSION}" -t "$HERMES_IMAGE" "$HERMES_DOCKER_CONTEXT"
}
prepare_fresh_state() {
mkdir -p "$HERMES_SMOKE_DIAG_DIR"
if [[ -e "$HERMES_SMOKE_STATE_DIR" && "$HERMES_SMOKE_KEEP" != "1" ]]; then
rm -rf "$HERMES_SMOKE_STATE_DIR"
fi
mkdir -p \
"${HERMES_SMOKE_STATE_DIR}/hermes-home" \
"${HERMES_SMOKE_STATE_DIR}/workspace" \
"${HERMES_SMOKE_STATE_DIR}/fake-host-home/.hermes"
# These host-created bind mounts must be readable and writable by the
# non-root hermes user (uid 10001) inside the container.
chmod 777 "${HERMES_SMOKE_STATE_DIR}/hermes-home" "${HERMES_SMOKE_STATE_DIR}/workspace" || true
echo "host hermes sentinel ${RUN_SUFFIX}" > "${HERMES_SMOKE_STATE_DIR}/fake-host-home/.hermes/host-sentinel.txt"
if find "${HERMES_SMOKE_STATE_DIR}/hermes-home" -mindepth 1 -print -quit | grep -q .; then
fail "Hermes state dir is not empty: ${HERMES_SMOKE_STATE_DIR}/hermes-home"
fi
}
yaml_single_quote() {
local value="$1"
value="${value//\'/\'\'}"
printf "'%s'" "$value"
}
write_hermes_model_config() {
local has_model_config=0
if [[ -n "$HERMES_SMOKE_MODEL_PROVIDER" || -n "$HERMES_SMOKE_MODEL_DEFAULT" || -n "$HERMES_SMOKE_MODEL_BASE_URL" ]]; then
has_model_config=1
fi
if [[ "$has_model_config" == "1" && ( -z "$HERMES_SMOKE_MODEL_PROVIDER" || -z "$HERMES_SMOKE_MODEL_DEFAULT" ) ]]; then
fail "HERMES_SMOKE_MODEL_PROVIDER and HERMES_SMOKE_MODEL_DEFAULT must be set together"
fi
local config_file="${HERMES_SMOKE_STATE_DIR}/hermes-home/config.yaml"
if [[ -e "$config_file" ]]; then
fail "Hermes model config already exists in fresh state: ${config_file}"
fi
{
if [[ "$has_model_config" == "1" ]]; then
echo "model:"
printf " default: %s\n" "$(yaml_single_quote "$HERMES_SMOKE_MODEL_DEFAULT")"
printf " provider: %s\n" "$(yaml_single_quote "$HERMES_SMOKE_MODEL_PROVIDER")"
if [[ -n "$HERMES_SMOKE_MODEL_BASE_URL" ]]; then
printf " base_url: %s\n" "$(yaml_single_quote "$HERMES_SMOKE_MODEL_BASE_URL")"
fi
echo "providers: {}"
fi
echo "command_allowlist:"
echo "- execute_code"
} > "$config_file"
chmod 644 "$config_file"
if [[ "$has_model_config" == "1" ]]; then
log "seeded Hermes model config provider=${HERMES_SMOKE_MODEL_PROVIDER} model=${HERMES_SMOKE_MODEL_DEFAULT}"
else
log "seeded Hermes smoke config"
fi
}
start_container() {
docker rm -f "$HERMES_CONTAINER_NAME" >/dev/null 2>&1 || true
local args=(
run -d
--name "$HERMES_CONTAINER_NAME"
-p "127.0.0.1:${HERMES_GATEWAY_PORT}:8642"
-e API_SERVER_ENABLED=true
-e API_SERVER_KEY="$HERMES_GATEWAY_API_KEY"
-e API_SERVER_HOST=0.0.0.0
-e API_SERVER_PORT=8642
-e PAPERCLIP_API_URL="$PAPERCLIP_API_URL_FOR_HERMES"
-e NO_COLOR=1
-v "${HERMES_SMOKE_STATE_DIR}/hermes-home:/home/hermes/.hermes"
-v "${HERMES_SMOKE_STATE_DIR}/workspace:/home/hermes/workspace"
)
local provider_key
local provider_keys=()
for provider_key in "${HERMES_PROVIDER_ENV_KEYS[@]}"; do
if [[ -n "${!provider_key-}" ]]; then
args+=(-e "${provider_key}=${!provider_key}")
provider_keys+=("$provider_key")
fi
done
if [[ ${#provider_keys[@]} -gt 0 ]]; then
log "passing Hermes inference provider env keys: ${provider_keys[*]}"
else
warn "no Hermes inference provider env keys set; direct run will fail unless Hermes state config already has a provider"
fi
if [[ -n "$HERMES_SMOKE_NETWORK" ]]; then
args+=(--network "$HERMES_SMOKE_NETWORK")
fi
if [[ "$HERMES_DOCKER_ADD_HOST" == "1" ]]; then
args+=(--add-host=host.docker.internal:host-gateway)
fi
args+=("$HERMES_IMAGE")
log "starting container ${HERMES_CONTAINER_NAME}"
docker "${args[@]}" >/dev/null
}
assert_fresh_container_state() {
log "asserting container does not see host Hermes state"
docker exec "$HERMES_CONTAINER_NAME" sh -lc 'test ! -e "$HERMES_HOME/host-sentinel.txt"'
docker exec "$HERMES_CONTAINER_NAME" sh -lc 'env | sort | grep -E "^(HOME|HERMES_HOME|XDG_|API_SERVER_|PAPERCLIP_API_URL)=" || true' > "${HERMES_SMOKE_DIAG_DIR}/container-env.txt"
docker exec "$HERMES_CONTAINER_NAME" sh -lc 'find "$HERMES_HOME" -maxdepth 2 -type f -print | sort' > "${HERMES_SMOKE_DIAG_DIR}/container-hermes-home-files-before.txt" || true
if docker exec "$HERMES_CONTAINER_NAME" sh -lc 'env | grep -q "^PAPERCLIP_API_KEY="'; then
fail "container unexpectedly has PAPERCLIP_API_KEY before join/key claim"
fi
}
probe_container_to_paperclip() {
log "probing container-to-Paperclip connectivity at ${PAPERCLIP_API_URL_FOR_HERMES}/api/health"
if ! docker exec "$HERMES_CONTAINER_NAME" curl -fsS --max-time 8 "${PAPERCLIP_API_URL_FOR_HERMES%/}/api/health" > "${HERMES_SMOKE_DIAG_DIR}/container-paperclip-health.json"; then
fail "Hermes container cannot reach Paperclip. Set PAPERCLIP_API_URL_FOR_HERMES to a URL reachable from inside Docker, or keep HERMES_DOCKER_ADD_HOST=1 for Linux host.docker.internal."
fi
}
probe_gateway_readiness() {
log "waiting for Hermes gateway health at ${HERMES_GATEWAY_PROBE_URL%/}/health"
if [[ "$HERMES_GATEWAY_PROBE_URL" != "$HERMES_GATEWAY_API_BASE_URL" ]]; then
log "Paperclip will store Hermes gateway URL ${HERMES_GATEWAY_API_BASE_URL}"
fi
wait_http_ready "${HERMES_GATEWAY_PROBE_URL%/}/health" "$GATEWAY_READY_TIMEOUT_SEC" || fail "Hermes gateway health did not become ready"
gateway_request "GET" "/health" "" "${HERMES_SMOKE_DIAG_DIR}/gateway-health.json"
assert_status "200"
local wrong_code
wrong_code="$(curl -sS -o /dev/null -w "%{http_code}" -H "Authorization: Bearer wrong-smoke-key" "${HERMES_GATEWAY_PROBE_URL%/}/v1/capabilities" || true)"
if [[ "$wrong_code" == "200" ]]; then
fail "Hermes protected endpoint accepted a wrong API key"
fi
}
assert_capabilities() {
log "asserting /v1/capabilities"
gateway_request "GET" "/v1/capabilities" "" "${HERMES_SMOKE_DIAG_DIR}/gateway-capabilities.json"
assert_status "200"
if ! jq -e 'type == "object"' <<<"$RESPONSE_BODY" >/dev/null; then
fail "capabilities response is not a JSON object"
fi
}
poll_gateway_run_terminal() {
local run_id="$1"
local timeout_sec="$2"
local label="$3"
local started now status
started="$(date +%s)"
while true; do
gateway_request "GET" "/v1/runs/${run_id}" "" "${HERMES_SMOKE_DIAG_DIR}/${label}-${run_id}-status.json"
if [[ "$RESPONSE_CODE" == "200" ]]; then
status="$(jq -r '.status // empty' <<<"$RESPONSE_BODY")"
case "$status" in
completed|failed|error|cancelled|canceled|stopped|interrupted)
echo "$status"
return 0
;;
esac
fi
now="$(date +%s)"
if (( now - started >= timeout_sec )); then
echo "timeout"
return 0
fi
sleep 2
done
}
assert_direct_gateway_run() {
local marker="HERMES_DIRECT_OK_${RUN_SUFFIX}"
local payload
payload="$(jq -nc \
--arg marker "$marker" \
--arg session "paperclip-smoke-direct-${RUN_SUFFIX}" \
'{input: ("Reply with exactly " + $marker + " and no other text."), instructions: "You are running a Paperclip Hermes gateway smoke direct API assertion.", session_id: $session}')"
log "asserting POST /v1/runs and SSE events"
gateway_request "POST" "/v1/runs" "$payload" "${HERMES_SMOKE_DIAG_DIR}/direct-run-create.json"
if [[ "$RESPONSE_CODE" != "200" && "$RESPONSE_CODE" != "202" ]]; then
redact_text "$RESPONSE_BODY" >&2
echo >&2
fail "expected HTTP 200 or 202, got HTTP ${RESPONSE_CODE}"
fi
DIRECT_RUN_ID="$(jq -r '.run_id // .runId // .id // empty' <<<"$RESPONSE_BODY")"
[[ -n "$DIRECT_RUN_ID" ]] || fail "direct run creation did not return run id"
local events_file="${HERMES_SMOKE_DIAG_DIR}/direct-run-${DIRECT_RUN_ID}-events.sse"
curl -sS --max-time "$HERMES_DIRECT_RUN_EVENTS_TIMEOUT_SEC" -N \
-H "Authorization: Bearer ${HERMES_GATEWAY_API_KEY}" \
"${HERMES_GATEWAY_PROBE_URL%/}/v1/runs/${DIRECT_RUN_ID}/events" \
> "${events_file}.raw" || true
redact_text "$(cat "${events_file}.raw")" > "$events_file"
rm -f "${events_file}.raw"
local events_seen=0
if grep -Eq '(^event:|^data:)' "$events_file"; then
events_seen=1
else
warn "SSE stream produced no event/data frames within ${HERMES_DIRECT_RUN_EVENTS_TIMEOUT_SEC}s; polling direct run status"
fi
local status
status="$(poll_gateway_run_terminal "$DIRECT_RUN_ID" "$HERMES_DIRECT_RUN_TIMEOUT_SEC" "direct-run")"
log "direct Hermes run ${DIRECT_RUN_ID} status=${status}"
[[ "$status" == "completed" ]] || fail "direct Hermes run did not complete successfully (status=${status})"
if [[ "$events_seen" != "1" ]]; then
warn "direct Hermes run completed, but the live SSE probe was quiet"
fi
}
assert_stop_behavior_if_deterministic() {
[[ "$HERMES_STOP_ASSERT" != "0" ]] || {
log "HERMES_STOP_ASSERT=0; skipping /stop assertion"
return
}
local payload
payload="$(jq -nc \
--arg session "paperclip-smoke-stop-${RUN_SUFFIX}" \
'{input: "Wait until stopped. If you cannot wait, emit a short acknowledgement.", instructions: "This run exists only to verify the Hermes gateway stop endpoint.", session_id: $session}')"
log "probing /stop behavior (mode=${HERMES_STOP_ASSERT})"
gateway_request "POST" "/v1/runs" "$payload" "${HERMES_SMOKE_DIAG_DIR}/stop-run-create.json"
if [[ "$RESPONSE_CODE" != "200" && "$RESPONSE_CODE" != "202" ]]; then
redact_text "$RESPONSE_BODY" >&2
echo >&2
fail "expected HTTP 200 or 202, got HTTP ${RESPONSE_CODE}"
fi
STOP_RUN_ID="$(jq -r '.run_id // .runId // .id // empty' <<<"$RESPONSE_BODY")"
[[ -n "$STOP_RUN_ID" ]] || fail "stop test run creation did not return run id"
gateway_request "POST" "/v1/runs/${STOP_RUN_ID}/stop" "{}" "${HERMES_SMOKE_DIAG_DIR}/stop-run-stop-response.json"
if [[ "$RESPONSE_CODE" != "200" && "$RESPONSE_CODE" != "202" && "$RESPONSE_CODE" != "204" ]]; then
if [[ "$HERMES_STOP_ASSERT" == "auto" ]]; then
warn "/stop returned HTTP ${RESPONSE_CODE}; treating stop assertion as non-deterministic"
return
fi
fail "/stop returned HTTP ${RESPONSE_CODE}"
fi
local status
status="$(poll_gateway_run_terminal "$STOP_RUN_ID" 45 "stop-run")"
case "$status" in
cancelled|canceled|stopped|interrupted)
log "stop run ${STOP_RUN_ID} reached ${status}"
;;
completed)
if [[ "$HERMES_STOP_ASSERT" == "auto" ]]; then
warn "stop run completed before cancellation could be observed; treating stop assertion as non-deterministic"
else
fail "stop run completed instead of stopping"
fi
;;
*)
if [[ "$HERMES_STOP_ASSERT" == "auto" ]]; then
warn "stop run terminal status ${status}; treating stop assertion as non-deterministic"
else
fail "stop run did not reach a stopped/cancelled terminal status (status=${status})"
fi
;;
esac
}
join_hermes_agent() {
log "running join-only smoke helper"
local join_log="${HERMES_SMOKE_DIAG_DIR}/hermes-gateway-join.log"
HERMES_AGENT_NAME="$HERMES_AGENT_NAME" \
HERMES_GATEWAY_API_BASE_URL="$HERMES_GATEWAY_API_BASE_URL" \
HERMES_GATEWAY_PROBE_URL="$HERMES_GATEWAY_PROBE_URL" \
HERMES_GATEWAY_API_KEY="$HERMES_GATEWAY_API_KEY" \
HERMES_GATEWAY_ALLOW_INSECURE_HTTP="$HERMES_GATEWAY_ALLOW_INSECURE_HTTP" \
HERMES_GATEWAY_SESSION_KEY_STRATEGY="$HERMES_GATEWAY_SESSION_KEY_STRATEGY" \
HERMES_GATEWAY_TIMEOUT_SEC="$HERMES_ADAPTER_TIMEOUT_SEC" \
PAPERCLIP_API_URL="$PAPERCLIP_API_URL" \
PAPERCLIP_API_URL_FOR_HERMES="$PAPERCLIP_API_URL_FOR_HERMES" \
PAPERCLIP_AUTH_HEADER="${PAPERCLIP_AUTH_HEADER:-}" \
PAPERCLIP_API_KEY="${PAPERCLIP_API_KEY:-}" \
PAPERCLIP_COOKIE="${PAPERCLIP_COOKIE:-}" \
COMPANY_ID="$COMPANY_ID" \
COMPANY_SELECTOR="$COMPANY_SELECTOR" \
HERMES_JOIN_OUTPUT_FILE="$JOIN_OUTPUT_FILE" \
bash scripts/smoke/hermes-gateway-join.sh > "${join_log}.raw" 2>&1 || {
redact_text "$(cat "${join_log}.raw")" > "$join_log"
rm -f "${join_log}.raw"
fail "join helper failed; see ${join_log}"
}
redact_text "$(cat "${join_log}.raw")" > "$join_log"
rm -f "${join_log}.raw"
[[ -f "$JOIN_OUTPUT_FILE" ]] || fail "join helper did not write ${JOIN_OUTPUT_FILE}"
AGENT_ID="$(jq -r '.agentId // empty' "$JOIN_OUTPUT_FILE")"
AGENT_API_KEY="$(jq -r '.agentApiKey // empty' "$JOIN_OUTPUT_FILE")"
INVITE_ID="$(jq -r '.inviteId // empty' "$JOIN_OUTPUT_FILE")"
JOIN_REQUEST_ID="$(jq -r '.joinRequestId // empty' "$JOIN_OUTPUT_FILE")"
KEY_ID="$(jq -r '.keyId // empty' "$JOIN_OUTPUT_FILE")"
[[ -n "$AGENT_ID" && -n "$AGENT_API_KEY" ]] || fail "join output missing agent id or API key"
log "joined Hermes gateway agent ${AGENT_ID} keyId=${KEY_ID} agentKeySha256=$(hash_prefix "$AGENT_API_KEY")"
}
install_claimed_key_in_container() {
log "placing newly claimed Paperclip key in container workspace"
local key_file="${HERMES_SMOKE_STATE_DIR}/workspace/paperclip-claimed-api-key.json"
jq -nc --arg token "$AGENT_API_KEY" '{token:$token,apiKey:$token}' > "$key_file"
# The host-created bind-mounted file must be readable by the non-root hermes
# user inside the container. The state dir is still per-run and deleted on
# success unless HERMES_SMOKE_KEEP=1.
chmod 644 "$key_file"
docker exec "$HERMES_CONTAINER_NAME" sh -lc 'test -f /home/hermes/workspace/paperclip-claimed-api-key.json && test ! -e "$HERMES_HOME/host-sentinel.txt"'
}
patch_agent_instructions_with_claimed_key() {
log "patching Hermes agent instructions with claimed Paperclip API context"
api_request "GET" "/agents/${AGENT_ID}"
assert_status "200"
local instructions patch_payload
instructions="For this smoke run only, call Paperclip at ${PAPERCLIP_API_URL_FOR_HERMES}. Read /home/hermes/workspace/paperclip-claimed-api-key.json and use its token as PAPERCLIP_API_KEY for Paperclip API requests. Do not reveal this key. When mutating Paperclip, include X-Paperclip-Run-Id with the current Paperclip run id when available."
patch_payload="$(jq -c --arg instructions "$instructions" '
{adapterConfig: ((.adapterConfig // {}) + {instructions: $instructions})}
' <<<"$RESPONSE_BODY")"
api_request "PATCH" "/agents/${AGENT_ID}" "$patch_payload"
assert_status "200"
}
create_smoke_issue() {
local marker="HERMES_PAPERCLIP_E2E_OK_${RUN_SUFFIX}"
local title="[Hermes Gateway Smoke] ${RUN_SUFFIX}"
local description
description="Hermes gateway full Docker e2e smoke.\n\n1. Read this issue.\n2. Post a Paperclip issue comment containing exactly: ${marker}\n3. Mark this issue done.\n\nUse the Paperclip API URL and key provided in your run instructions. Do not reveal secrets."
local payload
payload="$(jq -nc \
--arg title "$title" \
--arg description "$description" \
--arg assignee "$AGENT_ID" \
'{title:$title,description:$description,status:"todo",priority:"high",assigneeAgentId:$assignee}')"
api_request "POST" "/companies/${COMPANY_ID}/issues" "$payload"
assert_status "201"
SMOKE_ISSUE_ID="$(jq -r '.id // empty' <<<"$RESPONSE_BODY")"
SMOKE_ISSUE_IDENTIFIER="$(jq -r '.identifier // empty' <<<"$RESPONSE_BODY")"
[[ -n "$SMOKE_ISSUE_ID" ]] || fail "smoke issue create missing id"
log "created smoke issue ${SMOKE_ISSUE_ID} (${SMOKE_ISSUE_IDENTIFIER})"
echo "$marker" > "${HERMES_SMOKE_DIAG_DIR}/paperclip-marker.txt"
}
trigger_wakeup() {
local payload
payload="$(jq -nc --arg issueId "$SMOKE_ISSUE_ID" '{source:"on_demand",triggerDetail:"manual",reason:"hermes_gateway_docker_e2e_smoke",payload:{issueId:$issueId,taskId:$issueId}}')"
api_request "POST" "/agents/${AGENT_ID}/wakeup" "$payload"
if [[ "$RESPONSE_CODE" != "202" ]]; then
redact_text "$RESPONSE_BODY" >&2
echo >&2
fail "wakeup failed (HTTP ${RESPONSE_CODE})"
fi
RUN_ID="$(jq -r '.id // empty' <<<"$RESPONSE_BODY")"
[[ -n "$RUN_ID" ]] || fail "wakeup response missing run id"
log "triggered Paperclip run ${RUN_ID}"
}
get_run_status() {
local run_id="$1"
api_request "GET" "/companies/${COMPANY_ID}/heartbeat-runs?agentId=${AGENT_ID}&limit=200"
if [[ "$RESPONSE_CODE" != "200" ]]; then
echo ""
return 0
fi
jq -r --arg runId "$run_id" '.[] | select(.id == $runId) | .status' <<<"$RESPONSE_BODY" | head -n1
}
wait_for_run_terminal() {
local run_id="$1"
local timeout_sec="$2"
local started now status
started="$(date +%s)"
while true; do
status="$(get_run_status "$run_id")"
if [[ "$status" == "succeeded" || "$status" == "failed" || "$status" == "timed_out" || "$status" == "cancelled" ]]; then
echo "$status"
return
fi
now="$(date +%s)"
if (( now - started >= timeout_sec )); then
echo "timeout"
return
fi
sleep 3
done
}
get_issue_status() {
local issue_id="$1"
api_request "GET" "/issues/${issue_id}"
if [[ "$RESPONSE_CODE" != "200" ]]; then
echo ""
return 0
fi
jq -r '.status // empty' <<<"$RESPONSE_BODY"
}
wait_for_issue_terminal() {
local issue_id="$1"
local timeout_sec="$2"
local started now status
started="$(date +%s)"
while true; do
status="$(get_issue_status "$issue_id")"
if [[ "$status" == "done" || "$status" == "blocked" || "$status" == "cancelled" ]]; then
echo "$status"
return
fi
now="$(date +%s)"
if (( now - started >= timeout_sec )); then
echo "timeout"
return
fi
sleep 3
done
}
issue_comments_contain() {
local issue_id="$1"
local marker="$2"
api_request "GET" "/issues/${issue_id}/comments"
if [[ "$RESPONSE_CODE" != "200" ]]; then
echo "false"
return
fi
jq -r --arg marker "$marker" '[.[] | (.body // "") | contains($marker)] | any' <<<"$RESPONSE_BODY"
}
assert_paperclip_wake_success() {
local marker
marker="$(cat "${HERMES_SMOKE_DIAG_DIR}/paperclip-marker.txt")"
trigger_wakeup
local run_status issue_status marker_found
run_status="$(wait_for_run_terminal "$RUN_ID" "$RUN_TIMEOUT_SEC")"
log "Paperclip run ${RUN_ID} status=${run_status}"
issue_status="$(wait_for_issue_terminal "$SMOKE_ISSUE_ID" "$CASE_TIMEOUT_SEC")"
marker_found="$(issue_comments_contain "$SMOKE_ISSUE_ID" "$marker")"
log "smoke issue status=${issue_status} marker_found=${marker_found}"
if [[ "$run_status" != "succeeded" || "$issue_status" != "done" || "$marker_found" != "true" ]]; then
capture_diagnostics
fi
if [[ "$STRICT_CASES" == "1" ]]; then
[[ "$run_status" == "succeeded" ]] || fail "Paperclip Hermes gateway run did not succeed"
[[ "$issue_status" == "done" ]] || fail "smoke issue did not reach done"
[[ "$marker_found" == "true" ]] || fail "smoke marker was not found in issue comments"
fi
}
scan_diagnostics_for_secret_leaks() {
log "scanning diagnostics for raw secret leaks"
local secrets=()
[[ -n "$HERMES_GATEWAY_API_KEY" ]] && secrets+=("$HERMES_GATEWAY_API_KEY")
[[ -n "$AGENT_API_KEY" ]] && secrets+=("$AGENT_API_KEY")
local key
for key in "${HERMES_PROVIDER_ENV_KEYS[@]}"; do
[[ -n "${!key-}" ]] && secrets+=("${!key}")
done
[[ ${#secrets[@]} -gt 0 ]] || return
local file
while IFS= read -r file; do
[[ "$file" == "$JOIN_OUTPUT_FILE" ]] && continue
local secret
for secret in "${secrets[@]}"; do
if grep -Fq "$secret" "$file"; then
fail "raw secret leaked in diagnostics file ${file}"
fi
done
done < <(find "$HERMES_SMOKE_DIAG_DIR" -type f -print)
}
main() {
log "starting Hermes gateway Docker E2E smoke"
mkdir -p "$HERMES_SMOKE_DIAG_DIR"
log "diagnostics dir: ${HERMES_SMOKE_DIAG_DIR}"
require_cmd curl
require_cmd docker
require_cmd jq
if [[ -z "$HERMES_GATEWAY_API_KEY" ]]; then
HERMES_GATEWAY_API_KEY="$(generate_key)"
fi
log "Hermes API key sha256=$(hash_prefix "$HERMES_GATEWAY_API_KEY") len=${#HERMES_GATEWAY_API_KEY}"
assert_gateway_api_base_url_allowed
api_request "GET" "/health"
assert_status "200"
log "Paperclip health deploymentMode=$(jq -r '.deploymentMode // "unknown"' <<<"$RESPONSE_BODY") exposure=$(jq -r '.deploymentExposure // "unknown"' <<<"$RESPONSE_BODY")"
require_board_auth
resolve_company_id
prepare_fresh_state
write_hermes_model_config
build_image
start_container
assert_fresh_container_state
probe_container_to_paperclip
probe_gateway_readiness
assert_capabilities
assert_direct_gateway_run
assert_stop_behavior_if_deterministic
join_hermes_agent
install_claimed_key_in_container
patch_agent_instructions_with_claimed_key
create_smoke_issue
assert_paperclip_wake_success
capture_diagnostics
scan_diagnostics_for_secret_leaks
log "success"
log "companyId=${COMPANY_ID}"
log "agentId=${AGENT_ID}"
log "inviteId=${INVITE_ID}"
log "joinRequestId=${JOIN_REQUEST_ID}"
log "issueId=${SMOKE_ISSUE_ID}"
log "issueIdentifier=${SMOKE_ISSUE_IDENTIFIER}"
log "runId=${RUN_ID}"
log "directHermesRunId=${DIRECT_RUN_ID}"
log "diagnostics=${HERMES_SMOKE_DIAG_DIR}"
}
main "$@"