whisper-money/autoresearch.sh

#!/usr/bin/env bash
set -euo pipefail

RUN_ID="${RUN_ID:-}"
BRANCH="${BRANCH:-$(git branch --show-current)}"
WORKFLOW="${WORKFLOW:-CI}"

current_head() {
  git rev-parse HEAD
}

latest_run_for_head() {
  local event="$1"
  local head_sha="$2"
  gh run list --workflow="$WORKFLOW" --branch="$BRANCH" --event="$event" --limit=20 --json databaseId,headSha,status,conclusion \
    --jq ".[] | select(.headSha == \"$head_sha\") | .databaseId" | head -1
}

wait_for_run_for_head() {
  local event="$1"
  local head_sha="$2"
  local run_id=""

  for _ in {1..90}; do
    run_id="$(latest_run_for_head "$event" "$head_sha")"
    if [[ -n "$run_id" ]]; then
      echo "$run_id"
      return 0
    fi
    sleep 5
  done

  echo "No $event CI run appeared for $head_sha" >&2
  return 1
}

commit_and_push_if_dirty() {
  local message="$1"

  if ! git diff --quiet -- .github/workflows composer.json composer.lock tests/.pest autoresearch.md autoresearch.sh autoresearch.jsonl || \
     [[ -n "$(git ls-files --others --exclude-standard .github/workflows tests/.pest 2>/dev/null)" ]]; then
    git add .github/workflows composer.json composer.lock tests/.pest autoresearch.md autoresearch.sh autoresearch.jsonl
    git commit -m "$message"
    git push
  fi
}

# Time-balanced sharding experiment bootstrap:
# 1. Push Pest/update-shards workflow changes.
# 2. Run CI workflow_dispatch with build_only=false to produce tests/.pest/shards.json.
# 3. Download and commit shards.json.
# 4. Wait for PR CI for the committed shard timings and measure it.
if [[ -z "$RUN_ID" && ! -f tests/.pest/shards.json && "${GENERATE_BROWSER_SHARDS:-1}" == "1" ]]; then
  commit_and_push_if_dirty "Experiment: update Pest and enable browser shard timings"

  head_sha="$(current_head)"
  dispatch_run="$(latest_run_for_head workflow_dispatch "$head_sha")"
  if [[ -z "$dispatch_run" ]]; then
    gh workflow run "$WORKFLOW" --ref "$BRANCH" -f build_only=false
    dispatch_run="$(wait_for_run_for_head workflow_dispatch "$head_sha")"
  fi

  gh run watch "$dispatch_run" --exit-status --interval 10 >/dev/null

  tmp_dir="$(mktemp -d)"
  gh run download "$dispatch_run" --name browser-shards --dir "$tmp_dir" >/dev/null
  mkdir -p tests/.pest
  cp "$tmp_dir/shards.json" tests/.pest/shards.json
  rm -rf "$tmp_dir"

  commit_and_push_if_dirty "Experiment: add Pest browser shard timings"
  head_sha="$(current_head)"
  RUN_ID="$(wait_for_run_for_head pull_request "$head_sha")"
  gh run watch "$RUN_ID" --exit-status --interval 10 >/dev/null
elif [[ -z "$RUN_ID" ]]; then
  commit_and_push_if_dirty "Experiment CI change"
  head_sha="$(current_head)"
  RUN_ID="$(wait_for_run_for_head pull_request "$head_sha")"
  gh run watch "$RUN_ID" --exit-status --interval 10 >/dev/null
fi

python3 - <<'PY' "$RUN_ID" "$BRANCH" "$WORKFLOW"
from __future__ import annotations

import datetime as dt
import json
import subprocess
import sys
from collections import defaultdict

run_id, branch, workflow = sys.argv[1:4]


def gh_json(args: list[str]) -> object:
    raw = subprocess.check_output(['gh', *args], text=True, stderr=subprocess.DEVNULL)
    return json.loads(raw)

if not run_id:
    head_sha = subprocess.check_output(['git', 'rev-parse', 'HEAD'], text=True).strip()
    runs = gh_json([
        'run', 'list',
        '--workflow', workflow,
        '--branch', branch,
        '--event', 'pull_request',
        '--limit', '20',
        '--json', 'databaseId,status,conclusion,headSha',
    ])
    for run in runs:
        if run.get('headSha') == head_sha and run.get('status') == 'completed' and run.get('conclusion') == 'success':
            run_id = str(run['databaseId'])
            break

if not run_id:
    print('No successful completed pull_request CI run found for HEAD', file=sys.stderr)
    sys.exit(1)

payload = gh_json(['run', 'view', run_id, '--json', 'jobs,status,conclusion,headSha,headBranch,event'])
if payload.get('status') != 'completed' or payload.get('conclusion') != 'success':
    print(f'Run {run_id} not successful: {payload.get("status")} {payload.get("conclusion")}', file=sys.stderr)
    sys.exit(1)


def parse_time(value: str) -> dt.datetime:
    return dt.datetime.fromisoformat(value.replace('Z', '+00:00'))


def seconds(start: str, end: str) -> float:
    return max(0.0, (parse_time(end) - parse_time(start)).total_seconds())

jobs = [job for job in payload.get('jobs', []) if job.get('startedAt') and job.get('completedAt')]
if not jobs:
    print(f'Run {run_id} has no timed jobs', file=sys.stderr)
    sys.exit(1)

measured_jobs = [
    job for job in jobs
    if job.get('conclusion') != 'skipped' or job['name'] in {'build-assets'}
]
measured_jobs = [
    job for job in measured_jobs
    if not job['name'].startswith('build-image') and not job['name'].startswith('deploy')
]

start = min(parse_time(job['startedAt']) for job in measured_jobs)
end = max(parse_time(job['completedAt']) for job in measured_jobs)
total = (end - start).total_seconds()

buckets: dict[str, list[float]] = defaultdict(list)
for job in jobs:
    name = job['name']
    duration = seconds(job['startedAt'], job['completedAt'])
    if name == 'tests':
        buckets['tests_s'].append(duration)
    elif name == 'linter':
        buckets['linter_s'].append(duration)
    elif name == 'static-analysis':
        buckets['static_analysis_s'].append(duration)
    elif name == 'performance-tests':
        buckets['performance_tests_s'].append(duration)
    elif name == 'build-assets':
        buckets['build_assets_s'].append(duration if job.get('conclusion') != 'skipped' else 0.0)
    elif name == 'browser-tests':
        buckets['browser_aggregate_s'].append(duration)
    elif name.startswith('browser-tests-matrix'):
        buckets['browser_matrix_shard_s'].append(duration)
    elif name == 'update-browser-shards':
        buckets['update_browser_shards_s'].append(duration)

browser_shards = buckets['browser_matrix_shard_s']
metrics = {
    'github_ci_total_s': total,
    'tests_s': max(buckets['tests_s'] or [0.0]),
    'linter_s': max(buckets['linter_s'] or [0.0]),
    'static_analysis_s': max(buckets['static_analysis_s'] or [0.0]),
    'performance_tests_s': max(buckets['performance_tests_s'] or [0.0]),
    'build_assets_s': max(buckets['build_assets_s'] or [0.0]),
    'browser_matrix_s': max(browser_shards or [0.0]),
    'browser_aggregate_s': max(buckets['browser_aggregate_s'] or [0.0]),
    'browser_shards': float(len(browser_shards)),
    'job_count': float(len(jobs)),
    'run_id': float(run_id),
}

for key, value in metrics.items():
    print(f'METRIC {key}={value:.3f}')
PY