honcho/sandbox/seed.py

"""Populate the sandbox with a known fixture and verify it actually landed.

Run through sandbox.sh, which supplies the base URL and owns the Docker and
Postgres side. This script only talks to the API:

    uv run --no-project --with-editable ./sdks/python python sandbox/seed.py verify

Two phases, because seeded conclusions are written between them by
inject_conclusions.py, which sandbox.sh runs inside the api container:

    seed.py seed      peers, messages, derivation
    seed.py verify    the whole round trip, seeded conclusions included

The split is what keeps the deriver honest. The check that proves derivation ran
at all is "some conclusion exists", and once conclusions are seeded that would
pass against a completely dead deriver - so it runs in the seed phase, before
anything is injected.

It is provider-agnostic. Under the mock provider the *text* of a derived
conclusion is synthetic and unrelated to the messages, so nothing here asserts on
derived content. Seeded conclusions are the opposite: their text is committed, so
they are asserted exactly, in both modes. What is asserted strictly either way is
the round trip that harness integrations actually get wrong: which messages
landed, on which peers, with which observation topology.

Exits non-zero on any failure. An empty sandbox that reports success is the exact
failure mode this whole thing exists to prevent.
"""

from __future__ import annotations

import json
import os
import sys
import time
from pathlib import Path
from typing import Any

from honcho import Honcho
from honcho.api_types import MessageCreateParams, SessionPeerConfig

FIXTURE = Path(__file__).with_name("fixture.json")

# Generous: a cold deriver on a real provider is slow, and the failure we care
# about (nothing is processing at all) shows up as a timeout either way.
DRAIN_TIMEOUT_SECONDS = float(os.environ.get("SANDBOX_DRAIN_TIMEOUT", "300"))
DRAIN_POLL_SECONDS = 0.5


class SeedError(RuntimeError):
    """Seeding did not reach a usable state."""


def log(message: str) -> None:
    print(f"[seed] {message}", flush=True)


LEVELS = ("explicit", "deductive", "inductive")


def seeded_conclusions(
    fixture: dict[str, Any],
) -> dict[tuple[str, str], dict[str, list[str]]]:
    """What inject_conclusions.py should have written, keyed by (observer, observed).

    Mirrors the injector's resolution rules so verification is stated in terms of the
    fixture rather than of whatever happens to be in the database.
    """
    peers = fixture["peers"]
    planned: dict[tuple[str, str], dict[str, list[str]]] = {}
    for spec in peers:
        by_level = {
            level: [
                item if isinstance(item, str) else item["content"]
                for item in spec.get(level, [])
            ]
            for level in LEVELS
        }
        if not any(by_level.values()):
            continue
        override = spec.get("observer")
        observers = (
            [override]
            if override
            else [
                peer["id"]
                for peer in peers
                if peer.get("observe_others") and peer["id"] != spec["id"]
            ]
        )
        for observer in observers:
            planned[(observer, spec["id"])] = by_level
    return planned


def cited_premise_texts(spec: dict[str, Any], explicit: list[str]) -> set[str]:
    """The explicit conclusions this peer's derived conclusions name as premises."""
    return {
        explicit[index]
        for level in ("deductive", "inductive")
        for item in spec.get(level, [])
        if isinstance(item, dict)
        for index in item.get("premises", [])
    }


def count_conclusions(peers: dict[str, Any]) -> int:
    total = 0
    for observer in peers.values():
        for observed_id in peers:
            total += len(list(observer.conclusions_of(observed_id).list(size=100)))
    return total


def drain(honcho: Honcho, what: str) -> None:
    """Block until the deriver queue is empty, or fail loudly.

    Silence is the dangerous outcome here. A queue that never drains and a queue
    that never had anything queued look identical from the outside, so the timeout
    reports the last status it saw rather than just giving up.
    """
    deadline = time.monotonic() + DRAIN_TIMEOUT_SECONDS
    last: Any = None
    while time.monotonic() < deadline:
        last = honcho.queue_status()
        outstanding = last.pending_work_units + last.in_progress_work_units
        if outstanding == 0 and last.completed_work_units > 0:
            log(f"drained after {what} ({last.completed_work_units} work units)")
            return
        time.sleep(DRAIN_POLL_SECONDS)

    raise SeedError(
        f"queue did not drain after {what} within {DRAIN_TIMEOUT_SECONDS:.0f}s. "
        f"Last status: {last}. "
        "If completed_work_units is 0, nothing was ever enqueued - check that the "
        "deriver is running and that the provider base URL resolves."
    )


def main() -> int:
    phase = sys.argv[1] if len(sys.argv) > 1 else "seed"
    if phase not in ("seed", "verify"):
        raise SeedError(f"unknown phase {phase!r} (expected 'seed' or 'verify')")

    fixture = json.loads(FIXTURE.read_text())
    base_url = os.environ.get("SANDBOX_BASE_URL", "http://127.0.0.1:18000")
    workspace_id = fixture["workspace"]

    honcho = Honcho(base_url=base_url, workspace_id=workspace_id, api_key="sandbox")
    peers = {spec["id"]: honcho.peer(spec["id"]) for spec in fixture["peers"]}
    session = honcho.session(fixture["session"])

    if phase == "verify":
        verify(honcho, session, peers, fixture)
        log("verify complete")
        return 0

    log(f"seeding workspace {workspace_id!r} at {base_url}")

    session.add_peers(
        [
            (
                peers[spec["id"]],
                SessionPeerConfig(
                    observe_me=spec["observe_me"],
                    observe_others=spec["observe_others"],
                ),
            )
            for spec in fixture["peers"]
        ]
    )
    log(f"created {len(peers)} peers with explicit observation topology")

    session.add_messages(
        [
            MessageCreateParams(peer_id=msg["peer"], content=msg["content"])
            for msg in fixture["messages"]
        ]
    )
    log(f"posted {len(fixture['messages'])} messages")
    drain(honcho, "messages")

    for dream in fixture.get("dreams", []):
        honcho.schedule_dream(
            observer=dream["observer"],
            observed=dream["observed"],
            session=session,
        )
        log(f"scheduled dream: {dream['observer']} -> {dream['observed']}")
    if fixture.get("dreams"):
        drain(honcho, "dream")

    # The one check that proves derivation happened, made here rather than in verify
    # because seeded conclusions land afterwards and would satisfy it on their own.
    derived = count_conclusions(peers)
    if derived == 0:
        raise SeedError(
            "no conclusions were derived. The queue drained, so the deriver ran and "
            "produced nothing - check the provider wiring "
            "(LLM_OPENAI_BASE_URL / EMBEDDING_MODEL_CONFIG__OVERRIDES__BASE_URL) "
            "and the deriver logs."
        )
    log(f"verified {derived} derived conclusions present")
    log("seed complete")
    return 0


def verify(
    honcho: Honcho,
    session: Any,
    peers: dict[str, Any],
    fixture: dict[str, Any],
) -> None:
    """Assert the fixture round-tripped. Raises SeedError on any mismatch."""
    stored = list(session.messages(size=100))
    if len(stored) != len(fixture["messages"]):
        raise SeedError(
            f"expected {len(fixture['messages'])} messages, found {len(stored)}"
        )

    expected_authors = sorted(msg["peer"] for msg in fixture["messages"])
    actual_authors = sorted(msg.peer_id for msg in stored)
    if actual_authors != expected_authors:
        raise SeedError(
            f"messages landed on the wrong peers: expected {expected_authors}, "
            f"got {actual_authors}"
        )
    log(f"verified {len(stored)} messages on the expected peers")

    # Topology is the thing that silently breaks, so it is checked against the
    # server's view rather than assumed from the create call.
    for spec in fixture["peers"]:
        config = session.get_peer_configuration(spec["id"])
        if (config.observe_me, config.observe_others) != (
            spec["observe_me"],
            spec["observe_others"],
        ):
            raise SeedError(
                f"peer {spec['id']!r} topology mismatch: expected "
                f"observe_me={spec['observe_me']} "
                f"observe_others={spec['observe_others']}, got {config}"
            )
    log("verified observation topology")

    # Derived conclusions get presence only. Under the mock provider their text is
    # synthetic and says nothing about the messages, and the level mix is
    # explicit-only because the Dreamer specialists write via tool calls the mock
    # never emits. Asserting on either would pass in real mode and fail in mock mode.
    total = 0
    for observer_id, observer in peers.items():
        for observed_id in peers:
            found = len(list(observer.conclusions_of(observed_id).list(size=100)))
            if found:
                log(f"  {observer_id} -> {observed_id}: {found} conclusions")
            total += found

    if total == 0:
        raise SeedError(
            "no conclusions at all. Both derivation and injection produced nothing - "
            "check the provider wiring "
            "(LLM_OPENAI_BASE_URL / EMBEDDING_MODEL_CONFIG__OVERRIDES__BASE_URL) "
            "and the deriver logs."
        )
    log(f"verified {total} conclusions present")

    verify_seeded(peers, fixture, total)


def verify_seeded(
    peers: dict[str, Any],
    fixture: dict[str, Any],
    total: int,
) -> None:
    """Assert the seeded conclusions landed exactly, level by level.

    Seeded text is committed, so unlike derived conclusions it is asserted by content
    in both modes - which is the whole reason the fixture carries these keys.
    """
    planned = seeded_conclusions(fixture)
    if not planned:
        log("fixture declares no conclusions to seed")
        return

    seeded_total = 0
    for (observer_id, observed_id), by_level in planned.items():
        stored = list(peers[observer_id].conclusions_of(observed_id).list(size=100))
        for level, expected in by_level.items():
            if not expected:
                continue
            actual = {c.content for c in stored if c.level == level}
            missing = [content for content in expected if content not in actual]
            if missing:
                raise SeedError(
                    f"{observer_id} -> {observed_id}: {len(missing)} seeded {level} "
                    f"conclusion(s) missing, first is {missing[0]!r}. Honcho collapses "
                    "conclusions whose content matches something already stored, so "
                    "check for near-duplicate text in the fixture."
                )
            seeded_total += len(expected)
        summary = " ".join(f"{level}={len(by_level[level])}" for level in LEVELS)
        log(f"verified seeded {observer_id} -> {observed_id}: {summary}")

    # Derived and seeded conclusions must both be present. The seed phase already
    # proved derivation ran against an un-injected database; this catches the reverse
    # error of an injection that somehow replaced the derived rows.
    if total <= seeded_total:
        raise SeedError(
            f"found {total} conclusions but {seeded_total} were seeded, leaving none "
            "derived. Injection should add to the deriver's output, not replace it."
        )
    log(
        f"verified {total - seeded_total} derived conclusions alongside {seeded_total} seeded"
    )

    # Premise text is what the working representation renders for a derived
    # conclusion, and it is the only part of the reasoning tree an API client can
    # see - schemas.Conclusion does not expose source_ids.
    by_peer = {spec["id"]: spec for spec in fixture["peers"]}
    for (observer_id, observed_id), by_level in planned.items():
        premise_texts = cited_premise_texts(by_peer[observed_id], by_level["explicit"])
        if not premise_texts:
            continue
        rendered = peers[observer_id].representation(
            target=observed_id, max_conclusions=100
        )
        absent = [text for text in premise_texts if text not in rendered]
        if absent:
            raise SeedError(
                f"{observer_id} -> {observed_id}: premise text missing from the "
                f"representation, first is {absent[0]!r}. Premises render only for "
                "deductive conclusions and sources only for inductive ones, so a "
                "premise stored under the wrong metadata key renders as nothing."
            )
        log(
            f"verified premise text renders in {observer_id}'s representation of {observed_id}"
        )


if __name__ == "__main__":
    try:
        sys.exit(main())
    except SeedError as exc:
        print(f"[seed] FAILED: {exc}", file=sys.stderr)
        sys.exit(1)