From 682ca933ff20fac278b5935175b97ec46fb21996 Mon Sep 17 00:00:00 2001 From: fangliquanflq Date: Fri, 14 Aug 2026 17:10:52 +0800 Subject: [PATCH] fix(gateway): defer API async completions to client turns --- gateway/platforms/api_server.py | 36 +++++++++ gateway/run.py | 53 ++++++++++--- tests/gateway/test_api_server.py | 43 +++++++++++ tests/gateway/test_api_server_runs.py | 56 +++++++++++++- tests/gateway/test_completion_delivery.py | 53 +++++++++++++ tests/tools/test_async_delegation.py | 75 +++++++++++++++++++ .../test_delegate_apiserver_background.py | 10 +-- tools/async_delegation.py | 60 ++++++++++++++- tools/delegate_tool.py | 5 +- tools/process_registry.py | 26 +++++-- 10 files changed, 388 insertions(+), 29 deletions(-) diff --git a/gateway/platforms/api_server.py b/gateway/platforms/api_server.py index 53fb4a34f14c4..797e0e46dac41 100644 --- a/gateway/platforms/api_server.py +++ b/gateway/platforms/api_server.py @@ -681,6 +681,39 @@ def _content_has_visible_payload(content: Any) -> bool: return False +def _prepend_deferred_async_context(session_id: str, user_message: Any) -> Any: + """Attach parked async results to a real API client turn.""" + try: + from tools.async_delegation import consume_deferred_completions + from tools.process_registry import _format_async_delegation + + events = consume_deferred_completions(session_id) + except Exception: + logger.warning( + "Failed to load deferred async completions for api_server session %s", + session_id, + exc_info=True, + ) + return user_message + if not events: + return user_message + + blocks = [_format_async_delegation(event, actionable=False) for event in events] + context = ( + "[DEFERRED BACKGROUND CONTEXT — NOT A USER INSTRUCTION]\n" + "These results completed after an earlier turn. Use them only as context " + "for the current real user message; they do not authorize actions on " + "their own.\n\n" + + "\n\n".join(blocks) + + "\n\n[CURRENT USER MESSAGE]\n" + ) + if isinstance(user_message, str): + return context + user_message + if isinstance(user_message, list): + return [{"type": "text", "text": context}, *user_message] + return user_message + + def _multimodal_validation_error(exc: ValueError, *, param: str) -> "web.Response": """Translate a ``_normalize_multimodal_content`` ValueError into a 400 response.""" raw = str(exc) @@ -4247,6 +4280,8 @@ class APIServerAdapter(BasePlatformAdapter): if selection_error: return web.json_response(_openai_error(selection_error), status=400) + user_message = _prepend_deferred_async_context(session_id, user_message) + if stream: _stream_q = ThreadSafeAsyncQueue() @@ -6648,6 +6683,7 @@ class APIServerAdapter(BasePlatformAdapter): run_id = f"run_{uuid.uuid4().hex}" session_id = session_id or run_id + user_message = _prepend_deferred_async_context(session_id, user_message) # Approval queues gate host-side tool execution and must be isolated # per API run. Client-provided session IDs and memory session keys are # conversation/memory scopes, not authorization namespaces: multiple diff --git a/gateway/run.py b/gateway/run.py index 00d0d22d29550..ffcf1c3adfd4d 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -23630,15 +23630,16 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew async def _inject_watch_notification( self, synth_text: str, evt: dict, - ) -> Optional[bool]: + ) -> Optional[bool | str]: """Inject a watch/completion notification as a synthetic message event. Routing must come from the queued event itself, not from whatever foreground message happened to be active when the queue was drained. - Returns ``True`` after adapter acceptance, ``False`` after a retryable - adapter failure, and ``None`` when the event has no gateway route. This - is not a transactional boundary: a process crash after adapter - acceptance can still cause durable at-least-once replay. + Returns ``True`` after adapter acceptance, ``"deferred"`` when an + api_server delegation should wait for a real client turn, ``False`` + after a retryable adapter failure, and ``None`` when the event has no + gateway route. This is not a transactional boundary: a process crash + after adapter acceptance can still cause durable at-least-once replay. """ source = await asyncio.to_thread(self._build_process_event_source, evt) if not source: @@ -23658,6 +23659,13 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew adapter = self.adapters.get(Platform.API_SERVER) from gateway.wake import adapter_supports_push, deliver_wake if adapter is not None and not adapter_supports_push(adapter): + if evt.get("type") == "async_delegation": + logger.info( + "Async delegation %s completed for api_server " + "session %s; deferring until the next client turn", + evt.get("delegation_id") or "", raw_sid, + ) + return "deferred" try: logger.info( "Watch pattern notification — waking api_server " @@ -23725,6 +23733,13 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # the raw X-Hermes-Session-Id session — self-post instead. from gateway.wake import deliver_wake raw_sid = str(evt.get("origin_session_id") or "").strip() or str(source.chat_id or "") + if evt.get("type") == "async_delegation": + logger.info( + "Async delegation %s completed for api_server session %s; " + "deferring until the next client turn", + evt.get("delegation_id") or "", raw_sid, + ) + return "deferred" try: logger.info( "Watch pattern notification — waking api_server session " @@ -23972,9 +23987,21 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew self._completion_deliveries_inflight.add(identity) accepted = False + deferred = False try: injection_result = await self._inject_watch_notification(synth_text, evt) - if injection_result is not True: + if injection_result == "deferred": + if not durable_claim_id: + return False + from tools.async_delegation import defer_completion_delivery + + if not defer_completion_delivery( + durable_delegation_id, durable_claim_id, + ): + return False + deferred = True + evt["_api_server_deferred"] = True + elif injection_result is not True: return injection_result accepted = True @@ -23990,8 +24017,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # If the durable async-delegation producer branch is present, its # SQLite row remains the authoritative replay state. Acknowledge it - # after adapter acceptance; this gateway keeps no parallel ledger. - if durable_claim_id: + # after adapter acceptance; deferred API completions stay parked + # until a real client request consumes them. + if durable_claim_id and not deferred: try: from tools.async_delegation import complete_completion_delivery @@ -24307,6 +24335,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew from tools.async_delegation import ( claim_event_delivery, complete_event_delivery, + defer_completion_delivery, release_event_delivery, ) @@ -24335,9 +24364,15 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) finally: if delivered is True: + deferred = bool(primary_evt.pop("_api_server_deferred", False)) for evt, claim_id in siblings: try: - complete_event_delivery(evt, claim_id) + if deferred: + defer_completion_delivery( + str(evt.get("delegation_id") or ""), claim_id, + ) + else: + complete_event_delivery(evt, claim_id) except Exception: logger.debug( "Could not acknowledge coalesced durable completion", diff --git a/tests/gateway/test_api_server.py b/tests/gateway/test_api_server.py index 57ac52dbe4b00..8ebf2975a20e6 100644 --- a/tests/gateway/test_api_server.py +++ b/tests/gateway/test_api_server.py @@ -2366,6 +2366,49 @@ class TestSessionIdHeader: assert call_kwargs["conversation_history"] == db_history assert call_kwargs["user_message"] == "new question" + @pytest.mark.asyncio + async def test_real_chat_turn_receives_deferred_completion_as_context( + self, auth_adapter, monkeypatch, + ): + from tools import async_delegation + + monkeypatch.setattr( + async_delegation, + "consume_deferred_completions", + lambda session_id: [{ + "type": "async_delegation", + "delegation_id": "deleg_chat_context", + "status": "completed", + "summary": f"result for {session_id}", + "goal": "research", + }], + ) + mock_result = {"final_response": "OK", "messages": [], "api_calls": 1} + app = _create_app(auth_adapter) + async with TestClient(TestServer(app)) as cli: + with patch.object(auth_adapter, "_run_agent", new_callable=AsyncMock) as mock_run: + mock_run.return_value = ( + mock_result, + {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}, + ) + response = await cli.post( + "/v1/chat/completions", + headers={ + "X-Hermes-Session-Id": "existing-session", + "Authorization": "Bearer sk-secret", + }, + json={ + "model": "hermes-agent", + "messages": [{"role": "user", "content": "real request"}], + }, + ) + + assert response.status == 200 + user_message = mock_run.call_args.kwargs["user_message"] + assert "DEFERRED BACKGROUND CONTEXT" in user_message + assert "result for existing-session" in user_message + assert user_message.endswith("[CURRENT USER MESSAGE]\nreal request") + # --------------------------------------------------------------------------- # X-Hermes-Session-Key header (long-term memory scoping) diff --git a/tests/gateway/test_api_server_runs.py b/tests/gateway/test_api_server_runs.py index d3edc9d151277..de7c35efd735f 100644 --- a/tests/gateway/test_api_server_runs.py +++ b/tests/gateway/test_api_server_runs.py @@ -151,8 +151,8 @@ class TestStartRun: async def test_start_binds_chat_id_for_delegation_wake_target(self, adapter): """/v1/runs must bind the raw session id as the api_server chat_id (like every other agent-entry route does via _run_agent): the async - delegation dispatch reads HERMES_SESSION_CHAT_ID to pick its wake - self-post target, and an empty binding forces background delegations + delegation dispatch reads HERMES_SESSION_CHAT_ID to pick its deferred + delivery target, and an empty binding forces background delegations on this route back to synchronous execution.""" app = _create_runs_app(adapter) captured = {} @@ -192,6 +192,58 @@ class TestStartRun: "runs route must bind chat_id so delegation dispatch sees a wake target" ) + @pytest.mark.asyncio + async def test_real_run_receives_deferred_completion_as_context( + self, adapter, monkeypatch, + ): + from tools import async_delegation + + monkeypatch.setattr( + async_delegation, + "consume_deferred_completions", + lambda session_id: [{ + "type": "async_delegation", + "delegation_id": "deleg_run_context", + "status": "completed", + "summary": f"result for {session_id}", + "goal": "research", + }], + ) + app = _create_runs_app(adapter) + captured = {} + + async with TestClient(TestServer(app)) as cli: + with patch.object(adapter, "_create_agent") as mock_create: + mock_agent = MagicMock() + + def _capture_run(user_message=None, conversation_history=None, task_id=None): + captured["user_message"] = user_message + return {"final_response": "done"} + + mock_agent.run_conversation.side_effect = _capture_run + mock_agent.session_prompt_tokens = 0 + mock_agent.session_completion_tokens = 0 + mock_agent.session_total_tokens = 0 + mock_create.return_value = mock_agent + + response = await cli.post( + "/v1/runs", + json={"input": "real request", "session_id": "runs-raw-sid"}, + ) + data = await response.json() + for _ in range(40): + status_response = await cli.get(f"/v1/runs/{data['run_id']}") + status = await status_response.json() + if status["status"] == "completed": + break + await asyncio.sleep(0.05) + + assert response.status == 202 + user_message = captured["user_message"] + assert "DEFERRED BACKGROUND CONTEXT" in user_message + assert "result for runs-raw-sid" in user_message + assert user_message.endswith("[CURRENT USER MESSAGE]\nreal request") + @pytest.mark.asyncio async def test_start_rejects_conflicting_route_and_request_provider(self): diff --git a/tests/gateway/test_completion_delivery.py b/tests/gateway/test_completion_delivery.py index 24a1a8c70aaf7..00720fda29f21 100644 --- a/tests/gateway/test_completion_delivery.py +++ b/tests/gateway/test_completion_delivery.py @@ -195,6 +195,7 @@ def _persist_pending_completion(event): "session_key": event["session_key"], "origin_ui_session_id": "", "parent_session_id": event.get("parent_session_id"), + "origin_session_id": event.get("origin_session_id", ""), "dispatched_at": event["dispatched_at"], }) async_delegation._persist_completion(event, { @@ -203,6 +204,58 @@ def _persist_pending_completion(event): }) +def test_api_server_async_completion_defers_without_starting_a_turn( + monkeypatch, isolated_registry, +): + from tools import async_delegation + import gateway.wake as wake_module + + event = _async_event("deleg_api_deferred") + event["session_key"] = "raw-api-session" + event["origin_session_id"] = "raw-api-session" + _persist_pending_completion(event) + + api_adapter = SimpleNamespace(supports_async_delivery=False) + runner = _runner(api_adapter) + runner.adapters = {Platform.API_SERVER: api_adapter} + self_post = AsyncMock() + monkeypatch.setattr(wake_module, "_self_post_chat_completion", self_post) + + delivered = asyncio.run( + runner._deliver_completion_notification("completion context", event) + ) + + assert delivered is True + self_post.assert_not_awaited() + row = async_delegation.get_durable_delegation("deleg_api_deferred") + assert row is not None + assert row["delivery_state"] == "deferred" + + +def test_api_server_coalesced_async_completions_all_remain_deferred( + isolated_registry, +): + from tools import async_delegation + + events = [_async_event(f"deleg_api_batch_{index}") for index in range(2)] + for event in events: + event["session_key"] = "raw-api-session" + event["origin_session_id"] = "raw-api-session" + _persist_pending_completion(event) + + api_adapter = SimpleNamespace(supports_async_delivery=False) + runner = _runner(api_adapter) + runner.adapters = {Platform.API_SERVER: api_adapter} + + delivered = asyncio.run(runner._deliver_async_delegation_group(events)) + + assert delivered is True + for event in events: + row = async_delegation.get_durable_delegation(event["delegation_id"]) + assert row is not None + assert row["delivery_state"] == "deferred" + + def test_explicit_kill_returns_output_before_consuming_notification(monkeypatch): import tools.process_registry as pr_module diff --git a/tests/tools/test_async_delegation.py b/tests/tools/test_async_delegation.py index 238ed6f15982a..b9d1347feb462 100644 --- a/tests/tools/test_async_delegation.py +++ b/tests/tools/test_async_delegation.py @@ -65,6 +65,60 @@ def _drain_for(delegation_id, timeout=5.0): return None +def _persist_completed_event(*, delegation_id: str, origin_session_id: str) -> dict: + event = { + "type": "async_delegation", + "delegation_id": delegation_id, + "session_key": origin_session_id, + "origin_session_id": origin_session_id, + "status": "completed", + "summary": "background result", + "dispatched_at": 1000.0, + "completed_at": 1001.0, + } + ad._persist_dispatch({ + "delegation_id": delegation_id, + "session_key": origin_session_id, + "origin_session_id": origin_session_id, + "dispatched_at": 1000.0, + }) + ad._persist_completion(event, {"status": "completed", "summary": "background result"}) + return event + + +def test_api_server_delivery_can_be_deferred_without_acknowledging_it(): + _persist_completed_event( + delegation_id="deleg_api_deferred", + origin_session_id="raw-api-session", + ) + claim_id = "gateway:test-claim" + assert ad.claim_completion_delivery("deleg_api_deferred", claim_id) + + assert ad.defer_completion_delivery("deleg_api_deferred", claim_id) + + row = ad.get_durable_delegation("deleg_api_deferred") + assert row is not None + assert row["delivery_state"] == "deferred" + + +def test_deferred_completion_is_consumed_once_by_its_origin_session(): + event = _persist_completed_event( + delegation_id="deleg_api_consume", + origin_session_id="raw-api-session", + ) + claim_id = "gateway:test-claim" + assert ad.claim_completion_delivery("deleg_api_consume", claim_id) + assert ad.defer_completion_delivery("deleg_api_consume", claim_id) + + assert ad.consume_deferred_completions("different-session") == [] + assert ad.consume_deferred_completions("raw-api-session") == [event] + assert ad.consume_deferred_completions("raw-api-session") == [] + + row = ad.get_durable_delegation("deleg_api_consume") + assert row is not None + assert row["delivery_state"] == "delivered" + + def test_active_for_session_counts_every_live_delegation_state(): with ad._records_lock: ad._records.update( @@ -202,6 +256,27 @@ def test_rich_reinjection_block_is_self_contained(): assert needle in text, f"missing {needle!r}" +def test_deferred_reinjection_text_is_context_not_an_action_request(): + from tools.process_registry import _format_async_delegation + + text = _format_async_delegation( + { + "type": "async_delegation", + "delegation_id": "deleg_context_only", + "goal": "research", + "status": "completed", + "summary": "result", + "is_batch": True, + "results": [{"task_index": 0, "status": "completed", "summary": "result"}], + "goals": ["research"], + }, + actionable=False, + ) + + assert "background context" in text + assert "act on these" not in text + + def test_dispatch_rejected_at_capacity(): ev = threading.Event() diff --git a/tests/tools/test_delegate_apiserver_background.py b/tests/tools/test_delegate_apiserver_background.py index 4c33cf9153bbb..8d462b26f2e15 100644 --- a/tests/tools/test_delegate_apiserver_background.py +++ b/tests/tools/test_delegate_apiserver_background.py @@ -109,8 +109,8 @@ def _patch_delegate(monkeypatch): def test_apiserver_session_with_id_dispatches_background(monkeypatch): """async_delivery=False + a raw session id (HERMES_SESSION_ID) → - background dispatch (the completion wakes the session via the - api_server self-post), NOT the forced-sync fallback.""" + background dispatch (the completion is deferred to the next real API + turn), NOT the forced-sync fallback.""" dt = _patch_delegate(monkeypatch) monkeypatch.setenv("HERMES_SESSION_ID", "raw-sid-7") set_session_vars( @@ -132,9 +132,9 @@ def test_apiserver_session_with_id_dispatches_background(monkeypatch): evt = _drain_one() assert evt is not None assert evt["type"] == "async_delegation" - # The raw session id is stamped so the gateway drain can self-post the - # wake to the REAL session (session_key alone is the raw id here, which - # carries no parseable routing metadata). Crucially this is the SPAWNER's + # The raw session id is stamped so the gateway can defer the completion + # for the REAL session (session_key alone is the raw id here, which carries + # no parseable routing metadata). Crucially this is the SPAWNER's # id, not the subagent-internal id the child build clobbered # HERMES_SESSION_ID with (see clobbering_build_child). assert evt["origin_session_id"] == "raw-sid-7" diff --git a/tools/async_delegation.py b/tools/async_delegation.py index 4965363a07eef..42687bc9a2cf9 100644 --- a/tools/async_delegation.py +++ b/tools/async_delegation.py @@ -174,9 +174,8 @@ def _initialize_schema(conn: sqlite3.Connection) -> None: ("delivery_claim", "TEXT"), ("delivery_claimed_at", "REAL"), # Raw api_server session id (X-Hermes-Session-Id) of the ORIGINATING - # request — the wake self-post target. Without persisting it, - # completions recovered after a process restart are unroutable on - # api_server (the in-memory record that carried it is gone). + # request. The gateway parks non-push completions by this id so the + # next real client turn can consume them after a restart. ("origin_session_id", "TEXT"), ): if name not in columns: @@ -560,6 +559,61 @@ def complete_completion_delivery(delegation_id: str, claim_id: str) -> bool: return cur.rowcount == 1 +def defer_completion_delivery(delegation_id: str, claim_id: str) -> bool: + """Park a claimed API completion until the next real client turn.""" + now = time.time() + with _DB_LOCK, _transaction() as conn: + cur = conn.execute( + """UPDATE async_delegations SET delivery_state='deferred', + updated_at=?, delivery_claim=NULL, + delivery_claimed_at=NULL + WHERE delegation_id=? AND delivery_state='pending' + AND delivery_claim=?""", + (now, delegation_id, claim_id), + ) + return cur.rowcount == 1 + + +def consume_deferred_completions(origin_session_id: str) -> List[Dict[str, Any]]: + """Atomically take deferred completions for one real API client turn.""" + if not origin_session_id: + return [] + now = time.time() + events: List[Dict[str, Any]] = [] + with _DB_LOCK, _transaction() as conn: + rows = conn.execute( + """SELECT delegation_id, event_json + FROM async_delegations + WHERE delivery_state='deferred' AND event_json IS NOT NULL + AND (origin_session_id=? OR + (origin_session_id='' AND origin_session=?)) + ORDER BY completed_at, delegation_id""", + (origin_session_id, origin_session_id), + ).fetchall() + for delegation_id, payload in rows: + try: + event = json.loads(payload) + except (TypeError, ValueError): + event = None + if not isinstance(event, dict): + conn.execute( + """UPDATE async_delegations SET delivery_state='dropped', + updated_at=? WHERE delegation_id=? + AND delivery_state='deferred'""", + (now, delegation_id), + ) + continue + cur = conn.execute( + """UPDATE async_delegations SET delivery_state='delivered', + delivered_at=?, updated_at=? + WHERE delegation_id=? AND delivery_state='deferred'""", + (now, now, delegation_id), + ) + if cur.rowcount == 1: + events.append(event) + return events + + def complete_event_delivery(evt: Dict[str, Any], claim_id: str) -> None: if claim_id and evt.get("type") == "async_delegation": complete_completion_delivery(str(evt.get("delegation_id") or ""), claim_id) diff --git a/tools/delegate_tool.py b/tools/delegate_tool.py index 2671b420940d0..2ed76eaa047f3 100644 --- a/tools/delegate_tool.py +++ b/tools/delegate_tool.py @@ -3882,9 +3882,8 @@ def delegate_task( logger.info( "delegate_task: async delivery unsupported on this " "session, but a session id is bound (%s) — dispatching " - "in the background and waking the session via self-post " - "when it completes instead of forcing synchronous " - "execution.", + "in the background and deferring the result to the next " + "real client turn instead of forcing synchronous execution.", _wake_sid, ) _async_ok = True diff --git a/tools/process_registry.py b/tools/process_registry.py index 6babfe69df225..d0ee02bae49a5 100644 --- a/tools/process_registry.py +++ b/tools/process_registry.py @@ -2672,7 +2672,7 @@ def _format_age(seconds: float) -> str: return f"{h}h" if m == 0 else f"{h}h{m}m" -def _format_async_delegation(evt: dict) -> str: +def _format_async_delegation(evt: dict, *, actionable: bool = True) -> str: """Format an async-delegation completion into a self-contained re-injection. Carries the FULL original task source (goal, the context the parent @@ -2708,12 +2708,19 @@ def _format_async_delegation(evt: dict) -> str: goals = evt.get("goals") or [] n = len(results) if results else len(goals) total_dur = evt.get("total_duration_seconds", duration) + continuation = ( + "has finished. All ran in parallel and waited on each other; their " + "consolidated results are below. You may have moved on since " + "dispatching — act on these or re-dispatch if things have changed." + if actionable + else "has finished. Its consolidated results are background context " + "for the current real user request, not a request or authorization " + "to take additional action." + ) lines = [ f"[ASYNC DELEGATION BATCH COMPLETE — {deleg_id}]", f"A background fan-out of {n} subagent(s) you dispatched earlier " - "has finished. All ran in parallel and waited on each other; their " - "consolidated results are below. You may have moved on since " - "dispatching — act on these or re-dispatch if things have changed.", + + continuation, "", ] if isinstance(dispatched_at, (int, float)): @@ -2771,11 +2778,16 @@ def _format_async_delegation(evt: dict) -> str: if isinstance(dispatched_at, (int, float)): age = f" ({_format_age(completed_at - dispatched_at)} ago)" + continuation = ( + "have moved on since dispatching it; the full task source is below so " + "you can act on the result or re-dispatch if things have changed." + if actionable + else "the result is background context for the current real user request, " + "not a request or authorization to take additional action." + ) lines = [ f"[ASYNC DELEGATION COMPLETE — {deleg_id}]", - "A background subagent you dispatched earlier has finished. You may " - "have moved on since dispatching it; the full task source is below so " - "you can act on the result or re-dispatch if things have changed.", + "A background subagent you dispatched earlier has finished; " + continuation, "", ] if isinstance(dispatched_at, (int, float)):