from __future__ import annotations import json import logging import weakref from collections.abc import AsyncIterator from typing import Any, cast from openai import BadRequestError, LengthFinishReasonError from pydantic import BaseModel, ValidationError from src.exceptions import ValidationException from src.llm.backend import CompletionResult, StreamChunk, ToolCallResult from src.llm.request_builder import ( apply_sdk_passthroughs, request_timeout_from_extra_params, ) from src.llm.structured_output import ( StructuredOutputError, empty_structured_output, repair_response_model_json, validate_structured_output, ) logger = logging.getLogger(__name__) # The point of this being a WeakKeyDictionary, as opposed to a regular dict, is that it # does not hold a reference to the keyed BaseModel so that when a dynamically created # type is no longer referenced it becomes eligible for garbage collection. This avoids a # memory leak. _json_object_instruction_cache: weakref.WeakKeyDictionary[type[BaseModel], str] = ( weakref.WeakKeyDictionary() ) def _json_object_instruction(response_format: type[BaseModel]) -> str: """Schema-injection instruction for json_object mode. The JSON schema is static per response_format class, so cache the serialized instruction — the deriver would otherwise re-walk the schema + re-serialize it every call. """ cached = _json_object_instruction_cache.get(response_format) if cached is not None: return cached # Some OpenAI-compatible providers enforce this JSON-object precondition with # a case-sensitive substring check, so include lowercase "json" explicitly. instruction = ( "You must respond with a single JSON object (json) that conforms " "exactly to the following JSON schema. Do not include any text, " "markdown, or code fences outside the JSON object.\n\nJSON schema:\n" f"{json.dumps(response_format.model_json_schema())}" ) _json_object_instruction_cache[response_format] = instruction return instruction def _uses_max_completion_tokens(model: str) -> bool: """OpenAI reasoning models (gpt-5 family + o-series) require ``max_completion_tokens`` instead of the classic ``max_tokens`` parameter. Matches: gpt-5, gpt-5-anything, gpt-5.anything (incl. gpt-5.4, gpt-5.4-mini), o1*, o3*, o4*. Anything else (gpt-4.x, gpt-4o, chat models on proxies) stays on ``max_tokens``. """ m = model.lower() if m == "gpt-5" or m.startswith("gpt-5-") or m.startswith("gpt-5."): return True for prefix in ("o1", "o3", "o4"): if m == prefix or m.startswith(prefix + "-"): return True return False def extract_openai_reasoning_content(response: Any) -> str | None: try: message = response.choices[0].message if hasattr(message, "reasoning_details") and message.reasoning_details: reasoning_parts: list[str] = [] for detail in message.reasoning_details: detail_content = getattr(detail, "content", None) if isinstance(detail_content, str) and detail_content: reasoning_parts.append(detail_content) elif isinstance(detail, dict): detail_dict = cast(dict[str, Any], detail) dict_content = detail_dict.get("content") if isinstance(dict_content, str) and dict_content: reasoning_parts.append(dict_content) if reasoning_parts: return "\n".join(reasoning_parts) if hasattr(message, "reasoning_content") and message.reasoning_content: return message.reasoning_content except (AttributeError, IndexError, TypeError): return None return None def extract_openai_reasoning_details(response: Any) -> list[dict[str, Any]]: try: message = response.choices[0].message if hasattr(message, "reasoning_details") and message.reasoning_details: details: list[dict[str, Any]] = [] for detail in message.reasoning_details: if hasattr(detail, "model_dump"): dumped = detail.model_dump() if isinstance(dumped, dict): details.append(cast(dict[str, Any], dumped)) elif isinstance(detail, dict): details.append(cast(dict[str, Any], detail)) else: detail_content = getattr(detail, "content", None) if isinstance(detail_content, str) and detail_content: details.append({"content": detail_content}) return details except (AttributeError, IndexError, TypeError): return [] return [] def extract_openai_cache_tokens(usage: Any) -> tuple[int, int]: if not usage: return 0, 0 cache_read = 0 if hasattr(usage, "prompt_tokens_details") and usage.prompt_tokens_details: details = usage.prompt_tokens_details if hasattr(details, "cached_tokens") and details.cached_tokens: cache_read = details.cached_tokens if cache_read == 0: if hasattr(usage, "cache_read_input_tokens") and usage.cache_read_input_tokens: cache_read = usage.cache_read_input_tokens elif hasattr(usage, "cached_tokens") and usage.cached_tokens: cache_read = usage.cached_tokens cache_creation = 0 if ( hasattr(usage, "cache_creation_input_tokens") and usage.cache_creation_input_tokens ): cache_creation = usage.cache_creation_input_tokens return cache_creation, cache_read class OpenAIBackend: """Provider backend wrapping AsyncOpenAI.""" def __init__(self, client: Any) -> None: self._client: Any = client async def complete( self, *, model: str, messages: list[dict[str, Any]], max_tokens: int, temperature: float | None = None, stop: list[str] | None = None, tools: list[dict[str, Any]] | None = None, tool_choice: str | dict[str, Any] | None = None, response_format: type[BaseModel] | dict[str, Any] | None = None, thinking_budget_tokens: int | None = None, thinking_effort: str | None = None, max_output_tokens: int | None = None, extra_params: dict[str, Any] | None = None, ) -> CompletionResult: params = self._build_params( model=model, messages=messages, max_tokens=max_output_tokens or max_tokens, temperature=temperature, stop=stop, tools=tools, tool_choice=tool_choice, thinking_effort=thinking_effort, thinking_budget_tokens=thinking_budget_tokens, extra_params=extra_params, ) if isinstance(response_format, type): if self._structured_output_mode(extra_params) == "json_object": self._apply_json_object_mode(params, response_format) response = await self._client.chat.completions.create(**params) # A loose provider that returns nothing shouldn't crash the call. content = self._parse_or_repair_structured_content( response, response_format, model, empty_on_missing=True ) return self._normalize_response(response, content_override=content) if tools: # parse() refuses non-strict function tools, and our agent tool # schemas are deliberately non-strict (see _convert_tools), so # tool-loop iterations use create() with an explicit json_schema # response_format — same server-side schema enforcement, no # strict-tools requirement — mirroring the streaming path. params["response_format"] = self._json_schema_response_format( response_format ) response = await self._client.chat.completions.create(**params) # Tool-call turns carry no consumable content — the tool loop # ignores it — and parsing their empty text would raise. if getattr(response.choices[0].message, "tool_calls", None): return self._normalize_response(response) content = self._parse_or_repair_structured_content( response, response_format, model, empty_on_missing=False ) return self._normalize_response(response, content_override=content) params["response_format"] = response_format try: response = await self._client.chat.completions.parse(**params) except LengthFinishReasonError as exc: # Truncated output: repair the partial content directly. repair # handles empty/unrepairable JSON with its own model-aware fallback # (PromptRepresentation -> empty, others -> raise), which differs # from the parse-fallback terminal below, so it stays a direct call. truncated = exc.completion raw_content = truncated.choices[0].message.content or "" content = repair_response_model_json( raw_content, response_format, model, ) return self._normalize_response( truncated, content_override=content, ) except BadRequestError: # A 400 means the provider rejected the request shape — most # often it doesn't support OpenAI Structured Outputs (json_schema). # Retrying or re-requesting won't help (it rejects the same shape # again, the latency trap of #797), so return empty rather than # erroring existing flows. The warning is the signal to set # structured_output_mode=json_object. There is no response body to # account for, so token usage is legitimately zero here. logger.warning( "Structured output via json_schema rejected by model %s; " + "set structured_output_mode=json_object if the provider does " + "not support OpenAI Structured Outputs.", model, ) # empty_structured_output() validates {} against the model, which # itself raises if the model has required fields. Fall back to # empty string content rather than letting that escape the handler. try: fallback_content: Any = empty_structured_output(response_format) except ValidationError: fallback_content = "" return CompletionResult(content=fallback_content) parsed = response.choices[0].message.parsed if parsed is not None: return self._normalize_response( response, content_override=validate_structured_output( parsed, response_format ), ) # parse() returned no model: repair raw content, surface a refusal, # or raise so the retry/fallback chain engages on a junk response. content = self._parse_or_repair_structured_content( response, response_format, model, empty_on_missing=False ) return self._normalize_response(response, content_override=content) if response_format is not None: params["response_format"] = response_format if extra_params and extra_params.get("json_mode"): params["response_format"] = {"type": "json_object"} response = await self._client.chat.completions.create(**params) return self._normalize_response(response) async def stream( self, *, model: str, messages: list[dict[str, Any]], max_tokens: int, temperature: float | None = None, stop: list[str] | None = None, tools: list[dict[str, Any]] | None = None, tool_choice: str | dict[str, Any] | None = None, response_format: type[BaseModel] | dict[str, Any] | None = None, thinking_budget_tokens: int | None = None, thinking_effort: str | None = None, max_output_tokens: int | None = None, extra_params: dict[str, Any] | None = None, ) -> AsyncIterator[StreamChunk]: params = self._build_params( model=model, messages=messages, max_tokens=max_output_tokens or max_tokens, temperature=temperature, stop=stop, tools=tools, tool_choice=tool_choice, thinking_effort=thinking_effort, thinking_budget_tokens=thinking_budget_tokens, extra_params=extra_params, ) params["stream"] = True params["stream_options"] = {"include_usage": True} if isinstance(response_format, type): if self._structured_output_mode(extra_params) == "json_object": # Inject the schema into the prompt for providers without # json_schema support; repair happens downstream. self._apply_json_object_mode(params, response_format) else: # Streaming create() can't take a BaseModel like parse() does; # convert to a json_schema dict. params["response_format"] = self._json_schema_response_format( response_format ) elif response_format is not None: params["response_format"] = response_format elif extra_params and extra_params.get("json_mode"): params["response_format"] = {"type": "json_object"} response_stream = await self._client.chat.completions.create(**params) finish_reason: str | None = None usage_chunk_received = False async for chunk in response_stream: if chunk.choices and chunk.choices[0].delta.content: yield StreamChunk(content=chunk.choices[0].delta.content) if chunk.choices and chunk.choices[0].finish_reason: finish_reason = chunk.choices[0].finish_reason if hasattr(chunk, "usage") and chunk.usage: yield StreamChunk( is_done=True, finish_reason=finish_reason, output_tokens=chunk.usage.completion_tokens, ) usage_chunk_received = True if not usage_chunk_received and finish_reason: yield StreamChunk(is_done=True, finish_reason=finish_reason) def _build_params( self, *, model: str, messages: list[dict[str, Any]], max_tokens: int, temperature: float | None, stop: list[str] | None, tools: list[dict[str, Any]] | None, tool_choice: str | dict[str, Any] | None, thinking_effort: str | None, thinking_budget_tokens: int | None, extra_params: dict[str, Any] | None, ) -> dict[str, Any]: params: dict[str, Any] = { "model": model, "messages": messages, } if _uses_max_completion_tokens(model): params["max_completion_tokens"] = max_tokens if extra_params and extra_params.get("verbosity"): params["verbosity"] = extra_params["verbosity"] else: params["max_tokens"] = max_tokens if temperature is not None: params["temperature"] = temperature if thinking_effort: params["reasoning_effort"] = thinking_effort # Token-budget style thinking is not part of the native OpenAI API, but # OpenAI-compatible proxies (OpenRouter, etc.) accept a `reasoning` object # on the request body. Pass through via extra_body so it reaches those # backends. Operators on providers that need a different shape (e.g. # Anthropic-via-Vertex behind litellm wants `thinking`, not `reasoning`) # supply that shape via ModelConfig.provider_params.extra_body and unset # thinking_budget_tokens themselves — Honcho does not try to translate. if thinking_budget_tokens is not None and thinking_budget_tokens > 0: params.setdefault("extra_body", {}).setdefault("reasoning", {})[ "max_tokens" ] = thinking_budget_tokens if stop: params["stop"] = stop if tools: params["tools"] = self._convert_tools(tools) converted_tool_choice = self._convert_tool_choice(tool_choice) if converted_tool_choice is not None: params["tool_choice"] = converted_tool_choice if extra_params: for key in ( "top_p", "frequency_penalty", "presence_penalty", "seed", ): if key in extra_params: params[key] = extra_params[key] # Operator escape hatch: forward OpenAI SDK passthrough kwargs from # ModelConfig.provider_params. Shallow merge with operator-wins — # if the operator supplies `extra_body.reasoning`, it replaces any # value Honcho auto-injected above. apply_sdk_passthroughs(params, extra_params) timeout = request_timeout_from_extra_params(extra_params) if timeout is not None: params["timeout"] = timeout return params def _normalize_response( self, response: Any, *, content_override: Any | None = None, ) -> CompletionResult: usage = response.usage finish_reason = response.choices[0].finish_reason tool_calls: list[ToolCallResult] = [] message = response.choices[0].message if getattr(message, "tool_calls", None): for tool_call in message.tool_calls: tool_input: dict[str, Any] = {} if tool_call.function.arguments: try: tool_input = json.loads(tool_call.function.arguments) except (json.JSONDecodeError, TypeError) as exc: # Don't log the raw arguments payload — LLM-generated # tool calls can mirror user PII from the prompt into # their arguments, and this runs at WARN level. logger.warning( "Malformed tool arguments for %s (id=%s): %s", tool_call.function.name, tool_call.id, exc.__class__.__name__, ) tool_calls.append( ToolCallResult( id=tool_call.id, name=tool_call.function.name, input=tool_input, ) ) cache_creation, cache_read = extract_openai_cache_tokens(usage) return CompletionResult( content=content_override if content_override is not None else (message.content or ""), input_tokens=usage.prompt_tokens if usage else 0, output_tokens=usage.completion_tokens if usage else 0, cache_creation_input_tokens=cache_creation, cache_read_input_tokens=cache_read, finish_reason=finish_reason or "stop", tool_calls=tool_calls, thinking_content=extract_openai_reasoning_content(response), reasoning_details=extract_openai_reasoning_details(response), raw_response=response, ) @staticmethod def _json_schema_response_format( response_format: type[BaseModel], ) -> dict[str, Any]: """Build the response_format param for create() calls that can't use parse(): streaming, and requests carrying non-strict function tools.""" return { "type": "json_schema", "json_schema": { "name": response_format.__name__, "schema": response_format.model_json_schema(), }, } @staticmethod def _structured_output_mode(extra_params: dict[str, Any] | None) -> str | None: # Threaded in via extra_params (see build_config_extra_params). if not extra_params: return None return extra_params.get("structured_output_mode") def _apply_json_object_mode( self, params: dict[str, Any], response_format: type[BaseModel], ) -> None: """Configure params for json_object mode in place (shared by complete/stream). Injects the schema into the prompt and requests loose JSON, so providers without OpenAI Structured Outputs (json_schema) support still return JSON. """ params["messages"] = self._with_json_schema_instructions( params["messages"], response_format ) params["response_format"] = {"type": "json_object"} @staticmethod def _with_json_schema_instructions( messages: list[dict[str, Any]], response_format: type[BaseModel], ) -> list[dict[str, Any]]: """Add JSON-schema instructions to a copy of messages for json_object mode. The Anthropic backend has its own schema-into-prompt injection (``_append_text_to_last_message``); the two are intentionally kept separate since the providers want different placement and wording. """ instruction = _json_object_instruction(response_format) new_messages = [dict(message) for message in messages] first = new_messages[0] if new_messages else None # Only merge into a leading system message when its content is a plain # string; non-string content (e.g. a list of content parts) would be # corrupted by f-string coercion, so prepend a fresh system message. if ( first and first.get("role") == "system" and isinstance(first.get("content"), str) ): first["content"] = f"{first['content']}\n\n{instruction}".strip() else: new_messages.insert(0, {"role": "system", "content": instruction}) return new_messages @staticmethod def _parse_or_repair_structured_content( response: Any, response_format: type[BaseModel], model: str, *, empty_on_missing: bool, ) -> BaseModel | str: """Validate (or repair) the raw structured content of a response. Shared by the json_object path and the json_schema parse() fallbacks (truncation, parsed=None). On a contentless response with no refusal, ``empty_on_missing`` selects the terminal behavior: json_object returns a graceful empty so a loose provider can't crash the call, while json_schema raises so the retry/fallback chain engages on a junk response. """ message = response.choices[0].message raw_content = message.content or "" if raw_content: # Fast path: clean JSON validates directly. Only fall back to the # repair pipeline when validation fails — repair is comparatively # expensive and silently degrades malformed input to an empty model. try: return validate_structured_output(raw_content, response_format) except (StructuredOutputError, ValidationError): return repair_response_model_json(raw_content, response_format, model) refusal = getattr(message, "refusal", None) if refusal: return refusal if not empty_on_missing: raise ValidationException("No parsed content in structured response") # empty_structured_output() validates {} against the model, which itself # raises if the model has required fields. Fall back to empty string # content rather than letting that escape the handler. try: return empty_structured_output(response_format) except ValidationError: return "" @staticmethod def _convert_tool_choice( tool_choice: str | dict[str, Any] | None, ) -> str | dict[str, Any] | None: # Translate Honcho's canonical tool_choice vocabulary to OpenAI's. This # mirrors the Anthropic/Gemini backends so a single TOOL_CHOICE value # works regardless of which provider a fallback chain lands on. Notably # OpenAI has no "any" — it spells the same intent "required". if tool_choice is None: return None if isinstance(tool_choice, dict): if "name" in tool_choice: return { "type": "function", "function": {"name": tool_choice["name"]}, } return tool_choice if tool_choice in {"any", "required"}: return "required" if tool_choice in {"auto", "none"}: return tool_choice return {"type": "function", "function": {"name": tool_choice}} @staticmethod def _convert_tools(tools: list[dict[str, Any]]) -> list[dict[str, Any]]: if not tools or tools[0].get("type") == "function": return tools # Tool schemas in src/utils/agent_tools.py use optional fields with # defaults and don't declare additionalProperties: false. OpenAI's # strict function-calling mode forbids both, so we intentionally # don't set strict: True. Standard function calling on GPT-4.x / # GPT-5 remains reliable, and this stays compatible with # OpenAI-compatible proxies (OpenRouter, Together, vLLM, Ollama) # whose strict-mode support is inconsistent. return [ { "type": "function", "function": { "name": tool["name"], "description": tool["description"], "parameters": tool["input_schema"], }, } for tool in tools ]