From f0d487f3104743e195bdac1befbdf8bfd0ae3ce2 Mon Sep 17 00:00:00 2001 From: 666ghj <670939375@qq.com> Date: Thu, 23 Jul 2026 01:58:27 +0800 Subject: [PATCH] fix: harden ontology JSON generation --- backend/app/api/graph.py | 57 ++++- backend/app/services/ontology_generator.py | 6 +- backend/app/utils/llm_client.py | 233 ++++++++++++++++-- backend/tests/test_llm_json_responses.py | 259 +++++++++++++++++++++ backend/tests/test_ontology_api_errors.py | 70 ++++++ backend/tests/test_ontology_generator.py | 27 +++ frontend/src/api/index.js | 7 + 7 files changed, 631 insertions(+), 28 deletions(-) create mode 100644 backend/tests/test_llm_json_responses.py create mode 100644 backend/tests/test_ontology_api_errors.py diff --git a/backend/app/api/graph.py b/backend/app/api/graph.py index cfaa2627..bcd21eba 100644 --- a/backend/app/api/graph.py +++ b/backend/app/api/graph.py @@ -4,6 +4,7 @@ """ import os +import re import traceback import threading from contextlib import ExitStack, nullcontext @@ -24,6 +25,7 @@ from ..models.project import ProjectManager, ProjectStatus from ..services.simulation_manager import SimulationManager from ..services.simulation_runner import SimulationRunner, RunnerStatus from ..services.zep_graph_memory_updater import ZepGraphMemoryManager +from ..utils.llm_client import LLMResponseError # 获取日志器 logger = get_logger('mirofish.api') @@ -288,6 +290,7 @@ def generate_ontology(): } } """ + project = None try: logger.info("=== 开始生成本体定义 ===") @@ -388,12 +391,56 @@ def generate_ontology(): } }) - except Exception as e: - return jsonify({ + except Exception as error: + provider_status = getattr(error, "status_code", None) + request_id = getattr(error, "request_id", None) + + if isinstance(error, LLMResponseError): + public_error = str(error) + response_status = 502 + logger.exception("LLM returned an unusable ontology response") + elif isinstance(provider_status, int): + public_error = f"LLM provider request failed (HTTP {provider_status})" + if request_id: + safe_request_id = re.sub( + r"[^a-zA-Z0-9._:-]", "", str(request_id) + )[:128] + if safe_request_id: + public_error += f" (request_id: {safe_request_id})" + response_status = 502 + # Provider exception bodies may echo request content. Keep the + # server log useful without serializing the exception body. + logger.error( + "Ontology provider request failed: type=%s status=%s request_id=%s", + type(error).__name__, + provider_status, + request_id or "unknown", + ) + else: + public_error = "Ontology generation failed; check the server logs" + response_status = 500 + logger.exception("Unexpected ontology generation failure") + + response_data = None + if project is not None: + project.status = ProjectStatus.FAILED + project.error = public_error + try: + ProjectManager.save_project(project) + except Exception: + logger.exception( + "Failed to persist ontology failure for project %s", + project.project_id, + ) + response_data = {"project_id": project.project_id} + + payload = { "success": False, - "error": str(e), - "traceback": traceback.format_exc() - }), 500 + "error": public_error, + } + if response_data is not None: + payload["data"] = response_data + return jsonify(payload), response_status # ============== 接口2:构建图谱 ============== diff --git a/backend/app/services/ontology_generator.py b/backend/app/services/ontology_generator.py index ce0ea9ab..77f34ec2 100644 --- a/backend/app/services/ontology_generator.py +++ b/backend/app/services/ontology_generator.py @@ -235,7 +235,11 @@ class OntologyGenerator: result = self.llm_client.chat_json( messages=messages, temperature=0.3, - max_tokens=4096 + # Structured ontology responses can exceed 4096 completion tokens, + # especially when a compatible provider counts hidden reasoning in + # the same budget. Let the provider use its model-specific limit. + max_tokens=None, + max_attempts=2, ) # 验证和后处理 diff --git a/backend/app/utils/llm_client.py b/backend/app/utils/llm_client.py index fc316fce..9d85259b 100644 --- a/backend/app/utils/llm_client.py +++ b/backend/app/utils/llm_client.py @@ -4,6 +4,7 @@ LLM客户端封装 """ import json +import logging import re from typing import Optional, Dict, Any, List from openai import OpenAI @@ -12,6 +13,81 @@ from ..config import Config from .openai_chat_compat import create_chat_completion, extract_chat_completion_text +logger = logging.getLogger(__name__) + + +class LLMResponseError(ValueError): + """A safe, structured error for unusable model responses.""" + + def __init__(self, message: str, *, finish_reason: Optional[str] = None): + super().__init__(message) + self.finish_reason = finish_reason + + +def _is_response_format_unsupported(error: Exception) -> bool: + """Detect an explicit provider rejection of JSON response_format.""" + + if getattr(error, "status_code", None) not in {400, 422}: + return False + + body = getattr(error, "body", None) + if not isinstance(body, dict): + return False + + details = body.get("error", body) + if not isinstance(details, dict): + return False + + param = str(details.get("param") or "").strip().lower() + if param == "response_format" or param.startswith("response_format."): + return True + + message = str(details.get("message") or "").lower() + if "response_format" not in message: + return False + + code = str(details.get("code") or "").lower() + unsupported_codes = { + "unsupported_parameter", + "unsupported_value", + "unknown_parameter", + "invalid_parameter", + } + unsupported_phrases = ( + "not support", + "unsupported", + "unknown parameter", + "unrecognized parameter", + ) + return code in unsupported_codes or any( + phrase in message for phrase in unsupported_phrases + ) + + +def _clean_chat_text(content: str) -> str: + """Remove common reasoning wrappers and an outer Markdown JSON fence.""" + + cleaned = re.sub(r'[\s\S]*?', '', content).strip() + cleaned = cleaned.lstrip("\ufeff") + cleaned = re.sub(r'^```(?:json)?\s*\n?', '', cleaned, flags=re.IGNORECASE) + cleaned = re.sub(r'\n?```\s*$', '', cleaned) + return cleaned.strip() + + +def _contains_additional_json_container(content: str) -> bool: + """Return True when trailing text embeds another JSON object or array.""" + + decoder = json.JSONDecoder() + for match in re.finditer(r"[\[{]", content): + try: + value, _ = decoder.raw_decode(content[match.start():]) + except json.JSONDecodeError: + continue + if isinstance(value, (dict, list)): + return True + return False + + class LLMClient: """LLM客户端""" @@ -32,12 +108,31 @@ class LLMClient: api_key=self.api_key, base_url=self.base_url ) + + def _create_completion( + self, + *, + messages: List[Dict[str, str]], + temperature: Optional[float], + max_tokens: Optional[int], + response_format: Optional[Dict[str, Any]], + ) -> Any: + """Send one raw Chat Completions request through the compatibility layer.""" + + return create_chat_completion( + self.client, + model=self.model, + messages=messages, + temperature=temperature, + max_tokens=max_tokens, + response_format=response_format, + ) def chat( self, messages: List[Dict[str, str]], temperature: float = 0.7, - max_tokens: int = 4096, + max_tokens: Optional[int] = 4096, response_format: Optional[Dict] = None ) -> str: """ @@ -52,24 +147,21 @@ class LLMClient: Returns: 模型响应文本 """ - response = create_chat_completion( - self.client, - model=self.model, + response = self._create_completion( messages=messages, temperature=temperature, max_tokens=max_tokens, response_format=response_format, ) content = extract_chat_completion_text(response) - # 部分模型(如MiniMax M2.5)会在content中包含思考内容,需要移除 - content = re.sub(r'[\s\S]*?', '', content).strip() - return content + return _clean_chat_text(content) def chat_json( self, messages: List[Dict[str, str]], temperature: float = 0.3, - max_tokens: int = 4096 + max_tokens: Optional[int] = 4096, + max_attempts: int = 1, ) -> Dict[str, Any]: """ 发送聊天请求并返回JSON @@ -78,24 +170,121 @@ class LLMClient: messages: 消息列表 temperature: 温度参数 max_tokens: 最大token数 + max_attempts: 内容生成尝试次数(不含一次明确的JSON模式能力降级) Returns: 解析后的JSON对象 """ - response = self.chat( - messages=messages, - temperature=temperature, - max_tokens=max_tokens, - response_format={"type": "json_object"} - ) - # 清理markdown代码块标记 - cleaned_response = response.strip() - cleaned_response = re.sub(r'^```(?:json)?\s*\n?', '', cleaned_response, flags=re.IGNORECASE) - cleaned_response = re.sub(r'\n?```\s*$', '', cleaned_response) - cleaned_response = cleaned_response.strip() + if max_attempts < 1: + raise ValueError("max_attempts must be at least 1") + + response_format: Optional[Dict[str, str]] = {"type": "json_object"} + request_max_tokens = max_tokens + last_error: Optional[LLMResponseError] = None + + for attempt in range(1, max_attempts + 1): + # JSON-mode capability negotiation is separate from content + # regeneration. An explicit response_format rejection may add one + # request, but it must not consume a content attempt. + while True: + try: + response = self._create_completion( + messages=messages, + temperature=temperature, + max_tokens=request_max_tokens, + response_format=response_format, + ) + except Exception as error: + if ( + response_format is not None + and _is_response_format_unsupported(error) + ): + logger.warning( + "LLM provider explicitly rejected response_format; " + "retrying once with prompt-only JSON guidance" + ) + response_format = None + continue + raise + break + + try: + return self._parse_json_response(response) + except LLMResponseError as error: + last_error = error + if attempt >= max_attempts: + raise + + # A caller-supplied cap is the common cause of a partial JSON + # object. Omit it for the one bounded retry so the provider can + # use its model-specific output limit. + had_token_cap = request_max_tokens is not None + request_max_tokens = None + logger.warning( + "LLM returned unusable JSON (finish_reason=%s); " + "retrying content generation%s", + error.finish_reason or "unknown", + " without an output token cap" if had_token_cap else "", + ) + + if last_error is not None: # pragma: no cover - defensive loop guard + raise last_error + raise LLMResponseError("LLM did not produce a JSON response") + + @staticmethod + def _parse_json_response(response: Any) -> Dict[str, Any]: + choices = getattr(response, "choices", None) or [] + if not choices: + raise LLMResponseError("LLM returned no choices") + + choice = choices[0] + finish_reason = getattr(choice, "finish_reason", None) + if finish_reason == "length": + raise LLMResponseError( + "LLM JSON output was truncated at the token limit", + finish_reason=finish_reason, + ) + if finish_reason not in {None, "stop"}: + raise LLMResponseError( + f"LLM JSON generation stopped unexpectedly ({finish_reason})", + finish_reason=finish_reason, + ) + + content = _clean_chat_text(extract_chat_completion_text(response)) + if not content: + raise LLMResponseError( + "LLM returned empty JSON content", + finish_reason=finish_reason, + ) try: - return json.loads(cleaned_response) - except json.JSONDecodeError: - raise ValueError(f"LLM返回的JSON格式无效: {cleaned_response}") + value = json.loads(content) + except json.JSONDecodeError as strict_error: + # Some compatible providers append a short explanation after an + # otherwise complete JSON object. Accept only an object decoded + # from the beginning; never repair or invent truncated JSON. + try: + value, end = json.JSONDecoder().raw_decode(content) + except json.JSONDecodeError: + raise LLMResponseError( + "LLM returned invalid JSON " + f"(line {strict_error.lineno}, column {strict_error.colno})", + finish_reason=finish_reason, + ) from strict_error + trailing = content[end:].strip() + if trailing: + if _contains_additional_json_container(trailing): + raise LLMResponseError( + "LLM returned multiple JSON values", + finish_reason=finish_reason, + ) + logger.warning("Ignoring text after a complete LLM JSON object") + + if not isinstance(value, dict): + raise LLMResponseError( + "LLM JSON response must be a top-level JSON object", + finish_reason=finish_reason, + ) + + return value diff --git a/backend/tests/test_llm_json_responses.py b/backend/tests/test_llm_json_responses.py new file mode 100644 index 00000000..1616d956 --- /dev/null +++ b/backend/tests/test_llm_json_responses.py @@ -0,0 +1,259 @@ +from types import SimpleNamespace + +import pytest + +from app.utils.llm_client import LLMClient, LLMResponseError + + +class CompletionSequence: + def __init__(self, *results): + self.results = list(results) + self.calls = [] + + def create(self, **kwargs): + self.calls.append(kwargs) + result = self.results.pop(0) + if isinstance(result, Exception): + raise result + return result + + +class ProviderError(RuntimeError): + def __init__(self, *, status_code, body): + super().__init__(body.get("error", {}).get("message", "provider error")) + self.status_code = status_code + self.body = body + + +def _response(content, *, finish_reason="stop", include_choice=True): + choices = [] + if include_choice: + choices.append( + SimpleNamespace( + finish_reason=finish_reason, + message=SimpleNamespace(content=content), + ) + ) + return SimpleNamespace(choices=choices) + + +def _client_for(sequence): + client = object.__new__(LLMClient) + client.model = "compatible-model" + client.client = SimpleNamespace( + chat=SimpleNamespace(completions=sequence) + ) + return client + + +def test_chat_json_retries_truncated_completion_without_token_cap(): + sequence = CompletionSequence( + _response('{"items": [', finish_reason="length"), + _response('{"items": [1, 2]}'), + ) + client = _client_for(sequence) + + result = client.chat_json( + messages=[{"role": "user", "content": "Return JSON"}], + max_tokens=4096, + max_attempts=2, + ) + + assert result == {"items": [1, 2]} + assert sequence.calls[0]["max_tokens"] == 4096 + assert "max_tokens" not in sequence.calls[1] + + +def test_chat_json_omits_token_cap_when_requested(): + sequence = CompletionSequence(_response('{"ok": true}')) + client = _client_for(sequence) + + assert client.chat_json( + messages=[{"role": "user", "content": "Return JSON"}], + max_tokens=None, + ) == {"ok": True} + assert "max_tokens" not in sequence.calls[0] + + +def test_chat_json_retries_empty_content_once(): + sequence = CompletionSequence( + _response(None), + _response('{"ok": true}'), + ) + client = _client_for(sequence) + + result = client.chat_json( + messages=[{"role": "user", "content": "Return JSON"}], + max_attempts=2, + ) + + assert result == {"ok": True} + assert len(sequence.calls) == 2 + + +def test_chat_json_accepts_complete_object_before_trailing_text(): + sequence = CompletionSequence( + _response('{"ok": true}\nThis object is ready to use.') + ) + client = _client_for(sequence) + + assert client.chat_json( + messages=[{"role": "user", "content": "Return JSON"}], + ) == {"ok": True} + + +@pytest.mark.parametrize( + "content", + [ + '{"status": "draft"}\n{"status": "complete"}', + '{"status": "draft"}\n```json\n{"status": "complete"}\n```', + '{"status": "draft"}\nExplanation first.\n{"status": "complete"}', + ], +) +def test_chat_json_rejects_a_second_json_document(content): + sequence = CompletionSequence(_response(content)) + client = _client_for(sequence) + + with pytest.raises(LLMResponseError, match="multiple JSON"): + client.chat_json( + messages=[{"role": "user", "content": "Return JSON"}], + ) + + +def test_chat_json_rejects_top_level_array(): + sequence = CompletionSequence(_response('[{"ok": true}]')) + client = _client_for(sequence) + + with pytest.raises(LLMResponseError, match="JSON object"): + client.chat_json( + messages=[{"role": "user", "content": "Return JSON"}], + max_attempts=1, + ) + + +def test_chat_json_error_does_not_echo_partial_model_output(): + partial = '{"private_source_text": "SENTINEL-SHOULD-NOT-LEAK"' + sequence = CompletionSequence( + _response(partial, finish_reason="length"), + _response(partial, finish_reason="length"), + ) + client = _client_for(sequence) + + with pytest.raises(LLMResponseError) as captured: + client.chat_json( + messages=[{"role": "user", "content": "Return JSON"}], + max_attempts=2, + ) + + assert "SENTINEL-SHOULD-NOT-LEAK" not in str(captured.value) + assert captured.value.finish_reason == "length" + + +def test_chat_json_falls_back_only_for_explicit_response_format_rejection(): + unsupported = ProviderError( + status_code=400, + body={ + "error": { + "param": "response_format", + "code": "unsupported_parameter", + "message": "response_format is not supported by this model", + } + }, + ) + sequence = CompletionSequence(unsupported, _response('{"ok": true}')) + client = _client_for(sequence) + + result = client.chat_json( + messages=[{"role": "user", "content": "Return JSON"}], + max_attempts=1, + ) + + assert result == {"ok": True} + assert sequence.calls[0]["response_format"] == {"type": "json_object"} + assert "response_format" not in sequence.calls[1] + + +def test_response_format_fallback_keeps_content_retry_available(): + unsupported = ProviderError( + status_code=400, + body={ + "error": { + "param": "response_format", + "code": "unsupported_parameter", + "message": "response_format is not supported by this model", + } + }, + ) + sequence = CompletionSequence( + unsupported, + _response('{"items": [', finish_reason="length"), + _response('{"items": [1]}'), + ) + client = _client_for(sequence) + + result = client.chat_json( + messages=[{"role": "user", "content": "Return JSON"}], + max_attempts=2, + ) + + assert result == {"items": [1]} + assert sequence.calls[0]["response_format"] == {"type": "json_object"} + assert "response_format" not in sequence.calls[1] + assert "response_format" not in sequence.calls[2] + assert "max_tokens" not in sequence.calls[2] + + +def test_chat_json_defaults_to_one_content_attempt(): + sequence = CompletionSequence( + _response(None), + _response('{"ok": true}'), + ) + client = _client_for(sequence) + + with pytest.raises(LLMResponseError, match="empty JSON"): + client.chat_json( + messages=[{"role": "user", "content": "Return JSON"}], + ) + + assert len(sequence.calls) == 1 + + +@pytest.mark.parametrize("status_code", [400, 401, 429, 500]) +def test_chat_json_does_not_retry_unrelated_provider_errors(status_code): + provider_error = ProviderError( + status_code=status_code, + body={ + "error": { + "param": "messages", + "code": "invalid_request", + "message": "request failed for an unrelated reason", + } + }, + ) + sequence = CompletionSequence(provider_error, _response('{"ok": true}')) + client = _client_for(sequence) + + with pytest.raises(ProviderError) as captured: + client.chat_json( + messages=[{"role": "user", "content": "Return JSON"}], + max_attempts=2, + ) + + assert captured.value is provider_error + assert len(sequence.calls) == 1 + + +def test_chat_json_reports_missing_choices_without_retrying_forever(): + sequence = CompletionSequence( + _response(None, include_choice=False), + _response(None, include_choice=False), + ) + client = _client_for(sequence) + + with pytest.raises(LLMResponseError, match="no choices"): + client.chat_json( + messages=[{"role": "user", "content": "Return JSON"}], + max_attempts=2, + ) + + assert len(sequence.calls) == 2 diff --git a/backend/tests/test_ontology_api_errors.py b/backend/tests/test_ontology_api_errors.py new file mode 100644 index 00000000..cbfb320b --- /dev/null +++ b/backend/tests/test_ontology_api_errors.py @@ -0,0 +1,70 @@ +import io + +from app import create_app +from app.api import graph as graph_api +from app.models.project import ProjectManager, ProjectStatus +from app.utils.llm_client import LLMResponseError + + +def _post_ontology(client): + return client.post( + "/api/graph/ontology/generate", + data={ + "simulation_requirement": "Simulate the discussion.", + "files": (io.BytesIO(b"A short source document."), "source.md"), + }, + content_type="multipart/form-data", + ) + + +def test_ontology_api_returns_safe_truncation_error_and_failed_project( + tmp_path, + monkeypatch, +): + class FailingGenerator: + def generate(self, **kwargs): + raise LLMResponseError( + "LLM JSON output was truncated at the token limit", + finish_reason="length", + ) + + monkeypatch.setattr(ProjectManager, "PROJECTS_DIR", str(tmp_path)) + monkeypatch.setattr(graph_api, "OntologyGenerator", FailingGenerator) + + app = create_app() + app.config.update(TESTING=True) + response = _post_ontology(app.test_client()) + + assert response.status_code == 502 + assert response.json["success"] is False + assert "token limit" in response.json["error"] + assert "traceback" not in response.json + + project_id = response.json["data"]["project_id"] + project = ProjectManager.get_project(project_id) + assert project.status == ProjectStatus.FAILED + assert project.error == response.json["error"] + + +def test_ontology_api_does_not_expose_provider_error_body(tmp_path, monkeypatch): + class ProviderError(RuntimeError): + status_code = 401 + request_id = "request-safe-id" + body = {"error": {"message": "SECRET-PROVIDER-BODY"}} + + class FailingGenerator: + def generate(self, **kwargs): + raise ProviderError("SECRET-PROVIDER-BODY") + + monkeypatch.setattr(ProjectManager, "PROJECTS_DIR", str(tmp_path)) + monkeypatch.setattr(graph_api, "OntologyGenerator", FailingGenerator) + + app = create_app() + app.config.update(TESTING=True) + response = _post_ontology(app.test_client()) + + assert response.status_code == 502 + assert "HTTP 401" in response.json["error"] + assert "request-safe-id" in response.json["error"] + assert "SECRET-PROVIDER-BODY" not in response.get_data(as_text=True) + assert "traceback" not in response.json diff --git a/backend/tests/test_ontology_generator.py b/backend/tests/test_ontology_generator.py index 4d81d2ff..e5c930e2 100644 --- a/backend/tests/test_ontology_generator.py +++ b/backend/tests/test_ontology_generator.py @@ -1,6 +1,19 @@ from app.services.ontology_generator import OntologyGenerator +class RecordingLLMClient: + def __init__(self): + self.calls = [] + + def chat_json(self, **kwargs): + self.calls.append(kwargs) + return { + "entity_types": [], + "edge_types": [], + "analysis_summary": "ok", + } + + def _generator_for_test() -> OntologyGenerator: generator = OntologyGenerator(llm_client=object()) generator.MAX_TEXT_LENGTH_FOR_LLM = 2000 @@ -50,3 +63,17 @@ def test_very_long_ontology_context_selects_representative_chunks(): assert "BEGIN" in context assert "FINALEND" in context assert context.count("--- 文档 1 / 分块") == generator.MAX_LONG_TEXT_CHUNKS + + +def test_ontology_generation_does_not_cap_structured_output_tokens(): + llm = RecordingLLMClient() + generator = OntologyGenerator(llm_client=llm) + + result = generator.generate( + document_texts=["A short source document."], + simulation_requirement="Simulate the public discussion.", + ) + + assert result["analysis_summary"] == "ok" + assert llm.calls[0]["max_tokens"] is None + assert llm.calls[0]["max_attempts"] == 2 diff --git a/frontend/src/api/index.js b/frontend/src/api/index.js index 807a0b4e..2e5216f0 100644 --- a/frontend/src/api/index.js +++ b/frontend/src/api/index.js @@ -37,6 +37,7 @@ service.interceptors.response.use( }, error => { console.error('Response error:', error) + const apiError = error.response?.data?.error || error.response?.data?.message // 处理超时 if (error.code === 'ECONNABORTED' && error.message.includes('timeout')) { @@ -47,6 +48,12 @@ service.interceptors.response.use( if (error.message === 'Network Error') { console.error('Network error - please check your connection') } + + // Axios rejects non-2xx responses before the success interceptor can + // surface the backend's safe, actionable error message. + if (typeof apiError === 'string' && apiError) { + error.message = apiError + } return Promise.reject(error) }