From 1d96b098756166618febff816b529d9d72bc61ac Mon Sep 17 00:00:00 2001 From: MiroFish Contributor Date: Tue, 21 Apr 2026 00:02:58 +0800 Subject: [PATCH 1/2] fix: strip fabricated tool_result blocks to prevent LLM hallucination When the LLM generates a block followed by a self-generated block in the same response, the fake result must be stripped before appending to message history. The real tool result will be injected separately by the system. This fixes a React Hallucination bug where models could fabricate tool results that didn't actually come from tool invocations. --- backend/app/services/report_agent.py | 41 ++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/backend/app/services/report_agent.py b/backend/app/services/report_agent.py index cecd70b4..88bff23a 100644 --- a/backend/app/services/report_agent.py +++ b/backend/app/services/report_agent.py @@ -664,6 +664,12 @@ SECTION_SYSTEM_PROMPT_TEMPLATE = """\ - 不要添加模拟中不存在的信息 - 如果某方面信息不足，如实说明 +5. 【禁止捏造数据】 + - ❌ 禁止捏造用户名、引用、统计数字或互动数据 + - ❌ 禁止在回复中包含 块 — 只有系统会提供工具结果 + - ✅ 只能引用真实出现在工具结果中的实体、引用和数据 + - 如果工具结果中没有相关内容，应如实说明，而非编造 + ═══════════════════════════════════════════════════════════════ 【⚠️ 格式规范 - 极其重要！】 ═══════════════════════════════════════════════════════════════ @@ -1133,7 +1139,26 @@ class ReportAgent: if params_desc: desc_parts.append(f" 参数: {params_desc}") return "\n".join(desc_parts) - + + @staticmethod + def _strip_fake_tool_results(response: str) -> str: + """Strip any blocks the LLM fabricated in its response. + + When the LLM generates a block and then continues to generate + a block in the same response, we must strip the fake result + before appending to message history. The real tool result will be injected + separately by the system. + """ + import re + cleaned = re.sub( + r'.*?', + '', + response, + flags=re.DOTALL, + ) + cleaned = re.sub(r'\n{3,}', '\n\n', cleaned) + return cleaned.strip() + def plan_outline( self, progress_callback: Optional[Callable] = None @@ -1335,7 +1360,8 @@ class ReportAgent: if conflict_retries <= 2: # 前两次：丢弃本次响应，要求 LLM 重新回复 - messages.append({"role": "assistant", "content": response}) + cleaned_response = ReportAgent._strip_fake_tool_results(response) + messages.append({"role": "assistant", "content": cleaned_response}) messages.append({ "role": "user", "content": ( @@ -1375,7 +1401,8 @@ class ReportAgent: if has_final_answer: # 工具调用次数不足，拒绝并要求继续调工具 if tool_calls_count < min_tool_calls: - messages.append({"role": "assistant", "content": response}) + cleaned_response = ReportAgent._strip_fake_tool_results(response) + messages.append({"role": "assistant", "content": cleaned_response}) unused_tools = all_tools - used_tools unused_hint = f"（这些工具还未使用，推荐用一下他们: {', '.join(unused_tools)}）" if unused_tools else "" messages.append({ @@ -1451,9 +1478,10 @@ class ReportAgent: unused_tools = all_tools - used_tools unused_hint = "" if unused_tools and tool_calls_count < self.MAX_TOOL_CALLS_PER_SECTION: - unused_hint = REACT_UNUSED_TOOLS_HINT.format(unused_list="、".join(unused_tools)) + unlock_hint = REACT_UNUSED_TOOLS_HINT.format(unused_list="、".join(unused_tools)) - messages.append({"role": "assistant", "content": response}) + cleaned_response = ReportAgent._strip_fake_tool_results(response) + messages.append({"role": "assistant", "content": cleaned_response}) messages.append({ "role": "user", "content": REACT_OBSERVATION_TEMPLATE.format( @@ -1857,7 +1885,8 @@ class ReportAgent: tool_calls_made.append(call) # 将结果添加到消息 - messages.append({"role": "assistant", "content": response}) + cleaned_response = ReportAgent._strip_fake_tool_results(response) + messages.append({"role": "assistant", "content": cleaned_response}) observation = "\n".join([f"[{r['tool']}结果]\n{r['result']}" for r in tool_results]) messages.append({ "role": "user", From f0e379471a6c9b124eb23ca9d1331dbdf33a1412 Mon Sep 17 00:00:00 2001 From: 666ghj <670939375@qq.com> Date: Wed, 22 Jul 2026 01:24:00 +0800 Subject: [PATCH 2/2] fix: sanitize fabricated tool results everywhere --- backend/app/services/report_agent.py | 44 +++++++++++++------ .../test_report_tool_result_sanitizer.py | 28 ++++++++++++ 2 files changed, 59 insertions(+), 13 deletions(-) create mode 100644 backend/tests/test_report_tool_result_sanitizer.py diff --git a/backend/app/services/report_agent.py b/backend/app/services/report_agent.py index 88bff23a..1bb212b1 100644 --- a/backend/app/services/report_agent.py +++ b/backend/app/services/report_agent.py @@ -1149,13 +1149,27 @@ class ReportAgent: before appending to message history. The real tool result will be injected separately by the system. """ - import re - cleaned = re.sub( - r'.*?', - '', - response, - flags=re.DOTALL, - ) + tag_pattern = re.compile(r']*>', flags=re.IGNORECASE) + parts = [] + cursor = 0 + depth = 0 + + for match in tag_pattern.finditer(response): + if depth == 0: + parts.append(response[cursor:match.start()]) + + if match.group(0).lstrip().startswith('` as unsafe too. + cleaned = re.sub(r'= self.MAX_TOOL_CALLS_PER_SECTION: - messages.append({"role": "assistant", "content": response}) + cleaned_response = ReportAgent._strip_fake_tool_results(response) + messages.append({"role": "assistant", "content": cleaned_response}) messages.append({ "role": "user", "content": REACT_TOOL_LIMIT_MSG.format( @@ -1478,7 +1493,7 @@ class ReportAgent: unused_tools = all_tools - used_tools unused_hint = "" if unused_tools and tool_calls_count < self.MAX_TOOL_CALLS_PER_SECTION: - unlock_hint = REACT_UNUSED_TOOLS_HINT.format(unused_list="、".join(unused_tools)) + unused_hint = REACT_UNUSED_TOOLS_HINT.format(unused_list="、".join(unused_tools)) cleaned_response = ReportAgent._strip_fake_tool_results(response) messages.append({"role": "assistant", "content": cleaned_response}) @@ -1496,7 +1511,8 @@ class ReportAgent: continue # ── 情况3：既没有工具调用，也没有 Final Answer ── - messages.append({"role": "assistant", "content": response}) + cleaned_response = ReportAgent._strip_fake_tool_results(response) + messages.append({"role": "assistant", "content": cleaned_response}) if tool_calls_count < min_tool_calls: # 工具调用次数不足，推荐未用过的工具 @@ -1516,7 +1532,7 @@ class ReportAgent: # 工具调用已足够，LLM 输出了内容但没带 "Final Answer:" 前缀 # 直接将这段内容作为最终答案，不再空转 logger.info(t('report.sectionNoPrefix', title=section.title, count=tool_calls_count)) - final_answer = response.strip() + final_answer = cleaned_response if self.report_logger: self.report_logger.log_section_content( @@ -1865,6 +1881,7 @@ class ReportAgent: # 没有工具调用，直接返回响应 clean_response = re.sub(r'.*?', '', response, flags=re.DOTALL) clean_response = re.sub(r'\[TOOL_CALL\].*?\)', '', clean_response) + clean_response = ReportAgent._strip_fake_tool_results(clean_response) return { "response": clean_response.strip(), @@ -1902,6 +1919,7 @@ class ReportAgent: # 清理响应 clean_response = re.sub(r'.*?', '', final_response, flags=re.DOTALL) clean_response = re.sub(r'\[TOOL_CALL\].*?\)', '', clean_response) + clean_response = ReportAgent._strip_fake_tool_results(clean_response) return { "response": clean_response.strip(), diff --git a/backend/tests/test_report_tool_result_sanitizer.py b/backend/tests/test_report_tool_result_sanitizer.py new file mode 100644 index 00000000..0a79cdc1 --- /dev/null +++ b/backend/tests/test_report_tool_result_sanitizer.py @@ -0,0 +1,28 @@ +import pytest + +from app.services.report_agent import ReportAgent + + +@pytest.mark.parametrize( + ("response", "expected"), + [ + ("before fake after", "before after"), + ("before\nline 1\nline 2\nafter", "before\n\nafter"), + ("afakeb", "ab"), + ( + "aouterinnerendb", + "ab", + ), + ("safeunclosed fake", "safe"), + ("safeb", "ab"), + ], +) +def test_strip_fake_tool_results(response, expected): + assert ReportAgent._strip_fake_tool_results(response) == expected + + +def test_preserves_legitimate_text_without_tool_result_tags(): + response = "Final Answer: {} legitimate text" + + assert ReportAgent._strip_fake_tool_results(response) == response