diff --git a/.env.example b/.env.example index 78a3b72c..10276350 100644 --- a/.env.example +++ b/.env.example @@ -8,9 +8,13 @@ LLM_MODEL_NAME=qwen-plus # ===== ZEP记忆图谱配置 ===== # 每月免费额度即可支撑简单使用：https://app.getzep.com/ ZEP_API_KEY=your_zep_api_key_here +# 单次 Zep Cloud 请求超时（秒） +ZEP_REQUEST_TIMEOUT_SECONDS=30 +# 文档批次或模拟 episode 等待处理完成的最长时间（秒） +ZEP_INGESTION_TIMEOUT_SECONDS=600 # ===== 加速 LLM 配置（可选）===== # 注意如果不使用加速配置，env文件中就不要出现下面的配置项 LLM_BOOST_API_KEY=your_api_key_here LLM_BOOST_BASE_URL=your_base_url_here -LLM_BOOST_MODEL_NAME=your_model_name_here \ No newline at end of file +LLM_BOOST_MODEL_NAME=your_model_name_here diff --git a/backend/app/api/graph.py b/backend/app/api/graph.py index 82b8d713..cfaa2627 100644 --- a/backend/app/api/graph.py +++ b/backend/app/api/graph.py @@ -6,21 +6,115 @@ import os import traceback import threading +from contextlib import ExitStack, nullcontext from flask import request, jsonify +from zep_cloud import NotFoundError from . import graph_bp from ..config import Config from ..services.ontology_generator import OntologyGenerator -from ..services.graph_builder import GraphBuilderService +from ..services.graph_builder import BatchSubmission, GraphBuilderService from ..services.text_processor import TextProcessor from ..utils.file_parser import FileParser from ..utils.logger import get_logger from ..utils.locale import t, get_locale, set_locale +from ..utils.zep_lifecycle import get_graph_readers, graph_lifecycle_lock from ..models.task import TaskManager, TaskStatus from ..models.project import ProjectManager, ProjectStatus +from ..services.simulation_manager import SimulationManager +from ..services.simulation_runner import SimulationRunner, RunnerStatus +from ..services.zep_graph_memory_updater import ZepGraphMemoryManager # 获取日志器 logger = get_logger('mirofish.api') +_build_locks: dict[str, threading.Lock] = {} +_build_locks_guard = threading.Lock() + + +class GraphInUseError(RuntimeError): + pass + + +def _active_graph_consumers(graph_id: str) -> list[str]: + active = { + f"report:{reader_id}" + for reader_id in get_graph_readers(graph_id) + } + for simulation_id in ZepGraphMemoryManager.get_simulation_ids_for_graph(graph_id): + finalization_lock = SimulationRunner._finalization_lock(simulation_id) + if not finalization_lock.acquire(blocking=False): + active.add(simulation_id) + continue + try: + run_state = SimulationRunner.get_run_state(simulation_id) + if run_state and run_state.runner_status == RunnerStatus.FAILED: + # reset/delete is the explicit recovery path for an incomplete, + # non-replayable write. Serialize it against a retry drain. + ZepGraphMemoryManager.discard_inactive_updater(simulation_id) + SimulationRunner._graph_memory_enabled.pop(simulation_id, None) + continue + active.add(simulation_id) + finally: + finalization_lock.release() + active_runner_statuses = { + RunnerStatus.STARTING, + RunnerStatus.RUNNING, + RunnerStatus.PAUSED, + RunnerStatus.STOPPING, + } + for simulation in SimulationManager().list_simulations(): + if simulation.graph_id != graph_id: + continue + run_state = SimulationRunner.get_run_state(simulation.simulation_id) + if run_state and run_state.runner_status in active_runner_statuses: + active.add(simulation.simulation_id) + return sorted(active) + + +def _delete_cloud_graph_if_present(graph_id: str | None) -> None: + """Delete a referenced Cloud graph without retrying the mutation.""" + + if not graph_id: + return + # Keep the consumer check and Cloud mutation in one critical section. The + # callers that also clear local references hold this re-entrant lock around + # both operations. + with graph_lifecycle_lock(graph_id): + active_simulations = _active_graph_consumers(graph_id) + if active_simulations: + raise GraphInUseError( + f"Graph {graph_id} is in use by active consumer(s): " + f"{', '.join(active_simulations)}" + ) + try: + GraphBuilderService(api_key=Config.ZEP_API_KEY).delete_graph(graph_id) + except NotFoundError: + logger.info("Zep Cloud graph already absent: %s", graph_id) + + +def _clear_project_graph_reference(project) -> None: + project.graph_id = None + project.graph_build_task_id = None + project.zep_batch_id = None + project.zep_batch_operation_id = None + project.error = None + + +def _project_build_lock(project_id: str) -> threading.Lock: + with _build_locks_guard: + return _build_locks.setdefault(project_id, threading.Lock()) + + +def _project_has_active_build(project) -> bool: + if project.status != ProjectStatus.GRAPH_BUILDING: + return False + if not project.graph_build_task_id: + return False + task = TaskManager().get_task(project.graph_build_task_id) + return bool( + task + and task.status in {TaskStatus.PENDING, TaskStatus.PROCESSING} + ) def allowed_file(filename: str) -> bool: @@ -69,10 +163,38 @@ def list_projects(): @graph_bp.route('/project/', methods=['DELETE']) def delete_project(project_id: str): + with _project_build_lock(project_id): + return _delete_project_impl(project_id) + + +def _delete_project_impl(project_id: str): """ 删除项目 """ - success = ProjectManager.delete_project(project_id) + project = ProjectManager.get_project(project_id) + if not project: + return jsonify({ + "success": False, + "error": t('api.projectNotFound', id=project_id) + }), 404 + if _project_has_active_build(project): + return jsonify({ + "success": False, + "error": t('api.graphBuilding') + }), 409 + + graph_id = project.graph_id + graph_guard = ( + graph_lifecycle_lock(graph_id) if graph_id else nullcontext() + ) + with graph_guard: + try: + _delete_cloud_graph_if_present(graph_id) + except GraphInUseError as error: + return jsonify({"success": False, "error": str(error)}), 409 + # The local reference remains protected until it is removed, so a new + # simulation cannot claim the just-deleted graph in between. + success = ProjectManager.delete_project(project_id) if not success: return jsonify({ @@ -88,6 +210,11 @@ def delete_project(project_id: str): @graph_bp.route('/project//reset', methods=['POST']) def reset_project(project_id: str): + with _project_build_lock(project_id): + return _reset_project_impl(project_id) + + +def _reset_project_impl(project_id: str): """ 重置项目状态（用于重新构建图谱） """ @@ -99,16 +226,30 @@ def reset_project(project_id: str): "error": t('api.projectNotFound', id=project_id) }), 404 - # 重置到本体已生成状态 - if project.ontology: - project.status = ProjectStatus.ONTOLOGY_GENERATED - else: - project.status = ProjectStatus.CREATED - - project.graph_id = None - project.graph_build_task_id = None - project.error = None - ProjectManager.save_project(project) + if _project_has_active_build(project): + return jsonify({ + "success": False, + "error": t('api.graphBuilding') + }), 409 + + graph_id = project.graph_id + graph_guard = ( + graph_lifecycle_lock(graph_id) if graph_id else nullcontext() + ) + with graph_guard: + try: + _delete_cloud_graph_if_present(graph_id) + except GraphInUseError as error: + return jsonify({"success": False, "error": str(error)}), 409 + + # 重置到本体已生成状态 + if project.ontology: + project.status = ProjectStatus.ONTOLOGY_GENERATED + else: + project.status = ProjectStatus.CREATED + + _clear_project_graph_reference(project) + ProjectManager.save_project(project) return jsonify({ "success": True, @@ -259,6 +400,17 @@ def generate_ontology(): @graph_bp.route('/build', methods=['POST']) def build_graph(): + """Serialize build claims for the same project within this process.""" + + data = request.get_json(silent=True) or {} + project_id = data.get("project_id") + if not project_id: + return _build_graph_impl() + with _project_build_lock(project_id): + return _build_graph_impl() + + +def _build_graph_impl(): """ 接口2：根据project_id构建图谱 @@ -315,6 +467,11 @@ def build_graph(): # 检查项目状态 force = data.get('force', False) # 强制重新构建 + if not isinstance(force, bool): + return jsonify({ + "success": False, + "error": "force must be a JSON boolean" + }), 400 if project.status == ProjectStatus.CREATED: return jsonify({ @@ -322,24 +479,77 @@ def build_graph(): "error": t('api.ontologyNotGenerated') }), 400 - if project.status == ProjectStatus.GRAPH_BUILDING and not force: + resume_existing_batch = False + if project.status == ProjectStatus.GRAPH_BUILDING: + if _project_has_active_build(project): + return jsonify({ + "success": True, + "data": { + "project_id": project_id, + "task_id": project.graph_build_task_id, + "graph_id": project.graph_id, + "reused": True, + "message": t('api.graphBuilding') + } + }) + + if ( + not force + and project.graph_id + and project.zep_batch_id + and project.zep_batch_operation_id + ): + builder = GraphBuilderService(api_key=Config.ZEP_API_KEY) + batch_summary = builder.get_batch_summary(project.zep_batch_id) + if getattr(batch_summary, "status", None) in { + "queued", + "processing", + "succeeded", + }: + resume_existing_batch = True + + if not resume_existing_batch: + project.status = ProjectStatus.FAILED + project.error = ( + "Graph build task is no longer present; the persisted Zep " + "batch cannot be resumed automatically" + ) + ProjectManager.save_project(project) + if not force: + return jsonify({ + "success": False, + "error": project.error, + "task_id": project.graph_build_task_id, + "recoverable": True, + }), 409 + + if project.status == ProjectStatus.GRAPH_COMPLETED and not force: return jsonify({ - "success": False, - "error": t('api.graphBuilding'), - "task_id": project.graph_build_task_id - }), 400 - - # 如果强制重建，重置状态 - if force and project.status in [ProjectStatus.GRAPH_BUILDING, ProjectStatus.FAILED, ProjectStatus.GRAPH_COMPLETED]: - project.status = ProjectStatus.ONTOLOGY_GENERATED - project.graph_id = None - project.graph_build_task_id = None - project.error = None + "success": True, + "data": { + "project_id": project_id, + "task_id": project.graph_build_task_id, + "graph_id": project.graph_id, + "reused": True, + "message": t('progress.graphBuildComplete') + } + }) # 获取配置 graph_name = data.get('graph_name', project.name or 'MiroFish Graph') chunk_size = data.get('chunk_size', project.chunk_size or Config.DEFAULT_CHUNK_SIZE) chunk_overlap = data.get('chunk_overlap', project.chunk_overlap or Config.DEFAULT_CHUNK_OVERLAP) + if not isinstance(chunk_size, int) or chunk_size <= 0: + return jsonify({"success": False, "error": "chunk_size must be a positive integer"}), 400 + if ( + not isinstance(chunk_overlap, int) + or chunk_overlap < 0 + or chunk_overlap >= chunk_size + ): + return jsonify({ + "success": False, + "error": "chunk_overlap must satisfy 0 <= chunk_overlap < chunk_size" + }), 400 # 更新项目配置 project.chunk_size = chunk_size @@ -360,6 +570,22 @@ def build_graph(): "success": False, "error": t('api.ontologyNotFound') }), 400 + + # Only mutate Cloud state after the complete rebuild request validates. + if project.status == ProjectStatus.FAILED or ( + force and project.status == ProjectStatus.GRAPH_COMPLETED + ): + graph_id_to_delete = project.graph_id + graph_guard = ( + graph_lifecycle_lock(graph_id_to_delete) + if graph_id_to_delete + else nullcontext() + ) + with graph_guard: + _delete_cloud_graph_if_present(graph_id_to_delete) + project.status = ProjectStatus.ONTOLOGY_GENERATED + _clear_project_graph_reference(project) + ProjectManager.save_project(project) # 创建异步任务 task_manager = TaskManager() @@ -400,49 +626,79 @@ def build_graph(): chunk_size=chunk_size, overlap=chunk_overlap ) + builder.validate_batch_chunks(chunks, batch_size=350) total_chunks = len(chunks) - # 创建图谱 - task_manager.update_task( - task_id, - message=t('progress.creatingZepGraph'), - progress=10 - ) - graph_id = builder.create_graph(name=graph_name) - - # 更新项目的graph_id - project.graph_id = graph_id - ProjectManager.save_project(project) - - # 设置本体 - task_manager.update_task( - task_id, - message=t('progress.settingOntology'), - progress=15 - ) - builder.set_ontology(graph_id, ontology) - - # 添加文本（progress_callback 签名是 (msg, progress_ratio)） - def add_progress_callback(msg, progress_ratio): - progress = 15 + int(progress_ratio * 40) # 15% - 55% + if resume_existing_batch: + graph_id = project.graph_id + operation_id = builder.build_operation_id(graph_id, chunks) + if operation_id != project.zep_batch_operation_id: + raise RuntimeError( + "Persisted Zep batch does not match the current graph input" + ) + submission = BatchSubmission( + batch_id=project.zep_batch_id, + operation_id=operation_id, + episode_uuids=[], + item_count=total_chunks, + ) task_manager.update_task( task_id, - message=msg, - progress=progress + message=t('progress.waitingZepProcess'), + progress=55, + ) + else: + # 创建图谱 + task_manager.update_task( + task_id, + message=t('progress.creatingZepGraph'), + progress=10 + ) + + def remember_graph(graph_id): + project.graph_id = graph_id + ProjectManager.save_project(project) + + graph_id = builder.create_graph( + name=graph_name, + graph_id_callback=remember_graph, + ) + + # 设置本体 + task_manager.update_task( + task_id, + message=t('progress.settingOntology'), + progress=15 + ) + builder.set_ontology(graph_id, ontology) + + # 添加文本（progress_callback 签名是 (msg, progress_ratio)） + def add_progress_callback(msg, progress_ratio): + progress = 15 + int(progress_ratio * 40) # 15% - 55% + task_manager.update_task( + task_id, + message=msg, + progress=progress + ) + + task_manager.update_task( + task_id, + message=t('progress.addingChunks', count=total_chunks), + progress=15 + ) + + def remember_batch(batch_id, operation_id): + project.zep_batch_id = batch_id + project.zep_batch_operation_id = operation_id + ProjectManager.save_project(project) + + submission = builder.add_text_batches( + graph_id, + chunks, + batch_size=350, + progress_callback=add_progress_callback, + batch_created_callback=remember_batch, ) - - task_manager.update_task( - task_id, - message=t('progress.addingChunks', count=total_chunks), - progress=15 - ) - - episode_uuids = builder.add_text_batches( - graph_id, - chunks, - batch_size=3, - progress_callback=add_progress_callback - ) # 等待Zep处理完成（查询每个episode的processed状态） task_manager.update_task( @@ -459,7 +715,7 @@ def build_graph(): progress=progress ) - builder._wait_for_episodes(episode_uuids, wait_progress_callback) + builder._wait_for_batch(submission, wait_progress_callback) # 获取图谱数据 task_manager.update_task( @@ -469,44 +725,48 @@ def build_graph(): ) graph_data = builder.get_graph_data(graph_id) - # 更新项目状态 - project.status = ProjectStatus.GRAPH_COMPLETED - ProjectManager.save_project(project) - node_count = graph_data.get("node_count", 0) edge_count = graph_data.get("edge_count", 0) build_logger.info(f"[{task_id}] 图谱构建完成: graph_id={graph_id}, 节点={node_count}, 边={edge_count}") - - # 完成 - task_manager.update_task( - task_id, - status=TaskStatus.COMPLETED, - message=t('progress.graphBuildComplete'), - progress=100, - result={ - "project_id": project_id, - "graph_id": graph_id, - "node_count": node_count, - "edge_count": edge_count, - "chunk_count": total_chunks - } - ) + + # Publish local project/task terminal state under the same + # lifecycle lock used by reset/delete/build claims. This + # prevents a deletion from interleaving between the two saves. + with _project_build_lock(project_id): + project.status = ProjectStatus.GRAPH_COMPLETED + project.error = None + ProjectManager.save_project(project) + task_manager.update_task( + task_id, + status=TaskStatus.COMPLETED, + message=t('progress.graphBuildComplete'), + progress=100, + result={ + "project_id": project_id, + "graph_id": graph_id, + "node_count": node_count, + "edge_count": edge_count, + "chunk_count": total_chunks, + "zep_batch_id": submission.batch_id, + } + ) except Exception as e: # 更新项目状态为失败 build_logger.error(f"[{task_id}] 图谱构建失败: {str(e)}") build_logger.debug(traceback.format_exc()) - project.status = ProjectStatus.FAILED - project.error = str(e) - ProjectManager.save_project(project) - - task_manager.update_task( - task_id, - status=TaskStatus.FAILED, - message=t('progress.buildFailed', error=str(e)), - error=traceback.format_exc() - ) + with _project_build_lock(project_id): + project.status = ProjectStatus.FAILED + project.error = str(e) + ProjectManager.save_project(project) + + task_manager.update_task( + task_id, + status=TaskStatus.FAILED, + message=t('progress.buildFailed', error=str(e)), + error=traceback.format_exc() + ) # 启动后台线程 thread = threading.Thread(target=build_task, daemon=True) @@ -517,10 +777,13 @@ def build_graph(): "data": { "project_id": project_id, "task_id": task_id, + "resumed": resume_existing_batch, "message": t('api.graphBuildStarted', taskId=task_id) } }) + except GraphInUseError as e: + return jsonify({"success": False, "error": str(e)}), 409 except Exception as e: return jsonify({ "success": False, @@ -606,14 +869,45 @@ def delete_graph(graph_id: str): "error": t('api.zepApiKeyMissing') }), 500 - builder = GraphBuilderService(api_key=Config.ZEP_API_KEY) - builder.delete_graph(graph_id) + projects = ProjectManager.find_projects_by_graph_id(graph_id) + if not projects: + return jsonify({ + "success": False, + "error": "No local project references this graph" + }), 404 + project_ids = sorted({project.project_id for project in projects}) + with ExitStack() as stack: + for project_id in project_ids: + stack.enter_context(_project_build_lock(project_id)) + stack.enter_context(graph_lifecycle_lock(graph_id)) + + # Re-read under all owning project locks so a concurrent build + # claim cannot appear between validation and Cloud deletion. + projects = ProjectManager.find_projects_by_graph_id(graph_id) + if any(_project_has_active_build(project) for project in projects): + return jsonify({ + "success": False, + "error": t('api.graphBuilding') + }), 409 + + _delete_cloud_graph_if_present(graph_id) + + for project in projects: + _clear_project_graph_reference(project) + project.status = ( + ProjectStatus.ONTOLOGY_GENERATED + if project.ontology + else ProjectStatus.CREATED + ) + ProjectManager.save_project(project) return jsonify({ "success": True, "message": t('api.graphDeleted', id=graph_id) }) + except GraphInUseError as e: + return jsonify({"success": False, "error": str(e)}), 409 except Exception as e: return jsonify({ "success": False, diff --git a/backend/app/api/report.py b/backend/app/api/report.py index d7f2a4d0..a1e7dbcf 100644 --- a/backend/app/api/report.py +++ b/backend/app/api/report.py @@ -12,10 +12,17 @@ from . import report_bp from ..config import Config from ..services.report_agent import ReportAgent, ReportManager, ReportStatus from ..services.simulation_manager import SimulationManager -from ..models.project import ProjectManager +from ..services.simulation_runner import SimulationRunner, RunnerStatus +from ..services.zep_graph_memory_updater import ZepGraphMemoryManager +from ..models.project import ProjectManager, ProjectStatus from ..models.task import TaskManager, TaskStatus from ..utils.logger import get_logger from ..utils.locale import t, get_locale, set_locale +from ..utils.zep_lifecycle import ( + graph_lifecycle_lock, + register_graph_reader, + unregister_graph_reader, +) logger = get_logger('mirofish.api.report') @@ -58,6 +65,11 @@ def generate_report(): }), 400 force_regenerate = data.get('force_regenerate', False) + if not isinstance(force_regenerate, bool): + return jsonify({ + "success": False, + "error": "force_regenerate must be a JSON boolean", + }), 400 # 获取模拟信息 manager = SimulationManager() @@ -69,21 +81,41 @@ def generate_report(): "error": t('api.simulationNotFound', id=simulation_id) }), 404 - # 检查是否已有报告 - if not force_regenerate: - existing_report = ReportManager.get_report_by_simulation(simulation_id) - if existing_report and existing_report.status == ReportStatus.COMPLETED: - return jsonify({ - "success": True, - "data": { - "simulation_id": simulation_id, - "report_id": existing_report.report_id, - "status": "completed", - "message": t('api.reportAlreadyExists'), - "already_generated": True - } - }) - + run_state = SimulationRunner.get_run_state(simulation_id) + updater = ZepGraphMemoryManager.get_updater(simulation_id) + active_statuses = { + RunnerStatus.STARTING, + RunnerStatus.RUNNING, + RunnerStatus.PAUSED, + RunnerStatus.STOPPING, + } + if updater is not None or ( + run_state is not None and run_state.runner_status in active_statuses + ): + return jsonify({ + "success": False, + "error": ( + "Simulation or Zep graph ingestion is still active; " + "wait for a terminal run status before generating a report" + ), + "ingestion_pending": updater is not None, + }), 409 + successful_terminal_statuses = { + RunnerStatus.COMPLETED, + RunnerStatus.STOPPED, + } + if ( + run_state is None + or run_state.runner_status not in successful_terminal_statuses + ): + return jsonify({ + "success": False, + "error": ( + "A successfully completed or stopped simulation is required " + "before generating a report" + ), + }), 409 + # 获取项目信息 project = ProjectManager.get_project(state.project_id) if not project: @@ -92,12 +124,26 @@ def generate_report(): "error": t('api.projectNotFound', id=state.project_id) }), 404 - graph_id = state.graph_id or project.graph_id + if project.status != ProjectStatus.GRAPH_COMPLETED: + return jsonify({ + "success": False, + "error": "The project graph must be completely built before reporting", + }), 409 + + graph_id = project.graph_id if not graph_id: return jsonify({ "success": False, "error": t('api.missingGraphIdEnsure') }), 400 + if state.graph_id and state.graph_id != graph_id: + return jsonify({ + "success": False, + "error": ( + "The simulation references an older graph; prepare it " + "again before generating a report" + ), + }), 409 simulation_requirement = project.simulation_requirement if not simulation_requirement: @@ -110,74 +156,147 @@ def generate_report(): import uuid report_id = f"report_{uuid.uuid4().hex[:12]}" - # 创建异步任务 - task_manager = TaskManager() - task_id = task_manager.create_task( - task_type="report_generate", - metadata={ - "simulation_id": simulation_id, - "graph_id": graph_id, - "report_id": report_id - } - ) - - # Capture locale before spawning background thread - current_locale = get_locale() + # Register the background report as a graph reader under the same lock + # used by graph deletion and updater startup. A lock itself cannot be + # acquired in this request thread and released by the worker, so the + # durable reader registration is the cross-thread lease. + with graph_lifecycle_lock(graph_id): + refreshed_state = manager.get_simulation(simulation_id) + refreshed_project = ( + ProjectManager.get_project(refreshed_state.project_id) + if refreshed_state + else None + ) + refreshed_run_state = SimulationRunner.get_run_state(simulation_id) + refreshed_updater = ZepGraphMemoryManager.get_updater(simulation_id) + if ( + refreshed_state is None + or refreshed_project is None + or refreshed_project.graph_id != graph_id + or refreshed_project.status != ProjectStatus.GRAPH_COMPLETED + or ( + refreshed_state.graph_id + and refreshed_state.graph_id != graph_id + ) + ): + return jsonify({ + "success": False, + "error": "The project graph changed while reporting was starting", + }), 409 + if refreshed_updater is not None or ( + refreshed_run_state is not None + and refreshed_run_state.runner_status in active_statuses + ): + return jsonify({ + "success": False, + "error": ( + "Simulation or Zep graph ingestion became active; " + "retry after it reaches a terminal state" + ), + "ingestion_pending": refreshed_updater is not None, + }), 409 + if ( + refreshed_run_state is None + or refreshed_run_state.runner_status + not in successful_terminal_statuses + ): + return jsonify({ + "success": False, + "error": ( + "A successfully completed or stopped simulation is " + "required before generating a report" + ), + }), 409 - # 定义后台任务 - def run_generate(): - set_locale(current_locale) - try: - task_manager.update_task( - task_id, - status=TaskStatus.PROCESSING, - progress=0, - message=t('api.initReportAgent') + # Cached-report reuse is now part of the same atomic barrier, so a + # concurrent rerun cannot make the returned report stale between + # the status check and response. + if not force_regenerate: + existing_report = ReportManager.get_report_by_simulation( + simulation_id ) - - # 创建Report Agent - agent = ReportAgent( - graph_id=graph_id, - simulation_id=simulation_id, - simulation_requirement=simulation_requirement - ) - - # 进度回调 - def progress_callback(stage, progress, message): + if ( + existing_report + and existing_report.status == ReportStatus.COMPLETED + ): + return jsonify({ + "success": True, + "data": { + "simulation_id": simulation_id, + "report_id": existing_report.report_id, + "status": "completed", + "message": t('api.reportAlreadyExists'), + "already_generated": True + } + }) + + task_manager = TaskManager() + task_id = task_manager.create_task( + task_type="report_generate", + metadata={ + "simulation_id": simulation_id, + "graph_id": graph_id, + "report_id": report_id + } + ) + current_locale = get_locale() + register_graph_reader(graph_id, report_id) + + def run_generate(): + set_locale(current_locale) + try: task_manager.update_task( task_id, - progress=progress, - message=f"[{stage}] {message}" + status=TaskStatus.PROCESSING, + progress=0, + message=t('api.initReportAgent') ) - - # 生成报告（传入预先生成的 report_id） - report = agent.generate_report( - progress_callback=progress_callback, - report_id=report_id - ) - - # 保存报告 - ReportManager.save_report(report) - - if report.status == ReportStatus.COMPLETED: - task_manager.complete_task( - task_id, - result={ - "report_id": report.report_id, - "simulation_id": simulation_id, - "status": "completed" - } + + agent = ReportAgent( + graph_id=graph_id, + simulation_id=simulation_id, + simulation_requirement=simulation_requirement ) - else: - task_manager.fail_task(task_id, report.error or t('api.reportGenerateFailed')) - - except Exception as e: - logger.error(f"报告生成失败: {str(e)}") - task_manager.fail_task(task_id, str(e)) - - # 启动后台线程 - thread = threading.Thread(target=run_generate, daemon=True) - thread.start() + + def progress_callback(stage, progress, message): + task_manager.update_task( + task_id, + progress=progress, + message=f"[{stage}] {message}" + ) + + report = agent.generate_report( + progress_callback=progress_callback, + report_id=report_id + ) + ReportManager.save_report(report) + + if report.status == ReportStatus.COMPLETED: + task_manager.complete_task( + task_id, + result={ + "report_id": report.report_id, + "simulation_id": simulation_id, + "status": "completed" + } + ) + else: + task_manager.fail_task( + task_id, + report.error or t('api.reportGenerateFailed') + ) + except Exception as e: + logger.error(f"报告生成失败: {str(e)}") + task_manager.fail_task(task_id, str(e)) + finally: + unregister_graph_reader(graph_id, report_id) + + try: + thread = threading.Thread(target=run_generate, daemon=True) + thread.start() + except Exception: + unregister_graph_reader(graph_id, report_id) + raise return jsonify({ "success": True, diff --git a/backend/app/api/simulation.py b/backend/app/api/simulation.py index 54842436..aae76788 100644 --- a/backend/app/api/simulation.py +++ b/backend/app/api/simulation.py @@ -5,6 +5,7 @@ Step2: Zep实体读取与过滤、OASIS模拟准备与运行（全程自动化 import os import traceback +from contextlib import nullcontext from flask import request, jsonify, send_file from . import simulation_bp @@ -12,9 +13,15 @@ from ..config import Config from ..services.zep_entity_reader import ZepEntityReader from ..services.oasis_profile_generator import OasisProfileGenerator from ..services.simulation_manager import SimulationManager, SimulationStatus -from ..services.simulation_runner import SimulationRunner, RunnerStatus +from ..services.simulation_runner import ( + SimulationRunner, + RunnerStatus, + SimulationStopPending, +) +from ..services.zep_graph_memory_updater import ZepGraphMemoryManager from ..utils.logger import get_logger from ..utils.locale import t, get_locale, set_locale +from ..utils.zep_lifecycle import get_graph_readers, graph_lifecycle_lock from ..models.project import ProjectManager logger = get_logger('mirofish.api.simulation') @@ -1546,6 +1553,16 @@ def start_simulation(): max_rounds = data.get('max_rounds') # 可选：最大模拟轮数 enable_graph_memory_update = data.get('enable_graph_memory_update', False) # 可选：是否启用图谱记忆更新 force = data.get('force', False) # 可选：强制重新开始 + if not isinstance(enable_graph_memory_update, bool): + return jsonify({ + "success": False, + "error": "enable_graph_memory_update must be a JSON boolean", + }), 400 + if not isinstance(force, bool): + return jsonify({ + "success": False, + "error": "force must be a JSON boolean", + }), 400 # 验证 max_rounds 参数 if max_rounds is not None: @@ -1586,31 +1603,67 @@ def start_simulation(): is_prepared, prepare_info = _check_simulation_prepared(simulation_id) if is_prepared: - # 准备工作已完成，检查是否有正在运行的进程 - if state.status == SimulationStatus.RUNNING: - # 检查模拟进程是否真的在运行 - run_state = SimulationRunner.get_run_state(simulation_id) - if run_state and run_state.runner_status.value == "running": - # 进程确实在运行 - if force: - # 强制模式：停止运行中的模拟 - logger.info(f"强制模式：停止运行中的模拟 {simulation_id}") - try: - SimulationRunner.stop_simulation(simulation_id) - except Exception as e: - logger.warning(f"停止模拟时出现警告: {str(e)}") - else: - return jsonify({ - "success": False, - "error": t('api.simRunningForceHint') - }), 400 + run_state = SimulationRunner.get_run_state(simulation_id) + updater = ZepGraphMemoryManager.get_updater(simulation_id) + needs_finalization = bool( + run_state + and run_state.runner_status in { + RunnerStatus.RUNNING, + RunnerStatus.PAUSED, + RunnerStatus.STOPPING, + RunnerStatus.FAILED, + } + and ( + run_state.runner_status + in { + RunnerStatus.RUNNING, + RunnerStatus.PAUSED, + RunnerStatus.STOPPING, + } + or updater is not None + ) + ) + if needs_finalization: + if not force: + return jsonify({ + "success": False, + "error": t('api.simRunningForceHint') + }), 400 + logger.info(f"强制模式：先完成旧模拟终止 {simulation_id}") + try: + stopped = SimulationRunner.stop_simulation(simulation_id) + except SimulationStopPending as error: + return jsonify({ + "success": False, + "pending": True, + "error": str(error), + }), 409 + except Exception as error: + return jsonify({ + "success": False, + "error": ( + "Cannot restart until the previous simulation " + f"finalizes safely: {error}" + ), + }), 409 + if stopped.runner_status != RunnerStatus.STOPPED: + return jsonify({ + "success": False, + "error": "Previous simulation did not reach STOPPED", + }), 409 # 如果是强制模式，清理运行日志 if force: logger.info(f"强制模式：清理模拟日志 {simulation_id}") cleanup_result = SimulationRunner.cleanup_simulation_logs(simulation_id) if not cleanup_result.get("success"): - logger.warning(f"清理日志时出现警告: {cleanup_result.get('errors')}") + return jsonify({ + "success": False, + "error": ( + "Failed to clean previous simulation logs: " + f"{cleanup_result.get('errors')}" + ), + }), 500 force_restarted = True # 进程不存在或已结束，重置状态为 ready @@ -1627,34 +1680,82 @@ def start_simulation(): # 获取图谱ID（用于图谱记忆更新） graph_id = None if enable_graph_memory_update: - # 从模拟状态或项目中获取 graph_id - graph_id = state.graph_id - if not graph_id: - # 尝试从项目中获取 - project = ProjectManager.get_project(state.project_id) - if project: - graph_id = project.graph_id - + # The project is authoritative. A graph ID copied into an older + # simulation can outlive a project reset/rebuild and must not be + # used to resurrect writes to a deleted graph. + project = ProjectManager.get_project(state.project_id) + graph_id = project.graph_id if project else None if not graph_id: return jsonify({ "success": False, "error": t('api.graphIdRequiredForMemory') }), 400 - - logger.info(f"启用图谱记忆更新: simulation_id={simulation_id}, graph_id={graph_id}") - - # 启动模拟 - run_state = SimulationRunner.start_simulation( - simulation_id=simulation_id, - platform=platform, - max_rounds=max_rounds, - enable_graph_memory_update=enable_graph_memory_update, - graph_id=graph_id + + graph_guard = ( + graph_lifecycle_lock(graph_id) + if enable_graph_memory_update + else nullcontext() ) - - # 更新模拟状态 - state.status = SimulationStatus.RUNNING - manager._save_simulation_state(state) + with graph_guard: + if enable_graph_memory_update: + # Re-read both references under the same per-graph lock used + # by reset/delete. Keep the lock through updater creation in + # start_simulation so check -> claim is atomic. + refreshed_state = manager.get_simulation(simulation_id) + refreshed_project = ( + ProjectManager.get_project(refreshed_state.project_id) + if refreshed_state + else None + ) + current_graph_id = ( + refreshed_project.graph_id if refreshed_project else None + ) + if current_graph_id != graph_id: + return jsonify({ + "success": False, + "error": ( + "The project graph changed while the simulation " + "was starting; retry after refreshing the project" + ), + }), 409 + if ( + refreshed_state.graph_id + and refreshed_state.graph_id != current_graph_id + ): + return jsonify({ + "success": False, + "error": ( + "The simulation references an older graph; " + "prepare it again before enabling graph memory" + ), + }), 409 + active_reports = get_graph_readers(graph_id) + if active_reports: + return jsonify({ + "success": False, + "error": ( + "A report is currently reading this graph; wait " + "for report generation to finish before enabling " + "graph memory updates" + ), + "active_reports": active_reports, + }), 409 + state = refreshed_state + logger.info( + "启用图谱记忆更新: simulation_id=%s, graph_id=%s", + simulation_id, + graph_id, + ) + + # 启动模拟。启用图谱写入时仍持有 graph_guard，直到 updater + # claim 与进程资源全部发布完成。 + run_state = SimulationRunner.start_simulation( + simulation_id=simulation_id, + platform=platform, + max_rounds=max_rounds, + enable_graph_memory_update=enable_graph_memory_update, + graph_id=graph_id + ) response_data = run_state.to_dict() if max_rounds: @@ -1720,14 +1821,22 @@ def stop_simulation(): manager = SimulationManager() state = manager.get_simulation(simulation_id) if state: - state.status = SimulationStatus.PAUSED + state.status = SimulationStatus.STOPPED + state.error = None manager._save_simulation_state(state) return jsonify({ "success": True, "data": run_state.to_dict() }) - + + except SimulationStopPending as e: + return jsonify({ + "success": False, + "pending": True, + "error": str(e), + }), 202 + except ValueError as e: return jsonify({ "success": False, @@ -1736,6 +1845,14 @@ def stop_simulation(): except Exception as e: logger.error(f"停止模拟失败: {str(e)}") + simulation_id = (request.get_json(silent=True) or {}).get('simulation_id') + if simulation_id: + manager = SimulationManager() + state = manager.get_simulation(simulation_id) + if state: + state.status = SimulationStatus.FAILED + state.error = str(e) + manager._save_simulation_state(state) return jsonify({ "success": False, "error": str(e), diff --git a/backend/app/config.py b/backend/app/config.py index 280b089f..534e3baf 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -14,6 +14,29 @@ else: load_dotenv(override=True) +def _parse_number(name, raw_value, *, default, cast): + """Parse a numeric setting without making module import fail.""" + + try: + return cast(raw_value), None + except (TypeError, ValueError): + return default, f"{name} must be a number" + + +_zep_request_timeout, _zep_request_timeout_error = _parse_number( + "ZEP_REQUEST_TIMEOUT_SECONDS", + os.environ.get("ZEP_REQUEST_TIMEOUT_SECONDS", "30"), + default=30.0, + cast=float, +) +_zep_ingestion_timeout, _zep_ingestion_timeout_error = _parse_number( + "ZEP_INGESTION_TIMEOUT_SECONDS", + os.environ.get("ZEP_INGESTION_TIMEOUT_SECONDS", "600"), + default=600, + cast=int, +) + + class Config: """Flask配置类""" @@ -31,6 +54,13 @@ class Config: # Zep配置 ZEP_API_KEY = os.environ.get('ZEP_API_KEY') + ZEP_REQUEST_TIMEOUT_SECONDS = _zep_request_timeout + ZEP_INGESTION_TIMEOUT_SECONDS = _zep_ingestion_timeout + _ZEP_CONFIG_PARSE_ERRORS = tuple( + error + for error in (_zep_request_timeout_error, _zep_ingestion_timeout_error) + if error + ) # 文件上传配置 MAX_CONTENT_LENGTH = 50 * 1024 * 1024 # 50MB @@ -68,6 +98,13 @@ class Config: errors.append("LLM_API_KEY 未配置") if not cls.ZEP_API_KEY: errors.append("ZEP_API_KEY 未配置") + if os.environ.get("ZEP_API_URL"): + errors.append("ZEP_API_URL 不受支持；MiroFish 仅连接 Zep Cloud") + errors.extend(cls._ZEP_CONFIG_PARSE_ERRORS) + if cls.ZEP_REQUEST_TIMEOUT_SECONDS <= 0: + errors.append("ZEP_REQUEST_TIMEOUT_SECONDS 必须大于 0") + if cls.ZEP_INGESTION_TIMEOUT_SECONDS <= 0: + errors.append("ZEP_INGESTION_TIMEOUT_SECONDS 必须大于 0") if cls.DEBUG: import warnings warnings.warn("Flask DEBUG mode is enabled. Do not use in production.", RuntimeWarning) diff --git a/backend/app/models/project.py b/backend/app/models/project.py index 08978937..fa63ab94 100644 --- a/backend/app/models/project.py +++ b/backend/app/models/project.py @@ -43,6 +43,8 @@ class Project: # 图谱信息（接口2完成后填充） graph_id: Optional[str] = None graph_build_task_id: Optional[str] = None + zep_batch_id: Optional[str] = None + zep_batch_operation_id: Optional[str] = None # 配置 simulation_requirement: Optional[str] = None @@ -66,6 +68,8 @@ class Project: "analysis_summary": self.analysis_summary, "graph_id": self.graph_id, "graph_build_task_id": self.graph_build_task_id, + "zep_batch_id": self.zep_batch_id, + "zep_batch_operation_id": self.zep_batch_operation_id, "simulation_requirement": self.simulation_requirement, "chunk_size": self.chunk_size, "chunk_overlap": self.chunk_overlap, @@ -91,6 +95,8 @@ class Project: analysis_summary=data.get('analysis_summary'), graph_id=data.get('graph_id'), graph_build_task_id=data.get('graph_build_task_id'), + zep_batch_id=data.get('zep_batch_id'), + zep_batch_operation_id=data.get('zep_batch_operation_id'), simulation_requirement=data.get('simulation_requirement'), chunk_size=data.get('chunk_size', 500), chunk_overlap=data.get('chunk_overlap', 50), @@ -195,7 +201,7 @@ class ProjectManager: return Project.from_dict(data) @classmethod - def list_projects(cls, limit: int = 50) -> List[Project]: + def list_projects(cls, limit: Optional[int] = 50) -> List[Project]: """ 列出所有项目 @@ -216,7 +222,17 @@ class ProjectManager: # 按创建时间倒序排序 projects.sort(key=lambda p: p.created_at, reverse=True) - return projects[:limit] + return projects if limit is None else projects[:limit] + + @classmethod + def find_projects_by_graph_id(cls, graph_id: str) -> List[Project]: + """Return every persisted project that references a Cloud graph.""" + + return [ + project + for project in cls.list_projects(limit=None) + if project.graph_id == graph_id + ] @classmethod def delete_project(cls, project_id: str) -> bool: @@ -302,4 +318,3 @@ class ProjectManager: for f in os.listdir(files_dir) if os.path.isfile(os.path.join(files_dir, f)) ] - diff --git a/backend/app/services/graph_builder.py b/backend/app/services/graph_builder.py index 4a12c148..33706914 100644 --- a/backend/app/services/graph_builder.py +++ b/backend/app/services/graph_builder.py @@ -3,15 +3,14 @@ 接口2：使用Zep API构建Standalone Graph """ -import os +import hashlib import uuid import time import threading from typing import Dict, Any, List, Optional, Callable from dataclasses import dataclass -from zep_cloud.client import Zep -from zep_cloud import EpisodeData, EntityEdgeSourceTarget +from zep_cloud import BatchAddItem, EntityEdgeSourceTarget, NotFoundError from ..config import Config from ..models.task import TaskManager, TaskStatus @@ -20,6 +19,12 @@ from ..utils.ontology import ( MAX_ONTOLOGY_TYPES, RESERVED_ONTOLOGY_ATTRIBUTE_NAMES, normalize_ontology_attributes, + normalize_ontology_source_targets, +) +from ..utils.zep import ( + call_zep_read_with_retry, + get_zep_client, + is_retryable_zep_error, ) from .text_processor import TextProcessor from ..utils.locale import t, get_locale, set_locale @@ -42,6 +47,16 @@ class GraphInfo: } +@dataclass(frozen=True) +class BatchSubmission: + """Durable identity for one Zep Batch API ingestion operation.""" + + batch_id: str + operation_id: str + episode_uuids: List[str] + item_count: int + + class GraphBuilderService: """ 图谱构建服务 @@ -53,7 +68,7 @@ class GraphBuilderService: if not self.api_key: raise ValueError("ZEP_API_KEY 未配置") - self.client = Zep(api_key=self.api_key) + self.client = get_zep_client(self.api_key) self.task_manager = TaskManager() def build_graph_async( @@ -63,7 +78,7 @@ class GraphBuilderService: graph_name: str = "MiroFish Graph", chunk_size: int = 500, chunk_overlap: int = 50, - batch_size: int = 3 + batch_size: int = 350 ) -> str: """ 异步构建图谱 @@ -123,6 +138,12 @@ class GraphBuilderService: message=t('progress.startBuildingGraph') ) + # Validate the complete ingestion payload before the first Cloud + # mutation, including this legacy service entry point. + chunks = TextProcessor.split_text(text, chunk_size, chunk_overlap) + self.validate_batch_chunks(chunks, batch_size=batch_size) + total_chunks = len(chunks) + # 1. 创建图谱 graph_id = self.create_graph(graph_name) self.task_manager.update_task( @@ -139,9 +160,7 @@ class GraphBuilderService: message=t('progress.ontologySet') ) - # 3. 文本分块 - chunks = TextProcessor.split_text(text, chunk_size, chunk_overlap) - total_chunks = len(chunks) + # 3. 文本分块已在 Cloud mutation 前完成并验证 self.task_manager.update_task( task_id, progress=20, @@ -149,7 +168,7 @@ class GraphBuilderService: ) # 4. 分批发送数据 - episode_uuids = self.add_text_batches( + submission = self.add_text_batches( graph_id, chunks, batch_size, lambda msg, prog: self.task_manager.update_task( task_id, @@ -165,8 +184,8 @@ class GraphBuilderService: message=t('progress.waitingZepProcess') ) - self._wait_for_episodes( - episode_uuids, + self._wait_for_batch( + submission, lambda msg, prog: self.task_manager.update_task( task_id, progress=60 + int(prog * 0.3), # 60-90% @@ -195,17 +214,100 @@ class GraphBuilderService: error_msg = f"{str(e)}\n{traceback.format_exc()}" self.task_manager.fail_task(task_id, error_msg) - def create_graph(self, name: str) -> str: - """创建Zep图谱（公开方法）""" - graph_id = f"mirofish_{uuid.uuid4().hex[:16]}" - - self.client.graph.create( - graph_id=graph_id, - name=name, - description="MiroFish Social Simulation Graph" - ) - + def create_graph( + self, + name: str, + *, + graph_id: str | None = None, + graph_id_callback: Optional[Callable[[str], None]] = None, + ) -> str: + """Create a graph with a caller-durable ID and reconcile lost replies.""" + + graph_id = graph_id or f"mirofish_{uuid.uuid4().hex[:16]}" + # Persist the client-generated ID before the non-idempotent POST so a + # later reset can clean up a graph whose successful response was lost. + if graph_id_callback: + graph_id_callback(graph_id) + + try: + self.client.graph.create( + graph_id=graph_id, + name=name, + description="MiroFish Social Simulation Graph" + ) + except Exception as error: + if not is_retryable_zep_error(error): + raise + reconciliation_error = None + for attempt in range(3): + try: + call_zep_read_with_retry( + lambda: self.client.graph.get(graph_id), + operation_name=f"reconcile graph create {graph_id}", + ) + reconciliation_error = None + break + except NotFoundError as not_found: + reconciliation_error = not_found + if attempt < 2: + time.sleep(attempt + 1) + except Exception as read_error: + reconciliation_error = read_error + break + if reconciliation_error is not None: + raise error from reconciliation_error + return graph_id + + @staticmethod + def build_operation_id(graph_id: str, chunks: List[str]) -> str: + payload_hash = hashlib.sha256("\0".join(chunks).encode("utf-8")).hexdigest() + return hashlib.sha256( + f"{graph_id}:{payload_hash}".encode("utf-8") + ).hexdigest() + + def _find_batch_by_operation_id( + self, + graph_id: str, + operation_id: str, + *, + max_attempts: int = 3, + ) -> Any | None: + """Find one server-created batch after an ambiguous create reply.""" + + for attempt in range(1, max_attempts + 1): + matches: List[Any] = [] + cursor: int | None = None + seen_cursors: set[int] = set() + while True: + page = call_zep_read_with_retry( + lambda: self.client.batch.list(limit=100, cursor=cursor), + operation_name=f"reconcile batch create {operation_id}", + ) + for batch in getattr(page, "batches", None) or []: + metadata = getattr(batch, "metadata", None) or {} + if ( + metadata.get("mirofish_operation_id") == operation_id + and metadata.get("graph_id") == graph_id + ): + matches.append(batch) + next_cursor = getattr(page, "next_cursor", None) + if next_cursor is None: + break + if next_cursor == cursor or next_cursor in seen_cursors: + raise RuntimeError("Zep batch list cursor did not advance") + seen_cursors.add(next_cursor) + cursor = next_cursor + + if len(matches) > 1: + raise RuntimeError( + f"Multiple Zep batches match operation {operation_id}; refusing ambiguity" + ) + if matches: + return matches[0] + if attempt < max_attempts: + time.sleep(attempt) + return None def set_ontology(self, graph_id: str, ontology: Dict[str, Any]): """设置图谱本体（公开方法）""" @@ -278,7 +380,9 @@ class GraphBuilderService: # 构建source_targets source_targets = [] - for st in edge_def.get("source_targets", []): + for st in normalize_ontology_source_targets( + edge_def.get("source_targets", []) + ): source_targets.append( EntityEdgeSourceTarget( source=st.get("source", "Entity"), @@ -293,8 +397,8 @@ class GraphBuilderService: if entity_types or edge_definitions: self.client.graph.set_ontology( graph_ids=[graph_id], - # zep-cloud 3.13.0 iterates entities.items(), so an edge-only - # ontology must pass an empty dictionary rather than None. + # Zep iterates entities.items(), so edge-only ontologies must + # pass an empty dictionary rather than None. entities=entity_types, edges=edge_definitions if edge_definitions else None, ) @@ -303,13 +407,53 @@ class GraphBuilderService: self, graph_id: str, chunks: List[str], - batch_size: int = 3, - progress_callback: Optional[Callable] = None - ) -> List[str]: - """分批添加文本到图谱，返回所有 episode 的 uuid 列表""" - episode_uuids = [] + batch_size: int = 350, + progress_callback: Optional[Callable] = None, + batch_created_callback: Optional[Callable[[str | None, str], None]] = None, + ) -> BatchSubmission: + """Submit document chunks through Zep's current Batch API. + + Mutating calls are deliberately not retried: create/add are not + documented as idempotent, and an ambiguous replay can duplicate graph + episodes. The returned batch identity allows callers to persist and + reconcile the operation instead. + """ + + if not graph_id: + raise ValueError("graph_id is required") + self.validate_batch_chunks(chunks, batch_size=batch_size) + total_chunks = len(chunks) - + operation_id = self.build_operation_id(graph_id, chunks) + if batch_created_callback: + # Journal the deterministic operation before the server-generated + # batch ID POST. This leaves enough identity for later diagnosis + # even if both the response and immediate list reconciliation fail. + batch_created_callback(None, operation_id) + + try: + batch = self.client.batch.create( + metadata={ + "mirofish_operation_id": operation_id, + "graph_id": graph_id, + "chunk_count": total_chunks, + } + ) + except Exception as error: + if not is_retryable_zep_error(error): + raise + batch = self._find_batch_by_operation_id(graph_id, operation_id) + if batch is None: + raise RuntimeError( + "Zep batch creation is unconfirmed and no matching operation was found" + ) from error + batch_id = getattr(batch, "batch_id", None) + if not batch_id: + raise RuntimeError("Zep Batch API returned no batch_id") + if batch_created_callback: + batch_created_callback(batch_id, operation_id) + + episode_uuids: List[str] = [] for i in range(0, total_chunks, batch_size): batch_chunks = chunks[i:i + batch_size] batch_num = i // batch_size + 1 @@ -322,34 +466,254 @@ class GraphBuilderService: progress ) - # 构建episode数据 - episodes = [ - EpisodeData(data=chunk, type="text") - for chunk in batch_chunks - ] - - # 发送到Zep - try: - batch_result = self.client.graph.add_batch( + items = [ + BatchAddItem( + type="graph_episode", graph_id=graph_id, - episodes=episodes + data=chunk, + data_type="text", + source_description="MiroFish source document chunk", + metadata={ + "mirofish_operation_id": operation_id, + "chunk_index": i + offset, + "chunk_sha256": hashlib.sha256( + chunk.encode("utf-8") + ).hexdigest(), + }, + ) + for offset, chunk in enumerate(batch_chunks) + ] + + expected_item_count = i + len(items) + try: + item_details = self.client.batch.add( + batch_id=batch_id, + items=items, ) - - # 收集返回的 episode uuid - if batch_result and isinstance(batch_result, list): - for ep in batch_result: - ep_uuid = getattr(ep, 'uuid_', None) or getattr(ep, 'uuid', None) - if ep_uuid: - episode_uuids.append(ep_uuid) - - # 避免请求过快 - time.sleep(1) - except Exception as e: if progress_callback: progress_callback(t('progress.batchFailed', batch=batch_num, error=str(e)), 0) - raise - + if is_retryable_zep_error(e): + recovered_items = self._reconcile_batch_item_count( + batch_id, + expected_item_count, + ) + recovered_indexes = { + getattr(item, "sequence_index", None) + for item in recovered_items + } + if ( + len(recovered_items) == expected_item_count + and recovered_indexes == set(range(expected_item_count)) + ): + item_details = recovered_items[i:expected_item_count] + else: + raise RuntimeError( + f"Zep batch {batch_id} item submission is unconfirmed; " + "the draft was not processed or replayed" + ) from e + else: + raise RuntimeError( + f"Zep batch {batch_id} item submission failed" + ) from e + + if len(item_details or []) != len(items): + recovered_items = self._reconcile_batch_item_count( + batch_id, + expected_item_count, + ) + recovered_indexes = { + getattr(item, "sequence_index", None) + for item in recovered_items + } + if ( + len(recovered_items) == expected_item_count + and recovered_indexes == set(range(expected_item_count)) + ): + item_details = recovered_items[i:expected_item_count] + else: + raise RuntimeError( + f"Zep batch {batch_id} acknowledged {len(item_details or [])} " + f"of {len(items)} items" + ) + for item in item_details: + episode_uuid = getattr(item, "episode_uuid", None) + if episode_uuid: + episode_uuids.append(episode_uuid) + + try: + self.client.batch.process(batch_id=batch_id) + except Exception as error: + # A process response can be lost after the server accepted it. + # Reconcile with a safe GET instead of issuing a second POST. + summary = call_zep_read_with_retry( + lambda: self.client.batch.get(batch_id=batch_id), + operation_name=f"reconcile batch {batch_id}", + ) + if getattr(summary, "status", None) in {None, "draft"}: + raise RuntimeError( + f"Zep batch {batch_id} processing is unconfirmed" + ) from error + + return BatchSubmission( + batch_id=batch_id, + operation_id=operation_id, + episode_uuids=episode_uuids, + item_count=total_chunks, + ) + + @staticmethod + def validate_batch_chunks(chunks: List[str], *, batch_size: int = 350) -> None: + """Validate every Batch API limit before the first Cloud mutation.""" + + if not chunks: + raise ValueError("At least one text chunk is required") + if not 1 <= batch_size <= 350: + raise ValueError("batch_size must be between 1 and 350") + if len(chunks) > 50_000: + raise ValueError("A Zep batch cannot contain more than 50,000 items") + oversized = [index for index, chunk in enumerate(chunks) if len(chunk) > 10_000] + if oversized: + raise ValueError( + f"Zep batch item exceeds 10,000 characters at chunk {oversized[0]}" + ) + + def _list_batch_items(self, batch_id: str) -> List[Any]: + items: List[Any] = [] + cursor: int | None = None + seen_cursors: set[int] = set() + while True: + page = call_zep_read_with_retry( + lambda: self.client.batch.list_items( + batch_id=batch_id, + limit=100, + cursor=cursor, + ), + operation_name=f"list batch items {batch_id}", + ) + items.extend(getattr(page, "items", None) or []) + next_cursor = getattr(page, "next_cursor", None) + if next_cursor is None: + break + if next_cursor == cursor or next_cursor in seen_cursors: + raise RuntimeError(f"Zep batch {batch_id} item cursor did not advance") + seen_cursors.add(next_cursor) + cursor = next_cursor + return items + + def _reconcile_batch_item_count( + self, + batch_id: str, + expected_item_count: int, + *, + max_attempts: int = 3, + ) -> List[Any]: + """Allow a short propagation window after an ambiguous add reply.""" + + items: List[Any] = [] + for attempt in range(1, max_attempts + 1): + items = self._list_batch_items(batch_id) + if len(items) >= expected_item_count: + return items + if attempt < max_attempts: + time.sleep(attempt) + return items + + def get_batch_summary(self, batch_id: str) -> Any: + """Read a persisted batch identity for restart reconciliation.""" + + return call_zep_read_with_retry( + lambda: self.client.batch.get(batch_id=batch_id), + operation_name=f"get batch {batch_id}", + ) + + def _wait_for_batch( + self, + submission: BatchSubmission, + progress_callback: Optional[Callable] = None, + timeout: int | None = None, + ) -> List[str]: + """Wait for a Batch API terminal state and validate every item.""" + + timeout = timeout or Config.ZEP_INGESTION_TIMEOUT_SECONDS + start_time = time.time() + terminal_states = {"succeeded", "partial", "failed", "invalid", "canceled"} + + while True: + if time.time() - start_time > timeout: + raise TimeoutError( + f"Zep batch {submission.batch_id} did not finish within {timeout}s" + ) + + summary = call_zep_read_with_retry( + lambda: self.client.batch.get(batch_id=submission.batch_id), + operation_name=f"poll batch {submission.batch_id}", + ) + status = getattr(summary, "status", None) + progress = getattr(summary, "progress", None) + percent = float(getattr(progress, "percent_complete", 0) or 0) / 100 + if progress_callback: + completed = int(getattr(progress, "succeeded_items", 0) or 0) + progress_callback( + t( + 'progress.zepProcessing', + completed=completed, + total=submission.item_count, + pending=max(submission.item_count - completed, 0), + elapsed=int(time.time() - start_time), + ), + min(max(percent, 0.0), 1.0), + ) + + if status in terminal_states: + break + time.sleep(3) + + items = self._list_batch_items(submission.batch_id) + if status != "succeeded": + failed_items = [ + item for item in items + if getattr(item, "status", None) not in {"succeeded", "skipped"} + ] + first_error = getattr(failed_items[0], "error", None) if failed_items else None + raise RuntimeError( + f"Zep batch {submission.batch_id} ended as {status}; " + f"failed_items={len(failed_items)}; first_error={first_error}" + ) + if len(items) != submission.item_count: + raise RuntimeError( + f"Zep batch {submission.batch_id} contains {len(items)} items, " + f"expected {submission.item_count}" + ) + + ordered_items = sorted( + items, + key=lambda item: getattr(item, "sequence_index", 0) or 0, + ) + episode_uuids: List[str] = [] + for item in ordered_items: + item_status = getattr(item, "status", None) + episode_uuid = getattr(item, "episode_uuid", None) + source_uuid = getattr(item, "source_uuid", None) + if item_status != "succeeded" or not episode_uuid: + raise RuntimeError( + f"Zep batch {submission.batch_id} returned an incomplete item" + ) + if source_uuid and source_uuid != episode_uuid: + raise RuntimeError( + f"Zep batch {submission.batch_id} returned mismatched episode UUIDs" + ) + episode_uuids.append(episode_uuid) + + if progress_callback: + progress_callback( + t( + 'progress.processingComplete', + completed=len(episode_uuids), + total=submission.item_count, + ), + 1.0, + ) return episode_uuids def _wait_for_episodes( @@ -379,21 +743,22 @@ class GraphBuilderService: t('progress.episodesTimeout', completed=completed_count, total=total_episodes), completed_count / total_episodes ) - break + raise TimeoutError( + f"Zep episode processing timed out with " + f"{len(pending_episodes)} episode(s) still pending" + ) # 检查每个 episode 的处理状态 for ep_uuid in list(pending_episodes): - try: - episode = self.client.graph.episode.get(uuid_=ep_uuid) - is_processed = getattr(episode, 'processed', False) - - if is_processed: - pending_episodes.remove(ep_uuid) - completed_count += 1 - - except Exception as e: - # 忽略单个查询错误，继续 - pass + episode = call_zep_read_with_retry( + lambda: self.client.graph.episode.get(uuid_=ep_uuid), + operation_name=f"poll episode {ep_uuid}", + ) + is_processed = getattr(episode, 'processed', False) + + if is_processed: + pending_episodes.remove(ep_uuid) + completed_count += 1 elapsed = int(time.time() - start_time) if progress_callback: diff --git a/backend/app/services/oasis_profile_generator.py b/backend/app/services/oasis_profile_generator.py index add3a24f..de6ce6e3 100644 --- a/backend/app/services/oasis_profile_generator.py +++ b/backend/app/services/oasis_profile_generator.py @@ -16,12 +16,16 @@ from dataclasses import dataclass, field from datetime import datetime from openai import OpenAI -from zep_cloud.client import Zep - from ..config import Config from ..utils.logger import get_logger from ..utils.locale import get_language_instruction, get_locale, set_locale, t from ..utils.openai_chat_compat import create_chat_completion, extract_chat_completion_text +from ..utils.zep import ( + call_zep_read_with_retry, + get_zep_client, + is_retryable_zep_error, + normalize_zep_search_query, +) from .zep_entity_reader import EntityNode, ZepEntityReader logger = get_logger('mirofish.oasis_profile') @@ -263,7 +267,7 @@ class OasisProfileGenerator: if self.zep_api_key: try: - self.zep_client = Zep(api_key=self.zep_api_key) + self.zep_client = get_zep_client(self.zep_api_key) except Exception as e: logger.warning(f"Zep客户端初始化失败: {e}") @@ -372,57 +376,35 @@ class OasisProfileGenerator: logger.debug(f"跳过Zep检索：未设置graph_id") return results - comprehensive_query = t('progress.zepSearchQuery', name=entity_name) + comprehensive_query = normalize_zep_search_query( + t('progress.zepSearchQuery', name=entity_name) + ) def search_edges(): """搜索边（事实/关系）- 带重试机制""" - max_retries = 3 - last_exception = None - delay = 2.0 - - for attempt in range(max_retries): - try: - return self.zep_client.graph.search( + return call_zep_read_with_retry( + lambda: self.zep_client.graph.search( query=comprehensive_query, graph_id=self.graph_id, limit=30, scope="edges", reranker="rrf" - ) - except Exception as e: - last_exception = e - if attempt < max_retries - 1: - logger.debug(f"Zep边搜索第 {attempt + 1} 次失败: {str(e)[:80]}, 重试中...") - time.sleep(delay) - delay *= 2 - else: - logger.debug(f"Zep边搜索在 {max_retries} 次尝试后仍失败: {e}") - return None + ), + operation_name=f"profile edge search ({entity.uuid})", + ) def search_nodes(): """搜索节点（实体摘要）- 带重试机制""" - max_retries = 3 - last_exception = None - delay = 2.0 - - for attempt in range(max_retries): - try: - return self.zep_client.graph.search( + return call_zep_read_with_retry( + lambda: self.zep_client.graph.search( query=comprehensive_query, graph_id=self.graph_id, limit=20, scope="nodes", reranker="rrf" - ) - except Exception as e: - last_exception = e - if attempt < max_retries - 1: - logger.debug(f"Zep节点搜索第 {attempt + 1} 次失败: {str(e)[:80]}, 重试中...") - time.sleep(delay) - delay *= 2 - else: - logger.debug(f"Zep节点搜索在 {max_retries} 次尝试后仍失败: {e}") - return None + ), + operation_name=f"profile node search ({entity.uuid})", + ) try: # 并行执行edges和nodes搜索 @@ -431,8 +413,11 @@ class OasisProfileGenerator: node_future = executor.submit(search_nodes) # 获取结果 - edge_result = edge_future.result(timeout=30) - node_result = node_future.result(timeout=30) + # Each request already has the configured HTTP timeout and + # typed retry budget. A second hard-coded 30s future timeout + # discarded late successes while the executor still waited. + edge_result = edge_future.result() + node_result = node_future.result() # 处理边搜索结果 all_facts = set() @@ -462,10 +447,10 @@ class OasisProfileGenerator: logger.info(f"Zep混合检索完成: {entity_name}, 获取 {len(results['facts'])} 条事实, {len(results['node_summaries'])} 个相关节点") - except concurrent.futures.TimeoutError: - logger.warning(f"Zep检索超时 ({entity_name})") except Exception as e: logger.warning(f"Zep检索失败 ({entity_name}): {e}") + if not is_retryable_zep_error(e): + raise return results diff --git a/backend/app/services/ontology_generator.py b/backend/app/services/ontology_generator.py index b719a8dc..ce0ea9ab 100644 --- a/backend/app/services/ontology_generator.py +++ b/backend/app/services/ontology_generator.py @@ -13,6 +13,7 @@ from ..utils.file_parser import split_text_into_chunks from ..utils.ontology import ( MAX_ONTOLOGY_TYPES, normalize_ontology_attributes, + normalize_ontology_source_targets, ) logger = logging.getLogger(__name__) @@ -31,6 +32,18 @@ def _to_pascal_case(name: str) -> str: return result if result else 'Unknown' +def _to_upper_snake_case(name: str) -> str: + """Convert free-form or camelCase names to SCREAMING_SNAKE_CASE.""" + + separated = re.sub(r'([a-z0-9])([A-Z])', r'\1_\2', name.strip()) + normalized = re.sub(r'[^a-zA-Z0-9]+', '_', separated).strip('_').upper() + if not normalized: + return "UNKNOWN" + if normalized[0].isdigit(): + normalized = f"REL_{normalized}" + return normalized + + # 本体生成的系统提示词 ONTOLOGY_SYSTEM_PROMPT = """你是一个专业的知识图谱本体设计专家。你的任务是分析给定的文本内容和模拟需求，设计适合**社交媒体舆论模拟**的实体类型和关系类型。 @@ -414,76 +427,71 @@ class OntologyGenerator: def _validate_and_process(self, result: Dict[str, Any]) -> Dict[str, Any]: """验证和后处理结果""" - - # 确保必要字段存在 - if "entity_types" not in result: - result["entity_types"] = [] - if "edge_types" not in result: - result["edge_types"] = [] - if "analysis_summary" not in result: + if not isinstance(result, dict): + raise ValueError("Ontology result must be an object") + + raw_entities = result.get("entity_types") + raw_edges = result.get("edge_types") + if not isinstance(raw_entities, list): + raw_entities = [] + if not isinstance(raw_edges, list): + raw_edges = [] + if not isinstance(result.get("analysis_summary"), str): result["analysis_summary"] = "" - - # 验证实体类型 - # 记录原始名称到 PascalCase 的映射，用于后续修正 edge 的 source_targets 引用 - entity_name_map = {} - for entity in result["entity_types"]: - # 强制将 entity name 转为 PascalCase（Zep API 要求） - if "name" in entity: - original_name = entity["name"] - entity["name"] = _to_pascal_case(original_name) - if entity["name"] != original_name: - logger.warning(f"Entity type name '{original_name}' auto-converted to '{entity['name']}'") - entity_name_map[original_name] = entity["name"] - # Normalize LLM output, enforce Zep field limits, and guarantee at - # least one property for every custom ontology type. + + # Normalize entity entries before touching their fields. LLMs + # occasionally emit a bare string, null, or another scalar. + entity_name_map: Dict[str, str] = {} + processed_entities: List[Dict[str, Any]] = [] + seen_entity_names = set() + for raw_entity in raw_entities: + if isinstance(raw_entity, str): + entity = {"name": raw_entity} + elif isinstance(raw_entity, dict): + entity = dict(raw_entity) + else: + logger.warning("Ignoring non-object ontology entity entry") + continue + + original_name = entity.get("name") + if not isinstance(original_name, str) or not original_name.strip(): + logger.warning("Ignoring ontology entity without a usable name") + continue + original_name = original_name.strip() + normalized_name = _to_pascal_case(original_name) + if normalized_name == "Unknown": + continue + if normalized_name in seen_entity_names: + logger.warning(f"Duplicate entity type '{normalized_name}' removed during validation") + entity_name_map[original_name] = normalized_name + entity_name_map[original_name.lower()] = normalized_name + continue + + if normalized_name != original_name: + logger.warning( + f"Entity type name '{original_name}' auto-converted to '{normalized_name}'" + ) + entity["name"] = normalized_name entity["attributes"] = normalize_ontology_attributes( entity.get("attributes", []) ) - if "examples" not in entity: + if not isinstance(entity.get("examples"), list): entity["examples"] = [] - # 确保description不超过100字符 - if len(entity.get("description", "")) > 100: - entity["description"] = entity["description"][:97] + "..." - - # 验证关系类型 - for edge in result["edge_types"]: - # 强制将 edge name 转为 SCREAMING_SNAKE_CASE（Zep API 要求） - if "name" in edge: - original_name = edge["name"] - edge["name"] = original_name.upper() - if edge["name"] != original_name: - logger.warning(f"Edge type name '{original_name}' auto-converted to '{edge['name']}'") - # 修正 source_targets 中的实体名称引用，与转换后的 PascalCase 保持一致 - for st in edge.get("source_targets", []): - if st.get("source") in entity_name_map: - st["source"] = entity_name_map[st["source"]] - if st.get("target") in entity_name_map: - st["target"] = entity_name_map[st["target"]] - if "source_targets" not in edge: - edge["source_targets"] = [] - # Normalize LLM output, enforce Zep field limits, and guarantee at - # least one property for every custom ontology type. - edge["attributes"] = normalize_ontology_attributes( - edge.get("attributes", []) + description = entity.get("description") + if not isinstance(description, str) or not description: + description = f"A {normalized_name} entity." + entity["description"] = ( + description[:97] + "..." if len(description) > 100 else description ) - if len(edge.get("description", "")) > 100: - edge["description"] = edge["description"][:97] + "..." - - # Zep API 限制：最多 10 个自定义实体类型，最多 10 个自定义边类型 - MAX_ENTITY_TYPES = MAX_ONTOLOGY_TYPES - MAX_EDGE_TYPES = MAX_ONTOLOGY_TYPES - # 去重：按 name 去重，保留首次出现的 - seen_names = set() - deduped = [] - for entity in result["entity_types"]: - name = entity.get("name", "") - if name and name not in seen_names: - seen_names.add(name) - deduped.append(entity) - elif name in seen_names: - logger.warning(f"Duplicate entity type '{name}' removed during validation") - result["entity_types"] = deduped + seen_entity_names.add(normalized_name) + processed_entities.append(entity) + entity_name_map[original_name] = normalized_name + entity_name_map[original_name.lower()] = normalized_name + entity_name_map[normalized_name] = normalized_name + entity_name_map[normalized_name.lower()] = normalized_name + + result["entity_types"] = processed_entities # 兜底类型定义 person_fallback = { @@ -523,9 +531,9 @@ class OntologyGenerator: needed_slots = len(fallbacks_to_add) # 如果添加后会超过 10 个，需要移除一些现有类型 - if current_count + needed_slots > MAX_ENTITY_TYPES: + if current_count + needed_slots > MAX_ONTOLOGY_TYPES: # 计算需要移除多少个 - to_remove = current_count + needed_slots - MAX_ENTITY_TYPES + to_remove = current_count + needed_slots - MAX_ONTOLOGY_TYPES # 从末尾移除（保留前面更重要的具体类型） result["entity_types"] = result["entity_types"][:-to_remove] @@ -533,11 +541,82 @@ class OntologyGenerator: result["entity_types"].extend(fallbacks_to_add) # 最终确保不超过限制（防御性编程） - if len(result["entity_types"]) > MAX_ENTITY_TYPES: - result["entity_types"] = result["entity_types"][:MAX_ENTITY_TYPES] - - if len(result["edge_types"]) > MAX_EDGE_TYPES: - result["edge_types"] = result["edge_types"][:MAX_EDGE_TYPES] + result["entity_types"] = result["entity_types"][:MAX_ONTOLOGY_TYPES] + + # Resolve edge endpoints only after entity fallback/capping, so an edge + # cannot refer to a type that was removed to satisfy Zep's limits. + valid_entity_names = {entity["name"] for entity in result["entity_types"]} + for name in valid_entity_names: + entity_name_map[name] = name + entity_name_map[name.lower()] = name + + def resolve_entity_name(value: str) -> Optional[str]: + stripped = value.strip() + if stripped == "Entity": + return stripped + mapped = entity_name_map.get(stripped) or entity_name_map.get(stripped.lower()) + if mapped in valid_entity_names: + return mapped + pascal_name = _to_pascal_case(stripped) + return pascal_name if pascal_name in valid_entity_names else None + + processed_edges: List[Dict[str, Any]] = [] + seen_edge_names = set() + for raw_edge in raw_edges: + if isinstance(raw_edge, str): + # A bare edge name has no endpoints and cannot be installed in + # Zep safely. Ignore it instead of inventing a relationship. + logger.warning(f"Ignoring ontology edge without source_targets: {raw_edge}") + continue + elif isinstance(raw_edge, dict): + edge = dict(raw_edge) + else: + logger.warning("Ignoring non-object ontology edge entry") + continue + + original_name = edge.get("name") + if not isinstance(original_name, str) or not original_name.strip(): + logger.warning("Ignoring ontology edge without a usable name") + continue + normalized_name = _to_upper_snake_case(original_name) + if normalized_name == "UNKNOWN" or normalized_name in seen_edge_names: + if normalized_name in seen_edge_names: + logger.warning(f"Duplicate edge type '{normalized_name}' removed during validation") + continue + if normalized_name != original_name: + logger.warning( + f"Edge type name '{original_name}' auto-converted to '{normalized_name}'" + ) + edge["name"] = normalized_name + + normalized_targets = [] + for source_target in normalize_ontology_source_targets( + edge.get("source_targets", []), + limit=None, + ): + source = resolve_entity_name(source_target["source"]) + target = resolve_entity_name(source_target["target"]) + if source and target: + normalized_targets.append({"source": source, "target": target}) + edge["source_targets"] = normalize_ontology_source_targets( + normalized_targets + ) + edge["attributes"] = normalize_ontology_attributes( + edge.get("attributes", []) + ) + description = edge.get("description") + if not isinstance(description, str) or not description: + description = f"A {normalized_name} relationship." + edge["description"] = ( + description[:97] + "..." if len(description) > 100 else description + ) + + seen_edge_names.add(normalized_name) + processed_edges.append(edge) + if len(processed_edges) == MAX_ONTOLOGY_TYPES: + break + + result["edge_types"] = processed_edges return result diff --git a/backend/app/services/simulation_manager.py b/backend/app/services/simulation_manager.py index 76dc0c9a..73c32e0e 100644 --- a/backend/app/services/simulation_manager.py +++ b/backend/app/services/simulation_manager.py @@ -28,6 +28,7 @@ class SimulationStatus(str, Enum): PREPARING = "preparing" READY = "ready" RUNNING = "running" + STOPPING = "stopping" PAUSED = "paused" STOPPED = "stopped" # 模拟被手动停止 COMPLETED = "completed" # 模拟自然完成 diff --git a/backend/app/services/simulation_runner.py b/backend/app/services/simulation_runner.py index e86021f8..b3bdb39b 100644 --- a/backend/app/services/simulation_runner.py +++ b/backend/app/services/simulation_runner.py @@ -45,6 +45,10 @@ class RunnerStatus(str, Enum): FAILED = "failed" +class SimulationStopPending(TimeoutError): + """The monitor still owns a bounded graph-ingestion finalization.""" + + @dataclass class AgentAction: """Agent动作记录""" @@ -226,6 +230,56 @@ class SimulationRunner: # 图谱记忆更新配置 _graph_memory_enabled: Dict[str, bool] = {} # simulation_id -> enabled + _finalization_locks: Dict[str, threading.Lock] = {} + _finalization_locks_guard = threading.Lock() + _manual_stop_requests: set[str] = set() + + @classmethod + def _finalization_lock(cls, simulation_id: str) -> threading.Lock: + with cls._finalization_locks_guard: + return cls._finalization_locks.setdefault( + simulation_id, threading.Lock() + ) + + @classmethod + def _sync_simulation_status( + cls, + simulation_id: str, + runner_status: RunnerStatus, + error: str | None = None, + ) -> None: + """Keep persisted simulation metadata aligned with run_state.json.""" + + from .simulation_manager import SimulationManager, SimulationStatus + + status_map = { + RunnerStatus.RUNNING: SimulationStatus.RUNNING, + RunnerStatus.STOPPING: SimulationStatus.STOPPING, + RunnerStatus.STOPPED: SimulationStatus.STOPPED, + RunnerStatus.COMPLETED: SimulationStatus.COMPLETED, + RunnerStatus.FAILED: SimulationStatus.FAILED, + } + status = status_map.get(runner_status) + if status is None: + return + try: + manager = SimulationManager() + simulation = manager.get_simulation(simulation_id) + if simulation is None: + return + simulation.status = status + simulation.error = error + manager._save_simulation_state(simulation) + except Exception as sync_error: + # state.json is a secondary projection. Never let a projection + # failure skip the authoritative run-state finalization or Zep + # ingestion drain. + logger.error( + "同步模拟状态失败: simulation_id=%s, status=%s, error=%s", + simulation_id, + runner_status.value, + sync_error, + ) @classmethod def get_run_state(cls, simulation_id: str) -> Optional[SimulationRunState]: @@ -331,11 +385,6 @@ class SimulationRunner: Returns: SimulationRunState """ - # 检查是否已在运行 - existing = cls.get_run_state(simulation_id) - if existing and existing.runner_status in [RunnerStatus.RUNNING, RunnerStatus.STARTING]: - raise ValueError(f"模拟已在运行中: {simulation_id}") - # 加载模拟配置 sim_dir = os.path.join(cls.RUN_STATE_DIR, simulation_id) config_path = os.path.join(sim_dir, "simulation_config.json") @@ -367,7 +416,22 @@ class SimulationRunner: started_at=datetime.now().isoformat(), ) - cls._save_run_state(state) + # Atomically claim this simulation ID. The expensive updater/process + # startup happens after releasing the lock, while the persisted + # STARTING state makes every concurrent start fail closed. + with cls._finalization_lock(simulation_id): + existing = cls.get_run_state(simulation_id) + active_statuses = { + RunnerStatus.STARTING, + RunnerStatus.RUNNING, + RunnerStatus.PAUSED, + RunnerStatus.STOPPING, + } + if ( + existing and existing.runner_status in active_statuses + ) or ZepGraphMemoryManager.get_updater(simulation_id) is not None: + raise ValueError(f"模拟已在运行或结束处理中: {simulation_id}") + cls._save_run_state(state) # 如果启用图谱记忆更新，创建更新器 if enable_graph_memory_update: @@ -381,6 +445,16 @@ class SimulationRunner: except Exception as e: logger.error(f"创建图谱记忆更新器失败: {e}") cls._graph_memory_enabled[simulation_id] = False + state.runner_status = RunnerStatus.FAILED + state.error = f"Zep图谱更新器初始化失败: {e}" + with cls._finalization_lock(simulation_id): + cls._save_run_state(state) + cls._sync_simulation_status( + simulation_id, + RunnerStatus.FAILED, + state.error, + ) + raise RuntimeError(state.error) from e else: cls._graph_memory_enabled[simulation_id] = False @@ -399,12 +473,35 @@ class SimulationRunner: script_path = os.path.join(cls.SCRIPTS_DIR, script_name) if not os.path.exists(script_path): - raise ValueError(f"脚本不存在: {script_path}") + cleanup_error = None + if cls._graph_memory_enabled.get(simulation_id, False): + try: + ZepGraphMemoryManager.stop_updater(simulation_id) + cls._graph_memory_enabled.pop(simulation_id, None) + except Exception as error: + cleanup_error = error + state.runner_status = RunnerStatus.FAILED + state.twitter_running = False + state.reddit_running = False + state.error = f"脚本不存在: {script_path}" + if cleanup_error is not None: + state.error += f"; Zep图谱写入清理失败: {cleanup_error}" + with cls._finalization_lock(simulation_id): + cls._save_run_state(state) + cls._sync_simulation_status( + simulation_id, + RunnerStatus.FAILED, + state.error, + ) + raise ValueError(state.error) # 创建动作队列 action_queue = Queue() cls._action_queues[simulation_id] = action_queue - + + process = None + main_log_file = None + # 启动模拟进程 try: # 构建运行命令，使用完整路径 @@ -447,33 +544,70 @@ class SimulationRunner: start_new_session=True, # 创建新进程组，确保服务器关闭时能终止所有相关进程 ) - # 保存文件句柄以便后续关闭 - cls._stdout_files[simulation_id] = main_log_file - cls._stderr_files[simulation_id] = None # 不再需要单独的 stderr - - state.process_pid = process.pid - state.runner_status = RunnerStatus.RUNNING - cls._processes[simulation_id] = process - cls._save_run_state(state) - # Capture locale before spawning monitor thread current_locale = get_locale() - # 启动监控线程 monitor_thread = threading.Thread( target=cls._monitor_simulation, args=(simulation_id, current_locale), daemon=True ) - monitor_thread.start() - cls._monitor_threads[simulation_id] = monitor_thread + + # Atomically publish every resource needed by stop/finalization. + # The monitor is registered before start; if it exits immediately, + # it waits on the same lock until RUNNING is fully visible. + with cls._finalization_lock(simulation_id): + cls._stdout_files[simulation_id] = main_log_file + cls._stderr_files[simulation_id] = None + state.process_pid = process.pid + state.runner_status = RunnerStatus.RUNNING + cls._processes[simulation_id] = process + cls._monitor_threads[simulation_id] = monitor_thread + cls._save_run_state(state) + cls._sync_simulation_status( + simulation_id, + RunnerStatus.RUNNING, + ) + monitor_thread.start() logger.info(f"模拟启动成功: {simulation_id}, pid={process.pid}, platform={platform}") except Exception as e: + cleanup_errors = [] + if process is not None and process.poll() is None: + try: + cls._terminate_process(process, simulation_id) + except Exception as error: + cleanup_errors.append(f"子进程终止失败: {error}") + cls._processes.pop(simulation_id, None) + cls._monitor_threads.pop(simulation_id, None) + cls._action_queues.pop(simulation_id, None) + cls._stdout_files.pop(simulation_id, None) + cls._stderr_files.pop(simulation_id, None) + if main_log_file is not None: + try: + main_log_file.close() + except Exception as error: + cleanup_errors.append(f"日志关闭失败: {error}") + if cls._graph_memory_enabled.get(simulation_id, False): + try: + ZepGraphMemoryManager.stop_updater(simulation_id) + cls._graph_memory_enabled.pop(simulation_id, None) + except Exception as error: + cleanup_errors.append(f"Zep图谱写入清理失败: {error}") state.runner_status = RunnerStatus.FAILED + state.twitter_running = False + state.reddit_running = False state.error = str(e) - cls._save_run_state(state) + if cleanup_errors: + state.error += "; " + "; ".join(cleanup_errors) + with cls._finalization_lock(simulation_id): + cls._save_run_state(state) + cls._sync_simulation_status( + simulation_id, + RunnerStatus.FAILED, + state.error, + ) raise return state @@ -497,6 +631,8 @@ class SimulationRunner: twitter_position = 0 reddit_position = 0 + monitor_error: Exception | None = None + exit_code: int | None = None try: while process.poll() is None: # 进程仍在运行 # 读取 Twitter 动作日志 @@ -521,50 +657,93 @@ class SimulationRunner: if os.path.exists(reddit_actions_log): cls._read_action_log(reddit_actions_log, reddit_position, state, "reddit") - # 进程结束 exit_code = process.returncode - if exit_code == 0: - state.runner_status = RunnerStatus.COMPLETED - state.completed_at = datetime.now().isoformat() - logger.info(f"模拟完成: {simulation_id}") - else: - state.runner_status = RunnerStatus.FAILED - # 从主日志文件读取错误信息 - main_log_path = os.path.join(sim_dir, "simulation.log") - error_info = "" - try: - if os.path.exists(main_log_path): - with open(main_log_path, 'r', encoding='utf-8') as f: - error_info = f.read()[-2000:] # 取最后2000字符 - except Exception: - pass - state.error = f"进程退出码: {exit_code}, 错误: {error_info}" - logger.error(f"模拟失败: {simulation_id}, error={state.error}") - - state.twitter_running = False - state.reddit_running = False - cls._save_run_state(state) - except Exception as e: logger.error(f"监控线程异常: {simulation_id}, error={str(e)}") - state.runner_status = RunnerStatus.FAILED - state.error = str(e) - cls._save_run_state(state) + monitor_error = e finally: - # 停止图谱记忆更新器 - if cls._graph_memory_enabled.get(simulation_id, False): - try: - ZepGraphMemoryManager.stop_updater(simulation_id) - logger.info(f"已停止图谱记忆更新: simulation_id={simulation_id}") - except Exception as e: - logger.error(f"停止图谱记忆更新器失败: {e}") - cls._graph_memory_enabled.pop(simulation_id, None) + # Manual stop and natural completion can observe the same process + # exit. Serialize terminal state and updater drain so only one path + # owns the final result. + with cls._finalization_lock(simulation_id): + latest_state = cls.get_run_state(simulation_id) + if latest_state is not None: + state = latest_state + + if state.runner_status not in { + RunnerStatus.STOPPED, + RunnerStatus.FAILED, + }: + manual_stop = simulation_id in cls._manual_stop_requests + desired_status = ( + RunnerStatus.STOPPED + if manual_stop + else RunnerStatus.COMPLETED + ) + error_message = None + if not manual_stop and monitor_error is not None: + desired_status = RunnerStatus.FAILED + error_message = str(monitor_error) + elif not manual_stop and exit_code != 0: + desired_status = RunnerStatus.FAILED + main_log_path = os.path.join(sim_dir, "simulation.log") + error_info = "" + try: + if os.path.exists(main_log_path): + with open(main_log_path, 'r', encoding='utf-8') as f: + error_info = f.read()[-2000:] + except Exception: + pass + error_message = ( + f"进程退出码: {exit_code}, 错误: {error_info}" + ) + + state.twitter_running = False + state.reddit_running = False + + if cls._graph_memory_enabled.get(simulation_id, False): + # STOPPING is a non-terminal ingestion barrier. The UI + # and report API must not observe COMPLETED until every + # accepted episode is processed by Zep Cloud. + state.runner_status = RunnerStatus.STOPPING + cls._save_run_state(state) + cls._sync_simulation_status( + simulation_id, + RunnerStatus.STOPPING, + ) + try: + ZepGraphMemoryManager.stop_updater(simulation_id) + cls._graph_memory_enabled.pop(simulation_id, None) + logger.info( + "已停止图谱记忆更新: simulation_id=%s", + simulation_id, + ) + except Exception as error: + logger.error(f"停止图谱记忆更新器失败: {error}") + desired_status = RunnerStatus.FAILED + error_message = f"Zep图谱写入未完整完成: {error}" + + state.runner_status = desired_status + state.error = error_message + state.completed_at = datetime.now().isoformat() + cls._save_run_state(state) + cls._sync_simulation_status( + simulation_id, + desired_status, + error_message, + ) + if desired_status == RunnerStatus.COMPLETED: + logger.info(f"模拟完成: {simulation_id}") + else: + logger.error(f"模拟失败: {simulation_id}, error={state.error}") + cls._manual_stop_requests.discard(simulation_id) # 清理进程资源 cls._processes.pop(simulation_id, None) cls._action_queues.pop(simulation_id, None) + cls._monitor_threads.pop(simulation_id, None) # 关闭日志文件句柄 if simulation_id in cls._stdout_files: @@ -635,9 +814,14 @@ class SimulationRunner: # 如果运行了两个平台，需要两个都完成 all_completed = cls._check_all_platforms_completed(state) if all_completed: - state.runner_status = RunnerStatus.COMPLETED - state.completed_at = datetime.now().isoformat() - logger.info(f"所有平台模拟已完成: {state.simulation_id}") + # Platform completion is only an input + # signal. The monitor publishes the + # terminal status after the process has + # exited and Zep ingestion has drained. + logger.info( + f"所有平台已结束，等待进程与图谱写入完成: " + f"{state.simulation_id}" + ) # 更新轮次信息（从 round_end 事件） elif event_type == "round_end": @@ -776,51 +960,125 @@ class SimulationRunner: @classmethod def stop_simulation(cls, simulation_id: str) -> SimulationRunState: """停止模拟""" - state = cls.get_run_state(simulation_id) - if not state: - raise ValueError(f"模拟不存在: {simulation_id}") - - if state.runner_status not in [RunnerStatus.RUNNING, RunnerStatus.PAUSED]: - raise ValueError(f"模拟未在运行: {simulation_id}, status={state.runner_status}") - - state.runner_status = RunnerStatus.STOPPING - cls._save_run_state(state) - - # 终止进程 - process = cls._processes.get(simulation_id) - if process and process.poll() is None: - try: - cls._terminate_process(process, simulation_id) - except ProcessLookupError: - # 进程已经不存在 - pass - except Exception as e: - logger.error(f"终止进程组失败: {simulation_id}, error={e}") - # 回退到直接终止进程 + with cls._finalization_lock(simulation_id): + state = cls.get_run_state(simulation_id) + if not state: + raise ValueError(f"模拟不存在: {simulation_id}") + if state.runner_status == RunnerStatus.STOPPED: + return state + + pending_updater = ZepGraphMemoryManager.get_updater(simulation_id) + retrying_finalization = ( + pending_updater is not None + and state.runner_status in { + RunnerStatus.STOPPING, + RunnerStatus.FAILED, + } + ) + if ( + state.runner_status not in [ + RunnerStatus.STARTING, + RunnerStatus.RUNNING, + RunnerStatus.PAUSED, + RunnerStatus.STOPPING, + ] + and not retrying_finalization + ): + raise ValueError( + f"模拟未在运行: {simulation_id}, status={state.runner_status}" + ) + + state.runner_status = RunnerStatus.STOPPING + cls._manual_stop_requests.add(simulation_id) + cls._save_run_state(state) + cls._sync_simulation_status(simulation_id, RunnerStatus.STOPPING) + + # 终止进程 + process = cls._processes.get(simulation_id) + if process and process.poll() is None: try: - process.terminate() - process.wait(timeout=5) - except Exception: - process.kill() - - state.runner_status = RunnerStatus.STOPPED - state.twitter_running = False - state.reddit_running = False - state.completed_at = datetime.now().isoformat() - cls._save_run_state(state) - - # 停止图谱记忆更新器 - if cls._graph_memory_enabled.get(simulation_id, False): - try: - ZepGraphMemoryManager.stop_updater(simulation_id) - logger.info(f"已停止图谱记忆更新: simulation_id={simulation_id}") - except Exception as e: - logger.error(f"停止图谱记忆更新器失败: {e}") - cls._graph_memory_enabled.pop(simulation_id, None) - + cls._terminate_process(process, simulation_id) + except ProcessLookupError: + pass + except Exception as e: + logger.error(f"终止进程组失败: {simulation_id}, error={e}") + try: + process.terminate() + process.wait(timeout=5) + except Exception: + process.kill() + + # Let the monitor consume the final action-log tail and own the single + # updater drain. It will publish STOPPED (rather than COMPLETED) because + # the manual-stop marker is set above. + monitor = cls._monitor_threads.get(simulation_id) + if ( + not retrying_finalization + and + monitor is not None + and monitor is not threading.current_thread() + and monitor.is_alive() + ): + wait_timeout = max( + 30.0, + Config.ZEP_INGESTION_TIMEOUT_SECONDS + + Config.ZEP_REQUEST_TIMEOUT_SECONDS + + 5, + ) + monitor.join(timeout=wait_timeout) + if monitor.is_alive(): + # The monitor still owns finalization and may be inside one + # bounded HTTP request. Do not block on or overwrite its lock; + # leave the observable state as STOPPING and let polling expose + # the eventual STOPPED/FAILED result. + raise SimulationStopPending( + f"模拟仍在停止中，图谱写入未在 {wait_timeout:.0f}s 内完成" + ) + else: + # Restart recovery or tests may have no monitor thread. Complete + # the same barrier synchronously in this request. + with cls._finalization_lock(simulation_id): + state = cls.get_run_state(simulation_id) or state + if cls._graph_memory_enabled.get(simulation_id, False): + try: + ZepGraphMemoryManager.stop_updater(simulation_id) + cls._graph_memory_enabled.pop(simulation_id, None) + except Exception as error: + state.runner_status = RunnerStatus.FAILED + state.twitter_running = False + state.reddit_running = False + state.completed_at = datetime.now().isoformat() + state.error = f"Zep图谱写入未完整完成: {error}" + cls._save_run_state(state) + cls._sync_simulation_status( + simulation_id, + RunnerStatus.FAILED, + state.error, + ) + raise RuntimeError(state.error) from error + state.runner_status = RunnerStatus.STOPPED + state.twitter_running = False + state.reddit_running = False + state.completed_at = datetime.now().isoformat() + state.error = None + cls._save_run_state(state) + cls._sync_simulation_status( + simulation_id, + RunnerStatus.STOPPED, + ) + cls._manual_stop_requests.discard(simulation_id) + + state = cls.get_run_state(simulation_id) or state + if state.runner_status == RunnerStatus.FAILED: + raise RuntimeError(state.error or "模拟停止失败") + if state.runner_status != RunnerStatus.STOPPED: + raise RuntimeError( + f"模拟停止未达到终态: {simulation_id}, status={state.runner_status}" + ) + logger.info(f"模拟已停止: {simulation_id}") return state - + @classmethod def _read_actions_from_file( cls, @@ -1194,95 +1452,99 @@ class SimulationRunner: if cls._cleanup_done: return cls._cleanup_done = True - - # 检查是否有内容需要清理（避免空进程的进程打印无用日志） - has_processes = bool(cls._processes) - has_updaters = bool(cls._graph_memory_enabled) - - if not has_processes and not has_updaters: - return # 没有需要清理的内容，静默返回 - - logger.info("正在清理所有模拟进程...") - - # 首先停止所有图谱记忆更新器（stop_all 内部会打印日志） - try: - ZepGraphMemoryManager.stop_all() - except Exception as e: - logger.error(f"停止图谱记忆更新器失败: {e}") - cls._graph_memory_enabled.clear() - - # 复制字典以避免在迭代时修改 - processes = list(cls._processes.items()) - - for simulation_id, process in processes: + + updater_ids = set(ZepGraphMemoryManager.get_simulation_ids()) + simulation_ids = sorted( + set(cls._processes) + | set(cls._graph_memory_enabled) + | updater_ids + ) + if not simulation_ids: + return + + logger.info("正在安全完成所有模拟进程与图谱写入...") + cleanup_failed = False + + # Each simulation follows the normal stop/finalization path: terminate + # its producer, let the monitor consume the final action-log tail, and + # only then drain Zep. This avoids dropping actions emitted during + # SIGTERM handling. + for simulation_id in simulation_ids: try: - if process.poll() is None: # 进程仍在运行 - logger.info(f"终止模拟进程: {simulation_id}, pid={process.pid}") - - try: - # 使用跨平台的进程终止方法 + state = cls.get_run_state(simulation_id) + updater = ZepGraphMemoryManager.get_updater(simulation_id) + process = cls._processes.get(simulation_id) + + if state is None: + # Missing/corrupt state is exceptional, but retain the + # critical producer-before-consumer shutdown ordering. + if process is not None and process.poll() is None: cls._terminate_process(process, simulation_id, timeout=5) - except (ProcessLookupError, OSError): - # 进程可能已经不存在，尝试直接终止 - try: - process.terminate() - process.wait(timeout=3) - except Exception: - process.kill() - - # 更新 run_state.json - state = cls.get_run_state(simulation_id) - if state: - state.runner_status = RunnerStatus.STOPPED - state.twitter_running = False - state.reddit_running = False - state.completed_at = datetime.now().isoformat() - state.error = "服务器关闭，模拟被终止" + if updater is not None: + ZepGraphMemoryManager.stop_updater(simulation_id) + continue + + if updater is not None: + cls._graph_memory_enabled[simulation_id] = True + if state.runner_status in { + RunnerStatus.IDLE, + RunnerStatus.STOPPED, + RunnerStatus.COMPLETED, + }: + # A retained updater means the old terminal projection + # was premature. Restore the ingestion barrier first. + state.runner_status = RunnerStatus.STOPPING cls._save_run_state(state) - - # 同时更新 state.json，将状态设为 stopped - try: - sim_dir = os.path.join(cls.RUN_STATE_DIR, simulation_id) - state_file = os.path.join(sim_dir, "state.json") - logger.info(f"尝试更新 state.json: {state_file}") - if os.path.exists(state_file): - with open(state_file, 'r', encoding='utf-8') as f: - state_data = json.load(f) - state_data['status'] = 'stopped' - state_data['updated_at'] = datetime.now().isoformat() - with open(state_file, 'w', encoding='utf-8') as f: - json.dump(state_data, f, indent=2, ensure_ascii=False) - logger.info(f"已更新 state.json 状态为 stopped: {simulation_id}") - else: - logger.warning(f"state.json 不存在: {state_file}") - except Exception as state_err: - logger.warning(f"更新 state.json 失败: {simulation_id}, error={state_err}") - - except Exception as e: - logger.error(f"清理进程失败: {simulation_id}, error={e}") - - # 清理文件句柄 - for simulation_id, file_handle in list(cls._stdout_files.items()): - try: - if file_handle: - file_handle.close() - except Exception: - pass - cls._stdout_files.clear() - - for simulation_id, file_handle in list(cls._stderr_files.items()): - try: - if file_handle: - file_handle.close() - except Exception: - pass - cls._stderr_files.clear() - - # 清理内存中的状态 - cls._processes.clear() - cls._action_queues.clear() - - logger.info("模拟进程清理完成") + cls._sync_simulation_status( + simulation_id, + RunnerStatus.STOPPING, + ) + + needs_finalization = bool( + (process is not None and process.poll() is None) + or updater is not None + or state.runner_status in { + RunnerStatus.STARTING, + RunnerStatus.RUNNING, + RunnerStatus.PAUSED, + RunnerStatus.STOPPING, + } + ) + if needs_finalization: + cls.stop_simulation(simulation_id) + + # A recovery path without a monitor does not run the monitor's + # resource cleanup block. Release only successfully stopped + # resources; FAILED/STOPPING resources remain retryable. + latest = cls.get_run_state(simulation_id) + if latest and latest.runner_status == RunnerStatus.STOPPED: + stopped_process = cls._processes.get(simulation_id) + if stopped_process is None or stopped_process.poll() is not None: + cls._processes.pop(simulation_id, None) + cls._action_queues.pop(simulation_id, None) + cls._monitor_threads.pop(simulation_id, None) + for file_map in (cls._stdout_files, cls._stderr_files): + file_handle = file_map.pop(simulation_id, None) + if file_handle: + try: + file_handle.close() + except Exception: + pass + except Exception as error: + cleanup_failed = True + logger.error( + "清理模拟失败，保留状态以便重试: simulation_id=%s, error=%s", + simulation_id, + error, + ) + + if cleanup_failed: + # Retained updaters and FAILED run states continue to block report + # generation and graph deletion. Permit an explicit retry. + cls._cleanup_done = False + logger.error("部分模拟未安全完成清理") + else: + logger.info("模拟进程与图谱写入清理完成") @classmethod def register_cleanup(cls): @@ -1765,4 +2027,3 @@ class SimulationRunner: results = results[:limit] return results - diff --git a/backend/app/services/zep_entity_reader.py b/backend/app/services/zep_entity_reader.py index d94c6c3f..1e20c6a3 100644 --- a/backend/app/services/zep_entity_reader.py +++ b/backend/app/services/zep_entity_reader.py @@ -3,15 +3,14 @@ Zep实体读取与过滤服务 从Zep图谱中读取节点，筛选出符合预定义实体类型的节点 """ -import time from typing import Dict, Any, List, Optional, Set, Callable, TypeVar from dataclasses import dataclass, field - -from zep_cloud.client import Zep +from zep_cloud import NotFoundError from ..config import Config from ..utils.logger import get_logger from ..utils.zep_paging import fetch_all_nodes, fetch_all_edges +from ..utils.zep import call_zep_read_with_retry, get_zep_client logger = get_logger('mirofish.zep_entity_reader') @@ -83,7 +82,7 @@ class ZepEntityReader: if not self.api_key: raise ValueError("ZEP_API_KEY 未配置") - self.client = Zep(api_key=self.api_key) + self.client = get_zep_client(self.api_key) def _call_with_retry( self, @@ -104,25 +103,12 @@ class ZepEntityReader: Returns: API调用结果 """ - last_exception = None - delay = initial_delay - - for attempt in range(max_retries): - try: - return func() - except Exception as e: - last_exception = e - if attempt < max_retries - 1: - logger.warning( - f"Zep {operation_name} 第 {attempt + 1} 次尝试失败: {str(e)[:100]}, " - f"{delay:.1f}秒后重试..." - ) - time.sleep(delay) - delay *= 2 # 指数退避 - else: - logger.error(f"Zep {operation_name} 在 {max_retries} 次尝试后仍失败: {str(e)}") - - raise last_exception + return call_zep_read_with_retry( + func, + operation_name=operation_name, + max_attempts=max_retries, + initial_delay=initial_delay, + ) def get_all_nodes(self, graph_id: str) -> List[Dict[str, Any]]: """ @@ -209,8 +195,10 @@ class ZepEntityReader: return edges_data except Exception as e: - logger.warning(f"获取节点 {node_uuid} 的边失败: {str(e)}") - return [] + # An empty edge list is valid data. Authentication, permission and + # transport failures must not be made indistinguishable from it. + logger.error(f"获取节点 {node_uuid} 的边失败: {str(e)}") + raise def filter_defined_entities( self, @@ -406,9 +394,14 @@ class ZepEntityReader: related_nodes=related_nodes, ) - except Exception as e: - logger.error(f"获取实体 {entity_uuid} 失败: {str(e)}") + except NotFoundError: return None + except Exception as e: + # Only an actual Zep 404 means "entity not found". Propagate 401, + # 403 and exhausted transport errors so callers cannot prepare a + # simulation with silently incomplete graph context. + logger.error(f"获取实体 {entity_uuid} 失败: {str(e)}") + raise def get_entities_by_type( self, @@ -433,4 +426,3 @@ class ZepEntityReader: enrich_with_edges=enrich_with_edges ) return result.entities - diff --git a/backend/app/services/zep_graph_memory_updater.py b/backend/app/services/zep_graph_memory_updater.py index e034fee2..343ca1f2 100644 --- a/backend/app/services/zep_graph_memory_updater.py +++ b/backend/app/services/zep_graph_memory_updater.py @@ -3,20 +3,17 @@ Zep图谱记忆更新服务 将模拟中的Agent活动动态更新到Zep图谱中 """ -import os import time import threading -import json -from typing import Dict, Any, List, Optional, Callable +from typing import Dict, Any, List, Optional from dataclasses import dataclass from datetime import datetime from queue import Queue, Empty -from zep_cloud.client import Zep - from ..config import Config from ..utils.logger import get_logger from ..utils.locale import get_locale, set_locale +from ..utils.zep import call_zep_read_with_retry, get_zep_client logger = get_logger('mirofish.zep_graph_memory_updater') @@ -58,8 +55,12 @@ class AgentActivity: describe_func = action_descriptions.get(self.action_type, self._describe_generic) description = describe_func() - # 直接返回 "agent名称: 活动描述" 格式，不添加模拟前缀 - return f"{self.agent_name}: {description}" + # Keep the event time in the source text as well as episode metadata so + # temporal extraction does not collapse a multi-action batch. + return ( + f"[{self.timestamp}] [{self.platform} round {self.round_num}] " + f"{self.agent_name}: {description}" + ) def _describe_create_post(self) -> str: content = self.action_args.get("content", "") @@ -199,6 +200,12 @@ class AgentActivity: return f"执行了{self.action_type}操作" +class _DrainDeadlineExceeded(TimeoutError): + def __init__(self, processed_count: int): + super().__init__("Zep updater drain deadline elapsed") + self.processed_count = processed_count + + class ZepGraphMemoryUpdater: """ Zep图谱记忆更新器 @@ -225,11 +232,16 @@ class ZepGraphMemoryUpdater: # 发送间隔（秒），避免请求过快 SEND_INTERVAL = 0.5 - # 重试配置 - MAX_RETRIES = 3 - RETRY_DELAY = 2 # 秒 + # Zep recommends keeping an episode below 10,000 characters. Leave room + # for future source formatting changes. + MAX_EPISODE_CHARS = 9_500 - def __init__(self, graph_id: str, api_key: Optional[str] = None): + def __init__( + self, + graph_id: str, + api_key: Optional[str] = None, + simulation_id: Optional[str] = None, + ): """ 初始化更新器 @@ -238,12 +250,13 @@ class ZepGraphMemoryUpdater: api_key: Zep API Key（可选，默认从配置读取） """ self.graph_id = graph_id + self.simulation_id = simulation_id or "unknown" self.api_key = api_key or Config.ZEP_API_KEY if not self.api_key: raise ValueError("ZEP_API_KEY未配置") - self.client = Zep(api_key=self.api_key) + self.client = get_zep_client(self.api_key) # 活动队列 self._activity_queue: Queue = Queue() @@ -254,6 +267,7 @@ class ZepGraphMemoryUpdater: 'reddit': [], } self._buffer_lock = threading.Lock() + self._acceptance_lock = threading.Lock() # 控制标志 self._running = False @@ -265,6 +279,8 @@ class ZepGraphMemoryUpdater: self._total_items_sent = 0 # 成功发送到Zep的活动条数 self._failed_count = 0 # 发送失败的批次数 self._skipped_count = 0 # 被过滤跳过的活动数（DO_NOTHING） + self._failed_batches: List[Dict[str, Any]] = [] + self._pending_episode_uuids: List[str] = [] logger.info(f"ZepGraphMemoryUpdater 初始化完成: graph_id={graph_id}, batch_size={self.BATCH_SIZE}") @@ -291,14 +307,34 @@ class ZepGraphMemoryUpdater: logger.info(f"ZepGraphMemoryUpdater 已启动: graph_id={self.graph_id}") def stop(self): - """停止后台工作线程""" - self._running = False - - # 发送剩余的活动 - self._flush_remaining() - + """Drain the worker, flush tail events, and wait for Cloud ingestion.""" + deadline = time.time() + Config.ZEP_INGESTION_TIMEOUT_SECONDS + # Serialize the accepting->closed transition with add_activity's + # check+enqueue operation. This closes the small race where a producer + # could enqueue after both the worker and final flush had exited. + with self._acceptance_lock: + self._running = False + if self._worker_thread and self._worker_thread.is_alive(): - self._worker_thread.join(timeout=10) + join_timeout = max(0.0, deadline - time.time()) + self._worker_thread.join(timeout=join_timeout) + if self._worker_thread.is_alive(): + raise TimeoutError( + f"Zep updater worker did not stop within {join_timeout:.0f}s" + ) + + # The worker has drained the queue. Only now is it safe to flush + # buffers; doing this before join loses an item already dequeued by the + # worker but not yet buffered. + self._flush_remaining(deadline=deadline) + + if self._failed_batches: + raise RuntimeError( + f"{len(self._failed_batches)} Zep activity batch(es) failed; " + "simulation graph ingestion is incomplete" + ) + + self._wait_for_pending_episodes(deadline=deadline) logger.info(f"ZepGraphMemoryUpdater 已停止: graph_id={self.graph_id}, " f"total_activities={self._total_activities}, " @@ -332,9 +368,12 @@ class ZepGraphMemoryUpdater: if activity.action_type == "DO_NOTHING": self._skipped_count += 1 return - - self._activity_queue.put(activity) - self._total_activities += 1 + + with self._acceptance_lock: + if not self._running: + raise RuntimeError("Zep graph updater is not running") + self._activity_queue.put(activity) + self._total_activities += 1 logger.debug(f"添加活动到Zep队列: {activity.agent_name} - {activity.action_type}") def add_activity_from_dict(self, data: Dict[str, Any], platform: str): @@ -348,6 +387,9 @@ class ZepGraphMemoryUpdater: # 跳过事件类型的条目 if "event_type" in data: return + if data.get("success") is False: + self._skipped_count += 1 + return activity = AgentActivity( platform=platform, @@ -372,6 +414,7 @@ class ZepGraphMemoryUpdater: # 将活动添加到对应平台的缓冲区 platform = activity.platform.lower() + batch = None with self._buffer_lock: if platform not in self._platform_buffers: self._platform_buffers[platform] = [] @@ -381,10 +424,11 @@ class ZepGraphMemoryUpdater: if len(self._platform_buffers[platform]) >= self.BATCH_SIZE: batch = self._platform_buffers[platform][:self.BATCH_SIZE] self._platform_buffers[platform] = self._platform_buffers[platform][self.BATCH_SIZE:] - # 释放锁后再发送 - self._send_batch_activities(batch, platform) - # 发送间隔，避免请求过快 - time.sleep(self.SEND_INTERVAL) + + # Never hold the buffer lock across network I/O or sleep. + if batch: + self._send_batch_activities(batch, platform) + time.sleep(self.SEND_INTERVAL) except Empty: pass @@ -393,7 +437,41 @@ class ZepGraphMemoryUpdater: logger.error(f"工作循环异常: {e}") time.sleep(1) - def _send_batch_activities(self, activities: List[AgentActivity], platform: str): + def _build_episode_payloads( + self, + activities: List[AgentActivity], + ) -> List[tuple[List[AgentActivity], str]]: + payloads: List[tuple[List[AgentActivity], str]] = [] + current_activities: List[AgentActivity] = [] + current_lines: List[str] = [] + current_length = 0 + + for activity in activities: + text = activity.to_episode_text() + if len(text) > self.MAX_EPISODE_CHARS: + marker = "... [truncated by MiroFish]" + text = text[: self.MAX_EPISODE_CHARS - len(marker)] + marker + projected_length = current_length + (1 if current_lines else 0) + len(text) + if current_lines and projected_length > self.MAX_EPISODE_CHARS: + payloads.append((current_activities, "\n".join(current_lines))) + current_activities = [] + current_lines = [] + current_length = 0 + current_activities.append(activity) + current_lines.append(text) + current_length += (1 if len(current_lines) > 1 else 0) + len(text) + + if current_lines: + payloads.append((current_activities, "\n".join(current_lines))) + return payloads + + def _send_batch_activities( + self, + activities: List[AgentActivity], + platform: str, + *, + deadline: float | None = None, + ) -> int: """ 批量发送活动到Zep图谱（合并为一条文本） @@ -402,37 +480,80 @@ class ZepGraphMemoryUpdater: platform: 平台名称 """ if not activities: - return - - # 将多条活动合并为一条文本，用换行分隔 - episode_texts = [activity.to_episode_text() for activity in activities] - combined_text = "\n".join(episode_texts) - - # 带重试的发送 - for attempt in range(self.MAX_RETRIES): + return 0 + + processed_count = 0 + for payload_activities, combined_text in self._build_episode_payloads(activities): + if deadline is not None and time.time() >= deadline: + raise _DrainDeadlineExceeded(processed_count) try: - self.client.graph.add( + episode = self.client.graph.add( graph_id=self.graph_id, type="text", - data=combined_text + data=combined_text, + created_at=self._to_rfc3339(payload_activities[-1].timestamp), + source_description="MiroFish simulation activity batch", + metadata={ + "source": "mirofish_simulation", + "simulation_id": self.simulation_id, + "platform": platform, + "activity_count": len(payload_activities), + "first_round": min(a.round_num for a in payload_activities), + "last_round": max(a.round_num for a in payload_activities), + "agent_ids": ",".join( + str(value) + for value in sorted({a.agent_id for a in payload_activities}) + ), + "action_types": ",".join( + value + for value in sorted({a.action_type for a in payload_activities}) + if value + ) or "unknown", + }, ) - + + episode_uuid = ( + getattr(episode, "uuid_", None) + or getattr(episode, "uuid", None) + ) + if not episode_uuid: + raise RuntimeError("Zep graph.add returned no episode UUID") + self._pending_episode_uuids.append(str(episode_uuid)) self._total_sent += 1 - self._total_items_sent += len(activities) + self._total_items_sent += len(payload_activities) display_name = self._get_platform_display_name(platform) - logger.info(f"成功批量发送 {len(activities)} 条{display_name}活动到图谱 {self.graph_id}") + logger.info(f"成功批量发送 {len(payload_activities)} 条{display_name}活动到图谱 {self.graph_id}") logger.debug(f"批量内容预览: {combined_text[:200]}...") - return - + except Exception as e: - if attempt < self.MAX_RETRIES - 1: - logger.warning(f"批量发送到Zep失败 (尝试 {attempt + 1}/{self.MAX_RETRIES}): {e}") - time.sleep(self.RETRY_DELAY * (attempt + 1)) - else: - logger.error(f"批量发送到Zep失败，已重试{self.MAX_RETRIES}次: {e}") - self._failed_count += 1 - - def _flush_remaining(self): + # graph.add has no idempotency key. Replaying an ambiguous + # response can duplicate extracted facts, so fail closed and + # surface the incomplete batch to SimulationRunner. + logger.error(f"批量发送到Zep失败，未自动重放非幂等写入: {e}") + self._failed_count += 1 + self._failed_batches.append({ + "platform": platform, + "activities": payload_activities, + "error": str(e), + }) + finally: + # Successes have a confirmed episode UUID; failures are kept + # durably in _failed_batches and must never be replayed. Either + # way this payload is accounted for before moving on. + processed_count += len(payload_activities) + return processed_count + + @staticmethod + def _to_rfc3339(value: str) -> str: + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + if parsed.tzinfo is None: + parsed = parsed.astimezone() + return parsed.isoformat() + except (AttributeError, TypeError, ValueError): + return datetime.now().astimezone().isoformat() + + def _flush_remaining(self, *, deadline: float | None = None): """发送队列和缓冲区中剩余的活动""" # 首先处理队列中剩余的活动，添加到缓冲区 while not self._activity_queue.empty(): @@ -446,16 +567,54 @@ class ZepGraphMemoryUpdater: except Empty: break - # 然后发送各平台缓冲区中剩余的活动（即使不足BATCH_SIZE条） - with self._buffer_lock: - for platform, buffer in self._platform_buffers.items(): - if buffer: - display_name = self._get_platform_display_name(platform) - logger.info(f"发送{display_name}平台剩余的 {len(buffer)} 条活动") - self._send_batch_activities(buffer, platform) - # 清空所有缓冲区 - for platform in self._platform_buffers: - self._platform_buffers[platform] = [] + for platform in list(self._platform_buffers): + with self._buffer_lock: + buffer = list(self._platform_buffers.get(platform, [])) + if not buffer: + continue + display_name = self._get_platform_display_name(platform) + logger.info(f"发送{display_name}平台剩余的 {len(buffer)} 条活动") + if deadline is not None and time.time() >= deadline: + raise TimeoutError( + "Zep updater drain deadline elapsed before flushing all activities" + ) + try: + processed_count = self._send_batch_activities( + buffer, + platform, + deadline=deadline, + ) + except _DrainDeadlineExceeded as error: + with self._buffer_lock: + del self._platform_buffers[platform][:error.processed_count] + raise TimeoutError(str(error)) from error + else: + with self._buffer_lock: + del self._platform_buffers[platform][:processed_count] + + def _wait_for_pending_episodes(self, *, deadline: float | None = None) -> None: + pending = set(self._pending_episode_uuids) + if not pending: + return + + if deadline is None: + deadline = time.time() + Config.ZEP_INGESTION_TIMEOUT_SECONDS + while pending: + if time.time() >= deadline: + raise TimeoutError( + f"Zep simulation ingestion timed out with {len(pending)} " + "episode(s) pending" + ) + for episode_uuid in list(pending): + episode = call_zep_read_with_retry( + lambda: self.client.graph.episode.get(uuid_=episode_uuid), + operation_name=f"poll simulation episode {episode_uuid}", + ) + if getattr(episode, "processed", False): + pending.remove(episode_uuid) + if pending: + time.sleep(3) + self._pending_episode_uuids = [] def get_stats(self) -> Dict[str, Any]: """获取统计信息""" @@ -469,6 +628,7 @@ class ZepGraphMemoryUpdater: "batches_sent": self._total_sent, # 成功发送的批次数 "items_sent": self._total_items_sent, # 成功发送的活动条数 "failed_count": self._failed_count, # 发送失败的批次数 + "pending_episode_count": len(self._pending_episode_uuids), "skipped_count": self._skipped_count, # 被过滤跳过的活动数（DO_NOTHING） "queue_size": self._activity_queue.qsize(), "buffer_sizes": buffer_sizes, # 各平台缓冲区大小 @@ -503,9 +663,13 @@ class ZepGraphMemoryManager: if simulation_id in cls._updaters: cls._updaters[simulation_id].stop() - updater = ZepGraphMemoryUpdater(graph_id) + updater = ZepGraphMemoryUpdater( + graph_id, + simulation_id=simulation_id, + ) updater.start() cls._updaters[simulation_id] = updater + cls._stop_all_done = False logger.info(f"创建图谱记忆更新器: simulation_id={simulation_id}, graph_id={graph_id}") return updater @@ -513,16 +677,68 @@ class ZepGraphMemoryManager: @classmethod def get_updater(cls, simulation_id: str) -> Optional[ZepGraphMemoryUpdater]: """获取模拟的更新器""" - return cls._updaters.get(simulation_id) + with cls._lock: + return cls._updaters.get(simulation_id) + + @classmethod + def get_simulation_ids_for_graph(cls, graph_id: str) -> List[str]: + """Return simulations whose updater still owns or drains this graph.""" + + with cls._lock: + return sorted( + simulation_id + for simulation_id, updater in cls._updaters.items() + if updater.graph_id == graph_id + ) + + @classmethod + def get_simulation_ids(cls) -> List[str]: + """Return every simulation with a retained updater.""" + + with cls._lock: + return sorted(cls._updaters) + + @classmethod + def discard_inactive_updater(cls, simulation_id: str) -> bool: + """Discard a failed, fully stopped updater during graph destruction.""" + + with cls._lock: + updater = cls._updaters.get(simulation_id) + if updater is None: + return False + worker_alive = bool( + updater._worker_thread and updater._worker_thread.is_alive() + ) + if updater._running or worker_alive: + raise RuntimeError( + f"Zep updater for {simulation_id} is still active" + ) + cls._updaters.pop(simulation_id, None) + logger.warning( + "Discarded incomplete Zep updater during explicit graph deletion: " + "simulation_id=%s, graph_id=%s", + simulation_id, + updater.graph_id, + ) + return True @classmethod def stop_updater(cls, simulation_id: str): """停止并移除模拟的更新器""" with cls._lock: - if simulation_id in cls._updaters: - cls._updaters[simulation_id].stop() - del cls._updaters[simulation_id] - logger.info(f"已停止图谱记忆更新器: simulation_id={simulation_id}") + updater = cls._updaters.get(simulation_id) + if updater is None: + return + + # Do not hold the manager lock through up to several minutes of Cloud + # polling. Crucially, only remove the updater after a successful drain; + # on failure it remains visible to report/deletion barriers and can be + # stopped again. + updater.stop() + with cls._lock: + if cls._updaters.get(simulation_id) is updater: + cls._updaters.pop(simulation_id, None) + logger.info(f"已停止图谱记忆更新器: simulation_id={simulation_id}") # 防止 stop_all 重复调用的标志 _stop_all_done = False @@ -533,17 +749,34 @@ class ZepGraphMemoryManager: # 防止重复调用 if cls._stop_all_done: return - cls._stop_all_done = True - + with cls._lock: - if cls._updaters: - for simulation_id, updater in list(cls._updaters.items()): - try: - updater.stop() - except Exception as e: - logger.error(f"停止更新器失败: simulation_id={simulation_id}, error={e}") - cls._updaters.clear() - logger.info("已停止所有图谱记忆更新器") + simulation_ids = list(cls._updaters) + + errors = [] + for simulation_id in simulation_ids: + try: + cls.stop_updater(simulation_id) + except Exception as error: + # Keep a failed updater registered so the caller can retry and + # lifecycle/report guards still see the incomplete ingestion. + logger.error( + "停止更新器失败: simulation_id=%s, error=%s", + simulation_id, + error, + ) + errors.append((simulation_id, error)) + + with cls._lock: + cls._stop_all_done = not cls._updaters + + if errors: + details = "; ".join( + f"{simulation_id}: {error}" + for simulation_id, error in errors + ) + raise RuntimeError(f"部分图谱更新器未完整停止: {details}") + logger.info("已停止所有图谱记忆更新器") @classmethod def get_all_stats(cls) -> Dict[str, Dict[str, Any]]: diff --git a/backend/app/services/zep_tools.py b/backend/app/services/zep_tools.py index 3bc8a57a..7535e394 100644 --- a/backend/app/services/zep_tools.py +++ b/backend/app/services/zep_tools.py @@ -12,14 +12,19 @@ import time import json from typing import Dict, Any, List, Optional from dataclasses import dataclass, field - -from zep_cloud.client import Zep +from zep_cloud import NotFoundError from ..config import Config from ..utils.logger import get_logger from ..utils.llm_client import LLMClient from ..utils.locale import get_locale, t from ..utils.zep_paging import fetch_all_nodes, fetch_all_edges +from ..utils.zep import ( + call_zep_read_with_retry, + get_zep_client, + normalize_zep_search_limit, + normalize_zep_search_query, +) logger = get_logger('mirofish.zep_tools') @@ -427,7 +432,7 @@ class ZepToolsService: if not self.api_key: raise ValueError("ZEP_API_KEY 未配置") - self.client = Zep(api_key=self.api_key) + self.client = get_zep_client(self.api_key) # LLM客户端用于InsightForge生成子问题 self._llm_client = llm_client logger.info(t("console.zepToolsInitialized")) @@ -440,26 +445,14 @@ class ZepToolsService: return self._llm_client def _call_with_retry(self, func, operation_name: str, max_retries: int = None): - """带重试机制的API调用""" - max_retries = max_retries or self.MAX_RETRIES - last_exception = None - delay = self.RETRY_DELAY - - for attempt in range(max_retries): - try: - return func() - except Exception as e: - last_exception = e - if attempt < max_retries - 1: - logger.warning( - t("console.zepRetryAttempt", operation=operation_name, attempt=attempt + 1, error=str(e)[:100], delay=f"{delay:.1f}") - ) - time.sleep(delay) - delay *= 2 - else: - logger.error(t("console.zepAllRetriesFailed", operation=operation_name, retries=max_retries, error=str(e))) - - raise last_exception + """Retry one safe read using typed Zep/HTTPX error classification.""" + + return call_zep_read_with_retry( + func, + operation_name=operation_name, + max_attempts=max_retries or self.MAX_RETRIES, + initial_delay=self.RETRY_DELAY, + ) def search_graph( self, @@ -485,13 +478,15 @@ class ZepToolsService: """ logger.info(t("console.graphSearch", graphId=graph_id, query=query[:50])) - # 尝试使用Zep Cloud Search API + zep_query = normalize_zep_search_query(query) + zep_limit = normalize_zep_search_limit(limit) + try: search_results = self._call_with_retry( func=lambda: self.client.graph.search( graph_id=graph_id, - query=query, - limit=limit, + query=zep_query, + limit=zep_limit, scope=scope, reranker="cross_encoder" ), @@ -539,9 +534,10 @@ class ZepToolsService: ) except Exception as e: - logger.warning(t("console.zepSearchApiFallback", error=str(e))) - # 降级：使用本地关键词匹配搜索 - return self._local_search(graph_id, query, limit, scope) + # Authentication, invalid input, missing graphs, and exhausted + # transient failures must remain visible to the report workflow. + logger.error(t("console.zepSearchApiFallback", error=str(e))) + raise def _local_search( self, @@ -741,9 +737,11 @@ class ZepToolsService: summary=node.summary or "", attributes=node.attributes or {} ) + except NotFoundError: + return None except Exception as e: logger.error(t("console.fetchNodeDetailFailed", error=str(e))) - return None + raise def get_node_edges(self, graph_id: str, node_uuid: str) -> List[EdgeInfo]: """ @@ -774,8 +772,8 @@ class ZepToolsService: return result except Exception as e: - logger.warning(t("console.fetchNodeEdgesFailed", error=str(e))) - return [] + logger.error(t("console.fetchNodeEdgesFailed", error=str(e))) + raise def get_entities_by_type( self, diff --git a/backend/app/utils/ontology.py b/backend/app/utils/ontology.py index d1596c68..632c9995 100644 --- a/backend/app/utils/ontology.py +++ b/backend/app/utils/ontology.py @@ -5,6 +5,7 @@ from typing import Any, Dict, List, Optional MAX_ONTOLOGY_TYPES = 10 MAX_ONTOLOGY_ATTRIBUTES = 10 +MAX_ONTOLOGY_SOURCE_TARGETS = 10 RESERVED_ONTOLOGY_ATTRIBUTE_NAMES = frozenset({ "uuid", "name", @@ -67,3 +68,36 @@ def normalize_ontology_attributes(attributes: Any) -> List[Dict[str, Any]]: normalized_attributes.append(dict(_FALLBACK_ATTRIBUTE)) return normalized_attributes + + +def normalize_ontology_source_targets( + source_targets: Any, + *, + limit: int | None = MAX_ONTOLOGY_SOURCE_TARGETS, +) -> List[Dict[str, str]]: + """Return unique, structurally valid source-target pairs within Zep limits.""" + + if not isinstance(source_targets, list): + return [] + + normalized_targets: List[Dict[str, str]] = [] + seen: set[tuple[str, str]] = set() + for source_target in source_targets: + if not isinstance(source_target, dict): + continue + source = source_target.get("source") + target = source_target.get("target") + if not isinstance(source, str) or not source.strip(): + continue + if not isinstance(target, str) or not target.strip(): + continue + + pair = (source.strip(), target.strip()) + if pair in seen: + continue + seen.add(pair) + normalized_targets.append({"source": pair[0], "target": pair[1]}) + if limit is not None and len(normalized_targets) == limit: + break + + return normalized_targets diff --git a/backend/app/utils/zep.py b/backend/app/utils/zep.py new file mode 100644 index 00000000..16dbe20e --- /dev/null +++ b/backend/app/utils/zep.py @@ -0,0 +1,158 @@ +"""Shared Zep Cloud client, request limits, and retry policy.""" + +from __future__ import annotations + +import os +import time +from functools import lru_cache +from typing import Any, Callable, TypeVar + +import httpx +from zep_cloud.client import Zep +from zep_cloud.core.api_error import ApiError as ZepApiError + +from ..config import Config +from .logger import get_logger + +logger = get_logger("mirofish.zep") + +T = TypeVar("T") + +ZEP_CLOUD_BASE_URL = "https://api.getzep.com/api/v2" +MAX_ZEP_SEARCH_QUERY_CHARS = 400 +MAX_ZEP_SEARCH_RESULTS = 50 + + +def normalize_zep_search_query(query: Any) -> str: + """Return a non-empty query within Zep Cloud's endpoint limit.""" + + if not isinstance(query, str): + raise ValueError("Zep search query must be a string") + normalized = query.strip() + if not normalized: + raise ValueError("Zep search query must not be empty") + return normalized[:MAX_ZEP_SEARCH_QUERY_CHARS] + + +def normalize_zep_search_limit(limit: Any) -> int: + """Clamp a search result limit to the current Zep Cloud contract.""" + + try: + normalized = int(limit) + except (TypeError, ValueError) as exc: + raise ValueError("Zep search limit must be an integer") from exc + if normalized < 1: + raise ValueError("Zep search limit must be at least 1") + return min(normalized, MAX_ZEP_SEARCH_RESULTS) + + +@lru_cache(maxsize=4) +def _cached_zep_client(api_key: str, timeout: float) -> Zep: + return Zep( + api_key=api_key, + base_url=ZEP_CLOUD_BASE_URL, + timeout=timeout, + ) + + +def get_zep_client(api_key: str | None = None, timeout: float | None = None) -> Zep: + """Return a process-shared, explicitly configured Zep Cloud client.""" + + if Config._ZEP_CONFIG_PARSE_ERRORS: + raise ValueError("; ".join(Config._ZEP_CONFIG_PARSE_ERRORS)) + + # zep-cloud gives ZEP_API_URL precedence even when base_url is explicit. + # Reject it so this Cloud-only integration cannot silently target a + # self-hosted or compatibility endpoint. + if os.environ.get("ZEP_API_URL"): + raise ValueError("ZEP_API_URL is unsupported; unset it to use Zep Cloud") + + normalized_key = (api_key or Config.ZEP_API_KEY or "").strip() + if not normalized_key: + raise ValueError("ZEP_API_KEY 未配置") + + request_timeout = float( + timeout if timeout is not None else Config.ZEP_REQUEST_TIMEOUT_SECONDS + ) + if request_timeout <= 0: + raise ValueError("Zep request timeout must be greater than 0") + return _cached_zep_client(normalized_key, request_timeout) + + +def clear_zep_client_cache() -> None: + """Clear cached clients. Intended for tests and controlled reconfiguration.""" + + _cached_zep_client.cache_clear() + + +def is_retryable_zep_error(error: BaseException) -> bool: + """Return whether a failed *read* is safe and useful to retry.""" + + if isinstance(error, (httpx.TimeoutException, httpx.TransportError)): + return True + if isinstance(error, (ConnectionError, TimeoutError, OSError)): + return True + if isinstance(error, ZepApiError): + status_code = error.status_code + return status_code in {408, 429} or ( + status_code is not None and 500 <= status_code <= 599 + ) + return False + + +def _retry_after_seconds(error: BaseException) -> float | None: + if not isinstance(error, ZepApiError) or not error.headers: + return None + value = next( + ( + header_value + for header_name, header_value in error.headers.items() + if header_name.lower() == "retry-after" + ), + None, + ) + if value is None: + return None + try: + return max(0.0, float(value)) + except (TypeError, ValueError): + return None + + +def call_zep_read_with_retry( + operation: Callable[[], T], + *, + operation_name: str, + max_attempts: int = 3, + initial_delay: float = 2.0, + max_delay: float = 60.0, + sleep: Callable[[float], None] = time.sleep, +) -> T: + """Retry a safe Zep read only for transport, 408, 429, or 5xx errors.""" + + if max_attempts < 1: + raise ValueError("max_attempts must be at least 1") + + for attempt in range(1, max_attempts + 1): + try: + return operation() + except Exception as error: + if attempt == max_attempts or not is_retryable_zep_error(error): + raise + + retry_after = _retry_after_seconds(error) + delay = min( + retry_after if retry_after is not None else initial_delay * (2 ** (attempt - 1)), + max_delay, + ) + logger.warning( + "Zep %s attempt %s/%s failed (%s); retrying in %.1fs", + operation_name, + attempt, + max_attempts, + type(error).__name__, + delay, + ) + sleep(delay) + + raise AssertionError("unreachable") diff --git a/backend/app/utils/zep_lifecycle.py b/backend/app/utils/zep_lifecycle.py new file mode 100644 index 00000000..34171fe0 --- /dev/null +++ b/backend/app/utils/zep_lifecycle.py @@ -0,0 +1,51 @@ +"""Process-local lifecycle coordination for Zep Cloud graphs. + +The lock is intentionally keyed by graph ID so graph deletion/reset and a new +simulation updater claim cannot pass each other between validation and their +Cloud mutation. It complements (but does not replace) a distributed lock in +multi-worker deployments. +""" + +import threading + + +_graph_locks: dict[str, threading.RLock] = {} +_graph_locks_guard = threading.Lock() +_graph_readers: dict[str, set[str]] = {} + + +def graph_lifecycle_lock(graph_id: str) -> threading.RLock: + """Return the process-local re-entrant lifecycle lock for ``graph_id``.""" + + if not graph_id: + raise ValueError("graph_id is required for lifecycle locking") + with _graph_locks_guard: + return _graph_locks.setdefault(graph_id, threading.RLock()) + + +def register_graph_reader(graph_id: str, reader_id: str) -> None: + """Register a long-running read lease under the graph lifecycle lock.""" + + if not reader_id: + raise ValueError("reader_id is required") + with graph_lifecycle_lock(graph_id): + _graph_readers.setdefault(graph_id, set()).add(reader_id) + + +def unregister_graph_reader(graph_id: str, reader_id: str) -> None: + """Release a previously registered graph read lease.""" + + with graph_lifecycle_lock(graph_id): + readers = _graph_readers.get(graph_id) + if not readers: + return + readers.discard(reader_id) + if not readers: + _graph_readers.pop(graph_id, None) + + +def get_graph_readers(graph_id: str) -> list[str]: + """Return active reader IDs while serializing with lifecycle mutations.""" + + with graph_lifecycle_lock(graph_id): + return sorted(_graph_readers.get(graph_id, set())) diff --git a/backend/app/utils/zep_paging.py b/backend/app/utils/zep_paging.py index 22c20526..71f4ea58 100644 --- a/backend/app/utils/zep_paging.py +++ b/backend/app/utils/zep_paging.py @@ -1,106 +1,142 @@ -"""Zep Graph 分页读取工具。 - -Zep 的 node/edge 列表接口使用 UUID cursor 分页， -本模块封装自动翻页逻辑（含单页重试），对调用方透明地返回完整列表。 -""" +"""Complete Zep Graph node/edge pagination using opaque response cursors.""" from __future__ import annotations -import time from collections.abc import Callable from typing import Any -from zep_cloud import InternalServerError from zep_cloud.client import Zep from .logger import get_logger +from .zep import call_zep_read_with_retry -logger = get_logger('mirofish.zep_paging') +logger = get_logger("mirofish.zep_paging") _DEFAULT_PAGE_SIZE = 100 -_MAX_NODES = 2000 -_MAX_EDGES = 5000 _DEFAULT_MAX_RETRIES = 3 -_DEFAULT_RETRY_DELAY = 2.0 # seconds, doubles each retry +_DEFAULT_RETRY_DELAY = 2.0 +_NEXT_CURSOR_HEADER = "zep-next-cursor" def _fetch_page_with_retry( - api_call: Callable[..., list[Any]], + api_call: Callable[..., Any], *args: Any, max_retries: int = _DEFAULT_MAX_RETRIES, retry_delay: float = _DEFAULT_RETRY_DELAY, page_description: str = "page", **kwargs: Any, +) -> Any: + """Fetch one read-only page with the shared transient-error policy.""" + + return call_zep_read_with_retry( + lambda: api_call(*args, **kwargs), + operation_name=page_description, + max_attempts=max_retries, + initial_delay=retry_delay, + ) + + +def _header_value(headers: Any, name: str) -> str | None: + if not headers: + return None + direct = headers.get(name) + if direct is not None: + return str(direct) + return next( + ( + str(value) + for header_name, value in headers.items() + if str(header_name).lower() == name.lower() + ), + None, + ) + + +def _fetch_all( + api_call: Callable[..., Any], + graph_id: str, + *, + item_name: str, + page_size: int, + max_items: int | None, + max_retries: int, + retry_delay: float, ) -> list[Any]: - """单页请求，失败时指数退避重试。仅重试网络/IO类瞬态错误。""" - if max_retries < 1: - raise ValueError("max_retries must be >= 1") + if not 1 <= page_size <= 100: + raise ValueError("page_size must be between 1 and 100") + if max_items is not None and max_items < 1: + raise ValueError("max_items must be at least 1 when provided") - last_exception: Exception | None = None - delay = retry_delay + all_items: list[Any] = [] + cursor: str | None = None + seen_cursors: set[str] = set() + page_number = 0 - for attempt in range(max_retries): - try: - return api_call(*args, **kwargs) - except (ConnectionError, TimeoutError, OSError, InternalServerError) as e: - last_exception = e - if attempt < max_retries - 1: - logger.warning( - f"Zep {page_description} attempt {attempt + 1} failed: {str(e)[:100]}, retrying in {delay:.1f}s..." - ) - time.sleep(delay) - delay *= 2 - else: - logger.error(f"Zep {page_description} failed after {max_retries} attempts: {str(e)}") + while True: + kwargs: dict[str, Any] = {"limit": page_size} + if cursor is not None: + kwargs["cursor"] = cursor - assert last_exception is not None - raise last_exception + page_number += 1 + response = _fetch_page_with_retry( + api_call, + graph_id, + max_retries=max_retries, + retry_delay=retry_delay, + page_description=( + f"fetch {item_name} page {page_number} (graph={graph_id})" + ), + **kwargs, + ) + batch = list(getattr(response, "data", None) or []) + all_items.extend(batch) + + if max_items is not None and len(all_items) >= max_items: + if len(all_items) > max_items: + all_items = all_items[:max_items] + logger.warning( + "Zep %s pagination reached explicit max_items=%s for graph %s", + item_name, + max_items, + graph_id, + ) + break + + next_cursor = _header_value( + getattr(response, "headers", None), + _NEXT_CURSOR_HEADER, + ) + if next_cursor is None: + break + if next_cursor in seen_cursors or next_cursor == cursor: + raise RuntimeError( + f"Zep {item_name} pagination cursor did not advance for graph {graph_id}" + ) + seen_cursors.add(next_cursor) + cursor = next_cursor + + return all_items def fetch_all_nodes( client: Zep, graph_id: str, page_size: int = _DEFAULT_PAGE_SIZE, - max_items: int = _MAX_NODES, + max_items: int | None = None, max_retries: int = _DEFAULT_MAX_RETRIES, retry_delay: float = _DEFAULT_RETRY_DELAY, ) -> list[Any]: - """分页获取图谱节点，最多返回 max_items 条（默认 2000）。每页请求自带重试。""" - all_nodes: list[Any] = [] - cursor: str | None = None - page_num = 0 + """Fetch every graph node unless the caller supplies an explicit cap.""" - while True: - kwargs: dict[str, Any] = {"limit": page_size} - if cursor is not None: - kwargs["uuid_cursor"] = cursor - - page_num += 1 - batch = _fetch_page_with_retry( - client.graph.node.get_by_graph_id, - graph_id, - max_retries=max_retries, - retry_delay=retry_delay, - page_description=f"fetch nodes page {page_num} (graph={graph_id})", - **kwargs, - ) - if not batch: - break - - all_nodes.extend(batch) - if len(all_nodes) >= max_items: - all_nodes = all_nodes[:max_items] - logger.warning(f"Node count reached limit ({max_items}), stopping pagination for graph {graph_id}") - break - if len(batch) < page_size: - break - - cursor = getattr(batch[-1], "uuid_", None) or getattr(batch[-1], "uuid", None) - if cursor is None: - logger.warning(f"Node missing uuid field, stopping pagination at {len(all_nodes)} nodes") - break - - return all_nodes + return _fetch_all( + client.graph.node.with_raw_response.get_by_graph_id, + graph_id, + item_name="nodes", + page_size=page_size, + max_items=max_items, + max_retries=max_retries, + retry_delay=retry_delay, + ) def fetch_all_edges( @@ -109,41 +145,16 @@ def fetch_all_edges( page_size: int = _DEFAULT_PAGE_SIZE, max_retries: int = _DEFAULT_MAX_RETRIES, retry_delay: float = _DEFAULT_RETRY_DELAY, - max_items: int = _MAX_EDGES, + max_items: int | None = None, ) -> list[Any]: - """分页获取图谱所有边，最多返回 max_items 条（默认 5000）。每页请求自带重试。""" - all_edges: list[Any] = [] - cursor: str | None = None - page_num = 0 + """Fetch every graph edge unless the caller supplies an explicit cap.""" - while True: - kwargs: dict[str, Any] = {"limit": page_size} - if cursor is not None: - kwargs["uuid_cursor"] = cursor - - page_num += 1 - batch = _fetch_page_with_retry( - client.graph.edge.get_by_graph_id, - graph_id, - max_retries=max_retries, - retry_delay=retry_delay, - page_description=f"fetch edges page {page_num} (graph={graph_id})", - **kwargs, - ) - if not batch: - break - - all_edges.extend(batch) - if len(all_edges) >= max_items: - all_edges = all_edges[:max_items] - logger.warning(f"Edge count reached limit ({max_items}), stopping pagination for graph {graph_id}") - break - if len(batch) < page_size: - break - - cursor = getattr(batch[-1], "uuid_", None) or getattr(batch[-1], "uuid", None) - if cursor is None: - logger.warning(f"Edge missing uuid field, stopping pagination at {len(all_edges)} edges") - break - - return all_edges + return _fetch_all( + client.graph.edge.with_raw_response.get_by_graph_id, + graph_id, + item_name="edges", + page_size=page_size, + max_items=max_items, + max_retries=max_retries, + retry_delay=retry_delay, + ) diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 8c65b729..f972e120 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -17,7 +17,8 @@ dependencies = [ "openai>=1.0.0", # Zep Cloud - "zep-cloud==3.13.0", + "zep-cloud==3.25.0", + "httpx>=0.27.0", # OASIS 社交媒体模拟 "camel-oasis==0.2.5", diff --git a/backend/requirements.txt b/backend/requirements.txt index 4f146296..8fe1327e 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -14,7 +14,8 @@ flask-cors>=6.0.0 openai>=1.0.0 # ============= Zep Cloud ============= -zep-cloud==3.13.0 +zep-cloud==3.25.0 +httpx>=0.27.0 # ============= OASIS 社交媒体模拟 ============= # OASIS 社交模拟框架 diff --git a/backend/tests/test_ontology_attributes.py b/backend/tests/test_ontology_attributes.py index 38e96546..bbc7bd35 100644 --- a/backend/tests/test_ontology_attributes.py +++ b/backend/tests/test_ontology_attributes.py @@ -2,6 +2,7 @@ from app.services.ontology_generator import OntologyGenerator from app.services.graph_builder import GraphBuilderService from app.utils.ontology import ( MAX_ONTOLOGY_ATTRIBUTES, + MAX_ONTOLOGY_SOURCE_TARGETS, normalize_ontology_attribute, normalize_ontology_attributes, ) @@ -168,3 +169,89 @@ def test_graph_builder_passes_an_empty_entity_mapping_for_edge_only_ontology(): }) assert captured["entities"] == {} + + +def test_graph_builder_deduplicates_and_caps_edge_source_targets_for_zep(): + captured = {} + + class GraphApi: + def set_ontology(self, **kwargs): + captured.update(kwargs) + + class Client: + graph = GraphApi() + + source_targets = [ + {"source": f"Source{index}", "target": f"Target{index}"} + for index in range(MAX_ONTOLOGY_SOURCE_TARGETS + 2) + ] + source_targets.insert(1, dict(source_targets[0])) + + builder = object.__new__(GraphBuilderService) + builder.client = Client() + builder.set_ontology("graph-id", { + "entity_types": [], + "edge_types": [{ + "name": "RELATED_TO", + "attributes": ["reason"], + "source_targets": source_targets, + }], + }) + + _, normalized_targets = captured["edges"]["RELATED_TO"] + assert len(normalized_targets) == MAX_ONTOLOGY_SOURCE_TARGETS + assert [(item.source, item.target) for item in normalized_targets] == [ + (f"Source{index}", f"Target{index}") + for index in range(MAX_ONTOLOGY_SOURCE_TARGETS) + ] + + +def test_generator_ignores_invalid_entries_and_normalizes_edge_names(): + source_targets = [ + {"source": "speaker", "target": "news outlet"}, + {"source": "speaker", "target": "news outlet"}, + None, + ] + [ + {"source": "speaker", "target": "news outlet" if index == 0 else "Person"} + for index in range(12) + ] + + result = OntologyGenerator(llm_client=object())._validate_and_process({ + "entity_types": ["speaker", None, 7, {"name": "news outlet"}], + "edge_types": [ + "unusable edge", + None, + {"name": "worksFor", "source_targets": source_targets}, + {"name": "works-for", "source_targets": []}, + ], + }) + + assert [entity["name"] for entity in result["entity_types"][:2]] == [ + "Speaker", + "NewsOutlet", + ] + assert [edge["name"] for edge in result["edge_types"]] == ["WORKS_FOR"] + assert result["edge_types"][0]["source_targets"] == [ + {"source": "Speaker", "target": "NewsOutlet"}, + {"source": "Speaker", "target": "Person"}, + ] + + +def test_generator_caps_after_discarding_invalid_edge_endpoints(): + invalid_first = [ + {"source": f"Removed{index}", "target": "AlsoRemoved"} + for index in range(MAX_ONTOLOGY_SOURCE_TARGETS) + ] + result = OntologyGenerator(llm_client=object())._validate_and_process({ + "entity_types": [{"name": "person"}, {"name": "organization"}], + "edge_types": [{ + "name": "works_for", + "source_targets": invalid_first + [ + {"source": "person", "target": "organization"} + ], + }], + }) + + assert result["edge_types"][0]["source_targets"] == [ + {"source": "Person", "target": "Organization"} + ] diff --git a/backend/tests/test_zep_cloud_contracts.py b/backend/tests/test_zep_cloud_contracts.py new file mode 100644 index 00000000..0d0bad40 --- /dev/null +++ b/backend/tests/test_zep_cloud_contracts.py @@ -0,0 +1,437 @@ +from types import SimpleNamespace +import json + +import httpx +import pytest +from zep_cloud import Zep +from zep_cloud.core.api_error import ApiError as ZepApiError + +from app.services import graph_builder as graph_builder_module +from app.services.graph_builder import BatchSubmission, GraphBuilderService +from app.services.oasis_profile_generator import OasisProfileGenerator +from app.services.zep_entity_reader import EntityNode, ZepEntityReader +from app.services.zep_tools import ZepToolsService + + +def test_report_search_caps_the_query_sent_to_zep(): + calls = [] + + class GraphApi: + def search(self, **kwargs): + calls.append(kwargs) + return SimpleNamespace(edges=[], nodes=[]) + + service = object.__new__(ZepToolsService) + service.client = SimpleNamespace(graph=GraphApi()) + + original_query = "q" * 401 + result = service.search_graph("graph-id", original_query) + + assert calls[0]["query"] == original_query[:400] + assert result.query == original_query + + +def test_profile_context_search_caps_both_queries_sent_to_zep(): + calls = [] + + class GraphApi: + def search(self, **kwargs): + calls.append(kwargs) + return SimpleNamespace(edges=[], nodes=[]) + + generator = object.__new__(OasisProfileGenerator) + generator.zep_client = SimpleNamespace(graph=GraphApi()) + generator.graph_id = "graph-id" + + entity = EntityNode( + uuid="node-id", + name="n" * 500, + labels=["Entity", "Person"], + summary="", + attributes={}, + ) + generator._search_zep_for_entity(entity) + + assert len(calls) == 2 + assert all(0 < len(call["query"]) <= 400 for call in calls) + + +def test_entity_reader_does_not_turn_auth_failure_into_missing_entity(): + def unauthorized(**_kwargs): + raise ZepApiError(status_code=401, body={"message": "unauthorized"}) + + reader = object.__new__(ZepEntityReader) + reader.client = SimpleNamespace( + graph=SimpleNamespace(node=SimpleNamespace(get=unauthorized)) + ) + + with pytest.raises(ZepApiError) as error: + reader.get_entity_with_context("graph-id", "node-id") + + assert error.value.status_code == 401 + + +def test_entity_reader_does_not_turn_edge_failure_into_empty_data(): + def forbidden(**_kwargs): + raise ZepApiError(status_code=403, body={"message": "forbidden"}) + + reader = object.__new__(ZepEntityReader) + reader.client = SimpleNamespace( + graph=SimpleNamespace( + node=SimpleNamespace(get_edges=forbidden), + ) + ) + + with pytest.raises(ZepApiError) as error: + reader.get_node_edges("node-id") + + assert error.value.status_code == 403 + + +def test_report_tools_do_not_turn_zep_read_failures_into_empty_data(): + def unauthorized(**_kwargs): + raise ZepApiError(status_code=401, body={"message": "unauthorized"}) + + service = object.__new__(ZepToolsService) + service.client = SimpleNamespace( + graph=SimpleNamespace(node=SimpleNamespace(get=unauthorized)) + ) + + with pytest.raises(ZepApiError): + service.get_node_detail("node-id") + + service.get_all_edges = lambda _graph_id: (_ for _ in ()).throw( + ZepApiError(status_code=503, body={"message": "unavailable"}) + ) + with pytest.raises(ZepApiError): + service.get_node_edges("graph-id", "node-id") + + +def test_episode_processing_timeout_fails_instead_of_reporting_success(monkeypatch): + builder = object.__new__(GraphBuilderService) + builder.client = SimpleNamespace( + graph=SimpleNamespace( + episode=SimpleNamespace( + get=lambda **_kwargs: SimpleNamespace(processed=False) + ) + ) + ) + + timestamps = iter([0.0, 2.0]) + monkeypatch.setattr(graph_builder_module.time, "time", lambda: next(timestamps)) + monkeypatch.setattr(graph_builder_module.time, "sleep", lambda _seconds: None) + + with pytest.raises(TimeoutError, match="episode"): + builder._wait_for_episodes(["episode-1"], timeout=1) + + +def test_document_ingestion_uses_current_batch_api_and_persists_identity(): + calls = [] + + class BatchApi: + def create(self, **kwargs): + calls.append(("create", kwargs)) + return SimpleNamespace(batch_id="batch-1") + + def add(self, **kwargs): + calls.append(("add", kwargs)) + return [ + SimpleNamespace(episode_uuid=f"episode-{index}") + for index, _item in enumerate(kwargs["items"]) + ] + + def process(self, **kwargs): + calls.append(("process", kwargs)) + return SimpleNamespace(status="queued") + + builder = object.__new__(GraphBuilderService) + builder.client = SimpleNamespace(batch=BatchApi()) + persisted = [] + + submission = builder.add_text_batches( + "graph-id", + ["chunk one", "chunk two"], + batch_created_callback=lambda batch_id, operation_id: persisted.append( + (batch_id, operation_id) + ), + ) + + assert submission.batch_id == "batch-1" + assert submission.item_count == 2 + assert len(submission.operation_id) == 64 + assert persisted == [ + (None, submission.operation_id), + ("batch-1", submission.operation_id), + ] + assert [name for name, _kwargs in calls] == ["create", "add", "process"] + items = calls[1][1]["items"] + assert [item.type for item in items] == ["graph_episode", "graph_episode"] + assert all(item.graph_id == "graph-id" for item in items) + assert all(item.data_type == "text" for item in items) + + +def test_graph_create_persists_identity_before_post_and_reconciles_timeout(): + events = [] + + class GraphApi: + def create(self, **kwargs): + events.append(("create", kwargs["graph_id"])) + raise TimeoutError("response lost") + + def get(self, graph_id): + events.append(("get", graph_id)) + return SimpleNamespace(graph_id=graph_id) + + builder = object.__new__(GraphBuilderService) + builder.client = SimpleNamespace(graph=GraphApi()) + + graph_id = builder.create_graph( + "Graph", + graph_id="known-id", + graph_id_callback=lambda value: events.append(("persist", value)), + ) + + assert graph_id == "known-id" + assert events == [ + ("persist", "known-id"), + ("create", "known-id"), + ("get", "known-id"), + ] + + +def test_batch_create_timeout_is_reconciled_by_operation_metadata(monkeypatch): + calls = [] + list_count = 0 + + class BatchApi: + def create(self, **_kwargs): + calls.append("create") + raise TimeoutError("response lost") + + def list(self, **_kwargs): + nonlocal list_count + calls.append("list") + list_count += 1 + if list_count == 1: + return SimpleNamespace(batches=[], next_cursor=None) + return SimpleNamespace( + batches=[SimpleNamespace( + batch_id="batch-recovered", + metadata={ + "mirofish_operation_id": GraphBuilderService.build_operation_id( + "graph-id", ["chunk"] + ), + "graph_id": "graph-id", + }, + )], + next_cursor=None, + ) + + def add(self, **kwargs): + calls.append("add") + return [SimpleNamespace(episode_uuid="episode-1")] + + def process(self, **_kwargs): + calls.append("process") + return SimpleNamespace(status="queued") + + builder = object.__new__(GraphBuilderService) + builder.client = SimpleNamespace(batch=BatchApi()) + monkeypatch.setattr(graph_builder_module.time, "sleep", lambda _seconds: None) + + submission = builder.add_text_batches("graph-id", ["chunk"]) + + assert submission.batch_id == "batch-recovered" + assert calls == ["create", "list", "list", "add", "process"] + + +def test_batch_add_timeout_recovers_a_fully_accepted_group_without_replay(monkeypatch): + add_calls = [] + list_calls = [] + + class BatchApi: + def create(self, **_kwargs): + return SimpleNamespace(batch_id="batch-1") + + def add(self, **_kwargs): + add_calls.append(True) + raise TimeoutError("response lost") + + def list_items(self, **_kwargs): + list_calls.append(True) + if len(list_calls) == 1: + return SimpleNamespace(items=[], next_cursor=None) + return SimpleNamespace( + items=[ + SimpleNamespace(sequence_index=0, episode_uuid="episode-1"), + SimpleNamespace(sequence_index=1, episode_uuid="episode-2"), + ], + next_cursor=None, + ) + + def process(self, **_kwargs): + return SimpleNamespace(status="queued") + + builder = object.__new__(GraphBuilderService) + builder.client = SimpleNamespace(batch=BatchApi()) + monkeypatch.setattr(graph_builder_module.time, "sleep", lambda _seconds: None) + + submission = builder.add_text_batches( + "graph-id", ["chunk one", "chunk two"] + ) + + assert submission.item_count == 2 + assert add_calls == [True] + assert len(list_calls) == 2 + + +def test_batch_wait_validates_terminal_items_and_opaque_zero_cursor(): + list_calls = [] + + class BatchApi: + def get(self, **_kwargs): + return SimpleNamespace( + status="succeeded", + progress=SimpleNamespace( + percent_complete=100, + succeeded_items=2, + ), + ) + + def list_items(self, **kwargs): + list_calls.append(kwargs) + if kwargs["cursor"] is None: + return SimpleNamespace( + items=[SimpleNamespace( + sequence_index=0, + status="succeeded", + episode_uuid="episode-1", + source_uuid="episode-1", + )], + next_cursor=0, + ) + return SimpleNamespace( + items=[SimpleNamespace( + sequence_index=1, + status="succeeded", + episode_uuid="episode-2", + source_uuid="episode-2", + )], + next_cursor=None, + ) + + builder = object.__new__(GraphBuilderService) + builder.client = SimpleNamespace(batch=BatchApi()) + submission = BatchSubmission("batch-1", "operation", [], 2) + + assert builder._wait_for_batch(submission, timeout=1) == [ + "episode-1", + "episode-2", + ] + assert [call["cursor"] for call in list_calls] == [None, 0] + + +@pytest.mark.parametrize("status", ["partial", "failed", "invalid", "canceled"]) +def test_batch_non_success_terminal_states_fail(status): + builder = object.__new__(GraphBuilderService) + builder.client = SimpleNamespace( + batch=SimpleNamespace( + get=lambda **_kwargs: SimpleNamespace(status=status, progress=None), + list_items=lambda **_kwargs: SimpleNamespace( + items=[SimpleNamespace(status="failed", error={"message": "bad"})], + next_cursor=None, + ), + ) + ) + + with pytest.raises(RuntimeError, match=status): + builder._wait_for_batch( + BatchSubmission("batch-1", "operation", [], 1), + timeout=1, + ) + + +def test_batch_wait_times_out_while_status_remains_nonterminal(monkeypatch): + builder = object.__new__(GraphBuilderService) + builder.client = SimpleNamespace( + batch=SimpleNamespace( + get=lambda **_kwargs: SimpleNamespace(status="processing", progress=None) + ) + ) + timestamps = iter([0.0, 2.0]) + monkeypatch.setattr(graph_builder_module.time, "time", lambda: next(timestamps)) + monkeypatch.setattr(graph_builder_module.time, "sleep", lambda _seconds: None) + + with pytest.raises(TimeoutError, match="batch-1"): + builder._wait_for_batch( + BatchSubmission("batch-1", "operation", [], 1), + timeout=1, + ) + + +def test_installed_sdk_serializes_the_batch_325_contract(): + requests = [] + + def handler(request): + requests.append((request.method, request.url.path, request.content)) + path = request.url.path + if path.endswith("/batches") and request.method == "POST": + return httpx.Response( + 200, + json={"batch_id": "batch-1", "status": "draft", "item_count": 0}, + ) + if path.endswith("/batches/batch-1/items") and request.method == "POST": + return httpx.Response(200, json=[{ + "item_id": "item-1", + "sequence_index": 0, + "status": "pending", + "episode_uuid": "episode-1", + "source_uuid": "episode-1", + }]) + if path.endswith("/batches/batch-1/process"): + return httpx.Response( + 200, + json={"batch_id": "batch-1", "status": "queued", "item_count": 1}, + ) + if path.endswith("/batches/batch-1"): + return httpx.Response(200, json={ + "batch_id": "batch-1", + "status": "succeeded", + "item_count": 1, + "progress": {"percent_complete": 100, "succeeded_items": 1}, + }) + if path.endswith("/batches/batch-1/items") and request.method == "GET": + return httpx.Response(200, json={ + "items": [{ + "item_id": "item-1", + "sequence_index": 0, + "status": "succeeded", + "episode_uuid": "episode-1", + "source_uuid": "episode-1", + }], + "next_cursor": None, + }) + raise AssertionError(f"Unexpected request: {request.method} {path}") + + with httpx.Client(transport=httpx.MockTransport(handler)) as transport_client: + builder = object.__new__(GraphBuilderService) + builder.client = Zep(api_key="test-key", httpx_client=transport_client) + submission = builder.add_text_batches("graph-id", ["source chunk"]) + assert builder._wait_for_batch(submission, timeout=1) == ["episode-1"] + + assert [(method, path) for method, path, _body in requests] == [ + ("POST", "/api/v2/batches"), + ("POST", "/api/v2/batches/batch-1/items"), + ("POST", "/api/v2/batches/batch-1/process"), + ("GET", "/api/v2/batches/batch-1"), + ("GET", "/api/v2/batches/batch-1/items"), + ] + add_payload = json.loads(requests[1][2]) + assert add_payload["items"][0] == { + "data": "source chunk", + "data_type": "text", + "graph_id": "graph-id", + "metadata": add_payload["items"][0]["metadata"], + "source_description": "MiroFish source document chunk", + "type": "graph_episode", + } diff --git a/backend/tests/test_zep_edge_paging.py b/backend/tests/test_zep_edge_paging.py index d0e92ff4..d69ed0f1 100644 --- a/backend/tests/test_zep_edge_paging.py +++ b/backend/tests/test_zep_edge_paging.py @@ -4,14 +4,27 @@ from app.utils import zep_paging def _client(): - edge_api = SimpleNamespace(get_by_graph_id=lambda *args, **kwargs: []) + edge_api = SimpleNamespace( + with_raw_response=SimpleNamespace( + get_by_graph_id=lambda *args, **kwargs: SimpleNamespace( + data=[], + headers={}, + ) + ) + ) return SimpleNamespace(graph=SimpleNamespace(edge=edge_api)) def test_edge_cap_stops_pagination_at_requested_limit(monkeypatch): pages = [ - [SimpleNamespace(uuid_="e1"), SimpleNamespace(uuid_="e2")], - [SimpleNamespace(uuid_="e3"), SimpleNamespace(uuid_="e4")], + SimpleNamespace( + data=[SimpleNamespace(uuid_="e1"), SimpleNamespace(uuid_="e2")], + headers={"Zep-Next-Cursor": "opaque-page-2"}, + ), + SimpleNamespace( + data=[SimpleNamespace(uuid_="e3"), SimpleNamespace(uuid_="e4")], + headers={}, + ), ] calls = [] @@ -25,7 +38,7 @@ def test_edge_cap_stops_pagination_at_requested_limit(monkeypatch): assert [edge.uuid_ for edge in result] == ["e1", "e2", "e3"] assert len(calls) == 2 - assert calls[1]["uuid_cursor"] == "e2" + assert calls[1]["cursor"] == "opaque-page-2" def test_existing_positional_retry_arguments_keep_their_meaning(monkeypatch): @@ -33,7 +46,7 @@ def test_existing_positional_retry_arguments_keep_their_meaning(monkeypatch): def fake_fetch(*args, **kwargs): observed.update(kwargs) - return [] + return SimpleNamespace(data=[], headers={}) monkeypatch.setattr(zep_paging, "_fetch_page_with_retry", fake_fetch) @@ -42,3 +55,58 @@ def test_existing_positional_retry_arguments_keep_their_meaning(monkeypatch): assert observed["limit"] == 25 assert observed["max_retries"] == 7 assert observed["retry_delay"] == 0.25 + + +def test_pagination_uses_the_current_opaque_response_cursor(): + calls = [] + + class RawEdgeApi: + def get_by_graph_id(self, graph_id, **kwargs): + calls.append((graph_id, kwargs)) + if kwargs.get("cursor") is None: + return SimpleNamespace( + data=[SimpleNamespace(uuid_="e1"), SimpleNamespace(uuid_="e2")], + headers={"Zep-Next-Cursor": "opaque-page-2"}, + ) + return SimpleNamespace( + data=[SimpleNamespace(uuid_="e3")], + headers={}, + ) + + class EdgeApi: + with_raw_response = RawEdgeApi() + + def get_by_graph_id(self, *_args, **_kwargs): + raise AssertionError("pagination must read the response cursor header") + + client = SimpleNamespace( + graph=SimpleNamespace(edge=EdgeApi()) + ) + + result = zep_paging.fetch_all_edges(client, "graph", page_size=2) + + assert [edge.uuid_ for edge in result] == ["e1", "e2", "e3"] + assert calls == [ + ("graph", {"limit": 2}), + ("graph", {"limit": 2, "cursor": "opaque-page-2"}), + ] + + +def test_pagination_fails_if_the_service_repeats_a_cursor(): + class RawEdgeApi: + def get_by_graph_id(self, _graph_id, **_kwargs): + return SimpleNamespace( + data=[SimpleNamespace(uuid_="e1")], + headers={"Zep-Next-Cursor": "same-cursor"}, + ) + + client = SimpleNamespace( + graph=SimpleNamespace( + edge=SimpleNamespace(with_raw_response=RawEdgeApi()) + ) + ) + + import pytest + + with pytest.raises(RuntimeError, match="did not advance"): + zep_paging.fetch_all_edges(client, "graph", page_size=1) diff --git a/backend/tests/test_zep_graph_lifecycle.py b/backend/tests/test_zep_graph_lifecycle.py new file mode 100644 index 00000000..947ec0f2 --- /dev/null +++ b/backend/tests/test_zep_graph_lifecycle.py @@ -0,0 +1,418 @@ +from datetime import datetime +import threading + +from flask import Flask +from types import SimpleNamespace + +from app.api import graph as graph_api +from app.api import simulation as simulation_api +from app.models.project import Project, ProjectStatus +from app.services.simulation_manager import SimulationStatus +from app.models.task import TaskStatus + + +def _project(status, graph_id="graph-1"): + now = datetime.now().isoformat() + return Project( + project_id="proj-1", + name="Project", + status=status, + created_at=now, + updated_at=now, + ontology={"entity_types": [], "edge_types": []}, + graph_id=graph_id, + graph_build_task_id="task-1", + zep_batch_id="batch-1", + zep_batch_operation_id="operation-1", + ) + + +def _json_result(result): + if isinstance(result, tuple): + response, status = result + else: + response, status = result, result.status_code + return response.get_json(), status + + +def test_project_reset_deletes_the_cloud_graph_before_clearing_reference(monkeypatch): + project = _project(ProjectStatus.GRAPH_COMPLETED) + events = [] + + class Builder: + def __init__(self, **_kwargs): + pass + + def delete_graph(self, graph_id): + events.append(("cloud-delete", graph_id)) + + monkeypatch.setattr(graph_api, "GraphBuilderService", Builder) + monkeypatch.setattr(graph_api.Config, "ZEP_API_KEY", "test-key") + monkeypatch.setattr( + graph_api.ProjectManager, + "get_project", + classmethod(lambda _cls, _project_id: project), + ) + monkeypatch.setattr( + graph_api.ProjectManager, + "save_project", + classmethod(lambda _cls, saved: events.append(("save", saved.graph_id))), + ) + + app = Flask(__name__) + with app.test_request_context("/api/graph/project/proj-1/reset", method="POST"): + body, status = _json_result(graph_api.reset_project("proj-1")) + + assert status == 200 + assert body["success"] is True + assert events == [("cloud-delete", "graph-1"), ("save", None)] + assert project.zep_batch_id is None + assert project.status == ProjectStatus.ONTOLOGY_GENERATED + + +def test_project_reset_refuses_a_graph_with_an_active_simulation(monkeypatch): + project = _project(ProjectStatus.GRAPH_COMPLETED) + monkeypatch.setattr(graph_api.Config, "ZEP_API_KEY", "test-key") + monkeypatch.setattr( + graph_api.ProjectManager, + "get_project", + classmethod(lambda _cls, _project_id: project), + ) + monkeypatch.setattr( + graph_api.ZepGraphMemoryManager, + "get_simulation_ids_for_graph", + classmethod(lambda _cls, _graph_id: ["sim-active"]), + ) + + app = Flask(__name__) + with app.test_request_context("/api/graph/project/proj-1/reset", method="POST"): + body, status = _json_result(graph_api.reset_project("proj-1")) + + assert status == 409 + assert "sim-active" in body["error"] + + +def test_graph_delete_cannot_discard_an_updater_during_finalization(monkeypatch): + monkeypatch.setattr( + graph_api.ZepGraphMemoryManager, + "get_simulation_ids_for_graph", + classmethod(lambda _cls, _graph_id: ["sim-finalizing"]), + ) + discarded = [] + monkeypatch.setattr( + graph_api.ZepGraphMemoryManager, + "discard_inactive_updater", + classmethod( + lambda _cls, simulation_id: discarded.append(simulation_id) + ), + ) + lock = graph_api.SimulationRunner._finalization_lock("sim-finalizing") + lock.acquire() + try: + assert graph_api._active_graph_consumers("graph-1") == ["sim-finalizing"] + finally: + lock.release() + + assert discarded == [] + + +def test_repeated_build_request_reuses_the_existing_task(monkeypatch): + project = _project(ProjectStatus.GRAPH_BUILDING) + monkeypatch.setattr(graph_api.Config, "ZEP_API_KEY", "test-key") + monkeypatch.setattr( + graph_api.ProjectManager, + "get_project", + classmethod(lambda _cls, _project_id: project), + ) + monkeypatch.setattr( + graph_api, + "TaskManager", + lambda: SimpleNamespace( + get_task=lambda _task_id: SimpleNamespace(status=TaskStatus.PROCESSING) + ), + ) + + app = Flask(__name__) + with app.test_request_context( + "/api/graph/build", + method="POST", + json={"project_id": "proj-1", "force": True}, + ): + body, status = _json_result(graph_api.build_graph()) + + assert status == 200 + assert body["success"] is True + assert body["data"]["reused"] is True + assert body["data"]["task_id"] == "task-1" + assert body["data"]["graph_id"] == "graph-1" + + +def test_stale_build_after_restart_is_recoverable_instead_of_reused(monkeypatch): + project = _project(ProjectStatus.GRAPH_BUILDING) + project.zep_batch_id = None + project.zep_batch_operation_id = None + saved = [] + monkeypatch.setattr(graph_api.Config, "ZEP_API_KEY", "test-key") + monkeypatch.setattr( + graph_api.ProjectManager, + "get_project", + classmethod(lambda _cls, _project_id: project), + ) + monkeypatch.setattr( + graph_api.ProjectManager, + "save_project", + classmethod(lambda _cls, value: saved.append(value.status)), + ) + monkeypatch.setattr( + graph_api, + "TaskManager", + lambda: SimpleNamespace(get_task=lambda _task_id: None), + ) + + app = Flask(__name__) + with app.test_request_context( + "/api/graph/build", + method="POST", + json={"project_id": "proj-1"}, + ): + body, status = _json_result(graph_api.build_graph()) + + assert status == 409 + assert body["recoverable"] is True + assert project.status == ProjectStatus.FAILED + assert saved == [ProjectStatus.FAILED] + + +def test_stale_build_resumes_a_persisted_processing_batch(monkeypatch): + project = _project(ProjectStatus.GRAPH_BUILDING) + created_threads = [] + + class Tasks: + def get_task(self, _task_id): + return None + + def create_task(self, _description): + return "task-resumed" + + class Builder: + def __init__(self, **_kwargs): + pass + + def get_batch_summary(self, batch_id): + assert batch_id == "batch-1" + return SimpleNamespace(status="processing") + + class Thread: + def __init__(self, *, target, daemon): + created_threads.append((target, daemon)) + + def start(self): + pass + + monkeypatch.setattr(graph_api.Config, "ZEP_API_KEY", "test-key") + monkeypatch.setattr(graph_api, "TaskManager", Tasks) + monkeypatch.setattr(graph_api, "GraphBuilderService", Builder) + monkeypatch.setattr(graph_api.threading, "Thread", Thread) + monkeypatch.setattr( + graph_api.ProjectManager, + "get_project", + classmethod(lambda _cls, _project_id: project), + ) + monkeypatch.setattr( + graph_api.ProjectManager, + "get_extracted_text", + classmethod(lambda _cls, _project_id: "source text"), + ) + monkeypatch.setattr( + graph_api.ProjectManager, + "save_project", + classmethod(lambda _cls, _project: None), + ) + + app = Flask(__name__) + with app.test_request_context( + "/api/graph/build", + method="POST", + json={"project_id": "proj-1"}, + ): + body, status = _json_result(graph_api.build_graph()) + + assert status == 200 + assert body["data"]["resumed"] is True + assert body["data"]["task_id"] == "task-resumed" + assert project.graph_build_task_id == "task-resumed" + assert len(created_threads) == 1 + + +def test_project_delete_removes_cloud_graph_before_local_files(monkeypatch): + project = _project(ProjectStatus.GRAPH_COMPLETED) + events = [] + + class Builder: + def __init__(self, **_kwargs): + pass + + def delete_graph(self, graph_id): + events.append(("cloud-delete", graph_id)) + + monkeypatch.setattr(graph_api, "GraphBuilderService", Builder) + monkeypatch.setattr(graph_api.Config, "ZEP_API_KEY", "test-key") + monkeypatch.setattr( + graph_api.ProjectManager, + "get_project", + classmethod(lambda _cls, _project_id: project), + ) + monkeypatch.setattr( + graph_api.ProjectManager, + "delete_project", + classmethod( + lambda _cls, project_id: events.append(("local-delete", project_id)) or True + ), + ) + + app = Flask(__name__) + with app.test_request_context( + "/api/graph/project/proj-1", + method="DELETE", + ): + body, status = _json_result(graph_api.delete_project("proj-1")) + + assert status == 200 + assert body["success"] is True + assert events == [ + ("cloud-delete", "graph-1"), + ("local-delete", "proj-1"), + ] + + +def test_completed_build_request_is_idempotent_without_force(monkeypatch): + project = _project(ProjectStatus.GRAPH_COMPLETED) + monkeypatch.setattr(graph_api.Config, "ZEP_API_KEY", "test-key") + monkeypatch.setattr( + graph_api.ProjectManager, + "get_project", + classmethod(lambda _cls, _project_id: project), + ) + + app = Flask(__name__) + with app.test_request_context( + "/api/graph/build", + method="POST", + json={"project_id": "proj-1"}, + ): + body, status = _json_result(graph_api.build_graph()) + + assert status == 200 + assert body["data"]["reused"] is True + assert body["data"]["graph_id"] == "graph-1" + + +def test_force_must_be_a_json_boolean(monkeypatch): + project = _project(ProjectStatus.GRAPH_COMPLETED) + monkeypatch.setattr(graph_api.Config, "ZEP_API_KEY", "test-key") + monkeypatch.setattr( + graph_api.ProjectManager, + "get_project", + classmethod(lambda _cls, _project_id: project), + ) + + app = Flask(__name__) + with app.test_request_context( + "/api/graph/build", + method="POST", + json={"project_id": "proj-1", "force": "false"}, + ): + body, status = _json_result(graph_api.build_graph()) + + assert status == 400 + assert "boolean" in body["error"] + + +def test_graph_reset_and_memory_start_cannot_cross_between_delete_and_clear( + monkeypatch, +): + project = _project(ProjectStatus.GRAPH_COMPLETED) + simulation = SimpleNamespace( + simulation_id="sim-1", + project_id=project.project_id, + graph_id=project.graph_id, + status=SimulationStatus.READY, + ) + delete_entered = threading.Event() + allow_delete = threading.Event() + runner_called = [] + + class Builder: + def __init__(self, **_kwargs): + pass + + def delete_graph(self, graph_id): + assert graph_id == "graph-1" + delete_entered.set() + assert allow_delete.wait(timeout=2) + + class Simulations: + def get_simulation(self, _simulation_id): + return simulation + + monkeypatch.setattr(graph_api, "GraphBuilderService", Builder) + monkeypatch.setattr(graph_api.Config, "ZEP_API_KEY", "test-key") + monkeypatch.setattr( + graph_api.ProjectManager, + "get_project", + classmethod(lambda _cls, _project_id: project), + ) + monkeypatch.setattr( + graph_api.ProjectManager, + "save_project", + classmethod(lambda _cls, _project: None), + ) + monkeypatch.setattr(simulation_api, "SimulationManager", Simulations) + monkeypatch.setattr( + simulation_api.ProjectManager, + "get_project", + classmethod(lambda _cls, _project_id: project), + ) + monkeypatch.setattr( + simulation_api.SimulationRunner, + "start_simulation", + classmethod(lambda _cls, **_kwargs: runner_called.append(True)), + ) + + app = Flask(__name__) + results = {} + + def reset(): + with app.test_request_context( + "/api/graph/project/proj-1/reset", method="POST" + ): + results["reset"] = _json_result(graph_api.reset_project("proj-1")) + + def start(): + with app.test_request_context( + "/api/simulation/start", + method="POST", + json={ + "simulation_id": "sim-1", + "enable_graph_memory_update": True, + }, + ): + results["start"] = _json_result(simulation_api.start_simulation()) + + reset_thread = threading.Thread(target=reset) + reset_thread.start() + assert delete_entered.wait(timeout=2) + + start_thread = threading.Thread(target=start) + start_thread.start() + start_thread.join(timeout=0.05) + assert start_thread.is_alive() + + allow_delete.set() + reset_thread.join(timeout=2) + start_thread.join(timeout=2) + + assert results["reset"][1] == 200 + assert results["start"][1] == 409 + assert runner_called == [] + assert project.graph_id is None diff --git a/backend/tests/test_zep_graph_memory_updater.py b/backend/tests/test_zep_graph_memory_updater.py new file mode 100644 index 00000000..5f5531b0 --- /dev/null +++ b/backend/tests/test_zep_graph_memory_updater.py @@ -0,0 +1,237 @@ +from types import SimpleNamespace +import threading +from queue import Queue + +import pytest + +from app.services import zep_graph_memory_updater as updater_module +from app.services.zep_graph_memory_updater import ( + AgentActivity, + ZepGraphMemoryManager, + ZepGraphMemoryUpdater, +) + + +def _activity(index=1, content="hello"): + return AgentActivity( + platform="twitter", + agent_id=index, + agent_name=f"Agent {index}", + action_type="CREATE_POST", + action_args={"content": content}, + round_num=index, + timestamp="2026-07-22T12:00:00+08:00", + ) + + +def _client(add): + return SimpleNamespace( + graph=SimpleNamespace( + add=add, + episode=SimpleNamespace( + get=lambda **_kwargs: SimpleNamespace(processed=True) + ), + ) + ) + + +def _updater(monkeypatch, add, simulation_id="sim-1"): + client = _client(add) + monkeypatch.setattr(updater_module, "get_zep_client", lambda _key: client) + updater = ZepGraphMemoryUpdater( + "graph-1", + api_key="test-key", + simulation_id=simulation_id, + ) + updater.SEND_INTERVAL = 0 + return updater + + +def test_stop_drains_an_immediately_queued_tail_activity(monkeypatch): + writes = [] + updater = _updater( + monkeypatch, + lambda **kwargs: writes.append(kwargs) or SimpleNamespace(uuid_="episode-1"), + ) + + updater.start() + updater.add_activity(_activity()) + updater.stop() + + assert len(writes) == 1 + assert updater.get_stats()["items_sent"] == 1 + assert updater.get_stats()["queue_size"] == 0 + + +def test_network_write_happens_outside_the_buffer_lock(monkeypatch): + lock_was_available = [] + updater = None + + def add(**_kwargs): + acquired = updater._buffer_lock.acquire(blocking=False) + lock_was_available.append(acquired) + if acquired: + updater._buffer_lock.release() + return SimpleNamespace(uuid_="episode-1") + + updater = _updater(monkeypatch, add) + updater.start() + for index in range(updater.BATCH_SIZE): + updater.add_activity(_activity(index)) + updater.stop() + + assert lock_was_available == [True] + + +def test_activity_episode_has_provenance_time_and_a_safe_size(monkeypatch): + writes = [] + updater = _updater( + monkeypatch, + lambda **kwargs: writes.append(kwargs) or SimpleNamespace(uuid_="episode-1"), + simulation_id="sim-provenance", + ) + + updater._send_batch_activities( + [_activity(content="x" * 20_000)], + "twitter", + ) + + assert len(writes) == 1 + write = writes[0] + assert len(write["data"]) <= updater.MAX_EPISODE_CHARS + assert write["created_at"] == "2026-07-22T12:00:00+08:00" + assert write["source_description"] == "MiroFish simulation activity batch" + assert write["metadata"]["simulation_id"] == "sim-provenance" + assert write["metadata"]["platform"] == "twitter" + assert write["metadata"]["activity_count"] == 1 + + +def test_failed_non_idempotent_write_is_reported_by_stop(monkeypatch): + def add(**_kwargs): + raise RuntimeError("write failed") + + updater = _updater(monkeypatch, add) + updater.start() + updater.add_activity(_activity()) + + with pytest.raises(RuntimeError, match="ingestion is incomplete"): + updater.stop() + + assert updater.get_stats()["failed_count"] == 1 + + +def test_failed_simulation_action_is_not_ingested(monkeypatch): + updater = _updater( + monkeypatch, + lambda **_kwargs: SimpleNamespace(uuid_="unused"), + ) + + updater.add_activity_from_dict( + { + "agent_id": 1, + "agent_name": "Agent", + "action_type": "CREATE_POST", + "action_args": {"content": "not actually posted"}, + "success": False, + }, + "twitter", + ) + + assert updater.get_stats()["queue_size"] == 0 + assert updater.get_stats()["skipped_count"] == 1 + + +def test_stop_cannot_finish_between_acceptance_check_and_enqueue(monkeypatch): + writes = [] + updater = _updater( + monkeypatch, + lambda **kwargs: writes.append(kwargs) or SimpleNamespace(uuid_="episode-1"), + ) + + put_entered = threading.Event() + allow_put = threading.Event() + + class BlockingQueue(Queue): + def put(self, item, block=True, timeout=None): + put_entered.set() + assert allow_put.wait(timeout=2) + return super().put(item, block=block, timeout=timeout) + + updater._activity_queue = BlockingQueue() + updater.start() + producer = threading.Thread(target=updater.add_activity, args=(_activity(),)) + producer.start() + assert put_entered.wait(timeout=1) + + stopper = threading.Thread(target=updater.stop) + stopper.start() + stopper.join(timeout=0.1) + assert stopper.is_alive() + + allow_put.set() + producer.join(timeout=2) + stopper.join(timeout=2) + + assert not producer.is_alive() + assert not stopper.is_alive() + assert len(writes) == 1 + + +def test_pending_episode_wait_has_a_deadline(monkeypatch): + updater = _updater( + monkeypatch, + lambda **_kwargs: SimpleNamespace(uuid_="episode-1"), + ) + updater._pending_episode_uuids = ["episode-1"] + updater.client.graph.episode.get = lambda **_kwargs: SimpleNamespace( + processed=False + ) + timestamps = iter([0.0, 2.0]) + monkeypatch.setattr(updater_module.Config, "ZEP_INGESTION_TIMEOUT_SECONDS", 1) + monkeypatch.setattr(updater_module.time, "time", lambda: next(timestamps)) + monkeypatch.setattr(updater_module.time, "sleep", lambda _seconds: None) + + with pytest.raises(TimeoutError, match="pending"): + updater._wait_for_pending_episodes() + + +def test_explicit_graph_destruction_can_discard_a_stopped_failed_updater(): + updater = SimpleNamespace( + graph_id="graph-1", + _running=False, + _worker_thread=SimpleNamespace(is_alive=lambda: False), + ) + ZepGraphMemoryManager._updaters["sim-failed"] = updater + try: + assert ZepGraphMemoryManager.discard_inactive_updater("sim-failed") is True + assert "sim-failed" not in ZepGraphMemoryManager._updaters + finally: + ZepGraphMemoryManager._updaters.pop("sim-failed", None) + + +def test_flush_deadline_keeps_unattempted_platform_for_a_safe_retry(monkeypatch): + now = [0.0] + writes = [] + + def add(**kwargs): + writes.append(kwargs) + now[0] = 2.0 + return SimpleNamespace(uuid_=f"episode-{len(writes)}") + + updater = _updater(monkeypatch, add) + updater._platform_buffers["twitter"] = [_activity(1)] + reddit_activity = _activity(2) + reddit_activity.platform = "reddit" + updater._platform_buffers["reddit"] = [reddit_activity] + monkeypatch.setattr(updater_module.time, "time", lambda: now[0]) + + with pytest.raises(TimeoutError, match="deadline"): + updater._flush_remaining(deadline=1.0) + + assert updater._platform_buffers["twitter"] == [] + assert updater._platform_buffers["reddit"] == [reddit_activity] + + now[0] = 0.0 + updater._flush_remaining(deadline=1.0) + assert updater._platform_buffers["reddit"] == [] + assert len(writes) == 2 diff --git a/backend/tests/test_zep_report_barrier.py b/backend/tests/test_zep_report_barrier.py new file mode 100644 index 00000000..cda337dd --- /dev/null +++ b/backend/tests/test_zep_report_barrier.py @@ -0,0 +1,297 @@ +from types import SimpleNamespace + +import pytest +from flask import Flask + +from app.api import graph as graph_api +from app.api import report as report_api +from app.api import simulation as simulation_api +from app.models.project import ProjectStatus +from app.services.simulation_manager import SimulationStatus +from app.services.simulation_runner import RunnerStatus +from app.utils.zep_lifecycle import ( + get_graph_readers, + unregister_graph_reader, +) + + +def _json_result(result): + if isinstance(result, tuple): + response, status = result + else: + response, status = result, result.status_code + return response.get_json(), status + + +def test_report_generation_waits_for_zep_ingestion(monkeypatch): + simulation = SimpleNamespace(project_id="proj-1", graph_id="graph-1") + monkeypatch.setattr( + report_api, + "SimulationManager", + lambda: SimpleNamespace( + get_simulation=lambda _simulation_id: simulation + ), + ) + monkeypatch.setattr( + report_api.ReportManager, + "get_report_by_simulation", + classmethod(lambda _cls, _simulation_id: None), + ) + monkeypatch.setattr( + report_api.SimulationRunner, + "get_run_state", + classmethod( + lambda _cls, _simulation_id: SimpleNamespace( + runner_status=RunnerStatus.STOPPING + ) + ), + ) + monkeypatch.setattr( + report_api.ZepGraphMemoryManager, + "get_updater", + classmethod(lambda _cls, _simulation_id: object()), + ) + + app = Flask(__name__) + with app.test_request_context( + "/api/report/generate", + method="POST", + json={"simulation_id": "sim-1"}, + ): + body, status = _json_result(report_api.generate_report()) + + assert status == 409 + assert body["ingestion_pending"] is True + + +def test_active_rerun_does_not_return_a_stale_completed_report(monkeypatch): + simulation = SimpleNamespace(project_id="proj-1", graph_id="graph-1") + monkeypatch.setattr( + report_api, + "SimulationManager", + lambda: SimpleNamespace( + get_simulation=lambda _simulation_id: simulation + ), + ) + monkeypatch.setattr( + report_api.ReportManager, + "get_report_by_simulation", + classmethod( + lambda _cls, _simulation_id: SimpleNamespace( + report_id="old-report", + status=report_api.ReportStatus.COMPLETED, + ) + ), + ) + monkeypatch.setattr( + report_api.SimulationRunner, + "get_run_state", + classmethod( + lambda _cls, _simulation_id: SimpleNamespace( + runner_status=RunnerStatus.STOPPING + ) + ), + ) + monkeypatch.setattr( + report_api.ZepGraphMemoryManager, + "get_updater", + classmethod(lambda _cls, _simulation_id: object()), + ) + + app = Flask(__name__) + with app.test_request_context( + "/api/report/generate", + method="POST", + json={"simulation_id": "sim-1"}, + ): + body, status = _json_result(report_api.generate_report()) + + assert status == 409 + assert body["ingestion_pending"] is True + + +def test_failed_ingestion_cannot_generate_a_report_after_restart(monkeypatch): + simulation = SimpleNamespace(project_id="proj-1", graph_id="graph-1") + monkeypatch.setattr( + report_api, + "SimulationManager", + lambda: SimpleNamespace( + get_simulation=lambda _simulation_id: simulation + ), + ) + monkeypatch.setattr( + report_api.SimulationRunner, + "get_run_state", + classmethod( + lambda _cls, _simulation_id: SimpleNamespace( + runner_status=RunnerStatus.FAILED + ) + ), + ) + monkeypatch.setattr( + report_api.ZepGraphMemoryManager, + "get_updater", + classmethod(lambda _cls, _simulation_id: None), + ) + + app = Flask(__name__) + with app.test_request_context( + "/api/report/generate", + method="POST", + json={"simulation_id": "sim-1"}, + ): + body, status = _json_result(report_api.generate_report()) + + assert status == 409 + assert "successfully completed" in body["error"] + + +def test_report_reader_lease_blocks_graph_start_and_delete(monkeypatch): + simulation = SimpleNamespace( + simulation_id="sim-1", + project_id="proj-1", + graph_id="graph-1", + status=SimulationStatus.READY, + ) + project = SimpleNamespace( + project_id="proj-1", + graph_id="graph-1", + status=ProjectStatus.GRAPH_COMPLETED, + simulation_requirement="mock requirement", + ) + run_state = SimpleNamespace(runner_status=RunnerStatus.COMPLETED) + worker_targets = [] + runner_calls = [] + + class Tasks: + def create_task(self, **_kwargs): + return "task-1" + + def update_task(self, *_args, **_kwargs): + pass + + def complete_task(self, *_args, **_kwargs): + pass + + def fail_task(self, *_args, **_kwargs): + pass + + class ParkedThread: + def __init__(self, *, target, daemon): + assert daemon is True + self.target = target + + def start(self): + worker_targets.append(self.target) + + class Agent: + def __init__(self, **_kwargs): + pass + + def generate_report(self, *, progress_callback, report_id): + progress_callback("mock", 100, "done") + return SimpleNamespace( + report_id=report_id, + status=report_api.ReportStatus.COMPLETED, + error=None, + ) + + monkeypatch.setattr( + report_api, + "SimulationManager", + lambda: SimpleNamespace( + get_simulation=lambda _simulation_id: simulation + ), + ) + monkeypatch.setattr( + simulation_api, + "SimulationManager", + lambda: SimpleNamespace( + get_simulation=lambda _simulation_id: simulation + ), + ) + monkeypatch.setattr( + report_api.ProjectManager, + "get_project", + classmethod(lambda _cls, _project_id: project), + ) + monkeypatch.setattr( + report_api.SimulationRunner, + "get_run_state", + classmethod(lambda _cls, _simulation_id: run_state), + ) + monkeypatch.setattr( + report_api.ZepGraphMemoryManager, + "get_updater", + classmethod(lambda _cls, _simulation_id: None), + ) + monkeypatch.setattr( + report_api.ReportManager, + "get_report_by_simulation", + classmethod(lambda _cls, _simulation_id: None), + ) + monkeypatch.setattr( + report_api.ReportManager, + "save_report", + classmethod(lambda _cls, _report: None), + ) + monkeypatch.setattr(report_api, "TaskManager", Tasks) + monkeypatch.setattr(report_api, "ReportAgent", Agent) + monkeypatch.setattr(report_api.threading, "Thread", ParkedThread) + monkeypatch.setattr( + simulation_api.SimulationRunner, + "start_simulation", + classmethod( + lambda _cls, **_kwargs: runner_calls.append(True) + ), + ) + monkeypatch.setattr( + graph_api.ZepGraphMemoryManager, + "get_simulation_ids_for_graph", + classmethod(lambda _cls, _graph_id: []), + ) + monkeypatch.setattr( + graph_api, + "SimulationManager", + lambda: SimpleNamespace(list_simulations=lambda: []), + ) + + app = Flask(__name__) + report_id = None + try: + with app.test_request_context( + "/api/report/generate", + method="POST", + json={"simulation_id": "sim-1"}, + ): + body, status = _json_result(report_api.generate_report()) + assert status == 200 + report_id = body["data"]["report_id"] + assert get_graph_readers("graph-1") == [report_id] + assert len(worker_targets) == 1 + + with app.test_request_context( + "/api/simulation/start", + method="POST", + json={ + "simulation_id": "sim-1", + "enable_graph_memory_update": True, + }, + ): + start_body, start_status = _json_result( + simulation_api.start_simulation() + ) + assert start_status == 409 + assert start_body["active_reports"] == [report_id] + assert runner_calls == [] + + with pytest.raises(graph_api.GraphInUseError, match=f"report:{report_id}"): + graph_api._delete_cloud_graph_if_present("graph-1") + + # Let the parked background report finish; its finally block must + # release the lease even if report generation fails. + worker_targets[0]() + assert get_graph_readers("graph-1") == [] + finally: + if report_id: + unregister_graph_reader("graph-1", report_id) diff --git a/backend/tests/test_zep_retry_and_client.py b/backend/tests/test_zep_retry_and_client.py new file mode 100644 index 00000000..5e8d4494 --- /dev/null +++ b/backend/tests/test_zep_retry_and_client.py @@ -0,0 +1,102 @@ +from types import SimpleNamespace + +import pytest +from zep_cloud.core.api_error import ApiError as ZepApiError + +from app.utils import zep +from app import config + + +def test_permanent_zep_errors_fail_without_retry(): + calls = [] + + def operation(): + calls.append(True) + raise ZepApiError(status_code=400, body={"message": "bad query"}) + + with pytest.raises(ZepApiError): + zep.call_zep_read_with_retry( + operation, + operation_name="permanent failure", + sleep=lambda _seconds: None, + ) + + assert len(calls) == 1 + + +def test_rate_limit_retry_respects_retry_after(): + calls = [] + sleeps = [] + + def operation(): + calls.append(True) + if len(calls) == 1: + raise ZepApiError( + status_code=429, + headers={"Retry-After": "7"}, + body={"message": "slow down"}, + ) + return "ok" + + result = zep.call_zep_read_with_retry( + operation, + operation_name="rate limited read", + sleep=sleeps.append, + ) + + assert result == "ok" + assert len(calls) == 2 + assert sleeps == [7.0] + + +def test_zep_client_is_shared_and_uses_an_explicit_timeout(monkeypatch): + created = [] + + def fake_zep(**kwargs): + created.append(kwargs) + return SimpleNamespace(kwargs=kwargs) + + monkeypatch.delenv("ZEP_API_URL", raising=False) + monkeypatch.setattr(zep, "Zep", fake_zep) + zep.clear_zep_client_cache() + + first = zep.get_zep_client(" test-key ", timeout=12) + second = zep.get_zep_client("test-key", timeout=12) + + assert first is second + assert created == [{ + "api_key": "test-key", + "base_url": zep.ZEP_CLOUD_BASE_URL, + "timeout": 12.0, + }] + zep.clear_zep_client_cache() + + +def test_zep_client_rejects_self_hosted_endpoint_override(monkeypatch): + monkeypatch.setenv("ZEP_API_URL", "https://example.invalid") + + with pytest.raises(ValueError, match="ZEP_API_URL"): + zep.get_zep_client("test-key") + + +def test_malformed_timeout_config_is_reported_without_import_failure(): + value, error = config._parse_number( + "ZEP_REQUEST_TIMEOUT_SECONDS", + "not-a-number", + default=30.0, + cast=float, + ) + + assert value == 30.0 + assert "must be a number" in error + + +def test_zep_client_rejects_a_recorded_timeout_parse_error(monkeypatch): + monkeypatch.setattr( + zep.Config, + "_ZEP_CONFIG_PARSE_ERRORS", + ("ZEP_REQUEST_TIMEOUT_SECONDS must be a number",), + ) + + with pytest.raises(ValueError, match="must be a number"): + zep.get_zep_client("test-key") diff --git a/backend/tests/test_zep_simulation_barrier.py b/backend/tests/test_zep_simulation_barrier.py new file mode 100644 index 00000000..63289fce --- /dev/null +++ b/backend/tests/test_zep_simulation_barrier.py @@ -0,0 +1,484 @@ +from types import SimpleNamespace +import json + +import pytest +from flask import Flask + +from app.api import simulation as simulation_api +from app.services import simulation_runner as runner_module +from app.services.simulation_manager import SimulationStatus +from app.services.simulation_runner import ( + RunnerStatus, + SimulationRunState, + SimulationRunner, + SimulationStopPending, +) + + +def test_manual_stop_surfaces_graph_ingestion_failure(monkeypatch): + state = SimulationRunState( + simulation_id="sim-1", + runner_status=RunnerStatus.RUNNING, + ) + saved = [] + monkeypatch.setattr( + SimulationRunner, + "get_run_state", + classmethod(lambda _cls, _simulation_id: state), + ) + monkeypatch.setattr( + SimulationRunner, + "_save_run_state", + classmethod(lambda _cls, value: saved.append(value.runner_status)), + ) + monkeypatch.setattr( + runner_module.ZepGraphMemoryManager, + "stop_updater", + classmethod( + lambda _cls, _simulation_id: (_ for _ in ()).throw( + RuntimeError("ingestion incomplete") + ) + ), + ) + SimulationRunner._processes.pop("sim-1", None) + SimulationRunner._graph_memory_enabled["sim-1"] = True + + try: + with pytest.raises(RuntimeError, match="ingestion incomplete"): + SimulationRunner.stop_simulation("sim-1") + + assert state.runner_status == RunnerStatus.FAILED + assert "ingestion incomplete" in state.error + assert saved[-1] == RunnerStatus.FAILED + finally: + SimulationRunner._graph_memory_enabled.pop("sim-1", None) + SimulationRunner._manual_stop_requests.discard("sim-1") + + +def test_platform_completion_does_not_publish_terminal_success_before_barrier( + monkeypatch, tmp_path +): + simulation_id = "sim-1" + sim_dir = tmp_path / simulation_id / "twitter" + sim_dir.mkdir(parents=True) + log_path = sim_dir / "actions.jsonl" + log_path.write_text( + '{"event_type":"simulation_end","total_rounds":1,"total_actions":0}\n', + encoding="utf-8", + ) + monkeypatch.setattr(SimulationRunner, "RUN_STATE_DIR", str(tmp_path)) + state = SimulationRunState( + simulation_id=simulation_id, + runner_status=RunnerStatus.RUNNING, + twitter_running=True, + ) + + SimulationRunner._read_action_log(str(log_path), 0, state, "twitter") + + assert state.twitter_completed is True + assert state.runner_status == RunnerStatus.RUNNING + + +def test_manual_stop_timeout_leaves_monitor_owned_state_stopping(monkeypatch): + state = SimulationRunState( + simulation_id="sim-timeout", + runner_status=RunnerStatus.RUNNING, + ) + + class Monitor: + def join(self, timeout): + assert timeout >= 30 + + def is_alive(self): + return True + + monkeypatch.setattr( + SimulationRunner, + "get_run_state", + classmethod(lambda _cls, _simulation_id: state), + ) + monkeypatch.setattr( + SimulationRunner, + "_save_run_state", + classmethod(lambda _cls, _state: None), + ) + SimulationRunner._monitor_threads["sim-timeout"] = Monitor() + SimulationRunner._processes.pop("sim-timeout", None) + SimulationRunner._graph_memory_enabled.pop("sim-timeout", None) + + try: + with pytest.raises(TimeoutError, match="仍在停止中"): + SimulationRunner.stop_simulation("sim-timeout") + assert state.runner_status == RunnerStatus.STOPPING + finally: + SimulationRunner._monitor_threads.pop("sim-timeout", None) + SimulationRunner._manual_stop_requests.discard("sim-timeout") + + +def test_failed_ingestion_finalization_can_be_retried(monkeypatch): + state = SimulationRunState( + simulation_id="sim-retry", + runner_status=RunnerStatus.FAILED, + error="first drain timed out", + ) + monkeypatch.setattr( + SimulationRunner, + "get_run_state", + classmethod(lambda _cls, _simulation_id: state), + ) + monkeypatch.setattr( + SimulationRunner, + "_save_run_state", + classmethod(lambda _cls, _state: None), + ) + monkeypatch.setattr( + runner_module.ZepGraphMemoryManager, + "get_updater", + classmethod(lambda _cls, _simulation_id: object()), + ) + monkeypatch.setattr( + runner_module.ZepGraphMemoryManager, + "stop_updater", + classmethod(lambda _cls, _simulation_id: None), + ) + SimulationRunner._graph_memory_enabled["sim-retry"] = True + SimulationRunner._monitor_threads.pop("sim-retry", None) + + try: + result = SimulationRunner.stop_simulation("sim-retry") + assert result.runner_status == RunnerStatus.STOPPED + assert result.error is None + finally: + SimulationRunner._graph_memory_enabled.pop("sim-retry", None) + SimulationRunner._manual_stop_requests.discard("sim-retry") + + +def test_stop_api_keeps_pending_finalization_out_of_failed_state(monkeypatch): + simulation = SimpleNamespace(status=SimulationStatus.STOPPING, error=None) + saved = [] + monkeypatch.setattr( + simulation_api.SimulationRunner, + "stop_simulation", + classmethod( + lambda _cls, _simulation_id: (_ for _ in ()).throw( + SimulationStopPending("still draining") + ) + ), + ) + monkeypatch.setattr( + simulation_api, + "SimulationManager", + lambda: SimpleNamespace( + get_simulation=lambda _simulation_id: simulation, + _save_simulation_state=lambda state: saved.append(state.status), + ), + ) + + app = Flask(__name__) + with app.test_request_context( + "/api/simulation/stop", + method="POST", + json={"simulation_id": "sim-pending"}, + ): + response, status = simulation_api.stop_simulation() + + assert status == 202 + assert response.get_json()["pending"] is True + assert simulation.status == SimulationStatus.STOPPING + assert saved == [] + + +@pytest.mark.parametrize( + "field", + ["force", "enable_graph_memory_update"], +) +def test_simulation_start_rejects_string_booleans(field): + app = Flask(__name__) + with app.test_request_context( + "/api/simulation/start", + method="POST", + json={"simulation_id": "sim-1", field: "false"}, + ): + response, status = simulation_api.start_simulation() + + assert status == 400 + assert "JSON boolean" in response.get_json()["error"] + + +def test_force_restart_does_not_continue_while_old_ingestion_is_pending(monkeypatch): + simulation = SimpleNamespace( + simulation_id="sim-1", + project_id="proj-1", + graph_id="graph-1", + status=SimulationStatus.STOPPING, + ) + cleanup_called = [] + monkeypatch.setattr( + simulation_api, + "SimulationManager", + lambda: SimpleNamespace( + get_simulation=lambda _simulation_id: simulation, + _save_simulation_state=lambda _state: None, + ), + ) + monkeypatch.setattr( + simulation_api, + "_check_simulation_prepared", + lambda _simulation_id: (True, {}), + ) + monkeypatch.setattr( + simulation_api.SimulationRunner, + "get_run_state", + classmethod( + lambda _cls, _simulation_id: SimpleNamespace( + runner_status=RunnerStatus.STOPPING + ) + ), + ) + monkeypatch.setattr( + simulation_api.SimulationRunner, + "stop_simulation", + classmethod( + lambda _cls, _simulation_id: (_ for _ in ()).throw( + SimulationStopPending("still draining") + ) + ), + ) + monkeypatch.setattr( + simulation_api.SimulationRunner, + "cleanup_simulation_logs", + classmethod( + lambda _cls, _simulation_id: cleanup_called.append(True) + ), + ) + monkeypatch.setattr( + simulation_api.ZepGraphMemoryManager, + "get_updater", + classmethod(lambda _cls, _simulation_id: object()), + ) + + app = Flask(__name__) + with app.test_request_context( + "/api/simulation/start", + method="POST", + json={"simulation_id": "sim-1", "force": True}, + ): + response, status = simulation_api.start_simulation() + + assert status == 409 + assert response.get_json()["pending"] is True + assert cleanup_called == [] + + +def test_monitor_start_failure_terminates_the_spawned_process(monkeypatch, tmp_path): + simulation_id = "sim-start-failure" + sim_dir = tmp_path / "runs" / simulation_id + scripts_dir = tmp_path / "scripts" + sim_dir.mkdir(parents=True) + scripts_dir.mkdir() + (sim_dir / "simulation_config.json").write_text( + json.dumps({ + "time_config": { + "total_simulation_hours": 1, + "minutes_per_round": 60, + } + }), + encoding="utf-8", + ) + (scripts_dir / "run_twitter_simulation.py").write_text("pass\n", encoding="utf-8") + + class Process: + pid = 123 + + def poll(self): + return None + + class BrokenThread: + def __init__(self, **_kwargs): + pass + + def start(self): + raise RuntimeError("monitor failed") + + terminated = [] + monkeypatch.setattr(SimulationRunner, "RUN_STATE_DIR", str(tmp_path / "runs")) + monkeypatch.setattr(SimulationRunner, "SCRIPTS_DIR", str(scripts_dir)) + monkeypatch.setattr(runner_module.subprocess, "Popen", lambda *_args, **_kwargs: Process()) + monkeypatch.setattr(runner_module.threading, "Thread", BrokenThread) + monkeypatch.setattr( + SimulationRunner, + "_terminate_process", + classmethod(lambda _cls, _process, sim_id: terminated.append(sim_id)), + ) + monkeypatch.setattr( + SimulationRunner, + "_sync_simulation_status", + classmethod(lambda _cls, *_args, **_kwargs: None), + ) + + try: + with pytest.raises(RuntimeError, match="monitor failed"): + SimulationRunner.start_simulation( + simulation_id, + platform="twitter", + enable_graph_memory_update=False, + ) + assert terminated == [simulation_id] + assert simulation_id not in SimulationRunner._processes + assert simulation_id not in SimulationRunner._action_queues + assert simulation_id not in SimulationRunner._stdout_files + finally: + SimulationRunner._run_states.pop(simulation_id, None) + SimulationRunner._processes.pop(simulation_id, None) + SimulationRunner._action_queues.pop(simulation_id, None) + SimulationRunner._stdout_files.pop(simulation_id, None) + SimulationRunner._stderr_files.pop(simulation_id, None) + SimulationRunner._graph_memory_enabled.pop(simulation_id, None) + + +def test_shutdown_terminates_producer_before_tail_read_and_updater_drain( + monkeypatch, +): + simulation_id = "sim-shutdown-order" + state = SimulationRunState( + simulation_id=simulation_id, + runner_status=RunnerStatus.RUNNING, + ) + events = [] + + class Process: + pid = 123 + stopped = False + + def poll(self): + return 0 if self.stopped else None + + process = Process() + + class Monitor: + alive = True + + def join(self, timeout): + assert timeout >= 30 + events.extend(["tail-read", "updater-drain"]) + state.runner_status = RunnerStatus.STOPPED + SimulationRunner._graph_memory_enabled.pop(simulation_id, None) + self.alive = False + + def is_alive(self): + return self.alive + + monkeypatch.setattr( + SimulationRunner, + "get_run_state", + classmethod(lambda _cls, _simulation_id: state), + ) + monkeypatch.setattr( + SimulationRunner, + "_save_run_state", + classmethod(lambda _cls, _state: None), + ) + monkeypatch.setattr( + SimulationRunner, + "_sync_simulation_status", + classmethod(lambda _cls, *_args, **_kwargs: None), + ) + monkeypatch.setattr( + SimulationRunner, + "_terminate_process", + classmethod( + lambda _cls, proc, _simulation_id, **_kwargs: ( + events.append("producer-terminate"), + setattr(proc, "stopped", True), + ) + ), + ) + monkeypatch.setattr( + runner_module.ZepGraphMemoryManager, + "get_simulation_ids", + classmethod(lambda _cls: [simulation_id]), + ) + updater = object() + monkeypatch.setattr( + runner_module.ZepGraphMemoryManager, + "get_updater", + classmethod(lambda _cls, _simulation_id: updater), + ) + + SimulationRunner._cleanup_done = False + SimulationRunner._processes[simulation_id] = process + SimulationRunner._monitor_threads[simulation_id] = Monitor() + SimulationRunner._graph_memory_enabled[simulation_id] = True + try: + SimulationRunner.cleanup_all_simulations() + assert events == [ + "producer-terminate", + "tail-read", + "updater-drain", + ] + assert state.runner_status == RunnerStatus.STOPPED + finally: + SimulationRunner._cleanup_done = False + SimulationRunner._processes.pop(simulation_id, None) + SimulationRunner._monitor_threads.pop(simulation_id, None) + SimulationRunner._graph_memory_enabled.pop(simulation_id, None) + SimulationRunner._manual_stop_requests.discard(simulation_id) + + +def test_shutdown_drain_failure_remains_failed_and_retryable(monkeypatch): + simulation_id = "sim-shutdown-failure" + state = SimulationRunState( + simulation_id=simulation_id, + runner_status=RunnerStatus.RUNNING, + ) + updater = object() + + monkeypatch.setattr( + SimulationRunner, + "get_run_state", + classmethod(lambda _cls, _simulation_id: state), + ) + monkeypatch.setattr( + SimulationRunner, + "_save_run_state", + classmethod(lambda _cls, _state: None), + ) + monkeypatch.setattr( + SimulationRunner, + "_sync_simulation_status", + classmethod(lambda _cls, *_args, **_kwargs: None), + ) + monkeypatch.setattr( + runner_module.ZepGraphMemoryManager, + "get_simulation_ids", + classmethod(lambda _cls: [simulation_id]), + ) + monkeypatch.setattr( + runner_module.ZepGraphMemoryManager, + "get_updater", + classmethod(lambda _cls, _simulation_id: updater), + ) + monkeypatch.setattr( + runner_module.ZepGraphMemoryManager, + "stop_updater", + classmethod( + lambda _cls, _simulation_id: (_ for _ in ()).throw( + RuntimeError("drain incomplete") + ) + ), + ) + + SimulationRunner._cleanup_done = False + SimulationRunner._graph_memory_enabled[simulation_id] = True + SimulationRunner._monitor_threads.pop(simulation_id, None) + SimulationRunner._processes.pop(simulation_id, None) + try: + SimulationRunner.cleanup_all_simulations() + assert state.runner_status == RunnerStatus.FAILED + assert "drain incomplete" in state.error + assert SimulationRunner._graph_memory_enabled[simulation_id] is True + assert SimulationRunner._cleanup_done is False + finally: + SimulationRunner._cleanup_done = False + SimulationRunner._graph_memory_enabled.pop(simulation_id, None) + SimulationRunner._manual_stop_requests.discard(simulation_id) diff --git a/backend/uv.lock b/backend/uv.lock index b2f853cc..7f606a13 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -994,6 +994,7 @@ dependencies = [ { name = "charset-normalizer" }, { name = "flask" }, { name = "flask-cors" }, + { name = "httpx" }, { name = "openai" }, { name = "pydantic" }, { name = "pymupdf" }, @@ -1022,6 +1023,7 @@ requires-dist = [ { name = "charset-normalizer", specifier = ">=3.0.0" }, { name = "flask", specifier = ">=3.0.0" }, { name = "flask-cors", specifier = ">=6.0.0" }, + { name = "httpx", specifier = ">=0.27.0" }, { name = "openai", specifier = ">=1.0.0" }, { name = "pipreqs", marker = "extra == 'dev'", specifier = ">=0.5.0" }, { name = "pydantic", specifier = ">=2.0.0" }, @@ -1029,7 +1031,7 @@ requires-dist = [ { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0" }, { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.23.0" }, { name = "python-dotenv", specifier = ">=1.0.0" }, - { name = "zep-cloud", specifier = "==3.13.0" }, + { name = "zep-cloud", specifier = "==3.25.0" }, ] provides-extras = ["dev"] @@ -2775,7 +2777,7 @@ wheels = [ [[package]] name = "zep-cloud" -version = "3.13.0" +version = "3.25.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "httpx" }, @@ -2784,7 +2786,7 @@ dependencies = [ { name = "python-dateutil" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/32/c7/c835debf13302f8aaf8d0561ac6ff5a9bc15cc140cd692a1330fb1900c55/zep_cloud-3.13.0.tar.gz", hash = "sha256:c55d9c511773bb2177ae8e08546141404f87d2099affafabd7ec4b4505763e48", size = 63116, upload-time = "2025-11-20T15:25:40.745Z" } +sdist = { url = "https://files.pythonhosted.org/packages/81/0d/7a866e1e1e5f0f1353eedf3d0e17df9b3cf362c494bccd41eb121b49c43f/zep_cloud-3.25.0.tar.gz", hash = "sha256:a77867e02c2a9036a20623e85d03415e237e76510b6e05ff2e9ab2cdced0aeb9", size = 94772, upload-time = "2026-07-16T00:46:43.918Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f7/e1/bbf03c6c8007c0cb238780e7fc6d8e1a52633893933a41aa09678618985a/zep_cloud-3.13.0-py3-none-any.whl", hash = "sha256:b2fbdeef73e262194c8f67b58f76471de6ee87e1a629541a09d8f7bbf475f12b", size = 110601, upload-time = "2025-11-20T15:25:38.484Z" }, + { url = "https://files.pythonhosted.org/packages/77/fc/5ca2bdd20b83bd62bc2708580f756e704f69704709e0ed43e735aa590c1e/zep_cloud-3.25.0-py3-none-any.whl", hash = "sha256:94d9599038b154af9ad33cab4b4873ed9adb1bcaa48fc4c41cbd407c1a657ef5", size = 166438, upload-time = "2026-07-16T00:46:42.39Z" }, ] diff --git a/frontend/src/api/graph.js b/frontend/src/api/graph.js index ef90a2b6..85669559 100644 --- a/frontend/src/api/graph.js +++ b/frontend/src/api/graph.js @@ -1,4 +1,4 @@ -import service, { requestWithRetry } from './index' +import service from './index' /** * 生成本体（上传文档和模拟需求） @@ -6,16 +6,14 @@ import service, { requestWithRetry } from './index' * @returns {Promise} */ export function generateOntology(formData) { - return requestWithRetry(() => - service({ - url: '/api/graph/ontology/generate', - method: 'post', - data: formData, - headers: { - 'Content-Type': 'multipart/form-data' - } - }) - ) + return service({ + url: '/api/graph/ontology/generate', + method: 'post', + data: formData, + headers: { + 'Content-Type': 'multipart/form-data' + } + }) } /** @@ -24,13 +22,11 @@ export function generateOntology(formData) { * @returns {Promise} */ export function buildGraph(data) { - return requestWithRetry(() => - service({ - url: '/api/graph/build', - method: 'post', - data - }) - ) + return service({ + url: '/api/graph/build', + method: 'post', + data + }) } /** diff --git a/frontend/src/api/index.js b/frontend/src/api/index.js index e840e116..807a0b4e 100644 --- a/frontend/src/api/index.js +++ b/frontend/src/api/index.js @@ -52,18 +52,4 @@ service.interceptors.response.use( } ) -// 带重试的请求函数 -export const requestWithRetry = async (requestFn, maxRetries = 3, delay = 1000) => { - for (let i = 0; i < maxRetries; i++) { - try { - return await requestFn() - } catch (error) { - if (i === maxRetries - 1) throw error - - console.warn(`Request failed, retrying (${i + 1}/${maxRetries})...`) - await new Promise(resolve => setTimeout(resolve, delay * Math.pow(2, i))) - } - } -} - export default service diff --git a/frontend/src/api/report.js b/frontend/src/api/report.js index c89a67d8..9b61ce72 100644 --- a/frontend/src/api/report.js +++ b/frontend/src/api/report.js @@ -1,11 +1,11 @@ -import service, { requestWithRetry } from './index' +import service from './index' /** * 开始报告生成 * @param {Object} data - { simulation_id, force_regenerate? } */ export const generateReport = (data) => { - return requestWithRetry(() => service.post('/api/report/generate', data), 3, 1000) + return service.post('/api/report/generate', data) } /** @@ -47,5 +47,5 @@ export const getReport = (reportId) => { * @param {Object} data - { simulation_id, message, chat_history? } */ export const chatWithReport = (data) => { - return requestWithRetry(() => service.post('/api/report/chat', data), 3, 1000) + return service.post('/api/report/chat', data) } diff --git a/frontend/src/api/simulation.js b/frontend/src/api/simulation.js index 4dddd073..d495afae 100644 --- a/frontend/src/api/simulation.js +++ b/frontend/src/api/simulation.js @@ -1,11 +1,11 @@ -import service, { requestWithRetry } from './index' +import service from './index' /** * 创建模拟 * @param {Object} data - { project_id, graph_id?, enable_twitter?, enable_reddit? } */ export const createSimulation = (data) => { - return requestWithRetry(() => service.post('/api/simulation/create', data), 3, 1000) + return service.post('/api/simulation/create', data) } /** @@ -13,7 +13,7 @@ export const createSimulation = (data) => { * @param {Object} data - { simulation_id, entity_types?, use_llm_for_profiles?, parallel_profile_count?, force_regenerate? } */ export const prepareSimulation = (data) => { - return requestWithRetry(() => service.post('/api/simulation/prepare', data), 3, 1000) + return service.post('/api/simulation/prepare', data) } /** @@ -83,7 +83,7 @@ export const listSimulations = (projectId) => { * @param {Object} data - { simulation_id, platform?, max_rounds?, enable_graph_memory_update? } */ export const startSimulation = (data) => { - return requestWithRetry(() => service.post('/api/simulation/start', data), 3, 1000) + return service.post('/api/simulation/start', data) } /** @@ -175,7 +175,7 @@ export const getEnvStatus = (data) => { * @param {Object} data - { simulation_id, interviews: [{ agent_id, prompt }] } */ export const interviewAgents = (data) => { - return requestWithRetry(() => service.post('/api/simulation/interview/batch', data), 3, 1000) + return service.post('/api/simulation/interview/batch', data) } /** @@ -186,4 +186,3 @@ export const interviewAgents = (data) => { export const getSimulationHistory = (limit = 20) => { return service.get('/api/simulation/history', { params: { limit } }) } - diff --git a/frontend/src/components/Step3Simulation.vue b/frontend/src/components/Step3Simulation.vue index 65e3f520..e9dd3dc3 100644 --- a/frontend/src/components/Step3Simulation.vue +++ b/frontend/src/components/Step3Simulation.vue @@ -515,22 +515,14 @@ const fetchRunStatus = async () => { const isCompleted = data.runner_status === 'completed' || data.runner_status === 'stopped' const isFailed = data.runner_status === 'failed' - // 额外检查：如果后端还没来得及更新 runner_status，但平台已经报告完成 - // 通过检测 twitter_completed 和 reddit_completed 状态判断 - const platformsCompleted = checkPlatformsCompleted(data) - - // An explicit runner failure is authoritative. Platform completion flags - // can be stale or partial when one subprocess exits successfully before - // another subprocess fails, so inferred completion must not mask it. + // runner_status is authoritative because the backend only publishes a + // terminal state after the Zep ingestion barrier has completed. if (isFailed) { addLog(t('log.simFailed') + (data.error ? `: ${data.error}` : '')) phase.value = 2 stopPolling() emit('update-status', 'error') - } else if (isCompleted || platformsCompleted) { - if (platformsCompleted && !isCompleted) { - addLog(t('log.allPlatformsCompleted')) - } + } else if (isCompleted) { addLog(t('log.simCompleted')) phase.value = 2 stopPolling() diff --git a/frontend/src/views/MainView.vue b/frontend/src/views/MainView.vue index 513c70d8..cf600ff9 100644 --- a/frontend/src/views/MainView.vue +++ b/frontend/src/views/MainView.vue @@ -246,7 +246,6 @@ const loadProject = async () => { } else if (res.data.status === 'graph_building' && res.data.graph_build_task_id) { currentPhase.value = 1 startPollingTask(res.data.graph_build_task_id) - startGraphPolling() } else if (res.data.status === 'graph_completed' && res.data.graph_id) { currentPhase.value = 2 await loadGraph(res.data.graph_id) @@ -281,8 +280,18 @@ const startBuildGraph = async () => { const res = await buildGraph({ project_id: currentProjectId.value }) if (res.success) { + if (res.data.reused && res.data.graph_id) { + currentPhase.value = 2 + buildProgress.value = null + const projectRes = await getProject(currentProjectId.value) + if (projectRes.success) { + projectData.value = projectRes.data + } + await loadGraph(res.data.graph_id) + return + } + addLog(`Graph build task started. Task ID: ${res.data.task_id}`) - startGraphPolling() startPollingTask(res.data.task_id) } else { error.value = res.error diff --git a/frontend/src/views/Process.vue b/frontend/src/views/Process.vue index 2d2d3cc1..dcddeaec 100644 --- a/frontend/src/views/Process.vue +++ b/frontend/src/views/Process.vue @@ -686,14 +686,22 @@ const startBuildGraph = async () => { const response = await buildGraph({ project_id: currentProjectId.value }) if (response.success) { + if (response.data.reused && response.data.graph_id) { + currentPhase.value = 2 + buildProgress.value = null + const projectResponse = await getProject(currentProjectId.value) + if (projectResponse.success) { + projectData.value = projectResponse.data + } + await loadGraph(response.data.graph_id) + return + } + buildProgress.value.message = '图谱构建任务已启动...' // 保存 task_id 用于轮询 const taskId = response.data.task_id - // 启动图谱数据轮询（独立于任务状态轮询） - startGraphPolling() - // 启动任务状态轮询 startPollingTask(taskId) } else { @@ -2065,4 +2073,4 @@ onUnmounted(() => { display: none; } } - \ No newline at end of file +