fix: modernize Zep Cloud integration
This commit is contained in:
parent
dd7a646d6b
commit
e4aa38dff2
|
|
@ -8,9 +8,13 @@ LLM_MODEL_NAME=qwen-plus
|
|||
# ===== ZEP记忆图谱配置 =====
|
||||
# 每月免费额度即可支撑简单使用:https://app.getzep.com/
|
||||
ZEP_API_KEY=your_zep_api_key_here
|
||||
# 单次 Zep Cloud 请求超时(秒)
|
||||
ZEP_REQUEST_TIMEOUT_SECONDS=30
|
||||
# 文档批次或模拟 episode 等待处理完成的最长时间(秒)
|
||||
ZEP_INGESTION_TIMEOUT_SECONDS=600
|
||||
|
||||
# ===== 加速 LLM 配置(可选)=====
|
||||
# 注意如果不使用加速配置,env文件中就不要出现下面的配置项
|
||||
LLM_BOOST_API_KEY=your_api_key_here
|
||||
LLM_BOOST_BASE_URL=your_base_url_here
|
||||
LLM_BOOST_MODEL_NAME=your_model_name_here
|
||||
LLM_BOOST_MODEL_NAME=your_model_name_here
|
||||
|
|
|
|||
|
|
@ -6,21 +6,115 @@
|
|||
import os
|
||||
import traceback
|
||||
import threading
|
||||
from contextlib import ExitStack, nullcontext
|
||||
from flask import request, jsonify
|
||||
from zep_cloud import NotFoundError
|
||||
|
||||
from . import graph_bp
|
||||
from ..config import Config
|
||||
from ..services.ontology_generator import OntologyGenerator
|
||||
from ..services.graph_builder import GraphBuilderService
|
||||
from ..services.graph_builder import BatchSubmission, GraphBuilderService
|
||||
from ..services.text_processor import TextProcessor
|
||||
from ..utils.file_parser import FileParser
|
||||
from ..utils.logger import get_logger
|
||||
from ..utils.locale import t, get_locale, set_locale
|
||||
from ..utils.zep_lifecycle import get_graph_readers, graph_lifecycle_lock
|
||||
from ..models.task import TaskManager, TaskStatus
|
||||
from ..models.project import ProjectManager, ProjectStatus
|
||||
from ..services.simulation_manager import SimulationManager
|
||||
from ..services.simulation_runner import SimulationRunner, RunnerStatus
|
||||
from ..services.zep_graph_memory_updater import ZepGraphMemoryManager
|
||||
|
||||
# 获取日志器
|
||||
logger = get_logger('mirofish.api')
|
||||
_build_locks: dict[str, threading.Lock] = {}
|
||||
_build_locks_guard = threading.Lock()
|
||||
|
||||
|
||||
class GraphInUseError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def _active_graph_consumers(graph_id: str) -> list[str]:
|
||||
active = {
|
||||
f"report:{reader_id}"
|
||||
for reader_id in get_graph_readers(graph_id)
|
||||
}
|
||||
for simulation_id in ZepGraphMemoryManager.get_simulation_ids_for_graph(graph_id):
|
||||
finalization_lock = SimulationRunner._finalization_lock(simulation_id)
|
||||
if not finalization_lock.acquire(blocking=False):
|
||||
active.add(simulation_id)
|
||||
continue
|
||||
try:
|
||||
run_state = SimulationRunner.get_run_state(simulation_id)
|
||||
if run_state and run_state.runner_status == RunnerStatus.FAILED:
|
||||
# reset/delete is the explicit recovery path for an incomplete,
|
||||
# non-replayable write. Serialize it against a retry drain.
|
||||
ZepGraphMemoryManager.discard_inactive_updater(simulation_id)
|
||||
SimulationRunner._graph_memory_enabled.pop(simulation_id, None)
|
||||
continue
|
||||
active.add(simulation_id)
|
||||
finally:
|
||||
finalization_lock.release()
|
||||
active_runner_statuses = {
|
||||
RunnerStatus.STARTING,
|
||||
RunnerStatus.RUNNING,
|
||||
RunnerStatus.PAUSED,
|
||||
RunnerStatus.STOPPING,
|
||||
}
|
||||
for simulation in SimulationManager().list_simulations():
|
||||
if simulation.graph_id != graph_id:
|
||||
continue
|
||||
run_state = SimulationRunner.get_run_state(simulation.simulation_id)
|
||||
if run_state and run_state.runner_status in active_runner_statuses:
|
||||
active.add(simulation.simulation_id)
|
||||
return sorted(active)
|
||||
|
||||
|
||||
def _delete_cloud_graph_if_present(graph_id: str | None) -> None:
|
||||
"""Delete a referenced Cloud graph without retrying the mutation."""
|
||||
|
||||
if not graph_id:
|
||||
return
|
||||
# Keep the consumer check and Cloud mutation in one critical section. The
|
||||
# callers that also clear local references hold this re-entrant lock around
|
||||
# both operations.
|
||||
with graph_lifecycle_lock(graph_id):
|
||||
active_simulations = _active_graph_consumers(graph_id)
|
||||
if active_simulations:
|
||||
raise GraphInUseError(
|
||||
f"Graph {graph_id} is in use by active consumer(s): "
|
||||
f"{', '.join(active_simulations)}"
|
||||
)
|
||||
try:
|
||||
GraphBuilderService(api_key=Config.ZEP_API_KEY).delete_graph(graph_id)
|
||||
except NotFoundError:
|
||||
logger.info("Zep Cloud graph already absent: %s", graph_id)
|
||||
|
||||
|
||||
def _clear_project_graph_reference(project) -> None:
|
||||
project.graph_id = None
|
||||
project.graph_build_task_id = None
|
||||
project.zep_batch_id = None
|
||||
project.zep_batch_operation_id = None
|
||||
project.error = None
|
||||
|
||||
|
||||
def _project_build_lock(project_id: str) -> threading.Lock:
|
||||
with _build_locks_guard:
|
||||
return _build_locks.setdefault(project_id, threading.Lock())
|
||||
|
||||
|
||||
def _project_has_active_build(project) -> bool:
|
||||
if project.status != ProjectStatus.GRAPH_BUILDING:
|
||||
return False
|
||||
if not project.graph_build_task_id:
|
||||
return False
|
||||
task = TaskManager().get_task(project.graph_build_task_id)
|
||||
return bool(
|
||||
task
|
||||
and task.status in {TaskStatus.PENDING, TaskStatus.PROCESSING}
|
||||
)
|
||||
|
||||
|
||||
def allowed_file(filename: str) -> bool:
|
||||
|
|
@ -69,10 +163,38 @@ def list_projects():
|
|||
|
||||
@graph_bp.route('/project/<project_id>', methods=['DELETE'])
|
||||
def delete_project(project_id: str):
|
||||
with _project_build_lock(project_id):
|
||||
return _delete_project_impl(project_id)
|
||||
|
||||
|
||||
def _delete_project_impl(project_id: str):
|
||||
"""
|
||||
删除项目
|
||||
"""
|
||||
success = ProjectManager.delete_project(project_id)
|
||||
project = ProjectManager.get_project(project_id)
|
||||
if not project:
|
||||
return jsonify({
|
||||
"success": False,
|
||||
"error": t('api.projectNotFound', id=project_id)
|
||||
}), 404
|
||||
if _project_has_active_build(project):
|
||||
return jsonify({
|
||||
"success": False,
|
||||
"error": t('api.graphBuilding')
|
||||
}), 409
|
||||
|
||||
graph_id = project.graph_id
|
||||
graph_guard = (
|
||||
graph_lifecycle_lock(graph_id) if graph_id else nullcontext()
|
||||
)
|
||||
with graph_guard:
|
||||
try:
|
||||
_delete_cloud_graph_if_present(graph_id)
|
||||
except GraphInUseError as error:
|
||||
return jsonify({"success": False, "error": str(error)}), 409
|
||||
# The local reference remains protected until it is removed, so a new
|
||||
# simulation cannot claim the just-deleted graph in between.
|
||||
success = ProjectManager.delete_project(project_id)
|
||||
|
||||
if not success:
|
||||
return jsonify({
|
||||
|
|
@ -88,6 +210,11 @@ def delete_project(project_id: str):
|
|||
|
||||
@graph_bp.route('/project/<project_id>/reset', methods=['POST'])
|
||||
def reset_project(project_id: str):
|
||||
with _project_build_lock(project_id):
|
||||
return _reset_project_impl(project_id)
|
||||
|
||||
|
||||
def _reset_project_impl(project_id: str):
|
||||
"""
|
||||
重置项目状态(用于重新构建图谱)
|
||||
"""
|
||||
|
|
@ -99,16 +226,30 @@ def reset_project(project_id: str):
|
|||
"error": t('api.projectNotFound', id=project_id)
|
||||
}), 404
|
||||
|
||||
# 重置到本体已生成状态
|
||||
if project.ontology:
|
||||
project.status = ProjectStatus.ONTOLOGY_GENERATED
|
||||
else:
|
||||
project.status = ProjectStatus.CREATED
|
||||
|
||||
project.graph_id = None
|
||||
project.graph_build_task_id = None
|
||||
project.error = None
|
||||
ProjectManager.save_project(project)
|
||||
if _project_has_active_build(project):
|
||||
return jsonify({
|
||||
"success": False,
|
||||
"error": t('api.graphBuilding')
|
||||
}), 409
|
||||
|
||||
graph_id = project.graph_id
|
||||
graph_guard = (
|
||||
graph_lifecycle_lock(graph_id) if graph_id else nullcontext()
|
||||
)
|
||||
with graph_guard:
|
||||
try:
|
||||
_delete_cloud_graph_if_present(graph_id)
|
||||
except GraphInUseError as error:
|
||||
return jsonify({"success": False, "error": str(error)}), 409
|
||||
|
||||
# 重置到本体已生成状态
|
||||
if project.ontology:
|
||||
project.status = ProjectStatus.ONTOLOGY_GENERATED
|
||||
else:
|
||||
project.status = ProjectStatus.CREATED
|
||||
|
||||
_clear_project_graph_reference(project)
|
||||
ProjectManager.save_project(project)
|
||||
|
||||
return jsonify({
|
||||
"success": True,
|
||||
|
|
@ -259,6 +400,17 @@ def generate_ontology():
|
|||
|
||||
@graph_bp.route('/build', methods=['POST'])
|
||||
def build_graph():
|
||||
"""Serialize build claims for the same project within this process."""
|
||||
|
||||
data = request.get_json(silent=True) or {}
|
||||
project_id = data.get("project_id")
|
||||
if not project_id:
|
||||
return _build_graph_impl()
|
||||
with _project_build_lock(project_id):
|
||||
return _build_graph_impl()
|
||||
|
||||
|
||||
def _build_graph_impl():
|
||||
"""
|
||||
接口2:根据project_id构建图谱
|
||||
|
||||
|
|
@ -315,6 +467,11 @@ def build_graph():
|
|||
|
||||
# 检查项目状态
|
||||
force = data.get('force', False) # 强制重新构建
|
||||
if not isinstance(force, bool):
|
||||
return jsonify({
|
||||
"success": False,
|
||||
"error": "force must be a JSON boolean"
|
||||
}), 400
|
||||
|
||||
if project.status == ProjectStatus.CREATED:
|
||||
return jsonify({
|
||||
|
|
@ -322,24 +479,77 @@ def build_graph():
|
|||
"error": t('api.ontologyNotGenerated')
|
||||
}), 400
|
||||
|
||||
if project.status == ProjectStatus.GRAPH_BUILDING and not force:
|
||||
resume_existing_batch = False
|
||||
if project.status == ProjectStatus.GRAPH_BUILDING:
|
||||
if _project_has_active_build(project):
|
||||
return jsonify({
|
||||
"success": True,
|
||||
"data": {
|
||||
"project_id": project_id,
|
||||
"task_id": project.graph_build_task_id,
|
||||
"graph_id": project.graph_id,
|
||||
"reused": True,
|
||||
"message": t('api.graphBuilding')
|
||||
}
|
||||
})
|
||||
|
||||
if (
|
||||
not force
|
||||
and project.graph_id
|
||||
and project.zep_batch_id
|
||||
and project.zep_batch_operation_id
|
||||
):
|
||||
builder = GraphBuilderService(api_key=Config.ZEP_API_KEY)
|
||||
batch_summary = builder.get_batch_summary(project.zep_batch_id)
|
||||
if getattr(batch_summary, "status", None) in {
|
||||
"queued",
|
||||
"processing",
|
||||
"succeeded",
|
||||
}:
|
||||
resume_existing_batch = True
|
||||
|
||||
if not resume_existing_batch:
|
||||
project.status = ProjectStatus.FAILED
|
||||
project.error = (
|
||||
"Graph build task is no longer present; the persisted Zep "
|
||||
"batch cannot be resumed automatically"
|
||||
)
|
||||
ProjectManager.save_project(project)
|
||||
if not force:
|
||||
return jsonify({
|
||||
"success": False,
|
||||
"error": project.error,
|
||||
"task_id": project.graph_build_task_id,
|
||||
"recoverable": True,
|
||||
}), 409
|
||||
|
||||
if project.status == ProjectStatus.GRAPH_COMPLETED and not force:
|
||||
return jsonify({
|
||||
"success": False,
|
||||
"error": t('api.graphBuilding'),
|
||||
"task_id": project.graph_build_task_id
|
||||
}), 400
|
||||
|
||||
# 如果强制重建,重置状态
|
||||
if force and project.status in [ProjectStatus.GRAPH_BUILDING, ProjectStatus.FAILED, ProjectStatus.GRAPH_COMPLETED]:
|
||||
project.status = ProjectStatus.ONTOLOGY_GENERATED
|
||||
project.graph_id = None
|
||||
project.graph_build_task_id = None
|
||||
project.error = None
|
||||
"success": True,
|
||||
"data": {
|
||||
"project_id": project_id,
|
||||
"task_id": project.graph_build_task_id,
|
||||
"graph_id": project.graph_id,
|
||||
"reused": True,
|
||||
"message": t('progress.graphBuildComplete')
|
||||
}
|
||||
})
|
||||
|
||||
# 获取配置
|
||||
graph_name = data.get('graph_name', project.name or 'MiroFish Graph')
|
||||
chunk_size = data.get('chunk_size', project.chunk_size or Config.DEFAULT_CHUNK_SIZE)
|
||||
chunk_overlap = data.get('chunk_overlap', project.chunk_overlap or Config.DEFAULT_CHUNK_OVERLAP)
|
||||
if not isinstance(chunk_size, int) or chunk_size <= 0:
|
||||
return jsonify({"success": False, "error": "chunk_size must be a positive integer"}), 400
|
||||
if (
|
||||
not isinstance(chunk_overlap, int)
|
||||
or chunk_overlap < 0
|
||||
or chunk_overlap >= chunk_size
|
||||
):
|
||||
return jsonify({
|
||||
"success": False,
|
||||
"error": "chunk_overlap must satisfy 0 <= chunk_overlap < chunk_size"
|
||||
}), 400
|
||||
|
||||
# 更新项目配置
|
||||
project.chunk_size = chunk_size
|
||||
|
|
@ -360,6 +570,22 @@ def build_graph():
|
|||
"success": False,
|
||||
"error": t('api.ontologyNotFound')
|
||||
}), 400
|
||||
|
||||
# Only mutate Cloud state after the complete rebuild request validates.
|
||||
if project.status == ProjectStatus.FAILED or (
|
||||
force and project.status == ProjectStatus.GRAPH_COMPLETED
|
||||
):
|
||||
graph_id_to_delete = project.graph_id
|
||||
graph_guard = (
|
||||
graph_lifecycle_lock(graph_id_to_delete)
|
||||
if graph_id_to_delete
|
||||
else nullcontext()
|
||||
)
|
||||
with graph_guard:
|
||||
_delete_cloud_graph_if_present(graph_id_to_delete)
|
||||
project.status = ProjectStatus.ONTOLOGY_GENERATED
|
||||
_clear_project_graph_reference(project)
|
||||
ProjectManager.save_project(project)
|
||||
|
||||
# 创建异步任务
|
||||
task_manager = TaskManager()
|
||||
|
|
@ -400,49 +626,79 @@ def build_graph():
|
|||
chunk_size=chunk_size,
|
||||
overlap=chunk_overlap
|
||||
)
|
||||
builder.validate_batch_chunks(chunks, batch_size=350)
|
||||
total_chunks = len(chunks)
|
||||
|
||||
# 创建图谱
|
||||
task_manager.update_task(
|
||||
task_id,
|
||||
message=t('progress.creatingZepGraph'),
|
||||
progress=10
|
||||
)
|
||||
graph_id = builder.create_graph(name=graph_name)
|
||||
|
||||
# 更新项目的graph_id
|
||||
project.graph_id = graph_id
|
||||
ProjectManager.save_project(project)
|
||||
|
||||
# 设置本体
|
||||
task_manager.update_task(
|
||||
task_id,
|
||||
message=t('progress.settingOntology'),
|
||||
progress=15
|
||||
)
|
||||
builder.set_ontology(graph_id, ontology)
|
||||
|
||||
# 添加文本(progress_callback 签名是 (msg, progress_ratio))
|
||||
def add_progress_callback(msg, progress_ratio):
|
||||
progress = 15 + int(progress_ratio * 40) # 15% - 55%
|
||||
if resume_existing_batch:
|
||||
graph_id = project.graph_id
|
||||
operation_id = builder.build_operation_id(graph_id, chunks)
|
||||
if operation_id != project.zep_batch_operation_id:
|
||||
raise RuntimeError(
|
||||
"Persisted Zep batch does not match the current graph input"
|
||||
)
|
||||
submission = BatchSubmission(
|
||||
batch_id=project.zep_batch_id,
|
||||
operation_id=operation_id,
|
||||
episode_uuids=[],
|
||||
item_count=total_chunks,
|
||||
)
|
||||
task_manager.update_task(
|
||||
task_id,
|
||||
message=msg,
|
||||
progress=progress
|
||||
message=t('progress.waitingZepProcess'),
|
||||
progress=55,
|
||||
)
|
||||
else:
|
||||
# 创建图谱
|
||||
task_manager.update_task(
|
||||
task_id,
|
||||
message=t('progress.creatingZepGraph'),
|
||||
progress=10
|
||||
)
|
||||
|
||||
def remember_graph(graph_id):
|
||||
project.graph_id = graph_id
|
||||
ProjectManager.save_project(project)
|
||||
|
||||
graph_id = builder.create_graph(
|
||||
name=graph_name,
|
||||
graph_id_callback=remember_graph,
|
||||
)
|
||||
|
||||
# 设置本体
|
||||
task_manager.update_task(
|
||||
task_id,
|
||||
message=t('progress.settingOntology'),
|
||||
progress=15
|
||||
)
|
||||
builder.set_ontology(graph_id, ontology)
|
||||
|
||||
# 添加文本(progress_callback 签名是 (msg, progress_ratio))
|
||||
def add_progress_callback(msg, progress_ratio):
|
||||
progress = 15 + int(progress_ratio * 40) # 15% - 55%
|
||||
task_manager.update_task(
|
||||
task_id,
|
||||
message=msg,
|
||||
progress=progress
|
||||
)
|
||||
|
||||
task_manager.update_task(
|
||||
task_id,
|
||||
message=t('progress.addingChunks', count=total_chunks),
|
||||
progress=15
|
||||
)
|
||||
|
||||
def remember_batch(batch_id, operation_id):
|
||||
project.zep_batch_id = batch_id
|
||||
project.zep_batch_operation_id = operation_id
|
||||
ProjectManager.save_project(project)
|
||||
|
||||
submission = builder.add_text_batches(
|
||||
graph_id,
|
||||
chunks,
|
||||
batch_size=350,
|
||||
progress_callback=add_progress_callback,
|
||||
batch_created_callback=remember_batch,
|
||||
)
|
||||
|
||||
task_manager.update_task(
|
||||
task_id,
|
||||
message=t('progress.addingChunks', count=total_chunks),
|
||||
progress=15
|
||||
)
|
||||
|
||||
episode_uuids = builder.add_text_batches(
|
||||
graph_id,
|
||||
chunks,
|
||||
batch_size=3,
|
||||
progress_callback=add_progress_callback
|
||||
)
|
||||
|
||||
# 等待Zep处理完成(查询每个episode的processed状态)
|
||||
task_manager.update_task(
|
||||
|
|
@ -459,7 +715,7 @@ def build_graph():
|
|||
progress=progress
|
||||
)
|
||||
|
||||
builder._wait_for_episodes(episode_uuids, wait_progress_callback)
|
||||
builder._wait_for_batch(submission, wait_progress_callback)
|
||||
|
||||
# 获取图谱数据
|
||||
task_manager.update_task(
|
||||
|
|
@ -469,44 +725,48 @@ def build_graph():
|
|||
)
|
||||
graph_data = builder.get_graph_data(graph_id)
|
||||
|
||||
# 更新项目状态
|
||||
project.status = ProjectStatus.GRAPH_COMPLETED
|
||||
ProjectManager.save_project(project)
|
||||
|
||||
node_count = graph_data.get("node_count", 0)
|
||||
edge_count = graph_data.get("edge_count", 0)
|
||||
build_logger.info(f"[{task_id}] 图谱构建完成: graph_id={graph_id}, 节点={node_count}, 边={edge_count}")
|
||||
|
||||
# 完成
|
||||
task_manager.update_task(
|
||||
task_id,
|
||||
status=TaskStatus.COMPLETED,
|
||||
message=t('progress.graphBuildComplete'),
|
||||
progress=100,
|
||||
result={
|
||||
"project_id": project_id,
|
||||
"graph_id": graph_id,
|
||||
"node_count": node_count,
|
||||
"edge_count": edge_count,
|
||||
"chunk_count": total_chunks
|
||||
}
|
||||
)
|
||||
|
||||
# Publish local project/task terminal state under the same
|
||||
# lifecycle lock used by reset/delete/build claims. This
|
||||
# prevents a deletion from interleaving between the two saves.
|
||||
with _project_build_lock(project_id):
|
||||
project.status = ProjectStatus.GRAPH_COMPLETED
|
||||
project.error = None
|
||||
ProjectManager.save_project(project)
|
||||
task_manager.update_task(
|
||||
task_id,
|
||||
status=TaskStatus.COMPLETED,
|
||||
message=t('progress.graphBuildComplete'),
|
||||
progress=100,
|
||||
result={
|
||||
"project_id": project_id,
|
||||
"graph_id": graph_id,
|
||||
"node_count": node_count,
|
||||
"edge_count": edge_count,
|
||||
"chunk_count": total_chunks,
|
||||
"zep_batch_id": submission.batch_id,
|
||||
}
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
# 更新项目状态为失败
|
||||
build_logger.error(f"[{task_id}] 图谱构建失败: {str(e)}")
|
||||
build_logger.debug(traceback.format_exc())
|
||||
|
||||
project.status = ProjectStatus.FAILED
|
||||
project.error = str(e)
|
||||
ProjectManager.save_project(project)
|
||||
|
||||
task_manager.update_task(
|
||||
task_id,
|
||||
status=TaskStatus.FAILED,
|
||||
message=t('progress.buildFailed', error=str(e)),
|
||||
error=traceback.format_exc()
|
||||
)
|
||||
with _project_build_lock(project_id):
|
||||
project.status = ProjectStatus.FAILED
|
||||
project.error = str(e)
|
||||
ProjectManager.save_project(project)
|
||||
|
||||
task_manager.update_task(
|
||||
task_id,
|
||||
status=TaskStatus.FAILED,
|
||||
message=t('progress.buildFailed', error=str(e)),
|
||||
error=traceback.format_exc()
|
||||
)
|
||||
|
||||
# 启动后台线程
|
||||
thread = threading.Thread(target=build_task, daemon=True)
|
||||
|
|
@ -517,10 +777,13 @@ def build_graph():
|
|||
"data": {
|
||||
"project_id": project_id,
|
||||
"task_id": task_id,
|
||||
"resumed": resume_existing_batch,
|
||||
"message": t('api.graphBuildStarted', taskId=task_id)
|
||||
}
|
||||
})
|
||||
|
||||
except GraphInUseError as e:
|
||||
return jsonify({"success": False, "error": str(e)}), 409
|
||||
except Exception as e:
|
||||
return jsonify({
|
||||
"success": False,
|
||||
|
|
@ -606,14 +869,45 @@ def delete_graph(graph_id: str):
|
|||
"error": t('api.zepApiKeyMissing')
|
||||
}), 500
|
||||
|
||||
builder = GraphBuilderService(api_key=Config.ZEP_API_KEY)
|
||||
builder.delete_graph(graph_id)
|
||||
projects = ProjectManager.find_projects_by_graph_id(graph_id)
|
||||
if not projects:
|
||||
return jsonify({
|
||||
"success": False,
|
||||
"error": "No local project references this graph"
|
||||
}), 404
|
||||
project_ids = sorted({project.project_id for project in projects})
|
||||
with ExitStack() as stack:
|
||||
for project_id in project_ids:
|
||||
stack.enter_context(_project_build_lock(project_id))
|
||||
stack.enter_context(graph_lifecycle_lock(graph_id))
|
||||
|
||||
# Re-read under all owning project locks so a concurrent build
|
||||
# claim cannot appear between validation and Cloud deletion.
|
||||
projects = ProjectManager.find_projects_by_graph_id(graph_id)
|
||||
if any(_project_has_active_build(project) for project in projects):
|
||||
return jsonify({
|
||||
"success": False,
|
||||
"error": t('api.graphBuilding')
|
||||
}), 409
|
||||
|
||||
_delete_cloud_graph_if_present(graph_id)
|
||||
|
||||
for project in projects:
|
||||
_clear_project_graph_reference(project)
|
||||
project.status = (
|
||||
ProjectStatus.ONTOLOGY_GENERATED
|
||||
if project.ontology
|
||||
else ProjectStatus.CREATED
|
||||
)
|
||||
ProjectManager.save_project(project)
|
||||
|
||||
return jsonify({
|
||||
"success": True,
|
||||
"message": t('api.graphDeleted', id=graph_id)
|
||||
})
|
||||
|
||||
except GraphInUseError as e:
|
||||
return jsonify({"success": False, "error": str(e)}), 409
|
||||
except Exception as e:
|
||||
return jsonify({
|
||||
"success": False,
|
||||
|
|
|
|||
|
|
@ -12,10 +12,17 @@ from . import report_bp
|
|||
from ..config import Config
|
||||
from ..services.report_agent import ReportAgent, ReportManager, ReportStatus
|
||||
from ..services.simulation_manager import SimulationManager
|
||||
from ..models.project import ProjectManager
|
||||
from ..services.simulation_runner import SimulationRunner, RunnerStatus
|
||||
from ..services.zep_graph_memory_updater import ZepGraphMemoryManager
|
||||
from ..models.project import ProjectManager, ProjectStatus
|
||||
from ..models.task import TaskManager, TaskStatus
|
||||
from ..utils.logger import get_logger
|
||||
from ..utils.locale import t, get_locale, set_locale
|
||||
from ..utils.zep_lifecycle import (
|
||||
graph_lifecycle_lock,
|
||||
register_graph_reader,
|
||||
unregister_graph_reader,
|
||||
)
|
||||
|
||||
logger = get_logger('mirofish.api.report')
|
||||
|
||||
|
|
@ -58,6 +65,11 @@ def generate_report():
|
|||
}), 400
|
||||
|
||||
force_regenerate = data.get('force_regenerate', False)
|
||||
if not isinstance(force_regenerate, bool):
|
||||
return jsonify({
|
||||
"success": False,
|
||||
"error": "force_regenerate must be a JSON boolean",
|
||||
}), 400
|
||||
|
||||
# 获取模拟信息
|
||||
manager = SimulationManager()
|
||||
|
|
@ -69,21 +81,41 @@ def generate_report():
|
|||
"error": t('api.simulationNotFound', id=simulation_id)
|
||||
}), 404
|
||||
|
||||
# 检查是否已有报告
|
||||
if not force_regenerate:
|
||||
existing_report = ReportManager.get_report_by_simulation(simulation_id)
|
||||
if existing_report and existing_report.status == ReportStatus.COMPLETED:
|
||||
return jsonify({
|
||||
"success": True,
|
||||
"data": {
|
||||
"simulation_id": simulation_id,
|
||||
"report_id": existing_report.report_id,
|
||||
"status": "completed",
|
||||
"message": t('api.reportAlreadyExists'),
|
||||
"already_generated": True
|
||||
}
|
||||
})
|
||||
|
||||
run_state = SimulationRunner.get_run_state(simulation_id)
|
||||
updater = ZepGraphMemoryManager.get_updater(simulation_id)
|
||||
active_statuses = {
|
||||
RunnerStatus.STARTING,
|
||||
RunnerStatus.RUNNING,
|
||||
RunnerStatus.PAUSED,
|
||||
RunnerStatus.STOPPING,
|
||||
}
|
||||
if updater is not None or (
|
||||
run_state is not None and run_state.runner_status in active_statuses
|
||||
):
|
||||
return jsonify({
|
||||
"success": False,
|
||||
"error": (
|
||||
"Simulation or Zep graph ingestion is still active; "
|
||||
"wait for a terminal run status before generating a report"
|
||||
),
|
||||
"ingestion_pending": updater is not None,
|
||||
}), 409
|
||||
successful_terminal_statuses = {
|
||||
RunnerStatus.COMPLETED,
|
||||
RunnerStatus.STOPPED,
|
||||
}
|
||||
if (
|
||||
run_state is None
|
||||
or run_state.runner_status not in successful_terminal_statuses
|
||||
):
|
||||
return jsonify({
|
||||
"success": False,
|
||||
"error": (
|
||||
"A successfully completed or stopped simulation is required "
|
||||
"before generating a report"
|
||||
),
|
||||
}), 409
|
||||
|
||||
# 获取项目信息
|
||||
project = ProjectManager.get_project(state.project_id)
|
||||
if not project:
|
||||
|
|
@ -92,12 +124,26 @@ def generate_report():
|
|||
"error": t('api.projectNotFound', id=state.project_id)
|
||||
}), 404
|
||||
|
||||
graph_id = state.graph_id or project.graph_id
|
||||
if project.status != ProjectStatus.GRAPH_COMPLETED:
|
||||
return jsonify({
|
||||
"success": False,
|
||||
"error": "The project graph must be completely built before reporting",
|
||||
}), 409
|
||||
|
||||
graph_id = project.graph_id
|
||||
if not graph_id:
|
||||
return jsonify({
|
||||
"success": False,
|
||||
"error": t('api.missingGraphIdEnsure')
|
||||
}), 400
|
||||
if state.graph_id and state.graph_id != graph_id:
|
||||
return jsonify({
|
||||
"success": False,
|
||||
"error": (
|
||||
"The simulation references an older graph; prepare it "
|
||||
"again before generating a report"
|
||||
),
|
||||
}), 409
|
||||
|
||||
simulation_requirement = project.simulation_requirement
|
||||
if not simulation_requirement:
|
||||
|
|
@ -110,74 +156,147 @@ def generate_report():
|
|||
import uuid
|
||||
report_id = f"report_{uuid.uuid4().hex[:12]}"
|
||||
|
||||
# 创建异步任务
|
||||
task_manager = TaskManager()
|
||||
task_id = task_manager.create_task(
|
||||
task_type="report_generate",
|
||||
metadata={
|
||||
"simulation_id": simulation_id,
|
||||
"graph_id": graph_id,
|
||||
"report_id": report_id
|
||||
}
|
||||
)
|
||||
|
||||
# Capture locale before spawning background thread
|
||||
current_locale = get_locale()
|
||||
# Register the background report as a graph reader under the same lock
|
||||
# used by graph deletion and updater startup. A lock itself cannot be
|
||||
# acquired in this request thread and released by the worker, so the
|
||||
# durable reader registration is the cross-thread lease.
|
||||
with graph_lifecycle_lock(graph_id):
|
||||
refreshed_state = manager.get_simulation(simulation_id)
|
||||
refreshed_project = (
|
||||
ProjectManager.get_project(refreshed_state.project_id)
|
||||
if refreshed_state
|
||||
else None
|
||||
)
|
||||
refreshed_run_state = SimulationRunner.get_run_state(simulation_id)
|
||||
refreshed_updater = ZepGraphMemoryManager.get_updater(simulation_id)
|
||||
if (
|
||||
refreshed_state is None
|
||||
or refreshed_project is None
|
||||
or refreshed_project.graph_id != graph_id
|
||||
or refreshed_project.status != ProjectStatus.GRAPH_COMPLETED
|
||||
or (
|
||||
refreshed_state.graph_id
|
||||
and refreshed_state.graph_id != graph_id
|
||||
)
|
||||
):
|
||||
return jsonify({
|
||||
"success": False,
|
||||
"error": "The project graph changed while reporting was starting",
|
||||
}), 409
|
||||
if refreshed_updater is not None or (
|
||||
refreshed_run_state is not None
|
||||
and refreshed_run_state.runner_status in active_statuses
|
||||
):
|
||||
return jsonify({
|
||||
"success": False,
|
||||
"error": (
|
||||
"Simulation or Zep graph ingestion became active; "
|
||||
"retry after it reaches a terminal state"
|
||||
),
|
||||
"ingestion_pending": refreshed_updater is not None,
|
||||
}), 409
|
||||
if (
|
||||
refreshed_run_state is None
|
||||
or refreshed_run_state.runner_status
|
||||
not in successful_terminal_statuses
|
||||
):
|
||||
return jsonify({
|
||||
"success": False,
|
||||
"error": (
|
||||
"A successfully completed or stopped simulation is "
|
||||
"required before generating a report"
|
||||
),
|
||||
}), 409
|
||||
|
||||
# 定义后台任务
|
||||
def run_generate():
|
||||
set_locale(current_locale)
|
||||
try:
|
||||
task_manager.update_task(
|
||||
task_id,
|
||||
status=TaskStatus.PROCESSING,
|
||||
progress=0,
|
||||
message=t('api.initReportAgent')
|
||||
# Cached-report reuse is now part of the same atomic barrier, so a
|
||||
# concurrent rerun cannot make the returned report stale between
|
||||
# the status check and response.
|
||||
if not force_regenerate:
|
||||
existing_report = ReportManager.get_report_by_simulation(
|
||||
simulation_id
|
||||
)
|
||||
|
||||
# 创建Report Agent
|
||||
agent = ReportAgent(
|
||||
graph_id=graph_id,
|
||||
simulation_id=simulation_id,
|
||||
simulation_requirement=simulation_requirement
|
||||
)
|
||||
|
||||
# 进度回调
|
||||
def progress_callback(stage, progress, message):
|
||||
if (
|
||||
existing_report
|
||||
and existing_report.status == ReportStatus.COMPLETED
|
||||
):
|
||||
return jsonify({
|
||||
"success": True,
|
||||
"data": {
|
||||
"simulation_id": simulation_id,
|
||||
"report_id": existing_report.report_id,
|
||||
"status": "completed",
|
||||
"message": t('api.reportAlreadyExists'),
|
||||
"already_generated": True
|
||||
}
|
||||
})
|
||||
|
||||
task_manager = TaskManager()
|
||||
task_id = task_manager.create_task(
|
||||
task_type="report_generate",
|
||||
metadata={
|
||||
"simulation_id": simulation_id,
|
||||
"graph_id": graph_id,
|
||||
"report_id": report_id
|
||||
}
|
||||
)
|
||||
current_locale = get_locale()
|
||||
register_graph_reader(graph_id, report_id)
|
||||
|
||||
def run_generate():
|
||||
set_locale(current_locale)
|
||||
try:
|
||||
task_manager.update_task(
|
||||
task_id,
|
||||
progress=progress,
|
||||
message=f"[{stage}] {message}"
|
||||
status=TaskStatus.PROCESSING,
|
||||
progress=0,
|
||||
message=t('api.initReportAgent')
|
||||
)
|
||||
|
||||
# 生成报告(传入预先生成的 report_id)
|
||||
report = agent.generate_report(
|
||||
progress_callback=progress_callback,
|
||||
report_id=report_id
|
||||
)
|
||||
|
||||
# 保存报告
|
||||
ReportManager.save_report(report)
|
||||
|
||||
if report.status == ReportStatus.COMPLETED:
|
||||
task_manager.complete_task(
|
||||
task_id,
|
||||
result={
|
||||
"report_id": report.report_id,
|
||||
"simulation_id": simulation_id,
|
||||
"status": "completed"
|
||||
}
|
||||
|
||||
agent = ReportAgent(
|
||||
graph_id=graph_id,
|
||||
simulation_id=simulation_id,
|
||||
simulation_requirement=simulation_requirement
|
||||
)
|
||||
else:
|
||||
task_manager.fail_task(task_id, report.error or t('api.reportGenerateFailed'))
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"报告生成失败: {str(e)}")
|
||||
task_manager.fail_task(task_id, str(e))
|
||||
|
||||
# 启动后台线程
|
||||
thread = threading.Thread(target=run_generate, daemon=True)
|
||||
thread.start()
|
||||
|
||||
def progress_callback(stage, progress, message):
|
||||
task_manager.update_task(
|
||||
task_id,
|
||||
progress=progress,
|
||||
message=f"[{stage}] {message}"
|
||||
)
|
||||
|
||||
report = agent.generate_report(
|
||||
progress_callback=progress_callback,
|
||||
report_id=report_id
|
||||
)
|
||||
ReportManager.save_report(report)
|
||||
|
||||
if report.status == ReportStatus.COMPLETED:
|
||||
task_manager.complete_task(
|
||||
task_id,
|
||||
result={
|
||||
"report_id": report.report_id,
|
||||
"simulation_id": simulation_id,
|
||||
"status": "completed"
|
||||
}
|
||||
)
|
||||
else:
|
||||
task_manager.fail_task(
|
||||
task_id,
|
||||
report.error or t('api.reportGenerateFailed')
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"报告生成失败: {str(e)}")
|
||||
task_manager.fail_task(task_id, str(e))
|
||||
finally:
|
||||
unregister_graph_reader(graph_id, report_id)
|
||||
|
||||
try:
|
||||
thread = threading.Thread(target=run_generate, daemon=True)
|
||||
thread.start()
|
||||
except Exception:
|
||||
unregister_graph_reader(graph_id, report_id)
|
||||
raise
|
||||
|
||||
return jsonify({
|
||||
"success": True,
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ Step2: Zep实体读取与过滤、OASIS模拟准备与运行(全程自动化
|
|||
|
||||
import os
|
||||
import traceback
|
||||
from contextlib import nullcontext
|
||||
from flask import request, jsonify, send_file
|
||||
|
||||
from . import simulation_bp
|
||||
|
|
@ -12,9 +13,15 @@ from ..config import Config
|
|||
from ..services.zep_entity_reader import ZepEntityReader
|
||||
from ..services.oasis_profile_generator import OasisProfileGenerator
|
||||
from ..services.simulation_manager import SimulationManager, SimulationStatus
|
||||
from ..services.simulation_runner import SimulationRunner, RunnerStatus
|
||||
from ..services.simulation_runner import (
|
||||
SimulationRunner,
|
||||
RunnerStatus,
|
||||
SimulationStopPending,
|
||||
)
|
||||
from ..services.zep_graph_memory_updater import ZepGraphMemoryManager
|
||||
from ..utils.logger import get_logger
|
||||
from ..utils.locale import t, get_locale, set_locale
|
||||
from ..utils.zep_lifecycle import get_graph_readers, graph_lifecycle_lock
|
||||
from ..models.project import ProjectManager
|
||||
|
||||
logger = get_logger('mirofish.api.simulation')
|
||||
|
|
@ -1546,6 +1553,16 @@ def start_simulation():
|
|||
max_rounds = data.get('max_rounds') # 可选:最大模拟轮数
|
||||
enable_graph_memory_update = data.get('enable_graph_memory_update', False) # 可选:是否启用图谱记忆更新
|
||||
force = data.get('force', False) # 可选:强制重新开始
|
||||
if not isinstance(enable_graph_memory_update, bool):
|
||||
return jsonify({
|
||||
"success": False,
|
||||
"error": "enable_graph_memory_update must be a JSON boolean",
|
||||
}), 400
|
||||
if not isinstance(force, bool):
|
||||
return jsonify({
|
||||
"success": False,
|
||||
"error": "force must be a JSON boolean",
|
||||
}), 400
|
||||
|
||||
# 验证 max_rounds 参数
|
||||
if max_rounds is not None:
|
||||
|
|
@ -1586,31 +1603,67 @@ def start_simulation():
|
|||
is_prepared, prepare_info = _check_simulation_prepared(simulation_id)
|
||||
|
||||
if is_prepared:
|
||||
# 准备工作已完成,检查是否有正在运行的进程
|
||||
if state.status == SimulationStatus.RUNNING:
|
||||
# 检查模拟进程是否真的在运行
|
||||
run_state = SimulationRunner.get_run_state(simulation_id)
|
||||
if run_state and run_state.runner_status.value == "running":
|
||||
# 进程确实在运行
|
||||
if force:
|
||||
# 强制模式:停止运行中的模拟
|
||||
logger.info(f"强制模式:停止运行中的模拟 {simulation_id}")
|
||||
try:
|
||||
SimulationRunner.stop_simulation(simulation_id)
|
||||
except Exception as e:
|
||||
logger.warning(f"停止模拟时出现警告: {str(e)}")
|
||||
else:
|
||||
return jsonify({
|
||||
"success": False,
|
||||
"error": t('api.simRunningForceHint')
|
||||
}), 400
|
||||
run_state = SimulationRunner.get_run_state(simulation_id)
|
||||
updater = ZepGraphMemoryManager.get_updater(simulation_id)
|
||||
needs_finalization = bool(
|
||||
run_state
|
||||
and run_state.runner_status in {
|
||||
RunnerStatus.RUNNING,
|
||||
RunnerStatus.PAUSED,
|
||||
RunnerStatus.STOPPING,
|
||||
RunnerStatus.FAILED,
|
||||
}
|
||||
and (
|
||||
run_state.runner_status
|
||||
in {
|
||||
RunnerStatus.RUNNING,
|
||||
RunnerStatus.PAUSED,
|
||||
RunnerStatus.STOPPING,
|
||||
}
|
||||
or updater is not None
|
||||
)
|
||||
)
|
||||
if needs_finalization:
|
||||
if not force:
|
||||
return jsonify({
|
||||
"success": False,
|
||||
"error": t('api.simRunningForceHint')
|
||||
}), 400
|
||||
logger.info(f"强制模式:先完成旧模拟终止 {simulation_id}")
|
||||
try:
|
||||
stopped = SimulationRunner.stop_simulation(simulation_id)
|
||||
except SimulationStopPending as error:
|
||||
return jsonify({
|
||||
"success": False,
|
||||
"pending": True,
|
||||
"error": str(error),
|
||||
}), 409
|
||||
except Exception as error:
|
||||
return jsonify({
|
||||
"success": False,
|
||||
"error": (
|
||||
"Cannot restart until the previous simulation "
|
||||
f"finalizes safely: {error}"
|
||||
),
|
||||
}), 409
|
||||
if stopped.runner_status != RunnerStatus.STOPPED:
|
||||
return jsonify({
|
||||
"success": False,
|
||||
"error": "Previous simulation did not reach STOPPED",
|
||||
}), 409
|
||||
|
||||
# 如果是强制模式,清理运行日志
|
||||
if force:
|
||||
logger.info(f"强制模式:清理模拟日志 {simulation_id}")
|
||||
cleanup_result = SimulationRunner.cleanup_simulation_logs(simulation_id)
|
||||
if not cleanup_result.get("success"):
|
||||
logger.warning(f"清理日志时出现警告: {cleanup_result.get('errors')}")
|
||||
return jsonify({
|
||||
"success": False,
|
||||
"error": (
|
||||
"Failed to clean previous simulation logs: "
|
||||
f"{cleanup_result.get('errors')}"
|
||||
),
|
||||
}), 500
|
||||
force_restarted = True
|
||||
|
||||
# 进程不存在或已结束,重置状态为 ready
|
||||
|
|
@ -1627,34 +1680,82 @@ def start_simulation():
|
|||
# 获取图谱ID(用于图谱记忆更新)
|
||||
graph_id = None
|
||||
if enable_graph_memory_update:
|
||||
# 从模拟状态或项目中获取 graph_id
|
||||
graph_id = state.graph_id
|
||||
if not graph_id:
|
||||
# 尝试从项目中获取
|
||||
project = ProjectManager.get_project(state.project_id)
|
||||
if project:
|
||||
graph_id = project.graph_id
|
||||
|
||||
# The project is authoritative. A graph ID copied into an older
|
||||
# simulation can outlive a project reset/rebuild and must not be
|
||||
# used to resurrect writes to a deleted graph.
|
||||
project = ProjectManager.get_project(state.project_id)
|
||||
graph_id = project.graph_id if project else None
|
||||
if not graph_id:
|
||||
return jsonify({
|
||||
"success": False,
|
||||
"error": t('api.graphIdRequiredForMemory')
|
||||
}), 400
|
||||
|
||||
logger.info(f"启用图谱记忆更新: simulation_id={simulation_id}, graph_id={graph_id}")
|
||||
|
||||
# 启动模拟
|
||||
run_state = SimulationRunner.start_simulation(
|
||||
simulation_id=simulation_id,
|
||||
platform=platform,
|
||||
max_rounds=max_rounds,
|
||||
enable_graph_memory_update=enable_graph_memory_update,
|
||||
graph_id=graph_id
|
||||
|
||||
graph_guard = (
|
||||
graph_lifecycle_lock(graph_id)
|
||||
if enable_graph_memory_update
|
||||
else nullcontext()
|
||||
)
|
||||
|
||||
# 更新模拟状态
|
||||
state.status = SimulationStatus.RUNNING
|
||||
manager._save_simulation_state(state)
|
||||
with graph_guard:
|
||||
if enable_graph_memory_update:
|
||||
# Re-read both references under the same per-graph lock used
|
||||
# by reset/delete. Keep the lock through updater creation in
|
||||
# start_simulation so check -> claim is atomic.
|
||||
refreshed_state = manager.get_simulation(simulation_id)
|
||||
refreshed_project = (
|
||||
ProjectManager.get_project(refreshed_state.project_id)
|
||||
if refreshed_state
|
||||
else None
|
||||
)
|
||||
current_graph_id = (
|
||||
refreshed_project.graph_id if refreshed_project else None
|
||||
)
|
||||
if current_graph_id != graph_id:
|
||||
return jsonify({
|
||||
"success": False,
|
||||
"error": (
|
||||
"The project graph changed while the simulation "
|
||||
"was starting; retry after refreshing the project"
|
||||
),
|
||||
}), 409
|
||||
if (
|
||||
refreshed_state.graph_id
|
||||
and refreshed_state.graph_id != current_graph_id
|
||||
):
|
||||
return jsonify({
|
||||
"success": False,
|
||||
"error": (
|
||||
"The simulation references an older graph; "
|
||||
"prepare it again before enabling graph memory"
|
||||
),
|
||||
}), 409
|
||||
active_reports = get_graph_readers(graph_id)
|
||||
if active_reports:
|
||||
return jsonify({
|
||||
"success": False,
|
||||
"error": (
|
||||
"A report is currently reading this graph; wait "
|
||||
"for report generation to finish before enabling "
|
||||
"graph memory updates"
|
||||
),
|
||||
"active_reports": active_reports,
|
||||
}), 409
|
||||
state = refreshed_state
|
||||
logger.info(
|
||||
"启用图谱记忆更新: simulation_id=%s, graph_id=%s",
|
||||
simulation_id,
|
||||
graph_id,
|
||||
)
|
||||
|
||||
# 启动模拟。启用图谱写入时仍持有 graph_guard,直到 updater
|
||||
# claim 与进程资源全部发布完成。
|
||||
run_state = SimulationRunner.start_simulation(
|
||||
simulation_id=simulation_id,
|
||||
platform=platform,
|
||||
max_rounds=max_rounds,
|
||||
enable_graph_memory_update=enable_graph_memory_update,
|
||||
graph_id=graph_id
|
||||
)
|
||||
|
||||
response_data = run_state.to_dict()
|
||||
if max_rounds:
|
||||
|
|
@ -1720,14 +1821,22 @@ def stop_simulation():
|
|||
manager = SimulationManager()
|
||||
state = manager.get_simulation(simulation_id)
|
||||
if state:
|
||||
state.status = SimulationStatus.PAUSED
|
||||
state.status = SimulationStatus.STOPPED
|
||||
state.error = None
|
||||
manager._save_simulation_state(state)
|
||||
|
||||
return jsonify({
|
||||
"success": True,
|
||||
"data": run_state.to_dict()
|
||||
})
|
||||
|
||||
|
||||
except SimulationStopPending as e:
|
||||
return jsonify({
|
||||
"success": False,
|
||||
"pending": True,
|
||||
"error": str(e),
|
||||
}), 202
|
||||
|
||||
except ValueError as e:
|
||||
return jsonify({
|
||||
"success": False,
|
||||
|
|
@ -1736,6 +1845,14 @@ def stop_simulation():
|
|||
|
||||
except Exception as e:
|
||||
logger.error(f"停止模拟失败: {str(e)}")
|
||||
simulation_id = (request.get_json(silent=True) or {}).get('simulation_id')
|
||||
if simulation_id:
|
||||
manager = SimulationManager()
|
||||
state = manager.get_simulation(simulation_id)
|
||||
if state:
|
||||
state.status = SimulationStatus.FAILED
|
||||
state.error = str(e)
|
||||
manager._save_simulation_state(state)
|
||||
return jsonify({
|
||||
"success": False,
|
||||
"error": str(e),
|
||||
|
|
|
|||
|
|
@ -14,6 +14,29 @@ else:
|
|||
load_dotenv(override=True)
|
||||
|
||||
|
||||
def _parse_number(name, raw_value, *, default, cast):
|
||||
"""Parse a numeric setting without making module import fail."""
|
||||
|
||||
try:
|
||||
return cast(raw_value), None
|
||||
except (TypeError, ValueError):
|
||||
return default, f"{name} must be a number"
|
||||
|
||||
|
||||
_zep_request_timeout, _zep_request_timeout_error = _parse_number(
|
||||
"ZEP_REQUEST_TIMEOUT_SECONDS",
|
||||
os.environ.get("ZEP_REQUEST_TIMEOUT_SECONDS", "30"),
|
||||
default=30.0,
|
||||
cast=float,
|
||||
)
|
||||
_zep_ingestion_timeout, _zep_ingestion_timeout_error = _parse_number(
|
||||
"ZEP_INGESTION_TIMEOUT_SECONDS",
|
||||
os.environ.get("ZEP_INGESTION_TIMEOUT_SECONDS", "600"),
|
||||
default=600,
|
||||
cast=int,
|
||||
)
|
||||
|
||||
|
||||
class Config:
|
||||
"""Flask配置类"""
|
||||
|
||||
|
|
@ -31,6 +54,13 @@ class Config:
|
|||
|
||||
# Zep配置
|
||||
ZEP_API_KEY = os.environ.get('ZEP_API_KEY')
|
||||
ZEP_REQUEST_TIMEOUT_SECONDS = _zep_request_timeout
|
||||
ZEP_INGESTION_TIMEOUT_SECONDS = _zep_ingestion_timeout
|
||||
_ZEP_CONFIG_PARSE_ERRORS = tuple(
|
||||
error
|
||||
for error in (_zep_request_timeout_error, _zep_ingestion_timeout_error)
|
||||
if error
|
||||
)
|
||||
|
||||
# 文件上传配置
|
||||
MAX_CONTENT_LENGTH = 50 * 1024 * 1024 # 50MB
|
||||
|
|
@ -68,6 +98,13 @@ class Config:
|
|||
errors.append("LLM_API_KEY 未配置")
|
||||
if not cls.ZEP_API_KEY:
|
||||
errors.append("ZEP_API_KEY 未配置")
|
||||
if os.environ.get("ZEP_API_URL"):
|
||||
errors.append("ZEP_API_URL 不受支持;MiroFish 仅连接 Zep Cloud")
|
||||
errors.extend(cls._ZEP_CONFIG_PARSE_ERRORS)
|
||||
if cls.ZEP_REQUEST_TIMEOUT_SECONDS <= 0:
|
||||
errors.append("ZEP_REQUEST_TIMEOUT_SECONDS 必须大于 0")
|
||||
if cls.ZEP_INGESTION_TIMEOUT_SECONDS <= 0:
|
||||
errors.append("ZEP_INGESTION_TIMEOUT_SECONDS 必须大于 0")
|
||||
if cls.DEBUG:
|
||||
import warnings
|
||||
warnings.warn("Flask DEBUG mode is enabled. Do not use in production.", RuntimeWarning)
|
||||
|
|
|
|||
|
|
@ -43,6 +43,8 @@ class Project:
|
|||
# 图谱信息(接口2完成后填充)
|
||||
graph_id: Optional[str] = None
|
||||
graph_build_task_id: Optional[str] = None
|
||||
zep_batch_id: Optional[str] = None
|
||||
zep_batch_operation_id: Optional[str] = None
|
||||
|
||||
# 配置
|
||||
simulation_requirement: Optional[str] = None
|
||||
|
|
@ -66,6 +68,8 @@ class Project:
|
|||
"analysis_summary": self.analysis_summary,
|
||||
"graph_id": self.graph_id,
|
||||
"graph_build_task_id": self.graph_build_task_id,
|
||||
"zep_batch_id": self.zep_batch_id,
|
||||
"zep_batch_operation_id": self.zep_batch_operation_id,
|
||||
"simulation_requirement": self.simulation_requirement,
|
||||
"chunk_size": self.chunk_size,
|
||||
"chunk_overlap": self.chunk_overlap,
|
||||
|
|
@ -91,6 +95,8 @@ class Project:
|
|||
analysis_summary=data.get('analysis_summary'),
|
||||
graph_id=data.get('graph_id'),
|
||||
graph_build_task_id=data.get('graph_build_task_id'),
|
||||
zep_batch_id=data.get('zep_batch_id'),
|
||||
zep_batch_operation_id=data.get('zep_batch_operation_id'),
|
||||
simulation_requirement=data.get('simulation_requirement'),
|
||||
chunk_size=data.get('chunk_size', 500),
|
||||
chunk_overlap=data.get('chunk_overlap', 50),
|
||||
|
|
@ -195,7 +201,7 @@ class ProjectManager:
|
|||
return Project.from_dict(data)
|
||||
|
||||
@classmethod
|
||||
def list_projects(cls, limit: int = 50) -> List[Project]:
|
||||
def list_projects(cls, limit: Optional[int] = 50) -> List[Project]:
|
||||
"""
|
||||
列出所有项目
|
||||
|
||||
|
|
@ -216,7 +222,17 @@ class ProjectManager:
|
|||
# 按创建时间倒序排序
|
||||
projects.sort(key=lambda p: p.created_at, reverse=True)
|
||||
|
||||
return projects[:limit]
|
||||
return projects if limit is None else projects[:limit]
|
||||
|
||||
@classmethod
|
||||
def find_projects_by_graph_id(cls, graph_id: str) -> List[Project]:
|
||||
"""Return every persisted project that references a Cloud graph."""
|
||||
|
||||
return [
|
||||
project
|
||||
for project in cls.list_projects(limit=None)
|
||||
if project.graph_id == graph_id
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def delete_project(cls, project_id: str) -> bool:
|
||||
|
|
@ -302,4 +318,3 @@ class ProjectManager:
|
|||
for f in os.listdir(files_dir)
|
||||
if os.path.isfile(os.path.join(files_dir, f))
|
||||
]
|
||||
|
||||
|
|
|
|||
|
|
@ -3,15 +3,14 @@
|
|||
接口2:使用Zep API构建Standalone Graph
|
||||
"""
|
||||
|
||||
import os
|
||||
import hashlib
|
||||
import uuid
|
||||
import time
|
||||
import threading
|
||||
from typing import Dict, Any, List, Optional, Callable
|
||||
from dataclasses import dataclass
|
||||
|
||||
from zep_cloud.client import Zep
|
||||
from zep_cloud import EpisodeData, EntityEdgeSourceTarget
|
||||
from zep_cloud import BatchAddItem, EntityEdgeSourceTarget, NotFoundError
|
||||
|
||||
from ..config import Config
|
||||
from ..models.task import TaskManager, TaskStatus
|
||||
|
|
@ -20,6 +19,12 @@ from ..utils.ontology import (
|
|||
MAX_ONTOLOGY_TYPES,
|
||||
RESERVED_ONTOLOGY_ATTRIBUTE_NAMES,
|
||||
normalize_ontology_attributes,
|
||||
normalize_ontology_source_targets,
|
||||
)
|
||||
from ..utils.zep import (
|
||||
call_zep_read_with_retry,
|
||||
get_zep_client,
|
||||
is_retryable_zep_error,
|
||||
)
|
||||
from .text_processor import TextProcessor
|
||||
from ..utils.locale import t, get_locale, set_locale
|
||||
|
|
@ -42,6 +47,16 @@ class GraphInfo:
|
|||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BatchSubmission:
|
||||
"""Durable identity for one Zep Batch API ingestion operation."""
|
||||
|
||||
batch_id: str
|
||||
operation_id: str
|
||||
episode_uuids: List[str]
|
||||
item_count: int
|
||||
|
||||
|
||||
class GraphBuilderService:
|
||||
"""
|
||||
图谱构建服务
|
||||
|
|
@ -53,7 +68,7 @@ class GraphBuilderService:
|
|||
if not self.api_key:
|
||||
raise ValueError("ZEP_API_KEY 未配置")
|
||||
|
||||
self.client = Zep(api_key=self.api_key)
|
||||
self.client = get_zep_client(self.api_key)
|
||||
self.task_manager = TaskManager()
|
||||
|
||||
def build_graph_async(
|
||||
|
|
@ -63,7 +78,7 @@ class GraphBuilderService:
|
|||
graph_name: str = "MiroFish Graph",
|
||||
chunk_size: int = 500,
|
||||
chunk_overlap: int = 50,
|
||||
batch_size: int = 3
|
||||
batch_size: int = 350
|
||||
) -> str:
|
||||
"""
|
||||
异步构建图谱
|
||||
|
|
@ -123,6 +138,12 @@ class GraphBuilderService:
|
|||
message=t('progress.startBuildingGraph')
|
||||
)
|
||||
|
||||
# Validate the complete ingestion payload before the first Cloud
|
||||
# mutation, including this legacy service entry point.
|
||||
chunks = TextProcessor.split_text(text, chunk_size, chunk_overlap)
|
||||
self.validate_batch_chunks(chunks, batch_size=batch_size)
|
||||
total_chunks = len(chunks)
|
||||
|
||||
# 1. 创建图谱
|
||||
graph_id = self.create_graph(graph_name)
|
||||
self.task_manager.update_task(
|
||||
|
|
@ -139,9 +160,7 @@ class GraphBuilderService:
|
|||
message=t('progress.ontologySet')
|
||||
)
|
||||
|
||||
# 3. 文本分块
|
||||
chunks = TextProcessor.split_text(text, chunk_size, chunk_overlap)
|
||||
total_chunks = len(chunks)
|
||||
# 3. 文本分块已在 Cloud mutation 前完成并验证
|
||||
self.task_manager.update_task(
|
||||
task_id,
|
||||
progress=20,
|
||||
|
|
@ -149,7 +168,7 @@ class GraphBuilderService:
|
|||
)
|
||||
|
||||
# 4. 分批发送数据
|
||||
episode_uuids = self.add_text_batches(
|
||||
submission = self.add_text_batches(
|
||||
graph_id, chunks, batch_size,
|
||||
lambda msg, prog: self.task_manager.update_task(
|
||||
task_id,
|
||||
|
|
@ -165,8 +184,8 @@ class GraphBuilderService:
|
|||
message=t('progress.waitingZepProcess')
|
||||
)
|
||||
|
||||
self._wait_for_episodes(
|
||||
episode_uuids,
|
||||
self._wait_for_batch(
|
||||
submission,
|
||||
lambda msg, prog: self.task_manager.update_task(
|
||||
task_id,
|
||||
progress=60 + int(prog * 0.3), # 60-90%
|
||||
|
|
@ -195,17 +214,100 @@ class GraphBuilderService:
|
|||
error_msg = f"{str(e)}\n{traceback.format_exc()}"
|
||||
self.task_manager.fail_task(task_id, error_msg)
|
||||
|
||||
def create_graph(self, name: str) -> str:
|
||||
"""创建Zep图谱(公开方法)"""
|
||||
graph_id = f"mirofish_{uuid.uuid4().hex[:16]}"
|
||||
|
||||
self.client.graph.create(
|
||||
graph_id=graph_id,
|
||||
name=name,
|
||||
description="MiroFish Social Simulation Graph"
|
||||
)
|
||||
|
||||
def create_graph(
|
||||
self,
|
||||
name: str,
|
||||
*,
|
||||
graph_id: str | None = None,
|
||||
graph_id_callback: Optional[Callable[[str], None]] = None,
|
||||
) -> str:
|
||||
"""Create a graph with a caller-durable ID and reconcile lost replies."""
|
||||
|
||||
graph_id = graph_id or f"mirofish_{uuid.uuid4().hex[:16]}"
|
||||
# Persist the client-generated ID before the non-idempotent POST so a
|
||||
# later reset can clean up a graph whose successful response was lost.
|
||||
if graph_id_callback:
|
||||
graph_id_callback(graph_id)
|
||||
|
||||
try:
|
||||
self.client.graph.create(
|
||||
graph_id=graph_id,
|
||||
name=name,
|
||||
description="MiroFish Social Simulation Graph"
|
||||
)
|
||||
except Exception as error:
|
||||
if not is_retryable_zep_error(error):
|
||||
raise
|
||||
reconciliation_error = None
|
||||
for attempt in range(3):
|
||||
try:
|
||||
call_zep_read_with_retry(
|
||||
lambda: self.client.graph.get(graph_id),
|
||||
operation_name=f"reconcile graph create {graph_id}",
|
||||
)
|
||||
reconciliation_error = None
|
||||
break
|
||||
except NotFoundError as not_found:
|
||||
reconciliation_error = not_found
|
||||
if attempt < 2:
|
||||
time.sleep(attempt + 1)
|
||||
except Exception as read_error:
|
||||
reconciliation_error = read_error
|
||||
break
|
||||
if reconciliation_error is not None:
|
||||
raise error from reconciliation_error
|
||||
|
||||
return graph_id
|
||||
|
||||
@staticmethod
|
||||
def build_operation_id(graph_id: str, chunks: List[str]) -> str:
|
||||
payload_hash = hashlib.sha256("\0".join(chunks).encode("utf-8")).hexdigest()
|
||||
return hashlib.sha256(
|
||||
f"{graph_id}:{payload_hash}".encode("utf-8")
|
||||
).hexdigest()
|
||||
|
||||
def _find_batch_by_operation_id(
|
||||
self,
|
||||
graph_id: str,
|
||||
operation_id: str,
|
||||
*,
|
||||
max_attempts: int = 3,
|
||||
) -> Any | None:
|
||||
"""Find one server-created batch after an ambiguous create reply."""
|
||||
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
matches: List[Any] = []
|
||||
cursor: int | None = None
|
||||
seen_cursors: set[int] = set()
|
||||
while True:
|
||||
page = call_zep_read_with_retry(
|
||||
lambda: self.client.batch.list(limit=100, cursor=cursor),
|
||||
operation_name=f"reconcile batch create {operation_id}",
|
||||
)
|
||||
for batch in getattr(page, "batches", None) or []:
|
||||
metadata = getattr(batch, "metadata", None) or {}
|
||||
if (
|
||||
metadata.get("mirofish_operation_id") == operation_id
|
||||
and metadata.get("graph_id") == graph_id
|
||||
):
|
||||
matches.append(batch)
|
||||
next_cursor = getattr(page, "next_cursor", None)
|
||||
if next_cursor is None:
|
||||
break
|
||||
if next_cursor == cursor or next_cursor in seen_cursors:
|
||||
raise RuntimeError("Zep batch list cursor did not advance")
|
||||
seen_cursors.add(next_cursor)
|
||||
cursor = next_cursor
|
||||
|
||||
if len(matches) > 1:
|
||||
raise RuntimeError(
|
||||
f"Multiple Zep batches match operation {operation_id}; refusing ambiguity"
|
||||
)
|
||||
if matches:
|
||||
return matches[0]
|
||||
if attempt < max_attempts:
|
||||
time.sleep(attempt)
|
||||
return None
|
||||
|
||||
def set_ontology(self, graph_id: str, ontology: Dict[str, Any]):
|
||||
"""设置图谱本体(公开方法)"""
|
||||
|
|
@ -278,7 +380,9 @@ class GraphBuilderService:
|
|||
|
||||
# 构建source_targets
|
||||
source_targets = []
|
||||
for st in edge_def.get("source_targets", []):
|
||||
for st in normalize_ontology_source_targets(
|
||||
edge_def.get("source_targets", [])
|
||||
):
|
||||
source_targets.append(
|
||||
EntityEdgeSourceTarget(
|
||||
source=st.get("source", "Entity"),
|
||||
|
|
@ -293,8 +397,8 @@ class GraphBuilderService:
|
|||
if entity_types or edge_definitions:
|
||||
self.client.graph.set_ontology(
|
||||
graph_ids=[graph_id],
|
||||
# zep-cloud 3.13.0 iterates entities.items(), so an edge-only
|
||||
# ontology must pass an empty dictionary rather than None.
|
||||
# Zep iterates entities.items(), so edge-only ontologies must
|
||||
# pass an empty dictionary rather than None.
|
||||
entities=entity_types,
|
||||
edges=edge_definitions if edge_definitions else None,
|
||||
)
|
||||
|
|
@ -303,13 +407,53 @@ class GraphBuilderService:
|
|||
self,
|
||||
graph_id: str,
|
||||
chunks: List[str],
|
||||
batch_size: int = 3,
|
||||
progress_callback: Optional[Callable] = None
|
||||
) -> List[str]:
|
||||
"""分批添加文本到图谱,返回所有 episode 的 uuid 列表"""
|
||||
episode_uuids = []
|
||||
batch_size: int = 350,
|
||||
progress_callback: Optional[Callable] = None,
|
||||
batch_created_callback: Optional[Callable[[str | None, str], None]] = None,
|
||||
) -> BatchSubmission:
|
||||
"""Submit document chunks through Zep's current Batch API.
|
||||
|
||||
Mutating calls are deliberately not retried: create/add are not
|
||||
documented as idempotent, and an ambiguous replay can duplicate graph
|
||||
episodes. The returned batch identity allows callers to persist and
|
||||
reconcile the operation instead.
|
||||
"""
|
||||
|
||||
if not graph_id:
|
||||
raise ValueError("graph_id is required")
|
||||
self.validate_batch_chunks(chunks, batch_size=batch_size)
|
||||
|
||||
total_chunks = len(chunks)
|
||||
|
||||
operation_id = self.build_operation_id(graph_id, chunks)
|
||||
if batch_created_callback:
|
||||
# Journal the deterministic operation before the server-generated
|
||||
# batch ID POST. This leaves enough identity for later diagnosis
|
||||
# even if both the response and immediate list reconciliation fail.
|
||||
batch_created_callback(None, operation_id)
|
||||
|
||||
try:
|
||||
batch = self.client.batch.create(
|
||||
metadata={
|
||||
"mirofish_operation_id": operation_id,
|
||||
"graph_id": graph_id,
|
||||
"chunk_count": total_chunks,
|
||||
}
|
||||
)
|
||||
except Exception as error:
|
||||
if not is_retryable_zep_error(error):
|
||||
raise
|
||||
batch = self._find_batch_by_operation_id(graph_id, operation_id)
|
||||
if batch is None:
|
||||
raise RuntimeError(
|
||||
"Zep batch creation is unconfirmed and no matching operation was found"
|
||||
) from error
|
||||
batch_id = getattr(batch, "batch_id", None)
|
||||
if not batch_id:
|
||||
raise RuntimeError("Zep Batch API returned no batch_id")
|
||||
if batch_created_callback:
|
||||
batch_created_callback(batch_id, operation_id)
|
||||
|
||||
episode_uuids: List[str] = []
|
||||
for i in range(0, total_chunks, batch_size):
|
||||
batch_chunks = chunks[i:i + batch_size]
|
||||
batch_num = i // batch_size + 1
|
||||
|
|
@ -322,34 +466,254 @@ class GraphBuilderService:
|
|||
progress
|
||||
)
|
||||
|
||||
# 构建episode数据
|
||||
episodes = [
|
||||
EpisodeData(data=chunk, type="text")
|
||||
for chunk in batch_chunks
|
||||
]
|
||||
|
||||
# 发送到Zep
|
||||
try:
|
||||
batch_result = self.client.graph.add_batch(
|
||||
items = [
|
||||
BatchAddItem(
|
||||
type="graph_episode",
|
||||
graph_id=graph_id,
|
||||
episodes=episodes
|
||||
data=chunk,
|
||||
data_type="text",
|
||||
source_description="MiroFish source document chunk",
|
||||
metadata={
|
||||
"mirofish_operation_id": operation_id,
|
||||
"chunk_index": i + offset,
|
||||
"chunk_sha256": hashlib.sha256(
|
||||
chunk.encode("utf-8")
|
||||
).hexdigest(),
|
||||
},
|
||||
)
|
||||
for offset, chunk in enumerate(batch_chunks)
|
||||
]
|
||||
|
||||
expected_item_count = i + len(items)
|
||||
try:
|
||||
item_details = self.client.batch.add(
|
||||
batch_id=batch_id,
|
||||
items=items,
|
||||
)
|
||||
|
||||
# 收集返回的 episode uuid
|
||||
if batch_result and isinstance(batch_result, list):
|
||||
for ep in batch_result:
|
||||
ep_uuid = getattr(ep, 'uuid_', None) or getattr(ep, 'uuid', None)
|
||||
if ep_uuid:
|
||||
episode_uuids.append(ep_uuid)
|
||||
|
||||
# 避免请求过快
|
||||
time.sleep(1)
|
||||
|
||||
except Exception as e:
|
||||
if progress_callback:
|
||||
progress_callback(t('progress.batchFailed', batch=batch_num, error=str(e)), 0)
|
||||
raise
|
||||
|
||||
if is_retryable_zep_error(e):
|
||||
recovered_items = self._reconcile_batch_item_count(
|
||||
batch_id,
|
||||
expected_item_count,
|
||||
)
|
||||
recovered_indexes = {
|
||||
getattr(item, "sequence_index", None)
|
||||
for item in recovered_items
|
||||
}
|
||||
if (
|
||||
len(recovered_items) == expected_item_count
|
||||
and recovered_indexes == set(range(expected_item_count))
|
||||
):
|
||||
item_details = recovered_items[i:expected_item_count]
|
||||
else:
|
||||
raise RuntimeError(
|
||||
f"Zep batch {batch_id} item submission is unconfirmed; "
|
||||
"the draft was not processed or replayed"
|
||||
) from e
|
||||
else:
|
||||
raise RuntimeError(
|
||||
f"Zep batch {batch_id} item submission failed"
|
||||
) from e
|
||||
|
||||
if len(item_details or []) != len(items):
|
||||
recovered_items = self._reconcile_batch_item_count(
|
||||
batch_id,
|
||||
expected_item_count,
|
||||
)
|
||||
recovered_indexes = {
|
||||
getattr(item, "sequence_index", None)
|
||||
for item in recovered_items
|
||||
}
|
||||
if (
|
||||
len(recovered_items) == expected_item_count
|
||||
and recovered_indexes == set(range(expected_item_count))
|
||||
):
|
||||
item_details = recovered_items[i:expected_item_count]
|
||||
else:
|
||||
raise RuntimeError(
|
||||
f"Zep batch {batch_id} acknowledged {len(item_details or [])} "
|
||||
f"of {len(items)} items"
|
||||
)
|
||||
for item in item_details:
|
||||
episode_uuid = getattr(item, "episode_uuid", None)
|
||||
if episode_uuid:
|
||||
episode_uuids.append(episode_uuid)
|
||||
|
||||
try:
|
||||
self.client.batch.process(batch_id=batch_id)
|
||||
except Exception as error:
|
||||
# A process response can be lost after the server accepted it.
|
||||
# Reconcile with a safe GET instead of issuing a second POST.
|
||||
summary = call_zep_read_with_retry(
|
||||
lambda: self.client.batch.get(batch_id=batch_id),
|
||||
operation_name=f"reconcile batch {batch_id}",
|
||||
)
|
||||
if getattr(summary, "status", None) in {None, "draft"}:
|
||||
raise RuntimeError(
|
||||
f"Zep batch {batch_id} processing is unconfirmed"
|
||||
) from error
|
||||
|
||||
return BatchSubmission(
|
||||
batch_id=batch_id,
|
||||
operation_id=operation_id,
|
||||
episode_uuids=episode_uuids,
|
||||
item_count=total_chunks,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def validate_batch_chunks(chunks: List[str], *, batch_size: int = 350) -> None:
|
||||
"""Validate every Batch API limit before the first Cloud mutation."""
|
||||
|
||||
if not chunks:
|
||||
raise ValueError("At least one text chunk is required")
|
||||
if not 1 <= batch_size <= 350:
|
||||
raise ValueError("batch_size must be between 1 and 350")
|
||||
if len(chunks) > 50_000:
|
||||
raise ValueError("A Zep batch cannot contain more than 50,000 items")
|
||||
oversized = [index for index, chunk in enumerate(chunks) if len(chunk) > 10_000]
|
||||
if oversized:
|
||||
raise ValueError(
|
||||
f"Zep batch item exceeds 10,000 characters at chunk {oversized[0]}"
|
||||
)
|
||||
|
||||
def _list_batch_items(self, batch_id: str) -> List[Any]:
|
||||
items: List[Any] = []
|
||||
cursor: int | None = None
|
||||
seen_cursors: set[int] = set()
|
||||
while True:
|
||||
page = call_zep_read_with_retry(
|
||||
lambda: self.client.batch.list_items(
|
||||
batch_id=batch_id,
|
||||
limit=100,
|
||||
cursor=cursor,
|
||||
),
|
||||
operation_name=f"list batch items {batch_id}",
|
||||
)
|
||||
items.extend(getattr(page, "items", None) or [])
|
||||
next_cursor = getattr(page, "next_cursor", None)
|
||||
if next_cursor is None:
|
||||
break
|
||||
if next_cursor == cursor or next_cursor in seen_cursors:
|
||||
raise RuntimeError(f"Zep batch {batch_id} item cursor did not advance")
|
||||
seen_cursors.add(next_cursor)
|
||||
cursor = next_cursor
|
||||
return items
|
||||
|
||||
def _reconcile_batch_item_count(
|
||||
self,
|
||||
batch_id: str,
|
||||
expected_item_count: int,
|
||||
*,
|
||||
max_attempts: int = 3,
|
||||
) -> List[Any]:
|
||||
"""Allow a short propagation window after an ambiguous add reply."""
|
||||
|
||||
items: List[Any] = []
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
items = self._list_batch_items(batch_id)
|
||||
if len(items) >= expected_item_count:
|
||||
return items
|
||||
if attempt < max_attempts:
|
||||
time.sleep(attempt)
|
||||
return items
|
||||
|
||||
def get_batch_summary(self, batch_id: str) -> Any:
|
||||
"""Read a persisted batch identity for restart reconciliation."""
|
||||
|
||||
return call_zep_read_with_retry(
|
||||
lambda: self.client.batch.get(batch_id=batch_id),
|
||||
operation_name=f"get batch {batch_id}",
|
||||
)
|
||||
|
||||
def _wait_for_batch(
|
||||
self,
|
||||
submission: BatchSubmission,
|
||||
progress_callback: Optional[Callable] = None,
|
||||
timeout: int | None = None,
|
||||
) -> List[str]:
|
||||
"""Wait for a Batch API terminal state and validate every item."""
|
||||
|
||||
timeout = timeout or Config.ZEP_INGESTION_TIMEOUT_SECONDS
|
||||
start_time = time.time()
|
||||
terminal_states = {"succeeded", "partial", "failed", "invalid", "canceled"}
|
||||
|
||||
while True:
|
||||
if time.time() - start_time > timeout:
|
||||
raise TimeoutError(
|
||||
f"Zep batch {submission.batch_id} did not finish within {timeout}s"
|
||||
)
|
||||
|
||||
summary = call_zep_read_with_retry(
|
||||
lambda: self.client.batch.get(batch_id=submission.batch_id),
|
||||
operation_name=f"poll batch {submission.batch_id}",
|
||||
)
|
||||
status = getattr(summary, "status", None)
|
||||
progress = getattr(summary, "progress", None)
|
||||
percent = float(getattr(progress, "percent_complete", 0) or 0) / 100
|
||||
if progress_callback:
|
||||
completed = int(getattr(progress, "succeeded_items", 0) or 0)
|
||||
progress_callback(
|
||||
t(
|
||||
'progress.zepProcessing',
|
||||
completed=completed,
|
||||
total=submission.item_count,
|
||||
pending=max(submission.item_count - completed, 0),
|
||||
elapsed=int(time.time() - start_time),
|
||||
),
|
||||
min(max(percent, 0.0), 1.0),
|
||||
)
|
||||
|
||||
if status in terminal_states:
|
||||
break
|
||||
time.sleep(3)
|
||||
|
||||
items = self._list_batch_items(submission.batch_id)
|
||||
if status != "succeeded":
|
||||
failed_items = [
|
||||
item for item in items
|
||||
if getattr(item, "status", None) not in {"succeeded", "skipped"}
|
||||
]
|
||||
first_error = getattr(failed_items[0], "error", None) if failed_items else None
|
||||
raise RuntimeError(
|
||||
f"Zep batch {submission.batch_id} ended as {status}; "
|
||||
f"failed_items={len(failed_items)}; first_error={first_error}"
|
||||
)
|
||||
if len(items) != submission.item_count:
|
||||
raise RuntimeError(
|
||||
f"Zep batch {submission.batch_id} contains {len(items)} items, "
|
||||
f"expected {submission.item_count}"
|
||||
)
|
||||
|
||||
ordered_items = sorted(
|
||||
items,
|
||||
key=lambda item: getattr(item, "sequence_index", 0) or 0,
|
||||
)
|
||||
episode_uuids: List[str] = []
|
||||
for item in ordered_items:
|
||||
item_status = getattr(item, "status", None)
|
||||
episode_uuid = getattr(item, "episode_uuid", None)
|
||||
source_uuid = getattr(item, "source_uuid", None)
|
||||
if item_status != "succeeded" or not episode_uuid:
|
||||
raise RuntimeError(
|
||||
f"Zep batch {submission.batch_id} returned an incomplete item"
|
||||
)
|
||||
if source_uuid and source_uuid != episode_uuid:
|
||||
raise RuntimeError(
|
||||
f"Zep batch {submission.batch_id} returned mismatched episode UUIDs"
|
||||
)
|
||||
episode_uuids.append(episode_uuid)
|
||||
|
||||
if progress_callback:
|
||||
progress_callback(
|
||||
t(
|
||||
'progress.processingComplete',
|
||||
completed=len(episode_uuids),
|
||||
total=submission.item_count,
|
||||
),
|
||||
1.0,
|
||||
)
|
||||
return episode_uuids
|
||||
|
||||
def _wait_for_episodes(
|
||||
|
|
@ -379,21 +743,22 @@ class GraphBuilderService:
|
|||
t('progress.episodesTimeout', completed=completed_count, total=total_episodes),
|
||||
completed_count / total_episodes
|
||||
)
|
||||
break
|
||||
raise TimeoutError(
|
||||
f"Zep episode processing timed out with "
|
||||
f"{len(pending_episodes)} episode(s) still pending"
|
||||
)
|
||||
|
||||
# 检查每个 episode 的处理状态
|
||||
for ep_uuid in list(pending_episodes):
|
||||
try:
|
||||
episode = self.client.graph.episode.get(uuid_=ep_uuid)
|
||||
is_processed = getattr(episode, 'processed', False)
|
||||
|
||||
if is_processed:
|
||||
pending_episodes.remove(ep_uuid)
|
||||
completed_count += 1
|
||||
|
||||
except Exception as e:
|
||||
# 忽略单个查询错误,继续
|
||||
pass
|
||||
episode = call_zep_read_with_retry(
|
||||
lambda: self.client.graph.episode.get(uuid_=ep_uuid),
|
||||
operation_name=f"poll episode {ep_uuid}",
|
||||
)
|
||||
is_processed = getattr(episode, 'processed', False)
|
||||
|
||||
if is_processed:
|
||||
pending_episodes.remove(ep_uuid)
|
||||
completed_count += 1
|
||||
|
||||
elapsed = int(time.time() - start_time)
|
||||
if progress_callback:
|
||||
|
|
|
|||
|
|
@ -16,12 +16,16 @@ from dataclasses import dataclass, field
|
|||
from datetime import datetime
|
||||
|
||||
from openai import OpenAI
|
||||
from zep_cloud.client import Zep
|
||||
|
||||
from ..config import Config
|
||||
from ..utils.logger import get_logger
|
||||
from ..utils.locale import get_language_instruction, get_locale, set_locale, t
|
||||
from ..utils.openai_chat_compat import create_chat_completion, extract_chat_completion_text
|
||||
from ..utils.zep import (
|
||||
call_zep_read_with_retry,
|
||||
get_zep_client,
|
||||
is_retryable_zep_error,
|
||||
normalize_zep_search_query,
|
||||
)
|
||||
from .zep_entity_reader import EntityNode, ZepEntityReader
|
||||
|
||||
logger = get_logger('mirofish.oasis_profile')
|
||||
|
|
@ -263,7 +267,7 @@ class OasisProfileGenerator:
|
|||
|
||||
if self.zep_api_key:
|
||||
try:
|
||||
self.zep_client = Zep(api_key=self.zep_api_key)
|
||||
self.zep_client = get_zep_client(self.zep_api_key)
|
||||
except Exception as e:
|
||||
logger.warning(f"Zep客户端初始化失败: {e}")
|
||||
|
||||
|
|
@ -372,57 +376,35 @@ class OasisProfileGenerator:
|
|||
logger.debug(f"跳过Zep检索:未设置graph_id")
|
||||
return results
|
||||
|
||||
comprehensive_query = t('progress.zepSearchQuery', name=entity_name)
|
||||
comprehensive_query = normalize_zep_search_query(
|
||||
t('progress.zepSearchQuery', name=entity_name)
|
||||
)
|
||||
|
||||
def search_edges():
|
||||
"""搜索边(事实/关系)- 带重试机制"""
|
||||
max_retries = 3
|
||||
last_exception = None
|
||||
delay = 2.0
|
||||
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
return self.zep_client.graph.search(
|
||||
return call_zep_read_with_retry(
|
||||
lambda: self.zep_client.graph.search(
|
||||
query=comprehensive_query,
|
||||
graph_id=self.graph_id,
|
||||
limit=30,
|
||||
scope="edges",
|
||||
reranker="rrf"
|
||||
)
|
||||
except Exception as e:
|
||||
last_exception = e
|
||||
if attempt < max_retries - 1:
|
||||
logger.debug(f"Zep边搜索第 {attempt + 1} 次失败: {str(e)[:80]}, 重试中...")
|
||||
time.sleep(delay)
|
||||
delay *= 2
|
||||
else:
|
||||
logger.debug(f"Zep边搜索在 {max_retries} 次尝试后仍失败: {e}")
|
||||
return None
|
||||
),
|
||||
operation_name=f"profile edge search ({entity.uuid})",
|
||||
)
|
||||
|
||||
def search_nodes():
|
||||
"""搜索节点(实体摘要)- 带重试机制"""
|
||||
max_retries = 3
|
||||
last_exception = None
|
||||
delay = 2.0
|
||||
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
return self.zep_client.graph.search(
|
||||
return call_zep_read_with_retry(
|
||||
lambda: self.zep_client.graph.search(
|
||||
query=comprehensive_query,
|
||||
graph_id=self.graph_id,
|
||||
limit=20,
|
||||
scope="nodes",
|
||||
reranker="rrf"
|
||||
)
|
||||
except Exception as e:
|
||||
last_exception = e
|
||||
if attempt < max_retries - 1:
|
||||
logger.debug(f"Zep节点搜索第 {attempt + 1} 次失败: {str(e)[:80]}, 重试中...")
|
||||
time.sleep(delay)
|
||||
delay *= 2
|
||||
else:
|
||||
logger.debug(f"Zep节点搜索在 {max_retries} 次尝试后仍失败: {e}")
|
||||
return None
|
||||
),
|
||||
operation_name=f"profile node search ({entity.uuid})",
|
||||
)
|
||||
|
||||
try:
|
||||
# 并行执行edges和nodes搜索
|
||||
|
|
@ -431,8 +413,11 @@ class OasisProfileGenerator:
|
|||
node_future = executor.submit(search_nodes)
|
||||
|
||||
# 获取结果
|
||||
edge_result = edge_future.result(timeout=30)
|
||||
node_result = node_future.result(timeout=30)
|
||||
# Each request already has the configured HTTP timeout and
|
||||
# typed retry budget. A second hard-coded 30s future timeout
|
||||
# discarded late successes while the executor still waited.
|
||||
edge_result = edge_future.result()
|
||||
node_result = node_future.result()
|
||||
|
||||
# 处理边搜索结果
|
||||
all_facts = set()
|
||||
|
|
@ -462,10 +447,10 @@ class OasisProfileGenerator:
|
|||
|
||||
logger.info(f"Zep混合检索完成: {entity_name}, 获取 {len(results['facts'])} 条事实, {len(results['node_summaries'])} 个相关节点")
|
||||
|
||||
except concurrent.futures.TimeoutError:
|
||||
logger.warning(f"Zep检索超时 ({entity_name})")
|
||||
except Exception as e:
|
||||
logger.warning(f"Zep检索失败 ({entity_name}): {e}")
|
||||
if not is_retryable_zep_error(e):
|
||||
raise
|
||||
|
||||
return results
|
||||
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ from ..utils.file_parser import split_text_into_chunks
|
|||
from ..utils.ontology import (
|
||||
MAX_ONTOLOGY_TYPES,
|
||||
normalize_ontology_attributes,
|
||||
normalize_ontology_source_targets,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -31,6 +32,18 @@ def _to_pascal_case(name: str) -> str:
|
|||
return result if result else 'Unknown'
|
||||
|
||||
|
||||
def _to_upper_snake_case(name: str) -> str:
|
||||
"""Convert free-form or camelCase names to SCREAMING_SNAKE_CASE."""
|
||||
|
||||
separated = re.sub(r'([a-z0-9])([A-Z])', r'\1_\2', name.strip())
|
||||
normalized = re.sub(r'[^a-zA-Z0-9]+', '_', separated).strip('_').upper()
|
||||
if not normalized:
|
||||
return "UNKNOWN"
|
||||
if normalized[0].isdigit():
|
||||
normalized = f"REL_{normalized}"
|
||||
return normalized
|
||||
|
||||
|
||||
# 本体生成的系统提示词
|
||||
ONTOLOGY_SYSTEM_PROMPT = """你是一个专业的知识图谱本体设计专家。你的任务是分析给定的文本内容和模拟需求,设计适合**社交媒体舆论模拟**的实体类型和关系类型。
|
||||
|
||||
|
|
@ -414,76 +427,71 @@ class OntologyGenerator:
|
|||
|
||||
def _validate_and_process(self, result: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""验证和后处理结果"""
|
||||
|
||||
# 确保必要字段存在
|
||||
if "entity_types" not in result:
|
||||
result["entity_types"] = []
|
||||
if "edge_types" not in result:
|
||||
result["edge_types"] = []
|
||||
if "analysis_summary" not in result:
|
||||
if not isinstance(result, dict):
|
||||
raise ValueError("Ontology result must be an object")
|
||||
|
||||
raw_entities = result.get("entity_types")
|
||||
raw_edges = result.get("edge_types")
|
||||
if not isinstance(raw_entities, list):
|
||||
raw_entities = []
|
||||
if not isinstance(raw_edges, list):
|
||||
raw_edges = []
|
||||
if not isinstance(result.get("analysis_summary"), str):
|
||||
result["analysis_summary"] = ""
|
||||
|
||||
# 验证实体类型
|
||||
# 记录原始名称到 PascalCase 的映射,用于后续修正 edge 的 source_targets 引用
|
||||
entity_name_map = {}
|
||||
for entity in result["entity_types"]:
|
||||
# 强制将 entity name 转为 PascalCase(Zep API 要求)
|
||||
if "name" in entity:
|
||||
original_name = entity["name"]
|
||||
entity["name"] = _to_pascal_case(original_name)
|
||||
if entity["name"] != original_name:
|
||||
logger.warning(f"Entity type name '{original_name}' auto-converted to '{entity['name']}'")
|
||||
entity_name_map[original_name] = entity["name"]
|
||||
# Normalize LLM output, enforce Zep field limits, and guarantee at
|
||||
# least one property for every custom ontology type.
|
||||
|
||||
# Normalize entity entries before touching their fields. LLMs
|
||||
# occasionally emit a bare string, null, or another scalar.
|
||||
entity_name_map: Dict[str, str] = {}
|
||||
processed_entities: List[Dict[str, Any]] = []
|
||||
seen_entity_names = set()
|
||||
for raw_entity in raw_entities:
|
||||
if isinstance(raw_entity, str):
|
||||
entity = {"name": raw_entity}
|
||||
elif isinstance(raw_entity, dict):
|
||||
entity = dict(raw_entity)
|
||||
else:
|
||||
logger.warning("Ignoring non-object ontology entity entry")
|
||||
continue
|
||||
|
||||
original_name = entity.get("name")
|
||||
if not isinstance(original_name, str) or not original_name.strip():
|
||||
logger.warning("Ignoring ontology entity without a usable name")
|
||||
continue
|
||||
original_name = original_name.strip()
|
||||
normalized_name = _to_pascal_case(original_name)
|
||||
if normalized_name == "Unknown":
|
||||
continue
|
||||
if normalized_name in seen_entity_names:
|
||||
logger.warning(f"Duplicate entity type '{normalized_name}' removed during validation")
|
||||
entity_name_map[original_name] = normalized_name
|
||||
entity_name_map[original_name.lower()] = normalized_name
|
||||
continue
|
||||
|
||||
if normalized_name != original_name:
|
||||
logger.warning(
|
||||
f"Entity type name '{original_name}' auto-converted to '{normalized_name}'"
|
||||
)
|
||||
entity["name"] = normalized_name
|
||||
entity["attributes"] = normalize_ontology_attributes(
|
||||
entity.get("attributes", [])
|
||||
)
|
||||
if "examples" not in entity:
|
||||
if not isinstance(entity.get("examples"), list):
|
||||
entity["examples"] = []
|
||||
# 确保description不超过100字符
|
||||
if len(entity.get("description", "")) > 100:
|
||||
entity["description"] = entity["description"][:97] + "..."
|
||||
|
||||
# 验证关系类型
|
||||
for edge in result["edge_types"]:
|
||||
# 强制将 edge name 转为 SCREAMING_SNAKE_CASE(Zep API 要求)
|
||||
if "name" in edge:
|
||||
original_name = edge["name"]
|
||||
edge["name"] = original_name.upper()
|
||||
if edge["name"] != original_name:
|
||||
logger.warning(f"Edge type name '{original_name}' auto-converted to '{edge['name']}'")
|
||||
# 修正 source_targets 中的实体名称引用,与转换后的 PascalCase 保持一致
|
||||
for st in edge.get("source_targets", []):
|
||||
if st.get("source") in entity_name_map:
|
||||
st["source"] = entity_name_map[st["source"]]
|
||||
if st.get("target") in entity_name_map:
|
||||
st["target"] = entity_name_map[st["target"]]
|
||||
if "source_targets" not in edge:
|
||||
edge["source_targets"] = []
|
||||
# Normalize LLM output, enforce Zep field limits, and guarantee at
|
||||
# least one property for every custom ontology type.
|
||||
edge["attributes"] = normalize_ontology_attributes(
|
||||
edge.get("attributes", [])
|
||||
description = entity.get("description")
|
||||
if not isinstance(description, str) or not description:
|
||||
description = f"A {normalized_name} entity."
|
||||
entity["description"] = (
|
||||
description[:97] + "..." if len(description) > 100 else description
|
||||
)
|
||||
if len(edge.get("description", "")) > 100:
|
||||
edge["description"] = edge["description"][:97] + "..."
|
||||
|
||||
# Zep API 限制:最多 10 个自定义实体类型,最多 10 个自定义边类型
|
||||
MAX_ENTITY_TYPES = MAX_ONTOLOGY_TYPES
|
||||
MAX_EDGE_TYPES = MAX_ONTOLOGY_TYPES
|
||||
|
||||
# 去重:按 name 去重,保留首次出现的
|
||||
seen_names = set()
|
||||
deduped = []
|
||||
for entity in result["entity_types"]:
|
||||
name = entity.get("name", "")
|
||||
if name and name not in seen_names:
|
||||
seen_names.add(name)
|
||||
deduped.append(entity)
|
||||
elif name in seen_names:
|
||||
logger.warning(f"Duplicate entity type '{name}' removed during validation")
|
||||
result["entity_types"] = deduped
|
||||
seen_entity_names.add(normalized_name)
|
||||
processed_entities.append(entity)
|
||||
entity_name_map[original_name] = normalized_name
|
||||
entity_name_map[original_name.lower()] = normalized_name
|
||||
entity_name_map[normalized_name] = normalized_name
|
||||
entity_name_map[normalized_name.lower()] = normalized_name
|
||||
|
||||
result["entity_types"] = processed_entities
|
||||
|
||||
# 兜底类型定义
|
||||
person_fallback = {
|
||||
|
|
@ -523,9 +531,9 @@ class OntologyGenerator:
|
|||
needed_slots = len(fallbacks_to_add)
|
||||
|
||||
# 如果添加后会超过 10 个,需要移除一些现有类型
|
||||
if current_count + needed_slots > MAX_ENTITY_TYPES:
|
||||
if current_count + needed_slots > MAX_ONTOLOGY_TYPES:
|
||||
# 计算需要移除多少个
|
||||
to_remove = current_count + needed_slots - MAX_ENTITY_TYPES
|
||||
to_remove = current_count + needed_slots - MAX_ONTOLOGY_TYPES
|
||||
# 从末尾移除(保留前面更重要的具体类型)
|
||||
result["entity_types"] = result["entity_types"][:-to_remove]
|
||||
|
||||
|
|
@ -533,11 +541,82 @@ class OntologyGenerator:
|
|||
result["entity_types"].extend(fallbacks_to_add)
|
||||
|
||||
# 最终确保不超过限制(防御性编程)
|
||||
if len(result["entity_types"]) > MAX_ENTITY_TYPES:
|
||||
result["entity_types"] = result["entity_types"][:MAX_ENTITY_TYPES]
|
||||
|
||||
if len(result["edge_types"]) > MAX_EDGE_TYPES:
|
||||
result["edge_types"] = result["edge_types"][:MAX_EDGE_TYPES]
|
||||
result["entity_types"] = result["entity_types"][:MAX_ONTOLOGY_TYPES]
|
||||
|
||||
# Resolve edge endpoints only after entity fallback/capping, so an edge
|
||||
# cannot refer to a type that was removed to satisfy Zep's limits.
|
||||
valid_entity_names = {entity["name"] for entity in result["entity_types"]}
|
||||
for name in valid_entity_names:
|
||||
entity_name_map[name] = name
|
||||
entity_name_map[name.lower()] = name
|
||||
|
||||
def resolve_entity_name(value: str) -> Optional[str]:
|
||||
stripped = value.strip()
|
||||
if stripped == "Entity":
|
||||
return stripped
|
||||
mapped = entity_name_map.get(stripped) or entity_name_map.get(stripped.lower())
|
||||
if mapped in valid_entity_names:
|
||||
return mapped
|
||||
pascal_name = _to_pascal_case(stripped)
|
||||
return pascal_name if pascal_name in valid_entity_names else None
|
||||
|
||||
processed_edges: List[Dict[str, Any]] = []
|
||||
seen_edge_names = set()
|
||||
for raw_edge in raw_edges:
|
||||
if isinstance(raw_edge, str):
|
||||
# A bare edge name has no endpoints and cannot be installed in
|
||||
# Zep safely. Ignore it instead of inventing a relationship.
|
||||
logger.warning(f"Ignoring ontology edge without source_targets: {raw_edge}")
|
||||
continue
|
||||
elif isinstance(raw_edge, dict):
|
||||
edge = dict(raw_edge)
|
||||
else:
|
||||
logger.warning("Ignoring non-object ontology edge entry")
|
||||
continue
|
||||
|
||||
original_name = edge.get("name")
|
||||
if not isinstance(original_name, str) or not original_name.strip():
|
||||
logger.warning("Ignoring ontology edge without a usable name")
|
||||
continue
|
||||
normalized_name = _to_upper_snake_case(original_name)
|
||||
if normalized_name == "UNKNOWN" or normalized_name in seen_edge_names:
|
||||
if normalized_name in seen_edge_names:
|
||||
logger.warning(f"Duplicate edge type '{normalized_name}' removed during validation")
|
||||
continue
|
||||
if normalized_name != original_name:
|
||||
logger.warning(
|
||||
f"Edge type name '{original_name}' auto-converted to '{normalized_name}'"
|
||||
)
|
||||
edge["name"] = normalized_name
|
||||
|
||||
normalized_targets = []
|
||||
for source_target in normalize_ontology_source_targets(
|
||||
edge.get("source_targets", []),
|
||||
limit=None,
|
||||
):
|
||||
source = resolve_entity_name(source_target["source"])
|
||||
target = resolve_entity_name(source_target["target"])
|
||||
if source and target:
|
||||
normalized_targets.append({"source": source, "target": target})
|
||||
edge["source_targets"] = normalize_ontology_source_targets(
|
||||
normalized_targets
|
||||
)
|
||||
edge["attributes"] = normalize_ontology_attributes(
|
||||
edge.get("attributes", [])
|
||||
)
|
||||
description = edge.get("description")
|
||||
if not isinstance(description, str) or not description:
|
||||
description = f"A {normalized_name} relationship."
|
||||
edge["description"] = (
|
||||
description[:97] + "..." if len(description) > 100 else description
|
||||
)
|
||||
|
||||
seen_edge_names.add(normalized_name)
|
||||
processed_edges.append(edge)
|
||||
if len(processed_edges) == MAX_ONTOLOGY_TYPES:
|
||||
break
|
||||
|
||||
result["edge_types"] = processed_edges
|
||||
|
||||
return result
|
||||
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ class SimulationStatus(str, Enum):
|
|||
PREPARING = "preparing"
|
||||
READY = "ready"
|
||||
RUNNING = "running"
|
||||
STOPPING = "stopping"
|
||||
PAUSED = "paused"
|
||||
STOPPED = "stopped" # 模拟被手动停止
|
||||
COMPLETED = "completed" # 模拟自然完成
|
||||
|
|
|
|||
|
|
@ -45,6 +45,10 @@ class RunnerStatus(str, Enum):
|
|||
FAILED = "failed"
|
||||
|
||||
|
||||
class SimulationStopPending(TimeoutError):
|
||||
"""The monitor still owns a bounded graph-ingestion finalization."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class AgentAction:
|
||||
"""Agent动作记录"""
|
||||
|
|
@ -226,6 +230,56 @@ class SimulationRunner:
|
|||
|
||||
# 图谱记忆更新配置
|
||||
_graph_memory_enabled: Dict[str, bool] = {} # simulation_id -> enabled
|
||||
_finalization_locks: Dict[str, threading.Lock] = {}
|
||||
_finalization_locks_guard = threading.Lock()
|
||||
_manual_stop_requests: set[str] = set()
|
||||
|
||||
@classmethod
|
||||
def _finalization_lock(cls, simulation_id: str) -> threading.Lock:
|
||||
with cls._finalization_locks_guard:
|
||||
return cls._finalization_locks.setdefault(
|
||||
simulation_id, threading.Lock()
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _sync_simulation_status(
|
||||
cls,
|
||||
simulation_id: str,
|
||||
runner_status: RunnerStatus,
|
||||
error: str | None = None,
|
||||
) -> None:
|
||||
"""Keep persisted simulation metadata aligned with run_state.json."""
|
||||
|
||||
from .simulation_manager import SimulationManager, SimulationStatus
|
||||
|
||||
status_map = {
|
||||
RunnerStatus.RUNNING: SimulationStatus.RUNNING,
|
||||
RunnerStatus.STOPPING: SimulationStatus.STOPPING,
|
||||
RunnerStatus.STOPPED: SimulationStatus.STOPPED,
|
||||
RunnerStatus.COMPLETED: SimulationStatus.COMPLETED,
|
||||
RunnerStatus.FAILED: SimulationStatus.FAILED,
|
||||
}
|
||||
status = status_map.get(runner_status)
|
||||
if status is None:
|
||||
return
|
||||
try:
|
||||
manager = SimulationManager()
|
||||
simulation = manager.get_simulation(simulation_id)
|
||||
if simulation is None:
|
||||
return
|
||||
simulation.status = status
|
||||
simulation.error = error
|
||||
manager._save_simulation_state(simulation)
|
||||
except Exception as sync_error:
|
||||
# state.json is a secondary projection. Never let a projection
|
||||
# failure skip the authoritative run-state finalization or Zep
|
||||
# ingestion drain.
|
||||
logger.error(
|
||||
"同步模拟状态失败: simulation_id=%s, status=%s, error=%s",
|
||||
simulation_id,
|
||||
runner_status.value,
|
||||
sync_error,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_run_state(cls, simulation_id: str) -> Optional[SimulationRunState]:
|
||||
|
|
@ -331,11 +385,6 @@ class SimulationRunner:
|
|||
Returns:
|
||||
SimulationRunState
|
||||
"""
|
||||
# 检查是否已在运行
|
||||
existing = cls.get_run_state(simulation_id)
|
||||
if existing and existing.runner_status in [RunnerStatus.RUNNING, RunnerStatus.STARTING]:
|
||||
raise ValueError(f"模拟已在运行中: {simulation_id}")
|
||||
|
||||
# 加载模拟配置
|
||||
sim_dir = os.path.join(cls.RUN_STATE_DIR, simulation_id)
|
||||
config_path = os.path.join(sim_dir, "simulation_config.json")
|
||||
|
|
@ -367,7 +416,22 @@ class SimulationRunner:
|
|||
started_at=datetime.now().isoformat(),
|
||||
)
|
||||
|
||||
cls._save_run_state(state)
|
||||
# Atomically claim this simulation ID. The expensive updater/process
|
||||
# startup happens after releasing the lock, while the persisted
|
||||
# STARTING state makes every concurrent start fail closed.
|
||||
with cls._finalization_lock(simulation_id):
|
||||
existing = cls.get_run_state(simulation_id)
|
||||
active_statuses = {
|
||||
RunnerStatus.STARTING,
|
||||
RunnerStatus.RUNNING,
|
||||
RunnerStatus.PAUSED,
|
||||
RunnerStatus.STOPPING,
|
||||
}
|
||||
if (
|
||||
existing and existing.runner_status in active_statuses
|
||||
) or ZepGraphMemoryManager.get_updater(simulation_id) is not None:
|
||||
raise ValueError(f"模拟已在运行或结束处理中: {simulation_id}")
|
||||
cls._save_run_state(state)
|
||||
|
||||
# 如果启用图谱记忆更新,创建更新器
|
||||
if enable_graph_memory_update:
|
||||
|
|
@ -381,6 +445,16 @@ class SimulationRunner:
|
|||
except Exception as e:
|
||||
logger.error(f"创建图谱记忆更新器失败: {e}")
|
||||
cls._graph_memory_enabled[simulation_id] = False
|
||||
state.runner_status = RunnerStatus.FAILED
|
||||
state.error = f"Zep图谱更新器初始化失败: {e}"
|
||||
with cls._finalization_lock(simulation_id):
|
||||
cls._save_run_state(state)
|
||||
cls._sync_simulation_status(
|
||||
simulation_id,
|
||||
RunnerStatus.FAILED,
|
||||
state.error,
|
||||
)
|
||||
raise RuntimeError(state.error) from e
|
||||
else:
|
||||
cls._graph_memory_enabled[simulation_id] = False
|
||||
|
||||
|
|
@ -399,12 +473,35 @@ class SimulationRunner:
|
|||
script_path = os.path.join(cls.SCRIPTS_DIR, script_name)
|
||||
|
||||
if not os.path.exists(script_path):
|
||||
raise ValueError(f"脚本不存在: {script_path}")
|
||||
cleanup_error = None
|
||||
if cls._graph_memory_enabled.get(simulation_id, False):
|
||||
try:
|
||||
ZepGraphMemoryManager.stop_updater(simulation_id)
|
||||
cls._graph_memory_enabled.pop(simulation_id, None)
|
||||
except Exception as error:
|
||||
cleanup_error = error
|
||||
state.runner_status = RunnerStatus.FAILED
|
||||
state.twitter_running = False
|
||||
state.reddit_running = False
|
||||
state.error = f"脚本不存在: {script_path}"
|
||||
if cleanup_error is not None:
|
||||
state.error += f"; Zep图谱写入清理失败: {cleanup_error}"
|
||||
with cls._finalization_lock(simulation_id):
|
||||
cls._save_run_state(state)
|
||||
cls._sync_simulation_status(
|
||||
simulation_id,
|
||||
RunnerStatus.FAILED,
|
||||
state.error,
|
||||
)
|
||||
raise ValueError(state.error)
|
||||
|
||||
# 创建动作队列
|
||||
action_queue = Queue()
|
||||
cls._action_queues[simulation_id] = action_queue
|
||||
|
||||
|
||||
process = None
|
||||
main_log_file = None
|
||||
|
||||
# 启动模拟进程
|
||||
try:
|
||||
# 构建运行命令,使用完整路径
|
||||
|
|
@ -447,33 +544,70 @@ class SimulationRunner:
|
|||
start_new_session=True, # 创建新进程组,确保服务器关闭时能终止所有相关进程
|
||||
)
|
||||
|
||||
# 保存文件句柄以便后续关闭
|
||||
cls._stdout_files[simulation_id] = main_log_file
|
||||
cls._stderr_files[simulation_id] = None # 不再需要单独的 stderr
|
||||
|
||||
state.process_pid = process.pid
|
||||
state.runner_status = RunnerStatus.RUNNING
|
||||
cls._processes[simulation_id] = process
|
||||
cls._save_run_state(state)
|
||||
|
||||
# Capture locale before spawning monitor thread
|
||||
current_locale = get_locale()
|
||||
|
||||
# 启动监控线程
|
||||
monitor_thread = threading.Thread(
|
||||
target=cls._monitor_simulation,
|
||||
args=(simulation_id, current_locale),
|
||||
daemon=True
|
||||
)
|
||||
monitor_thread.start()
|
||||
cls._monitor_threads[simulation_id] = monitor_thread
|
||||
|
||||
# Atomically publish every resource needed by stop/finalization.
|
||||
# The monitor is registered before start; if it exits immediately,
|
||||
# it waits on the same lock until RUNNING is fully visible.
|
||||
with cls._finalization_lock(simulation_id):
|
||||
cls._stdout_files[simulation_id] = main_log_file
|
||||
cls._stderr_files[simulation_id] = None
|
||||
state.process_pid = process.pid
|
||||
state.runner_status = RunnerStatus.RUNNING
|
||||
cls._processes[simulation_id] = process
|
||||
cls._monitor_threads[simulation_id] = monitor_thread
|
||||
cls._save_run_state(state)
|
||||
cls._sync_simulation_status(
|
||||
simulation_id,
|
||||
RunnerStatus.RUNNING,
|
||||
)
|
||||
monitor_thread.start()
|
||||
|
||||
logger.info(f"模拟启动成功: {simulation_id}, pid={process.pid}, platform={platform}")
|
||||
|
||||
except Exception as e:
|
||||
cleanup_errors = []
|
||||
if process is not None and process.poll() is None:
|
||||
try:
|
||||
cls._terminate_process(process, simulation_id)
|
||||
except Exception as error:
|
||||
cleanup_errors.append(f"子进程终止失败: {error}")
|
||||
cls._processes.pop(simulation_id, None)
|
||||
cls._monitor_threads.pop(simulation_id, None)
|
||||
cls._action_queues.pop(simulation_id, None)
|
||||
cls._stdout_files.pop(simulation_id, None)
|
||||
cls._stderr_files.pop(simulation_id, None)
|
||||
if main_log_file is not None:
|
||||
try:
|
||||
main_log_file.close()
|
||||
except Exception as error:
|
||||
cleanup_errors.append(f"日志关闭失败: {error}")
|
||||
if cls._graph_memory_enabled.get(simulation_id, False):
|
||||
try:
|
||||
ZepGraphMemoryManager.stop_updater(simulation_id)
|
||||
cls._graph_memory_enabled.pop(simulation_id, None)
|
||||
except Exception as error:
|
||||
cleanup_errors.append(f"Zep图谱写入清理失败: {error}")
|
||||
state.runner_status = RunnerStatus.FAILED
|
||||
state.twitter_running = False
|
||||
state.reddit_running = False
|
||||
state.error = str(e)
|
||||
cls._save_run_state(state)
|
||||
if cleanup_errors:
|
||||
state.error += "; " + "; ".join(cleanup_errors)
|
||||
with cls._finalization_lock(simulation_id):
|
||||
cls._save_run_state(state)
|
||||
cls._sync_simulation_status(
|
||||
simulation_id,
|
||||
RunnerStatus.FAILED,
|
||||
state.error,
|
||||
)
|
||||
raise
|
||||
|
||||
return state
|
||||
|
|
@ -497,6 +631,8 @@ class SimulationRunner:
|
|||
twitter_position = 0
|
||||
reddit_position = 0
|
||||
|
||||
monitor_error: Exception | None = None
|
||||
exit_code: int | None = None
|
||||
try:
|
||||
while process.poll() is None: # 进程仍在运行
|
||||
# 读取 Twitter 动作日志
|
||||
|
|
@ -521,50 +657,93 @@ class SimulationRunner:
|
|||
if os.path.exists(reddit_actions_log):
|
||||
cls._read_action_log(reddit_actions_log, reddit_position, state, "reddit")
|
||||
|
||||
# 进程结束
|
||||
exit_code = process.returncode
|
||||
|
||||
if exit_code == 0:
|
||||
state.runner_status = RunnerStatus.COMPLETED
|
||||
state.completed_at = datetime.now().isoformat()
|
||||
logger.info(f"模拟完成: {simulation_id}")
|
||||
else:
|
||||
state.runner_status = RunnerStatus.FAILED
|
||||
# 从主日志文件读取错误信息
|
||||
main_log_path = os.path.join(sim_dir, "simulation.log")
|
||||
error_info = ""
|
||||
try:
|
||||
if os.path.exists(main_log_path):
|
||||
with open(main_log_path, 'r', encoding='utf-8') as f:
|
||||
error_info = f.read()[-2000:] # 取最后2000字符
|
||||
except Exception:
|
||||
pass
|
||||
state.error = f"进程退出码: {exit_code}, 错误: {error_info}"
|
||||
logger.error(f"模拟失败: {simulation_id}, error={state.error}")
|
||||
|
||||
state.twitter_running = False
|
||||
state.reddit_running = False
|
||||
cls._save_run_state(state)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"监控线程异常: {simulation_id}, error={str(e)}")
|
||||
state.runner_status = RunnerStatus.FAILED
|
||||
state.error = str(e)
|
||||
cls._save_run_state(state)
|
||||
monitor_error = e
|
||||
|
||||
finally:
|
||||
# 停止图谱记忆更新器
|
||||
if cls._graph_memory_enabled.get(simulation_id, False):
|
||||
try:
|
||||
ZepGraphMemoryManager.stop_updater(simulation_id)
|
||||
logger.info(f"已停止图谱记忆更新: simulation_id={simulation_id}")
|
||||
except Exception as e:
|
||||
logger.error(f"停止图谱记忆更新器失败: {e}")
|
||||
cls._graph_memory_enabled.pop(simulation_id, None)
|
||||
# Manual stop and natural completion can observe the same process
|
||||
# exit. Serialize terminal state and updater drain so only one path
|
||||
# owns the final result.
|
||||
with cls._finalization_lock(simulation_id):
|
||||
latest_state = cls.get_run_state(simulation_id)
|
||||
if latest_state is not None:
|
||||
state = latest_state
|
||||
|
||||
if state.runner_status not in {
|
||||
RunnerStatus.STOPPED,
|
||||
RunnerStatus.FAILED,
|
||||
}:
|
||||
manual_stop = simulation_id in cls._manual_stop_requests
|
||||
desired_status = (
|
||||
RunnerStatus.STOPPED
|
||||
if manual_stop
|
||||
else RunnerStatus.COMPLETED
|
||||
)
|
||||
error_message = None
|
||||
if not manual_stop and monitor_error is not None:
|
||||
desired_status = RunnerStatus.FAILED
|
||||
error_message = str(monitor_error)
|
||||
elif not manual_stop and exit_code != 0:
|
||||
desired_status = RunnerStatus.FAILED
|
||||
main_log_path = os.path.join(sim_dir, "simulation.log")
|
||||
error_info = ""
|
||||
try:
|
||||
if os.path.exists(main_log_path):
|
||||
with open(main_log_path, 'r', encoding='utf-8') as f:
|
||||
error_info = f.read()[-2000:]
|
||||
except Exception:
|
||||
pass
|
||||
error_message = (
|
||||
f"进程退出码: {exit_code}, 错误: {error_info}"
|
||||
)
|
||||
|
||||
state.twitter_running = False
|
||||
state.reddit_running = False
|
||||
|
||||
if cls._graph_memory_enabled.get(simulation_id, False):
|
||||
# STOPPING is a non-terminal ingestion barrier. The UI
|
||||
# and report API must not observe COMPLETED until every
|
||||
# accepted episode is processed by Zep Cloud.
|
||||
state.runner_status = RunnerStatus.STOPPING
|
||||
cls._save_run_state(state)
|
||||
cls._sync_simulation_status(
|
||||
simulation_id,
|
||||
RunnerStatus.STOPPING,
|
||||
)
|
||||
try:
|
||||
ZepGraphMemoryManager.stop_updater(simulation_id)
|
||||
cls._graph_memory_enabled.pop(simulation_id, None)
|
||||
logger.info(
|
||||
"已停止图谱记忆更新: simulation_id=%s",
|
||||
simulation_id,
|
||||
)
|
||||
except Exception as error:
|
||||
logger.error(f"停止图谱记忆更新器失败: {error}")
|
||||
desired_status = RunnerStatus.FAILED
|
||||
error_message = f"Zep图谱写入未完整完成: {error}"
|
||||
|
||||
state.runner_status = desired_status
|
||||
state.error = error_message
|
||||
state.completed_at = datetime.now().isoformat()
|
||||
cls._save_run_state(state)
|
||||
cls._sync_simulation_status(
|
||||
simulation_id,
|
||||
desired_status,
|
||||
error_message,
|
||||
)
|
||||
if desired_status == RunnerStatus.COMPLETED:
|
||||
logger.info(f"模拟完成: {simulation_id}")
|
||||
else:
|
||||
logger.error(f"模拟失败: {simulation_id}, error={state.error}")
|
||||
cls._manual_stop_requests.discard(simulation_id)
|
||||
|
||||
# 清理进程资源
|
||||
cls._processes.pop(simulation_id, None)
|
||||
cls._action_queues.pop(simulation_id, None)
|
||||
cls._monitor_threads.pop(simulation_id, None)
|
||||
|
||||
# 关闭日志文件句柄
|
||||
if simulation_id in cls._stdout_files:
|
||||
|
|
@ -635,9 +814,14 @@ class SimulationRunner:
|
|||
# 如果运行了两个平台,需要两个都完成
|
||||
all_completed = cls._check_all_platforms_completed(state)
|
||||
if all_completed:
|
||||
state.runner_status = RunnerStatus.COMPLETED
|
||||
state.completed_at = datetime.now().isoformat()
|
||||
logger.info(f"所有平台模拟已完成: {state.simulation_id}")
|
||||
# Platform completion is only an input
|
||||
# signal. The monitor publishes the
|
||||
# terminal status after the process has
|
||||
# exited and Zep ingestion has drained.
|
||||
logger.info(
|
||||
f"所有平台已结束,等待进程与图谱写入完成: "
|
||||
f"{state.simulation_id}"
|
||||
)
|
||||
|
||||
# 更新轮次信息(从 round_end 事件)
|
||||
elif event_type == "round_end":
|
||||
|
|
@ -776,51 +960,125 @@ class SimulationRunner:
|
|||
@classmethod
|
||||
def stop_simulation(cls, simulation_id: str) -> SimulationRunState:
|
||||
"""停止模拟"""
|
||||
state = cls.get_run_state(simulation_id)
|
||||
if not state:
|
||||
raise ValueError(f"模拟不存在: {simulation_id}")
|
||||
|
||||
if state.runner_status not in [RunnerStatus.RUNNING, RunnerStatus.PAUSED]:
|
||||
raise ValueError(f"模拟未在运行: {simulation_id}, status={state.runner_status}")
|
||||
|
||||
state.runner_status = RunnerStatus.STOPPING
|
||||
cls._save_run_state(state)
|
||||
|
||||
# 终止进程
|
||||
process = cls._processes.get(simulation_id)
|
||||
if process and process.poll() is None:
|
||||
try:
|
||||
cls._terminate_process(process, simulation_id)
|
||||
except ProcessLookupError:
|
||||
# 进程已经不存在
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.error(f"终止进程组失败: {simulation_id}, error={e}")
|
||||
# 回退到直接终止进程
|
||||
with cls._finalization_lock(simulation_id):
|
||||
state = cls.get_run_state(simulation_id)
|
||||
if not state:
|
||||
raise ValueError(f"模拟不存在: {simulation_id}")
|
||||
if state.runner_status == RunnerStatus.STOPPED:
|
||||
return state
|
||||
|
||||
pending_updater = ZepGraphMemoryManager.get_updater(simulation_id)
|
||||
retrying_finalization = (
|
||||
pending_updater is not None
|
||||
and state.runner_status in {
|
||||
RunnerStatus.STOPPING,
|
||||
RunnerStatus.FAILED,
|
||||
}
|
||||
)
|
||||
if (
|
||||
state.runner_status not in [
|
||||
RunnerStatus.STARTING,
|
||||
RunnerStatus.RUNNING,
|
||||
RunnerStatus.PAUSED,
|
||||
RunnerStatus.STOPPING,
|
||||
]
|
||||
and not retrying_finalization
|
||||
):
|
||||
raise ValueError(
|
||||
f"模拟未在运行: {simulation_id}, status={state.runner_status}"
|
||||
)
|
||||
|
||||
state.runner_status = RunnerStatus.STOPPING
|
||||
cls._manual_stop_requests.add(simulation_id)
|
||||
cls._save_run_state(state)
|
||||
cls._sync_simulation_status(simulation_id, RunnerStatus.STOPPING)
|
||||
|
||||
# 终止进程
|
||||
process = cls._processes.get(simulation_id)
|
||||
if process and process.poll() is None:
|
||||
try:
|
||||
process.terminate()
|
||||
process.wait(timeout=5)
|
||||
except Exception:
|
||||
process.kill()
|
||||
|
||||
state.runner_status = RunnerStatus.STOPPED
|
||||
state.twitter_running = False
|
||||
state.reddit_running = False
|
||||
state.completed_at = datetime.now().isoformat()
|
||||
cls._save_run_state(state)
|
||||
|
||||
# 停止图谱记忆更新器
|
||||
if cls._graph_memory_enabled.get(simulation_id, False):
|
||||
try:
|
||||
ZepGraphMemoryManager.stop_updater(simulation_id)
|
||||
logger.info(f"已停止图谱记忆更新: simulation_id={simulation_id}")
|
||||
except Exception as e:
|
||||
logger.error(f"停止图谱记忆更新器失败: {e}")
|
||||
cls._graph_memory_enabled.pop(simulation_id, None)
|
||||
|
||||
cls._terminate_process(process, simulation_id)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.error(f"终止进程组失败: {simulation_id}, error={e}")
|
||||
try:
|
||||
process.terminate()
|
||||
process.wait(timeout=5)
|
||||
except Exception:
|
||||
process.kill()
|
||||
|
||||
# Let the monitor consume the final action-log tail and own the single
|
||||
# updater drain. It will publish STOPPED (rather than COMPLETED) because
|
||||
# the manual-stop marker is set above.
|
||||
monitor = cls._monitor_threads.get(simulation_id)
|
||||
if (
|
||||
not retrying_finalization
|
||||
and
|
||||
monitor is not None
|
||||
and monitor is not threading.current_thread()
|
||||
and monitor.is_alive()
|
||||
):
|
||||
wait_timeout = max(
|
||||
30.0,
|
||||
Config.ZEP_INGESTION_TIMEOUT_SECONDS
|
||||
+ Config.ZEP_REQUEST_TIMEOUT_SECONDS
|
||||
+ 5,
|
||||
)
|
||||
monitor.join(timeout=wait_timeout)
|
||||
if monitor.is_alive():
|
||||
# The monitor still owns finalization and may be inside one
|
||||
# bounded HTTP request. Do not block on or overwrite its lock;
|
||||
# leave the observable state as STOPPING and let polling expose
|
||||
# the eventual STOPPED/FAILED result.
|
||||
raise SimulationStopPending(
|
||||
f"模拟仍在停止中,图谱写入未在 {wait_timeout:.0f}s 内完成"
|
||||
)
|
||||
else:
|
||||
# Restart recovery or tests may have no monitor thread. Complete
|
||||
# the same barrier synchronously in this request.
|
||||
with cls._finalization_lock(simulation_id):
|
||||
state = cls.get_run_state(simulation_id) or state
|
||||
if cls._graph_memory_enabled.get(simulation_id, False):
|
||||
try:
|
||||
ZepGraphMemoryManager.stop_updater(simulation_id)
|
||||
cls._graph_memory_enabled.pop(simulation_id, None)
|
||||
except Exception as error:
|
||||
state.runner_status = RunnerStatus.FAILED
|
||||
state.twitter_running = False
|
||||
state.reddit_running = False
|
||||
state.completed_at = datetime.now().isoformat()
|
||||
state.error = f"Zep图谱写入未完整完成: {error}"
|
||||
cls._save_run_state(state)
|
||||
cls._sync_simulation_status(
|
||||
simulation_id,
|
||||
RunnerStatus.FAILED,
|
||||
state.error,
|
||||
)
|
||||
raise RuntimeError(state.error) from error
|
||||
state.runner_status = RunnerStatus.STOPPED
|
||||
state.twitter_running = False
|
||||
state.reddit_running = False
|
||||
state.completed_at = datetime.now().isoformat()
|
||||
state.error = None
|
||||
cls._save_run_state(state)
|
||||
cls._sync_simulation_status(
|
||||
simulation_id,
|
||||
RunnerStatus.STOPPED,
|
||||
)
|
||||
cls._manual_stop_requests.discard(simulation_id)
|
||||
|
||||
state = cls.get_run_state(simulation_id) or state
|
||||
if state.runner_status == RunnerStatus.FAILED:
|
||||
raise RuntimeError(state.error or "模拟停止失败")
|
||||
if state.runner_status != RunnerStatus.STOPPED:
|
||||
raise RuntimeError(
|
||||
f"模拟停止未达到终态: {simulation_id}, status={state.runner_status}"
|
||||
)
|
||||
|
||||
logger.info(f"模拟已停止: {simulation_id}")
|
||||
return state
|
||||
|
||||
|
||||
@classmethod
|
||||
def _read_actions_from_file(
|
||||
cls,
|
||||
|
|
@ -1194,95 +1452,99 @@ class SimulationRunner:
|
|||
if cls._cleanup_done:
|
||||
return
|
||||
cls._cleanup_done = True
|
||||
|
||||
# 检查是否有内容需要清理(避免空进程的进程打印无用日志)
|
||||
has_processes = bool(cls._processes)
|
||||
has_updaters = bool(cls._graph_memory_enabled)
|
||||
|
||||
if not has_processes and not has_updaters:
|
||||
return # 没有需要清理的内容,静默返回
|
||||
|
||||
logger.info("正在清理所有模拟进程...")
|
||||
|
||||
# 首先停止所有图谱记忆更新器(stop_all 内部会打印日志)
|
||||
try:
|
||||
ZepGraphMemoryManager.stop_all()
|
||||
except Exception as e:
|
||||
logger.error(f"停止图谱记忆更新器失败: {e}")
|
||||
cls._graph_memory_enabled.clear()
|
||||
|
||||
# 复制字典以避免在迭代时修改
|
||||
processes = list(cls._processes.items())
|
||||
|
||||
for simulation_id, process in processes:
|
||||
|
||||
updater_ids = set(ZepGraphMemoryManager.get_simulation_ids())
|
||||
simulation_ids = sorted(
|
||||
set(cls._processes)
|
||||
| set(cls._graph_memory_enabled)
|
||||
| updater_ids
|
||||
)
|
||||
if not simulation_ids:
|
||||
return
|
||||
|
||||
logger.info("正在安全完成所有模拟进程与图谱写入...")
|
||||
cleanup_failed = False
|
||||
|
||||
# Each simulation follows the normal stop/finalization path: terminate
|
||||
# its producer, let the monitor consume the final action-log tail, and
|
||||
# only then drain Zep. This avoids dropping actions emitted during
|
||||
# SIGTERM handling.
|
||||
for simulation_id in simulation_ids:
|
||||
try:
|
||||
if process.poll() is None: # 进程仍在运行
|
||||
logger.info(f"终止模拟进程: {simulation_id}, pid={process.pid}")
|
||||
|
||||
try:
|
||||
# 使用跨平台的进程终止方法
|
||||
state = cls.get_run_state(simulation_id)
|
||||
updater = ZepGraphMemoryManager.get_updater(simulation_id)
|
||||
process = cls._processes.get(simulation_id)
|
||||
|
||||
if state is None:
|
||||
# Missing/corrupt state is exceptional, but retain the
|
||||
# critical producer-before-consumer shutdown ordering.
|
||||
if process is not None and process.poll() is None:
|
||||
cls._terminate_process(process, simulation_id, timeout=5)
|
||||
except (ProcessLookupError, OSError):
|
||||
# 进程可能已经不存在,尝试直接终止
|
||||
try:
|
||||
process.terminate()
|
||||
process.wait(timeout=3)
|
||||
except Exception:
|
||||
process.kill()
|
||||
|
||||
# 更新 run_state.json
|
||||
state = cls.get_run_state(simulation_id)
|
||||
if state:
|
||||
state.runner_status = RunnerStatus.STOPPED
|
||||
state.twitter_running = False
|
||||
state.reddit_running = False
|
||||
state.completed_at = datetime.now().isoformat()
|
||||
state.error = "服务器关闭,模拟被终止"
|
||||
if updater is not None:
|
||||
ZepGraphMemoryManager.stop_updater(simulation_id)
|
||||
continue
|
||||
|
||||
if updater is not None:
|
||||
cls._graph_memory_enabled[simulation_id] = True
|
||||
if state.runner_status in {
|
||||
RunnerStatus.IDLE,
|
||||
RunnerStatus.STOPPED,
|
||||
RunnerStatus.COMPLETED,
|
||||
}:
|
||||
# A retained updater means the old terminal projection
|
||||
# was premature. Restore the ingestion barrier first.
|
||||
state.runner_status = RunnerStatus.STOPPING
|
||||
cls._save_run_state(state)
|
||||
|
||||
# 同时更新 state.json,将状态设为 stopped
|
||||
try:
|
||||
sim_dir = os.path.join(cls.RUN_STATE_DIR, simulation_id)
|
||||
state_file = os.path.join(sim_dir, "state.json")
|
||||
logger.info(f"尝试更新 state.json: {state_file}")
|
||||
if os.path.exists(state_file):
|
||||
with open(state_file, 'r', encoding='utf-8') as f:
|
||||
state_data = json.load(f)
|
||||
state_data['status'] = 'stopped'
|
||||
state_data['updated_at'] = datetime.now().isoformat()
|
||||
with open(state_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(state_data, f, indent=2, ensure_ascii=False)
|
||||
logger.info(f"已更新 state.json 状态为 stopped: {simulation_id}")
|
||||
else:
|
||||
logger.warning(f"state.json 不存在: {state_file}")
|
||||
except Exception as state_err:
|
||||
logger.warning(f"更新 state.json 失败: {simulation_id}, error={state_err}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"清理进程失败: {simulation_id}, error={e}")
|
||||
|
||||
# 清理文件句柄
|
||||
for simulation_id, file_handle in list(cls._stdout_files.items()):
|
||||
try:
|
||||
if file_handle:
|
||||
file_handle.close()
|
||||
except Exception:
|
||||
pass
|
||||
cls._stdout_files.clear()
|
||||
|
||||
for simulation_id, file_handle in list(cls._stderr_files.items()):
|
||||
try:
|
||||
if file_handle:
|
||||
file_handle.close()
|
||||
except Exception:
|
||||
pass
|
||||
cls._stderr_files.clear()
|
||||
|
||||
# 清理内存中的状态
|
||||
cls._processes.clear()
|
||||
cls._action_queues.clear()
|
||||
|
||||
logger.info("模拟进程清理完成")
|
||||
cls._sync_simulation_status(
|
||||
simulation_id,
|
||||
RunnerStatus.STOPPING,
|
||||
)
|
||||
|
||||
needs_finalization = bool(
|
||||
(process is not None and process.poll() is None)
|
||||
or updater is not None
|
||||
or state.runner_status in {
|
||||
RunnerStatus.STARTING,
|
||||
RunnerStatus.RUNNING,
|
||||
RunnerStatus.PAUSED,
|
||||
RunnerStatus.STOPPING,
|
||||
}
|
||||
)
|
||||
if needs_finalization:
|
||||
cls.stop_simulation(simulation_id)
|
||||
|
||||
# A recovery path without a monitor does not run the monitor's
|
||||
# resource cleanup block. Release only successfully stopped
|
||||
# resources; FAILED/STOPPING resources remain retryable.
|
||||
latest = cls.get_run_state(simulation_id)
|
||||
if latest and latest.runner_status == RunnerStatus.STOPPED:
|
||||
stopped_process = cls._processes.get(simulation_id)
|
||||
if stopped_process is None or stopped_process.poll() is not None:
|
||||
cls._processes.pop(simulation_id, None)
|
||||
cls._action_queues.pop(simulation_id, None)
|
||||
cls._monitor_threads.pop(simulation_id, None)
|
||||
for file_map in (cls._stdout_files, cls._stderr_files):
|
||||
file_handle = file_map.pop(simulation_id, None)
|
||||
if file_handle:
|
||||
try:
|
||||
file_handle.close()
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as error:
|
||||
cleanup_failed = True
|
||||
logger.error(
|
||||
"清理模拟失败,保留状态以便重试: simulation_id=%s, error=%s",
|
||||
simulation_id,
|
||||
error,
|
||||
)
|
||||
|
||||
if cleanup_failed:
|
||||
# Retained updaters and FAILED run states continue to block report
|
||||
# generation and graph deletion. Permit an explicit retry.
|
||||
cls._cleanup_done = False
|
||||
logger.error("部分模拟未安全完成清理")
|
||||
else:
|
||||
logger.info("模拟进程与图谱写入清理完成")
|
||||
|
||||
@classmethod
|
||||
def register_cleanup(cls):
|
||||
|
|
@ -1765,4 +2027,3 @@ class SimulationRunner:
|
|||
results = results[:limit]
|
||||
|
||||
return results
|
||||
|
||||
|
|
|
|||
|
|
@ -3,15 +3,14 @@ Zep实体读取与过滤服务
|
|||
从Zep图谱中读取节点,筛选出符合预定义实体类型的节点
|
||||
"""
|
||||
|
||||
import time
|
||||
from typing import Dict, Any, List, Optional, Set, Callable, TypeVar
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from zep_cloud.client import Zep
|
||||
from zep_cloud import NotFoundError
|
||||
|
||||
from ..config import Config
|
||||
from ..utils.logger import get_logger
|
||||
from ..utils.zep_paging import fetch_all_nodes, fetch_all_edges
|
||||
from ..utils.zep import call_zep_read_with_retry, get_zep_client
|
||||
|
||||
logger = get_logger('mirofish.zep_entity_reader')
|
||||
|
||||
|
|
@ -83,7 +82,7 @@ class ZepEntityReader:
|
|||
if not self.api_key:
|
||||
raise ValueError("ZEP_API_KEY 未配置")
|
||||
|
||||
self.client = Zep(api_key=self.api_key)
|
||||
self.client = get_zep_client(self.api_key)
|
||||
|
||||
def _call_with_retry(
|
||||
self,
|
||||
|
|
@ -104,25 +103,12 @@ class ZepEntityReader:
|
|||
Returns:
|
||||
API调用结果
|
||||
"""
|
||||
last_exception = None
|
||||
delay = initial_delay
|
||||
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
return func()
|
||||
except Exception as e:
|
||||
last_exception = e
|
||||
if attempt < max_retries - 1:
|
||||
logger.warning(
|
||||
f"Zep {operation_name} 第 {attempt + 1} 次尝试失败: {str(e)[:100]}, "
|
||||
f"{delay:.1f}秒后重试..."
|
||||
)
|
||||
time.sleep(delay)
|
||||
delay *= 2 # 指数退避
|
||||
else:
|
||||
logger.error(f"Zep {operation_name} 在 {max_retries} 次尝试后仍失败: {str(e)}")
|
||||
|
||||
raise last_exception
|
||||
return call_zep_read_with_retry(
|
||||
func,
|
||||
operation_name=operation_name,
|
||||
max_attempts=max_retries,
|
||||
initial_delay=initial_delay,
|
||||
)
|
||||
|
||||
def get_all_nodes(self, graph_id: str) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
|
|
@ -209,8 +195,10 @@ class ZepEntityReader:
|
|||
|
||||
return edges_data
|
||||
except Exception as e:
|
||||
logger.warning(f"获取节点 {node_uuid} 的边失败: {str(e)}")
|
||||
return []
|
||||
# An empty edge list is valid data. Authentication, permission and
|
||||
# transport failures must not be made indistinguishable from it.
|
||||
logger.error(f"获取节点 {node_uuid} 的边失败: {str(e)}")
|
||||
raise
|
||||
|
||||
def filter_defined_entities(
|
||||
self,
|
||||
|
|
@ -406,9 +394,14 @@ class ZepEntityReader:
|
|||
related_nodes=related_nodes,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"获取实体 {entity_uuid} 失败: {str(e)}")
|
||||
except NotFoundError:
|
||||
return None
|
||||
except Exception as e:
|
||||
# Only an actual Zep 404 means "entity not found". Propagate 401,
|
||||
# 403 and exhausted transport errors so callers cannot prepare a
|
||||
# simulation with silently incomplete graph context.
|
||||
logger.error(f"获取实体 {entity_uuid} 失败: {str(e)}")
|
||||
raise
|
||||
|
||||
def get_entities_by_type(
|
||||
self,
|
||||
|
|
@ -433,4 +426,3 @@ class ZepEntityReader:
|
|||
enrich_with_edges=enrich_with_edges
|
||||
)
|
||||
return result.entities
|
||||
|
||||
|
|
|
|||
|
|
@ -3,20 +3,17 @@ Zep图谱记忆更新服务
|
|||
将模拟中的Agent活动动态更新到Zep图谱中
|
||||
"""
|
||||
|
||||
import os
|
||||
import time
|
||||
import threading
|
||||
import json
|
||||
from typing import Dict, Any, List, Optional, Callable
|
||||
from typing import Dict, Any, List, Optional
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from queue import Queue, Empty
|
||||
|
||||
from zep_cloud.client import Zep
|
||||
|
||||
from ..config import Config
|
||||
from ..utils.logger import get_logger
|
||||
from ..utils.locale import get_locale, set_locale
|
||||
from ..utils.zep import call_zep_read_with_retry, get_zep_client
|
||||
|
||||
logger = get_logger('mirofish.zep_graph_memory_updater')
|
||||
|
||||
|
|
@ -58,8 +55,12 @@ class AgentActivity:
|
|||
describe_func = action_descriptions.get(self.action_type, self._describe_generic)
|
||||
description = describe_func()
|
||||
|
||||
# 直接返回 "agent名称: 活动描述" 格式,不添加模拟前缀
|
||||
return f"{self.agent_name}: {description}"
|
||||
# Keep the event time in the source text as well as episode metadata so
|
||||
# temporal extraction does not collapse a multi-action batch.
|
||||
return (
|
||||
f"[{self.timestamp}] [{self.platform} round {self.round_num}] "
|
||||
f"{self.agent_name}: {description}"
|
||||
)
|
||||
|
||||
def _describe_create_post(self) -> str:
|
||||
content = self.action_args.get("content", "")
|
||||
|
|
@ -199,6 +200,12 @@ class AgentActivity:
|
|||
return f"执行了{self.action_type}操作"
|
||||
|
||||
|
||||
class _DrainDeadlineExceeded(TimeoutError):
|
||||
def __init__(self, processed_count: int):
|
||||
super().__init__("Zep updater drain deadline elapsed")
|
||||
self.processed_count = processed_count
|
||||
|
||||
|
||||
class ZepGraphMemoryUpdater:
|
||||
"""
|
||||
Zep图谱记忆更新器
|
||||
|
|
@ -225,11 +232,16 @@ class ZepGraphMemoryUpdater:
|
|||
# 发送间隔(秒),避免请求过快
|
||||
SEND_INTERVAL = 0.5
|
||||
|
||||
# 重试配置
|
||||
MAX_RETRIES = 3
|
||||
RETRY_DELAY = 2 # 秒
|
||||
# Zep recommends keeping an episode below 10,000 characters. Leave room
|
||||
# for future source formatting changes.
|
||||
MAX_EPISODE_CHARS = 9_500
|
||||
|
||||
def __init__(self, graph_id: str, api_key: Optional[str] = None):
|
||||
def __init__(
|
||||
self,
|
||||
graph_id: str,
|
||||
api_key: Optional[str] = None,
|
||||
simulation_id: Optional[str] = None,
|
||||
):
|
||||
"""
|
||||
初始化更新器
|
||||
|
||||
|
|
@ -238,12 +250,13 @@ class ZepGraphMemoryUpdater:
|
|||
api_key: Zep API Key(可选,默认从配置读取)
|
||||
"""
|
||||
self.graph_id = graph_id
|
||||
self.simulation_id = simulation_id or "unknown"
|
||||
self.api_key = api_key or Config.ZEP_API_KEY
|
||||
|
||||
if not self.api_key:
|
||||
raise ValueError("ZEP_API_KEY未配置")
|
||||
|
||||
self.client = Zep(api_key=self.api_key)
|
||||
self.client = get_zep_client(self.api_key)
|
||||
|
||||
# 活动队列
|
||||
self._activity_queue: Queue = Queue()
|
||||
|
|
@ -254,6 +267,7 @@ class ZepGraphMemoryUpdater:
|
|||
'reddit': [],
|
||||
}
|
||||
self._buffer_lock = threading.Lock()
|
||||
self._acceptance_lock = threading.Lock()
|
||||
|
||||
# 控制标志
|
||||
self._running = False
|
||||
|
|
@ -265,6 +279,8 @@ class ZepGraphMemoryUpdater:
|
|||
self._total_items_sent = 0 # 成功发送到Zep的活动条数
|
||||
self._failed_count = 0 # 发送失败的批次数
|
||||
self._skipped_count = 0 # 被过滤跳过的活动数(DO_NOTHING)
|
||||
self._failed_batches: List[Dict[str, Any]] = []
|
||||
self._pending_episode_uuids: List[str] = []
|
||||
|
||||
logger.info(f"ZepGraphMemoryUpdater 初始化完成: graph_id={graph_id}, batch_size={self.BATCH_SIZE}")
|
||||
|
||||
|
|
@ -291,14 +307,34 @@ class ZepGraphMemoryUpdater:
|
|||
logger.info(f"ZepGraphMemoryUpdater 已启动: graph_id={self.graph_id}")
|
||||
|
||||
def stop(self):
|
||||
"""停止后台工作线程"""
|
||||
self._running = False
|
||||
|
||||
# 发送剩余的活动
|
||||
self._flush_remaining()
|
||||
|
||||
"""Drain the worker, flush tail events, and wait for Cloud ingestion."""
|
||||
deadline = time.time() + Config.ZEP_INGESTION_TIMEOUT_SECONDS
|
||||
# Serialize the accepting->closed transition with add_activity's
|
||||
# check+enqueue operation. This closes the small race where a producer
|
||||
# could enqueue after both the worker and final flush had exited.
|
||||
with self._acceptance_lock:
|
||||
self._running = False
|
||||
|
||||
if self._worker_thread and self._worker_thread.is_alive():
|
||||
self._worker_thread.join(timeout=10)
|
||||
join_timeout = max(0.0, deadline - time.time())
|
||||
self._worker_thread.join(timeout=join_timeout)
|
||||
if self._worker_thread.is_alive():
|
||||
raise TimeoutError(
|
||||
f"Zep updater worker did not stop within {join_timeout:.0f}s"
|
||||
)
|
||||
|
||||
# The worker has drained the queue. Only now is it safe to flush
|
||||
# buffers; doing this before join loses an item already dequeued by the
|
||||
# worker but not yet buffered.
|
||||
self._flush_remaining(deadline=deadline)
|
||||
|
||||
if self._failed_batches:
|
||||
raise RuntimeError(
|
||||
f"{len(self._failed_batches)} Zep activity batch(es) failed; "
|
||||
"simulation graph ingestion is incomplete"
|
||||
)
|
||||
|
||||
self._wait_for_pending_episodes(deadline=deadline)
|
||||
|
||||
logger.info(f"ZepGraphMemoryUpdater 已停止: graph_id={self.graph_id}, "
|
||||
f"total_activities={self._total_activities}, "
|
||||
|
|
@ -332,9 +368,12 @@ class ZepGraphMemoryUpdater:
|
|||
if activity.action_type == "DO_NOTHING":
|
||||
self._skipped_count += 1
|
||||
return
|
||||
|
||||
self._activity_queue.put(activity)
|
||||
self._total_activities += 1
|
||||
|
||||
with self._acceptance_lock:
|
||||
if not self._running:
|
||||
raise RuntimeError("Zep graph updater is not running")
|
||||
self._activity_queue.put(activity)
|
||||
self._total_activities += 1
|
||||
logger.debug(f"添加活动到Zep队列: {activity.agent_name} - {activity.action_type}")
|
||||
|
||||
def add_activity_from_dict(self, data: Dict[str, Any], platform: str):
|
||||
|
|
@ -348,6 +387,9 @@ class ZepGraphMemoryUpdater:
|
|||
# 跳过事件类型的条目
|
||||
if "event_type" in data:
|
||||
return
|
||||
if data.get("success") is False:
|
||||
self._skipped_count += 1
|
||||
return
|
||||
|
||||
activity = AgentActivity(
|
||||
platform=platform,
|
||||
|
|
@ -372,6 +414,7 @@ class ZepGraphMemoryUpdater:
|
|||
|
||||
# 将活动添加到对应平台的缓冲区
|
||||
platform = activity.platform.lower()
|
||||
batch = None
|
||||
with self._buffer_lock:
|
||||
if platform not in self._platform_buffers:
|
||||
self._platform_buffers[platform] = []
|
||||
|
|
@ -381,10 +424,11 @@ class ZepGraphMemoryUpdater:
|
|||
if len(self._platform_buffers[platform]) >= self.BATCH_SIZE:
|
||||
batch = self._platform_buffers[platform][:self.BATCH_SIZE]
|
||||
self._platform_buffers[platform] = self._platform_buffers[platform][self.BATCH_SIZE:]
|
||||
# 释放锁后再发送
|
||||
self._send_batch_activities(batch, platform)
|
||||
# 发送间隔,避免请求过快
|
||||
time.sleep(self.SEND_INTERVAL)
|
||||
|
||||
# Never hold the buffer lock across network I/O or sleep.
|
||||
if batch:
|
||||
self._send_batch_activities(batch, platform)
|
||||
time.sleep(self.SEND_INTERVAL)
|
||||
|
||||
except Empty:
|
||||
pass
|
||||
|
|
@ -393,7 +437,41 @@ class ZepGraphMemoryUpdater:
|
|||
logger.error(f"工作循环异常: {e}")
|
||||
time.sleep(1)
|
||||
|
||||
def _send_batch_activities(self, activities: List[AgentActivity], platform: str):
|
||||
def _build_episode_payloads(
|
||||
self,
|
||||
activities: List[AgentActivity],
|
||||
) -> List[tuple[List[AgentActivity], str]]:
|
||||
payloads: List[tuple[List[AgentActivity], str]] = []
|
||||
current_activities: List[AgentActivity] = []
|
||||
current_lines: List[str] = []
|
||||
current_length = 0
|
||||
|
||||
for activity in activities:
|
||||
text = activity.to_episode_text()
|
||||
if len(text) > self.MAX_EPISODE_CHARS:
|
||||
marker = "... [truncated by MiroFish]"
|
||||
text = text[: self.MAX_EPISODE_CHARS - len(marker)] + marker
|
||||
projected_length = current_length + (1 if current_lines else 0) + len(text)
|
||||
if current_lines and projected_length > self.MAX_EPISODE_CHARS:
|
||||
payloads.append((current_activities, "\n".join(current_lines)))
|
||||
current_activities = []
|
||||
current_lines = []
|
||||
current_length = 0
|
||||
current_activities.append(activity)
|
||||
current_lines.append(text)
|
||||
current_length += (1 if len(current_lines) > 1 else 0) + len(text)
|
||||
|
||||
if current_lines:
|
||||
payloads.append((current_activities, "\n".join(current_lines)))
|
||||
return payloads
|
||||
|
||||
def _send_batch_activities(
|
||||
self,
|
||||
activities: List[AgentActivity],
|
||||
platform: str,
|
||||
*,
|
||||
deadline: float | None = None,
|
||||
) -> int:
|
||||
"""
|
||||
批量发送活动到Zep图谱(合并为一条文本)
|
||||
|
||||
|
|
@ -402,37 +480,80 @@ class ZepGraphMemoryUpdater:
|
|||
platform: 平台名称
|
||||
"""
|
||||
if not activities:
|
||||
return
|
||||
|
||||
# 将多条活动合并为一条文本,用换行分隔
|
||||
episode_texts = [activity.to_episode_text() for activity in activities]
|
||||
combined_text = "\n".join(episode_texts)
|
||||
|
||||
# 带重试的发送
|
||||
for attempt in range(self.MAX_RETRIES):
|
||||
return 0
|
||||
|
||||
processed_count = 0
|
||||
for payload_activities, combined_text in self._build_episode_payloads(activities):
|
||||
if deadline is not None and time.time() >= deadline:
|
||||
raise _DrainDeadlineExceeded(processed_count)
|
||||
try:
|
||||
self.client.graph.add(
|
||||
episode = self.client.graph.add(
|
||||
graph_id=self.graph_id,
|
||||
type="text",
|
||||
data=combined_text
|
||||
data=combined_text,
|
||||
created_at=self._to_rfc3339(payload_activities[-1].timestamp),
|
||||
source_description="MiroFish simulation activity batch",
|
||||
metadata={
|
||||
"source": "mirofish_simulation",
|
||||
"simulation_id": self.simulation_id,
|
||||
"platform": platform,
|
||||
"activity_count": len(payload_activities),
|
||||
"first_round": min(a.round_num for a in payload_activities),
|
||||
"last_round": max(a.round_num for a in payload_activities),
|
||||
"agent_ids": ",".join(
|
||||
str(value)
|
||||
for value in sorted({a.agent_id for a in payload_activities})
|
||||
),
|
||||
"action_types": ",".join(
|
||||
value
|
||||
for value in sorted({a.action_type for a in payload_activities})
|
||||
if value
|
||||
) or "unknown",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
episode_uuid = (
|
||||
getattr(episode, "uuid_", None)
|
||||
or getattr(episode, "uuid", None)
|
||||
)
|
||||
if not episode_uuid:
|
||||
raise RuntimeError("Zep graph.add returned no episode UUID")
|
||||
self._pending_episode_uuids.append(str(episode_uuid))
|
||||
self._total_sent += 1
|
||||
self._total_items_sent += len(activities)
|
||||
self._total_items_sent += len(payload_activities)
|
||||
display_name = self._get_platform_display_name(platform)
|
||||
logger.info(f"成功批量发送 {len(activities)} 条{display_name}活动到图谱 {self.graph_id}")
|
||||
logger.info(f"成功批量发送 {len(payload_activities)} 条{display_name}活动到图谱 {self.graph_id}")
|
||||
logger.debug(f"批量内容预览: {combined_text[:200]}...")
|
||||
return
|
||||
|
||||
|
||||
except Exception as e:
|
||||
if attempt < self.MAX_RETRIES - 1:
|
||||
logger.warning(f"批量发送到Zep失败 (尝试 {attempt + 1}/{self.MAX_RETRIES}): {e}")
|
||||
time.sleep(self.RETRY_DELAY * (attempt + 1))
|
||||
else:
|
||||
logger.error(f"批量发送到Zep失败,已重试{self.MAX_RETRIES}次: {e}")
|
||||
self._failed_count += 1
|
||||
|
||||
def _flush_remaining(self):
|
||||
# graph.add has no idempotency key. Replaying an ambiguous
|
||||
# response can duplicate extracted facts, so fail closed and
|
||||
# surface the incomplete batch to SimulationRunner.
|
||||
logger.error(f"批量发送到Zep失败,未自动重放非幂等写入: {e}")
|
||||
self._failed_count += 1
|
||||
self._failed_batches.append({
|
||||
"platform": platform,
|
||||
"activities": payload_activities,
|
||||
"error": str(e),
|
||||
})
|
||||
finally:
|
||||
# Successes have a confirmed episode UUID; failures are kept
|
||||
# durably in _failed_batches and must never be replayed. Either
|
||||
# way this payload is accounted for before moving on.
|
||||
processed_count += len(payload_activities)
|
||||
return processed_count
|
||||
|
||||
@staticmethod
|
||||
def _to_rfc3339(value: str) -> str:
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
if parsed.tzinfo is None:
|
||||
parsed = parsed.astimezone()
|
||||
return parsed.isoformat()
|
||||
except (AttributeError, TypeError, ValueError):
|
||||
return datetime.now().astimezone().isoformat()
|
||||
|
||||
def _flush_remaining(self, *, deadline: float | None = None):
|
||||
"""发送队列和缓冲区中剩余的活动"""
|
||||
# 首先处理队列中剩余的活动,添加到缓冲区
|
||||
while not self._activity_queue.empty():
|
||||
|
|
@ -446,16 +567,54 @@ class ZepGraphMemoryUpdater:
|
|||
except Empty:
|
||||
break
|
||||
|
||||
# 然后发送各平台缓冲区中剩余的活动(即使不足BATCH_SIZE条)
|
||||
with self._buffer_lock:
|
||||
for platform, buffer in self._platform_buffers.items():
|
||||
if buffer:
|
||||
display_name = self._get_platform_display_name(platform)
|
||||
logger.info(f"发送{display_name}平台剩余的 {len(buffer)} 条活动")
|
||||
self._send_batch_activities(buffer, platform)
|
||||
# 清空所有缓冲区
|
||||
for platform in self._platform_buffers:
|
||||
self._platform_buffers[platform] = []
|
||||
for platform in list(self._platform_buffers):
|
||||
with self._buffer_lock:
|
||||
buffer = list(self._platform_buffers.get(platform, []))
|
||||
if not buffer:
|
||||
continue
|
||||
display_name = self._get_platform_display_name(platform)
|
||||
logger.info(f"发送{display_name}平台剩余的 {len(buffer)} 条活动")
|
||||
if deadline is not None and time.time() >= deadline:
|
||||
raise TimeoutError(
|
||||
"Zep updater drain deadline elapsed before flushing all activities"
|
||||
)
|
||||
try:
|
||||
processed_count = self._send_batch_activities(
|
||||
buffer,
|
||||
platform,
|
||||
deadline=deadline,
|
||||
)
|
||||
except _DrainDeadlineExceeded as error:
|
||||
with self._buffer_lock:
|
||||
del self._platform_buffers[platform][:error.processed_count]
|
||||
raise TimeoutError(str(error)) from error
|
||||
else:
|
||||
with self._buffer_lock:
|
||||
del self._platform_buffers[platform][:processed_count]
|
||||
|
||||
def _wait_for_pending_episodes(self, *, deadline: float | None = None) -> None:
|
||||
pending = set(self._pending_episode_uuids)
|
||||
if not pending:
|
||||
return
|
||||
|
||||
if deadline is None:
|
||||
deadline = time.time() + Config.ZEP_INGESTION_TIMEOUT_SECONDS
|
||||
while pending:
|
||||
if time.time() >= deadline:
|
||||
raise TimeoutError(
|
||||
f"Zep simulation ingestion timed out with {len(pending)} "
|
||||
"episode(s) pending"
|
||||
)
|
||||
for episode_uuid in list(pending):
|
||||
episode = call_zep_read_with_retry(
|
||||
lambda: self.client.graph.episode.get(uuid_=episode_uuid),
|
||||
operation_name=f"poll simulation episode {episode_uuid}",
|
||||
)
|
||||
if getattr(episode, "processed", False):
|
||||
pending.remove(episode_uuid)
|
||||
if pending:
|
||||
time.sleep(3)
|
||||
self._pending_episode_uuids = []
|
||||
|
||||
def get_stats(self) -> Dict[str, Any]:
|
||||
"""获取统计信息"""
|
||||
|
|
@ -469,6 +628,7 @@ class ZepGraphMemoryUpdater:
|
|||
"batches_sent": self._total_sent, # 成功发送的批次数
|
||||
"items_sent": self._total_items_sent, # 成功发送的活动条数
|
||||
"failed_count": self._failed_count, # 发送失败的批次数
|
||||
"pending_episode_count": len(self._pending_episode_uuids),
|
||||
"skipped_count": self._skipped_count, # 被过滤跳过的活动数(DO_NOTHING)
|
||||
"queue_size": self._activity_queue.qsize(),
|
||||
"buffer_sizes": buffer_sizes, # 各平台缓冲区大小
|
||||
|
|
@ -503,9 +663,13 @@ class ZepGraphMemoryManager:
|
|||
if simulation_id in cls._updaters:
|
||||
cls._updaters[simulation_id].stop()
|
||||
|
||||
updater = ZepGraphMemoryUpdater(graph_id)
|
||||
updater = ZepGraphMemoryUpdater(
|
||||
graph_id,
|
||||
simulation_id=simulation_id,
|
||||
)
|
||||
updater.start()
|
||||
cls._updaters[simulation_id] = updater
|
||||
cls._stop_all_done = False
|
||||
|
||||
logger.info(f"创建图谱记忆更新器: simulation_id={simulation_id}, graph_id={graph_id}")
|
||||
return updater
|
||||
|
|
@ -513,16 +677,68 @@ class ZepGraphMemoryManager:
|
|||
@classmethod
|
||||
def get_updater(cls, simulation_id: str) -> Optional[ZepGraphMemoryUpdater]:
|
||||
"""获取模拟的更新器"""
|
||||
return cls._updaters.get(simulation_id)
|
||||
with cls._lock:
|
||||
return cls._updaters.get(simulation_id)
|
||||
|
||||
@classmethod
|
||||
def get_simulation_ids_for_graph(cls, graph_id: str) -> List[str]:
|
||||
"""Return simulations whose updater still owns or drains this graph."""
|
||||
|
||||
with cls._lock:
|
||||
return sorted(
|
||||
simulation_id
|
||||
for simulation_id, updater in cls._updaters.items()
|
||||
if updater.graph_id == graph_id
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_simulation_ids(cls) -> List[str]:
|
||||
"""Return every simulation with a retained updater."""
|
||||
|
||||
with cls._lock:
|
||||
return sorted(cls._updaters)
|
||||
|
||||
@classmethod
|
||||
def discard_inactive_updater(cls, simulation_id: str) -> bool:
|
||||
"""Discard a failed, fully stopped updater during graph destruction."""
|
||||
|
||||
with cls._lock:
|
||||
updater = cls._updaters.get(simulation_id)
|
||||
if updater is None:
|
||||
return False
|
||||
worker_alive = bool(
|
||||
updater._worker_thread and updater._worker_thread.is_alive()
|
||||
)
|
||||
if updater._running or worker_alive:
|
||||
raise RuntimeError(
|
||||
f"Zep updater for {simulation_id} is still active"
|
||||
)
|
||||
cls._updaters.pop(simulation_id, None)
|
||||
logger.warning(
|
||||
"Discarded incomplete Zep updater during explicit graph deletion: "
|
||||
"simulation_id=%s, graph_id=%s",
|
||||
simulation_id,
|
||||
updater.graph_id,
|
||||
)
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def stop_updater(cls, simulation_id: str):
|
||||
"""停止并移除模拟的更新器"""
|
||||
with cls._lock:
|
||||
if simulation_id in cls._updaters:
|
||||
cls._updaters[simulation_id].stop()
|
||||
del cls._updaters[simulation_id]
|
||||
logger.info(f"已停止图谱记忆更新器: simulation_id={simulation_id}")
|
||||
updater = cls._updaters.get(simulation_id)
|
||||
if updater is None:
|
||||
return
|
||||
|
||||
# Do not hold the manager lock through up to several minutes of Cloud
|
||||
# polling. Crucially, only remove the updater after a successful drain;
|
||||
# on failure it remains visible to report/deletion barriers and can be
|
||||
# stopped again.
|
||||
updater.stop()
|
||||
with cls._lock:
|
||||
if cls._updaters.get(simulation_id) is updater:
|
||||
cls._updaters.pop(simulation_id, None)
|
||||
logger.info(f"已停止图谱记忆更新器: simulation_id={simulation_id}")
|
||||
|
||||
# 防止 stop_all 重复调用的标志
|
||||
_stop_all_done = False
|
||||
|
|
@ -533,17 +749,34 @@ class ZepGraphMemoryManager:
|
|||
# 防止重复调用
|
||||
if cls._stop_all_done:
|
||||
return
|
||||
cls._stop_all_done = True
|
||||
|
||||
|
||||
with cls._lock:
|
||||
if cls._updaters:
|
||||
for simulation_id, updater in list(cls._updaters.items()):
|
||||
try:
|
||||
updater.stop()
|
||||
except Exception as e:
|
||||
logger.error(f"停止更新器失败: simulation_id={simulation_id}, error={e}")
|
||||
cls._updaters.clear()
|
||||
logger.info("已停止所有图谱记忆更新器")
|
||||
simulation_ids = list(cls._updaters)
|
||||
|
||||
errors = []
|
||||
for simulation_id in simulation_ids:
|
||||
try:
|
||||
cls.stop_updater(simulation_id)
|
||||
except Exception as error:
|
||||
# Keep a failed updater registered so the caller can retry and
|
||||
# lifecycle/report guards still see the incomplete ingestion.
|
||||
logger.error(
|
||||
"停止更新器失败: simulation_id=%s, error=%s",
|
||||
simulation_id,
|
||||
error,
|
||||
)
|
||||
errors.append((simulation_id, error))
|
||||
|
||||
with cls._lock:
|
||||
cls._stop_all_done = not cls._updaters
|
||||
|
||||
if errors:
|
||||
details = "; ".join(
|
||||
f"{simulation_id}: {error}"
|
||||
for simulation_id, error in errors
|
||||
)
|
||||
raise RuntimeError(f"部分图谱更新器未完整停止: {details}")
|
||||
logger.info("已停止所有图谱记忆更新器")
|
||||
|
||||
@classmethod
|
||||
def get_all_stats(cls) -> Dict[str, Dict[str, Any]]:
|
||||
|
|
|
|||
|
|
@ -12,14 +12,19 @@ import time
|
|||
import json
|
||||
from typing import Dict, Any, List, Optional
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from zep_cloud.client import Zep
|
||||
from zep_cloud import NotFoundError
|
||||
|
||||
from ..config import Config
|
||||
from ..utils.logger import get_logger
|
||||
from ..utils.llm_client import LLMClient
|
||||
from ..utils.locale import get_locale, t
|
||||
from ..utils.zep_paging import fetch_all_nodes, fetch_all_edges
|
||||
from ..utils.zep import (
|
||||
call_zep_read_with_retry,
|
||||
get_zep_client,
|
||||
normalize_zep_search_limit,
|
||||
normalize_zep_search_query,
|
||||
)
|
||||
|
||||
logger = get_logger('mirofish.zep_tools')
|
||||
|
||||
|
|
@ -427,7 +432,7 @@ class ZepToolsService:
|
|||
if not self.api_key:
|
||||
raise ValueError("ZEP_API_KEY 未配置")
|
||||
|
||||
self.client = Zep(api_key=self.api_key)
|
||||
self.client = get_zep_client(self.api_key)
|
||||
# LLM客户端用于InsightForge生成子问题
|
||||
self._llm_client = llm_client
|
||||
logger.info(t("console.zepToolsInitialized"))
|
||||
|
|
@ -440,26 +445,14 @@ class ZepToolsService:
|
|||
return self._llm_client
|
||||
|
||||
def _call_with_retry(self, func, operation_name: str, max_retries: int = None):
|
||||
"""带重试机制的API调用"""
|
||||
max_retries = max_retries or self.MAX_RETRIES
|
||||
last_exception = None
|
||||
delay = self.RETRY_DELAY
|
||||
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
return func()
|
||||
except Exception as e:
|
||||
last_exception = e
|
||||
if attempt < max_retries - 1:
|
||||
logger.warning(
|
||||
t("console.zepRetryAttempt", operation=operation_name, attempt=attempt + 1, error=str(e)[:100], delay=f"{delay:.1f}")
|
||||
)
|
||||
time.sleep(delay)
|
||||
delay *= 2
|
||||
else:
|
||||
logger.error(t("console.zepAllRetriesFailed", operation=operation_name, retries=max_retries, error=str(e)))
|
||||
|
||||
raise last_exception
|
||||
"""Retry one safe read using typed Zep/HTTPX error classification."""
|
||||
|
||||
return call_zep_read_with_retry(
|
||||
func,
|
||||
operation_name=operation_name,
|
||||
max_attempts=max_retries or self.MAX_RETRIES,
|
||||
initial_delay=self.RETRY_DELAY,
|
||||
)
|
||||
|
||||
def search_graph(
|
||||
self,
|
||||
|
|
@ -485,13 +478,15 @@ class ZepToolsService:
|
|||
"""
|
||||
logger.info(t("console.graphSearch", graphId=graph_id, query=query[:50]))
|
||||
|
||||
# 尝试使用Zep Cloud Search API
|
||||
zep_query = normalize_zep_search_query(query)
|
||||
zep_limit = normalize_zep_search_limit(limit)
|
||||
|
||||
try:
|
||||
search_results = self._call_with_retry(
|
||||
func=lambda: self.client.graph.search(
|
||||
graph_id=graph_id,
|
||||
query=query,
|
||||
limit=limit,
|
||||
query=zep_query,
|
||||
limit=zep_limit,
|
||||
scope=scope,
|
||||
reranker="cross_encoder"
|
||||
),
|
||||
|
|
@ -539,9 +534,10 @@ class ZepToolsService:
|
|||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(t("console.zepSearchApiFallback", error=str(e)))
|
||||
# 降级:使用本地关键词匹配搜索
|
||||
return self._local_search(graph_id, query, limit, scope)
|
||||
# Authentication, invalid input, missing graphs, and exhausted
|
||||
# transient failures must remain visible to the report workflow.
|
||||
logger.error(t("console.zepSearchApiFallback", error=str(e)))
|
||||
raise
|
||||
|
||||
def _local_search(
|
||||
self,
|
||||
|
|
@ -741,9 +737,11 @@ class ZepToolsService:
|
|||
summary=node.summary or "",
|
||||
attributes=node.attributes or {}
|
||||
)
|
||||
except NotFoundError:
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(t("console.fetchNodeDetailFailed", error=str(e)))
|
||||
return None
|
||||
raise
|
||||
|
||||
def get_node_edges(self, graph_id: str, node_uuid: str) -> List[EdgeInfo]:
|
||||
"""
|
||||
|
|
@ -774,8 +772,8 @@ class ZepToolsService:
|
|||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(t("console.fetchNodeEdgesFailed", error=str(e)))
|
||||
return []
|
||||
logger.error(t("console.fetchNodeEdgesFailed", error=str(e)))
|
||||
raise
|
||||
|
||||
def get_entities_by_type(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ from typing import Any, Dict, List, Optional
|
|||
|
||||
MAX_ONTOLOGY_TYPES = 10
|
||||
MAX_ONTOLOGY_ATTRIBUTES = 10
|
||||
MAX_ONTOLOGY_SOURCE_TARGETS = 10
|
||||
RESERVED_ONTOLOGY_ATTRIBUTE_NAMES = frozenset({
|
||||
"uuid",
|
||||
"name",
|
||||
|
|
@ -67,3 +68,36 @@ def normalize_ontology_attributes(attributes: Any) -> List[Dict[str, Any]]:
|
|||
normalized_attributes.append(dict(_FALLBACK_ATTRIBUTE))
|
||||
|
||||
return normalized_attributes
|
||||
|
||||
|
||||
def normalize_ontology_source_targets(
|
||||
source_targets: Any,
|
||||
*,
|
||||
limit: int | None = MAX_ONTOLOGY_SOURCE_TARGETS,
|
||||
) -> List[Dict[str, str]]:
|
||||
"""Return unique, structurally valid source-target pairs within Zep limits."""
|
||||
|
||||
if not isinstance(source_targets, list):
|
||||
return []
|
||||
|
||||
normalized_targets: List[Dict[str, str]] = []
|
||||
seen: set[tuple[str, str]] = set()
|
||||
for source_target in source_targets:
|
||||
if not isinstance(source_target, dict):
|
||||
continue
|
||||
source = source_target.get("source")
|
||||
target = source_target.get("target")
|
||||
if not isinstance(source, str) or not source.strip():
|
||||
continue
|
||||
if not isinstance(target, str) or not target.strip():
|
||||
continue
|
||||
|
||||
pair = (source.strip(), target.strip())
|
||||
if pair in seen:
|
||||
continue
|
||||
seen.add(pair)
|
||||
normalized_targets.append({"source": pair[0], "target": pair[1]})
|
||||
if limit is not None and len(normalized_targets) == limit:
|
||||
break
|
||||
|
||||
return normalized_targets
|
||||
|
|
|
|||
|
|
@ -0,0 +1,158 @@
|
|||
"""Shared Zep Cloud client, request limits, and retry policy."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import time
|
||||
from functools import lru_cache
|
||||
from typing import Any, Callable, TypeVar
|
||||
|
||||
import httpx
|
||||
from zep_cloud.client import Zep
|
||||
from zep_cloud.core.api_error import ApiError as ZepApiError
|
||||
|
||||
from ..config import Config
|
||||
from .logger import get_logger
|
||||
|
||||
logger = get_logger("mirofish.zep")
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
ZEP_CLOUD_BASE_URL = "https://api.getzep.com/api/v2"
|
||||
MAX_ZEP_SEARCH_QUERY_CHARS = 400
|
||||
MAX_ZEP_SEARCH_RESULTS = 50
|
||||
|
||||
|
||||
def normalize_zep_search_query(query: Any) -> str:
|
||||
"""Return a non-empty query within Zep Cloud's endpoint limit."""
|
||||
|
||||
if not isinstance(query, str):
|
||||
raise ValueError("Zep search query must be a string")
|
||||
normalized = query.strip()
|
||||
if not normalized:
|
||||
raise ValueError("Zep search query must not be empty")
|
||||
return normalized[:MAX_ZEP_SEARCH_QUERY_CHARS]
|
||||
|
||||
|
||||
def normalize_zep_search_limit(limit: Any) -> int:
|
||||
"""Clamp a search result limit to the current Zep Cloud contract."""
|
||||
|
||||
try:
|
||||
normalized = int(limit)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ValueError("Zep search limit must be an integer") from exc
|
||||
if normalized < 1:
|
||||
raise ValueError("Zep search limit must be at least 1")
|
||||
return min(normalized, MAX_ZEP_SEARCH_RESULTS)
|
||||
|
||||
|
||||
@lru_cache(maxsize=4)
|
||||
def _cached_zep_client(api_key: str, timeout: float) -> Zep:
|
||||
return Zep(
|
||||
api_key=api_key,
|
||||
base_url=ZEP_CLOUD_BASE_URL,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
|
||||
def get_zep_client(api_key: str | None = None, timeout: float | None = None) -> Zep:
|
||||
"""Return a process-shared, explicitly configured Zep Cloud client."""
|
||||
|
||||
if Config._ZEP_CONFIG_PARSE_ERRORS:
|
||||
raise ValueError("; ".join(Config._ZEP_CONFIG_PARSE_ERRORS))
|
||||
|
||||
# zep-cloud gives ZEP_API_URL precedence even when base_url is explicit.
|
||||
# Reject it so this Cloud-only integration cannot silently target a
|
||||
# self-hosted or compatibility endpoint.
|
||||
if os.environ.get("ZEP_API_URL"):
|
||||
raise ValueError("ZEP_API_URL is unsupported; unset it to use Zep Cloud")
|
||||
|
||||
normalized_key = (api_key or Config.ZEP_API_KEY or "").strip()
|
||||
if not normalized_key:
|
||||
raise ValueError("ZEP_API_KEY 未配置")
|
||||
|
||||
request_timeout = float(
|
||||
timeout if timeout is not None else Config.ZEP_REQUEST_TIMEOUT_SECONDS
|
||||
)
|
||||
if request_timeout <= 0:
|
||||
raise ValueError("Zep request timeout must be greater than 0")
|
||||
return _cached_zep_client(normalized_key, request_timeout)
|
||||
|
||||
|
||||
def clear_zep_client_cache() -> None:
|
||||
"""Clear cached clients. Intended for tests and controlled reconfiguration."""
|
||||
|
||||
_cached_zep_client.cache_clear()
|
||||
|
||||
|
||||
def is_retryable_zep_error(error: BaseException) -> bool:
|
||||
"""Return whether a failed *read* is safe and useful to retry."""
|
||||
|
||||
if isinstance(error, (httpx.TimeoutException, httpx.TransportError)):
|
||||
return True
|
||||
if isinstance(error, (ConnectionError, TimeoutError, OSError)):
|
||||
return True
|
||||
if isinstance(error, ZepApiError):
|
||||
status_code = error.status_code
|
||||
return status_code in {408, 429} or (
|
||||
status_code is not None and 500 <= status_code <= 599
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
def _retry_after_seconds(error: BaseException) -> float | None:
|
||||
if not isinstance(error, ZepApiError) or not error.headers:
|
||||
return None
|
||||
value = next(
|
||||
(
|
||||
header_value
|
||||
for header_name, header_value in error.headers.items()
|
||||
if header_name.lower() == "retry-after"
|
||||
),
|
||||
None,
|
||||
)
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return max(0.0, float(value))
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def call_zep_read_with_retry(
|
||||
operation: Callable[[], T],
|
||||
*,
|
||||
operation_name: str,
|
||||
max_attempts: int = 3,
|
||||
initial_delay: float = 2.0,
|
||||
max_delay: float = 60.0,
|
||||
sleep: Callable[[float], None] = time.sleep,
|
||||
) -> T:
|
||||
"""Retry a safe Zep read only for transport, 408, 429, or 5xx errors."""
|
||||
|
||||
if max_attempts < 1:
|
||||
raise ValueError("max_attempts must be at least 1")
|
||||
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
try:
|
||||
return operation()
|
||||
except Exception as error:
|
||||
if attempt == max_attempts or not is_retryable_zep_error(error):
|
||||
raise
|
||||
|
||||
retry_after = _retry_after_seconds(error)
|
||||
delay = min(
|
||||
retry_after if retry_after is not None else initial_delay * (2 ** (attempt - 1)),
|
||||
max_delay,
|
||||
)
|
||||
logger.warning(
|
||||
"Zep %s attempt %s/%s failed (%s); retrying in %.1fs",
|
||||
operation_name,
|
||||
attempt,
|
||||
max_attempts,
|
||||
type(error).__name__,
|
||||
delay,
|
||||
)
|
||||
sleep(delay)
|
||||
|
||||
raise AssertionError("unreachable")
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
"""Process-local lifecycle coordination for Zep Cloud graphs.
|
||||
|
||||
The lock is intentionally keyed by graph ID so graph deletion/reset and a new
|
||||
simulation updater claim cannot pass each other between validation and their
|
||||
Cloud mutation. It complements (but does not replace) a distributed lock in
|
||||
multi-worker deployments.
|
||||
"""
|
||||
|
||||
import threading
|
||||
|
||||
|
||||
_graph_locks: dict[str, threading.RLock] = {}
|
||||
_graph_locks_guard = threading.Lock()
|
||||
_graph_readers: dict[str, set[str]] = {}
|
||||
|
||||
|
||||
def graph_lifecycle_lock(graph_id: str) -> threading.RLock:
|
||||
"""Return the process-local re-entrant lifecycle lock for ``graph_id``."""
|
||||
|
||||
if not graph_id:
|
||||
raise ValueError("graph_id is required for lifecycle locking")
|
||||
with _graph_locks_guard:
|
||||
return _graph_locks.setdefault(graph_id, threading.RLock())
|
||||
|
||||
|
||||
def register_graph_reader(graph_id: str, reader_id: str) -> None:
|
||||
"""Register a long-running read lease under the graph lifecycle lock."""
|
||||
|
||||
if not reader_id:
|
||||
raise ValueError("reader_id is required")
|
||||
with graph_lifecycle_lock(graph_id):
|
||||
_graph_readers.setdefault(graph_id, set()).add(reader_id)
|
||||
|
||||
|
||||
def unregister_graph_reader(graph_id: str, reader_id: str) -> None:
|
||||
"""Release a previously registered graph read lease."""
|
||||
|
||||
with graph_lifecycle_lock(graph_id):
|
||||
readers = _graph_readers.get(graph_id)
|
||||
if not readers:
|
||||
return
|
||||
readers.discard(reader_id)
|
||||
if not readers:
|
||||
_graph_readers.pop(graph_id, None)
|
||||
|
||||
|
||||
def get_graph_readers(graph_id: str) -> list[str]:
|
||||
"""Return active reader IDs while serializing with lifecycle mutations."""
|
||||
|
||||
with graph_lifecycle_lock(graph_id):
|
||||
return sorted(_graph_readers.get(graph_id, set()))
|
||||
|
|
@ -1,106 +1,142 @@
|
|||
"""Zep Graph 分页读取工具。
|
||||
|
||||
Zep 的 node/edge 列表接口使用 UUID cursor 分页,
|
||||
本模块封装自动翻页逻辑(含单页重试),对调用方透明地返回完整列表。
|
||||
"""
|
||||
"""Complete Zep Graph node/edge pagination using opaque response cursors."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from zep_cloud import InternalServerError
|
||||
from zep_cloud.client import Zep
|
||||
|
||||
from .logger import get_logger
|
||||
from .zep import call_zep_read_with_retry
|
||||
|
||||
logger = get_logger('mirofish.zep_paging')
|
||||
logger = get_logger("mirofish.zep_paging")
|
||||
|
||||
_DEFAULT_PAGE_SIZE = 100
|
||||
_MAX_NODES = 2000
|
||||
_MAX_EDGES = 5000
|
||||
_DEFAULT_MAX_RETRIES = 3
|
||||
_DEFAULT_RETRY_DELAY = 2.0 # seconds, doubles each retry
|
||||
_DEFAULT_RETRY_DELAY = 2.0
|
||||
_NEXT_CURSOR_HEADER = "zep-next-cursor"
|
||||
|
||||
|
||||
def _fetch_page_with_retry(
|
||||
api_call: Callable[..., list[Any]],
|
||||
api_call: Callable[..., Any],
|
||||
*args: Any,
|
||||
max_retries: int = _DEFAULT_MAX_RETRIES,
|
||||
retry_delay: float = _DEFAULT_RETRY_DELAY,
|
||||
page_description: str = "page",
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
"""Fetch one read-only page with the shared transient-error policy."""
|
||||
|
||||
return call_zep_read_with_retry(
|
||||
lambda: api_call(*args, **kwargs),
|
||||
operation_name=page_description,
|
||||
max_attempts=max_retries,
|
||||
initial_delay=retry_delay,
|
||||
)
|
||||
|
||||
|
||||
def _header_value(headers: Any, name: str) -> str | None:
|
||||
if not headers:
|
||||
return None
|
||||
direct = headers.get(name)
|
||||
if direct is not None:
|
||||
return str(direct)
|
||||
return next(
|
||||
(
|
||||
str(value)
|
||||
for header_name, value in headers.items()
|
||||
if str(header_name).lower() == name.lower()
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def _fetch_all(
|
||||
api_call: Callable[..., Any],
|
||||
graph_id: str,
|
||||
*,
|
||||
item_name: str,
|
||||
page_size: int,
|
||||
max_items: int | None,
|
||||
max_retries: int,
|
||||
retry_delay: float,
|
||||
) -> list[Any]:
|
||||
"""单页请求,失败时指数退避重试。仅重试网络/IO类瞬态错误。"""
|
||||
if max_retries < 1:
|
||||
raise ValueError("max_retries must be >= 1")
|
||||
if not 1 <= page_size <= 100:
|
||||
raise ValueError("page_size must be between 1 and 100")
|
||||
if max_items is not None and max_items < 1:
|
||||
raise ValueError("max_items must be at least 1 when provided")
|
||||
|
||||
last_exception: Exception | None = None
|
||||
delay = retry_delay
|
||||
all_items: list[Any] = []
|
||||
cursor: str | None = None
|
||||
seen_cursors: set[str] = set()
|
||||
page_number = 0
|
||||
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
return api_call(*args, **kwargs)
|
||||
except (ConnectionError, TimeoutError, OSError, InternalServerError) as e:
|
||||
last_exception = e
|
||||
if attempt < max_retries - 1:
|
||||
logger.warning(
|
||||
f"Zep {page_description} attempt {attempt + 1} failed: {str(e)[:100]}, retrying in {delay:.1f}s..."
|
||||
)
|
||||
time.sleep(delay)
|
||||
delay *= 2
|
||||
else:
|
||||
logger.error(f"Zep {page_description} failed after {max_retries} attempts: {str(e)}")
|
||||
while True:
|
||||
kwargs: dict[str, Any] = {"limit": page_size}
|
||||
if cursor is not None:
|
||||
kwargs["cursor"] = cursor
|
||||
|
||||
assert last_exception is not None
|
||||
raise last_exception
|
||||
page_number += 1
|
||||
response = _fetch_page_with_retry(
|
||||
api_call,
|
||||
graph_id,
|
||||
max_retries=max_retries,
|
||||
retry_delay=retry_delay,
|
||||
page_description=(
|
||||
f"fetch {item_name} page {page_number} (graph={graph_id})"
|
||||
),
|
||||
**kwargs,
|
||||
)
|
||||
batch = list(getattr(response, "data", None) or [])
|
||||
all_items.extend(batch)
|
||||
|
||||
if max_items is not None and len(all_items) >= max_items:
|
||||
if len(all_items) > max_items:
|
||||
all_items = all_items[:max_items]
|
||||
logger.warning(
|
||||
"Zep %s pagination reached explicit max_items=%s for graph %s",
|
||||
item_name,
|
||||
max_items,
|
||||
graph_id,
|
||||
)
|
||||
break
|
||||
|
||||
next_cursor = _header_value(
|
||||
getattr(response, "headers", None),
|
||||
_NEXT_CURSOR_HEADER,
|
||||
)
|
||||
if next_cursor is None:
|
||||
break
|
||||
if next_cursor in seen_cursors or next_cursor == cursor:
|
||||
raise RuntimeError(
|
||||
f"Zep {item_name} pagination cursor did not advance for graph {graph_id}"
|
||||
)
|
||||
seen_cursors.add(next_cursor)
|
||||
cursor = next_cursor
|
||||
|
||||
return all_items
|
||||
|
||||
|
||||
def fetch_all_nodes(
|
||||
client: Zep,
|
||||
graph_id: str,
|
||||
page_size: int = _DEFAULT_PAGE_SIZE,
|
||||
max_items: int = _MAX_NODES,
|
||||
max_items: int | None = None,
|
||||
max_retries: int = _DEFAULT_MAX_RETRIES,
|
||||
retry_delay: float = _DEFAULT_RETRY_DELAY,
|
||||
) -> list[Any]:
|
||||
"""分页获取图谱节点,最多返回 max_items 条(默认 2000)。每页请求自带重试。"""
|
||||
all_nodes: list[Any] = []
|
||||
cursor: str | None = None
|
||||
page_num = 0
|
||||
"""Fetch every graph node unless the caller supplies an explicit cap."""
|
||||
|
||||
while True:
|
||||
kwargs: dict[str, Any] = {"limit": page_size}
|
||||
if cursor is not None:
|
||||
kwargs["uuid_cursor"] = cursor
|
||||
|
||||
page_num += 1
|
||||
batch = _fetch_page_with_retry(
|
||||
client.graph.node.get_by_graph_id,
|
||||
graph_id,
|
||||
max_retries=max_retries,
|
||||
retry_delay=retry_delay,
|
||||
page_description=f"fetch nodes page {page_num} (graph={graph_id})",
|
||||
**kwargs,
|
||||
)
|
||||
if not batch:
|
||||
break
|
||||
|
||||
all_nodes.extend(batch)
|
||||
if len(all_nodes) >= max_items:
|
||||
all_nodes = all_nodes[:max_items]
|
||||
logger.warning(f"Node count reached limit ({max_items}), stopping pagination for graph {graph_id}")
|
||||
break
|
||||
if len(batch) < page_size:
|
||||
break
|
||||
|
||||
cursor = getattr(batch[-1], "uuid_", None) or getattr(batch[-1], "uuid", None)
|
||||
if cursor is None:
|
||||
logger.warning(f"Node missing uuid field, stopping pagination at {len(all_nodes)} nodes")
|
||||
break
|
||||
|
||||
return all_nodes
|
||||
return _fetch_all(
|
||||
client.graph.node.with_raw_response.get_by_graph_id,
|
||||
graph_id,
|
||||
item_name="nodes",
|
||||
page_size=page_size,
|
||||
max_items=max_items,
|
||||
max_retries=max_retries,
|
||||
retry_delay=retry_delay,
|
||||
)
|
||||
|
||||
|
||||
def fetch_all_edges(
|
||||
|
|
@ -109,41 +145,16 @@ def fetch_all_edges(
|
|||
page_size: int = _DEFAULT_PAGE_SIZE,
|
||||
max_retries: int = _DEFAULT_MAX_RETRIES,
|
||||
retry_delay: float = _DEFAULT_RETRY_DELAY,
|
||||
max_items: int = _MAX_EDGES,
|
||||
max_items: int | None = None,
|
||||
) -> list[Any]:
|
||||
"""分页获取图谱所有边,最多返回 max_items 条(默认 5000)。每页请求自带重试。"""
|
||||
all_edges: list[Any] = []
|
||||
cursor: str | None = None
|
||||
page_num = 0
|
||||
"""Fetch every graph edge unless the caller supplies an explicit cap."""
|
||||
|
||||
while True:
|
||||
kwargs: dict[str, Any] = {"limit": page_size}
|
||||
if cursor is not None:
|
||||
kwargs["uuid_cursor"] = cursor
|
||||
|
||||
page_num += 1
|
||||
batch = _fetch_page_with_retry(
|
||||
client.graph.edge.get_by_graph_id,
|
||||
graph_id,
|
||||
max_retries=max_retries,
|
||||
retry_delay=retry_delay,
|
||||
page_description=f"fetch edges page {page_num} (graph={graph_id})",
|
||||
**kwargs,
|
||||
)
|
||||
if not batch:
|
||||
break
|
||||
|
||||
all_edges.extend(batch)
|
||||
if len(all_edges) >= max_items:
|
||||
all_edges = all_edges[:max_items]
|
||||
logger.warning(f"Edge count reached limit ({max_items}), stopping pagination for graph {graph_id}")
|
||||
break
|
||||
if len(batch) < page_size:
|
||||
break
|
||||
|
||||
cursor = getattr(batch[-1], "uuid_", None) or getattr(batch[-1], "uuid", None)
|
||||
if cursor is None:
|
||||
logger.warning(f"Edge missing uuid field, stopping pagination at {len(all_edges)} edges")
|
||||
break
|
||||
|
||||
return all_edges
|
||||
return _fetch_all(
|
||||
client.graph.edge.with_raw_response.get_by_graph_id,
|
||||
graph_id,
|
||||
item_name="edges",
|
||||
page_size=page_size,
|
||||
max_items=max_items,
|
||||
max_retries=max_retries,
|
||||
retry_delay=retry_delay,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -17,7 +17,8 @@ dependencies = [
|
|||
"openai>=1.0.0",
|
||||
|
||||
# Zep Cloud
|
||||
"zep-cloud==3.13.0",
|
||||
"zep-cloud==3.25.0",
|
||||
"httpx>=0.27.0",
|
||||
|
||||
# OASIS 社交媒体模拟
|
||||
"camel-oasis==0.2.5",
|
||||
|
|
|
|||
|
|
@ -14,7 +14,8 @@ flask-cors>=6.0.0
|
|||
openai>=1.0.0
|
||||
|
||||
# ============= Zep Cloud =============
|
||||
zep-cloud==3.13.0
|
||||
zep-cloud==3.25.0
|
||||
httpx>=0.27.0
|
||||
|
||||
# ============= OASIS 社交媒体模拟 =============
|
||||
# OASIS 社交模拟框架
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ from app.services.ontology_generator import OntologyGenerator
|
|||
from app.services.graph_builder import GraphBuilderService
|
||||
from app.utils.ontology import (
|
||||
MAX_ONTOLOGY_ATTRIBUTES,
|
||||
MAX_ONTOLOGY_SOURCE_TARGETS,
|
||||
normalize_ontology_attribute,
|
||||
normalize_ontology_attributes,
|
||||
)
|
||||
|
|
@ -168,3 +169,89 @@ def test_graph_builder_passes_an_empty_entity_mapping_for_edge_only_ontology():
|
|||
})
|
||||
|
||||
assert captured["entities"] == {}
|
||||
|
||||
|
||||
def test_graph_builder_deduplicates_and_caps_edge_source_targets_for_zep():
|
||||
captured = {}
|
||||
|
||||
class GraphApi:
|
||||
def set_ontology(self, **kwargs):
|
||||
captured.update(kwargs)
|
||||
|
||||
class Client:
|
||||
graph = GraphApi()
|
||||
|
||||
source_targets = [
|
||||
{"source": f"Source{index}", "target": f"Target{index}"}
|
||||
for index in range(MAX_ONTOLOGY_SOURCE_TARGETS + 2)
|
||||
]
|
||||
source_targets.insert(1, dict(source_targets[0]))
|
||||
|
||||
builder = object.__new__(GraphBuilderService)
|
||||
builder.client = Client()
|
||||
builder.set_ontology("graph-id", {
|
||||
"entity_types": [],
|
||||
"edge_types": [{
|
||||
"name": "RELATED_TO",
|
||||
"attributes": ["reason"],
|
||||
"source_targets": source_targets,
|
||||
}],
|
||||
})
|
||||
|
||||
_, normalized_targets = captured["edges"]["RELATED_TO"]
|
||||
assert len(normalized_targets) == MAX_ONTOLOGY_SOURCE_TARGETS
|
||||
assert [(item.source, item.target) for item in normalized_targets] == [
|
||||
(f"Source{index}", f"Target{index}")
|
||||
for index in range(MAX_ONTOLOGY_SOURCE_TARGETS)
|
||||
]
|
||||
|
||||
|
||||
def test_generator_ignores_invalid_entries_and_normalizes_edge_names():
|
||||
source_targets = [
|
||||
{"source": "speaker", "target": "news outlet"},
|
||||
{"source": "speaker", "target": "news outlet"},
|
||||
None,
|
||||
] + [
|
||||
{"source": "speaker", "target": "news outlet" if index == 0 else "Person"}
|
||||
for index in range(12)
|
||||
]
|
||||
|
||||
result = OntologyGenerator(llm_client=object())._validate_and_process({
|
||||
"entity_types": ["speaker", None, 7, {"name": "news outlet"}],
|
||||
"edge_types": [
|
||||
"unusable edge",
|
||||
None,
|
||||
{"name": "worksFor", "source_targets": source_targets},
|
||||
{"name": "works-for", "source_targets": []},
|
||||
],
|
||||
})
|
||||
|
||||
assert [entity["name"] for entity in result["entity_types"][:2]] == [
|
||||
"Speaker",
|
||||
"NewsOutlet",
|
||||
]
|
||||
assert [edge["name"] for edge in result["edge_types"]] == ["WORKS_FOR"]
|
||||
assert result["edge_types"][0]["source_targets"] == [
|
||||
{"source": "Speaker", "target": "NewsOutlet"},
|
||||
{"source": "Speaker", "target": "Person"},
|
||||
]
|
||||
|
||||
|
||||
def test_generator_caps_after_discarding_invalid_edge_endpoints():
|
||||
invalid_first = [
|
||||
{"source": f"Removed{index}", "target": "AlsoRemoved"}
|
||||
for index in range(MAX_ONTOLOGY_SOURCE_TARGETS)
|
||||
]
|
||||
result = OntologyGenerator(llm_client=object())._validate_and_process({
|
||||
"entity_types": [{"name": "person"}, {"name": "organization"}],
|
||||
"edge_types": [{
|
||||
"name": "works_for",
|
||||
"source_targets": invalid_first + [
|
||||
{"source": "person", "target": "organization"}
|
||||
],
|
||||
}],
|
||||
})
|
||||
|
||||
assert result["edge_types"][0]["source_targets"] == [
|
||||
{"source": "Person", "target": "Organization"}
|
||||
]
|
||||
|
|
|
|||
|
|
@ -0,0 +1,437 @@
|
|||
from types import SimpleNamespace
|
||||
import json
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from zep_cloud import Zep
|
||||
from zep_cloud.core.api_error import ApiError as ZepApiError
|
||||
|
||||
from app.services import graph_builder as graph_builder_module
|
||||
from app.services.graph_builder import BatchSubmission, GraphBuilderService
|
||||
from app.services.oasis_profile_generator import OasisProfileGenerator
|
||||
from app.services.zep_entity_reader import EntityNode, ZepEntityReader
|
||||
from app.services.zep_tools import ZepToolsService
|
||||
|
||||
|
||||
def test_report_search_caps_the_query_sent_to_zep():
|
||||
calls = []
|
||||
|
||||
class GraphApi:
|
||||
def search(self, **kwargs):
|
||||
calls.append(kwargs)
|
||||
return SimpleNamespace(edges=[], nodes=[])
|
||||
|
||||
service = object.__new__(ZepToolsService)
|
||||
service.client = SimpleNamespace(graph=GraphApi())
|
||||
|
||||
original_query = "q" * 401
|
||||
result = service.search_graph("graph-id", original_query)
|
||||
|
||||
assert calls[0]["query"] == original_query[:400]
|
||||
assert result.query == original_query
|
||||
|
||||
|
||||
def test_profile_context_search_caps_both_queries_sent_to_zep():
|
||||
calls = []
|
||||
|
||||
class GraphApi:
|
||||
def search(self, **kwargs):
|
||||
calls.append(kwargs)
|
||||
return SimpleNamespace(edges=[], nodes=[])
|
||||
|
||||
generator = object.__new__(OasisProfileGenerator)
|
||||
generator.zep_client = SimpleNamespace(graph=GraphApi())
|
||||
generator.graph_id = "graph-id"
|
||||
|
||||
entity = EntityNode(
|
||||
uuid="node-id",
|
||||
name="n" * 500,
|
||||
labels=["Entity", "Person"],
|
||||
summary="",
|
||||
attributes={},
|
||||
)
|
||||
generator._search_zep_for_entity(entity)
|
||||
|
||||
assert len(calls) == 2
|
||||
assert all(0 < len(call["query"]) <= 400 for call in calls)
|
||||
|
||||
|
||||
def test_entity_reader_does_not_turn_auth_failure_into_missing_entity():
|
||||
def unauthorized(**_kwargs):
|
||||
raise ZepApiError(status_code=401, body={"message": "unauthorized"})
|
||||
|
||||
reader = object.__new__(ZepEntityReader)
|
||||
reader.client = SimpleNamespace(
|
||||
graph=SimpleNamespace(node=SimpleNamespace(get=unauthorized))
|
||||
)
|
||||
|
||||
with pytest.raises(ZepApiError) as error:
|
||||
reader.get_entity_with_context("graph-id", "node-id")
|
||||
|
||||
assert error.value.status_code == 401
|
||||
|
||||
|
||||
def test_entity_reader_does_not_turn_edge_failure_into_empty_data():
|
||||
def forbidden(**_kwargs):
|
||||
raise ZepApiError(status_code=403, body={"message": "forbidden"})
|
||||
|
||||
reader = object.__new__(ZepEntityReader)
|
||||
reader.client = SimpleNamespace(
|
||||
graph=SimpleNamespace(
|
||||
node=SimpleNamespace(get_edges=forbidden),
|
||||
)
|
||||
)
|
||||
|
||||
with pytest.raises(ZepApiError) as error:
|
||||
reader.get_node_edges("node-id")
|
||||
|
||||
assert error.value.status_code == 403
|
||||
|
||||
|
||||
def test_report_tools_do_not_turn_zep_read_failures_into_empty_data():
|
||||
def unauthorized(**_kwargs):
|
||||
raise ZepApiError(status_code=401, body={"message": "unauthorized"})
|
||||
|
||||
service = object.__new__(ZepToolsService)
|
||||
service.client = SimpleNamespace(
|
||||
graph=SimpleNamespace(node=SimpleNamespace(get=unauthorized))
|
||||
)
|
||||
|
||||
with pytest.raises(ZepApiError):
|
||||
service.get_node_detail("node-id")
|
||||
|
||||
service.get_all_edges = lambda _graph_id: (_ for _ in ()).throw(
|
||||
ZepApiError(status_code=503, body={"message": "unavailable"})
|
||||
)
|
||||
with pytest.raises(ZepApiError):
|
||||
service.get_node_edges("graph-id", "node-id")
|
||||
|
||||
|
||||
def test_episode_processing_timeout_fails_instead_of_reporting_success(monkeypatch):
|
||||
builder = object.__new__(GraphBuilderService)
|
||||
builder.client = SimpleNamespace(
|
||||
graph=SimpleNamespace(
|
||||
episode=SimpleNamespace(
|
||||
get=lambda **_kwargs: SimpleNamespace(processed=False)
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
timestamps = iter([0.0, 2.0])
|
||||
monkeypatch.setattr(graph_builder_module.time, "time", lambda: next(timestamps))
|
||||
monkeypatch.setattr(graph_builder_module.time, "sleep", lambda _seconds: None)
|
||||
|
||||
with pytest.raises(TimeoutError, match="episode"):
|
||||
builder._wait_for_episodes(["episode-1"], timeout=1)
|
||||
|
||||
|
||||
def test_document_ingestion_uses_current_batch_api_and_persists_identity():
|
||||
calls = []
|
||||
|
||||
class BatchApi:
|
||||
def create(self, **kwargs):
|
||||
calls.append(("create", kwargs))
|
||||
return SimpleNamespace(batch_id="batch-1")
|
||||
|
||||
def add(self, **kwargs):
|
||||
calls.append(("add", kwargs))
|
||||
return [
|
||||
SimpleNamespace(episode_uuid=f"episode-{index}")
|
||||
for index, _item in enumerate(kwargs["items"])
|
||||
]
|
||||
|
||||
def process(self, **kwargs):
|
||||
calls.append(("process", kwargs))
|
||||
return SimpleNamespace(status="queued")
|
||||
|
||||
builder = object.__new__(GraphBuilderService)
|
||||
builder.client = SimpleNamespace(batch=BatchApi())
|
||||
persisted = []
|
||||
|
||||
submission = builder.add_text_batches(
|
||||
"graph-id",
|
||||
["chunk one", "chunk two"],
|
||||
batch_created_callback=lambda batch_id, operation_id: persisted.append(
|
||||
(batch_id, operation_id)
|
||||
),
|
||||
)
|
||||
|
||||
assert submission.batch_id == "batch-1"
|
||||
assert submission.item_count == 2
|
||||
assert len(submission.operation_id) == 64
|
||||
assert persisted == [
|
||||
(None, submission.operation_id),
|
||||
("batch-1", submission.operation_id),
|
||||
]
|
||||
assert [name for name, _kwargs in calls] == ["create", "add", "process"]
|
||||
items = calls[1][1]["items"]
|
||||
assert [item.type for item in items] == ["graph_episode", "graph_episode"]
|
||||
assert all(item.graph_id == "graph-id" for item in items)
|
||||
assert all(item.data_type == "text" for item in items)
|
||||
|
||||
|
||||
def test_graph_create_persists_identity_before_post_and_reconciles_timeout():
|
||||
events = []
|
||||
|
||||
class GraphApi:
|
||||
def create(self, **kwargs):
|
||||
events.append(("create", kwargs["graph_id"]))
|
||||
raise TimeoutError("response lost")
|
||||
|
||||
def get(self, graph_id):
|
||||
events.append(("get", graph_id))
|
||||
return SimpleNamespace(graph_id=graph_id)
|
||||
|
||||
builder = object.__new__(GraphBuilderService)
|
||||
builder.client = SimpleNamespace(graph=GraphApi())
|
||||
|
||||
graph_id = builder.create_graph(
|
||||
"Graph",
|
||||
graph_id="known-id",
|
||||
graph_id_callback=lambda value: events.append(("persist", value)),
|
||||
)
|
||||
|
||||
assert graph_id == "known-id"
|
||||
assert events == [
|
||||
("persist", "known-id"),
|
||||
("create", "known-id"),
|
||||
("get", "known-id"),
|
||||
]
|
||||
|
||||
|
||||
def test_batch_create_timeout_is_reconciled_by_operation_metadata(monkeypatch):
|
||||
calls = []
|
||||
list_count = 0
|
||||
|
||||
class BatchApi:
|
||||
def create(self, **_kwargs):
|
||||
calls.append("create")
|
||||
raise TimeoutError("response lost")
|
||||
|
||||
def list(self, **_kwargs):
|
||||
nonlocal list_count
|
||||
calls.append("list")
|
||||
list_count += 1
|
||||
if list_count == 1:
|
||||
return SimpleNamespace(batches=[], next_cursor=None)
|
||||
return SimpleNamespace(
|
||||
batches=[SimpleNamespace(
|
||||
batch_id="batch-recovered",
|
||||
metadata={
|
||||
"mirofish_operation_id": GraphBuilderService.build_operation_id(
|
||||
"graph-id", ["chunk"]
|
||||
),
|
||||
"graph_id": "graph-id",
|
||||
},
|
||||
)],
|
||||
next_cursor=None,
|
||||
)
|
||||
|
||||
def add(self, **kwargs):
|
||||
calls.append("add")
|
||||
return [SimpleNamespace(episode_uuid="episode-1")]
|
||||
|
||||
def process(self, **_kwargs):
|
||||
calls.append("process")
|
||||
return SimpleNamespace(status="queued")
|
||||
|
||||
builder = object.__new__(GraphBuilderService)
|
||||
builder.client = SimpleNamespace(batch=BatchApi())
|
||||
monkeypatch.setattr(graph_builder_module.time, "sleep", lambda _seconds: None)
|
||||
|
||||
submission = builder.add_text_batches("graph-id", ["chunk"])
|
||||
|
||||
assert submission.batch_id == "batch-recovered"
|
||||
assert calls == ["create", "list", "list", "add", "process"]
|
||||
|
||||
|
||||
def test_batch_add_timeout_recovers_a_fully_accepted_group_without_replay(monkeypatch):
|
||||
add_calls = []
|
||||
list_calls = []
|
||||
|
||||
class BatchApi:
|
||||
def create(self, **_kwargs):
|
||||
return SimpleNamespace(batch_id="batch-1")
|
||||
|
||||
def add(self, **_kwargs):
|
||||
add_calls.append(True)
|
||||
raise TimeoutError("response lost")
|
||||
|
||||
def list_items(self, **_kwargs):
|
||||
list_calls.append(True)
|
||||
if len(list_calls) == 1:
|
||||
return SimpleNamespace(items=[], next_cursor=None)
|
||||
return SimpleNamespace(
|
||||
items=[
|
||||
SimpleNamespace(sequence_index=0, episode_uuid="episode-1"),
|
||||
SimpleNamespace(sequence_index=1, episode_uuid="episode-2"),
|
||||
],
|
||||
next_cursor=None,
|
||||
)
|
||||
|
||||
def process(self, **_kwargs):
|
||||
return SimpleNamespace(status="queued")
|
||||
|
||||
builder = object.__new__(GraphBuilderService)
|
||||
builder.client = SimpleNamespace(batch=BatchApi())
|
||||
monkeypatch.setattr(graph_builder_module.time, "sleep", lambda _seconds: None)
|
||||
|
||||
submission = builder.add_text_batches(
|
||||
"graph-id", ["chunk one", "chunk two"]
|
||||
)
|
||||
|
||||
assert submission.item_count == 2
|
||||
assert add_calls == [True]
|
||||
assert len(list_calls) == 2
|
||||
|
||||
|
||||
def test_batch_wait_validates_terminal_items_and_opaque_zero_cursor():
|
||||
list_calls = []
|
||||
|
||||
class BatchApi:
|
||||
def get(self, **_kwargs):
|
||||
return SimpleNamespace(
|
||||
status="succeeded",
|
||||
progress=SimpleNamespace(
|
||||
percent_complete=100,
|
||||
succeeded_items=2,
|
||||
),
|
||||
)
|
||||
|
||||
def list_items(self, **kwargs):
|
||||
list_calls.append(kwargs)
|
||||
if kwargs["cursor"] is None:
|
||||
return SimpleNamespace(
|
||||
items=[SimpleNamespace(
|
||||
sequence_index=0,
|
||||
status="succeeded",
|
||||
episode_uuid="episode-1",
|
||||
source_uuid="episode-1",
|
||||
)],
|
||||
next_cursor=0,
|
||||
)
|
||||
return SimpleNamespace(
|
||||
items=[SimpleNamespace(
|
||||
sequence_index=1,
|
||||
status="succeeded",
|
||||
episode_uuid="episode-2",
|
||||
source_uuid="episode-2",
|
||||
)],
|
||||
next_cursor=None,
|
||||
)
|
||||
|
||||
builder = object.__new__(GraphBuilderService)
|
||||
builder.client = SimpleNamespace(batch=BatchApi())
|
||||
submission = BatchSubmission("batch-1", "operation", [], 2)
|
||||
|
||||
assert builder._wait_for_batch(submission, timeout=1) == [
|
||||
"episode-1",
|
||||
"episode-2",
|
||||
]
|
||||
assert [call["cursor"] for call in list_calls] == [None, 0]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status", ["partial", "failed", "invalid", "canceled"])
|
||||
def test_batch_non_success_terminal_states_fail(status):
|
||||
builder = object.__new__(GraphBuilderService)
|
||||
builder.client = SimpleNamespace(
|
||||
batch=SimpleNamespace(
|
||||
get=lambda **_kwargs: SimpleNamespace(status=status, progress=None),
|
||||
list_items=lambda **_kwargs: SimpleNamespace(
|
||||
items=[SimpleNamespace(status="failed", error={"message": "bad"})],
|
||||
next_cursor=None,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match=status):
|
||||
builder._wait_for_batch(
|
||||
BatchSubmission("batch-1", "operation", [], 1),
|
||||
timeout=1,
|
||||
)
|
||||
|
||||
|
||||
def test_batch_wait_times_out_while_status_remains_nonterminal(monkeypatch):
|
||||
builder = object.__new__(GraphBuilderService)
|
||||
builder.client = SimpleNamespace(
|
||||
batch=SimpleNamespace(
|
||||
get=lambda **_kwargs: SimpleNamespace(status="processing", progress=None)
|
||||
)
|
||||
)
|
||||
timestamps = iter([0.0, 2.0])
|
||||
monkeypatch.setattr(graph_builder_module.time, "time", lambda: next(timestamps))
|
||||
monkeypatch.setattr(graph_builder_module.time, "sleep", lambda _seconds: None)
|
||||
|
||||
with pytest.raises(TimeoutError, match="batch-1"):
|
||||
builder._wait_for_batch(
|
||||
BatchSubmission("batch-1", "operation", [], 1),
|
||||
timeout=1,
|
||||
)
|
||||
|
||||
|
||||
def test_installed_sdk_serializes_the_batch_325_contract():
|
||||
requests = []
|
||||
|
||||
def handler(request):
|
||||
requests.append((request.method, request.url.path, request.content))
|
||||
path = request.url.path
|
||||
if path.endswith("/batches") and request.method == "POST":
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={"batch_id": "batch-1", "status": "draft", "item_count": 0},
|
||||
)
|
||||
if path.endswith("/batches/batch-1/items") and request.method == "POST":
|
||||
return httpx.Response(200, json=[{
|
||||
"item_id": "item-1",
|
||||
"sequence_index": 0,
|
||||
"status": "pending",
|
||||
"episode_uuid": "episode-1",
|
||||
"source_uuid": "episode-1",
|
||||
}])
|
||||
if path.endswith("/batches/batch-1/process"):
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={"batch_id": "batch-1", "status": "queued", "item_count": 1},
|
||||
)
|
||||
if path.endswith("/batches/batch-1"):
|
||||
return httpx.Response(200, json={
|
||||
"batch_id": "batch-1",
|
||||
"status": "succeeded",
|
||||
"item_count": 1,
|
||||
"progress": {"percent_complete": 100, "succeeded_items": 1},
|
||||
})
|
||||
if path.endswith("/batches/batch-1/items") and request.method == "GET":
|
||||
return httpx.Response(200, json={
|
||||
"items": [{
|
||||
"item_id": "item-1",
|
||||
"sequence_index": 0,
|
||||
"status": "succeeded",
|
||||
"episode_uuid": "episode-1",
|
||||
"source_uuid": "episode-1",
|
||||
}],
|
||||
"next_cursor": None,
|
||||
})
|
||||
raise AssertionError(f"Unexpected request: {request.method} {path}")
|
||||
|
||||
with httpx.Client(transport=httpx.MockTransport(handler)) as transport_client:
|
||||
builder = object.__new__(GraphBuilderService)
|
||||
builder.client = Zep(api_key="test-key", httpx_client=transport_client)
|
||||
submission = builder.add_text_batches("graph-id", ["source chunk"])
|
||||
assert builder._wait_for_batch(submission, timeout=1) == ["episode-1"]
|
||||
|
||||
assert [(method, path) for method, path, _body in requests] == [
|
||||
("POST", "/api/v2/batches"),
|
||||
("POST", "/api/v2/batches/batch-1/items"),
|
||||
("POST", "/api/v2/batches/batch-1/process"),
|
||||
("GET", "/api/v2/batches/batch-1"),
|
||||
("GET", "/api/v2/batches/batch-1/items"),
|
||||
]
|
||||
add_payload = json.loads(requests[1][2])
|
||||
assert add_payload["items"][0] == {
|
||||
"data": "source chunk",
|
||||
"data_type": "text",
|
||||
"graph_id": "graph-id",
|
||||
"metadata": add_payload["items"][0]["metadata"],
|
||||
"source_description": "MiroFish source document chunk",
|
||||
"type": "graph_episode",
|
||||
}
|
||||
|
|
@ -4,14 +4,27 @@ from app.utils import zep_paging
|
|||
|
||||
|
||||
def _client():
|
||||
edge_api = SimpleNamespace(get_by_graph_id=lambda *args, **kwargs: [])
|
||||
edge_api = SimpleNamespace(
|
||||
with_raw_response=SimpleNamespace(
|
||||
get_by_graph_id=lambda *args, **kwargs: SimpleNamespace(
|
||||
data=[],
|
||||
headers={},
|
||||
)
|
||||
)
|
||||
)
|
||||
return SimpleNamespace(graph=SimpleNamespace(edge=edge_api))
|
||||
|
||||
|
||||
def test_edge_cap_stops_pagination_at_requested_limit(monkeypatch):
|
||||
pages = [
|
||||
[SimpleNamespace(uuid_="e1"), SimpleNamespace(uuid_="e2")],
|
||||
[SimpleNamespace(uuid_="e3"), SimpleNamespace(uuid_="e4")],
|
||||
SimpleNamespace(
|
||||
data=[SimpleNamespace(uuid_="e1"), SimpleNamespace(uuid_="e2")],
|
||||
headers={"Zep-Next-Cursor": "opaque-page-2"},
|
||||
),
|
||||
SimpleNamespace(
|
||||
data=[SimpleNamespace(uuid_="e3"), SimpleNamespace(uuid_="e4")],
|
||||
headers={},
|
||||
),
|
||||
]
|
||||
calls = []
|
||||
|
||||
|
|
@ -25,7 +38,7 @@ def test_edge_cap_stops_pagination_at_requested_limit(monkeypatch):
|
|||
|
||||
assert [edge.uuid_ for edge in result] == ["e1", "e2", "e3"]
|
||||
assert len(calls) == 2
|
||||
assert calls[1]["uuid_cursor"] == "e2"
|
||||
assert calls[1]["cursor"] == "opaque-page-2"
|
||||
|
||||
|
||||
def test_existing_positional_retry_arguments_keep_their_meaning(monkeypatch):
|
||||
|
|
@ -33,7 +46,7 @@ def test_existing_positional_retry_arguments_keep_their_meaning(monkeypatch):
|
|||
|
||||
def fake_fetch(*args, **kwargs):
|
||||
observed.update(kwargs)
|
||||
return []
|
||||
return SimpleNamespace(data=[], headers={})
|
||||
|
||||
monkeypatch.setattr(zep_paging, "_fetch_page_with_retry", fake_fetch)
|
||||
|
||||
|
|
@ -42,3 +55,58 @@ def test_existing_positional_retry_arguments_keep_their_meaning(monkeypatch):
|
|||
assert observed["limit"] == 25
|
||||
assert observed["max_retries"] == 7
|
||||
assert observed["retry_delay"] == 0.25
|
||||
|
||||
|
||||
def test_pagination_uses_the_current_opaque_response_cursor():
|
||||
calls = []
|
||||
|
||||
class RawEdgeApi:
|
||||
def get_by_graph_id(self, graph_id, **kwargs):
|
||||
calls.append((graph_id, kwargs))
|
||||
if kwargs.get("cursor") is None:
|
||||
return SimpleNamespace(
|
||||
data=[SimpleNamespace(uuid_="e1"), SimpleNamespace(uuid_="e2")],
|
||||
headers={"Zep-Next-Cursor": "opaque-page-2"},
|
||||
)
|
||||
return SimpleNamespace(
|
||||
data=[SimpleNamespace(uuid_="e3")],
|
||||
headers={},
|
||||
)
|
||||
|
||||
class EdgeApi:
|
||||
with_raw_response = RawEdgeApi()
|
||||
|
||||
def get_by_graph_id(self, *_args, **_kwargs):
|
||||
raise AssertionError("pagination must read the response cursor header")
|
||||
|
||||
client = SimpleNamespace(
|
||||
graph=SimpleNamespace(edge=EdgeApi())
|
||||
)
|
||||
|
||||
result = zep_paging.fetch_all_edges(client, "graph", page_size=2)
|
||||
|
||||
assert [edge.uuid_ for edge in result] == ["e1", "e2", "e3"]
|
||||
assert calls == [
|
||||
("graph", {"limit": 2}),
|
||||
("graph", {"limit": 2, "cursor": "opaque-page-2"}),
|
||||
]
|
||||
|
||||
|
||||
def test_pagination_fails_if_the_service_repeats_a_cursor():
|
||||
class RawEdgeApi:
|
||||
def get_by_graph_id(self, _graph_id, **_kwargs):
|
||||
return SimpleNamespace(
|
||||
data=[SimpleNamespace(uuid_="e1")],
|
||||
headers={"Zep-Next-Cursor": "same-cursor"},
|
||||
)
|
||||
|
||||
client = SimpleNamespace(
|
||||
graph=SimpleNamespace(
|
||||
edge=SimpleNamespace(with_raw_response=RawEdgeApi())
|
||||
)
|
||||
)
|
||||
|
||||
import pytest
|
||||
|
||||
with pytest.raises(RuntimeError, match="did not advance"):
|
||||
zep_paging.fetch_all_edges(client, "graph", page_size=1)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,418 @@
|
|||
from datetime import datetime
|
||||
import threading
|
||||
|
||||
from flask import Flask
|
||||
from types import SimpleNamespace
|
||||
|
||||
from app.api import graph as graph_api
|
||||
from app.api import simulation as simulation_api
|
||||
from app.models.project import Project, ProjectStatus
|
||||
from app.services.simulation_manager import SimulationStatus
|
||||
from app.models.task import TaskStatus
|
||||
|
||||
|
||||
def _project(status, graph_id="graph-1"):
|
||||
now = datetime.now().isoformat()
|
||||
return Project(
|
||||
project_id="proj-1",
|
||||
name="Project",
|
||||
status=status,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
ontology={"entity_types": [], "edge_types": []},
|
||||
graph_id=graph_id,
|
||||
graph_build_task_id="task-1",
|
||||
zep_batch_id="batch-1",
|
||||
zep_batch_operation_id="operation-1",
|
||||
)
|
||||
|
||||
|
||||
def _json_result(result):
|
||||
if isinstance(result, tuple):
|
||||
response, status = result
|
||||
else:
|
||||
response, status = result, result.status_code
|
||||
return response.get_json(), status
|
||||
|
||||
|
||||
def test_project_reset_deletes_the_cloud_graph_before_clearing_reference(monkeypatch):
|
||||
project = _project(ProjectStatus.GRAPH_COMPLETED)
|
||||
events = []
|
||||
|
||||
class Builder:
|
||||
def __init__(self, **_kwargs):
|
||||
pass
|
||||
|
||||
def delete_graph(self, graph_id):
|
||||
events.append(("cloud-delete", graph_id))
|
||||
|
||||
monkeypatch.setattr(graph_api, "GraphBuilderService", Builder)
|
||||
monkeypatch.setattr(graph_api.Config, "ZEP_API_KEY", "test-key")
|
||||
monkeypatch.setattr(
|
||||
graph_api.ProjectManager,
|
||||
"get_project",
|
||||
classmethod(lambda _cls, _project_id: project),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
graph_api.ProjectManager,
|
||||
"save_project",
|
||||
classmethod(lambda _cls, saved: events.append(("save", saved.graph_id))),
|
||||
)
|
||||
|
||||
app = Flask(__name__)
|
||||
with app.test_request_context("/api/graph/project/proj-1/reset", method="POST"):
|
||||
body, status = _json_result(graph_api.reset_project("proj-1"))
|
||||
|
||||
assert status == 200
|
||||
assert body["success"] is True
|
||||
assert events == [("cloud-delete", "graph-1"), ("save", None)]
|
||||
assert project.zep_batch_id is None
|
||||
assert project.status == ProjectStatus.ONTOLOGY_GENERATED
|
||||
|
||||
|
||||
def test_project_reset_refuses_a_graph_with_an_active_simulation(monkeypatch):
|
||||
project = _project(ProjectStatus.GRAPH_COMPLETED)
|
||||
monkeypatch.setattr(graph_api.Config, "ZEP_API_KEY", "test-key")
|
||||
monkeypatch.setattr(
|
||||
graph_api.ProjectManager,
|
||||
"get_project",
|
||||
classmethod(lambda _cls, _project_id: project),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
graph_api.ZepGraphMemoryManager,
|
||||
"get_simulation_ids_for_graph",
|
||||
classmethod(lambda _cls, _graph_id: ["sim-active"]),
|
||||
)
|
||||
|
||||
app = Flask(__name__)
|
||||
with app.test_request_context("/api/graph/project/proj-1/reset", method="POST"):
|
||||
body, status = _json_result(graph_api.reset_project("proj-1"))
|
||||
|
||||
assert status == 409
|
||||
assert "sim-active" in body["error"]
|
||||
|
||||
|
||||
def test_graph_delete_cannot_discard_an_updater_during_finalization(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
graph_api.ZepGraphMemoryManager,
|
||||
"get_simulation_ids_for_graph",
|
||||
classmethod(lambda _cls, _graph_id: ["sim-finalizing"]),
|
||||
)
|
||||
discarded = []
|
||||
monkeypatch.setattr(
|
||||
graph_api.ZepGraphMemoryManager,
|
||||
"discard_inactive_updater",
|
||||
classmethod(
|
||||
lambda _cls, simulation_id: discarded.append(simulation_id)
|
||||
),
|
||||
)
|
||||
lock = graph_api.SimulationRunner._finalization_lock("sim-finalizing")
|
||||
lock.acquire()
|
||||
try:
|
||||
assert graph_api._active_graph_consumers("graph-1") == ["sim-finalizing"]
|
||||
finally:
|
||||
lock.release()
|
||||
|
||||
assert discarded == []
|
||||
|
||||
|
||||
def test_repeated_build_request_reuses_the_existing_task(monkeypatch):
|
||||
project = _project(ProjectStatus.GRAPH_BUILDING)
|
||||
monkeypatch.setattr(graph_api.Config, "ZEP_API_KEY", "test-key")
|
||||
monkeypatch.setattr(
|
||||
graph_api.ProjectManager,
|
||||
"get_project",
|
||||
classmethod(lambda _cls, _project_id: project),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
graph_api,
|
||||
"TaskManager",
|
||||
lambda: SimpleNamespace(
|
||||
get_task=lambda _task_id: SimpleNamespace(status=TaskStatus.PROCESSING)
|
||||
),
|
||||
)
|
||||
|
||||
app = Flask(__name__)
|
||||
with app.test_request_context(
|
||||
"/api/graph/build",
|
||||
method="POST",
|
||||
json={"project_id": "proj-1", "force": True},
|
||||
):
|
||||
body, status = _json_result(graph_api.build_graph())
|
||||
|
||||
assert status == 200
|
||||
assert body["success"] is True
|
||||
assert body["data"]["reused"] is True
|
||||
assert body["data"]["task_id"] == "task-1"
|
||||
assert body["data"]["graph_id"] == "graph-1"
|
||||
|
||||
|
||||
def test_stale_build_after_restart_is_recoverable_instead_of_reused(monkeypatch):
|
||||
project = _project(ProjectStatus.GRAPH_BUILDING)
|
||||
project.zep_batch_id = None
|
||||
project.zep_batch_operation_id = None
|
||||
saved = []
|
||||
monkeypatch.setattr(graph_api.Config, "ZEP_API_KEY", "test-key")
|
||||
monkeypatch.setattr(
|
||||
graph_api.ProjectManager,
|
||||
"get_project",
|
||||
classmethod(lambda _cls, _project_id: project),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
graph_api.ProjectManager,
|
||||
"save_project",
|
||||
classmethod(lambda _cls, value: saved.append(value.status)),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
graph_api,
|
||||
"TaskManager",
|
||||
lambda: SimpleNamespace(get_task=lambda _task_id: None),
|
||||
)
|
||||
|
||||
app = Flask(__name__)
|
||||
with app.test_request_context(
|
||||
"/api/graph/build",
|
||||
method="POST",
|
||||
json={"project_id": "proj-1"},
|
||||
):
|
||||
body, status = _json_result(graph_api.build_graph())
|
||||
|
||||
assert status == 409
|
||||
assert body["recoverable"] is True
|
||||
assert project.status == ProjectStatus.FAILED
|
||||
assert saved == [ProjectStatus.FAILED]
|
||||
|
||||
|
||||
def test_stale_build_resumes_a_persisted_processing_batch(monkeypatch):
|
||||
project = _project(ProjectStatus.GRAPH_BUILDING)
|
||||
created_threads = []
|
||||
|
||||
class Tasks:
|
||||
def get_task(self, _task_id):
|
||||
return None
|
||||
|
||||
def create_task(self, _description):
|
||||
return "task-resumed"
|
||||
|
||||
class Builder:
|
||||
def __init__(self, **_kwargs):
|
||||
pass
|
||||
|
||||
def get_batch_summary(self, batch_id):
|
||||
assert batch_id == "batch-1"
|
||||
return SimpleNamespace(status="processing")
|
||||
|
||||
class Thread:
|
||||
def __init__(self, *, target, daemon):
|
||||
created_threads.append((target, daemon))
|
||||
|
||||
def start(self):
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(graph_api.Config, "ZEP_API_KEY", "test-key")
|
||||
monkeypatch.setattr(graph_api, "TaskManager", Tasks)
|
||||
monkeypatch.setattr(graph_api, "GraphBuilderService", Builder)
|
||||
monkeypatch.setattr(graph_api.threading, "Thread", Thread)
|
||||
monkeypatch.setattr(
|
||||
graph_api.ProjectManager,
|
||||
"get_project",
|
||||
classmethod(lambda _cls, _project_id: project),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
graph_api.ProjectManager,
|
||||
"get_extracted_text",
|
||||
classmethod(lambda _cls, _project_id: "source text"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
graph_api.ProjectManager,
|
||||
"save_project",
|
||||
classmethod(lambda _cls, _project: None),
|
||||
)
|
||||
|
||||
app = Flask(__name__)
|
||||
with app.test_request_context(
|
||||
"/api/graph/build",
|
||||
method="POST",
|
||||
json={"project_id": "proj-1"},
|
||||
):
|
||||
body, status = _json_result(graph_api.build_graph())
|
||||
|
||||
assert status == 200
|
||||
assert body["data"]["resumed"] is True
|
||||
assert body["data"]["task_id"] == "task-resumed"
|
||||
assert project.graph_build_task_id == "task-resumed"
|
||||
assert len(created_threads) == 1
|
||||
|
||||
|
||||
def test_project_delete_removes_cloud_graph_before_local_files(monkeypatch):
|
||||
project = _project(ProjectStatus.GRAPH_COMPLETED)
|
||||
events = []
|
||||
|
||||
class Builder:
|
||||
def __init__(self, **_kwargs):
|
||||
pass
|
||||
|
||||
def delete_graph(self, graph_id):
|
||||
events.append(("cloud-delete", graph_id))
|
||||
|
||||
monkeypatch.setattr(graph_api, "GraphBuilderService", Builder)
|
||||
monkeypatch.setattr(graph_api.Config, "ZEP_API_KEY", "test-key")
|
||||
monkeypatch.setattr(
|
||||
graph_api.ProjectManager,
|
||||
"get_project",
|
||||
classmethod(lambda _cls, _project_id: project),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
graph_api.ProjectManager,
|
||||
"delete_project",
|
||||
classmethod(
|
||||
lambda _cls, project_id: events.append(("local-delete", project_id)) or True
|
||||
),
|
||||
)
|
||||
|
||||
app = Flask(__name__)
|
||||
with app.test_request_context(
|
||||
"/api/graph/project/proj-1",
|
||||
method="DELETE",
|
||||
):
|
||||
body, status = _json_result(graph_api.delete_project("proj-1"))
|
||||
|
||||
assert status == 200
|
||||
assert body["success"] is True
|
||||
assert events == [
|
||||
("cloud-delete", "graph-1"),
|
||||
("local-delete", "proj-1"),
|
||||
]
|
||||
|
||||
|
||||
def test_completed_build_request_is_idempotent_without_force(monkeypatch):
|
||||
project = _project(ProjectStatus.GRAPH_COMPLETED)
|
||||
monkeypatch.setattr(graph_api.Config, "ZEP_API_KEY", "test-key")
|
||||
monkeypatch.setattr(
|
||||
graph_api.ProjectManager,
|
||||
"get_project",
|
||||
classmethod(lambda _cls, _project_id: project),
|
||||
)
|
||||
|
||||
app = Flask(__name__)
|
||||
with app.test_request_context(
|
||||
"/api/graph/build",
|
||||
method="POST",
|
||||
json={"project_id": "proj-1"},
|
||||
):
|
||||
body, status = _json_result(graph_api.build_graph())
|
||||
|
||||
assert status == 200
|
||||
assert body["data"]["reused"] is True
|
||||
assert body["data"]["graph_id"] == "graph-1"
|
||||
|
||||
|
||||
def test_force_must_be_a_json_boolean(monkeypatch):
|
||||
project = _project(ProjectStatus.GRAPH_COMPLETED)
|
||||
monkeypatch.setattr(graph_api.Config, "ZEP_API_KEY", "test-key")
|
||||
monkeypatch.setattr(
|
||||
graph_api.ProjectManager,
|
||||
"get_project",
|
||||
classmethod(lambda _cls, _project_id: project),
|
||||
)
|
||||
|
||||
app = Flask(__name__)
|
||||
with app.test_request_context(
|
||||
"/api/graph/build",
|
||||
method="POST",
|
||||
json={"project_id": "proj-1", "force": "false"},
|
||||
):
|
||||
body, status = _json_result(graph_api.build_graph())
|
||||
|
||||
assert status == 400
|
||||
assert "boolean" in body["error"]
|
||||
|
||||
|
||||
def test_graph_reset_and_memory_start_cannot_cross_between_delete_and_clear(
|
||||
monkeypatch,
|
||||
):
|
||||
project = _project(ProjectStatus.GRAPH_COMPLETED)
|
||||
simulation = SimpleNamespace(
|
||||
simulation_id="sim-1",
|
||||
project_id=project.project_id,
|
||||
graph_id=project.graph_id,
|
||||
status=SimulationStatus.READY,
|
||||
)
|
||||
delete_entered = threading.Event()
|
||||
allow_delete = threading.Event()
|
||||
runner_called = []
|
||||
|
||||
class Builder:
|
||||
def __init__(self, **_kwargs):
|
||||
pass
|
||||
|
||||
def delete_graph(self, graph_id):
|
||||
assert graph_id == "graph-1"
|
||||
delete_entered.set()
|
||||
assert allow_delete.wait(timeout=2)
|
||||
|
||||
class Simulations:
|
||||
def get_simulation(self, _simulation_id):
|
||||
return simulation
|
||||
|
||||
monkeypatch.setattr(graph_api, "GraphBuilderService", Builder)
|
||||
monkeypatch.setattr(graph_api.Config, "ZEP_API_KEY", "test-key")
|
||||
monkeypatch.setattr(
|
||||
graph_api.ProjectManager,
|
||||
"get_project",
|
||||
classmethod(lambda _cls, _project_id: project),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
graph_api.ProjectManager,
|
||||
"save_project",
|
||||
classmethod(lambda _cls, _project: None),
|
||||
)
|
||||
monkeypatch.setattr(simulation_api, "SimulationManager", Simulations)
|
||||
monkeypatch.setattr(
|
||||
simulation_api.ProjectManager,
|
||||
"get_project",
|
||||
classmethod(lambda _cls, _project_id: project),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
simulation_api.SimulationRunner,
|
||||
"start_simulation",
|
||||
classmethod(lambda _cls, **_kwargs: runner_called.append(True)),
|
||||
)
|
||||
|
||||
app = Flask(__name__)
|
||||
results = {}
|
||||
|
||||
def reset():
|
||||
with app.test_request_context(
|
||||
"/api/graph/project/proj-1/reset", method="POST"
|
||||
):
|
||||
results["reset"] = _json_result(graph_api.reset_project("proj-1"))
|
||||
|
||||
def start():
|
||||
with app.test_request_context(
|
||||
"/api/simulation/start",
|
||||
method="POST",
|
||||
json={
|
||||
"simulation_id": "sim-1",
|
||||
"enable_graph_memory_update": True,
|
||||
},
|
||||
):
|
||||
results["start"] = _json_result(simulation_api.start_simulation())
|
||||
|
||||
reset_thread = threading.Thread(target=reset)
|
||||
reset_thread.start()
|
||||
assert delete_entered.wait(timeout=2)
|
||||
|
||||
start_thread = threading.Thread(target=start)
|
||||
start_thread.start()
|
||||
start_thread.join(timeout=0.05)
|
||||
assert start_thread.is_alive()
|
||||
|
||||
allow_delete.set()
|
||||
reset_thread.join(timeout=2)
|
||||
start_thread.join(timeout=2)
|
||||
|
||||
assert results["reset"][1] == 200
|
||||
assert results["start"][1] == 409
|
||||
assert runner_called == []
|
||||
assert project.graph_id is None
|
||||
|
|
@ -0,0 +1,237 @@
|
|||
from types import SimpleNamespace
|
||||
import threading
|
||||
from queue import Queue
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services import zep_graph_memory_updater as updater_module
|
||||
from app.services.zep_graph_memory_updater import (
|
||||
AgentActivity,
|
||||
ZepGraphMemoryManager,
|
||||
ZepGraphMemoryUpdater,
|
||||
)
|
||||
|
||||
|
||||
def _activity(index=1, content="hello"):
|
||||
return AgentActivity(
|
||||
platform="twitter",
|
||||
agent_id=index,
|
||||
agent_name=f"Agent {index}",
|
||||
action_type="CREATE_POST",
|
||||
action_args={"content": content},
|
||||
round_num=index,
|
||||
timestamp="2026-07-22T12:00:00+08:00",
|
||||
)
|
||||
|
||||
|
||||
def _client(add):
|
||||
return SimpleNamespace(
|
||||
graph=SimpleNamespace(
|
||||
add=add,
|
||||
episode=SimpleNamespace(
|
||||
get=lambda **_kwargs: SimpleNamespace(processed=True)
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _updater(monkeypatch, add, simulation_id="sim-1"):
|
||||
client = _client(add)
|
||||
monkeypatch.setattr(updater_module, "get_zep_client", lambda _key: client)
|
||||
updater = ZepGraphMemoryUpdater(
|
||||
"graph-1",
|
||||
api_key="test-key",
|
||||
simulation_id=simulation_id,
|
||||
)
|
||||
updater.SEND_INTERVAL = 0
|
||||
return updater
|
||||
|
||||
|
||||
def test_stop_drains_an_immediately_queued_tail_activity(monkeypatch):
|
||||
writes = []
|
||||
updater = _updater(
|
||||
monkeypatch,
|
||||
lambda **kwargs: writes.append(kwargs) or SimpleNamespace(uuid_="episode-1"),
|
||||
)
|
||||
|
||||
updater.start()
|
||||
updater.add_activity(_activity())
|
||||
updater.stop()
|
||||
|
||||
assert len(writes) == 1
|
||||
assert updater.get_stats()["items_sent"] == 1
|
||||
assert updater.get_stats()["queue_size"] == 0
|
||||
|
||||
|
||||
def test_network_write_happens_outside_the_buffer_lock(monkeypatch):
|
||||
lock_was_available = []
|
||||
updater = None
|
||||
|
||||
def add(**_kwargs):
|
||||
acquired = updater._buffer_lock.acquire(blocking=False)
|
||||
lock_was_available.append(acquired)
|
||||
if acquired:
|
||||
updater._buffer_lock.release()
|
||||
return SimpleNamespace(uuid_="episode-1")
|
||||
|
||||
updater = _updater(monkeypatch, add)
|
||||
updater.start()
|
||||
for index in range(updater.BATCH_SIZE):
|
||||
updater.add_activity(_activity(index))
|
||||
updater.stop()
|
||||
|
||||
assert lock_was_available == [True]
|
||||
|
||||
|
||||
def test_activity_episode_has_provenance_time_and_a_safe_size(monkeypatch):
|
||||
writes = []
|
||||
updater = _updater(
|
||||
monkeypatch,
|
||||
lambda **kwargs: writes.append(kwargs) or SimpleNamespace(uuid_="episode-1"),
|
||||
simulation_id="sim-provenance",
|
||||
)
|
||||
|
||||
updater._send_batch_activities(
|
||||
[_activity(content="x" * 20_000)],
|
||||
"twitter",
|
||||
)
|
||||
|
||||
assert len(writes) == 1
|
||||
write = writes[0]
|
||||
assert len(write["data"]) <= updater.MAX_EPISODE_CHARS
|
||||
assert write["created_at"] == "2026-07-22T12:00:00+08:00"
|
||||
assert write["source_description"] == "MiroFish simulation activity batch"
|
||||
assert write["metadata"]["simulation_id"] == "sim-provenance"
|
||||
assert write["metadata"]["platform"] == "twitter"
|
||||
assert write["metadata"]["activity_count"] == 1
|
||||
|
||||
|
||||
def test_failed_non_idempotent_write_is_reported_by_stop(monkeypatch):
|
||||
def add(**_kwargs):
|
||||
raise RuntimeError("write failed")
|
||||
|
||||
updater = _updater(monkeypatch, add)
|
||||
updater.start()
|
||||
updater.add_activity(_activity())
|
||||
|
||||
with pytest.raises(RuntimeError, match="ingestion is incomplete"):
|
||||
updater.stop()
|
||||
|
||||
assert updater.get_stats()["failed_count"] == 1
|
||||
|
||||
|
||||
def test_failed_simulation_action_is_not_ingested(monkeypatch):
|
||||
updater = _updater(
|
||||
monkeypatch,
|
||||
lambda **_kwargs: SimpleNamespace(uuid_="unused"),
|
||||
)
|
||||
|
||||
updater.add_activity_from_dict(
|
||||
{
|
||||
"agent_id": 1,
|
||||
"agent_name": "Agent",
|
||||
"action_type": "CREATE_POST",
|
||||
"action_args": {"content": "not actually posted"},
|
||||
"success": False,
|
||||
},
|
||||
"twitter",
|
||||
)
|
||||
|
||||
assert updater.get_stats()["queue_size"] == 0
|
||||
assert updater.get_stats()["skipped_count"] == 1
|
||||
|
||||
|
||||
def test_stop_cannot_finish_between_acceptance_check_and_enqueue(monkeypatch):
|
||||
writes = []
|
||||
updater = _updater(
|
||||
monkeypatch,
|
||||
lambda **kwargs: writes.append(kwargs) or SimpleNamespace(uuid_="episode-1"),
|
||||
)
|
||||
|
||||
put_entered = threading.Event()
|
||||
allow_put = threading.Event()
|
||||
|
||||
class BlockingQueue(Queue):
|
||||
def put(self, item, block=True, timeout=None):
|
||||
put_entered.set()
|
||||
assert allow_put.wait(timeout=2)
|
||||
return super().put(item, block=block, timeout=timeout)
|
||||
|
||||
updater._activity_queue = BlockingQueue()
|
||||
updater.start()
|
||||
producer = threading.Thread(target=updater.add_activity, args=(_activity(),))
|
||||
producer.start()
|
||||
assert put_entered.wait(timeout=1)
|
||||
|
||||
stopper = threading.Thread(target=updater.stop)
|
||||
stopper.start()
|
||||
stopper.join(timeout=0.1)
|
||||
assert stopper.is_alive()
|
||||
|
||||
allow_put.set()
|
||||
producer.join(timeout=2)
|
||||
stopper.join(timeout=2)
|
||||
|
||||
assert not producer.is_alive()
|
||||
assert not stopper.is_alive()
|
||||
assert len(writes) == 1
|
||||
|
||||
|
||||
def test_pending_episode_wait_has_a_deadline(monkeypatch):
|
||||
updater = _updater(
|
||||
monkeypatch,
|
||||
lambda **_kwargs: SimpleNamespace(uuid_="episode-1"),
|
||||
)
|
||||
updater._pending_episode_uuids = ["episode-1"]
|
||||
updater.client.graph.episode.get = lambda **_kwargs: SimpleNamespace(
|
||||
processed=False
|
||||
)
|
||||
timestamps = iter([0.0, 2.0])
|
||||
monkeypatch.setattr(updater_module.Config, "ZEP_INGESTION_TIMEOUT_SECONDS", 1)
|
||||
monkeypatch.setattr(updater_module.time, "time", lambda: next(timestamps))
|
||||
monkeypatch.setattr(updater_module.time, "sleep", lambda _seconds: None)
|
||||
|
||||
with pytest.raises(TimeoutError, match="pending"):
|
||||
updater._wait_for_pending_episodes()
|
||||
|
||||
|
||||
def test_explicit_graph_destruction_can_discard_a_stopped_failed_updater():
|
||||
updater = SimpleNamespace(
|
||||
graph_id="graph-1",
|
||||
_running=False,
|
||||
_worker_thread=SimpleNamespace(is_alive=lambda: False),
|
||||
)
|
||||
ZepGraphMemoryManager._updaters["sim-failed"] = updater
|
||||
try:
|
||||
assert ZepGraphMemoryManager.discard_inactive_updater("sim-failed") is True
|
||||
assert "sim-failed" not in ZepGraphMemoryManager._updaters
|
||||
finally:
|
||||
ZepGraphMemoryManager._updaters.pop("sim-failed", None)
|
||||
|
||||
|
||||
def test_flush_deadline_keeps_unattempted_platform_for_a_safe_retry(monkeypatch):
|
||||
now = [0.0]
|
||||
writes = []
|
||||
|
||||
def add(**kwargs):
|
||||
writes.append(kwargs)
|
||||
now[0] = 2.0
|
||||
return SimpleNamespace(uuid_=f"episode-{len(writes)}")
|
||||
|
||||
updater = _updater(monkeypatch, add)
|
||||
updater._platform_buffers["twitter"] = [_activity(1)]
|
||||
reddit_activity = _activity(2)
|
||||
reddit_activity.platform = "reddit"
|
||||
updater._platform_buffers["reddit"] = [reddit_activity]
|
||||
monkeypatch.setattr(updater_module.time, "time", lambda: now[0])
|
||||
|
||||
with pytest.raises(TimeoutError, match="deadline"):
|
||||
updater._flush_remaining(deadline=1.0)
|
||||
|
||||
assert updater._platform_buffers["twitter"] == []
|
||||
assert updater._platform_buffers["reddit"] == [reddit_activity]
|
||||
|
||||
now[0] = 0.0
|
||||
updater._flush_remaining(deadline=1.0)
|
||||
assert updater._platform_buffers["reddit"] == []
|
||||
assert len(writes) == 2
|
||||
|
|
@ -0,0 +1,297 @@
|
|||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from flask import Flask
|
||||
|
||||
from app.api import graph as graph_api
|
||||
from app.api import report as report_api
|
||||
from app.api import simulation as simulation_api
|
||||
from app.models.project import ProjectStatus
|
||||
from app.services.simulation_manager import SimulationStatus
|
||||
from app.services.simulation_runner import RunnerStatus
|
||||
from app.utils.zep_lifecycle import (
|
||||
get_graph_readers,
|
||||
unregister_graph_reader,
|
||||
)
|
||||
|
||||
|
||||
def _json_result(result):
|
||||
if isinstance(result, tuple):
|
||||
response, status = result
|
||||
else:
|
||||
response, status = result, result.status_code
|
||||
return response.get_json(), status
|
||||
|
||||
|
||||
def test_report_generation_waits_for_zep_ingestion(monkeypatch):
|
||||
simulation = SimpleNamespace(project_id="proj-1", graph_id="graph-1")
|
||||
monkeypatch.setattr(
|
||||
report_api,
|
||||
"SimulationManager",
|
||||
lambda: SimpleNamespace(
|
||||
get_simulation=lambda _simulation_id: simulation
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
report_api.ReportManager,
|
||||
"get_report_by_simulation",
|
||||
classmethod(lambda _cls, _simulation_id: None),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
report_api.SimulationRunner,
|
||||
"get_run_state",
|
||||
classmethod(
|
||||
lambda _cls, _simulation_id: SimpleNamespace(
|
||||
runner_status=RunnerStatus.STOPPING
|
||||
)
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
report_api.ZepGraphMemoryManager,
|
||||
"get_updater",
|
||||
classmethod(lambda _cls, _simulation_id: object()),
|
||||
)
|
||||
|
||||
app = Flask(__name__)
|
||||
with app.test_request_context(
|
||||
"/api/report/generate",
|
||||
method="POST",
|
||||
json={"simulation_id": "sim-1"},
|
||||
):
|
||||
body, status = _json_result(report_api.generate_report())
|
||||
|
||||
assert status == 409
|
||||
assert body["ingestion_pending"] is True
|
||||
|
||||
|
||||
def test_active_rerun_does_not_return_a_stale_completed_report(monkeypatch):
|
||||
simulation = SimpleNamespace(project_id="proj-1", graph_id="graph-1")
|
||||
monkeypatch.setattr(
|
||||
report_api,
|
||||
"SimulationManager",
|
||||
lambda: SimpleNamespace(
|
||||
get_simulation=lambda _simulation_id: simulation
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
report_api.ReportManager,
|
||||
"get_report_by_simulation",
|
||||
classmethod(
|
||||
lambda _cls, _simulation_id: SimpleNamespace(
|
||||
report_id="old-report",
|
||||
status=report_api.ReportStatus.COMPLETED,
|
||||
)
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
report_api.SimulationRunner,
|
||||
"get_run_state",
|
||||
classmethod(
|
||||
lambda _cls, _simulation_id: SimpleNamespace(
|
||||
runner_status=RunnerStatus.STOPPING
|
||||
)
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
report_api.ZepGraphMemoryManager,
|
||||
"get_updater",
|
||||
classmethod(lambda _cls, _simulation_id: object()),
|
||||
)
|
||||
|
||||
app = Flask(__name__)
|
||||
with app.test_request_context(
|
||||
"/api/report/generate",
|
||||
method="POST",
|
||||
json={"simulation_id": "sim-1"},
|
||||
):
|
||||
body, status = _json_result(report_api.generate_report())
|
||||
|
||||
assert status == 409
|
||||
assert body["ingestion_pending"] is True
|
||||
|
||||
|
||||
def test_failed_ingestion_cannot_generate_a_report_after_restart(monkeypatch):
|
||||
simulation = SimpleNamespace(project_id="proj-1", graph_id="graph-1")
|
||||
monkeypatch.setattr(
|
||||
report_api,
|
||||
"SimulationManager",
|
||||
lambda: SimpleNamespace(
|
||||
get_simulation=lambda _simulation_id: simulation
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
report_api.SimulationRunner,
|
||||
"get_run_state",
|
||||
classmethod(
|
||||
lambda _cls, _simulation_id: SimpleNamespace(
|
||||
runner_status=RunnerStatus.FAILED
|
||||
)
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
report_api.ZepGraphMemoryManager,
|
||||
"get_updater",
|
||||
classmethod(lambda _cls, _simulation_id: None),
|
||||
)
|
||||
|
||||
app = Flask(__name__)
|
||||
with app.test_request_context(
|
||||
"/api/report/generate",
|
||||
method="POST",
|
||||
json={"simulation_id": "sim-1"},
|
||||
):
|
||||
body, status = _json_result(report_api.generate_report())
|
||||
|
||||
assert status == 409
|
||||
assert "successfully completed" in body["error"]
|
||||
|
||||
|
||||
def test_report_reader_lease_blocks_graph_start_and_delete(monkeypatch):
|
||||
simulation = SimpleNamespace(
|
||||
simulation_id="sim-1",
|
||||
project_id="proj-1",
|
||||
graph_id="graph-1",
|
||||
status=SimulationStatus.READY,
|
||||
)
|
||||
project = SimpleNamespace(
|
||||
project_id="proj-1",
|
||||
graph_id="graph-1",
|
||||
status=ProjectStatus.GRAPH_COMPLETED,
|
||||
simulation_requirement="mock requirement",
|
||||
)
|
||||
run_state = SimpleNamespace(runner_status=RunnerStatus.COMPLETED)
|
||||
worker_targets = []
|
||||
runner_calls = []
|
||||
|
||||
class Tasks:
|
||||
def create_task(self, **_kwargs):
|
||||
return "task-1"
|
||||
|
||||
def update_task(self, *_args, **_kwargs):
|
||||
pass
|
||||
|
||||
def complete_task(self, *_args, **_kwargs):
|
||||
pass
|
||||
|
||||
def fail_task(self, *_args, **_kwargs):
|
||||
pass
|
||||
|
||||
class ParkedThread:
|
||||
def __init__(self, *, target, daemon):
|
||||
assert daemon is True
|
||||
self.target = target
|
||||
|
||||
def start(self):
|
||||
worker_targets.append(self.target)
|
||||
|
||||
class Agent:
|
||||
def __init__(self, **_kwargs):
|
||||
pass
|
||||
|
||||
def generate_report(self, *, progress_callback, report_id):
|
||||
progress_callback("mock", 100, "done")
|
||||
return SimpleNamespace(
|
||||
report_id=report_id,
|
||||
status=report_api.ReportStatus.COMPLETED,
|
||||
error=None,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
report_api,
|
||||
"SimulationManager",
|
||||
lambda: SimpleNamespace(
|
||||
get_simulation=lambda _simulation_id: simulation
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
simulation_api,
|
||||
"SimulationManager",
|
||||
lambda: SimpleNamespace(
|
||||
get_simulation=lambda _simulation_id: simulation
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
report_api.ProjectManager,
|
||||
"get_project",
|
||||
classmethod(lambda _cls, _project_id: project),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
report_api.SimulationRunner,
|
||||
"get_run_state",
|
||||
classmethod(lambda _cls, _simulation_id: run_state),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
report_api.ZepGraphMemoryManager,
|
||||
"get_updater",
|
||||
classmethod(lambda _cls, _simulation_id: None),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
report_api.ReportManager,
|
||||
"get_report_by_simulation",
|
||||
classmethod(lambda _cls, _simulation_id: None),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
report_api.ReportManager,
|
||||
"save_report",
|
||||
classmethod(lambda _cls, _report: None),
|
||||
)
|
||||
monkeypatch.setattr(report_api, "TaskManager", Tasks)
|
||||
monkeypatch.setattr(report_api, "ReportAgent", Agent)
|
||||
monkeypatch.setattr(report_api.threading, "Thread", ParkedThread)
|
||||
monkeypatch.setattr(
|
||||
simulation_api.SimulationRunner,
|
||||
"start_simulation",
|
||||
classmethod(
|
||||
lambda _cls, **_kwargs: runner_calls.append(True)
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
graph_api.ZepGraphMemoryManager,
|
||||
"get_simulation_ids_for_graph",
|
||||
classmethod(lambda _cls, _graph_id: []),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
graph_api,
|
||||
"SimulationManager",
|
||||
lambda: SimpleNamespace(list_simulations=lambda: []),
|
||||
)
|
||||
|
||||
app = Flask(__name__)
|
||||
report_id = None
|
||||
try:
|
||||
with app.test_request_context(
|
||||
"/api/report/generate",
|
||||
method="POST",
|
||||
json={"simulation_id": "sim-1"},
|
||||
):
|
||||
body, status = _json_result(report_api.generate_report())
|
||||
assert status == 200
|
||||
report_id = body["data"]["report_id"]
|
||||
assert get_graph_readers("graph-1") == [report_id]
|
||||
assert len(worker_targets) == 1
|
||||
|
||||
with app.test_request_context(
|
||||
"/api/simulation/start",
|
||||
method="POST",
|
||||
json={
|
||||
"simulation_id": "sim-1",
|
||||
"enable_graph_memory_update": True,
|
||||
},
|
||||
):
|
||||
start_body, start_status = _json_result(
|
||||
simulation_api.start_simulation()
|
||||
)
|
||||
assert start_status == 409
|
||||
assert start_body["active_reports"] == [report_id]
|
||||
assert runner_calls == []
|
||||
|
||||
with pytest.raises(graph_api.GraphInUseError, match=f"report:{report_id}"):
|
||||
graph_api._delete_cloud_graph_if_present("graph-1")
|
||||
|
||||
# Let the parked background report finish; its finally block must
|
||||
# release the lease even if report generation fails.
|
||||
worker_targets[0]()
|
||||
assert get_graph_readers("graph-1") == []
|
||||
finally:
|
||||
if report_id:
|
||||
unregister_graph_reader("graph-1", report_id)
|
||||
|
|
@ -0,0 +1,102 @@
|
|||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from zep_cloud.core.api_error import ApiError as ZepApiError
|
||||
|
||||
from app.utils import zep
|
||||
from app import config
|
||||
|
||||
|
||||
def test_permanent_zep_errors_fail_without_retry():
|
||||
calls = []
|
||||
|
||||
def operation():
|
||||
calls.append(True)
|
||||
raise ZepApiError(status_code=400, body={"message": "bad query"})
|
||||
|
||||
with pytest.raises(ZepApiError):
|
||||
zep.call_zep_read_with_retry(
|
||||
operation,
|
||||
operation_name="permanent failure",
|
||||
sleep=lambda _seconds: None,
|
||||
)
|
||||
|
||||
assert len(calls) == 1
|
||||
|
||||
|
||||
def test_rate_limit_retry_respects_retry_after():
|
||||
calls = []
|
||||
sleeps = []
|
||||
|
||||
def operation():
|
||||
calls.append(True)
|
||||
if len(calls) == 1:
|
||||
raise ZepApiError(
|
||||
status_code=429,
|
||||
headers={"Retry-After": "7"},
|
||||
body={"message": "slow down"},
|
||||
)
|
||||
return "ok"
|
||||
|
||||
result = zep.call_zep_read_with_retry(
|
||||
operation,
|
||||
operation_name="rate limited read",
|
||||
sleep=sleeps.append,
|
||||
)
|
||||
|
||||
assert result == "ok"
|
||||
assert len(calls) == 2
|
||||
assert sleeps == [7.0]
|
||||
|
||||
|
||||
def test_zep_client_is_shared_and_uses_an_explicit_timeout(monkeypatch):
|
||||
created = []
|
||||
|
||||
def fake_zep(**kwargs):
|
||||
created.append(kwargs)
|
||||
return SimpleNamespace(kwargs=kwargs)
|
||||
|
||||
monkeypatch.delenv("ZEP_API_URL", raising=False)
|
||||
monkeypatch.setattr(zep, "Zep", fake_zep)
|
||||
zep.clear_zep_client_cache()
|
||||
|
||||
first = zep.get_zep_client(" test-key ", timeout=12)
|
||||
second = zep.get_zep_client("test-key", timeout=12)
|
||||
|
||||
assert first is second
|
||||
assert created == [{
|
||||
"api_key": "test-key",
|
||||
"base_url": zep.ZEP_CLOUD_BASE_URL,
|
||||
"timeout": 12.0,
|
||||
}]
|
||||
zep.clear_zep_client_cache()
|
||||
|
||||
|
||||
def test_zep_client_rejects_self_hosted_endpoint_override(monkeypatch):
|
||||
monkeypatch.setenv("ZEP_API_URL", "https://example.invalid")
|
||||
|
||||
with pytest.raises(ValueError, match="ZEP_API_URL"):
|
||||
zep.get_zep_client("test-key")
|
||||
|
||||
|
||||
def test_malformed_timeout_config_is_reported_without_import_failure():
|
||||
value, error = config._parse_number(
|
||||
"ZEP_REQUEST_TIMEOUT_SECONDS",
|
||||
"not-a-number",
|
||||
default=30.0,
|
||||
cast=float,
|
||||
)
|
||||
|
||||
assert value == 30.0
|
||||
assert "must be a number" in error
|
||||
|
||||
|
||||
def test_zep_client_rejects_a_recorded_timeout_parse_error(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
zep.Config,
|
||||
"_ZEP_CONFIG_PARSE_ERRORS",
|
||||
("ZEP_REQUEST_TIMEOUT_SECONDS must be a number",),
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="must be a number"):
|
||||
zep.get_zep_client("test-key")
|
||||
|
|
@ -0,0 +1,484 @@
|
|||
from types import SimpleNamespace
|
||||
import json
|
||||
|
||||
import pytest
|
||||
from flask import Flask
|
||||
|
||||
from app.api import simulation as simulation_api
|
||||
from app.services import simulation_runner as runner_module
|
||||
from app.services.simulation_manager import SimulationStatus
|
||||
from app.services.simulation_runner import (
|
||||
RunnerStatus,
|
||||
SimulationRunState,
|
||||
SimulationRunner,
|
||||
SimulationStopPending,
|
||||
)
|
||||
|
||||
|
||||
def test_manual_stop_surfaces_graph_ingestion_failure(monkeypatch):
|
||||
state = SimulationRunState(
|
||||
simulation_id="sim-1",
|
||||
runner_status=RunnerStatus.RUNNING,
|
||||
)
|
||||
saved = []
|
||||
monkeypatch.setattr(
|
||||
SimulationRunner,
|
||||
"get_run_state",
|
||||
classmethod(lambda _cls, _simulation_id: state),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
SimulationRunner,
|
||||
"_save_run_state",
|
||||
classmethod(lambda _cls, value: saved.append(value.runner_status)),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
runner_module.ZepGraphMemoryManager,
|
||||
"stop_updater",
|
||||
classmethod(
|
||||
lambda _cls, _simulation_id: (_ for _ in ()).throw(
|
||||
RuntimeError("ingestion incomplete")
|
||||
)
|
||||
),
|
||||
)
|
||||
SimulationRunner._processes.pop("sim-1", None)
|
||||
SimulationRunner._graph_memory_enabled["sim-1"] = True
|
||||
|
||||
try:
|
||||
with pytest.raises(RuntimeError, match="ingestion incomplete"):
|
||||
SimulationRunner.stop_simulation("sim-1")
|
||||
|
||||
assert state.runner_status == RunnerStatus.FAILED
|
||||
assert "ingestion incomplete" in state.error
|
||||
assert saved[-1] == RunnerStatus.FAILED
|
||||
finally:
|
||||
SimulationRunner._graph_memory_enabled.pop("sim-1", None)
|
||||
SimulationRunner._manual_stop_requests.discard("sim-1")
|
||||
|
||||
|
||||
def test_platform_completion_does_not_publish_terminal_success_before_barrier(
|
||||
monkeypatch, tmp_path
|
||||
):
|
||||
simulation_id = "sim-1"
|
||||
sim_dir = tmp_path / simulation_id / "twitter"
|
||||
sim_dir.mkdir(parents=True)
|
||||
log_path = sim_dir / "actions.jsonl"
|
||||
log_path.write_text(
|
||||
'{"event_type":"simulation_end","total_rounds":1,"total_actions":0}\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setattr(SimulationRunner, "RUN_STATE_DIR", str(tmp_path))
|
||||
state = SimulationRunState(
|
||||
simulation_id=simulation_id,
|
||||
runner_status=RunnerStatus.RUNNING,
|
||||
twitter_running=True,
|
||||
)
|
||||
|
||||
SimulationRunner._read_action_log(str(log_path), 0, state, "twitter")
|
||||
|
||||
assert state.twitter_completed is True
|
||||
assert state.runner_status == RunnerStatus.RUNNING
|
||||
|
||||
|
||||
def test_manual_stop_timeout_leaves_monitor_owned_state_stopping(monkeypatch):
|
||||
state = SimulationRunState(
|
||||
simulation_id="sim-timeout",
|
||||
runner_status=RunnerStatus.RUNNING,
|
||||
)
|
||||
|
||||
class Monitor:
|
||||
def join(self, timeout):
|
||||
assert timeout >= 30
|
||||
|
||||
def is_alive(self):
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(
|
||||
SimulationRunner,
|
||||
"get_run_state",
|
||||
classmethod(lambda _cls, _simulation_id: state),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
SimulationRunner,
|
||||
"_save_run_state",
|
||||
classmethod(lambda _cls, _state: None),
|
||||
)
|
||||
SimulationRunner._monitor_threads["sim-timeout"] = Monitor()
|
||||
SimulationRunner._processes.pop("sim-timeout", None)
|
||||
SimulationRunner._graph_memory_enabled.pop("sim-timeout", None)
|
||||
|
||||
try:
|
||||
with pytest.raises(TimeoutError, match="仍在停止中"):
|
||||
SimulationRunner.stop_simulation("sim-timeout")
|
||||
assert state.runner_status == RunnerStatus.STOPPING
|
||||
finally:
|
||||
SimulationRunner._monitor_threads.pop("sim-timeout", None)
|
||||
SimulationRunner._manual_stop_requests.discard("sim-timeout")
|
||||
|
||||
|
||||
def test_failed_ingestion_finalization_can_be_retried(monkeypatch):
|
||||
state = SimulationRunState(
|
||||
simulation_id="sim-retry",
|
||||
runner_status=RunnerStatus.FAILED,
|
||||
error="first drain timed out",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
SimulationRunner,
|
||||
"get_run_state",
|
||||
classmethod(lambda _cls, _simulation_id: state),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
SimulationRunner,
|
||||
"_save_run_state",
|
||||
classmethod(lambda _cls, _state: None),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
runner_module.ZepGraphMemoryManager,
|
||||
"get_updater",
|
||||
classmethod(lambda _cls, _simulation_id: object()),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
runner_module.ZepGraphMemoryManager,
|
||||
"stop_updater",
|
||||
classmethod(lambda _cls, _simulation_id: None),
|
||||
)
|
||||
SimulationRunner._graph_memory_enabled["sim-retry"] = True
|
||||
SimulationRunner._monitor_threads.pop("sim-retry", None)
|
||||
|
||||
try:
|
||||
result = SimulationRunner.stop_simulation("sim-retry")
|
||||
assert result.runner_status == RunnerStatus.STOPPED
|
||||
assert result.error is None
|
||||
finally:
|
||||
SimulationRunner._graph_memory_enabled.pop("sim-retry", None)
|
||||
SimulationRunner._manual_stop_requests.discard("sim-retry")
|
||||
|
||||
|
||||
def test_stop_api_keeps_pending_finalization_out_of_failed_state(monkeypatch):
|
||||
simulation = SimpleNamespace(status=SimulationStatus.STOPPING, error=None)
|
||||
saved = []
|
||||
monkeypatch.setattr(
|
||||
simulation_api.SimulationRunner,
|
||||
"stop_simulation",
|
||||
classmethod(
|
||||
lambda _cls, _simulation_id: (_ for _ in ()).throw(
|
||||
SimulationStopPending("still draining")
|
||||
)
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
simulation_api,
|
||||
"SimulationManager",
|
||||
lambda: SimpleNamespace(
|
||||
get_simulation=lambda _simulation_id: simulation,
|
||||
_save_simulation_state=lambda state: saved.append(state.status),
|
||||
),
|
||||
)
|
||||
|
||||
app = Flask(__name__)
|
||||
with app.test_request_context(
|
||||
"/api/simulation/stop",
|
||||
method="POST",
|
||||
json={"simulation_id": "sim-pending"},
|
||||
):
|
||||
response, status = simulation_api.stop_simulation()
|
||||
|
||||
assert status == 202
|
||||
assert response.get_json()["pending"] is True
|
||||
assert simulation.status == SimulationStatus.STOPPING
|
||||
assert saved == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"field",
|
||||
["force", "enable_graph_memory_update"],
|
||||
)
|
||||
def test_simulation_start_rejects_string_booleans(field):
|
||||
app = Flask(__name__)
|
||||
with app.test_request_context(
|
||||
"/api/simulation/start",
|
||||
method="POST",
|
||||
json={"simulation_id": "sim-1", field: "false"},
|
||||
):
|
||||
response, status = simulation_api.start_simulation()
|
||||
|
||||
assert status == 400
|
||||
assert "JSON boolean" in response.get_json()["error"]
|
||||
|
||||
|
||||
def test_force_restart_does_not_continue_while_old_ingestion_is_pending(monkeypatch):
|
||||
simulation = SimpleNamespace(
|
||||
simulation_id="sim-1",
|
||||
project_id="proj-1",
|
||||
graph_id="graph-1",
|
||||
status=SimulationStatus.STOPPING,
|
||||
)
|
||||
cleanup_called = []
|
||||
monkeypatch.setattr(
|
||||
simulation_api,
|
||||
"SimulationManager",
|
||||
lambda: SimpleNamespace(
|
||||
get_simulation=lambda _simulation_id: simulation,
|
||||
_save_simulation_state=lambda _state: None,
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
simulation_api,
|
||||
"_check_simulation_prepared",
|
||||
lambda _simulation_id: (True, {}),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
simulation_api.SimulationRunner,
|
||||
"get_run_state",
|
||||
classmethod(
|
||||
lambda _cls, _simulation_id: SimpleNamespace(
|
||||
runner_status=RunnerStatus.STOPPING
|
||||
)
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
simulation_api.SimulationRunner,
|
||||
"stop_simulation",
|
||||
classmethod(
|
||||
lambda _cls, _simulation_id: (_ for _ in ()).throw(
|
||||
SimulationStopPending("still draining")
|
||||
)
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
simulation_api.SimulationRunner,
|
||||
"cleanup_simulation_logs",
|
||||
classmethod(
|
||||
lambda _cls, _simulation_id: cleanup_called.append(True)
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
simulation_api.ZepGraphMemoryManager,
|
||||
"get_updater",
|
||||
classmethod(lambda _cls, _simulation_id: object()),
|
||||
)
|
||||
|
||||
app = Flask(__name__)
|
||||
with app.test_request_context(
|
||||
"/api/simulation/start",
|
||||
method="POST",
|
||||
json={"simulation_id": "sim-1", "force": True},
|
||||
):
|
||||
response, status = simulation_api.start_simulation()
|
||||
|
||||
assert status == 409
|
||||
assert response.get_json()["pending"] is True
|
||||
assert cleanup_called == []
|
||||
|
||||
|
||||
def test_monitor_start_failure_terminates_the_spawned_process(monkeypatch, tmp_path):
|
||||
simulation_id = "sim-start-failure"
|
||||
sim_dir = tmp_path / "runs" / simulation_id
|
||||
scripts_dir = tmp_path / "scripts"
|
||||
sim_dir.mkdir(parents=True)
|
||||
scripts_dir.mkdir()
|
||||
(sim_dir / "simulation_config.json").write_text(
|
||||
json.dumps({
|
||||
"time_config": {
|
||||
"total_simulation_hours": 1,
|
||||
"minutes_per_round": 60,
|
||||
}
|
||||
}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(scripts_dir / "run_twitter_simulation.py").write_text("pass\n", encoding="utf-8")
|
||||
|
||||
class Process:
|
||||
pid = 123
|
||||
|
||||
def poll(self):
|
||||
return None
|
||||
|
||||
class BrokenThread:
|
||||
def __init__(self, **_kwargs):
|
||||
pass
|
||||
|
||||
def start(self):
|
||||
raise RuntimeError("monitor failed")
|
||||
|
||||
terminated = []
|
||||
monkeypatch.setattr(SimulationRunner, "RUN_STATE_DIR", str(tmp_path / "runs"))
|
||||
monkeypatch.setattr(SimulationRunner, "SCRIPTS_DIR", str(scripts_dir))
|
||||
monkeypatch.setattr(runner_module.subprocess, "Popen", lambda *_args, **_kwargs: Process())
|
||||
monkeypatch.setattr(runner_module.threading, "Thread", BrokenThread)
|
||||
monkeypatch.setattr(
|
||||
SimulationRunner,
|
||||
"_terminate_process",
|
||||
classmethod(lambda _cls, _process, sim_id: terminated.append(sim_id)),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
SimulationRunner,
|
||||
"_sync_simulation_status",
|
||||
classmethod(lambda _cls, *_args, **_kwargs: None),
|
||||
)
|
||||
|
||||
try:
|
||||
with pytest.raises(RuntimeError, match="monitor failed"):
|
||||
SimulationRunner.start_simulation(
|
||||
simulation_id,
|
||||
platform="twitter",
|
||||
enable_graph_memory_update=False,
|
||||
)
|
||||
assert terminated == [simulation_id]
|
||||
assert simulation_id not in SimulationRunner._processes
|
||||
assert simulation_id not in SimulationRunner._action_queues
|
||||
assert simulation_id not in SimulationRunner._stdout_files
|
||||
finally:
|
||||
SimulationRunner._run_states.pop(simulation_id, None)
|
||||
SimulationRunner._processes.pop(simulation_id, None)
|
||||
SimulationRunner._action_queues.pop(simulation_id, None)
|
||||
SimulationRunner._stdout_files.pop(simulation_id, None)
|
||||
SimulationRunner._stderr_files.pop(simulation_id, None)
|
||||
SimulationRunner._graph_memory_enabled.pop(simulation_id, None)
|
||||
|
||||
|
||||
def test_shutdown_terminates_producer_before_tail_read_and_updater_drain(
|
||||
monkeypatch,
|
||||
):
|
||||
simulation_id = "sim-shutdown-order"
|
||||
state = SimulationRunState(
|
||||
simulation_id=simulation_id,
|
||||
runner_status=RunnerStatus.RUNNING,
|
||||
)
|
||||
events = []
|
||||
|
||||
class Process:
|
||||
pid = 123
|
||||
stopped = False
|
||||
|
||||
def poll(self):
|
||||
return 0 if self.stopped else None
|
||||
|
||||
process = Process()
|
||||
|
||||
class Monitor:
|
||||
alive = True
|
||||
|
||||
def join(self, timeout):
|
||||
assert timeout >= 30
|
||||
events.extend(["tail-read", "updater-drain"])
|
||||
state.runner_status = RunnerStatus.STOPPED
|
||||
SimulationRunner._graph_memory_enabled.pop(simulation_id, None)
|
||||
self.alive = False
|
||||
|
||||
def is_alive(self):
|
||||
return self.alive
|
||||
|
||||
monkeypatch.setattr(
|
||||
SimulationRunner,
|
||||
"get_run_state",
|
||||
classmethod(lambda _cls, _simulation_id: state),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
SimulationRunner,
|
||||
"_save_run_state",
|
||||
classmethod(lambda _cls, _state: None),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
SimulationRunner,
|
||||
"_sync_simulation_status",
|
||||
classmethod(lambda _cls, *_args, **_kwargs: None),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
SimulationRunner,
|
||||
"_terminate_process",
|
||||
classmethod(
|
||||
lambda _cls, proc, _simulation_id, **_kwargs: (
|
||||
events.append("producer-terminate"),
|
||||
setattr(proc, "stopped", True),
|
||||
)
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
runner_module.ZepGraphMemoryManager,
|
||||
"get_simulation_ids",
|
||||
classmethod(lambda _cls: [simulation_id]),
|
||||
)
|
||||
updater = object()
|
||||
monkeypatch.setattr(
|
||||
runner_module.ZepGraphMemoryManager,
|
||||
"get_updater",
|
||||
classmethod(lambda _cls, _simulation_id: updater),
|
||||
)
|
||||
|
||||
SimulationRunner._cleanup_done = False
|
||||
SimulationRunner._processes[simulation_id] = process
|
||||
SimulationRunner._monitor_threads[simulation_id] = Monitor()
|
||||
SimulationRunner._graph_memory_enabled[simulation_id] = True
|
||||
try:
|
||||
SimulationRunner.cleanup_all_simulations()
|
||||
assert events == [
|
||||
"producer-terminate",
|
||||
"tail-read",
|
||||
"updater-drain",
|
||||
]
|
||||
assert state.runner_status == RunnerStatus.STOPPED
|
||||
finally:
|
||||
SimulationRunner._cleanup_done = False
|
||||
SimulationRunner._processes.pop(simulation_id, None)
|
||||
SimulationRunner._monitor_threads.pop(simulation_id, None)
|
||||
SimulationRunner._graph_memory_enabled.pop(simulation_id, None)
|
||||
SimulationRunner._manual_stop_requests.discard(simulation_id)
|
||||
|
||||
|
||||
def test_shutdown_drain_failure_remains_failed_and_retryable(monkeypatch):
|
||||
simulation_id = "sim-shutdown-failure"
|
||||
state = SimulationRunState(
|
||||
simulation_id=simulation_id,
|
||||
runner_status=RunnerStatus.RUNNING,
|
||||
)
|
||||
updater = object()
|
||||
|
||||
monkeypatch.setattr(
|
||||
SimulationRunner,
|
||||
"get_run_state",
|
||||
classmethod(lambda _cls, _simulation_id: state),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
SimulationRunner,
|
||||
"_save_run_state",
|
||||
classmethod(lambda _cls, _state: None),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
SimulationRunner,
|
||||
"_sync_simulation_status",
|
||||
classmethod(lambda _cls, *_args, **_kwargs: None),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
runner_module.ZepGraphMemoryManager,
|
||||
"get_simulation_ids",
|
||||
classmethod(lambda _cls: [simulation_id]),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
runner_module.ZepGraphMemoryManager,
|
||||
"get_updater",
|
||||
classmethod(lambda _cls, _simulation_id: updater),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
runner_module.ZepGraphMemoryManager,
|
||||
"stop_updater",
|
||||
classmethod(
|
||||
lambda _cls, _simulation_id: (_ for _ in ()).throw(
|
||||
RuntimeError("drain incomplete")
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
SimulationRunner._cleanup_done = False
|
||||
SimulationRunner._graph_memory_enabled[simulation_id] = True
|
||||
SimulationRunner._monitor_threads.pop(simulation_id, None)
|
||||
SimulationRunner._processes.pop(simulation_id, None)
|
||||
try:
|
||||
SimulationRunner.cleanup_all_simulations()
|
||||
assert state.runner_status == RunnerStatus.FAILED
|
||||
assert "drain incomplete" in state.error
|
||||
assert SimulationRunner._graph_memory_enabled[simulation_id] is True
|
||||
assert SimulationRunner._cleanup_done is False
|
||||
finally:
|
||||
SimulationRunner._cleanup_done = False
|
||||
SimulationRunner._graph_memory_enabled.pop(simulation_id, None)
|
||||
SimulationRunner._manual_stop_requests.discard(simulation_id)
|
||||
|
|
@ -994,6 +994,7 @@ dependencies = [
|
|||
{ name = "charset-normalizer" },
|
||||
{ name = "flask" },
|
||||
{ name = "flask-cors" },
|
||||
{ name = "httpx" },
|
||||
{ name = "openai" },
|
||||
{ name = "pydantic" },
|
||||
{ name = "pymupdf" },
|
||||
|
|
@ -1022,6 +1023,7 @@ requires-dist = [
|
|||
{ name = "charset-normalizer", specifier = ">=3.0.0" },
|
||||
{ name = "flask", specifier = ">=3.0.0" },
|
||||
{ name = "flask-cors", specifier = ">=6.0.0" },
|
||||
{ name = "httpx", specifier = ">=0.27.0" },
|
||||
{ name = "openai", specifier = ">=1.0.0" },
|
||||
{ name = "pipreqs", marker = "extra == 'dev'", specifier = ">=0.5.0" },
|
||||
{ name = "pydantic", specifier = ">=2.0.0" },
|
||||
|
|
@ -1029,7 +1031,7 @@ requires-dist = [
|
|||
{ name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0" },
|
||||
{ name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.23.0" },
|
||||
{ name = "python-dotenv", specifier = ">=1.0.0" },
|
||||
{ name = "zep-cloud", specifier = "==3.13.0" },
|
||||
{ name = "zep-cloud", specifier = "==3.25.0" },
|
||||
]
|
||||
provides-extras = ["dev"]
|
||||
|
||||
|
|
@ -2775,7 +2777,7 @@ wheels = [
|
|||
|
||||
[[package]]
|
||||
name = "zep-cloud"
|
||||
version = "3.13.0"
|
||||
version = "3.25.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "httpx" },
|
||||
|
|
@ -2784,7 +2786,7 @@ dependencies = [
|
|||
{ name = "python-dateutil" },
|
||||
{ name = "typing-extensions" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/32/c7/c835debf13302f8aaf8d0561ac6ff5a9bc15cc140cd692a1330fb1900c55/zep_cloud-3.13.0.tar.gz", hash = "sha256:c55d9c511773bb2177ae8e08546141404f87d2099affafabd7ec4b4505763e48", size = 63116, upload-time = "2025-11-20T15:25:40.745Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/81/0d/7a866e1e1e5f0f1353eedf3d0e17df9b3cf362c494bccd41eb121b49c43f/zep_cloud-3.25.0.tar.gz", hash = "sha256:a77867e02c2a9036a20623e85d03415e237e76510b6e05ff2e9ab2cdced0aeb9", size = 94772, upload-time = "2026-07-16T00:46:43.918Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/f7/e1/bbf03c6c8007c0cb238780e7fc6d8e1a52633893933a41aa09678618985a/zep_cloud-3.13.0-py3-none-any.whl", hash = "sha256:b2fbdeef73e262194c8f67b58f76471de6ee87e1a629541a09d8f7bbf475f12b", size = 110601, upload-time = "2025-11-20T15:25:38.484Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/77/fc/5ca2bdd20b83bd62bc2708580f756e704f69704709e0ed43e735aa590c1e/zep_cloud-3.25.0-py3-none-any.whl", hash = "sha256:94d9599038b154af9ad33cab4b4873ed9adb1bcaa48fc4c41cbd407c1a657ef5", size = 166438, upload-time = "2026-07-16T00:46:42.39Z" },
|
||||
]
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import service, { requestWithRetry } from './index'
|
||||
import service from './index'
|
||||
|
||||
/**
|
||||
* 生成本体(上传文档和模拟需求)
|
||||
|
|
@ -6,16 +6,14 @@ import service, { requestWithRetry } from './index'
|
|||
* @returns {Promise}
|
||||
*/
|
||||
export function generateOntology(formData) {
|
||||
return requestWithRetry(() =>
|
||||
service({
|
||||
url: '/api/graph/ontology/generate',
|
||||
method: 'post',
|
||||
data: formData,
|
||||
headers: {
|
||||
'Content-Type': 'multipart/form-data'
|
||||
}
|
||||
})
|
||||
)
|
||||
return service({
|
||||
url: '/api/graph/ontology/generate',
|
||||
method: 'post',
|
||||
data: formData,
|
||||
headers: {
|
||||
'Content-Type': 'multipart/form-data'
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -24,13 +22,11 @@ export function generateOntology(formData) {
|
|||
* @returns {Promise}
|
||||
*/
|
||||
export function buildGraph(data) {
|
||||
return requestWithRetry(() =>
|
||||
service({
|
||||
url: '/api/graph/build',
|
||||
method: 'post',
|
||||
data
|
||||
})
|
||||
)
|
||||
return service({
|
||||
url: '/api/graph/build',
|
||||
method: 'post',
|
||||
data
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -52,18 +52,4 @@ service.interceptors.response.use(
|
|||
}
|
||||
)
|
||||
|
||||
// 带重试的请求函数
|
||||
export const requestWithRetry = async (requestFn, maxRetries = 3, delay = 1000) => {
|
||||
for (let i = 0; i < maxRetries; i++) {
|
||||
try {
|
||||
return await requestFn()
|
||||
} catch (error) {
|
||||
if (i === maxRetries - 1) throw error
|
||||
|
||||
console.warn(`Request failed, retrying (${i + 1}/${maxRetries})...`)
|
||||
await new Promise(resolve => setTimeout(resolve, delay * Math.pow(2, i)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default service
|
||||
|
|
|
|||
|
|
@ -1,11 +1,11 @@
|
|||
import service, { requestWithRetry } from './index'
|
||||
import service from './index'
|
||||
|
||||
/**
|
||||
* 开始报告生成
|
||||
* @param {Object} data - { simulation_id, force_regenerate? }
|
||||
*/
|
||||
export const generateReport = (data) => {
|
||||
return requestWithRetry(() => service.post('/api/report/generate', data), 3, 1000)
|
||||
return service.post('/api/report/generate', data)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -47,5 +47,5 @@ export const getReport = (reportId) => {
|
|||
* @param {Object} data - { simulation_id, message, chat_history? }
|
||||
*/
|
||||
export const chatWithReport = (data) => {
|
||||
return requestWithRetry(() => service.post('/api/report/chat', data), 3, 1000)
|
||||
return service.post('/api/report/chat', data)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,11 +1,11 @@
|
|||
import service, { requestWithRetry } from './index'
|
||||
import service from './index'
|
||||
|
||||
/**
|
||||
* 创建模拟
|
||||
* @param {Object} data - { project_id, graph_id?, enable_twitter?, enable_reddit? }
|
||||
*/
|
||||
export const createSimulation = (data) => {
|
||||
return requestWithRetry(() => service.post('/api/simulation/create', data), 3, 1000)
|
||||
return service.post('/api/simulation/create', data)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -13,7 +13,7 @@ export const createSimulation = (data) => {
|
|||
* @param {Object} data - { simulation_id, entity_types?, use_llm_for_profiles?, parallel_profile_count?, force_regenerate? }
|
||||
*/
|
||||
export const prepareSimulation = (data) => {
|
||||
return requestWithRetry(() => service.post('/api/simulation/prepare', data), 3, 1000)
|
||||
return service.post('/api/simulation/prepare', data)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -83,7 +83,7 @@ export const listSimulations = (projectId) => {
|
|||
* @param {Object} data - { simulation_id, platform?, max_rounds?, enable_graph_memory_update? }
|
||||
*/
|
||||
export const startSimulation = (data) => {
|
||||
return requestWithRetry(() => service.post('/api/simulation/start', data), 3, 1000)
|
||||
return service.post('/api/simulation/start', data)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -175,7 +175,7 @@ export const getEnvStatus = (data) => {
|
|||
* @param {Object} data - { simulation_id, interviews: [{ agent_id, prompt }] }
|
||||
*/
|
||||
export const interviewAgents = (data) => {
|
||||
return requestWithRetry(() => service.post('/api/simulation/interview/batch', data), 3, 1000)
|
||||
return service.post('/api/simulation/interview/batch', data)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -186,4 +186,3 @@ export const interviewAgents = (data) => {
|
|||
export const getSimulationHistory = (limit = 20) => {
|
||||
return service.get('/api/simulation/history', { params: { limit } })
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -515,22 +515,14 @@ const fetchRunStatus = async () => {
|
|||
const isCompleted = data.runner_status === 'completed' || data.runner_status === 'stopped'
|
||||
const isFailed = data.runner_status === 'failed'
|
||||
|
||||
// 额外检查:如果后端还没来得及更新 runner_status,但平台已经报告完成
|
||||
// 通过检测 twitter_completed 和 reddit_completed 状态判断
|
||||
const platformsCompleted = checkPlatformsCompleted(data)
|
||||
|
||||
// An explicit runner failure is authoritative. Platform completion flags
|
||||
// can be stale or partial when one subprocess exits successfully before
|
||||
// another subprocess fails, so inferred completion must not mask it.
|
||||
// runner_status is authoritative because the backend only publishes a
|
||||
// terminal state after the Zep ingestion barrier has completed.
|
||||
if (isFailed) {
|
||||
addLog(t('log.simFailed') + (data.error ? `: ${data.error}` : ''))
|
||||
phase.value = 2
|
||||
stopPolling()
|
||||
emit('update-status', 'error')
|
||||
} else if (isCompleted || platformsCompleted) {
|
||||
if (platformsCompleted && !isCompleted) {
|
||||
addLog(t('log.allPlatformsCompleted'))
|
||||
}
|
||||
} else if (isCompleted) {
|
||||
addLog(t('log.simCompleted'))
|
||||
phase.value = 2
|
||||
stopPolling()
|
||||
|
|
|
|||
|
|
@ -246,7 +246,6 @@ const loadProject = async () => {
|
|||
} else if (res.data.status === 'graph_building' && res.data.graph_build_task_id) {
|
||||
currentPhase.value = 1
|
||||
startPollingTask(res.data.graph_build_task_id)
|
||||
startGraphPolling()
|
||||
} else if (res.data.status === 'graph_completed' && res.data.graph_id) {
|
||||
currentPhase.value = 2
|
||||
await loadGraph(res.data.graph_id)
|
||||
|
|
@ -281,8 +280,18 @@ const startBuildGraph = async () => {
|
|||
|
||||
const res = await buildGraph({ project_id: currentProjectId.value })
|
||||
if (res.success) {
|
||||
if (res.data.reused && res.data.graph_id) {
|
||||
currentPhase.value = 2
|
||||
buildProgress.value = null
|
||||
const projectRes = await getProject(currentProjectId.value)
|
||||
if (projectRes.success) {
|
||||
projectData.value = projectRes.data
|
||||
}
|
||||
await loadGraph(res.data.graph_id)
|
||||
return
|
||||
}
|
||||
|
||||
addLog(`Graph build task started. Task ID: ${res.data.task_id}`)
|
||||
startGraphPolling()
|
||||
startPollingTask(res.data.task_id)
|
||||
} else {
|
||||
error.value = res.error
|
||||
|
|
|
|||
|
|
@ -686,14 +686,22 @@ const startBuildGraph = async () => {
|
|||
const response = await buildGraph({ project_id: currentProjectId.value })
|
||||
|
||||
if (response.success) {
|
||||
if (response.data.reused && response.data.graph_id) {
|
||||
currentPhase.value = 2
|
||||
buildProgress.value = null
|
||||
const projectResponse = await getProject(currentProjectId.value)
|
||||
if (projectResponse.success) {
|
||||
projectData.value = projectResponse.data
|
||||
}
|
||||
await loadGraph(response.data.graph_id)
|
||||
return
|
||||
}
|
||||
|
||||
buildProgress.value.message = '图谱构建任务已启动...'
|
||||
|
||||
// 保存 task_id 用于轮询
|
||||
const taskId = response.data.task_id
|
||||
|
||||
// 启动图谱数据轮询(独立于任务状态轮询)
|
||||
startGraphPolling()
|
||||
|
||||
// 启动任务状态轮询
|
||||
startPollingTask(taskId)
|
||||
} else {
|
||||
|
|
@ -2065,4 +2073,4 @@ onUnmounted(() => {
|
|||
display: none;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
</style>
|
||||
|
|
|
|||
Loading…
Reference in New Issue