1769 lines
70 KiB
Python
1769 lines
70 KiB
Python
"""
|
||
OASISSimulation Runner
|
||
Run simulations in the background and record eachAgentaction，Support real-time status monitoring
|
||
"""
|
||
|
||
import os
|
||
import sys
|
||
import json
|
||
import time
|
||
import asyncio
|
||
import threading
|
||
import subprocess
|
||
import signal
|
||
import atexit
|
||
from typing import Dict, Any, List, Optional, Union
|
||
from dataclasses import dataclass, field
|
||
from datetime import datetime
|
||
from enum import Enum
|
||
from queue import Queue
|
||
|
||
from ..config import Config
|
||
from ..utils.logger import get_logger
|
||
from ..utils.locale import get_locale, set_locale
|
||
from .zep_graph_memory_updater import ZepGraphMemoryManager
|
||
from .simulation_ipc import SimulationIPCClient, CommandType, IPCResponse
|
||
|
||
logger = get_logger('mirofish.simulation_runner')
|
||
|
||
# Flag indicating whether the cleanup function has been registered
|
||
_cleanup_registered = False
|
||
|
||
# Platform detection
|
||
IS_WINDOWS = sys.platform == 'win32'
|
||
|
||
|
||
class RunnerStatus(str, Enum):
|
||
"""Runner state"""
|
||
IDLE = "idle"
|
||
STARTING = "starting"
|
||
RUNNING = "running"
|
||
PAUSED = "paused"
|
||
STOPPING = "stopping"
|
||
STOPPED = "stopped"
|
||
COMPLETED = "completed"
|
||
FAILED = "failed"
|
||
|
||
|
||
@dataclass
|
||
class AgentAction:
|
||
"""Agentaction record"""
|
||
round_num: int
|
||
timestamp: str
|
||
platform: str # twitter / reddit
|
||
agent_id: int
|
||
agent_name: str
|
||
action_type: str # CREATE_POST, LIKE_POST, etc.
|
||
action_args: Dict[str, Any] = field(default_factory=dict)
|
||
result: Optional[str] = None
|
||
success: bool = True
|
||
|
||
def to_dict(self) -> Dict[str, Any]:
|
||
return {
|
||
"round_num": self.round_num,
|
||
"timestamp": self.timestamp,
|
||
"platform": self.platform,
|
||
"agent_id": self.agent_id,
|
||
"agent_name": self.agent_name,
|
||
"action_type": self.action_type,
|
||
"action_args": self.action_args,
|
||
"result": self.result,
|
||
"success": self.success,
|
||
}
|
||
|
||
|
||
@dataclass
|
||
class RoundSummary:
|
||
"""per-round summary"""
|
||
round_num: int
|
||
start_time: str
|
||
end_time: Optional[str] = None
|
||
simulated_hour: int = 0
|
||
twitter_actions: int = 0
|
||
reddit_actions: int = 0
|
||
active_agents: List[int] = field(default_factory=list)
|
||
actions: List[AgentAction] = field(default_factory=list)
|
||
|
||
def to_dict(self) -> Dict[str, Any]:
|
||
return {
|
||
"round_num": self.round_num,
|
||
"start_time": self.start_time,
|
||
"end_time": self.end_time,
|
||
"simulated_hour": self.simulated_hour,
|
||
"twitter_actions": self.twitter_actions,
|
||
"reddit_actions": self.reddit_actions,
|
||
"active_agents": self.active_agents,
|
||
"actions_count": len(self.actions),
|
||
"actions": [a.to_dict() for a in self.actions],
|
||
}
|
||
|
||
|
||
@dataclass
|
||
class SimulationRunState:
|
||
"""simulation running state（real-time）"""
|
||
simulation_id: str
|
||
runner_status: RunnerStatus = RunnerStatus.IDLE
|
||
|
||
# progress information
|
||
current_round: int = 0
|
||
total_rounds: int = 0
|
||
simulated_hours: int = 0
|
||
total_simulation_hours: int = 0
|
||
|
||
# Per-platform independent rounds and simulation time（Used for parallel display of both platforms）
|
||
twitter_current_round: int = 0
|
||
reddit_current_round: int = 0
|
||
twitter_simulated_hours: int = 0
|
||
reddit_simulated_hours: int = 0
|
||
|
||
# platform state
|
||
twitter_running: bool = False
|
||
reddit_running: bool = False
|
||
twitter_actions_count: int = 0
|
||
reddit_actions_count: int = 0
|
||
|
||
# platform completion state（through detection actions.jsonl in the simulation_end event）
|
||
twitter_completed: bool = False
|
||
reddit_completed: bool = False
|
||
|
||
# per-round summary
|
||
rounds: List[RoundSummary] = field(default_factory=list)
|
||
|
||
# recent action（Used for real-time frontend display）
|
||
recent_actions: List[AgentAction] = field(default_factory=list)
|
||
max_recent_actions: int = 50
|
||
|
||
# timestamp
|
||
started_at: Optional[str] = None
|
||
updated_at: str = field(default_factory=lambda: datetime.now().isoformat())
|
||
completed_at: Optional[str] = None
|
||
|
||
# error message
|
||
error: Optional[str] = None
|
||
|
||
# processID（used to stop）
|
||
process_pid: Optional[int] = None
|
||
|
||
def add_action(self, action: AgentAction):
|
||
"""Add action to the recent action list"""
|
||
self.recent_actions.insert(0, action)
|
||
if len(self.recent_actions) > self.max_recent_actions:
|
||
self.recent_actions = self.recent_actions[:self.max_recent_actions]
|
||
|
||
if action.platform == "twitter":
|
||
self.twitter_actions_count += 1
|
||
else:
|
||
self.reddit_actions_count += 1
|
||
|
||
self.updated_at = datetime.now().isoformat()
|
||
|
||
def to_dict(self) -> Dict[str, Any]:
|
||
return {
|
||
"simulation_id": self.simulation_id,
|
||
"runner_status": self.runner_status.value,
|
||
"current_round": self.current_round,
|
||
"total_rounds": self.total_rounds,
|
||
"simulated_hours": self.simulated_hours,
|
||
"total_simulation_hours": self.total_simulation_hours,
|
||
"progress_percent": round(self.current_round / max(self.total_rounds, 1) * 100, 1),
|
||
# Per-platform independent rounds and time
|
||
"twitter_current_round": self.twitter_current_round,
|
||
"reddit_current_round": self.reddit_current_round,
|
||
"twitter_simulated_hours": self.twitter_simulated_hours,
|
||
"reddit_simulated_hours": self.reddit_simulated_hours,
|
||
"twitter_running": self.twitter_running,
|
||
"reddit_running": self.reddit_running,
|
||
"twitter_completed": self.twitter_completed,
|
||
"reddit_completed": self.reddit_completed,
|
||
"twitter_actions_count": self.twitter_actions_count,
|
||
"reddit_actions_count": self.reddit_actions_count,
|
||
"total_actions_count": self.twitter_actions_count + self.reddit_actions_count,
|
||
"started_at": self.started_at,
|
||
"updated_at": self.updated_at,
|
||
"completed_at": self.completed_at,
|
||
"error": self.error,
|
||
"process_pid": self.process_pid,
|
||
}
|
||
|
||
def to_detail_dict(self) -> Dict[str, Any]:
|
||
"""Contains detailed information of recent actions"""
|
||
result = self.to_dict()
|
||
result["recent_actions"] = [a.to_dict() for a in self.recent_actions]
|
||
result["rounds_count"] = len(self.rounds)
|
||
return result
|
||
|
||
|
||
class SimulationRunner:
|
||
"""
|
||
Simulation Runner
|
||
|
||
responsible for：
|
||
1. Run in a background processOASISsimulation
|
||
2. Parse run logs，record eachAgentaction
|
||
3. Provide a real-time status query interface
|
||
4. Support pause/stop/resume operation
|
||
"""
|
||
|
||
# Running state storage directory
|
||
RUN_STATE_DIR = os.path.join(
|
||
os.path.dirname(__file__),
|
||
'../../uploads/simulations'
|
||
)
|
||
|
||
# script directory
|
||
SCRIPTS_DIR = os.path.join(
|
||
os.path.dirname(__file__),
|
||
'../../scripts'
|
||
)
|
||
|
||
# In-memory running state
|
||
_run_states: Dict[str, SimulationRunState] = {}
|
||
_processes: Dict[str, subprocess.Popen] = {}
|
||
_action_queues: Dict[str, Queue] = {}
|
||
_monitor_threads: Dict[str, threading.Thread] = {}
|
||
_stdout_files: Dict[str, Any] = {} # storage stdout file handle
|
||
_stderr_files: Dict[str, Any] = {} # storage stderr file handle
|
||
|
||
# Graph memory update configuration
|
||
_graph_memory_enabled: Dict[str, bool] = {} # simulation_id -> enabled
|
||
|
||
@classmethod
|
||
def get_run_state(cls, simulation_id: str) -> Optional[SimulationRunState]:
|
||
"""Get the running state"""
|
||
if simulation_id in cls._run_states:
|
||
return cls._run_states[simulation_id]
|
||
|
||
# Try to load from file
|
||
state = cls._load_run_state(simulation_id)
|
||
if state:
|
||
cls._run_states[simulation_id] = state
|
||
return state
|
||
|
||
@classmethod
|
||
def _load_run_state(cls, simulation_id: str) -> Optional[SimulationRunState]:
|
||
"""Load running state from file"""
|
||
state_file = os.path.join(cls.RUN_STATE_DIR, simulation_id, "run_state.json")
|
||
if not os.path.exists(state_file):
|
||
return None
|
||
|
||
try:
|
||
with open(state_file, 'r', encoding='utf-8') as f:
|
||
data = json.load(f)
|
||
|
||
state = SimulationRunState(
|
||
simulation_id=simulation_id,
|
||
runner_status=RunnerStatus(data.get("runner_status", "idle")),
|
||
current_round=data.get("current_round", 0),
|
||
total_rounds=data.get("total_rounds", 0),
|
||
simulated_hours=data.get("simulated_hours", 0),
|
||
total_simulation_hours=data.get("total_simulation_hours", 0),
|
||
# Per-platform independent rounds and time
|
||
twitter_current_round=data.get("twitter_current_round", 0),
|
||
reddit_current_round=data.get("reddit_current_round", 0),
|
||
twitter_simulated_hours=data.get("twitter_simulated_hours", 0),
|
||
reddit_simulated_hours=data.get("reddit_simulated_hours", 0),
|
||
twitter_running=data.get("twitter_running", False),
|
||
reddit_running=data.get("reddit_running", False),
|
||
twitter_completed=data.get("twitter_completed", False),
|
||
reddit_completed=data.get("reddit_completed", False),
|
||
twitter_actions_count=data.get("twitter_actions_count", 0),
|
||
reddit_actions_count=data.get("reddit_actions_count", 0),
|
||
started_at=data.get("started_at"),
|
||
updated_at=data.get("updated_at", datetime.now().isoformat()),
|
||
completed_at=data.get("completed_at"),
|
||
error=data.get("error"),
|
||
process_pid=data.get("process_pid"),
|
||
)
|
||
|
||
# Load recent actions
|
||
actions_data = data.get("recent_actions", [])
|
||
for a in actions_data:
|
||
state.recent_actions.append(AgentAction(
|
||
round_num=a.get("round_num", 0),
|
||
timestamp=a.get("timestamp", ""),
|
||
platform=a.get("platform", ""),
|
||
agent_id=a.get("agent_id", 0),
|
||
agent_name=a.get("agent_name", ""),
|
||
action_type=a.get("action_type", ""),
|
||
action_args=a.get("action_args", {}),
|
||
result=a.get("result"),
|
||
success=a.get("success", True),
|
||
))
|
||
|
||
return state
|
||
except Exception as e:
|
||
logger.error(f"Failed to load running state: {str(e)}")
|
||
return None
|
||
|
||
@classmethod
|
||
def _save_run_state(cls, state: SimulationRunState):
|
||
"""Save running state to file"""
|
||
sim_dir = os.path.join(cls.RUN_STATE_DIR, state.simulation_id)
|
||
os.makedirs(sim_dir, exist_ok=True)
|
||
state_file = os.path.join(sim_dir, "run_state.json")
|
||
|
||
data = state.to_detail_dict()
|
||
|
||
with open(state_file, 'w', encoding='utf-8') as f:
|
||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||
|
||
cls._run_states[state.simulation_id] = state
|
||
|
||
@classmethod
|
||
def start_simulation(
|
||
cls,
|
||
simulation_id: str,
|
||
platform: str = "parallel", # twitter / reddit / parallel
|
||
max_rounds: int = None, # maximum simulation rounds（optional，Used to truncate overly long simulations）
|
||
enable_graph_memory_update: bool = False, # whether to update activities toZepgraph
|
||
graph_id: str = None # ZepgraphID（Required when graph update is enabled）
|
||
) -> SimulationRunState:
|
||
"""
|
||
Start simulation
|
||
|
||
Args:
|
||
simulation_id: simulationID
|
||
platform: running platform (twitter/reddit/parallel)
|
||
max_rounds: maximum simulation rounds（optional，Used to truncate overly long simulations）
|
||
enable_graph_memory_update: whether toAgentactivity updates toZepgraph
|
||
graph_id: ZepgraphID（Required when graph update is enabled）
|
||
|
||
Returns:
|
||
SimulationRunState
|
||
"""
|
||
# Check whether it is already running
|
||
existing = cls.get_run_state(simulation_id)
|
||
if existing and existing.runner_status in [RunnerStatus.RUNNING, RunnerStatus.STARTING]:
|
||
raise ValueError(f"Simulation is already running: {simulation_id}")
|
||
|
||
# Load simulation configuration
|
||
sim_dir = os.path.join(cls.RUN_STATE_DIR, simulation_id)
|
||
config_path = os.path.join(sim_dir, "simulation_config.json")
|
||
|
||
if not os.path.exists(config_path):
|
||
raise ValueError(f"Simulation configuration does not exist，please first call /prepare interface")
|
||
|
||
with open(config_path, 'r', encoding='utf-8') as f:
|
||
config = json.load(f)
|
||
|
||
# Initialize the running state
|
||
time_config = config.get("time_config", {})
|
||
total_hours = time_config.get("total_simulation_hours", 72)
|
||
minutes_per_round = time_config.get("minutes_per_round", 30)
|
||
total_rounds = int(total_hours * 60 / minutes_per_round)
|
||
|
||
# If the maximum rounds are specified，then truncate
|
||
if max_rounds is not None and max_rounds > 0:
|
||
original_rounds = total_rounds
|
||
total_rounds = min(total_rounds, max_rounds)
|
||
if total_rounds < original_rounds:
|
||
logger.info(f"Rounds have been truncated: {original_rounds} -> {total_rounds} (max_rounds={max_rounds})")
|
||
|
||
state = SimulationRunState(
|
||
simulation_id=simulation_id,
|
||
runner_status=RunnerStatus.STARTING,
|
||
total_rounds=total_rounds,
|
||
total_simulation_hours=total_hours,
|
||
started_at=datetime.now().isoformat(),
|
||
)
|
||
|
||
cls._save_run_state(state)
|
||
|
||
# If graph memory update is enabled, create the updater
|
||
if enable_graph_memory_update:
|
||
if not graph_id:
|
||
raise ValueError("Must be provided when graph memory update is enabled graph_id")
|
||
|
||
try:
|
||
ZepGraphMemoryManager.create_updater(simulation_id, graph_id)
|
||
cls._graph_memory_enabled[simulation_id] = True
|
||
logger.info(f"Graph memory update has been enabled: simulation_id={simulation_id}, graph_id={graph_id}")
|
||
except Exception as e:
|
||
logger.error(f"Failed to create graph memory updater: {e}")
|
||
cls._graph_memory_enabled[simulation_id] = False
|
||
else:
|
||
cls._graph_memory_enabled[simulation_id] = False
|
||
|
||
# Determine which script to run（Script is located in backend/scripts/ directory）
|
||
if platform == "twitter":
|
||
script_name = "run_twitter_simulation.py"
|
||
state.twitter_running = True
|
||
elif platform == "reddit":
|
||
script_name = "run_reddit_simulation.py"
|
||
state.reddit_running = True
|
||
else:
|
||
script_name = "run_parallel_simulation.py"
|
||
state.twitter_running = True
|
||
state.reddit_running = True
|
||
|
||
script_path = os.path.join(cls.SCRIPTS_DIR, script_name)
|
||
|
||
if not os.path.exists(script_path):
|
||
raise ValueError(f"Script does not exist: {script_path}")
|
||
|
||
# Create the action queue
|
||
action_queue = Queue()
|
||
cls._action_queues[simulation_id] = action_queue
|
||
|
||
# Start the simulation process
|
||
try:
|
||
# Build the run command，Use the full path
|
||
# New log structure：
|
||
# twitter/actions.jsonl - Twitter action log
|
||
# reddit/actions.jsonl - Reddit action log
|
||
# simulation.log - main process log
|
||
|
||
cmd = [
|
||
sys.executable, # Pythoninterpreter
|
||
script_path,
|
||
"--config", config_path, # Use the full configuration file path
|
||
]
|
||
|
||
# If the maximum rounds are specified，Add to command-line arguments
|
||
if max_rounds is not None and max_rounds > 0:
|
||
cmd.extend(["--max-rounds", str(max_rounds)])
|
||
|
||
# Create the main log file，avoid stdout/stderr pipeline buffer full causes the process to block
|
||
main_log_path = os.path.join(sim_dir, "simulation.log")
|
||
main_log_file = open(main_log_path, 'w', encoding='utf-8')
|
||
|
||
# Set subprocess environment variables，ensure Windows on the UTF-8 encoding
|
||
# This can fix third-party libraries（e.g. OASIS）problem of un-specified encoding when reading files
|
||
env = os.environ.copy()
|
||
env['PYTHONUTF8'] = '1' # Python 3.7+ support，let all open() use by default UTF-8
|
||
env['PYTHONIOENCODING'] = 'utf-8' # ensure stdout/stderr use UTF-8
|
||
|
||
# Set the working directory to the simulation directory（database and other files will be generated here）
|
||
# use start_new_session=True Create a new process group，ensure it can be terminated through os.killpg terminate all child processes
|
||
process = subprocess.Popen(
|
||
cmd,
|
||
cwd=sim_dir,
|
||
stdout=main_log_file,
|
||
stderr=subprocess.STDOUT, # stderr also written to the same file
|
||
text=True,
|
||
encoding='utf-8', # Explicitly specify encoding
|
||
bufsize=1,
|
||
env=env, # passing with UTF-8 set environment variables
|
||
start_new_session=True, # create a new process group，Ensure all related processes are terminated when the server shuts down
|
||
)
|
||
|
||
# Save file handle for later closing
|
||
cls._stdout_files[simulation_id] = main_log_file
|
||
cls._stderr_files[simulation_id] = None # No longer need a separate stderr
|
||
|
||
state.process_pid = process.pid
|
||
state.runner_status = RunnerStatus.RUNNING
|
||
cls._processes[simulation_id] = process
|
||
cls._save_run_state(state)
|
||
|
||
# Capture locale before spawning monitor thread
|
||
current_locale = get_locale()
|
||
|
||
# Start the monitor thread
|
||
monitor_thread = threading.Thread(
|
||
target=cls._monitor_simulation,
|
||
args=(simulation_id, current_locale),
|
||
daemon=True
|
||
)
|
||
monitor_thread.start()
|
||
cls._monitor_threads[simulation_id] = monitor_thread
|
||
|
||
logger.info(f"Simulation started successfully: {simulation_id}, pid={process.pid}, platform={platform}")
|
||
|
||
except Exception as e:
|
||
state.runner_status = RunnerStatus.FAILED
|
||
state.error = str(e)
|
||
cls._save_run_state(state)
|
||
raise
|
||
|
||
return state
|
||
|
||
@classmethod
|
||
def _monitor_simulation(cls, simulation_id: str, locale: str = 'zh'):
|
||
"""Monitor the simulation process，Parse action logs"""
|
||
set_locale(locale)
|
||
sim_dir = os.path.join(cls.RUN_STATE_DIR, simulation_id)
|
||
|
||
# New log structure：Per-platform action logs
|
||
twitter_actions_log = os.path.join(sim_dir, "twitter", "actions.jsonl")
|
||
reddit_actions_log = os.path.join(sim_dir, "reddit", "actions.jsonl")
|
||
|
||
process = cls._processes.get(simulation_id)
|
||
state = cls.get_run_state(simulation_id)
|
||
|
||
if not process or not state:
|
||
return
|
||
|
||
twitter_position = 0
|
||
reddit_position = 0
|
||
|
||
try:
|
||
while process.poll() is None: # Process is still running
|
||
# read Twitter action log
|
||
if os.path.exists(twitter_actions_log):
|
||
twitter_position = cls._read_action_log(
|
||
twitter_actions_log, twitter_position, state, "twitter"
|
||
)
|
||
|
||
# read Reddit action log
|
||
if os.path.exists(reddit_actions_log):
|
||
reddit_position = cls._read_action_log(
|
||
reddit_actions_log, reddit_position, state, "reddit"
|
||
)
|
||
|
||
# Update state
|
||
cls._save_run_state(state)
|
||
time.sleep(2)
|
||
|
||
# After the process ends，Read the log one last time
|
||
if os.path.exists(twitter_actions_log):
|
||
cls._read_action_log(twitter_actions_log, twitter_position, state, "twitter")
|
||
if os.path.exists(reddit_actions_log):
|
||
cls._read_action_log(reddit_actions_log, reddit_position, state, "reddit")
|
||
|
||
# process end
|
||
exit_code = process.returncode
|
||
|
||
if exit_code == 0:
|
||
state.runner_status = RunnerStatus.COMPLETED
|
||
state.completed_at = datetime.now().isoformat()
|
||
logger.info(f"Simulation complete: {simulation_id}")
|
||
else:
|
||
state.runner_status = RunnerStatus.FAILED
|
||
# Read error information from the main log file
|
||
main_log_path = os.path.join(sim_dir, "simulation.log")
|
||
error_info = ""
|
||
try:
|
||
if os.path.exists(main_log_path):
|
||
with open(main_log_path, 'r', encoding='utf-8') as f:
|
||
error_info = f.read()[-2000:] # take the last2000characters
|
||
except Exception:
|
||
pass
|
||
state.error = f"Process exit code: {exit_code}, error: {error_info}"
|
||
logger.error(f"Simulation failed: {simulation_id}, error={state.error}")
|
||
|
||
state.twitter_running = False
|
||
state.reddit_running = False
|
||
cls._save_run_state(state)
|
||
|
||
except Exception as e:
|
||
logger.error(f"Monitor thread exception: {simulation_id}, error={str(e)}")
|
||
state.runner_status = RunnerStatus.FAILED
|
||
state.error = str(e)
|
||
cls._save_run_state(state)
|
||
|
||
finally:
|
||
# Stop the graph memory updater
|
||
if cls._graph_memory_enabled.get(simulation_id, False):
|
||
try:
|
||
ZepGraphMemoryManager.stop_updater(simulation_id)
|
||
logger.info(f"Graph memory update has been stopped: simulation_id={simulation_id}")
|
||
except Exception as e:
|
||
logger.error(f"Failed to stop graph memory updater: {e}")
|
||
cls._graph_memory_enabled.pop(simulation_id, None)
|
||
|
||
# Clean up process resources
|
||
cls._processes.pop(simulation_id, None)
|
||
cls._action_queues.pop(simulation_id, None)
|
||
|
||
# Close the log file handle
|
||
if simulation_id in cls._stdout_files:
|
||
try:
|
||
cls._stdout_files[simulation_id].close()
|
||
except Exception:
|
||
pass
|
||
cls._stdout_files.pop(simulation_id, None)
|
||
if simulation_id in cls._stderr_files and cls._stderr_files[simulation_id]:
|
||
try:
|
||
cls._stderr_files[simulation_id].close()
|
||
except Exception:
|
||
pass
|
||
cls._stderr_files.pop(simulation_id, None)
|
||
|
||
@classmethod
|
||
def _read_action_log(
|
||
cls,
|
||
log_path: str,
|
||
position: int,
|
||
state: SimulationRunState,
|
||
platform: str
|
||
) -> int:
|
||
"""
|
||
Read the action log file
|
||
|
||
Args:
|
||
log_path: log file path
|
||
position: last read position
|
||
state: running state object
|
||
platform: platform name (twitter/reddit)
|
||
|
||
Returns:
|
||
new read position
|
||
"""
|
||
# Check whether graph memory update is enabled
|
||
graph_memory_enabled = cls._graph_memory_enabled.get(state.simulation_id, False)
|
||
graph_updater = None
|
||
if graph_memory_enabled:
|
||
graph_updater = ZepGraphMemoryManager.get_updater(state.simulation_id)
|
||
|
||
try:
|
||
with open(log_path, 'r', encoding='utf-8') as f:
|
||
f.seek(position)
|
||
for line in f:
|
||
line = line.strip()
|
||
if line:
|
||
try:
|
||
action_data = json.loads(line)
|
||
|
||
# Process event-type entries
|
||
if "event_type" in action_data:
|
||
event_type = action_data.get("event_type")
|
||
|
||
# detection simulation_end event，Mark the platform as completed
|
||
if event_type == "simulation_end":
|
||
if platform == "twitter":
|
||
state.twitter_completed = True
|
||
state.twitter_running = False
|
||
logger.info(f"Twitter Simulation has completed: {state.simulation_id}, total_rounds={action_data.get('total_rounds')}, total_actions={action_data.get('total_actions')}")
|
||
elif platform == "reddit":
|
||
state.reddit_completed = True
|
||
state.reddit_running = False
|
||
logger.info(f"Reddit Simulation has completed: {state.simulation_id}, total_rounds={action_data.get('total_rounds')}, total_actions={action_data.get('total_actions')}")
|
||
|
||
# Check whether all enabled platforms have completed
|
||
# If only one platform is running，only check that platform
|
||
# If two platforms are running，need both to be completed
|
||
all_completed = cls._check_all_platforms_completed(state)
|
||
if all_completed:
|
||
state.runner_status = RunnerStatus.COMPLETED
|
||
state.completed_at = datetime.now().isoformat()
|
||
logger.info(f"All platforms simulation has completed: {state.simulation_id}")
|
||
|
||
# Update round information（from round_end event）
|
||
elif event_type == "round_end":
|
||
round_num = action_data.get("round", 0)
|
||
simulated_hours = action_data.get("simulated_hours", 0)
|
||
|
||
# Update per-platform independent rounds and time
|
||
if platform == "twitter":
|
||
if round_num > state.twitter_current_round:
|
||
state.twitter_current_round = round_num
|
||
state.twitter_simulated_hours = simulated_hours
|
||
elif platform == "reddit":
|
||
if round_num > state.reddit_current_round:
|
||
state.reddit_current_round = round_num
|
||
state.reddit_simulated_hours = simulated_hours
|
||
|
||
# Overall rounds take the maximum of the two platforms
|
||
if round_num > state.current_round:
|
||
state.current_round = round_num
|
||
# Overall time takes the maximum of the two platforms
|
||
state.simulated_hours = max(state.twitter_simulated_hours, state.reddit_simulated_hours)
|
||
|
||
continue
|
||
|
||
action = AgentAction(
|
||
round_num=action_data.get("round", 0),
|
||
timestamp=action_data.get("timestamp", datetime.now().isoformat()),
|
||
platform=platform,
|
||
agent_id=action_data.get("agent_id", 0),
|
||
agent_name=action_data.get("agent_name", ""),
|
||
action_type=action_data.get("action_type", ""),
|
||
action_args=action_data.get("action_args", {}),
|
||
result=action_data.get("result"),
|
||
success=action_data.get("success", True),
|
||
)
|
||
state.add_action(action)
|
||
|
||
# update round
|
||
if action.round_num and action.round_num > state.current_round:
|
||
state.current_round = action.round_num
|
||
|
||
# If graph memory update is enabled，send activities toZep
|
||
if graph_updater:
|
||
graph_updater.add_activity_from_dict(action_data, platform)
|
||
|
||
except json.JSONDecodeError:
|
||
pass
|
||
return f.tell()
|
||
except Exception as e:
|
||
logger.warning(f"Failed to read the action log: {log_path}, error={e}")
|
||
return position
|
||
|
||
@classmethod
|
||
def _check_all_platforms_completed(cls, state: SimulationRunState) -> bool:
|
||
"""
|
||
Check whether all enabled platforms have completed simulation
|
||
|
||
by checking the corresponding actions.jsonl check whether a file exists to determine whether a platform is enabled
|
||
|
||
Returns:
|
||
True If all enabled platforms have completed
|
||
"""
|
||
sim_dir = os.path.join(cls.RUN_STATE_DIR, state.simulation_id)
|
||
twitter_log = os.path.join(sim_dir, "twitter", "actions.jsonl")
|
||
reddit_log = os.path.join(sim_dir, "reddit", "actions.jsonl")
|
||
|
||
# Check which platforms are enabled（determined by file existence）
|
||
twitter_enabled = os.path.exists(twitter_log)
|
||
reddit_enabled = os.path.exists(reddit_log)
|
||
|
||
# If the platform is enabled but not completed，then return False
|
||
if twitter_enabled and not state.twitter_completed:
|
||
return False
|
||
if reddit_enabled and not state.reddit_completed:
|
||
return False
|
||
|
||
# At least one platform is enabled and has completed
|
||
return twitter_enabled or reddit_enabled
|
||
|
||
@classmethod
|
||
def _terminate_process(cls, process: subprocess.Popen, simulation_id: str, timeout: int = 10):
|
||
"""
|
||
Cross-platform terminate a process and its child processes
|
||
|
||
Args:
|
||
process: process to terminate
|
||
simulation_id: simulationID（used for logs）
|
||
timeout: Timeout for waiting for the process to exit（seconds）
|
||
"""
|
||
if IS_WINDOWS:
|
||
# Windows: use taskkill command to terminate the process tree
|
||
# /F = force terminate, /T = terminate the process tree（including child processes）
|
||
logger.info(f"terminate the process tree (Windows): simulation={simulation_id}, pid={process.pid}")
|
||
try:
|
||
# First try graceful termination
|
||
subprocess.run(
|
||
['taskkill', '/PID', str(process.pid), '/T'],
|
||
capture_output=True,
|
||
timeout=5
|
||
)
|
||
try:
|
||
process.wait(timeout=timeout)
|
||
except subprocess.TimeoutExpired:
|
||
# force terminate
|
||
logger.warning(f"Process did not respond，force terminate: {simulation_id}")
|
||
subprocess.run(
|
||
['taskkill', '/F', '/PID', str(process.pid), '/T'],
|
||
capture_output=True,
|
||
timeout=5
|
||
)
|
||
process.wait(timeout=5)
|
||
except Exception as e:
|
||
logger.warning(f"taskkill failed，try terminate: {e}")
|
||
process.terminate()
|
||
try:
|
||
process.wait(timeout=5)
|
||
except subprocess.TimeoutExpired:
|
||
process.kill()
|
||
else:
|
||
# Unix: Use process group to terminate
|
||
# Because we used start_new_session=True，process group ID equal to the main process PID
|
||
pgid = os.getpgid(process.pid)
|
||
logger.info(f"terminate the process group (Unix): simulation={simulation_id}, pgid={pgid}")
|
||
|
||
# first send SIGTERM to the entire process group
|
||
os.killpg(pgid, signal.SIGTERM)
|
||
|
||
try:
|
||
process.wait(timeout=timeout)
|
||
except subprocess.TimeoutExpired:
|
||
# If not finished after timeout，force send SIGKILL
|
||
logger.warning(f"Process group did not respond SIGTERM，force terminate: {simulation_id}")
|
||
os.killpg(pgid, signal.SIGKILL)
|
||
process.wait(timeout=5)
|
||
|
||
@classmethod
|
||
def stop_simulation(cls, simulation_id: str) -> SimulationRunState:
|
||
"""Stop simulation"""
|
||
state = cls.get_run_state(simulation_id)
|
||
if not state:
|
||
raise ValueError(f"Simulation does not exist: {simulation_id}")
|
||
|
||
if state.runner_status not in [RunnerStatus.RUNNING, RunnerStatus.PAUSED]:
|
||
raise ValueError(f"Simulation is not running: {simulation_id}, status={state.runner_status}")
|
||
|
||
state.runner_status = RunnerStatus.STOPPING
|
||
cls._save_run_state(state)
|
||
|
||
# terminate process
|
||
process = cls._processes.get(simulation_id)
|
||
if process and process.poll() is None:
|
||
try:
|
||
cls._terminate_process(process, simulation_id)
|
||
except ProcessLookupError:
|
||
# Process has already been gone
|
||
pass
|
||
except Exception as e:
|
||
logger.error(f"Failed to terminate the process group: {simulation_id}, error={e}")
|
||
# Fall back to terminating the process directly
|
||
try:
|
||
process.terminate()
|
||
process.wait(timeout=5)
|
||
except Exception:
|
||
process.kill()
|
||
|
||
state.runner_status = RunnerStatus.STOPPED
|
||
state.twitter_running = False
|
||
state.reddit_running = False
|
||
state.completed_at = datetime.now().isoformat()
|
||
cls._save_run_state(state)
|
||
|
||
# Stop the graph memory updater
|
||
if cls._graph_memory_enabled.get(simulation_id, False):
|
||
try:
|
||
ZepGraphMemoryManager.stop_updater(simulation_id)
|
||
logger.info(f"Graph memory update has been stopped: simulation_id={simulation_id}")
|
||
except Exception as e:
|
||
logger.error(f"Failed to stop graph memory updater: {e}")
|
||
cls._graph_memory_enabled.pop(simulation_id, None)
|
||
|
||
logger.info(f"Simulation has been stopped: {simulation_id}")
|
||
return state
|
||
|
||
@classmethod
|
||
def _read_actions_from_file(
|
||
cls,
|
||
file_path: str,
|
||
default_platform: Optional[str] = None,
|
||
platform_filter: Optional[str] = None,
|
||
agent_id: Optional[int] = None,
|
||
round_num: Optional[int] = None
|
||
) -> List[AgentAction]:
|
||
"""
|
||
Read actions from a single action file
|
||
|
||
Args:
|
||
file_path: Action log file path
|
||
default_platform: default platform（When the action record has no platform field is used）
|
||
platform_filter: filter platform
|
||
agent_id: filter Agent ID
|
||
round_num: filter rounds
|
||
"""
|
||
if not os.path.exists(file_path):
|
||
return []
|
||
|
||
actions = []
|
||
|
||
with open(file_path, 'r', encoding='utf-8') as f:
|
||
for line in f:
|
||
line = line.strip()
|
||
if not line:
|
||
continue
|
||
|
||
try:
|
||
data = json.loads(line)
|
||
|
||
# Skip non-action records（e.g. simulation_start, round_start, round_end and other events）
|
||
if "event_type" in data:
|
||
continue
|
||
|
||
# skip records without agent_id record（non- Agent action）
|
||
if "agent_id" not in data:
|
||
continue
|
||
|
||
# get platform：Prefer the one in the record platform，otherwise use the default platform
|
||
record_platform = data.get("platform") or default_platform or ""
|
||
|
||
# filter
|
||
if platform_filter and record_platform != platform_filter:
|
||
continue
|
||
if agent_id is not None and data.get("agent_id") != agent_id:
|
||
continue
|
||
if round_num is not None and data.get("round") != round_num:
|
||
continue
|
||
|
||
actions.append(AgentAction(
|
||
round_num=data.get("round", 0),
|
||
timestamp=data.get("timestamp", ""),
|
||
platform=record_platform,
|
||
agent_id=data.get("agent_id", 0),
|
||
agent_name=data.get("agent_name", ""),
|
||
action_type=data.get("action_type", ""),
|
||
action_args=data.get("action_args", {}),
|
||
result=data.get("result"),
|
||
success=data.get("success", True),
|
||
))
|
||
|
||
except json.JSONDecodeError:
|
||
continue
|
||
|
||
return actions
|
||
|
||
@classmethod
|
||
def get_all_actions(
|
||
cls,
|
||
simulation_id: str,
|
||
platform: Optional[str] = None,
|
||
agent_id: Optional[int] = None,
|
||
round_num: Optional[int] = None
|
||
) -> List[AgentAction]:
|
||
"""
|
||
Get complete action history of all platforms（no pagination limit）
|
||
|
||
Args:
|
||
simulation_id: simulationID
|
||
platform: filter platform（twitter/reddit）
|
||
agent_id: filterAgent
|
||
round_num: filter rounds
|
||
|
||
Returns:
|
||
Complete action list（sort by timestamp，newest first）
|
||
"""
|
||
sim_dir = os.path.join(cls.RUN_STATE_DIR, simulation_id)
|
||
actions = []
|
||
|
||
# read Twitter action file（Automatically set based on the file path platform as twitter）
|
||
twitter_actions_log = os.path.join(sim_dir, "twitter", "actions.jsonl")
|
||
if not platform or platform == "twitter":
|
||
actions.extend(cls._read_actions_from_file(
|
||
twitter_actions_log,
|
||
default_platform="twitter", # auto-populate platform field
|
||
platform_filter=platform,
|
||
agent_id=agent_id,
|
||
round_num=round_num
|
||
))
|
||
|
||
# read Reddit action file（Automatically set based on the file path platform as reddit）
|
||
reddit_actions_log = os.path.join(sim_dir, "reddit", "actions.jsonl")
|
||
if not platform or platform == "reddit":
|
||
actions.extend(cls._read_actions_from_file(
|
||
reddit_actions_log,
|
||
default_platform="reddit", # auto-populate platform field
|
||
platform_filter=platform,
|
||
agent_id=agent_id,
|
||
round_num=round_num
|
||
))
|
||
|
||
# If the per-platform file does not exist，Try to read the old single-file format
|
||
if not actions:
|
||
actions_log = os.path.join(sim_dir, "actions.jsonl")
|
||
actions = cls._read_actions_from_file(
|
||
actions_log,
|
||
default_platform=None, # The old format file should have platform field
|
||
platform_filter=platform,
|
||
agent_id=agent_id,
|
||
round_num=round_num
|
||
)
|
||
|
||
# sort by timestamp（newest first）
|
||
actions.sort(key=lambda x: x.timestamp, reverse=True)
|
||
|
||
return actions
|
||
|
||
@classmethod
|
||
def get_actions(
|
||
cls,
|
||
simulation_id: str,
|
||
limit: int = 100,
|
||
offset: int = 0,
|
||
platform: Optional[str] = None,
|
||
agent_id: Optional[int] = None,
|
||
round_num: Optional[int] = None
|
||
) -> List[AgentAction]:
|
||
"""
|
||
Get action history（with pagination）
|
||
|
||
Args:
|
||
simulation_id: simulationID
|
||
limit: quantity return limit
|
||
offset: offset
|
||
platform: filter platform
|
||
agent_id: filterAgent
|
||
round_num: filter rounds
|
||
|
||
Returns:
|
||
action list
|
||
"""
|
||
actions = cls.get_all_actions(
|
||
simulation_id=simulation_id,
|
||
platform=platform,
|
||
agent_id=agent_id,
|
||
round_num=round_num
|
||
)
|
||
|
||
# pagination
|
||
return actions[offset:offset + limit]
|
||
|
||
@classmethod
|
||
def get_timeline(
|
||
cls,
|
||
simulation_id: str,
|
||
start_round: int = 0,
|
||
end_round: Optional[int] = None
|
||
) -> List[Dict[str, Any]]:
|
||
"""
|
||
Get the simulation timeline（summarized by round）
|
||
|
||
Args:
|
||
simulation_id: simulationID
|
||
start_round: start round
|
||
end_round: end round
|
||
|
||
Returns:
|
||
Summary information per round
|
||
"""
|
||
actions = cls.get_actions(simulation_id, limit=10000)
|
||
|
||
# group by round
|
||
rounds: Dict[int, Dict[str, Any]] = {}
|
||
|
||
for action in actions:
|
||
round_num = action.round_num
|
||
|
||
if round_num < start_round:
|
||
continue
|
||
if end_round is not None and round_num > end_round:
|
||
continue
|
||
|
||
if round_num not in rounds:
|
||
rounds[round_num] = {
|
||
"round_num": round_num,
|
||
"twitter_actions": 0,
|
||
"reddit_actions": 0,
|
||
"active_agents": set(),
|
||
"action_types": {},
|
||
"first_action_time": action.timestamp,
|
||
"last_action_time": action.timestamp,
|
||
}
|
||
|
||
r = rounds[round_num]
|
||
|
||
if action.platform == "twitter":
|
||
r["twitter_actions"] += 1
|
||
else:
|
||
r["reddit_actions"] += 1
|
||
|
||
r["active_agents"].add(action.agent_id)
|
||
r["action_types"][action.action_type] = r["action_types"].get(action.action_type, 0) + 1
|
||
r["last_action_time"] = action.timestamp
|
||
|
||
# Convert to list
|
||
result = []
|
||
for round_num in sorted(rounds.keys()):
|
||
r = rounds[round_num]
|
||
result.append({
|
||
"round_num": round_num,
|
||
"twitter_actions": r["twitter_actions"],
|
||
"reddit_actions": r["reddit_actions"],
|
||
"total_actions": r["twitter_actions"] + r["reddit_actions"],
|
||
"active_agents_count": len(r["active_agents"]),
|
||
"active_agents": list(r["active_agents"]),
|
||
"action_types": r["action_types"],
|
||
"first_action_time": r["first_action_time"],
|
||
"last_action_time": r["last_action_time"],
|
||
})
|
||
|
||
return result
|
||
|
||
@classmethod
|
||
def get_agent_stats(cls, simulation_id: str) -> List[Dict[str, Any]]:
|
||
"""
|
||
get eachAgentstatistics
|
||
|
||
Returns:
|
||
Agentstatistics list
|
||
"""
|
||
actions = cls.get_actions(simulation_id, limit=10000)
|
||
|
||
agent_stats: Dict[int, Dict[str, Any]] = {}
|
||
|
||
for action in actions:
|
||
agent_id = action.agent_id
|
||
|
||
if agent_id not in agent_stats:
|
||
agent_stats[agent_id] = {
|
||
"agent_id": agent_id,
|
||
"agent_name": action.agent_name,
|
||
"total_actions": 0,
|
||
"twitter_actions": 0,
|
||
"reddit_actions": 0,
|
||
"action_types": {},
|
||
"first_action_time": action.timestamp,
|
||
"last_action_time": action.timestamp,
|
||
}
|
||
|
||
stats = agent_stats[agent_id]
|
||
stats["total_actions"] += 1
|
||
|
||
if action.platform == "twitter":
|
||
stats["twitter_actions"] += 1
|
||
else:
|
||
stats["reddit_actions"] += 1
|
||
|
||
stats["action_types"][action.action_type] = stats["action_types"].get(action.action_type, 0) + 1
|
||
stats["last_action_time"] = action.timestamp
|
||
|
||
# Sort by total action count
|
||
result = sorted(agent_stats.values(), key=lambda x: x["total_actions"], reverse=True)
|
||
|
||
return result
|
||
|
||
@classmethod
|
||
def cleanup_simulation_logs(cls, simulation_id: str) -> Dict[str, Any]:
|
||
"""
|
||
Clean up simulation runtime logs（Used to force-restart the simulation）
|
||
|
||
will delete the following files：
|
||
- run_state.json
|
||
- twitter/actions.jsonl
|
||
- reddit/actions.jsonl
|
||
- simulation.log
|
||
- stdout.log / stderr.log
|
||
- twitter_simulation.db（simulation database）
|
||
- reddit_simulation.db（simulation database）
|
||
- env_status.json（environment state）
|
||
|
||
Note：will not delete the configuration file（simulation_config.json） and profile file
|
||
|
||
Args:
|
||
simulation_id: simulationID
|
||
|
||
Returns:
|
||
cleanup result information
|
||
"""
|
||
import shutil
|
||
|
||
sim_dir = os.path.join(cls.RUN_STATE_DIR, simulation_id)
|
||
|
||
if not os.path.exists(sim_dir):
|
||
return {"success": True, "message": "Simulation directory does not exist，no need to clean up"}
|
||
|
||
cleaned_files = []
|
||
errors = []
|
||
|
||
# List of files to delete（including database files）
|
||
files_to_delete = [
|
||
"run_state.json",
|
||
"simulation.log",
|
||
"stdout.log",
|
||
"stderr.log",
|
||
"twitter_simulation.db", # Twitter platform database
|
||
"reddit_simulation.db", # Reddit platform database
|
||
"env_status.json", # environment state file
|
||
]
|
||
|
||
# List of directories to delete（contains action logs）
|
||
dirs_to_clean = ["twitter", "reddit"]
|
||
|
||
# delete files
|
||
for filename in files_to_delete:
|
||
file_path = os.path.join(sim_dir, filename)
|
||
if os.path.exists(file_path):
|
||
try:
|
||
os.remove(file_path)
|
||
cleaned_files.append(filename)
|
||
except Exception as e:
|
||
errors.append(f"delete {filename} failed: {str(e)}")
|
||
|
||
# Clean up action logs in the platform directory
|
||
for dir_name in dirs_to_clean:
|
||
dir_path = os.path.join(sim_dir, dir_name)
|
||
if os.path.exists(dir_path):
|
||
actions_file = os.path.join(dir_path, "actions.jsonl")
|
||
if os.path.exists(actions_file):
|
||
try:
|
||
os.remove(actions_file)
|
||
cleaned_files.append(f"{dir_name}/actions.jsonl")
|
||
except Exception as e:
|
||
errors.append(f"delete {dir_name}/actions.jsonl failed: {str(e)}")
|
||
|
||
# Clean up the in-memory running state
|
||
if simulation_id in cls._run_states:
|
||
del cls._run_states[simulation_id]
|
||
|
||
logger.info(f"Simulation log cleanup completed: {simulation_id}, delete files: {cleaned_files}")
|
||
|
||
return {
|
||
"success": len(errors) == 0,
|
||
"cleaned_files": cleaned_files,
|
||
"errors": errors if errors else None
|
||
}
|
||
|
||
# Flag to prevent repeated cleanup
|
||
_cleanup_done = False
|
||
|
||
@classmethod
|
||
def cleanup_all_simulations(cls):
|
||
"""
|
||
Clean up all running simulation processes
|
||
|
||
Called when the server shuts down，Ensure all child processes are terminated
|
||
"""
|
||
# prevent repeated cleanup
|
||
if cls._cleanup_done:
|
||
return
|
||
cls._cleanup_done = True
|
||
|
||
# Check whether there is anything to clean up（Avoid empty processes printing useless logs）
|
||
has_processes = bool(cls._processes)
|
||
has_updaters = bool(cls._graph_memory_enabled)
|
||
|
||
if not has_processes and not has_updaters:
|
||
return # No content to clean up，silently return
|
||
|
||
logger.info("Cleaning up all simulation processes...")
|
||
|
||
# First stop all graph memory updaters（stop_all logs will be printed internally）
|
||
try:
|
||
ZepGraphMemoryManager.stop_all()
|
||
except Exception as e:
|
||
logger.error(f"Failed to stop graph memory updater: {e}")
|
||
cls._graph_memory_enabled.clear()
|
||
|
||
# Copy the dictionary to avoid modification during iteration
|
||
processes = list(cls._processes.items())
|
||
|
||
for simulation_id, process in processes:
|
||
try:
|
||
if process.poll() is None: # Process is still running
|
||
logger.info(f"Terminate the simulation process: {simulation_id}, pid={process.pid}")
|
||
|
||
try:
|
||
# Use a cross-platform process termination method
|
||
cls._terminate_process(process, simulation_id, timeout=5)
|
||
except (ProcessLookupError, OSError):
|
||
# Process may have already been gone，try to terminate directly
|
||
try:
|
||
process.terminate()
|
||
process.wait(timeout=3)
|
||
except Exception:
|
||
process.kill()
|
||
|
||
# update run_state.json
|
||
state = cls.get_run_state(simulation_id)
|
||
if state:
|
||
state.runner_status = RunnerStatus.STOPPED
|
||
state.twitter_running = False
|
||
state.reddit_running = False
|
||
state.completed_at = datetime.now().isoformat()
|
||
state.error = "Server shutdown，Simulation has been terminated"
|
||
cls._save_run_state(state)
|
||
|
||
# also update state.json，set the status to stopped
|
||
try:
|
||
sim_dir = os.path.join(cls.RUN_STATE_DIR, simulation_id)
|
||
state_file = os.path.join(sim_dir, "state.json")
|
||
logger.info(f"try to update state.json: {state_file}")
|
||
if os.path.exists(state_file):
|
||
with open(state_file, 'r', encoding='utf-8') as f:
|
||
state_data = json.load(f)
|
||
state_data['status'] = 'stopped'
|
||
state_data['updated_at'] = datetime.now().isoformat()
|
||
with open(state_file, 'w', encoding='utf-8') as f:
|
||
json.dump(state_data, f, indent=2, ensure_ascii=False)
|
||
logger.info(f"updated state.json status is stopped: {simulation_id}")
|
||
else:
|
||
logger.warning(f"state.json does not exist: {state_file}")
|
||
except Exception as state_err:
|
||
logger.warning(f"update state.json failed: {simulation_id}, error={state_err}")
|
||
|
||
except Exception as e:
|
||
logger.error(f"failed to clean up the process: {simulation_id}, error={e}")
|
||
|
||
# clean up file handles
|
||
for simulation_id, file_handle in list(cls._stdout_files.items()):
|
||
try:
|
||
if file_handle:
|
||
file_handle.close()
|
||
except Exception:
|
||
pass
|
||
cls._stdout_files.clear()
|
||
|
||
for simulation_id, file_handle in list(cls._stderr_files.items()):
|
||
try:
|
||
if file_handle:
|
||
file_handle.close()
|
||
except Exception:
|
||
pass
|
||
cls._stderr_files.clear()
|
||
|
||
# Clean up in-memory state
|
||
cls._processes.clear()
|
||
cls._action_queues.clear()
|
||
|
||
logger.info("Simulation process cleanup completed")
|
||
|
||
@classmethod
|
||
def register_cleanup(cls):
|
||
"""
|
||
Register cleanup function
|
||
|
||
in Flask Called at application startup，Ensure all simulation processes are cleaned up when the server shuts down
|
||
"""
|
||
global _cleanup_registered
|
||
|
||
if _cleanup_registered:
|
||
return
|
||
|
||
# Flask debug mode，only in reloader register cleanup in the child process（the process actually running the application）
|
||
# WERKZEUG_RUN_MAIN=true indicates it is reloader child process
|
||
# if it is not debug mode，this environment variable is not set，also needs to register
|
||
is_reloader_process = os.environ.get('WERKZEUG_RUN_MAIN') == 'true'
|
||
is_debug_mode = os.environ.get('FLASK_DEBUG') == '1' or os.environ.get('WERKZEUG_RUN_MAIN') is not None
|
||
|
||
# in debug mode，only in reloader register in the child process；non- debug always register in this mode
|
||
if is_debug_mode and not is_reloader_process:
|
||
_cleanup_registered = True # Mark as registered，Prevent the child process from trying again
|
||
return
|
||
|
||
# Save the original signal handler
|
||
original_sigint = signal.getsignal(signal.SIGINT)
|
||
original_sigterm = signal.getsignal(signal.SIGTERM)
|
||
# SIGHUP only in Unix system exists（macOS/Linux），Windows have no
|
||
original_sighup = None
|
||
has_sighup = hasattr(signal, 'SIGHUP')
|
||
if has_sighup:
|
||
original_sighup = signal.getsignal(signal.SIGHUP)
|
||
|
||
def cleanup_handler(signum=None, frame=None):
|
||
"""signal handler：first clean up the simulation processes，then call the original handler"""
|
||
# Only print logs when there are processes to clean up
|
||
if cls._processes or cls._graph_memory_enabled:
|
||
logger.info(f"received signal {signum}，start cleanup...")
|
||
cls.cleanup_all_simulations()
|
||
|
||
# Call the original signal handler，let Flask exit normally
|
||
if signum == signal.SIGINT and callable(original_sigint):
|
||
original_sigint(signum, frame)
|
||
elif signum == signal.SIGTERM and callable(original_sigterm):
|
||
original_sigterm(signum, frame)
|
||
elif has_sighup and signum == signal.SIGHUP:
|
||
# SIGHUP: sent when the terminal is closed
|
||
if callable(original_sighup):
|
||
original_sighup(signum, frame)
|
||
else:
|
||
# default behavior：exit normally
|
||
sys.exit(0)
|
||
else:
|
||
# If the original handler is not callable（e.g. SIG_DFL），use the default behavior
|
||
raise KeyboardInterrupt
|
||
|
||
# register atexit handler（as a fallback）
|
||
atexit.register(cls.cleanup_all_simulations)
|
||
|
||
# Register signal handler（only in the main thread）
|
||
try:
|
||
# SIGTERM: kill command default signal
|
||
signal.signal(signal.SIGTERM, cleanup_handler)
|
||
# SIGINT: Ctrl+C
|
||
signal.signal(signal.SIGINT, cleanup_handler)
|
||
# SIGHUP: terminal close (Unix systems only)
|
||
if has_sighup:
|
||
signal.signal(signal.SIGHUP, cleanup_handler)
|
||
except ValueError:
|
||
# not in the main thread，can only use atexit
|
||
logger.warning("Cannot register signal handler（not in the main thread），only use atexit")
|
||
|
||
_cleanup_registered = True
|
||
|
||
@classmethod
|
||
def get_running_simulations(cls) -> List[str]:
|
||
"""
|
||
Get all running simulationsIDlist
|
||
"""
|
||
running = []
|
||
for sim_id, process in cls._processes.items():
|
||
if process.poll() is None:
|
||
running.append(sim_id)
|
||
return running
|
||
|
||
# ============== Interview function ==============
|
||
|
||
@classmethod
|
||
def check_env_alive(cls, simulation_id: str) -> bool:
|
||
"""
|
||
Check whether the simulation environment is alive（can receiveInterviewcommand）
|
||
|
||
Args:
|
||
simulation_id: simulationID
|
||
|
||
Returns:
|
||
True indicates the environment is alive，False indicates the environment has been closed
|
||
"""
|
||
sim_dir = os.path.join(cls.RUN_STATE_DIR, simulation_id)
|
||
if not os.path.exists(sim_dir):
|
||
return False
|
||
|
||
ipc_client = SimulationIPCClient(sim_dir)
|
||
return ipc_client.check_env_alive()
|
||
|
||
@classmethod
|
||
def get_env_status_detail(cls, simulation_id: str) -> Dict[str, Any]:
|
||
"""
|
||
Get detailed status information of the simulation environment
|
||
|
||
Args:
|
||
simulation_id: simulationID
|
||
|
||
Returns:
|
||
status detail dictionary，contains status, twitter_available, reddit_available, timestamp
|
||
"""
|
||
sim_dir = os.path.join(cls.RUN_STATE_DIR, simulation_id)
|
||
status_file = os.path.join(sim_dir, "env_status.json")
|
||
|
||
default_status = {
|
||
"status": "stopped",
|
||
"twitter_available": False,
|
||
"reddit_available": False,
|
||
"timestamp": None
|
||
}
|
||
|
||
if not os.path.exists(status_file):
|
||
return default_status
|
||
|
||
try:
|
||
with open(status_file, 'r', encoding='utf-8') as f:
|
||
status = json.load(f)
|
||
return {
|
||
"status": status.get("status", "stopped"),
|
||
"twitter_available": status.get("twitter_available", False),
|
||
"reddit_available": status.get("reddit_available", False),
|
||
"timestamp": status.get("timestamp")
|
||
}
|
||
except (json.JSONDecodeError, OSError):
|
||
return default_status
|
||
|
||
@classmethod
|
||
def interview_agent(
|
||
cls,
|
||
simulation_id: str,
|
||
agent_id: int,
|
||
prompt: str,
|
||
platform: str = None,
|
||
timeout: float = 60.0
|
||
) -> Dict[str, Any]:
|
||
"""
|
||
interview a singleAgent
|
||
|
||
Args:
|
||
simulation_id: simulationID
|
||
agent_id: Agent ID
|
||
prompt: interview question
|
||
platform: specify platform（optional）
|
||
- "twitter": only interviewTwitterplatform
|
||
- "reddit": only interviewRedditplatform
|
||
- None: In dual-platform simulations, interview both platforms simultaneously，return integrated result
|
||
timeout: timeout（seconds）
|
||
|
||
Returns:
|
||
interview result dictionary
|
||
|
||
Raises:
|
||
ValueError: Simulation does not exist or environment is not running
|
||
TimeoutError: Waiting for response timed out
|
||
"""
|
||
sim_dir = os.path.join(cls.RUN_STATE_DIR, simulation_id)
|
||
if not os.path.exists(sim_dir):
|
||
raise ValueError(f"Simulation does not exist: {simulation_id}")
|
||
|
||
ipc_client = SimulationIPCClient(sim_dir)
|
||
|
||
if not ipc_client.check_env_alive():
|
||
raise ValueError(f"Simulation environment not running or has been closed，cannot executeInterview: {simulation_id}")
|
||
|
||
logger.info(f"sendInterviewcommand: simulation_id={simulation_id}, agent_id={agent_id}, platform={platform}")
|
||
|
||
response = ipc_client.send_interview(
|
||
agent_id=agent_id,
|
||
prompt=prompt,
|
||
platform=platform,
|
||
timeout=timeout
|
||
)
|
||
|
||
if response.status.value == "completed":
|
||
return {
|
||
"success": True,
|
||
"agent_id": agent_id,
|
||
"prompt": prompt,
|
||
"result": response.result,
|
||
"timestamp": response.timestamp
|
||
}
|
||
else:
|
||
return {
|
||
"success": False,
|
||
"agent_id": agent_id,
|
||
"prompt": prompt,
|
||
"error": response.error,
|
||
"timestamp": response.timestamp
|
||
}
|
||
|
||
@classmethod
|
||
def interview_agents_batch(
|
||
cls,
|
||
simulation_id: str,
|
||
interviews: List[Dict[str, Any]],
|
||
platform: str = None,
|
||
timeout: float = 120.0
|
||
) -> Dict[str, Any]:
|
||
"""
|
||
Batch interview multipleAgent
|
||
|
||
Args:
|
||
simulation_id: simulationID
|
||
interviews: interview list，Each element contains {"agent_id": int, "prompt": str, "platform": str(optional)}
|
||
platform: default platform（optional，will be overridden by each interview item'splatformoverride）
|
||
- "twitter": by default only interviewTwitterplatform
|
||
- "reddit": by default only interviewRedditplatform
|
||
- None: In dual-platform simulations, eachAgentInterview both platforms simultaneously
|
||
timeout: timeout（seconds）
|
||
|
||
Returns:
|
||
Batch interview result dictionary
|
||
|
||
Raises:
|
||
ValueError: Simulation does not exist or environment is not running
|
||
TimeoutError: Waiting for response timed out
|
||
"""
|
||
sim_dir = os.path.join(cls.RUN_STATE_DIR, simulation_id)
|
||
if not os.path.exists(sim_dir):
|
||
raise ValueError(f"Simulation does not exist: {simulation_id}")
|
||
|
||
ipc_client = SimulationIPCClient(sim_dir)
|
||
|
||
if not ipc_client.check_env_alive():
|
||
raise ValueError(f"Simulation environment not running or has been closed，cannot executeInterview: {simulation_id}")
|
||
|
||
logger.info(f"send batchInterviewcommand: simulation_id={simulation_id}, count={len(interviews)}, platform={platform}")
|
||
|
||
response = ipc_client.send_batch_interview(
|
||
interviews=interviews,
|
||
platform=platform,
|
||
timeout=timeout
|
||
)
|
||
|
||
if response.status.value == "completed":
|
||
return {
|
||
"success": True,
|
||
"interviews_count": len(interviews),
|
||
"result": response.result,
|
||
"timestamp": response.timestamp
|
||
}
|
||
else:
|
||
return {
|
||
"success": False,
|
||
"interviews_count": len(interviews),
|
||
"error": response.error,
|
||
"timestamp": response.timestamp
|
||
}
|
||
|
||
@classmethod
|
||
def interview_all_agents(
|
||
cls,
|
||
simulation_id: str,
|
||
prompt: str,
|
||
platform: str = None,
|
||
timeout: float = 180.0
|
||
) -> Dict[str, Any]:
|
||
"""
|
||
interview allAgent（global interview）
|
||
|
||
Use the same question to interview all simulatedAgent
|
||
|
||
Args:
|
||
simulation_id: simulationID
|
||
prompt: interview question（allAgentuse the same question）
|
||
platform: specify platform（optional）
|
||
- "twitter": only interviewTwitterplatform
|
||
- "reddit": only interviewRedditplatform
|
||
- None: In dual-platform simulations, eachAgentInterview both platforms simultaneously
|
||
timeout: timeout（seconds）
|
||
|
||
Returns:
|
||
Global interview result dictionary
|
||
"""
|
||
sim_dir = os.path.join(cls.RUN_STATE_DIR, simulation_id)
|
||
if not os.path.exists(sim_dir):
|
||
raise ValueError(f"Simulation does not exist: {simulation_id}")
|
||
|
||
# Get all from the configuration fileAgentinformation
|
||
config_path = os.path.join(sim_dir, "simulation_config.json")
|
||
if not os.path.exists(config_path):
|
||
raise ValueError(f"Simulation configuration does not exist: {simulation_id}")
|
||
|
||
with open(config_path, 'r', encoding='utf-8') as f:
|
||
config = json.load(f)
|
||
|
||
agent_configs = config.get("agent_configs", [])
|
||
if not agent_configs:
|
||
raise ValueError(f"Simulation configuration has noAgent: {simulation_id}")
|
||
|
||
# Build the batch interview list
|
||
interviews = []
|
||
for agent_config in agent_configs:
|
||
agent_id = agent_config.get("agent_id")
|
||
if agent_id is not None:
|
||
interviews.append({
|
||
"agent_id": agent_id,
|
||
"prompt": prompt
|
||
})
|
||
|
||
logger.info(f"send globalInterviewcommand: simulation_id={simulation_id}, agent_count={len(interviews)}, platform={platform}")
|
||
|
||
return cls.interview_agents_batch(
|
||
simulation_id=simulation_id,
|
||
interviews=interviews,
|
||
platform=platform,
|
||
timeout=timeout
|
||
)
|
||
|
||
@classmethod
|
||
def close_simulation_env(
|
||
cls,
|
||
simulation_id: str,
|
||
timeout: float = 30.0
|
||
) -> Dict[str, Any]:
|
||
"""
|
||
close the simulation environment（rather than stopping the simulation process）
|
||
|
||
Send a close-environment command to the simulation，cause it to gracefully exit the wait-for-command mode
|
||
|
||
Args:
|
||
simulation_id: simulationID
|
||
timeout: timeout（seconds）
|
||
|
||
Returns:
|
||
operation result dictionary
|
||
"""
|
||
sim_dir = os.path.join(cls.RUN_STATE_DIR, simulation_id)
|
||
if not os.path.exists(sim_dir):
|
||
raise ValueError(f"Simulation does not exist: {simulation_id}")
|
||
|
||
ipc_client = SimulationIPCClient(sim_dir)
|
||
|
||
if not ipc_client.check_env_alive():
|
||
return {
|
||
"success": True,
|
||
"message": "Environment has already been closed"
|
||
}
|
||
|
||
logger.info(f"Send the close-environment command: simulation_id={simulation_id}")
|
||
|
||
try:
|
||
response = ipc_client.send_close_env(timeout=timeout)
|
||
|
||
return {
|
||
"success": response.status.value == "completed",
|
||
"message": "Environment close command has been sent",
|
||
"result": response.result,
|
||
"timestamp": response.timestamp
|
||
}
|
||
except TimeoutError:
|
||
# Timeout may be because the environment is shutting down
|
||
return {
|
||
"success": True,
|
||
"message": "Environment close command has been sent（Waiting for response timed out，Environment may be shutting down）"
|
||
}
|
||
|
||
@classmethod
|
||
def _get_interview_history_from_db(
|
||
cls,
|
||
db_path: str,
|
||
platform_name: str,
|
||
agent_id: Optional[int] = None,
|
||
limit: int = 100
|
||
) -> List[Dict[str, Any]]:
|
||
"""Get from a single databaseInterviewhistory"""
|
||
import sqlite3
|
||
|
||
if not os.path.exists(db_path):
|
||
return []
|
||
|
||
results = []
|
||
|
||
try:
|
||
conn = sqlite3.connect(db_path)
|
||
cursor = conn.cursor()
|
||
|
||
if agent_id is not None:
|
||
cursor.execute("""
|
||
SELECT user_id, info, created_at
|
||
FROM trace
|
||
WHERE action = 'interview' AND user_id = ?
|
||
ORDER BY created_at DESC
|
||
LIMIT ?
|
||
""", (agent_id, limit))
|
||
else:
|
||
cursor.execute("""
|
||
SELECT user_id, info, created_at
|
||
FROM trace
|
||
WHERE action = 'interview'
|
||
ORDER BY created_at DESC
|
||
LIMIT ?
|
||
""", (limit,))
|
||
|
||
for user_id, info_json, created_at in cursor.fetchall():
|
||
try:
|
||
info = json.loads(info_json) if info_json else {}
|
||
except json.JSONDecodeError:
|
||
info = {"raw": info_json}
|
||
|
||
results.append({
|
||
"agent_id": user_id,
|
||
"response": info.get("response", info),
|
||
"prompt": info.get("prompt", ""),
|
||
"timestamp": created_at,
|
||
"platform": platform_name
|
||
})
|
||
|
||
conn.close()
|
||
|
||
except Exception as e:
|
||
logger.error(f"readInterviewhistory failed ({platform_name}): {e}")
|
||
|
||
return results
|
||
|
||
@classmethod
|
||
def get_interview_history(
|
||
cls,
|
||
simulation_id: str,
|
||
platform: str = None,
|
||
agent_id: Optional[int] = None,
|
||
limit: int = 100
|
||
) -> List[Dict[str, Any]]:
|
||
"""
|
||
getInterviewhistory record（read from database）
|
||
|
||
Args:
|
||
simulation_id: simulationID
|
||
platform: platform type（reddit/twitter/None）
|
||
- "reddit": only getRedditplatform history
|
||
- "twitter": only getTwitterplatform history
|
||
- None: Get history of both platforms
|
||
agent_id: specifyAgent ID（optional，only get theAgenthistory）
|
||
limit: Quantity limit per platform return
|
||
|
||
Returns:
|
||
Interviewhistory record list
|
||
"""
|
||
sim_dir = os.path.join(cls.RUN_STATE_DIR, simulation_id)
|
||
|
||
results = []
|
||
|
||
# Determine the platforms to query
|
||
if platform in ("reddit", "twitter"):
|
||
platforms = [platform]
|
||
else:
|
||
# not specifiedplatformtime，Query both platforms
|
||
platforms = ["twitter", "reddit"]
|
||
|
||
for p in platforms:
|
||
db_path = os.path.join(sim_dir, f"{p}_simulation.db")
|
||
platform_results = cls._get_interview_history_from_db(
|
||
db_path=db_path,
|
||
platform_name=p,
|
||
agent_id=agent_id,
|
||
limit=limit
|
||
)
|
||
results.extend(platform_results)
|
||
|
||
# Sort by time in descending order
|
||
results.sort(key=lambda x: x.get("timestamp", ""), reverse=True)
|
||
|
||
# If multiple platforms are queried，limit total count
|
||
if len(platforms) > 1 and len(results) > limit:
|
||
results = results[:limit]
|
||
|
||
return results
|
||
|