485 lines
15 KiB
Python
485 lines
15 KiB
Python
from types import SimpleNamespace
|
|
import json
|
|
|
|
import pytest
|
|
from flask import Flask
|
|
|
|
from app.api import simulation as simulation_api
|
|
from app.services import simulation_runner as runner_module
|
|
from app.services.simulation_manager import SimulationStatus
|
|
from app.services.simulation_runner import (
|
|
RunnerStatus,
|
|
SimulationRunState,
|
|
SimulationRunner,
|
|
SimulationStopPending,
|
|
)
|
|
|
|
|
|
def test_manual_stop_surfaces_graph_ingestion_failure(monkeypatch):
|
|
state = SimulationRunState(
|
|
simulation_id="sim-1",
|
|
runner_status=RunnerStatus.RUNNING,
|
|
)
|
|
saved = []
|
|
monkeypatch.setattr(
|
|
SimulationRunner,
|
|
"get_run_state",
|
|
classmethod(lambda _cls, _simulation_id: state),
|
|
)
|
|
monkeypatch.setattr(
|
|
SimulationRunner,
|
|
"_save_run_state",
|
|
classmethod(lambda _cls, value: saved.append(value.runner_status)),
|
|
)
|
|
monkeypatch.setattr(
|
|
runner_module.ZepGraphMemoryManager,
|
|
"stop_updater",
|
|
classmethod(
|
|
lambda _cls, _simulation_id: (_ for _ in ()).throw(
|
|
RuntimeError("ingestion incomplete")
|
|
)
|
|
),
|
|
)
|
|
SimulationRunner._processes.pop("sim-1", None)
|
|
SimulationRunner._graph_memory_enabled["sim-1"] = True
|
|
|
|
try:
|
|
with pytest.raises(RuntimeError, match="ingestion incomplete"):
|
|
SimulationRunner.stop_simulation("sim-1")
|
|
|
|
assert state.runner_status == RunnerStatus.FAILED
|
|
assert "ingestion incomplete" in state.error
|
|
assert saved[-1] == RunnerStatus.FAILED
|
|
finally:
|
|
SimulationRunner._graph_memory_enabled.pop("sim-1", None)
|
|
SimulationRunner._manual_stop_requests.discard("sim-1")
|
|
|
|
|
|
def test_platform_completion_does_not_publish_terminal_success_before_barrier(
|
|
monkeypatch, tmp_path
|
|
):
|
|
simulation_id = "sim-1"
|
|
sim_dir = tmp_path / simulation_id / "twitter"
|
|
sim_dir.mkdir(parents=True)
|
|
log_path = sim_dir / "actions.jsonl"
|
|
log_path.write_text(
|
|
'{"event_type":"simulation_end","total_rounds":1,"total_actions":0}\n',
|
|
encoding="utf-8",
|
|
)
|
|
monkeypatch.setattr(SimulationRunner, "RUN_STATE_DIR", str(tmp_path))
|
|
state = SimulationRunState(
|
|
simulation_id=simulation_id,
|
|
runner_status=RunnerStatus.RUNNING,
|
|
twitter_running=True,
|
|
)
|
|
|
|
SimulationRunner._read_action_log(str(log_path), 0, state, "twitter")
|
|
|
|
assert state.twitter_completed is True
|
|
assert state.runner_status == RunnerStatus.RUNNING
|
|
|
|
|
|
def test_manual_stop_timeout_leaves_monitor_owned_state_stopping(monkeypatch):
|
|
state = SimulationRunState(
|
|
simulation_id="sim-timeout",
|
|
runner_status=RunnerStatus.RUNNING,
|
|
)
|
|
|
|
class Monitor:
|
|
def join(self, timeout):
|
|
assert timeout >= 30
|
|
|
|
def is_alive(self):
|
|
return True
|
|
|
|
monkeypatch.setattr(
|
|
SimulationRunner,
|
|
"get_run_state",
|
|
classmethod(lambda _cls, _simulation_id: state),
|
|
)
|
|
monkeypatch.setattr(
|
|
SimulationRunner,
|
|
"_save_run_state",
|
|
classmethod(lambda _cls, _state: None),
|
|
)
|
|
SimulationRunner._monitor_threads["sim-timeout"] = Monitor()
|
|
SimulationRunner._processes.pop("sim-timeout", None)
|
|
SimulationRunner._graph_memory_enabled.pop("sim-timeout", None)
|
|
|
|
try:
|
|
with pytest.raises(TimeoutError, match="仍在停止中"):
|
|
SimulationRunner.stop_simulation("sim-timeout")
|
|
assert state.runner_status == RunnerStatus.STOPPING
|
|
finally:
|
|
SimulationRunner._monitor_threads.pop("sim-timeout", None)
|
|
SimulationRunner._manual_stop_requests.discard("sim-timeout")
|
|
|
|
|
|
def test_failed_ingestion_finalization_can_be_retried(monkeypatch):
|
|
state = SimulationRunState(
|
|
simulation_id="sim-retry",
|
|
runner_status=RunnerStatus.FAILED,
|
|
error="first drain timed out",
|
|
)
|
|
monkeypatch.setattr(
|
|
SimulationRunner,
|
|
"get_run_state",
|
|
classmethod(lambda _cls, _simulation_id: state),
|
|
)
|
|
monkeypatch.setattr(
|
|
SimulationRunner,
|
|
"_save_run_state",
|
|
classmethod(lambda _cls, _state: None),
|
|
)
|
|
monkeypatch.setattr(
|
|
runner_module.ZepGraphMemoryManager,
|
|
"get_updater",
|
|
classmethod(lambda _cls, _simulation_id: object()),
|
|
)
|
|
monkeypatch.setattr(
|
|
runner_module.ZepGraphMemoryManager,
|
|
"stop_updater",
|
|
classmethod(lambda _cls, _simulation_id: None),
|
|
)
|
|
SimulationRunner._graph_memory_enabled["sim-retry"] = True
|
|
SimulationRunner._monitor_threads.pop("sim-retry", None)
|
|
|
|
try:
|
|
result = SimulationRunner.stop_simulation("sim-retry")
|
|
assert result.runner_status == RunnerStatus.STOPPED
|
|
assert result.error is None
|
|
finally:
|
|
SimulationRunner._graph_memory_enabled.pop("sim-retry", None)
|
|
SimulationRunner._manual_stop_requests.discard("sim-retry")
|
|
|
|
|
|
def test_stop_api_keeps_pending_finalization_out_of_failed_state(monkeypatch):
|
|
simulation = SimpleNamespace(status=SimulationStatus.STOPPING, error=None)
|
|
saved = []
|
|
monkeypatch.setattr(
|
|
simulation_api.SimulationRunner,
|
|
"stop_simulation",
|
|
classmethod(
|
|
lambda _cls, _simulation_id: (_ for _ in ()).throw(
|
|
SimulationStopPending("still draining")
|
|
)
|
|
),
|
|
)
|
|
monkeypatch.setattr(
|
|
simulation_api,
|
|
"SimulationManager",
|
|
lambda: SimpleNamespace(
|
|
get_simulation=lambda _simulation_id: simulation,
|
|
_save_simulation_state=lambda state: saved.append(state.status),
|
|
),
|
|
)
|
|
|
|
app = Flask(__name__)
|
|
with app.test_request_context(
|
|
"/api/simulation/stop",
|
|
method="POST",
|
|
json={"simulation_id": "sim-pending"},
|
|
):
|
|
response, status = simulation_api.stop_simulation()
|
|
|
|
assert status == 202
|
|
assert response.get_json()["pending"] is True
|
|
assert simulation.status == SimulationStatus.STOPPING
|
|
assert saved == []
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"field",
|
|
["force", "enable_graph_memory_update"],
|
|
)
|
|
def test_simulation_start_rejects_string_booleans(field):
|
|
app = Flask(__name__)
|
|
with app.test_request_context(
|
|
"/api/simulation/start",
|
|
method="POST",
|
|
json={"simulation_id": "sim-1", field: "false"},
|
|
):
|
|
response, status = simulation_api.start_simulation()
|
|
|
|
assert status == 400
|
|
assert "JSON boolean" in response.get_json()["error"]
|
|
|
|
|
|
def test_force_restart_does_not_continue_while_old_ingestion_is_pending(monkeypatch):
|
|
simulation = SimpleNamespace(
|
|
simulation_id="sim-1",
|
|
project_id="proj-1",
|
|
graph_id="graph-1",
|
|
status=SimulationStatus.STOPPING,
|
|
)
|
|
cleanup_called = []
|
|
monkeypatch.setattr(
|
|
simulation_api,
|
|
"SimulationManager",
|
|
lambda: SimpleNamespace(
|
|
get_simulation=lambda _simulation_id: simulation,
|
|
_save_simulation_state=lambda _state: None,
|
|
),
|
|
)
|
|
monkeypatch.setattr(
|
|
simulation_api,
|
|
"_check_simulation_prepared",
|
|
lambda _simulation_id: (True, {}),
|
|
)
|
|
monkeypatch.setattr(
|
|
simulation_api.SimulationRunner,
|
|
"get_run_state",
|
|
classmethod(
|
|
lambda _cls, _simulation_id: SimpleNamespace(
|
|
runner_status=RunnerStatus.STOPPING
|
|
)
|
|
),
|
|
)
|
|
monkeypatch.setattr(
|
|
simulation_api.SimulationRunner,
|
|
"stop_simulation",
|
|
classmethod(
|
|
lambda _cls, _simulation_id: (_ for _ in ()).throw(
|
|
SimulationStopPending("still draining")
|
|
)
|
|
),
|
|
)
|
|
monkeypatch.setattr(
|
|
simulation_api.SimulationRunner,
|
|
"cleanup_simulation_logs",
|
|
classmethod(
|
|
lambda _cls, _simulation_id: cleanup_called.append(True)
|
|
),
|
|
)
|
|
monkeypatch.setattr(
|
|
simulation_api.ZepGraphMemoryManager,
|
|
"get_updater",
|
|
classmethod(lambda _cls, _simulation_id: object()),
|
|
)
|
|
|
|
app = Flask(__name__)
|
|
with app.test_request_context(
|
|
"/api/simulation/start",
|
|
method="POST",
|
|
json={"simulation_id": "sim-1", "force": True},
|
|
):
|
|
response, status = simulation_api.start_simulation()
|
|
|
|
assert status == 409
|
|
assert response.get_json()["pending"] is True
|
|
assert cleanup_called == []
|
|
|
|
|
|
def test_monitor_start_failure_terminates_the_spawned_process(monkeypatch, tmp_path):
|
|
simulation_id = "sim-start-failure"
|
|
sim_dir = tmp_path / "runs" / simulation_id
|
|
scripts_dir = tmp_path / "scripts"
|
|
sim_dir.mkdir(parents=True)
|
|
scripts_dir.mkdir()
|
|
(sim_dir / "simulation_config.json").write_text(
|
|
json.dumps({
|
|
"time_config": {
|
|
"total_simulation_hours": 1,
|
|
"minutes_per_round": 60,
|
|
}
|
|
}),
|
|
encoding="utf-8",
|
|
)
|
|
(scripts_dir / "run_twitter_simulation.py").write_text("pass\n", encoding="utf-8")
|
|
|
|
class Process:
|
|
pid = 123
|
|
|
|
def poll(self):
|
|
return None
|
|
|
|
class BrokenThread:
|
|
def __init__(self, **_kwargs):
|
|
pass
|
|
|
|
def start(self):
|
|
raise RuntimeError("monitor failed")
|
|
|
|
terminated = []
|
|
monkeypatch.setattr(SimulationRunner, "RUN_STATE_DIR", str(tmp_path / "runs"))
|
|
monkeypatch.setattr(SimulationRunner, "SCRIPTS_DIR", str(scripts_dir))
|
|
monkeypatch.setattr(runner_module.subprocess, "Popen", lambda *_args, **_kwargs: Process())
|
|
monkeypatch.setattr(runner_module.threading, "Thread", BrokenThread)
|
|
monkeypatch.setattr(
|
|
SimulationRunner,
|
|
"_terminate_process",
|
|
classmethod(lambda _cls, _process, sim_id: terminated.append(sim_id)),
|
|
)
|
|
monkeypatch.setattr(
|
|
SimulationRunner,
|
|
"_sync_simulation_status",
|
|
classmethod(lambda _cls, *_args, **_kwargs: None),
|
|
)
|
|
|
|
try:
|
|
with pytest.raises(RuntimeError, match="monitor failed"):
|
|
SimulationRunner.start_simulation(
|
|
simulation_id,
|
|
platform="twitter",
|
|
enable_graph_memory_update=False,
|
|
)
|
|
assert terminated == [simulation_id]
|
|
assert simulation_id not in SimulationRunner._processes
|
|
assert simulation_id not in SimulationRunner._action_queues
|
|
assert simulation_id not in SimulationRunner._stdout_files
|
|
finally:
|
|
SimulationRunner._run_states.pop(simulation_id, None)
|
|
SimulationRunner._processes.pop(simulation_id, None)
|
|
SimulationRunner._action_queues.pop(simulation_id, None)
|
|
SimulationRunner._stdout_files.pop(simulation_id, None)
|
|
SimulationRunner._stderr_files.pop(simulation_id, None)
|
|
SimulationRunner._graph_memory_enabled.pop(simulation_id, None)
|
|
|
|
|
|
def test_shutdown_terminates_producer_before_tail_read_and_updater_drain(
|
|
monkeypatch,
|
|
):
|
|
simulation_id = "sim-shutdown-order"
|
|
state = SimulationRunState(
|
|
simulation_id=simulation_id,
|
|
runner_status=RunnerStatus.RUNNING,
|
|
)
|
|
events = []
|
|
|
|
class Process:
|
|
pid = 123
|
|
stopped = False
|
|
|
|
def poll(self):
|
|
return 0 if self.stopped else None
|
|
|
|
process = Process()
|
|
|
|
class Monitor:
|
|
alive = True
|
|
|
|
def join(self, timeout):
|
|
assert timeout >= 30
|
|
events.extend(["tail-read", "updater-drain"])
|
|
state.runner_status = RunnerStatus.STOPPED
|
|
SimulationRunner._graph_memory_enabled.pop(simulation_id, None)
|
|
self.alive = False
|
|
|
|
def is_alive(self):
|
|
return self.alive
|
|
|
|
monkeypatch.setattr(
|
|
SimulationRunner,
|
|
"get_run_state",
|
|
classmethod(lambda _cls, _simulation_id: state),
|
|
)
|
|
monkeypatch.setattr(
|
|
SimulationRunner,
|
|
"_save_run_state",
|
|
classmethod(lambda _cls, _state: None),
|
|
)
|
|
monkeypatch.setattr(
|
|
SimulationRunner,
|
|
"_sync_simulation_status",
|
|
classmethod(lambda _cls, *_args, **_kwargs: None),
|
|
)
|
|
monkeypatch.setattr(
|
|
SimulationRunner,
|
|
"_terminate_process",
|
|
classmethod(
|
|
lambda _cls, proc, _simulation_id, **_kwargs: (
|
|
events.append("producer-terminate"),
|
|
setattr(proc, "stopped", True),
|
|
)
|
|
),
|
|
)
|
|
monkeypatch.setattr(
|
|
runner_module.ZepGraphMemoryManager,
|
|
"get_simulation_ids",
|
|
classmethod(lambda _cls: [simulation_id]),
|
|
)
|
|
updater = object()
|
|
monkeypatch.setattr(
|
|
runner_module.ZepGraphMemoryManager,
|
|
"get_updater",
|
|
classmethod(lambda _cls, _simulation_id: updater),
|
|
)
|
|
|
|
SimulationRunner._cleanup_done = False
|
|
SimulationRunner._processes[simulation_id] = process
|
|
SimulationRunner._monitor_threads[simulation_id] = Monitor()
|
|
SimulationRunner._graph_memory_enabled[simulation_id] = True
|
|
try:
|
|
SimulationRunner.cleanup_all_simulations()
|
|
assert events == [
|
|
"producer-terminate",
|
|
"tail-read",
|
|
"updater-drain",
|
|
]
|
|
assert state.runner_status == RunnerStatus.STOPPED
|
|
finally:
|
|
SimulationRunner._cleanup_done = False
|
|
SimulationRunner._processes.pop(simulation_id, None)
|
|
SimulationRunner._monitor_threads.pop(simulation_id, None)
|
|
SimulationRunner._graph_memory_enabled.pop(simulation_id, None)
|
|
SimulationRunner._manual_stop_requests.discard(simulation_id)
|
|
|
|
|
|
def test_shutdown_drain_failure_remains_failed_and_retryable(monkeypatch):
|
|
simulation_id = "sim-shutdown-failure"
|
|
state = SimulationRunState(
|
|
simulation_id=simulation_id,
|
|
runner_status=RunnerStatus.RUNNING,
|
|
)
|
|
updater = object()
|
|
|
|
monkeypatch.setattr(
|
|
SimulationRunner,
|
|
"get_run_state",
|
|
classmethod(lambda _cls, _simulation_id: state),
|
|
)
|
|
monkeypatch.setattr(
|
|
SimulationRunner,
|
|
"_save_run_state",
|
|
classmethod(lambda _cls, _state: None),
|
|
)
|
|
monkeypatch.setattr(
|
|
SimulationRunner,
|
|
"_sync_simulation_status",
|
|
classmethod(lambda _cls, *_args, **_kwargs: None),
|
|
)
|
|
monkeypatch.setattr(
|
|
runner_module.ZepGraphMemoryManager,
|
|
"get_simulation_ids",
|
|
classmethod(lambda _cls: [simulation_id]),
|
|
)
|
|
monkeypatch.setattr(
|
|
runner_module.ZepGraphMemoryManager,
|
|
"get_updater",
|
|
classmethod(lambda _cls, _simulation_id: updater),
|
|
)
|
|
monkeypatch.setattr(
|
|
runner_module.ZepGraphMemoryManager,
|
|
"stop_updater",
|
|
classmethod(
|
|
lambda _cls, _simulation_id: (_ for _ in ()).throw(
|
|
RuntimeError("drain incomplete")
|
|
)
|
|
),
|
|
)
|
|
|
|
SimulationRunner._cleanup_done = False
|
|
SimulationRunner._graph_memory_enabled[simulation_id] = True
|
|
SimulationRunner._monitor_threads.pop(simulation_id, None)
|
|
SimulationRunner._processes.pop(simulation_id, None)
|
|
try:
|
|
SimulationRunner.cleanup_all_simulations()
|
|
assert state.runner_status == RunnerStatus.FAILED
|
|
assert "drain incomplete" in state.error
|
|
assert SimulationRunner._graph_memory_enabled[simulation_id] is True
|
|
assert SimulationRunner._cleanup_done is False
|
|
finally:
|
|
SimulationRunner._cleanup_done = False
|
|
SimulationRunner._graph_memory_enabled.pop(simulation_id, None)
|
|
SimulationRunner._manual_stop_requests.discard(simulation_id)
|