mirror of https://github.com/aliasrobotics/cai.git
fix: prevent indefinite blocking on interactive commands (#370)
Add idle detection to prevent CAI from freezing when executing interactive tools (cat, python -i, gdb, radare2) that wait for user input indefinitely. - Add non-blocking I/O with asyncio.wait_for() timeouts - Add 10s idle timeout to terminate idle processes - Apply to streaming and non-streaming execution paths - Add select.select() to ShellSession._read_output() Resolves blocking I/O issue where interactive tools would hang CAI for the full timeout period (100s).
This commit is contained in:
parent
09ccb6e0ba
commit
e932e30850
|
|
@ -12,6 +12,7 @@ import time
|
||||||
import uuid
|
import uuid
|
||||||
import sys
|
import sys
|
||||||
import shlex
|
import shlex
|
||||||
|
import select
|
||||||
from wasabi import color # pylint: disable=import-error
|
from wasabi import color # pylint: disable=import-error
|
||||||
from cai.util import format_time, start_active_timer, stop_active_timer, start_idle_timer, stop_idle_timer, cli_print_tool_output
|
from cai.util import format_time, start_active_timer, stop_active_timer, start_idle_timer, stop_idle_timer, cli_print_tool_output
|
||||||
|
|
||||||
|
|
@ -265,16 +266,22 @@ class ShellSession: # pylint: disable=too-many-instance-attributes
|
||||||
self.is_running = False
|
self.is_running = False
|
||||||
return str(e)
|
return str(e)
|
||||||
def _read_output(self):
|
def _read_output(self):
|
||||||
"""Read output from the process"""
|
"""Read output with non-blocking select"""
|
||||||
try:
|
try:
|
||||||
while self.is_running and self.master is not None:
|
while self.is_running and self.master is not None:
|
||||||
try:
|
try:
|
||||||
# Check if process has exited before reading
|
|
||||||
if self.process and self.process.poll() is not None:
|
if self.process and self.process.poll() is not None:
|
||||||
self.is_running = False
|
self.is_running = False
|
||||||
break
|
break
|
||||||
|
|
||||||
# Read raw output chunk from PTY (don't require newlines)
|
# Non-blocking check for data
|
||||||
|
ready, _, _ = select.select([self.master], [], [], 0.5)
|
||||||
|
if not ready:
|
||||||
|
if self.process and self.process.poll() is not None:
|
||||||
|
self.is_running = False
|
||||||
|
break
|
||||||
|
continue
|
||||||
|
|
||||||
output = os.read(self.master, 4096).decode('utf-8', errors='replace')
|
output = os.read(self.master, 4096).decode('utf-8', errors='replace')
|
||||||
|
|
||||||
if output is not None and output != "":
|
if output is not None and output != "":
|
||||||
|
|
@ -694,27 +701,43 @@ async def _run_local_async(command, stdout=False, timeout=100, stream=False, cal
|
||||||
# Don't add refresh_rate to tool_args as it affects command deduplication
|
# Don't add refresh_rate to tool_args as it affects command deduplication
|
||||||
# The refresh behavior is already handled by the streaming update logic
|
# The refresh behavior is already handled by the streaming update logic
|
||||||
|
|
||||||
# Stream stdout in real-time
|
# Stream stdout with idle detection
|
||||||
async for line in process.stdout:
|
last_output = time.time()
|
||||||
line_str = line.decode('utf-8', errors='replace')
|
while True:
|
||||||
|
if process.returncode is not None:
|
||||||
# Add to output collection
|
break
|
||||||
output_buffer.append(line_str)
|
try:
|
||||||
buffer_size += 1
|
line = await asyncio.wait_for(process.stdout.readline(), timeout=0.5)
|
||||||
|
if line:
|
||||||
# Only update periodically to reduce UI refreshes
|
output_buffer.append(line.decode('utf-8', errors='replace'))
|
||||||
if buffer_size >= update_interval:
|
buffer_size += 1
|
||||||
current_output = ''.join(output_buffer)
|
last_output = time.time()
|
||||||
update_tool_streaming(tool_name, tool_args, current_output, call_id, token_info)
|
if buffer_size >= update_interval:
|
||||||
buffer_size = 0
|
update_tool_streaming(tool_name, tool_args, ''.join(output_buffer), call_id, token_info)
|
||||||
|
buffer_size = 0
|
||||||
|
else:
|
||||||
|
break
|
||||||
|
except asyncio.TimeoutError:
|
||||||
|
if time.time() - last_output > 10:
|
||||||
|
process.terminate()
|
||||||
|
try:
|
||||||
|
await asyncio.wait_for(process.wait(), timeout=1.0)
|
||||||
|
except asyncio.TimeoutError:
|
||||||
|
process.kill()
|
||||||
|
await process.wait()
|
||||||
|
output_buffer.append("\n[Terminated: idle 10s, likely waiting for input]")
|
||||||
|
break
|
||||||
|
|
||||||
# Wait for process to complete with timeout
|
# Wait for process to complete
|
||||||
try:
|
if process.returncode is None:
|
||||||
return_code = await asyncio.wait_for(process.wait(), timeout=timeout)
|
try:
|
||||||
except asyncio.TimeoutError:
|
return_code = await asyncio.wait_for(process.wait(), timeout=timeout)
|
||||||
process.kill()
|
except asyncio.TimeoutError:
|
||||||
await process.wait()
|
process.kill()
|
||||||
raise subprocess.TimeoutExpired(command, timeout)
|
await process.wait()
|
||||||
|
raise subprocess.TimeoutExpired(command, timeout)
|
||||||
|
else:
|
||||||
|
return_code = process.returncode
|
||||||
|
|
||||||
process_execution_time = time.time() - process_start_time
|
process_execution_time = time.time() - process_start_time
|
||||||
|
|
||||||
|
|
@ -743,7 +766,7 @@ async def _run_local_async(command, stdout=False, timeout=100, stream=False, cal
|
||||||
|
|
||||||
return final_output
|
return final_output
|
||||||
else:
|
else:
|
||||||
# Standard non-streaming async execution
|
# Non-streaming with idle detection
|
||||||
process = await asyncio.create_subprocess_shell(
|
process = await asyncio.create_subprocess_shell(
|
||||||
command,
|
command,
|
||||||
stdout=asyncio.subprocess.PIPE,
|
stdout=asyncio.subprocess.PIPE,
|
||||||
|
|
@ -751,15 +774,45 @@ async def _run_local_async(command, stdout=False, timeout=100, stream=False, cal
|
||||||
cwd=target_dir
|
cwd=target_dir
|
||||||
)
|
)
|
||||||
|
|
||||||
try:
|
stdout_chunks, stderr_chunks = [], []
|
||||||
stdout_data, stderr_data = await asyncio.wait_for(
|
last_output = time.time()
|
||||||
process.communicate(),
|
start = time.time()
|
||||||
timeout=timeout
|
|
||||||
)
|
while True:
|
||||||
except asyncio.TimeoutError:
|
if time.time() - start > timeout:
|
||||||
process.kill()
|
process.kill()
|
||||||
await process.wait()
|
await process.wait()
|
||||||
raise subprocess.TimeoutExpired(command, timeout)
|
raise subprocess.TimeoutExpired(command, timeout)
|
||||||
|
if process.returncode is not None:
|
||||||
|
break
|
||||||
|
try:
|
||||||
|
out_task = asyncio.create_task(process.stdout.read(4096))
|
||||||
|
err_task = asyncio.create_task(process.stderr.read(4096))
|
||||||
|
done, pending = await asyncio.wait([out_task, err_task], timeout=0.5, return_when=asyncio.FIRST_COMPLETED)
|
||||||
|
for task in pending:
|
||||||
|
task.cancel()
|
||||||
|
for task in done:
|
||||||
|
data = await task
|
||||||
|
if data:
|
||||||
|
(stdout_chunks if task == out_task else stderr_chunks).append(data)
|
||||||
|
last_output = time.time()
|
||||||
|
except asyncio.TimeoutError:
|
||||||
|
pass
|
||||||
|
if time.time() - last_output > 10:
|
||||||
|
try:
|
||||||
|
await asyncio.wait_for(process.wait(), timeout=0.1)
|
||||||
|
break
|
||||||
|
except asyncio.TimeoutError:
|
||||||
|
process.terminate()
|
||||||
|
try:
|
||||||
|
await asyncio.wait_for(process.wait(), timeout=1.0)
|
||||||
|
except asyncio.TimeoutError:
|
||||||
|
process.kill()
|
||||||
|
await process.wait()
|
||||||
|
stderr_chunks.append(b"\n[Terminated: idle 10s]")
|
||||||
|
break
|
||||||
|
|
||||||
|
stdout_data, stderr_data = b''.join(stdout_chunks), b''.join(stderr_chunks)
|
||||||
|
|
||||||
# Decode output
|
# Decode output
|
||||||
output = stdout_data.decode('utf-8', errors='replace') if stdout_data else ""
|
output = stdout_data.decode('utf-8', errors='replace') if stdout_data else ""
|
||||||
|
|
@ -968,26 +1021,43 @@ async def _run_docker_async(command, container_id, stdout=False, timeout=100, st
|
||||||
|
|
||||||
start_time = time.time()
|
start_time = time.time()
|
||||||
|
|
||||||
# Read stdout line by line
|
# Read stdout with idle detection
|
||||||
async for line in process.stdout:
|
last_output = time.time()
|
||||||
line_str = line.decode('utf-8', errors='replace')
|
while True:
|
||||||
output_buffer.append(line_str)
|
if process.returncode is not None:
|
||||||
buffer_size += 1
|
break
|
||||||
|
try:
|
||||||
# Only update periodically to reduce UI refreshes
|
line = await asyncio.wait_for(process.stdout.readline(), timeout=0.5)
|
||||||
if buffer_size >= update_interval:
|
if line:
|
||||||
# Show actual output as it's being collected
|
output_buffer.append(line.decode('utf-8', errors='replace'))
|
||||||
current_output = ''.join(output_buffer)
|
buffer_size += 1
|
||||||
update_tool_streaming(tool_name, tool_args, current_output, call_id, token_info)
|
last_output = time.time()
|
||||||
buffer_size = 0
|
if buffer_size >= update_interval:
|
||||||
|
update_tool_streaming(tool_name, tool_args, ''.join(output_buffer), call_id, token_info)
|
||||||
|
buffer_size = 0
|
||||||
|
else:
|
||||||
|
break
|
||||||
|
except asyncio.TimeoutError:
|
||||||
|
if time.time() - last_output > 10:
|
||||||
|
process.terminate()
|
||||||
|
try:
|
||||||
|
await asyncio.wait_for(process.wait(), timeout=1.0)
|
||||||
|
except asyncio.TimeoutError:
|
||||||
|
process.kill()
|
||||||
|
await process.wait()
|
||||||
|
output_buffer.append("\n[Terminated: idle 10s]")
|
||||||
|
break
|
||||||
|
|
||||||
# Wait for process completion
|
# Wait for process completion
|
||||||
try:
|
if process.returncode is None:
|
||||||
return_code = await asyncio.wait_for(process.wait(), timeout=timeout)
|
try:
|
||||||
except asyncio.TimeoutError:
|
return_code = await asyncio.wait_for(process.wait(), timeout=timeout)
|
||||||
process.kill()
|
except asyncio.TimeoutError:
|
||||||
await process.wait()
|
process.kill()
|
||||||
raise subprocess.TimeoutExpired(command, timeout)
|
await process.wait()
|
||||||
|
raise subprocess.TimeoutExpired(command, timeout)
|
||||||
|
else:
|
||||||
|
return_code = process.returncode
|
||||||
|
|
||||||
execution_time = time.time() - start_time
|
execution_time = time.time() - start_time
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue