diff --git a/.gitignore b/.gitignore index 919ab838..0d6bba99 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,5 @@ archive/ .claw/sessions/ .clawhip/ status-help.txt +.port_sessions/ +.port_workers/ diff --git a/src/agent_loop.py b/src/agent_loop.py index 9212d350..920e5501 100644 --- a/src/agent_loop.py +++ b/src/agent_loop.py @@ -122,6 +122,15 @@ def run_agent_task( if trace: trace_events.append(f'stopped=max_turns:{max_turns}') + if edit_required and has_successful_edit(tool_trace, required_edit_target): + return AgentRunResult( + content=f'Completed the required edit but reached max_turns={max_turns} before a final response.', + turns=max_turns, + tool_calls=tuple(tool_calls), + tool_trace=tuple(tool_trace), + stop_reason='completed_after_edit_max_turns', + trace_events=tuple(trace_events), + ) raise LLMBackendError(f'Agent stopped after reaching max_turns={max_turns}.') @@ -227,6 +236,25 @@ def prompt_requires_edit(prompt: str) -> bool: def infer_required_edit_target(prompt: str) -> str | None: + file_pattern = r'[\w./-]+\.(?:py|md|txt|json|yaml|yml|toml)' + for pattern in ( + rf'\bin\s+(?P{file_pattern})(?![\w./-])', + rf'\b(?:modify|edit|change|update|write|create|add(?:\s+to)?)\s+only\s+(?P{file_pattern})(?![\w./-])', + rf'\b(?:modify|edit|change|update|write|create|add(?:\s+to)?)\s+(?P{file_pattern})(?![\w./-])', + ): + match = re.search(pattern, prompt, flags=re.IGNORECASE) + if match: + return match.group('path').rstrip('.,:;') + + file_matches = re.findall( + rf'(?{file_pattern})(?![\w./-])', + prompt, + flags=re.IGNORECASE, + ) + unique_files = list(dict.fromkeys(path.rstrip('.,:;') for path in file_matches)) + if len(unique_files) == 1: + return unique_files[0] + match = re.search( r'(?:modify|edit|change|update|write|create|add(?:\s+to)?)\s+(?P[\w./-]+)', prompt, diff --git a/src/autoresearch_worker.py b/src/autoresearch_worker.py index e9089d63..382a599b 100644 --- a/src/autoresearch_worker.py +++ b/src/autoresearch_worker.py @@ -1,6 +1,8 @@ from __future__ import annotations import json +import hashlib +import os import subprocess from dataclasses import asdict, dataclass from datetime import datetime, timezone @@ -25,6 +27,7 @@ DEFAULT_AUTORESEARCH_TIMEOUT_SECONDS = 600 DEFAULT_STATE_FILE = '.autoresearch_state.json' DEFAULT_MEMORY_FILE = 'experiment_memory.jsonl' AUTORESEARCH_BRANCH_PREFIX = 'autoresearch/' +NO_LEGACY_COMMITS_ENV = 'AUTORESEARCH_NO_LEGACY_COMMITS' @dataclass(frozen=True) @@ -170,6 +173,7 @@ def run_autoresearch_packet( packet = packet_source if isinstance(packet_source, AutoresearchExperimentPacket) else load_autoresearch_packet(packet_source) state = load_autoresearch_state(repo_root) base_commit = short_head_commit(repo_root) + train_hash_before = _file_sha256(repo_root / 'train.py') worker = create_worker(root=repo_root, model=model, host=host) worker_run = run_worker( @@ -192,7 +196,29 @@ def run_autoresearch_packet( append_memory_event(repo_root, 'run_skipped', result) return result - commit = _commit_train_change(repo_root, packet.description) + train_hash_after = _file_sha256(repo_root / 'train.py') + if train_hash_before and train_hash_after == train_hash_before: + result = { + 'setup': setup.to_dict(), + 'state': state.to_dict(), + 'worker': worker_run.to_dict(), + 'experiment': None, + 'results_tsv': str((repo_root / packet.results_tsv).resolve()), + 'recommended_status': 'discard', + 'error': 'no_op_patch', + 'no_op_patch': True, + 'base_commit': base_commit, + 'train_hash_before': train_hash_before, + 'train_hash_after': train_hash_after, + } + append_memory_event(repo_root, 'run_skipped', result) + return result + + commit = ( + _dirty_candidate_ref(base_commit) + if os.environ.get(NO_LEGACY_COMMITS_ENV) == '1' + else _commit_train_change(repo_root, packet.description) + ) experiment = run_experiment( root=repo_root, command=packet.train_command, @@ -342,7 +368,7 @@ def discard_autoresearch_candidate(root: str | Path | None = None) -> dict[str, pending = _require_pending_experiment(state) metrics = ExperimentMetrics(**pending.experiment) final_status = 'crash' if not metrics.success or metrics.val_bpb is None else 'discard' - _git(repo_root, ['git', 'reset', '--hard', pending.base_commit], check=False) + _git(repo_root, ['git', 'checkout', pending.base_commit, '--', 'train.py'], check=False) append_results_row( repo_root, commit=pending.commit, @@ -597,7 +623,7 @@ def _commit_train_change(root: Path, description: str) -> str: message = f'autoresearch: {description.strip()}' subprocess.run(['git', 'add', '--', 'train.py'], cwd=root, capture_output=True, text=True, check=False) commit = subprocess.run( - ['git', 'commit', '-m', message[:120]], + ['git', 'commit', '-m', message[:120], '--', 'train.py'], cwd=root, capture_output=True, text=True, @@ -609,6 +635,10 @@ def _commit_train_change(root: Path, description: str) -> str: return f'{head}-dirty' if head != 'unknown' else 'uncommitted' +def _dirty_candidate_ref(base_commit: str) -> str: + return f'{base_commit}-dirty' if base_commit != 'unknown' else 'uncommitted' + + def _require_pending_experiment(state: AutoresearchState) -> PendingExperiment: if state.pending_experiment is None: raise RuntimeError('No pending autoresearch experiment is recorded.') @@ -644,3 +674,11 @@ def _suggest_branch_name() -> str: def _is_isolated_branch(branch: str) -> bool: return branch.startswith(AUTORESEARCH_BRANCH_PREFIX) + + +def _file_sha256(path: Path) -> str: + if not path.exists(): + return '' + digest = hashlib.sha256() + digest.update(path.read_bytes()) + return digest.hexdigest() diff --git a/tests/test_autoresearch_integration.py b/tests/test_autoresearch_integration.py index 9d2df834..c8b18c3b 100644 --- a/tests/test_autoresearch_integration.py +++ b/tests/test_autoresearch_integration.py @@ -139,10 +139,12 @@ class AutoresearchIntegrationTests(unittest.TestCase): 'verification': {'acceptance_tests': []}, }, ) - (root / 'train.py').write_text('print("changed")\n', encoding='utf-8') + def fake_run_worker(*args, **kwargs): + (root / 'train.py').write_text('print("changed")\n', encoding='utf-8') + return worker with patch('src.autoresearch_worker.create_worker', return_value=worker): - with patch('src.autoresearch_worker.run_worker', return_value=worker): + with patch('src.autoresearch_worker.run_worker', side_effect=fake_run_worker): with patch( 'src.autoresearch_worker.run_experiment', return_value=ExperimentMetrics( @@ -188,10 +190,12 @@ class AutoresearchIntegrationTests(unittest.TestCase): 'verification': {'acceptance_tests': []}, }, ) - (root / 'train.py').write_text('print("changed")\n', encoding='utf-8') + def fake_run_worker(*args, **kwargs): + (root / 'train.py').write_text('print("changed")\n', encoding='utf-8') + return worker with patch('src.autoresearch_worker.create_worker', return_value=worker): - with patch('src.autoresearch_worker.run_worker', return_value=worker): + with patch('src.autoresearch_worker.run_worker', side_effect=fake_run_worker): with patch( 'src.autoresearch_worker.run_experiment', return_value=ExperimentMetrics( @@ -240,10 +244,12 @@ class AutoresearchIntegrationTests(unittest.TestCase): 'verification': {'acceptance_tests': []}, }, ) - (root / 'train.py').write_text('print("candidate")\n', encoding='utf-8') + def run_candidate(*args, **kwargs): + (root / 'train.py').write_text('print("candidate")\n', encoding='utf-8') + return worker with patch('src.autoresearch_worker.create_worker', return_value=worker): - with patch('src.autoresearch_worker.run_worker', return_value=worker): + with patch('src.autoresearch_worker.run_worker', side_effect=run_candidate): with patch( 'src.autoresearch_worker.run_experiment', return_value=ExperimentMetrics( @@ -259,9 +265,13 @@ class AutoresearchIntegrationTests(unittest.TestCase): self.assertEqual(result['recommended_status'], 'keep') keep_autoresearch_candidate(root) - (root / 'train.py').write_text('print("worse")\n', encoding='utf-8') + + def run_worse(*args, **kwargs): + (root / 'train.py').write_text('print("worse")\n', encoding='utf-8') + return worker + with patch('src.autoresearch_worker.create_worker', return_value=worker): - with patch('src.autoresearch_worker.run_worker', return_value=worker): + with patch('src.autoresearch_worker.run_worker', side_effect=run_worse): with patch( 'src.autoresearch_worker.run_experiment', return_value=ExperimentMetrics( @@ -277,7 +287,7 @@ class AutoresearchIntegrationTests(unittest.TestCase): discard = discard_autoresearch_candidate(root) self.assertEqual(discard['decision'], 'discard') - self.assertEqual(short_head_commit(root), discard['reverted_to_commit']) + self.assertEqual(short_head_commit(root), discard['current_commit']) self.assertIn('candidate', (root / 'train.py').read_text(encoding='utf-8')) self.assertNotEqual(short_head_commit(root), 'unknown') @@ -349,14 +359,16 @@ class AutoresearchIntegrationTests(unittest.TestCase): 'verification': {'acceptance_tests': []}, }, ) - (root / 'train.py').write_text('print("changed")\n', encoding='utf-8') + def fake_run_worker(*args, **kwargs): + (root / 'train.py').write_text('print("changed")\n', encoding='utf-8') + return worker with patch( 'src.autoresearch_worker.ensure_autoresearch_baseline', return_value={'baseline_created': False, 'reason': 'baseline already recorded'}, ): with patch('src.autoresearch_worker.create_worker', return_value=worker): - with patch('src.autoresearch_worker.run_worker', return_value=worker): + with patch('src.autoresearch_worker.run_worker', side_effect=fake_run_worker): with patch( 'src.autoresearch_worker.run_experiment', return_value=ExperimentMetrics( diff --git a/tests/test_porting_workspace.py b/tests/test_porting_workspace.py index 754c20a2..dbd3cb50 100644 --- a/tests/test_porting_workspace.py +++ b/tests/test_porting_workspace.py @@ -371,6 +371,14 @@ class PortingWorkspaceTests(unittest.TestCase): 'train.py', ) + def test_infer_required_edit_target_prefers_unique_file_reference(self) -> None: + prompt = ( + 'Objective: In train.py, Change WEIGHT_DECAY from 1e-4 to 5e-5. ' + 'Scope: Modify only train.py in the autoresearch repo. ' + 'You may read program.md and README.md but do not edit prepare.py or pyproject.toml.' + ) + self.assertEqual(infer_required_edit_target(prompt), 'train.py') + def test_infer_tool_call_from_prompt_recognizes_read_and_search(self) -> None: read_call = infer_tool_call_from_prompt('read src/main.py lines 1 to 40') search_call = infer_tool_call_from_prompt("search the src tree for 'agent_parser'") @@ -544,6 +552,37 @@ class PortingWorkspaceTests(unittest.TestCase): self.assertTrue(target.read_text(encoding='utf-8').startswith('# smoke-test comment: 2\n')) + def test_run_agent_task_accepts_successful_edit_at_max_turns(self) -> None: + with TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + target = root / 'train.py' + target.write_text('WEIGHT_DECAY = 1e-4\n', encoding='utf-8') + responses = [ + normalize_ollama_response( + { + 'message': { + 'content': ( + '{"type":"tool_call","name":"edit_file","arguments":' + '{"path":"train.py","old_text":"WEIGHT_DECAY = 1e-4",' + '"new_text":"WEIGHT_DECAY = 5e-5"}}' + ) + } + } + ), + ] + with patch('src.agent_loop.OllamaBackend.chat', side_effect=responses): + result = run_agent_task( + 'Objective: In train.py, Change WEIGHT_DECAY from 1e-4 to 5e-5.', + root=root, + max_turns=1, + trace=True, + ) + + self.assertEqual(target.read_text(encoding='utf-8'), 'WEIGHT_DECAY = 5e-5\n') + self.assertEqual(result.stop_reason, 'completed_after_edit_max_turns') + self.assertEqual(result.tool_calls, ('edit_file',)) + self.assertTrue(any('stopped=max_turns:1' in event for event in result.trace_events)) + def test_has_successful_edit_requires_target_path(self) -> None: tool_trace = [ {