Add autoresearch control plane, repeatable smoke notebook, and stricter train.py edit enforcement

This commit is contained in:
Dowling_Wong 2026-04-08 01:51:32 +02:00
parent c55c56e24a
commit c527beaa19
90 changed files with 7347 additions and 5 deletions

BIN
.DS_Store vendored Normal file

Binary file not shown.

View File

@ -0,0 +1,9 @@
{
"session_id": "0228969d992f401687edd415914d88d0",
"messages": [
"review MCP tool",
"review MCP tool"
],
"input_tokens": 6,
"output_tokens": 32
}

View File

@ -0,0 +1,9 @@
{
"session_id": "03a8acb08e0f406ea9f8e5e4205db9ac",
"messages": [
"review MCP tool",
"review MCP tool"
],
"input_tokens": 6,
"output_tokens": 32
}

View File

@ -0,0 +1,9 @@
{
"session_id": "07537f9e81ab4513b9fe2c7e14cf3aff",
"messages": [
"review MCP tool",
"review MCP tool"
],
"input_tokens": 6,
"output_tokens": 32
}

View File

@ -0,0 +1,9 @@
{
"session_id": "0f1ca23f64584b9aa5d3ea8f00675b88",
"messages": [
"review MCP tool",
"review MCP tool"
],
"input_tokens": 6,
"output_tokens": 32
}

View File

@ -0,0 +1,9 @@
{
"session_id": "139fbf7a78f24609bfc56b1717d1158c",
"messages": [
"review MCP tool",
"review MCP tool"
],
"input_tokens": 6,
"output_tokens": 32
}

View File

@ -0,0 +1,9 @@
{
"session_id": "147b50bd6de34372b7289dac48fbb75e",
"messages": [
"review MCP tool",
"review MCP tool"
],
"input_tokens": 6,
"output_tokens": 32
}

View File

@ -0,0 +1,9 @@
{
"session_id": "1a4a2a8f32ce4568a2870c8f10f86a39",
"messages": [
"review MCP tool",
"review MCP tool"
],
"input_tokens": 6,
"output_tokens": 32
}

View File

@ -0,0 +1,8 @@
{
"session_id": "21e1ac4fd9964346842410cfd7981d8d",
"messages": [
"review MCP tool"
],
"input_tokens": 3,
"output_tokens": 13
}

View File

@ -0,0 +1,9 @@
{
"session_id": "2aff5051829a463593d722bfe966d7fc",
"messages": [
"review MCP tool",
"review MCP tool"
],
"input_tokens": 6,
"output_tokens": 32
}

View File

@ -0,0 +1,8 @@
{
"session_id": "2c8b2ac16df04df2bc52664e3706999c",
"messages": [
"review MCP tool"
],
"input_tokens": 3,
"output_tokens": 13
}

View File

@ -0,0 +1,8 @@
{
"session_id": "2d95afef7d7d47329a907c629257171e",
"messages": [
"review MCP tool"
],
"input_tokens": 3,
"output_tokens": 13
}

View File

@ -0,0 +1,9 @@
{
"session_id": "32e45820fb8149d186803d55c3262793",
"messages": [
"review MCP tool",
"review MCP tool"
],
"input_tokens": 6,
"output_tokens": 32
}

View File

@ -0,0 +1,9 @@
{
"session_id": "348077f8479f48449f5fa48f696bf813",
"messages": [
"review MCP tool",
"review MCP tool"
],
"input_tokens": 6,
"output_tokens": 32
}

View File

@ -0,0 +1,9 @@
{
"session_id": "35eba01342de4de4b29e0971673cae9f",
"messages": [
"review MCP tool",
"review MCP tool"
],
"input_tokens": 6,
"output_tokens": 32
}

View File

@ -0,0 +1,9 @@
{
"session_id": "39a28ffb334249749d1a2d47c889a2ad",
"messages": [
"review MCP tool",
"review MCP tool"
],
"input_tokens": 6,
"output_tokens": 32
}

View File

@ -0,0 +1,8 @@
{
"session_id": "39cc42b06ec44c2880bbe98af7b98e47",
"messages": [
"review MCP tool"
],
"input_tokens": 3,
"output_tokens": 13
}

View File

@ -0,0 +1,9 @@
{
"session_id": "3afaa2caaf854ae7be20f635c61d35a5",
"messages": [
"review MCP tool",
"review MCP tool"
],
"input_tokens": 6,
"output_tokens": 32
}

View File

@ -0,0 +1,9 @@
{
"session_id": "431cc294fada4b629a5afcc576349741",
"messages": [
"review MCP tool",
"review MCP tool"
],
"input_tokens": 6,
"output_tokens": 32
}

View File

@ -0,0 +1,9 @@
{
"session_id": "4ebbccbc34e140d08af4623d2a1f41ce",
"messages": [
"review MCP tool",
"review MCP tool"
],
"input_tokens": 6,
"output_tokens": 32
}

View File

@ -0,0 +1,8 @@
{
"session_id": "553b6516bcad467aa18931eb1882f2cb",
"messages": [
"review MCP tool"
],
"input_tokens": 3,
"output_tokens": 13
}

View File

@ -0,0 +1,8 @@
{
"session_id": "560c179d748b464b8bfc7ad3aaf9bab8",
"messages": [
"review MCP tool"
],
"input_tokens": 3,
"output_tokens": 13
}

View File

@ -0,0 +1,9 @@
{
"session_id": "5a4e69899681442cbb2f53a68572857f",
"messages": [
"review MCP tool",
"review MCP tool"
],
"input_tokens": 6,
"output_tokens": 32
}

View File

@ -0,0 +1,8 @@
{
"session_id": "5afdc8073a44407caf745e3fbdb25335",
"messages": [
"review MCP tool"
],
"input_tokens": 3,
"output_tokens": 13
}

View File

@ -0,0 +1,9 @@
{
"session_id": "65e9cfe029244fd89c341c822c2e7567",
"messages": [
"review MCP tool",
"review MCP tool"
],
"input_tokens": 6,
"output_tokens": 32
}

View File

@ -0,0 +1,9 @@
{
"session_id": "6ec47e7bcdc2407eb5b91de25aa4ad49",
"messages": [
"review MCP tool",
"review MCP tool"
],
"input_tokens": 6,
"output_tokens": 32
}

View File

@ -0,0 +1,9 @@
{
"session_id": "6ec578dbd7a64d14b2baa2c2a6fd3386",
"messages": [
"review MCP tool",
"review MCP tool"
],
"input_tokens": 6,
"output_tokens": 32
}

View File

@ -0,0 +1,9 @@
{
"session_id": "7601f3f91ec648bb999078465187e9b1",
"messages": [
"review MCP tool",
"review MCP tool"
],
"input_tokens": 6,
"output_tokens": 32
}

View File

@ -0,0 +1,9 @@
{
"session_id": "7af44d40f76348f793db71281a424c62",
"messages": [
"review MCP tool",
"review MCP tool"
],
"input_tokens": 6,
"output_tokens": 32
}

View File

@ -0,0 +1,8 @@
{
"session_id": "7beee467ce7f40c0b3e5e40da43404cb",
"messages": [
"review MCP tool"
],
"input_tokens": 3,
"output_tokens": 13
}

View File

@ -0,0 +1,8 @@
{
"session_id": "7d1e429766564c4491396fdfb6636634",
"messages": [
"review MCP tool"
],
"input_tokens": 3,
"output_tokens": 13
}

View File

@ -0,0 +1,9 @@
{
"session_id": "8b04b17b8791423a88f67db8dfd672e7",
"messages": [
"review MCP tool",
"review MCP tool"
],
"input_tokens": 6,
"output_tokens": 32
}

View File

@ -0,0 +1,9 @@
{
"session_id": "8b3a6522d9b947aa88b449a64f68b10a",
"messages": [
"review MCP tool",
"review MCP tool"
],
"input_tokens": 6,
"output_tokens": 32
}

View File

@ -0,0 +1,9 @@
{
"session_id": "8b8817685e1246b1af232888ca0f2489",
"messages": [
"review MCP tool",
"review MCP tool"
],
"input_tokens": 6,
"output_tokens": 32
}

View File

@ -0,0 +1,9 @@
{
"session_id": "9bc6445d7f044d249e0e9c28772dec2c",
"messages": [
"review MCP tool",
"review MCP tool"
],
"input_tokens": 6,
"output_tokens": 32
}

View File

@ -0,0 +1,8 @@
{
"session_id": "a8f2c525a64b431a8020d71e19eb57c1",
"messages": [
"review MCP tool"
],
"input_tokens": 3,
"output_tokens": 13
}

View File

@ -0,0 +1,9 @@
{
"session_id": "aa4e2a7e10ac4180aa7ea89a1f88adef",
"messages": [
"review MCP tool",
"review MCP tool"
],
"input_tokens": 6,
"output_tokens": 32
}

View File

@ -0,0 +1,9 @@
{
"session_id": "ab7dc8fd4d7b4f35b1a76dd924a41051",
"messages": [
"review MCP tool",
"review MCP tool"
],
"input_tokens": 6,
"output_tokens": 32
}

View File

@ -0,0 +1,9 @@
{
"session_id": "adcee2ef02814bbdb6021ba4da9714b1",
"messages": [
"review MCP tool",
"review MCP tool"
],
"input_tokens": 6,
"output_tokens": 32
}

View File

@ -0,0 +1,8 @@
{
"session_id": "af8f0ebd67934d9d94824c33b21df598",
"messages": [
"review MCP tool"
],
"input_tokens": 3,
"output_tokens": 13
}

View File

@ -0,0 +1,9 @@
{
"session_id": "b1bd6e8778e54b46b7953965c7719d03",
"messages": [
"review MCP tool",
"review MCP tool"
],
"input_tokens": 6,
"output_tokens": 32
}

View File

@ -0,0 +1,9 @@
{
"session_id": "b40eb42c98de45b88aaaacb36194e993",
"messages": [
"review MCP tool",
"review MCP tool"
],
"input_tokens": 6,
"output_tokens": 32
}

View File

@ -0,0 +1,9 @@
{
"session_id": "b6dc4ee47ec541b495fc71ee43d453e5",
"messages": [
"review MCP tool",
"review MCP tool"
],
"input_tokens": 6,
"output_tokens": 32
}

View File

@ -0,0 +1,9 @@
{
"session_id": "bb09985ad52f4ac0b2c48b38d7e486a3",
"messages": [
"review MCP tool",
"review MCP tool"
],
"input_tokens": 6,
"output_tokens": 32
}

View File

@ -0,0 +1,8 @@
{
"session_id": "bf47615c12f14e969df7e38667ff62fd",
"messages": [
"review MCP tool"
],
"input_tokens": 3,
"output_tokens": 13
}

View File

@ -0,0 +1,9 @@
{
"session_id": "c27d59a40d0a4a1aa9511905c61999bf",
"messages": [
"review MCP tool",
"review MCP tool"
],
"input_tokens": 6,
"output_tokens": 32
}

View File

@ -0,0 +1,9 @@
{
"session_id": "c2bfbd2d1fb04240a1bcc64358ef7509",
"messages": [
"review MCP tool",
"review MCP tool"
],
"input_tokens": 6,
"output_tokens": 32
}

View File

@ -0,0 +1,9 @@
{
"session_id": "c7e4fd25750d426fad047f8275735bdb",
"messages": [
"review MCP tool",
"review MCP tool"
],
"input_tokens": 6,
"output_tokens": 32
}

View File

@ -0,0 +1,8 @@
{
"session_id": "d04738b5d89643a086b5165348bc2276",
"messages": [
"review MCP tool"
],
"input_tokens": 3,
"output_tokens": 13
}

View File

@ -0,0 +1,9 @@
{
"session_id": "d1ffb054a6c742b2a8bba91b403a9835",
"messages": [
"review MCP tool",
"review MCP tool"
],
"input_tokens": 6,
"output_tokens": 32
}

View File

@ -0,0 +1,9 @@
{
"session_id": "df32ee126b80460c82951a8c0ddc8823",
"messages": [
"review MCP tool",
"review MCP tool"
],
"input_tokens": 6,
"output_tokens": 32
}

View File

@ -0,0 +1,9 @@
{
"session_id": "e719c34729384c3aacdb64ecdda8e51e",
"messages": [
"review MCP tool",
"review MCP tool"
],
"input_tokens": 6,
"output_tokens": 32
}

View File

@ -0,0 +1,9 @@
{
"session_id": "e84d4e6b3958494198e9ddc8514f871c",
"messages": [
"review MCP tool",
"review MCP tool"
],
"input_tokens": 6,
"output_tokens": 32
}

View File

@ -0,0 +1,8 @@
{
"session_id": "eab9da3a242d41ceb8d77138389c6220",
"messages": [
"review MCP tool"
],
"input_tokens": 3,
"output_tokens": 13
}

View File

@ -0,0 +1,9 @@
{
"session_id": "eaf733c1e6634d44bb05ea3b65aaab00",
"messages": [
"review MCP tool",
"review MCP tool"
],
"input_tokens": 6,
"output_tokens": 32
}

View File

@ -0,0 +1,9 @@
{
"session_id": "f4d50466bf2f4aa19af71dfa1adb6cb7",
"messages": [
"review MCP tool",
"review MCP tool"
],
"input_tokens": 6,
"output_tokens": 32
}

View File

@ -0,0 +1,9 @@
{
"session_id": "f5ce51832d5b42a1805ca0d62b7ca0e0",
"messages": [
"review MCP tool",
"review MCP tool"
],
"input_tokens": 6,
"output_tokens": 32
}

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -0,0 +1,13 @@
{
"worker_id": "7f1b0bdfb3d342bbaa4add09409ddd61",
"root": "/Users/wongdowling/Documents/autoresearch_harness/harness/claw-code",
"model": "qwen2.5-coder:7b",
"host": "http://localhost:11434",
"state": "closed",
"created_at": "2026-04-07T14:51:40Z",
"updated_at": "2026-04-07T14:51:40Z",
"run_count": 0,
"last_packet": null,
"last_result": null,
"last_error": null
}

View File

@ -0,0 +1,24 @@
{
"worker_id": "a4b91e78881349feaf46cd58acf7ab9c",
"root": "/Users/wongdowling/Documents/autoresearch_harness/nodes/autoresearch-macos",
"model": "qwen2.5-coder:7b",
"host": "http://localhost:11434",
"state": "running",
"created_at": "2026-04-07T21:15:02Z",
"updated_at": "2026-04-07T21:15:02Z",
"run_count": 0,
"last_packet": {
"objective": "Modify train.py by adding a single harmless comment line near the top of the file. You must change train.py. A no-op is failure.",
"scope": "Modify only train.py in the autoresearch repo. Read program.md, README.md, and train.py for context. Do not edit prepare.py, pyproject.toml, or dependencies. Do not run the full training experiment yourself; the manager will run it after your code change. Use syntax and git inspection tools when helpful.",
"repo": "/Users/wongdowling/Documents/autoresearch_harness/nodes/autoresearch-macos",
"branch_policy": "Stay on the current branch. Do not create or switch branches.",
"acceptance_tests": [
"python3 -m py_compile train.py"
],
"commit_policy": "Do not commit. The manager records and evaluates the experiment after your edit.",
"reporting_contract": "Return a concise summary of the train.py change, intended effect on val_bpb, and any blockers.",
"escalation_policy": "Stop if a required file outside train.py would need modification or if syntax cannot be preserved."
},
"last_result": null,
"last_error": null
}

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -0,0 +1,13 @@
{
"worker_id": "bf5663d0826b4bc9a84589982332e34b",
"root": "/Users/wongdowling/Documents/autoresearch_harness/harness/claw-code",
"model": "qwen2.5-coder:7b",
"host": "http://localhost:11434",
"state": "ready",
"created_at": "2026-04-07T14:47:08Z",
"updated_at": "2026-04-07T14:47:08Z",
"run_count": 0,
"last_packet": null,
"last_result": null,
"last_error": null
}

View File

@ -0,0 +1,41 @@
{
"worker_id": "d5c05c98476b4965b3adf7180f5f4d83",
"root": "/Users/wongdowling/Documents/autoresearch_harness/nodes/autoresearch-macos",
"model": "qwen2.5-coder:7b",
"host": "http://localhost:11434",
"state": "failed",
"created_at": "2026-04-07T21:58:21Z",
"updated_at": "2026-04-07T22:00:27Z",
"run_count": 1,
"last_packet": {
"objective": "Modify train.py by incrementing or inserting a harmless smoke-test comment counter near the top of the file. You must change train.py on every run. A no-op is failure.",
"scope": "Modify only train.py in the autoresearch repo. You may read program.md, README.md, and train.py for context, but train.py is the only file you may change. Editing any other path is failure. Do not edit prepare.py, pyproject.toml, or dependencies. Do not run the full training experiment yourself; the manager will run it after your code change. Use syntax and git inspection tools when helpful.",
"repo": "/Users/wongdowling/Documents/autoresearch_harness/nodes/autoresearch-macos",
"branch_policy": "Stay on the current branch. Do not create or switch branches.",
"acceptance_tests": [
"python3 -m py_compile train.py"
],
"commit_policy": "Do not commit. The manager records and evaluates the experiment after your edit.",
"reporting_contract": "Return a concise summary of the train.py change, intended effect on val_bpb, and any blockers.",
"escalation_policy": "Stop if a required file outside train.py would need modification or if syntax cannot be preserved."
},
"last_result": {
"worker_id": "d5c05c98476b4965b3adf7180f5f4d83",
"state": "failed",
"tool_calls": [],
"tool_trace": [],
"artifacts": [],
"changed_files": [],
"verification": {
"acceptance_tests": [],
"tool_runs": {
"run_tests": [],
"run_build": []
}
},
"final_answer": "",
"stop_reason": "error",
"error": "Agent stopped after reaching max_turns=8."
},
"last_error": "Agent stopped after reaching max_turns=8."
}

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -12,6 +12,8 @@ The current Ollama integration lives in the Python workspace:
- [`src/task.py`](./src/task.py): one-shot local chat wrapper
- [`src/runtime_tools.py`](./src/runtime_tools.py): executable workspace tools
- [`src/agent_loop.py`](./src/agent_loop.py): multi-turn agent loop with tool use
- [`src/task_packet.py`](./src/task_packet.py): structured worker task packet contract
- [`src/worker_api.py`](./src/worker_api.py): persisted worker control-plane API
- [`src/main.py`](./src/main.py): CLI entrypoints
## Default Configuration
@ -132,6 +134,19 @@ If the model returns invalid JSON or tries to answer directly without using a re
The current agent exposes these real tools:
### `list_dir`
Arguments:
- optional `path`
- optional `max_entries`
Behavior:
- lists directory entries inside the workspace root
- returns `[D]` and `[F]` records
- avoids shelling out for basic file discovery
### `read_file`
Arguments:
@ -187,6 +202,54 @@ Behavior:
- captures stdout, stderr, and exit code
- marks non-zero exit as failure
### `git_status`
Arguments:
- none
Behavior:
- runs `git status --short --branch`
- fails when the workspace is not a git repo
### `git_diff`
Arguments:
- optional `path`
- optional `staged`
Behavior:
- runs `git diff`
- can scope to a path
- can inspect staged changes
### `run_tests`
Arguments:
- optional `command`
- optional `timeout_seconds`
Behavior:
- defaults to `python3 -m unittest discover -s tests -v`
- can be overridden per project or per task
### `run_build`
Arguments:
- optional `command`
- optional `timeout_seconds`
Behavior:
- defaults to `python3 -m compileall src`
- can be overridden per project or per task
## CLI Commands
### One-shot local chat
@ -252,6 +315,51 @@ Expected stderr output looks like:
[trace] assistant_raw[2]={"type":"final","content":"..."}
```
### Structured worker API
Use the `worker` subcommands when you want a manager agent to control a local Ollama worker through machine-readable JSON state.
Create a worker:
```bash
python3 -m src.main worker create --root .
```
Run a task packet:
```bash
python3 -m src.main worker run <worker_id> --packet task.json --trace
```
Check worker state:
```bash
python3 -m src.main worker status <worker_id>
```
Resume the last stored packet:
```bash
python3 -m src.main worker resume <worker_id> --trace
```
Close a worker:
```bash
python3 -m src.main worker close <worker_id>
```
Worker results are emitted as JSON and include:
- `state`
- `tool_calls`
- `tool_trace`
- `artifacts`
- `changed_files`
- `verification`
- `final_answer`
- `stop_reason`
### Deny tools
You can block specific tools or prefixes in agent mode:
@ -313,6 +421,78 @@ Before building on this integration, verify:
3. `python3 -m src.main chat "hello"` works
4. `python3 -m src.main agent --trace "read src/main.py lines 1 to 20"` shows a real `tool_call`
## Worker Smoke Test
This is the recommended local smoke test for the structured worker API.
If you prefer a runnable walkthrough, use [`notebooks/ollama_worker_demo.ipynb`](./notebooks/ollama_worker_demo.ipynb). It handles `worker_id` capture automatically.
### 1. Start Ollama and ensure the model exists
```bash
ollama run qwen2.5-coder:7b
```
### 2. Create a task packet
Save this as `task.json` in the repo root:
```json
{
"objective": "Inspect the Python CLI and report the worker-related commands.",
"scope": "Only inspect src/main.py and related worker modules when needed.",
"repo": "claw-code",
"branch_policy": "Do not create or switch branches.",
"acceptance_tests": [
"python3 -m unittest tests/test_porting_workspace.py"
],
"commit_policy": "Do not commit.",
"reporting_contract": "Report the worker commands, changed files, and verification results.",
"escalation_policy": "Stop if Ollama is unavailable or a required tool fails repeatedly."
}
```
### 3. Create a worker
```bash
python3 -m src.main worker create --root .
```
Copy the `worker_id` from the JSON response.
### 4. Run the worker
```bash
python3 -m src.main worker run <worker_id> --packet task.json --trace
```
Success criteria:
- JSON output is returned
- `last_result.tool_calls` is non-empty
- `last_result.verification.acceptance_tests` is present
- `last_result.stop_reason` is `completed` or `verification_failed`
### 5. Inspect worker state
```bash
python3 -m src.main worker status <worker_id>
```
### 6. Resume the last packet
```bash
python3 -m src.main worker resume <worker_id> --trace
```
### 7. Close the worker
```bash
python3 -m src.main worker close <worker_id>
```
If all of those steps work, you have a usable manager/worker foundation for a local Ollama worker node.
## Example End-to-End Session
Command:

BIN
notebooks/.DS_Store vendored Normal file

Binary file not shown.

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -0,0 +1,9 @@
{
"objective": "Modify train.py by incrementing or inserting a harmless smoke-test comment counter near the top of the file. You must change train.py on every run. A no-op is failure.",
"description": "Increment the smoke-test comment counter in train.py.",
"train_command": "python3 -c 'from pathlib import Path; Path('\"'\"'run.log'\"'\"').write_text('\"'\"'---\\nval_bpb: 0.998000\\ntraining_seconds: 300.0\\ntotal_seconds: 320.0\\npeak_vram_mb: 1024.0\\nmfu_percent: 10.00\\ntotal_tokens_M: 1.0\\nnum_steps: 1\\nnum_params_M: 1.0\\ndepth: 1\\n'\"'\"', encoding='\"'\"'utf-8'\"'\"')'",
"timeout_seconds": 30,
"log_path": "run.log",
"results_tsv": "results.tsv",
"syntax_check_command": "python3 -m py_compile train.py"
}

12
notebooks/demo_task.json Normal file
View File

@ -0,0 +1,12 @@
{
"objective": "Inspect the Python CLI and report the worker-related commands.",
"scope": "Inspect src/main.py and worker modules only when needed.",
"repo": "claw-code",
"branch_policy": "Do not create or switch branches.",
"acceptance_tests": [
"python3 -m unittest tests/test_porting_workspace.py"
],
"commit_policy": "Do not commit.",
"reporting_contract": "Report worker commands, changed files, verification results, and blockers.",
"escalation_policy": "Stop if Ollama is unavailable or a required tool fails repeatedly."
}

View File

@ -0,0 +1,12 @@
{
"objective": "Inspect the Python CLI and report the worker-related commands.",
"scope": "Inspect src/main.py and worker modules only when needed.",
"repo": "claw-code",
"branch_policy": "Do not create or switch branches.",
"acceptance_tests": [
"python3 -m unittest tests/test_porting_workspace.py"
],
"commit_policy": "Do not commit.",
"reporting_contract": "Report worker commands, changed files, verification results, and blockers.",
"escalation_policy": "Stop if Ollama is unavailable or a required tool fails repeatedly."
}

File diff suppressed because one or more lines are too long

View File

@ -17,6 +17,7 @@ DEFAULT_AGENT_SYSTEM_PROMPT = (
'{"type":"final","content":"..."}. '
'Do not wrap JSON in markdown fences.'
)
MUTATING_TOOL_NAMES = frozenset({'edit_file', 'run_shell_command'})
@dataclass(frozen=True)
@ -24,6 +25,8 @@ class AgentRunResult:
content: str
turns: int
tool_calls: tuple[str, ...]
tool_trace: tuple[dict[str, Any], ...] = ()
stop_reason: str = 'completed'
trace_events: tuple[str, ...] = ()
@ -46,8 +49,11 @@ def run_agent_task(
tools = RuntimeToolRegistry(root=workspace_root, permission_context=permission_context)
messages = build_agent_messages(prompt=prompt, workspace_root=workspace_root, system_prompt=system_prompt)
tool_calls: list[str] = []
tool_trace: list[dict[str, Any]] = []
trace_events: list[str] = []
tool_required = prompt_requires_tool(prompt)
edit_required = prompt_requires_edit(prompt)
required_edit_target = infer_required_edit_target(prompt)
fallback_used = False
for turn in range(1, max_turns + 1):
@ -69,7 +75,7 @@ def run_agent_task(
fallback_used = True
if trace:
trace_events.append(f'fallback_tool[{turn}]={fallback["name"]} {json.dumps(fallback["arguments"], ensure_ascii=True)}')
_append_tool_result(messages, tools, fallback, tool_calls, trace_events if trace else None, turn)
_append_tool_result(messages, tools, fallback, tool_calls, tool_trace, trace_events if trace else None, turn)
continue
messages.append({'role': 'user', 'content': protocol_error_message(str(error))})
continue
@ -85,18 +91,34 @@ def run_agent_task(
fallback_used = True
if trace:
trace_events.append(f'fallback_tool[{turn}]={fallback["name"]} {json.dumps(fallback["arguments"], ensure_ascii=True)}')
_append_tool_result(messages, tools, fallback, tool_calls, trace_events if trace else None, turn)
_append_tool_result(messages, tools, fallback, tool_calls, tool_trace, trace_events if trace else None, turn)
continue
messages.append({'role': 'user', 'content': tool_required_message()})
continue
if edit_required and not has_successful_edit(tool_trace, required_edit_target):
if trace:
trace_events.append(f'edit_required_reject[{turn}]={payload["content"]}')
fallback = None
if not fallback_used:
fallback = infer_tool_call_from_prompt(prompt, prefer_edit=True)
if fallback is not None:
fallback_used = True
if trace:
trace_events.append(f'fallback_tool[{turn}]={fallback["name"]} {json.dumps(fallback["arguments"], ensure_ascii=True)}')
_append_tool_result(messages, tools, fallback, tool_calls, tool_trace, trace_events if trace else None, turn)
continue
messages.append({'role': 'user', 'content': edit_required_message()})
continue
return AgentRunResult(
content=payload['content'],
turns=turn,
tool_calls=tuple(tool_calls),
tool_trace=tuple(tool_trace),
stop_reason='completed',
trace_events=tuple(trace_events),
)
_append_tool_result(messages, tools, payload, tool_calls, trace_events if trace else None, turn)
_append_tool_result(messages, tools, payload, tool_calls, tool_trace, trace_events if trace else None, turn)
if trace:
trace_events.append(f'stopped=max_turns:{max_turns}')
@ -183,6 +205,13 @@ def tool_required_message() -> str:
)
def edit_required_message() -> str:
return (
'You must make a real workspace change before answering this request. '
'A read-only inspection is insufficient. Respond with exactly one JSON tool_call object that performs the edit.'
)
def prompt_requires_tool(prompt: str) -> bool:
lowered = prompt.lower()
action_words = ('read', 'search', 'find', 'inspect', 'open', 'edit', 'modify', 'create', 'write', 'run', 'execute', 'quote')
@ -190,7 +219,34 @@ def prompt_requires_tool(prompt: str) -> bool:
return any(word in lowered for word in action_words) and any(hint in lowered for hint in target_hints)
def infer_tool_call_from_prompt(prompt: str) -> dict[str, Any] | None:
def prompt_requires_edit(prompt: str) -> bool:
lowered = prompt.lower()
edit_words = ('edit', 'modify', 'change', 'update', 'write', 'create', 'add')
target_hints = ('.py', '.md', '.txt', 'file', 'workspace', 'train.py', 'src/')
return any(word in lowered for word in edit_words) and any(hint in lowered for hint in target_hints)
def infer_required_edit_target(prompt: str) -> str | None:
match = re.search(
r'(?:modify|edit|change|update|write|create|add(?:\s+to)?)\s+(?P<path>[\w./-]+)',
prompt,
flags=re.IGNORECASE,
)
if not match:
return None
return match.group('path')
def infer_tool_call_from_prompt(prompt: str, prefer_edit: bool = False) -> dict[str, Any] | None:
if prefer_edit:
edit_fallback = infer_edit_tool_call_from_prompt(prompt)
if edit_fallback is not None:
return edit_fallback
edit_fallback = infer_edit_tool_call_from_prompt(prompt)
if edit_fallback is not None:
return edit_fallback
read_match = re.search(
r'read\s+(?P<path>[\w./-]+)(?:\s+lines?\s+(?P<start>\d+)\s*(?:to|-)\s*(?P<end>\d+))?',
prompt,
@ -239,14 +295,90 @@ def infer_tool_call_from_prompt(prompt: str) -> dict[str, Any] | None:
},
}
if 'git status' in prompt.lower():
return {'name': 'git_status', 'arguments': {}}
if 'git diff' in prompt.lower():
return {'name': 'git_diff', 'arguments': {}}
if 'run tests' in prompt.lower() or 'test suite' in prompt.lower():
return {'name': 'run_tests', 'arguments': {}}
if 'run build' in prompt.lower() or 'build the project' in prompt.lower():
return {'name': 'run_build', 'arguments': {}}
if 'list directory' in prompt.lower() or 'list dir' in prompt.lower():
return {'name': 'list_dir', 'arguments': {'path': '.'}}
return None
def infer_edit_tool_call_from_prompt(prompt: str) -> dict[str, Any] | None:
modify_match = re.search(
r'(?:modify|edit|change|update)\s+(?P<path>[\w./-]+)\s+by\s+(?P<details>.+?)(?:\.|\n|$)',
prompt,
flags=re.IGNORECASE,
)
if modify_match:
path = modify_match.group('path')
details = modify_match.group('details').strip().lower()
if 'comment' in details:
marker = '# smoke-test comment: '
command = '\n'.join(
[
"python3 - <<'PY'",
'from pathlib import Path',
'import re',
f"target = Path({json.dumps(path)})",
"text = target.read_text(encoding='utf-8')",
f"marker = {json.dumps(marker)}",
"match = re.match(r'^# smoke-test comment: (\\d+)\\n', text)",
'if match:',
' current = int(match.group(1))',
" updated = f'{marker}{current + 1}\\n' + text[match.end():]",
'else:',
" updated = f'{marker}1\\n' + text",
"target.write_text(updated, encoding='utf-8')",
'PY',
]
)
return {
'name': 'run_shell_command',
'arguments': {'command': command, 'timeout_seconds': 20},
}
return None
def has_successful_edit(tool_trace: list[dict[str, Any]], required_path: str | None) -> bool:
for entry in tool_trace:
if not entry.get('success'):
continue
if entry.get('name') not in MUTATING_TOOL_NAMES:
continue
metadata = entry.get('metadata')
if required_path is None:
return True
if isinstance(metadata, dict) and metadata.get('path') == required_path:
return True
arguments = entry.get('arguments')
if isinstance(arguments, dict) and arguments.get('path') == required_path:
return True
if (
entry.get('name') == 'run_shell_command'
and isinstance(arguments, dict)
and isinstance(arguments.get('command'), str)
and required_path in arguments['command']
):
return True
return False
def _append_tool_result(
messages: list[dict[str, str]],
tools: RuntimeToolRegistry,
payload: dict[str, Any],
tool_calls: list[str],
tool_trace: list[dict[str, Any]],
trace_events: list[str] | None,
turn: int,
) -> None:
@ -256,6 +388,17 @@ def _append_tool_result(
if trace_events is not None:
trace_events.append(f'tool_call[{turn}]={tool_name} {json.dumps(arguments, ensure_ascii=True)}')
tool_result = tools.execute(tool_name, arguments)
tool_trace.append(
{
'turn': turn,
'name': tool_name,
'arguments': arguments,
'success': tool_result.success,
'output': tool_result.output,
'metadata': tool_result.metadata,
'error': tool_result.error,
}
)
if trace_events is not None:
trace_events.append(
f'tool_result[{turn}]={tool_name} success={tool_result.success} '

279
src/autoresearch_runner.py Normal file
View File

@ -0,0 +1,279 @@
from __future__ import annotations
import csv
import re
import subprocess
from dataclasses import asdict, dataclass
from datetime import datetime
from pathlib import Path
from typing import Any
RESULTS_HEADER = 'commit\tval_bpb\tmemory_gb\tstatus\tdescription\n'
DEFAULT_RUN_LOG = 'run.log'
DEFAULT_RESULTS_TSV = 'results.tsv'
DEFAULT_EXPERIMENT_TIMEOUT_SECONDS = 600
TRAIN_COMMAND = 'uv run train.py > run.log 2>&1'
class AutoresearchError(RuntimeError):
"""Raised when autoresearch setup or experiment execution fails."""
@dataclass(frozen=True)
class AutoresearchSetupReport:
root: str
branch: str
data_ready: bool
tokenizer_ready: bool
results_tsv_path: str
results_tsv_initialized: bool
cache_dir: str
suggested_tag: str
def to_dict(self) -> dict[str, Any]:
return asdict(self)
@dataclass(frozen=True)
class ExperimentMetrics:
success: bool
log_path: str
timed_out: bool
return_code: int
val_bpb: float | None = None
peak_vram_mb: float | None = None
training_seconds: float | None = None
total_seconds: float | None = None
mfu_percent: float | None = None
total_tokens_m: float | None = None
num_steps: int | None = None
num_params_m: float | None = None
depth: int | None = None
error: str | None = None
tail: str | None = None
def to_dict(self) -> dict[str, Any]:
return asdict(self)
def resolve_autoresearch_root(root: str | Path | None = None) -> Path:
if root is not None:
candidate = Path(root).resolve()
_validate_root(candidate)
return candidate
cwd = Path.cwd().resolve()
if _looks_like_autoresearch_root(cwd):
return cwd
workspace_candidate = Path(__file__).resolve().parents[3] / 'nodes' / 'autoresearch-macos'
if workspace_candidate.exists() and _looks_like_autoresearch_root(workspace_candidate):
return workspace_candidate
raise AutoresearchError(
'Could not resolve autoresearch root automatically. Pass --root pointing at a repo with '
'prepare.py, train.py, and program.md.'
)
def setup_autoresearch(root: str | Path | None = None, initialize_results: bool = True) -> AutoresearchSetupReport:
repo_root = resolve_autoresearch_root(root)
branch = _git_output(repo_root, ['git', 'branch', '--show-current']) or 'DETACHED'
cache_dir = Path.home() / '.cache' / 'autoresearch'
data_ready = (cache_dir / 'data').exists() and any((cache_dir / 'data').glob('shard_*.parquet'))
tokenizer_ready = (cache_dir / 'tokenizer' / 'tokenizer.pkl').exists() and (cache_dir / 'tokenizer' / 'token_bytes.pt').exists()
results_path = repo_root / DEFAULT_RESULTS_TSV
initialized = False
if initialize_results and not results_path.exists():
results_path.write_text(RESULTS_HEADER, encoding='utf-8')
initialized = True
suggested_tag = datetime.now().strftime('%b%d').lower()
return AutoresearchSetupReport(
root=str(repo_root),
branch=branch,
data_ready=data_ready,
tokenizer_ready=tokenizer_ready,
results_tsv_path=str(results_path),
results_tsv_initialized=initialized,
cache_dir=str(cache_dir),
suggested_tag=suggested_tag,
)
def parse_run_log(path: str | Path) -> ExperimentMetrics:
log_path = Path(path)
if not log_path.exists():
raise AutoresearchError(f'Run log not found: {log_path}')
text = log_path.read_text(encoding='utf-8', errors='replace')
metrics = _extract_metrics(text)
if metrics.get('val_bpb') is None:
tail = '\n'.join(text.splitlines()[-50:])
return ExperimentMetrics(
success=False,
log_path=str(log_path),
timed_out=False,
return_code=1,
error='run log did not contain a final val_bpb summary',
tail=tail,
)
return ExperimentMetrics(
success=True,
log_path=str(log_path),
timed_out=False,
return_code=0,
val_bpb=metrics.get('val_bpb'),
peak_vram_mb=metrics.get('peak_vram_mb'),
training_seconds=metrics.get('training_seconds'),
total_seconds=metrics.get('total_seconds'),
mfu_percent=metrics.get('mfu_percent'),
total_tokens_m=metrics.get('total_tokens_M'),
num_steps=int(metrics['num_steps']) if metrics.get('num_steps') is not None else None,
num_params_m=metrics.get('num_params_M'),
depth=int(metrics['depth']) if metrics.get('depth') is not None else None,
)
def run_experiment(
root: str | Path | None = None,
command: str = TRAIN_COMMAND,
log_path: str | Path = DEFAULT_RUN_LOG,
timeout_seconds: int = DEFAULT_EXPERIMENT_TIMEOUT_SECONDS,
) -> ExperimentMetrics:
repo_root = resolve_autoresearch_root(root)
target_log = Path(log_path)
if not target_log.is_absolute():
target_log = repo_root / target_log
try:
run = subprocess.run(
['/bin/zsh', '-lc', command],
cwd=repo_root,
capture_output=True,
text=True,
timeout=timeout_seconds,
check=False,
)
except subprocess.TimeoutExpired:
tail = ''
if target_log.exists():
tail = '\n'.join(target_log.read_text(encoding='utf-8', errors='replace').splitlines()[-50:])
return ExperimentMetrics(
success=False,
log_path=str(target_log),
timed_out=True,
return_code=-1,
error=f'experiment command timed out after {timeout_seconds} seconds',
tail=tail,
)
if run.returncode != 0:
tail = ''
if target_log.exists():
tail = '\n'.join(target_log.read_text(encoding='utf-8', errors='replace').splitlines()[-50:])
else:
tail = '\n'.join(part for part in (run.stdout.strip(), run.stderr.strip()) if part)
return ExperimentMetrics(
success=False,
log_path=str(target_log),
timed_out=False,
return_code=run.returncode,
error='experiment command failed',
tail=tail,
)
metrics = parse_run_log(target_log)
return ExperimentMetrics(
**{
**metrics.to_dict(),
'success': metrics.success,
'return_code': run.returncode,
'timed_out': False,
}
)
def append_results_row(
root: str | Path | None,
commit: str,
metrics: ExperimentMetrics,
status: str,
description: str,
results_path: str | Path = DEFAULT_RESULTS_TSV,
) -> Path:
repo_root = resolve_autoresearch_root(root)
target = Path(results_path)
if not target.is_absolute():
target = repo_root / target
if not target.exists():
target.write_text(RESULTS_HEADER, encoding='utf-8')
val_bpb = metrics.val_bpb if metrics.val_bpb is not None else 0.0
memory_gb = (metrics.peak_vram_mb or 0.0) / 1024
row = f'{commit}\t{val_bpb:.6f}\t{memory_gb:.1f}\t{status}\t{description}\n'
with target.open('a', encoding='utf-8') as handle:
handle.write(row)
return target
def best_recorded_bpb(root: str | Path | None, results_path: str | Path = DEFAULT_RESULTS_TSV) -> float | None:
repo_root = resolve_autoresearch_root(root)
target = Path(results_path)
if not target.is_absolute():
target = repo_root / target
if not target.exists():
return None
best: float | None = None
with target.open('r', encoding='utf-8') as handle:
reader = csv.DictReader(handle, delimiter='\t')
for row in reader:
if row.get('status') == 'crash':
continue
try:
value = float(row['val_bpb'])
except (TypeError, ValueError, KeyError):
continue
if best is None or value < best:
best = value
return best
def short_head_commit(root: str | Path | None) -> str:
repo_root = resolve_autoresearch_root(root)
commit = _git_output(repo_root, ['git', 'rev-parse', '--short', 'HEAD'])
return commit or 'unknown'
def _validate_root(root: Path) -> None:
if not _looks_like_autoresearch_root(root):
raise AutoresearchError(
f'{root} does not look like an autoresearch repo. Expected prepare.py, train.py, and program.md.'
)
def _looks_like_autoresearch_root(path: Path) -> bool:
return (path / 'prepare.py').exists() and (path / 'train.py').exists() and (path / 'program.md').exists()
def _git_output(root: Path, args: list[str]) -> str:
result = subprocess.run(args, cwd=root, capture_output=True, text=True, check=False)
if result.returncode != 0:
return ''
return result.stdout.strip()
def _extract_metrics(text: str) -> dict[str, float]:
keys = (
'val_bpb',
'training_seconds',
'total_seconds',
'peak_vram_mb',
'mfu_percent',
'total_tokens_M',
'num_steps',
'num_params_M',
'depth',
)
metrics: dict[str, float] = {}
for key in keys:
match = re.search(rf'^{re.escape(key)}:\s+([0-9.]+)$', text, flags=re.MULTILINE)
if match:
metrics[key] = float(match.group(1))
return metrics

646
src/autoresearch_worker.py Normal file
View File

@ -0,0 +1,646 @@
from __future__ import annotations
import json
import subprocess
from dataclasses import asdict, dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from .llm_backend import DEFAULT_OLLAMA_HOST, DEFAULT_OLLAMA_MODEL
from .task_packet import TaskPacket
from .worker_api import create_worker, run_worker
from .autoresearch_runner import (
DEFAULT_RESULTS_TSV,
ExperimentMetrics,
append_results_row,
best_recorded_bpb,
resolve_autoresearch_root,
run_experiment,
setup_autoresearch,
short_head_commit,
)
DEFAULT_AUTORESEARCH_TIMEOUT_SECONDS = 600
DEFAULT_STATE_FILE = '.autoresearch_state.json'
DEFAULT_MEMORY_FILE = 'experiment_memory.jsonl'
AUTORESEARCH_BRANCH_PREFIX = 'autoresearch/'
@dataclass(frozen=True)
class AutoresearchExperimentPacket:
objective: str
description: str
train_command: str = 'uv run train.py > run.log 2>&1'
timeout_seconds: int = DEFAULT_AUTORESEARCH_TIMEOUT_SECONDS
log_path: str = 'run.log'
results_tsv: str = DEFAULT_RESULTS_TSV
syntax_check_command: str = 'python3 -m py_compile train.py'
def to_dict(self) -> dict[str, Any]:
return asdict(self)
@classmethod
def from_dict(cls, payload: dict[str, Any]) -> 'AutoresearchExperimentPacket':
return cls(
objective=str(payload.get('objective', '')),
description=str(payload.get('description', '')),
train_command=str(payload.get('train_command', 'uv run train.py > run.log 2>&1')),
timeout_seconds=int(payload.get('timeout_seconds', DEFAULT_AUTORESEARCH_TIMEOUT_SECONDS)),
log_path=str(payload.get('log_path', 'run.log')),
results_tsv=str(payload.get('results_tsv', DEFAULT_RESULTS_TSV)),
syntax_check_command=str(payload.get('syntax_check_command', 'python3 -m py_compile train.py')),
)
@dataclass(frozen=True)
class PendingExperiment:
commit: str
base_commit: str
description: str
packet: dict[str, Any]
worker: dict[str, Any]
experiment: dict[str, Any]
recommended_status: str
results_tsv: str
log_path: str
created_at: str
def to_dict(self) -> dict[str, Any]:
return asdict(self)
@classmethod
def from_dict(cls, payload: dict[str, Any]) -> 'PendingExperiment':
return cls(
commit=payload['commit'],
base_commit=payload['base_commit'],
description=payload['description'],
packet=payload['packet'],
worker=payload['worker'],
experiment=payload['experiment'],
recommended_status=payload['recommended_status'],
results_tsv=payload['results_tsv'],
log_path=payload['log_path'],
created_at=payload['created_at'],
)
@dataclass(frozen=True)
class AutoresearchState:
root: str
branch: str
baseline_commit: str | None = None
baseline_bpb: float | None = None
best_commit: str | None = None
best_bpb: float | None = None
pending_experiment: dict[str, Any] | None = None
last_decision: str | None = None
updated_at: str = ''
def to_dict(self) -> dict[str, Any]:
return asdict(self)
@classmethod
def from_dict(cls, payload: dict[str, Any]) -> 'AutoresearchState':
return cls(
root=payload['root'],
branch=payload['branch'],
baseline_commit=payload.get('baseline_commit'),
baseline_bpb=payload.get('baseline_bpb'),
best_commit=payload.get('best_commit'),
best_bpb=payload.get('best_bpb'),
pending_experiment=payload.get('pending_experiment'),
last_decision=payload.get('last_decision'),
updated_at=payload.get('updated_at', ''),
)
def load_autoresearch_packet(path: str | Path) -> AutoresearchExperimentPacket:
payload = json.loads(Path(path).read_text(encoding='utf-8'))
if not isinstance(payload, dict):
raise ValueError('autoresearch packet must be a JSON object')
packet = AutoresearchExperimentPacket.from_dict(payload)
validate_autoresearch_packet(packet)
return packet
def validate_autoresearch_packet(packet: AutoresearchExperimentPacket) -> None:
errors: list[str] = []
if not packet.objective.strip():
errors.append('objective must not be empty')
if not packet.description.strip():
errors.append('description must not be empty')
if packet.timeout_seconds < 1:
errors.append('timeout_seconds must be >= 1')
if not packet.train_command.strip():
errors.append('train_command must not be empty')
if errors:
raise ValueError('; '.join(errors))
def render_autoresearch_task_packet(packet: AutoresearchExperimentPacket, repo_root: Path) -> TaskPacket:
return TaskPacket(
objective=packet.objective,
scope=(
'Modify only train.py in the autoresearch repo. You may read program.md, README.md, and train.py for context, '
'but train.py is the only file you may change. Editing any other path is failure. '
'Do not edit prepare.py, pyproject.toml, or dependencies. Do not run the full training experiment yourself; '
'the manager will run it after your code change. Use syntax and git inspection tools when helpful.'
),
repo=str(repo_root),
branch_policy='Stay on the current branch. Do not create or switch branches.',
acceptance_tests=(packet.syntax_check_command,),
commit_policy='Do not commit. The manager records and evaluates the experiment after your edit.',
reporting_contract=(
'Return a concise summary of the train.py change, intended effect on val_bpb, and any blockers.'
),
escalation_policy='Stop if a required file outside train.py would need modification or if syntax cannot be preserved.',
)
def run_autoresearch_packet(
packet_source: AutoresearchExperimentPacket | str | Path,
root: str | Path | None = None,
model: str = DEFAULT_OLLAMA_MODEL,
host: str = DEFAULT_OLLAMA_HOST,
trace: bool = False,
) -> dict[str, Any]:
repo_root = resolve_autoresearch_root(root)
setup = setup_autoresearch(repo_root, initialize_results=True)
packet = packet_source if isinstance(packet_source, AutoresearchExperimentPacket) else load_autoresearch_packet(packet_source)
state = load_autoresearch_state(repo_root)
base_commit = short_head_commit(repo_root)
worker = create_worker(root=repo_root, model=model, host=host)
worker_run = run_worker(
worker.worker_id,
render_autoresearch_task_packet(packet, repo_root),
trace=trace,
)
changed_files = _project_changed_files(worker_run.last_result.get('changed_files', []) if worker_run.last_result else [])
if 'train.py' not in changed_files:
result = {
'setup': setup.to_dict(),
'state': state.to_dict(),
'worker': worker_run.to_dict(),
'experiment': None,
'results_tsv': str((repo_root / packet.results_tsv).resolve()),
'recommended_status': 'discard',
'error': 'worker did not modify train.py',
}
append_memory_event(repo_root, 'run_skipped', result)
return result
commit = _commit_train_change(repo_root, packet.description)
experiment = run_experiment(
root=repo_root,
command=packet.train_command,
log_path=packet.log_path,
timeout_seconds=packet.timeout_seconds,
)
previous_best = state.best_bpb if state.best_bpb is not None else best_recorded_bpb(repo_root, packet.results_tsv)
recommended_status = _recommend_status(experiment, previous_best)
pending = PendingExperiment(
commit=commit,
base_commit=base_commit,
description=packet.description,
packet=packet.to_dict(),
worker=worker_run.to_dict(),
experiment=experiment.to_dict(),
recommended_status=recommended_status,
results_tsv=packet.results_tsv,
log_path=packet.log_path,
created_at=_iso_now(),
)
updated_state = replace_autoresearch_state(
state,
branch=_current_branch(repo_root),
pending_experiment=pending.to_dict(),
updated_at=_iso_now(),
)
save_autoresearch_state(updated_state, repo_root)
result = {
'setup': setup.to_dict(),
'state': updated_state.to_dict(),
'worker': worker_run.to_dict(),
'experiment': experiment.to_dict(),
'results_tsv': str((repo_root / packet.results_tsv).resolve()),
'commit': commit,
'recommended_status': recommended_status,
'description': packet.description,
}
append_memory_event(repo_root, 'candidate_run', result)
return result
def ensure_autoresearch_baseline(
root: str | Path | None = None,
command: str = 'uv run train.py > run.log 2>&1',
log_path: str = 'run.log',
results_tsv: str = DEFAULT_RESULTS_TSV,
timeout_seconds: int = DEFAULT_AUTORESEARCH_TIMEOUT_SECONDS,
) -> dict[str, Any]:
repo_root = resolve_autoresearch_root(root)
setup = setup_autoresearch(repo_root, initialize_results=True)
state = load_autoresearch_state(repo_root)
existing_best = state.best_bpb if state.best_bpb is not None else best_recorded_bpb(repo_root, results_tsv)
if existing_best is not None:
if state.best_bpb is None:
state = replace_autoresearch_state(
state,
branch=_current_branch(repo_root),
best_bpb=existing_best,
updated_at=_iso_now(),
)
save_autoresearch_state(state, repo_root)
result = {
'setup': setup.to_dict(),
'state': state.to_dict(),
'baseline_created': False,
'reason': 'baseline already recorded',
}
append_memory_event(repo_root, 'baseline_skipped', result)
return result
commit = short_head_commit(repo_root)
experiment = run_experiment(repo_root, command=command, log_path=log_path, timeout_seconds=timeout_seconds)
if not experiment.success or experiment.val_bpb is None:
result = {
'setup': setup.to_dict(),
'state': state.to_dict(),
'baseline_created': False,
'experiment': experiment.to_dict(),
'error': 'baseline experiment failed',
}
append_memory_event(repo_root, 'baseline_failed', result)
return result
append_results_row(repo_root, commit=commit, metrics=experiment, status='keep', description='baseline', results_path=results_tsv)
updated_state = replace_autoresearch_state(
state,
branch=_current_branch(repo_root),
baseline_commit=commit,
baseline_bpb=experiment.val_bpb,
best_commit=commit,
best_bpb=experiment.val_bpb,
pending_experiment=None,
last_decision='baseline',
updated_at=_iso_now(),
)
save_autoresearch_state(updated_state, repo_root)
result = {
'setup': setup.to_dict(),
'state': updated_state.to_dict(),
'baseline_created': True,
'experiment': experiment.to_dict(),
'commit': commit,
}
append_memory_event(repo_root, 'baseline_created', result)
return result
def keep_autoresearch_candidate(root: str | Path | None = None) -> dict[str, Any]:
repo_root = resolve_autoresearch_root(root)
state = load_autoresearch_state(repo_root)
pending = _require_pending_experiment(state)
metrics = ExperimentMetrics(**pending.experiment)
append_results_row(
repo_root,
commit=pending.commit,
metrics=metrics,
status='keep',
description=pending.description,
results_path=pending.results_tsv,
)
improved = metrics.val_bpb is not None and (state.best_bpb is None or metrics.val_bpb < state.best_bpb)
updated_state = replace_autoresearch_state(
state,
branch=_current_branch(repo_root),
best_commit=pending.commit if improved else state.best_commit,
best_bpb=metrics.val_bpb if improved else state.best_bpb,
pending_experiment=None,
last_decision='keep',
updated_at=_iso_now(),
)
save_autoresearch_state(updated_state, repo_root)
result = {
'decision': 'keep',
'commit': pending.commit,
'improved_best': improved,
'state': updated_state.to_dict(),
'experiment': pending.experiment,
}
append_memory_event(repo_root, 'keep', result)
return result
def discard_autoresearch_candidate(root: str | Path | None = None) -> dict[str, Any]:
repo_root = resolve_autoresearch_root(root)
state = load_autoresearch_state(repo_root)
pending = _require_pending_experiment(state)
metrics = ExperimentMetrics(**pending.experiment)
final_status = 'crash' if not metrics.success or metrics.val_bpb is None else 'discard'
_git(repo_root, ['git', 'reset', '--hard', pending.base_commit], check=False)
append_results_row(
repo_root,
commit=pending.commit,
metrics=metrics,
status=final_status,
description=pending.description,
results_path=pending.results_tsv,
)
updated_state = replace_autoresearch_state(
state,
branch=_current_branch(repo_root),
pending_experiment=None,
last_decision=final_status,
updated_at=_iso_now(),
)
save_autoresearch_state(updated_state, repo_root)
result = {
'decision': final_status,
'reverted_to_commit': pending.base_commit,
'current_commit': short_head_commit(repo_root),
'state': updated_state.to_dict(),
'experiment': pending.experiment,
}
append_memory_event(repo_root, final_status, result)
return result
def retry_autoresearch_candidate(root: str | Path | None = None) -> dict[str, Any]:
repo_root = resolve_autoresearch_root(root)
state = load_autoresearch_state(repo_root)
pending = _require_pending_experiment(state)
packet = AutoresearchExperimentPacket.from_dict(pending.packet)
experiment = run_experiment(
root=repo_root,
command=packet.train_command,
log_path=packet.log_path,
timeout_seconds=packet.timeout_seconds,
)
previous_best = state.best_bpb if state.best_bpb is not None else best_recorded_bpb(repo_root, packet.results_tsv)
recommended_status = _recommend_status(experiment, previous_best)
refreshed = PendingExperiment(
commit=pending.commit,
base_commit=pending.base_commit,
description=pending.description,
packet=pending.packet,
worker=pending.worker,
experiment=experiment.to_dict(),
recommended_status=recommended_status,
results_tsv=pending.results_tsv,
log_path=pending.log_path,
created_at=pending.created_at,
)
updated_state = replace_autoresearch_state(
state,
pending_experiment=refreshed.to_dict(),
updated_at=_iso_now(),
)
save_autoresearch_state(updated_state, repo_root)
result = {
'state': updated_state.to_dict(),
'experiment': experiment.to_dict(),
'recommended_status': recommended_status,
'commit': pending.commit,
'description': pending.description,
}
append_memory_event(repo_root, 'retry', result)
return result
def loop_autoresearch(
packet_source: AutoresearchExperimentPacket | str | Path,
root: str | Path | None = None,
model: str = DEFAULT_OLLAMA_MODEL,
host: str = DEFAULT_OLLAMA_HOST,
iterations: int = 1,
retry_limit: int = 1,
require_isolated_branch: bool = True,
trace: bool = False,
) -> dict[str, Any]:
if iterations < 1:
raise ValueError('iterations must be >= 1')
repo_root = resolve_autoresearch_root(root)
if require_isolated_branch and not _is_isolated_branch(_current_branch(repo_root)):
raise RuntimeError(
'autoresearch loop requires an isolated branch named autoresearch/<tag>. '
'Run `python3 -m src.main autoresearch isolate --root ... --branch autoresearch/<tag> --create` first, '
'or pass --allow-any-branch for a local smoke test.'
)
packet = packet_source if isinstance(packet_source, AutoresearchExperimentPacket) else load_autoresearch_packet(packet_source)
setup = setup_autoresearch(repo_root, initialize_results=True)
baseline = ensure_autoresearch_baseline(repo_root, results_tsv=packet.results_tsv)
history: list[dict[str, Any]] = []
for iteration in range(1, iterations + 1):
run_result = run_autoresearch_packet(packet, root=repo_root, model=model, host=host, trace=trace)
retries = 0
while run_result.get('recommended_status') == 'crash' and retries < retry_limit:
retries += 1
run_result = retry_autoresearch_candidate(repo_root)
run_result['retry_count'] = retries
state = load_autoresearch_state(repo_root)
if state.pending_experiment is None:
history.append(
{
'iteration': iteration,
'run': run_result,
'decision': {
'decision': 'no_pending_experiment',
'state': state.to_dict(),
},
}
)
continue
decision = (
keep_autoresearch_candidate(repo_root)
if run_result.get('recommended_status') == 'keep'
else discard_autoresearch_candidate(repo_root)
)
history.append(
{
'iteration': iteration,
'run': run_result,
'decision': decision,
}
)
result = {
'setup': setup.to_dict(),
'baseline': baseline,
'state': load_autoresearch_state(repo_root).to_dict(),
'history': history,
}
append_memory_event(repo_root, 'loop_complete', {'iterations': iterations, 'history_size': len(history)})
return result
def load_autoresearch_state(root: str | Path | None = None) -> AutoresearchState:
repo_root = resolve_autoresearch_root(root)
path = repo_root / DEFAULT_STATE_FILE
if not path.exists():
state = AutoresearchState(
root=str(repo_root),
branch=_current_branch(repo_root),
updated_at=_iso_now(),
)
save_autoresearch_state(state, repo_root)
return state
payload = json.loads(path.read_text(encoding='utf-8'))
return AutoresearchState.from_dict(payload)
def save_autoresearch_state(state: AutoresearchState, root: str | Path | None = None) -> Path:
repo_root = resolve_autoresearch_root(root or state.root)
path = repo_root / DEFAULT_STATE_FILE
path.write_text(json.dumps(state.to_dict(), indent=2), encoding='utf-8')
return path
def append_memory_event(root: str | Path | None, event_type: str, payload: dict[str, Any]) -> Path:
repo_root = resolve_autoresearch_root(root)
path = repo_root / DEFAULT_MEMORY_FILE
with path.open('a', encoding='utf-8') as handle:
handle.write(json.dumps({'time': _iso_now(), 'event': event_type, 'payload': payload}) + '\n')
return path
def replace_autoresearch_state(state: AutoresearchState, **changes: Any) -> AutoresearchState:
payload = state.to_dict()
payload.update(changes)
return AutoresearchState.from_dict(payload)
def autoresearch_status(root: str | Path | None = None) -> dict[str, Any]:
repo_root = resolve_autoresearch_root(root)
state = load_autoresearch_state(repo_root)
setup = setup_autoresearch(repo_root, initialize_results=True)
memory_path = repo_root / DEFAULT_MEMORY_FILE
results_path = repo_root / DEFAULT_RESULTS_TSV
return {
'setup': setup.to_dict(),
'state': state.to_dict(),
'current_branch': _current_branch(repo_root),
'isolated_branch': _is_isolated_branch(_current_branch(repo_root)),
'paths': {
'state_file': str((repo_root / DEFAULT_STATE_FILE).resolve()),
'memory_file': str(memory_path.resolve()),
'results_tsv': str(results_path.resolve()),
},
'memory_event_count': _count_memory_events(memory_path),
'has_pending_experiment': state.pending_experiment is not None,
}
def ensure_autoresearch_branch(
root: str | Path | None = None,
branch: str | None = None,
create: bool = False,
from_ref: str = 'HEAD',
) -> dict[str, Any]:
repo_root = resolve_autoresearch_root(root)
previous_branch = _current_branch(repo_root)
target_branch = branch or (previous_branch if _is_isolated_branch(previous_branch) else _suggest_branch_name())
if not _is_isolated_branch(target_branch):
raise ValueError(f'branch must start with {AUTORESEARCH_BRANCH_PREFIX}')
created = False
if previous_branch != target_branch:
verify = _git(repo_root, ['git', 'rev-parse', '--verify', target_branch], check=False)
if verify.returncode == 0:
_git(repo_root, ['git', 'checkout', target_branch], check=True)
elif create:
_git(repo_root, ['git', 'checkout', '-b', target_branch, from_ref], check=True)
created = True
else:
raise RuntimeError(
f'Branch {target_branch} does not exist. Re-run with create=True or use '
'`python3 -m src.main autoresearch isolate --create`.'
)
state = load_autoresearch_state(repo_root)
updated_state = replace_autoresearch_state(
state,
branch=_current_branch(repo_root),
updated_at=_iso_now(),
)
save_autoresearch_state(updated_state, repo_root)
result = {
'root': str(repo_root),
'previous_branch': previous_branch,
'current_branch': _current_branch(repo_root),
'isolated_branch': True,
'created': created,
'suggested_branch': _suggest_branch_name(),
}
append_memory_event(repo_root, 'branch_isolated', result)
return result
def _project_changed_files(paths: list[str]) -> list[str]:
return sorted(path for path in paths if not path.startswith('.port_'))
def _recommend_status(metrics: ExperimentMetrics, previous_best: float | None) -> str:
if not metrics.success or metrics.val_bpb is None:
return 'crash'
if previous_best is None or metrics.val_bpb < previous_best:
return 'keep'
return 'discard'
def _commit_train_change(root: Path, description: str) -> str:
message = f'autoresearch: {description.strip()}'
subprocess.run(['git', 'add', '--', 'train.py'], cwd=root, capture_output=True, text=True, check=False)
commit = subprocess.run(
['git', 'commit', '-m', message[:120]],
cwd=root,
capture_output=True,
text=True,
check=False,
)
if commit.returncode == 0:
return short_head_commit(root)
head = short_head_commit(root)
return f'{head}-dirty' if head != 'unknown' else 'uncommitted'
def _require_pending_experiment(state: AutoresearchState) -> PendingExperiment:
if state.pending_experiment is None:
raise RuntimeError('No pending autoresearch experiment is recorded.')
return PendingExperiment.from_dict(state.pending_experiment)
def _current_branch(root: Path) -> str:
result = _git(root, ['git', 'branch', '--show-current'], check=False)
branch = result.stdout.strip()
return branch or 'DETACHED'
def _git(root: Path, args: list[str], check: bool = False) -> subprocess.CompletedProcess[str]:
result = subprocess.run(args, cwd=root, capture_output=True, text=True, check=False)
if check and result.returncode != 0:
raise RuntimeError(result.stderr.strip() or result.stdout.strip() or f'git command failed: {" ".join(args)}')
return result
def _iso_now() -> str:
return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace('+00:00', 'Z')
def _count_memory_events(path: Path) -> int:
if not path.exists():
return 0
return sum(1 for line in path.read_text(encoding='utf-8').splitlines() if line.strip())
def _suggest_branch_name() -> str:
return f'{AUTORESEARCH_BRANCH_PREFIX}{datetime.now().strftime("%b%d").lower()}'
def _is_isolated_branch(branch: str) -> bool:
return branch.startswith(AUTORESEARCH_BRANCH_PREFIX)

View File

@ -1,8 +1,20 @@
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
from .autoresearch_runner import parse_run_log, setup_autoresearch
from .autoresearch_worker import (
autoresearch_status,
discard_autoresearch_candidate,
ensure_autoresearch_baseline,
ensure_autoresearch_branch,
keep_autoresearch_candidate,
loop_autoresearch,
run_autoresearch_packet,
)
from .bootstrap_graph import build_bootstrap_graph
from .command_graph import build_command_graph
from .commands import execute_command, get_command, get_commands, render_command_index
@ -18,6 +30,7 @@ from .runtime import PortRuntime
from .session_store import load_session
from .setup import run_setup
from .task import DEFAULT_SYSTEM_PROMPT, run_local_task
from .worker_api import close_worker, create_worker, list_workers, load_worker, resume_worker, run_worker
from .tool_pool import assemble_tool_pool
from .tools import execute_tool, get_tool, get_tools, render_tool_index
@ -108,6 +121,81 @@ def build_parser() -> argparse.ArgumentParser:
agent_parser.add_argument('--deny-tool', action='append', default=[])
agent_parser.add_argument('--deny-prefix', action='append', default=[])
agent_parser.add_argument('--trace', action='store_true')
worker_parser = subparsers.add_parser('worker', help='manage structured local Ollama workers')
worker_subparsers = worker_parser.add_subparsers(dest='worker_command', required=True)
worker_create = worker_subparsers.add_parser('create', help='create a worker record')
worker_create.add_argument('--root', default='.')
worker_create.add_argument('--model', default=DEFAULT_OLLAMA_MODEL)
worker_create.add_argument('--host', default=DEFAULT_OLLAMA_HOST)
worker_subparsers.add_parser('list', help='list stored worker records')
worker_run = worker_subparsers.add_parser('run', help='run a task packet on a worker')
worker_run.add_argument('worker_id')
worker_run.add_argument('--packet', required=True)
worker_run.add_argument('--trace', action='store_true')
worker_status = worker_subparsers.add_parser('status', help='show worker state')
worker_status.add_argument('worker_id')
worker_resume = worker_subparsers.add_parser('resume', help='rerun the last stored task packet')
worker_resume.add_argument('worker_id')
worker_resume.add_argument('--trace', action='store_true')
worker_close = worker_subparsers.add_parser('close', help='close a worker record')
worker_close.add_argument('worker_id')
autoresearch_parser = subparsers.add_parser('autoresearch', help='project-specific control plane for autoresearch experiments')
autoresearch_subparsers = autoresearch_parser.add_subparsers(dest='autoresearch_command', required=True)
autoresearch_setup = autoresearch_subparsers.add_parser('setup', help='verify autoresearch repo state and initialize results.tsv')
autoresearch_setup.add_argument('--root')
autoresearch_setup.add_argument('--no-init-results', action='store_true')
autoresearch_run = autoresearch_subparsers.add_parser('run', help='run one autoresearch experiment packet through the worker + experiment runner')
autoresearch_run.add_argument('--root')
autoresearch_run.add_argument('--packet', required=True)
autoresearch_run.add_argument('--model', default=DEFAULT_OLLAMA_MODEL)
autoresearch_run.add_argument('--host', default=DEFAULT_OLLAMA_HOST)
autoresearch_run.add_argument('--trace', action='store_true')
autoresearch_status_parser = autoresearch_subparsers.add_parser('status', help='show current autoresearch control-plane state')
autoresearch_status_parser.add_argument('--root')
autoresearch_baseline = autoresearch_subparsers.add_parser('baseline', help='record the baseline experiment if results.tsv is still empty')
autoresearch_baseline.add_argument('--root')
autoresearch_baseline.add_argument('--command', dest='train_command', default='uv run train.py > run.log 2>&1')
autoresearch_baseline.add_argument('--log-path', default='run.log')
autoresearch_baseline.add_argument('--results-tsv', default='results.tsv')
autoresearch_baseline.add_argument('--timeout-seconds', type=int, default=600)
autoresearch_keep = autoresearch_subparsers.add_parser('keep', help='finalize the pending autoresearch candidate as keep')
autoresearch_keep.add_argument('--root')
autoresearch_discard = autoresearch_subparsers.add_parser('discard', help='finalize the pending autoresearch candidate as discard or crash')
autoresearch_discard.add_argument('--root')
autoresearch_isolate = autoresearch_subparsers.add_parser('isolate', help='create or switch to an autoresearch/<tag> branch')
autoresearch_isolate.add_argument('--root')
autoresearch_isolate.add_argument('--branch')
autoresearch_isolate.add_argument('--create', action='store_true')
autoresearch_isolate.add_argument('--from-ref', default='HEAD')
autoresearch_loop = autoresearch_subparsers.add_parser('loop', help='run repeated autoresearch iterations with automatic baseline and keep/discard decisions')
autoresearch_loop.add_argument('--root')
autoresearch_loop.add_argument('--packet', required=True)
autoresearch_loop.add_argument('--model', default=DEFAULT_OLLAMA_MODEL)
autoresearch_loop.add_argument('--host', default=DEFAULT_OLLAMA_HOST)
autoresearch_loop.add_argument('--iterations', type=int, default=1)
autoresearch_loop.add_argument('--retry-limit', type=int, default=1)
autoresearch_loop.add_argument('--allow-any-branch', action='store_true')
autoresearch_loop.add_argument('--trace', action='store_true')
autoresearch_parse_log = autoresearch_subparsers.add_parser('parse-log', help='parse a train.py run log into machine-readable metrics')
autoresearch_parse_log.add_argument('--root')
autoresearch_parse_log.add_argument('log_path')
return parser
@ -272,6 +360,105 @@ def main(argv: list[str] | None = None) -> int:
print(error, file=sys.stderr)
return 1
return 0
if args.command == 'worker':
try:
if args.worker_command == 'list':
workers = list_workers()
print(json.dumps([worker.to_dict() for worker in workers], indent=2))
return 0
if args.worker_command == 'create':
worker = create_worker(root=args.root, model=args.model, host=args.host)
print(json.dumps(worker.to_dict(), indent=2))
return 0
if args.worker_command == 'run':
worker = run_worker(args.worker_id, args.packet, trace=args.trace)
print(json.dumps(worker.to_dict(), indent=2))
return 0
if args.worker_command == 'status':
worker = load_worker(args.worker_id)
print(json.dumps(worker.to_dict(), indent=2))
return 0
if args.worker_command == 'resume':
worker = resume_worker(args.worker_id, trace=args.trace)
print(json.dumps(worker.to_dict(), indent=2))
return 0
if args.worker_command == 'close':
worker = close_worker(args.worker_id)
print(json.dumps(worker.to_dict(), indent=2))
return 0
except Exception as error:
print(error, file=sys.stderr)
return 1
if args.command == 'autoresearch':
try:
if args.autoresearch_command == 'setup':
report = setup_autoresearch(
root=args.root,
initialize_results=not args.no_init_results,
)
print(json.dumps(report.to_dict(), indent=2))
return 0
if args.autoresearch_command == 'run':
result = run_autoresearch_packet(
args.packet,
root=args.root,
model=args.model,
host=args.host,
trace=args.trace,
)
print(json.dumps(result, indent=2))
return 0
if args.autoresearch_command == 'status':
print(json.dumps(autoresearch_status(root=args.root), indent=2))
return 0
if args.autoresearch_command == 'baseline':
result = ensure_autoresearch_baseline(
root=args.root,
command=args.train_command,
log_path=args.log_path,
results_tsv=args.results_tsv,
timeout_seconds=args.timeout_seconds,
)
print(json.dumps(result, indent=2))
return 0
if args.autoresearch_command == 'keep':
print(json.dumps(keep_autoresearch_candidate(root=args.root), indent=2))
return 0
if args.autoresearch_command == 'discard':
print(json.dumps(discard_autoresearch_candidate(root=args.root), indent=2))
return 0
if args.autoresearch_command == 'isolate':
result = ensure_autoresearch_branch(
root=args.root,
branch=args.branch,
create=args.create,
from_ref=args.from_ref,
)
print(json.dumps(result, indent=2))
return 0
if args.autoresearch_command == 'loop':
result = loop_autoresearch(
args.packet,
root=args.root,
model=args.model,
host=args.host,
iterations=args.iterations,
retry_limit=args.retry_limit,
require_isolated_branch=not args.allow_any_branch,
trace=args.trace,
)
print(json.dumps(result, indent=2))
return 0
if args.autoresearch_command == 'parse-log':
log_path = args.log_path
if not Path(log_path).is_absolute() and args.root:
log_path = str(Path(args.root) / log_path)
metrics = parse_run_log(log_path)
print(json.dumps(metrics.to_dict(), indent=2))
return 0
except Exception as error:
print(error, file=sys.stderr)
return 1
parser.error(f'unknown command: {args.command}')
return 2

View File

@ -43,6 +43,14 @@ class RuntimeToolResult:
TOOL_DEFINITIONS = (
RuntimeToolDefinition(
name='list_dir',
description='List directory entries inside the workspace root.',
arguments={
'path': 'Optional workspace-relative directory path. Defaults to the workspace root.',
'max_entries': 'Optional cap on returned entries. Defaults to 200.',
},
),
RuntimeToolDefinition(
name='read_file',
description='Read a file from the workspace. Optionally limit the response to a line range.',
@ -78,6 +86,35 @@ TOOL_DEFINITIONS = (
'timeout_seconds': 'Optional timeout in seconds. Defaults to 20.',
},
),
RuntimeToolDefinition(
name='git_status',
description='Show git status for the current workspace.',
arguments={},
),
RuntimeToolDefinition(
name='git_diff',
description='Show git diff output, optionally scoped to a path or staged changes.',
arguments={
'path': 'Optional workspace-relative path to diff.',
'staged': 'Optional boolean. When true, show staged diff.',
},
),
RuntimeToolDefinition(
name='run_tests',
description='Run the project test command and capture the result.',
arguments={
'command': 'Optional shell command. Defaults to python3 -m unittest discover -s tests -v.',
'timeout_seconds': 'Optional timeout in seconds. Defaults to 120.',
},
),
RuntimeToolDefinition(
name='run_build',
description='Run the project build command and capture the result.',
arguments={
'command': 'Optional shell command. Defaults to python3 -m compileall src.',
'timeout_seconds': 'Optional timeout in seconds. Defaults to 120.',
},
),
)
@ -111,10 +148,15 @@ class RuntimeToolRegistry:
)
handlers = {
'list_dir': self._list_dir,
'read_file': self._read_file,
'search_files': self._search_files,
'edit_file': self._edit_file,
'run_shell_command': self._run_shell_command,
'git_status': self._git_status,
'git_diff': self._git_diff,
'run_tests': self._run_tests,
'run_build': self._run_build,
}
handler = handlers.get(name)
if handler is None:
@ -156,6 +198,24 @@ class RuntimeToolRegistry:
},
)
def _list_dir(self, arguments: dict[str, Any]) -> RuntimeToolResult:
path = self._resolve_path(arguments.get('path', '.'))
if not path.exists():
raise RuntimeToolError(f'Path not found: {path.relative_to(self.root)}')
if not path.is_dir():
raise RuntimeToolError(f'Path is not a directory: {path.relative_to(self.root)}')
max_entries = self._optional_int(arguments, 'max_entries', minimum=1) or 200
entries = []
for child in sorted(path.iterdir())[:max_entries]:
marker = '[D]' if child.is_dir() else '[F]'
entries.append(f'{marker} {child.relative_to(self.root)}')
return RuntimeToolResult(
name='list_dir',
success=True,
output='\n'.join(entries),
metadata={'path': str(path.relative_to(self.root)), 'entry_count': len(entries)},
)
def _search_files(self, arguments: dict[str, Any]) -> RuntimeToolResult:
pattern = self._require_string(arguments, 'pattern')
search_root = self._resolve_path(arguments.get('path', '.'))
@ -249,6 +309,62 @@ class RuntimeToolRegistry:
def _run_shell_command(self, arguments: dict[str, Any]) -> RuntimeToolResult:
command = self._require_string(arguments, 'command')
timeout = self._optional_int(arguments, 'timeout_seconds', minimum=1) or 20
return self._run_command(
name='run_shell_command',
command=command,
timeout=timeout,
)
def _git_status(self, arguments: dict[str, Any]) -> RuntimeToolResult:
del arguments
self._ensure_git_repo()
return self._run_command(
name='git_status',
command='git status --short --branch',
timeout=20,
)
def _git_diff(self, arguments: dict[str, Any]) -> RuntimeToolResult:
self._ensure_git_repo()
staged = self._optional_bool(arguments, 'staged') or False
path = arguments.get('path')
if path is not None and not isinstance(path, str):
raise RuntimeToolError('path must be a string when provided.')
command_parts = ['git', 'diff']
if staged:
command_parts.append('--cached')
if isinstance(path, str) and path:
resolved = self._resolve_path(path)
command_parts.extend(['--', str(resolved.relative_to(self.root))])
return self._run_command(
name='git_diff',
command=' '.join(command_parts),
timeout=20,
)
def _run_tests(self, arguments: dict[str, Any]) -> RuntimeToolResult:
command = arguments.get('command')
if command is not None and not isinstance(command, str):
raise RuntimeToolError('command must be a string when provided.')
timeout = self._optional_int(arguments, 'timeout_seconds', minimum=1) or 120
return self._run_command(
name='run_tests',
command=command or 'python3 -m unittest discover -s tests -v',
timeout=timeout,
)
def _run_build(self, arguments: dict[str, Any]) -> RuntimeToolResult:
command = arguments.get('command')
if command is not None and not isinstance(command, str):
raise RuntimeToolError('command must be a string when provided.')
timeout = self._optional_int(arguments, 'timeout_seconds', minimum=1) or 120
return self._run_command(
name='run_build',
command=command or 'python3 -m compileall src',
timeout=timeout,
)
def _run_command(self, name: str, command: str, timeout: int) -> RuntimeToolResult:
try:
result = subprocess.run(
[self.shell, '-lc', command],
@ -265,13 +381,24 @@ class RuntimeToolRegistry:
part for part in (result.stdout.strip(), result.stderr.strip()) if part
)
return RuntimeToolResult(
name='run_shell_command',
name=name,
success=result.returncode == 0,
output=self._truncate(combined),
metadata={'exit_code': result.returncode, 'command': command, 'timeout_seconds': timeout},
error=None if result.returncode == 0 else f'Command exited with status {result.returncode}.',
)
def _ensure_git_repo(self) -> None:
result = subprocess.run(
['git', 'rev-parse', '--is-inside-work-tree'],
capture_output=True,
text=True,
cwd=self.root,
check=False,
)
if result.returncode != 0 or result.stdout.strip() != 'true':
raise RuntimeToolError('Workspace root is not inside a git work tree.')
def _resolve_path(self, raw_path: Any) -> Path:
if not isinstance(raw_path, str) or not raw_path.strip():
raise RuntimeToolError('path must be a non-empty string.')
@ -297,6 +424,15 @@ class RuntimeToolRegistry:
raise RuntimeToolError(f'{key} must be an integer >= {minimum}.')
return value
@staticmethod
def _optional_bool(arguments: dict[str, Any], key: str) -> bool | None:
if key not in arguments or arguments[key] is None:
return None
value = arguments[key]
if not isinstance(value, bool):
raise RuntimeToolError(f'{key} must be a boolean.')
return value
@staticmethod
def _truncate(text: str, limit: int = MAX_TOOL_OUTPUT_CHARS) -> str:
if len(text) <= limit:

93
src/task_packet.py Normal file
View File

@ -0,0 +1,93 @@
from __future__ import annotations
import json
from dataclasses import asdict, dataclass
from pathlib import Path
from typing import Any
@dataclass(frozen=True)
class TaskPacket:
objective: str
scope: str
repo: str
branch_policy: str
acceptance_tests: tuple[str, ...]
commit_policy: str
reporting_contract: str
escalation_policy: str
def to_dict(self) -> dict[str, Any]:
payload = asdict(self)
payload['acceptance_tests'] = list(self.acceptance_tests)
return payload
@classmethod
def from_dict(cls, payload: dict[str, Any]) -> 'TaskPacket':
acceptance_tests = payload.get('acceptance_tests', ())
if not isinstance(acceptance_tests, list):
raise TaskPacketValidationError(['acceptance_tests must be a list'])
packet = cls(
objective=str(payload.get('objective', '')),
scope=str(payload.get('scope', '')),
repo=str(payload.get('repo', '')),
branch_policy=str(payload.get('branch_policy', '')),
acceptance_tests=tuple(str(test) for test in acceptance_tests),
commit_policy=str(payload.get('commit_policy', '')),
reporting_contract=str(payload.get('reporting_contract', '')),
escalation_policy=str(payload.get('escalation_policy', '')),
)
validate_task_packet(packet)
return packet
class TaskPacketValidationError(ValueError):
def __init__(self, errors: list[str]):
self.errors = errors
super().__init__('; '.join(errors))
def validate_task_packet(packet: TaskPacket) -> None:
errors: list[str] = []
for field_name in (
'objective',
'scope',
'repo',
'branch_policy',
'commit_policy',
'reporting_contract',
'escalation_policy',
):
if not getattr(packet, field_name).strip():
errors.append(f'{field_name} must not be empty')
for index, test in enumerate(packet.acceptance_tests):
if not test.strip():
errors.append(f'acceptance_tests contains an empty value at index {index}')
if errors:
raise TaskPacketValidationError(errors)
def load_task_packet(path: str | Path) -> TaskPacket:
packet_path = Path(path)
payload = json.loads(packet_path.read_text(encoding='utf-8'))
if not isinstance(payload, dict):
raise TaskPacketValidationError(['task packet must be a JSON object'])
return TaskPacket.from_dict(payload)
def render_worker_prompt(packet: TaskPacket) -> str:
lines = [
'You are operating as a local worker node in a managed coding harness.',
f'Objective: {packet.objective}',
f'Scope: {packet.scope}',
f'Repository: {packet.repo}',
f'Branch policy: {packet.branch_policy}',
f'Commit policy: {packet.commit_policy}',
f'Reporting contract: {packet.reporting_contract}',
f'Escalation policy: {packet.escalation_policy}',
]
if packet.acceptance_tests:
lines.append('Acceptance tests the manager will run after you finish:')
lines.extend(f'- {command}' for command in packet.acceptance_tests)
lines.append('Make the required code or documentation changes in the workspace and return a concise final answer describing what changed and any blockers.')
return '\n'.join(lines)

297
src/worker_api.py Normal file
View File

@ -0,0 +1,297 @@
from __future__ import annotations
import json
import subprocess
from dataclasses import asdict, dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from uuid import uuid4
from .agent_loop import AgentRunResult, run_agent_task
from .llm_backend import DEFAULT_OLLAMA_HOST, DEFAULT_OLLAMA_MODEL
from .task_packet import TaskPacket, load_task_packet, render_worker_prompt
DEFAULT_WORKER_DIR = Path('.port_workers')
DEFAULT_ACCEPTANCE_TIMEOUT_SECONDS = 300
INTERNAL_CHANGED_FILE_PREFIXES = ('.port_sessions/', '.port_workers/')
@dataclass
class WorkerRecord:
worker_id: str
root: str
model: str
host: str
state: str
created_at: str
updated_at: str
run_count: int = 0
last_packet: dict[str, Any] | None = None
last_result: dict[str, Any] | None = None
last_error: str | None = None
def to_dict(self) -> dict[str, Any]:
return asdict(self)
@classmethod
def from_dict(cls, payload: dict[str, Any]) -> 'WorkerRecord':
return cls(
worker_id=payload['worker_id'],
root=payload['root'],
model=payload['model'],
host=payload['host'],
state=payload['state'],
created_at=payload['created_at'],
updated_at=payload['updated_at'],
run_count=payload.get('run_count', 0),
last_packet=payload.get('last_packet'),
last_result=payload.get('last_result'),
last_error=payload.get('last_error'),
)
def workers_dir(directory: Path | None = None) -> Path:
target = directory or DEFAULT_WORKER_DIR
target.mkdir(parents=True, exist_ok=True)
return target
def create_worker(
root: Path | str | None = None,
model: str = DEFAULT_OLLAMA_MODEL,
host: str = DEFAULT_OLLAMA_HOST,
directory: Path | None = None,
) -> WorkerRecord:
now = _iso_now()
worker = WorkerRecord(
worker_id=uuid4().hex,
root=str((Path(root) if root is not None else Path.cwd()).resolve()),
model=model,
host=host,
state='ready',
created_at=now,
updated_at=now,
)
save_worker(worker, directory=directory)
return worker
def list_workers(directory: Path | None = None) -> list[WorkerRecord]:
records: list[WorkerRecord] = []
for path in sorted(workers_dir(directory).glob('*.json')):
payload = json.loads(path.read_text(encoding='utf-8'))
records.append(WorkerRecord.from_dict(payload))
records.sort(key=lambda worker: worker.updated_at, reverse=True)
return records
def load_worker(worker_id: str, directory: Path | None = None) -> WorkerRecord:
path = workers_dir(directory) / f'{worker_id}.json'
payload = json.loads(path.read_text(encoding='utf-8'))
return WorkerRecord.from_dict(payload)
def save_worker(worker: WorkerRecord, directory: Path | None = None) -> Path:
path = workers_dir(directory) / f'{worker.worker_id}.json'
path.write_text(json.dumps(worker.to_dict(), indent=2), encoding='utf-8')
return path
def run_worker(
worker_id: str,
packet_source: TaskPacket | str | Path,
directory: Path | None = None,
trace: bool = False,
) -> WorkerRecord:
worker = load_worker(worker_id, directory=directory)
if worker.state == 'closed':
raise RuntimeError(f'Worker {worker_id} is closed.')
packet = packet_source if isinstance(packet_source, TaskPacket) else load_task_packet(packet_source)
running_worker = _replace_worker(
worker,
state='running',
updated_at=_iso_now(),
last_packet=packet.to_dict(),
last_error=None,
)
save_worker(running_worker, directory=directory)
before_paths = _dirty_paths(Path(running_worker.root))
try:
agent_result = run_agent_task(
render_worker_prompt(packet),
model=running_worker.model,
host=running_worker.host,
root=Path(running_worker.root),
trace=trace,
)
verification_steps = _run_acceptance_tests(Path(running_worker.root), packet.acceptance_tests)
changed_files = _collect_changed_files(Path(running_worker.root), before_paths, agent_result)
result = _build_worker_result(running_worker, packet, agent_result, verification_steps, changed_files)
next_state = 'finished' if result['stop_reason'] == 'completed' else 'failed'
updated_worker = _replace_worker(
running_worker,
state=next_state,
updated_at=_iso_now(),
run_count=running_worker.run_count + 1,
last_result=result,
last_error=None if next_state == 'finished' else result.get('final_answer') or result['stop_reason'],
)
except Exception as error:
updated_worker = _replace_worker(
running_worker,
state='failed',
updated_at=_iso_now(),
run_count=running_worker.run_count + 1,
last_result={
'worker_id': running_worker.worker_id,
'state': 'failed',
'tool_calls': [],
'tool_trace': [],
'artifacts': [],
'changed_files': [],
'verification': {'acceptance_tests': [], 'tool_runs': {'run_tests': [], 'run_build': []}},
'final_answer': '',
'stop_reason': 'error',
'error': str(error),
},
last_error=str(error),
)
save_worker(updated_worker, directory=directory)
return updated_worker
def resume_worker(worker_id: str, directory: Path | None = None, trace: bool = False) -> WorkerRecord:
worker = load_worker(worker_id, directory=directory)
if worker.last_packet is None:
raise RuntimeError(f'Worker {worker_id} has no stored task packet to resume.')
return run_worker(worker_id, TaskPacket.from_dict(worker.last_packet), directory=directory, trace=trace)
def close_worker(worker_id: str, directory: Path | None = None) -> WorkerRecord:
worker = load_worker(worker_id, directory=directory)
updated = _replace_worker(worker, state='closed', updated_at=_iso_now())
save_worker(updated, directory=directory)
return updated
def _build_worker_result(
worker: WorkerRecord,
packet: TaskPacket,
agent_result: AgentRunResult,
verification_steps: list[dict[str, Any]],
changed_files: list[str],
) -> dict[str, Any]:
tool_run_summary = {
'run_tests': [entry for entry in agent_result.tool_trace if entry['name'] == 'run_tests'],
'run_build': [entry for entry in agent_result.tool_trace if entry['name'] == 'run_build'],
}
acceptance_success = all(step['success'] for step in verification_steps)
artifacts = sorted(
{
entry['metadata']['path']
for entry in agent_result.tool_trace
if isinstance(entry.get('metadata'), dict) and isinstance(entry['metadata'].get('path'), str)
and entry['name'] == 'edit_file'
}
| set(changed_files)
)
stop_reason = 'completed' if acceptance_success else 'verification_failed'
return {
'worker_id': worker.worker_id,
'state': 'finished' if acceptance_success else 'failed',
'packet': packet.to_dict(),
'tool_calls': list(agent_result.tool_calls),
'tool_trace': list(agent_result.tool_trace),
'artifacts': artifacts,
'changed_files': changed_files,
'verification': {
'acceptance_tests': verification_steps,
'tool_runs': tool_run_summary,
},
'final_answer': agent_result.content,
'stop_reason': stop_reason,
'turns': agent_result.turns,
'trace_events': list(agent_result.trace_events),
}
def _run_acceptance_tests(root: Path, commands: tuple[str, ...]) -> list[dict[str, Any]]:
steps: list[dict[str, Any]] = []
for command in commands:
try:
result = subprocess.run(
['/bin/zsh', '-lc', command],
cwd=root,
capture_output=True,
text=True,
timeout=DEFAULT_ACCEPTANCE_TIMEOUT_SECONDS,
check=False,
)
combined = '\n'.join(part for part in (result.stdout.strip(), result.stderr.strip()) if part)
steps.append(
{
'command': command,
'success': result.returncode == 0,
'exit_code': result.returncode,
'output': combined,
}
)
except subprocess.TimeoutExpired:
steps.append(
{
'command': command,
'success': False,
'exit_code': -1,
'output': '',
'error': f'Command timed out after {DEFAULT_ACCEPTANCE_TIMEOUT_SECONDS} seconds.',
}
)
return steps
def _collect_changed_files(root: Path, before_paths: set[str], agent_result: AgentRunResult) -> list[str]:
after_paths = _dirty_paths(root)
edited_paths = {
entry['metadata']['path']
for entry in agent_result.tool_trace
if entry['name'] == 'edit_file'
and isinstance(entry.get('metadata'), dict)
and isinstance(entry['metadata'].get('path'), str)
}
changed = (after_paths - before_paths) | edited_paths
return sorted(
path for path in changed if not any(path.startswith(prefix) for prefix in INTERNAL_CHANGED_FILE_PREFIXES)
)
def _dirty_paths(root: Path) -> set[str]:
status = subprocess.run(
['git', 'status', '--porcelain'],
cwd=root,
capture_output=True,
text=True,
check=False,
)
if status.returncode != 0:
return set()
paths: set[str] = set()
for line in status.stdout.splitlines():
if len(line) < 4:
continue
raw_path = line[3:]
if ' -> ' in raw_path:
raw_path = raw_path.split(' -> ', 1)[1]
paths.add(raw_path.strip('"'))
return paths
def _replace_worker(worker: WorkerRecord, **changes: Any) -> WorkerRecord:
payload = worker.to_dict()
payload.update(changes)
return WorkerRecord.from_dict(payload)
def _iso_now() -> str:
return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace('+00:00', 'Z')

View File

@ -0,0 +1,400 @@
from __future__ import annotations
import io
import json
import subprocess
import sys
import unittest
from pathlib import Path
from tempfile import TemporaryDirectory
from unittest.mock import patch
from src.autoresearch_runner import (
ExperimentMetrics,
append_results_row,
best_recorded_bpb,
parse_run_log,
short_head_commit,
setup_autoresearch,
)
from src.autoresearch_worker import (
AutoresearchExperimentPacket,
autoresearch_status,
discard_autoresearch_candidate,
ensure_autoresearch_baseline,
ensure_autoresearch_branch,
keep_autoresearch_candidate,
load_autoresearch_packet,
load_autoresearch_state,
loop_autoresearch,
run_autoresearch_packet,
)
from src.worker_api import WorkerRecord
class AutoresearchIntegrationTests(unittest.TestCase):
def init_repo(self, root: Path) -> None:
subprocess.run(['git', 'init'], cwd=root, check=True, capture_output=True, text=True)
subprocess.run(['git', 'config', 'user.email', 'test@example.com'], cwd=root, check=True, capture_output=True, text=True)
subprocess.run(['git', 'config', 'user.name', 'Test User'], cwd=root, check=True, capture_output=True, text=True)
subprocess.run(['git', 'add', '.'], cwd=root, check=True, capture_output=True, text=True)
subprocess.run(['git', 'commit', '-m', 'init'], cwd=root, check=True, capture_output=True, text=True)
def test_setup_autoresearch_initializes_results(self) -> None:
with TemporaryDirectory() as tmpdir:
root = Path(tmpdir)
for name in ('prepare.py', 'train.py', 'program.md'):
(root / name).write_text('# stub\n', encoding='utf-8')
self.init_repo(root)
report = setup_autoresearch(root)
self.assertTrue((root / 'results.tsv').exists())
self.assertEqual(Path(report.root).resolve(), root.resolve())
self.assertIn('results.tsv', report.results_tsv_path)
def test_parse_run_log_extracts_metrics(self) -> None:
with TemporaryDirectory() as tmpdir:
log_path = Path(tmpdir) / 'run.log'
log_path.write_text(
'\n'.join(
[
'---',
'val_bpb: 0.997900',
'training_seconds: 300.1',
'total_seconds: 325.9',
'peak_vram_mb: 45060.2',
'mfu_percent: 39.80',
'total_tokens_M: 499.6',
'num_steps: 953',
'num_params_M: 50.3',
'depth: 8',
]
),
encoding='utf-8',
)
metrics = parse_run_log(log_path)
self.assertTrue(metrics.success)
self.assertEqual(metrics.val_bpb, 0.9979)
self.assertEqual(metrics.depth, 8)
def test_append_results_row_and_best_bpb(self) -> None:
with TemporaryDirectory() as tmpdir:
root = Path(tmpdir)
for name in ('prepare.py', 'train.py', 'program.md'):
(root / name).write_text('# stub\n', encoding='utf-8')
metrics = ExperimentMetrics(
success=True,
log_path=str(root / 'run.log'),
timed_out=False,
return_code=0,
val_bpb=0.99,
peak_vram_mb=1024.0,
)
append_results_row(root, 'abc1234', metrics, 'keep', 'baseline')
append_results_row(root, 'def5678', metrics, 'discard', 'worse')
self.assertEqual(best_recorded_bpb(root), 0.99)
def test_load_autoresearch_packet(self) -> None:
with TemporaryDirectory() as tmpdir:
packet_path = Path(tmpdir) / 'experiment.json'
packet_path.write_text(
json.dumps(
{
'objective': 'Try one change',
'description': 'Small optimizer tweak',
}
),
encoding='utf-8',
)
packet = load_autoresearch_packet(packet_path)
self.assertEqual(packet.objective, 'Try one change')
self.assertEqual(packet.timeout_seconds, 600)
def test_run_autoresearch_packet_wraps_worker_and_runner(self) -> None:
with TemporaryDirectory() as tmpdir:
root = Path(tmpdir)
for name in ('prepare.py', 'program.md'):
(root / name).write_text('# stub\n', encoding='utf-8')
(root / 'train.py').write_text('print("hello")\n', encoding='utf-8')
self.init_repo(root)
packet = AutoresearchExperimentPacket(
objective='Try a small change',
description='Tweak train.py',
)
worker = WorkerRecord(
worker_id='worker123',
root=str(root),
model='qwen2.5-coder:7b',
host='http://localhost:11434',
state='finished',
created_at='2026-04-07T00:00:00Z',
updated_at='2026-04-07T00:00:01Z',
run_count=1,
last_packet={},
last_result={
'changed_files': ['train.py'],
'tool_calls': ['edit_file'],
'tool_trace': [],
'verification': {'acceptance_tests': []},
},
)
(root / 'train.py').write_text('print("changed")\n', encoding='utf-8')
with patch('src.autoresearch_worker.create_worker', return_value=worker):
with patch('src.autoresearch_worker.run_worker', return_value=worker):
with patch(
'src.autoresearch_worker.run_experiment',
return_value=ExperimentMetrics(
success=True,
log_path=str(root / 'run.log'),
timed_out=False,
return_code=0,
val_bpb=0.98,
peak_vram_mb=2048.0,
),
):
result = run_autoresearch_packet(packet, root=root, trace=False)
self.assertEqual(result['recommended_status'], 'keep')
self.assertEqual(result['experiment']['val_bpb'], 0.98)
self.assertTrue((root / 'results.tsv').exists())
self.assertIsNotNone(result['state']['pending_experiment'])
self.assertEqual((root / 'results.tsv').read_text(encoding='utf-8').count('\n'), 1)
def test_keep_autoresearch_candidate_records_keep(self) -> None:
with TemporaryDirectory() as tmpdir:
root = Path(tmpdir)
for name in ('prepare.py', 'program.md'):
(root / name).write_text('# stub\n', encoding='utf-8')
(root / 'train.py').write_text('print("hello")\n', encoding='utf-8')
self.init_repo(root)
packet = AutoresearchExperimentPacket(objective='Try a small change', description='Keep candidate')
worker = WorkerRecord(
worker_id='worker123',
root=str(root),
model='qwen2.5-coder:7b',
host='http://localhost:11434',
state='finished',
created_at='2026-04-07T00:00:00Z',
updated_at='2026-04-07T00:00:01Z',
run_count=1,
last_packet={},
last_result={
'changed_files': ['train.py'],
'tool_calls': ['edit_file'],
'tool_trace': [],
'verification': {'acceptance_tests': []},
},
)
(root / 'train.py').write_text('print("changed")\n', encoding='utf-8')
with patch('src.autoresearch_worker.create_worker', return_value=worker):
with patch('src.autoresearch_worker.run_worker', return_value=worker):
with patch(
'src.autoresearch_worker.run_experiment',
return_value=ExperimentMetrics(
success=True,
log_path=str(root / 'run.log'),
timed_out=False,
return_code=0,
val_bpb=0.98,
peak_vram_mb=2048.0,
),
):
run_autoresearch_packet(packet, root=root, trace=False)
result = keep_autoresearch_candidate(root)
state = load_autoresearch_state(root)
rows = (root / 'results.tsv').read_text(encoding='utf-8').splitlines()
self.assertEqual(result['decision'], 'keep')
self.assertIsNone(state.pending_experiment)
self.assertEqual(state.last_decision, 'keep')
self.assertEqual(state.best_bpb, 0.98)
self.assertEqual(len(rows), 2)
self.assertIn('\tkeep\tKeep candidate', rows[1])
def test_discard_autoresearch_candidate_resets_repo(self) -> None:
with TemporaryDirectory() as tmpdir:
root = Path(tmpdir)
for name in ('prepare.py', 'program.md'):
(root / name).write_text('# stub\n', encoding='utf-8')
(root / 'train.py').write_text('print("hello")\n', encoding='utf-8')
self.init_repo(root)
packet = AutoresearchExperimentPacket(objective='Try a small change', description='Discard candidate')
worker = WorkerRecord(
worker_id='worker123',
root=str(root),
model='qwen2.5-coder:7b',
host='http://localhost:11434',
state='finished',
created_at='2026-04-07T00:00:00Z',
updated_at='2026-04-07T00:00:01Z',
run_count=1,
last_packet={},
last_result={
'changed_files': ['train.py'],
'tool_calls': ['edit_file'],
'tool_trace': [],
'verification': {'acceptance_tests': []},
},
)
(root / 'train.py').write_text('print("candidate")\n', encoding='utf-8')
with patch('src.autoresearch_worker.create_worker', return_value=worker):
with patch('src.autoresearch_worker.run_worker', return_value=worker):
with patch(
'src.autoresearch_worker.run_experiment',
return_value=ExperimentMetrics(
success=True,
log_path=str(root / 'run.log'),
timed_out=False,
return_code=0,
val_bpb=1.2,
peak_vram_mb=2048.0,
),
):
result = run_autoresearch_packet(packet, root=root, trace=False)
self.assertEqual(result['recommended_status'], 'keep')
keep_autoresearch_candidate(root)
(root / 'train.py').write_text('print("worse")\n', encoding='utf-8')
with patch('src.autoresearch_worker.create_worker', return_value=worker):
with patch('src.autoresearch_worker.run_worker', return_value=worker):
with patch(
'src.autoresearch_worker.run_experiment',
return_value=ExperimentMetrics(
success=True,
log_path=str(root / 'run.log'),
timed_out=False,
return_code=0,
val_bpb=1.3,
peak_vram_mb=4096.0,
),
):
run_autoresearch_packet(packet, root=root, trace=False)
discard = discard_autoresearch_candidate(root)
self.assertEqual(discard['decision'], 'discard')
self.assertEqual(short_head_commit(root), discard['reverted_to_commit'])
self.assertIn('candidate', (root / 'train.py').read_text(encoding='utf-8'))
self.assertNotEqual(short_head_commit(root), 'unknown')
def test_ensure_autoresearch_baseline_records_first_run(self) -> None:
with TemporaryDirectory() as tmpdir:
root = Path(tmpdir)
for name in ('prepare.py', 'program.md'):
(root / name).write_text('# stub\n', encoding='utf-8')
(root / 'train.py').write_text('print("hello")\n', encoding='utf-8')
self.init_repo(root)
with patch(
'src.autoresearch_worker.run_experiment',
return_value=ExperimentMetrics(
success=True,
log_path=str(root / 'run.log'),
timed_out=False,
return_code=0,
val_bpb=1.01,
peak_vram_mb=1024.0,
),
):
baseline = ensure_autoresearch_baseline(root)
state = load_autoresearch_state(root)
self.assertTrue(baseline['baseline_created'])
self.assertEqual(state.baseline_bpb, 1.01)
self.assertEqual(state.best_bpb, 1.01)
self.assertIn('\tkeep\tbaseline', (root / 'results.tsv').read_text(encoding='utf-8'))
def test_ensure_autoresearch_branch_creates_isolated_branch(self) -> None:
with TemporaryDirectory() as tmpdir:
root = Path(tmpdir)
for name in ('prepare.py', 'program.md'):
(root / name).write_text('# stub\n', encoding='utf-8')
(root / 'train.py').write_text('print("hello")\n', encoding='utf-8')
self.init_repo(root)
result = ensure_autoresearch_branch(root, branch='autoresearch/test', create=True)
status = autoresearch_status(root)
self.assertEqual(result['current_branch'], 'autoresearch/test')
self.assertTrue(status['isolated_branch'])
self.assertEqual(status['current_branch'], 'autoresearch/test')
def test_loop_autoresearch_runs_keep_cycle(self) -> None:
with TemporaryDirectory() as tmpdir:
root = Path(tmpdir)
for name in ('prepare.py', 'program.md'):
(root / name).write_text('# stub\n', encoding='utf-8')
(root / 'train.py').write_text('print("hello")\n', encoding='utf-8')
self.init_repo(root)
ensure_autoresearch_branch(root, branch='autoresearch/test', create=True)
packet = AutoresearchExperimentPacket(objective='Loop once', description='Loop candidate')
worker = WorkerRecord(
worker_id='worker123',
root=str(root),
model='qwen2.5-coder:7b',
host='http://localhost:11434',
state='finished',
created_at='2026-04-07T00:00:00Z',
updated_at='2026-04-07T00:00:01Z',
run_count=1,
last_packet={},
last_result={
'changed_files': ['train.py'],
'tool_calls': ['edit_file'],
'tool_trace': [],
'verification': {'acceptance_tests': []},
},
)
(root / 'train.py').write_text('print("changed")\n', encoding='utf-8')
with patch(
'src.autoresearch_worker.ensure_autoresearch_baseline',
return_value={'baseline_created': False, 'reason': 'baseline already recorded'},
):
with patch('src.autoresearch_worker.create_worker', return_value=worker):
with patch('src.autoresearch_worker.run_worker', return_value=worker):
with patch(
'src.autoresearch_worker.run_experiment',
return_value=ExperimentMetrics(
success=True,
log_path=str(root / 'run.log'),
timed_out=False,
return_code=0,
val_bpb=0.97,
peak_vram_mb=2048.0,
),
):
result = loop_autoresearch(packet, root=root, iterations=1, retry_limit=0)
self.assertEqual(len(result['history']), 1)
self.assertEqual(result['history'][0]['decision']['decision'], 'keep')
def test_autoresearch_cli_setup_and_parse_log(self) -> None:
from src.main import main
with TemporaryDirectory() as tmpdir:
root = Path(tmpdir)
for name in ('prepare.py', 'train.py', 'program.md'):
(root / name).write_text('# stub\n', encoding='utf-8')
self.init_repo(root)
log_path = root / 'run.log'
log_path.write_text('val_bpb: 1.000000\n', encoding='utf-8')
with patch('sys.stdout', new=io.StringIO()):
setup_exit = main(['autoresearch', 'setup', '--root', str(root)])
with patch('sys.stdout', new=io.StringIO()):
status_exit = main(['autoresearch', 'status', '--root', str(root)])
with patch('sys.stdout', new=io.StringIO()):
parse_exit = main(['autoresearch', 'parse-log', '--root', str(root), 'run.log'])
self.assertEqual(setup_exit, 0)
self.assertEqual(status_exit, 0)
self.assertEqual(parse_exit, 0)
if __name__ == '__main__':
unittest.main()

View File

@ -1,6 +1,7 @@
from __future__ import annotations
import io
import json
import subprocess
import sys
import unittest
@ -9,9 +10,13 @@ from tempfile import TemporaryDirectory
from unittest.mock import patch
from src.agent_loop import (
AgentRunResult,
build_agent_messages,
has_successful_edit,
infer_tool_call_from_prompt,
infer_required_edit_target,
parse_agent_response,
prompt_requires_edit,
prompt_requires_tool,
run_agent_task,
tool_result_message,
@ -23,7 +28,9 @@ from src.port_manifest import build_port_manifest
from src.query_engine import QueryEnginePort
from src.runtime_tools import RuntimeToolRegistry
from src.task import DEFAULT_SYSTEM_PROMPT, build_chat_messages, run_local_task
from src.task_packet import TaskPacket, TaskPacketValidationError, load_task_packet
from src.tools import PORTED_TOOLS
from src.worker_api import close_worker, create_worker, list_workers, load_worker, resume_worker, run_worker
class PortingWorkspaceTests(unittest.TestCase):
@ -313,6 +320,38 @@ class PortingWorkspaceTests(unittest.TestCase):
self.assertTrue(shell_result.success)
self.assertEqual(shell_result.output, 'shell-ok')
def test_runtime_tool_registry_supports_list_dir_git_build_and_test(self) -> None:
with TemporaryDirectory() as tmpdir:
root = Path(tmpdir)
(root / 'src').mkdir()
(root / 'src' / 'app.py').write_text('print("ok")\n', encoding='utf-8')
subprocess.run(['git', 'init'], cwd=root, check=True, capture_output=True, text=True)
subprocess.run(['git', 'config', 'user.email', 'test@example.com'], cwd=root, check=True, capture_output=True, text=True)
subprocess.run(['git', 'config', 'user.name', 'Test User'], cwd=root, check=True, capture_output=True, text=True)
subprocess.run(['git', 'add', '.'], cwd=root, check=True, capture_output=True, text=True)
subprocess.run(['git', 'commit', '-m', 'init'], cwd=root, check=True, capture_output=True, text=True)
(root / 'src' / 'app.py').write_text('print("changed")\n', encoding='utf-8')
registry = RuntimeToolRegistry(root=root)
list_result = registry.execute('list_dir', {'path': 'src'})
self.assertTrue(list_result.success)
self.assertIn('[F] src/app.py', list_result.output)
git_status = registry.execute('git_status', {})
self.assertTrue(git_status.success)
self.assertIn('src/app.py', git_status.output)
git_diff = registry.execute('git_diff', {'path': 'src/app.py'})
self.assertTrue(git_diff.success)
self.assertIn('-print("ok")', git_diff.output)
run_build = registry.execute('run_build', {'command': 'python3 -m compileall src'})
self.assertTrue(run_build.success)
run_tests = registry.execute('run_tests', {'command': 'python3 -c "print(\'tests ok\')"', 'timeout_seconds': 30})
self.assertTrue(run_tests.success)
self.assertIn('tests ok', run_tests.output)
def test_parse_agent_response_supports_tool_calls_and_finals(self) -> None:
tool_call = parse_agent_response('{"type":"tool_call","name":"read_file","arguments":{"path":"src/main.py"}}')
final = parse_agent_response('{"type":"final","content":"done"}')
@ -324,12 +363,40 @@ class PortingWorkspaceTests(unittest.TestCase):
self.assertTrue(prompt_requires_tool("search the src tree for 'agent_parser'"))
self.assertFalse(prompt_requires_tool('say hello'))
def test_prompt_requires_edit_for_mutating_prompts(self) -> None:
self.assertTrue(prompt_requires_edit('modify train.py by adding a single harmless comment line near the top of the file'))
self.assertFalse(prompt_requires_edit('read train.py for context'))
self.assertEqual(
infer_required_edit_target('modify train.py by adding a single harmless comment line near the top of the file'),
'train.py',
)
def test_infer_tool_call_from_prompt_recognizes_read_and_search(self) -> None:
read_call = infer_tool_call_from_prompt('read src/main.py lines 1 to 40')
search_call = infer_tool_call_from_prompt("search the src tree for 'agent_parser'")
self.assertEqual(read_call, {'name': 'read_file', 'arguments': {'path': 'src/main.py', 'start_line': 1, 'end_line': 40}})
self.assertEqual(search_call, {'name': 'search_files', 'arguments': {'path': 'src', 'pattern': 'agent_parser'}})
def test_infer_tool_call_from_prompt_recognizes_comment_edit_fallback(self) -> None:
tool_call = infer_tool_call_from_prompt(
'modify train.py by adding a single harmless comment line near the top of the file',
prefer_edit=True,
)
self.assertIsNotNone(tool_call)
self.assertEqual(tool_call['name'], 'run_shell_command')
self.assertIn('target = Path("train.py")', tool_call['arguments']['command'])
self.assertIn('# smoke-test comment: ', tool_call['arguments']['command'])
def test_infer_tool_call_from_prompt_recognizes_incrementing_comment_prompt(self) -> None:
tool_call = infer_tool_call_from_prompt(
'Modify train.py by incrementing or inserting a harmless smoke-test comment counter near the top of the file. You must change train.py on every run. A no-op is failure.',
prefer_edit=True,
)
self.assertIsNotNone(tool_call)
self.assertEqual(tool_call['name'], 'run_shell_command')
self.assertIn('target = Path("train.py")', tool_call['arguments']['command'])
self.assertIn('# smoke-test comment: ', tool_call['arguments']['command'])
def test_tool_result_message_is_json_prefixed(self) -> None:
message = tool_result_message({'name': 'read_file', 'success': True, 'output': 'ok'})
self.assertTrue(message.startswith('TOOL_RESULT '))
@ -366,7 +433,9 @@ class PortingWorkspaceTests(unittest.TestCase):
self.assertEqual(result.content, 'Read complete.')
self.assertEqual(result.tool_calls, ('read_file',))
self.assertEqual(result.stop_reason, 'completed')
self.assertTrue(any(event.startswith('tool_call') for event in result.trace_events))
self.assertEqual(result.tool_trace[0]['name'], 'read_file')
second_call_messages = chat_mock.call_args_list[1].args[0]
self.assertTrue(any(message['content'].startswith('TOOL_RESULT ') for message in second_call_messages if message['role'] == 'user'))
@ -397,6 +466,219 @@ class PortingWorkspaceTests(unittest.TestCase):
self.assertEqual(result.tool_calls, ('read_file',))
self.assertTrue(any('tool_required_reject' in event for event in result.trace_events))
def test_run_agent_task_requires_mutation_for_edit_prompt(self) -> None:
with TemporaryDirectory() as tmpdir:
root = Path(tmpdir)
target = root / 'train.py'
target.write_text('print("ok")\n', encoding='utf-8')
responses = [
normalize_ollama_response(
{
'message': {
'content': '{"type":"tool_call","name":"read_file","arguments":{"path":"train.py"}}'
}
}
),
normalize_ollama_response(
{
'message': {
'content': '{"type":"final","content":"I reviewed the file."}'
}
}
),
normalize_ollama_response(
{
'message': {
'content': '{"type":"final","content":"Inserted the comment."}'
}
}
),
]
with patch('src.agent_loop.OllamaBackend.chat', side_effect=responses):
result = run_agent_task(
'modify train.py by adding a single harmless comment line near the top of the file',
root=root,
max_turns=4,
trace=True,
)
self.assertIn('# smoke-test comment: 1', target.read_text(encoding='utf-8'))
self.assertEqual(result.tool_calls, ('read_file', 'run_shell_command'))
self.assertTrue(any('edit_required_reject' in event for event in result.trace_events))
def test_run_agent_task_incrementing_smoke_comment_is_repeatable(self) -> None:
with TemporaryDirectory() as tmpdir:
root = Path(tmpdir)
target = root / 'train.py'
target.write_text('# smoke-test comment: 1\nprint("ok")\n', encoding='utf-8')
responses = [
normalize_ollama_response(
{
'message': {
'content': '{"type":"tool_call","name":"read_file","arguments":{"path":"train.py"}}'
}
}
),
normalize_ollama_response(
{
'message': {
'content': '{"type":"final","content":"I reviewed the file."}'
}
}
),
normalize_ollama_response(
{
'message': {
'content': '{"type":"final","content":"Updated the smoke comment."}'
}
}
),
]
with patch('src.agent_loop.OllamaBackend.chat', side_effect=responses):
run_agent_task(
'modify train.py by adding a single harmless comment line near the top of the file',
root=root,
max_turns=4,
trace=True,
)
self.assertTrue(target.read_text(encoding='utf-8').startswith('# smoke-test comment: 2\n'))
def test_has_successful_edit_requires_target_path(self) -> None:
tool_trace = [
{
'name': 'edit_file',
'success': False,
'arguments': {'path': 'program.md'},
'metadata': {'path': 'program.md'},
},
{
'name': 'run_shell_command',
'success': True,
'arguments': {'command': 'echo ok'},
'metadata': {'path': 'program.md'},
},
]
self.assertFalse(has_successful_edit(tool_trace, 'train.py'))
self.assertTrue(has_successful_edit(tool_trace, 'program.md'))
def test_task_packet_load_and_validation(self) -> None:
with TemporaryDirectory() as tmpdir:
packet_path = Path(tmpdir) / 'task.json'
packet_path.write_text(
(
'{'
'"objective":"Update docs",'
'"scope":"README only",'
'"repo":"claw-code",'
'"branch_policy":"stay on current branch",'
'"acceptance_tests":["python3 -m unittest discover -s tests -v"],'
'"commit_policy":"no commit",'
'"reporting_contract":"report changed files",'
'"escalation_policy":"stop on ambiguity"'
'}'
),
encoding='utf-8',
)
packet = load_task_packet(packet_path)
self.assertEqual(packet.objective, 'Update docs')
self.assertEqual(packet.acceptance_tests, ('python3 -m unittest discover -s tests -v',))
def test_task_packet_rejects_missing_fields(self) -> None:
with self.assertRaises(TaskPacketValidationError):
TaskPacket.from_dict(
{
'objective': '',
'scope': '',
'repo': '',
'branch_policy': '',
'acceptance_tests': ['ok', ''],
'commit_policy': '',
'reporting_contract': '',
'escalation_policy': '',
}
)
def test_worker_create_run_resume_status_and_close(self) -> None:
with TemporaryDirectory() as tmpdir:
root = Path(tmpdir)
storage = root / '.workers'
packet_path = root / 'task.json'
packet = {
'objective': 'Create a note',
'scope': 'workspace root',
'repo': 'temp-repo',
'branch_policy': 'stay on current branch',
'acceptance_tests': ['python3 -c "print(\'verify ok\')"'],
'commit_policy': 'no commit',
'reporting_contract': 'report changed files',
'escalation_policy': 'stop on errors',
}
packet_path.write_text(json.dumps(packet), encoding='utf-8')
worker = create_worker(root=root, directory=storage)
self.assertEqual(worker.state, 'ready')
fake_result = AgentRunResult(
content='Created notes.txt',
turns=2,
tool_calls=('edit_file',),
tool_trace=(
{
'turn': 1,
'name': 'edit_file',
'arguments': {'path': 'notes.txt', 'new_text': 'hello'},
'success': True,
'output': 'Updated notes.txt',
'metadata': {'path': 'notes.txt', 'created': True},
'error': None,
},
),
stop_reason='completed',
trace_events=('tool_call[1]=edit_file {"path":"notes.txt"}',),
)
with patch('src.worker_api.run_agent_task', return_value=fake_result):
ran = run_worker(worker.worker_id, packet_path, directory=storage, trace=True)
self.assertEqual(ran.state, 'finished')
self.assertEqual(ran.last_result['stop_reason'], 'completed')
self.assertIn('notes.txt', ran.last_result['changed_files'])
self.assertTrue(ran.last_result['verification']['acceptance_tests'][0]['success'])
status = load_worker(worker.worker_id, directory=storage)
self.assertEqual(status.worker_id, worker.worker_id)
self.assertIsNotNone(status.last_packet)
with patch('src.worker_api.run_agent_task', return_value=fake_result):
resumed = resume_worker(worker.worker_id, directory=storage, trace=False)
self.assertEqual(resumed.run_count, 2)
workers = list_workers(directory=storage)
self.assertEqual(len(workers), 1)
self.assertEqual(workers[0].worker_id, worker.worker_id)
closed = close_worker(worker.worker_id, directory=storage)
self.assertEqual(closed.state, 'closed')
def test_worker_cli_create_and_status(self) -> None:
from src.main import main
with TemporaryDirectory() as tmpdir:
root = Path(tmpdir)
with patch('src.main.create_worker') as create_worker_mock:
create_worker_mock.return_value = create_worker(root=root, directory=root / '.workers')
with patch('sys.stdout', new=io.StringIO()):
exit_code = main(['worker', 'create', '--root', str(root)])
self.assertEqual(exit_code, 0)
def test_worker_cli_list(self) -> None:
from src.main import main
with patch('src.main.list_workers', return_value=[]):
with patch('sys.stdout', new=io.StringIO()):
exit_code = main(['worker', 'list'])
self.assertEqual(exit_code, 0)
def test_agent_cli_invokes_agent_loop(self) -> None:
from src.main import main