diff --git a/notebooks/autoresearch_memory_smoke_test.ipynb b/notebooks/autoresearch_memory_smoke_test.ipynb new file mode 100644 index 00000000..75ee9dc9 --- /dev/null +++ b/notebooks/autoresearch_memory_smoke_test.ipynb @@ -0,0 +1,308 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Autoresearch Memory Smoke Test\n", + "\n", + "This notebook validates the first synthesized memory layer for `autoresearch-macos`.\n", + "\n", + "It checks:\n", + "\n", + "1. the `memory/` directory exists\n", + "2. `MEMORY.md` exists and links the expected topic files\n", + "3. each topic file has frontmatter with `name`, `description`, and `type`\n", + "4. operational memory files still exist:\n", + " - `results.tsv`\n", + " - `experiment_memory.jsonl`\n", + "5. the synthesized memory layer complements, rather than replaces, the exact run logs\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'claw_root': '/Users/wongdowling/Documents/autoresearch_harness/harness/claw-code', 'autoresearch_root': '/Users/wongdowling/Documents/autoresearch_harness/nodes/autoresearch-macos', 'memory_root': '/Users/wongdowling/Documents/autoresearch_harness/nodes/autoresearch-macos/memory'}\n" + ] + } + ], + "source": [ + "from __future__ import annotations\n", + "\n", + "from pathlib import Path\n", + "import json\n", + "import re\n", + "import sys\n", + "\n", + "def is_claw_root(candidate: Path) -> bool:\n", + " return (candidate / 'src' / 'main.py').exists() and (candidate / 'tests' / 'test_porting_workspace.py').exists()\n", + "\n", + "def find_claw_root(start: Path | None = None) -> Path:\n", + " current = (start or Path.cwd()).resolve()\n", + " for candidate in (current, *current.parents):\n", + " if is_claw_root(candidate):\n", + " return candidate\n", + " for nested in (candidate / 'claw-code', candidate / 'harness' / 'claw-code'):\n", + " if is_claw_root(nested):\n", + " return nested\n", + " raise RuntimeError(f'Could not locate claw-code root from {current}')\n", + "\n", + "CLAW_ROOT = find_claw_root()\n", + "WORKSPACE_ROOT = CLAW_ROOT.parent.parent\n", + "AUTORESEARCH_ROOT = WORKSPACE_ROOT / 'nodes' / 'autoresearch-macos'\n", + "MEMORY_ROOT = AUTORESEARCH_ROOT / 'memory'\n", + "\n", + "print({\n", + " 'claw_root': str(CLAW_ROOT),\n", + " 'autoresearch_root': str(AUTORESEARCH_ROOT),\n", + " 'memory_root': str(MEMORY_ROOT),\n", + "})" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['MEMORY.md',\n", + " 'project_current_strategy.md',\n", + " 'project_failed_ideas.md',\n", + " 'project_promising_ideas.md',\n", + " 'reference_external_sources.md']" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "expected_files = [\n", + " 'MEMORY.md',\n", + " 'project_current_strategy.md',\n", + " 'project_failed_ideas.md',\n", + " 'project_promising_ideas.md',\n", + " 'reference_external_sources.md',\n", + "]\n", + "missing = [name for name in expected_files if not (MEMORY_ROOT / name).exists()]\n", + "if missing:\n", + " raise RuntimeError(f'Missing memory files: {missing}')\n", + "sorted(path.name for path in MEMORY_ROOT.iterdir())" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "# Memory Index\n", + "\n", + "This directory is the synthesized project memory layer for `autoresearch-macos`.\n", + "\n", + "It sits between:\n", + "\n", + "- raw operational records such as [`results.tsv`](/Users/wongdowling/Documents/autoresearch_harness/nodes/autoresearch-macos/results.tsv) and [`experiment_memory.jsonl`](/Users/wongdowling/Documents/autoresearch_harness/nodes/autoresearch-macos/experiment_memory.jsonl)\n", + "- future higher-level retrieval or wiki tooling\n", + "\n", + "The goal is not to replace exact run logs. The goal is to keep a compact, maintained set of topic pages that accumulate what the manager and worker learn over time.\n", + "\n", + "## Topics\n", + "\n", + "- [Current strategy](./project_current_strategy.md) - Current operating model, control-plane shape, and near-term execution priorities.\n", + "- [Failed ideas](./project_failed_ideas.md) - Approaches or patterns that did not work, or that should be treated cautiously.\n", + "- [Promising ideas](./project_promising_ideas.md) - Directions worth revisiting or expanding in future experiment cycles.\n", + "- [External references](./reference_external_sources.md) - Pointers to papers, repos, dashboards, or notes that are useful but live outside the codebase.\n", + "\n", + "## Memory policy\n", + "\n", + "- Update existing topic files before creating new ones.\n", + "- Prefer synthesized conclusions over dumping raw logs.\n", + "- Treat concrete file/function/flag references as historical claims and re-verify before acting on them.\n", + "- Keep the exact experiment ledger in `results.tsv`; keep the append-only event log in `experiment_memory.jsonl`.\n", + "- Use this directory for cross-run synthesis, strategy, and retrieval-friendly summaries.\n", + "\n" + ] + } + ], + "source": [ + "index_text = (MEMORY_ROOT / 'MEMORY.md').read_text(encoding='utf-8')\n", + "for required_link in [\n", + " './project_current_strategy.md',\n", + " './project_failed_ideas.md',\n", + " './project_promising_ideas.md',\n", + " './reference_external_sources.md',\n", + "]:\n", + " if required_link not in index_text:\n", + " raise RuntimeError(f'MEMORY.md is missing link: {required_link}')\n", + "print(index_text[:2000])" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'project_current_strategy.md': {'name': 'Current strategy',\n", + " 'description': 'Current operating strategy for the autoresearch manager/worker system and how to use the control plane safely.',\n", + " 'type': 'project'},\n", + " 'project_failed_ideas.md': {'name': 'Failed ideas',\n", + " 'description': 'Failed or cautionary patterns for the autoresearch system, especially around control-plane validation and worker behavior.',\n", + " 'type': 'project'},\n", + " 'project_promising_ideas.md': {'name': 'Promising ideas',\n", + " 'description': 'Promising next steps for memory, orchestration, and experiment quality in autoresearch.',\n", + " 'type': 'project'},\n", + " 'reference_external_sources.md': {'name': 'External references',\n", + " 'description': 'External sources, tools, or future systems that are relevant to the autoresearch memory and orchestration roadmap.',\n", + " 'type': 'reference'}}" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def parse_frontmatter(path: Path) -> dict[str, str]:\n", + " text = path.read_text(encoding='utf-8')\n", + " match = re.match(r'^---\\n(.*?)\\n---\\n', text, re.DOTALL)\n", + " if not match:\n", + " raise RuntimeError(f'Missing frontmatter in {path.name}')\n", + " result = {}\n", + " for line in match.group(1).splitlines():\n", + " if ':' not in line:\n", + " continue\n", + " key, value = line.split(':', 1)\n", + " result[key.strip()] = value.strip()\n", + " for field in ('name', 'description', 'type'):\n", + " if field not in result or not result[field]:\n", + " raise RuntimeError(f'Missing {field} in {path.name}')\n", + " return result\n", + "\n", + "metadata = {\n", + " name: parse_frontmatter(MEMORY_ROOT / name)\n", + " for name in expected_files\n", + " if name != 'MEMORY.md'\n", + "}\n", + "metadata" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'results_tsv': '/Users/wongdowling/Documents/autoresearch_harness/nodes/autoresearch-macos/results.tsv', 'experiment_memory_jsonl': '/Users/wongdowling/Documents/autoresearch_harness/nodes/autoresearch-macos/experiment_memory.jsonl', 'event_count': 30}\n" + ] + } + ], + "source": [ + "results_path = AUTORESEARCH_ROOT / 'results.tsv'\n", + "event_log_path = AUTORESEARCH_ROOT / 'experiment_memory.jsonl'\n", + "if not results_path.exists():\n", + " raise RuntimeError('results.tsv is missing')\n", + "if not event_log_path.exists():\n", + " raise RuntimeError('experiment_memory.jsonl is missing')\n", + "\n", + "event_count = sum(1 for line in event_log_path.read_text(encoding='utf-8').splitlines() if line.strip())\n", + "print({\n", + " 'results_tsv': str(results_path),\n", + " 'experiment_memory_jsonl': str(event_log_path),\n", + " 'event_count': event_count,\n", + "})" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'memory_dir_exists': True,\n", + " 'topic_file_count': 4,\n", + " 'has_exact_ledger': True,\n", + " 'has_append_only_event_log': True,\n", + " 'project_topic_types': ['project', 'reference']}" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "summary = {\n", + " 'memory_dir_exists': MEMORY_ROOT.exists(),\n", + " 'topic_file_count': len(expected_files) - 1,\n", + " 'has_exact_ledger': results_path.exists(),\n", + " 'has_append_only_event_log': event_log_path.exists(),\n", + " 'project_topic_types': sorted({item['type'] for item in metadata.values()}),\n", + "}\n", + "summary" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Success Criteria\n", + "\n", + "This notebook is a successful smoke test when:\n", + "\n", + "- the `memory/` directory exists\n", + "- `MEMORY.md` exists and links the expected topic files\n", + "- each topic file contains valid frontmatter with `name`, `description`, and `type`\n", + "- `results.tsv` still exists as the exact experiment ledger\n", + "- `experiment_memory.jsonl` still exists as the append-only event log\n", + "\n", + "That confirms the project now has the intended hybrid short-term memory layout:\n", + "\n", + "- exact operational records\n", + "- append-only event history\n", + "- small synthesized project memory\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "base", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.20" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}