From 6fbb702ae48f6e637b43e008311f7808eb771709 Mon Sep 17 00:00:00 2001 From: vintro Date: Mon, 17 Nov 2025 10:36:26 -0500 Subject: [PATCH 1/6] feat: beam benchmark initial commit --- tests/bench/README.md | 13 +- tests/bench/beam.py | 1095 ++++++++++++++++++++++++++++++++++++++++ tests/bench/harness.py | 6 +- 3 files changed, 1110 insertions(+), 4 deletions(-) create mode 100644 tests/bench/beam.py diff --git a/tests/bench/README.md b/tests/bench/README.md index 2f124efb..48a4b649 100644 --- a/tests/bench/README.md +++ b/tests/bench/README.md @@ -1,6 +1,15 @@ -# Honcho Development Harness +# Honcho Benchmark Suite -This directory contains a development harness script that makes it easy to run Honcho locally with a Docker database. +This directory contains benchmarking tools for evaluating Honcho's long-term memory capabilities. + +## Available Benchmarks + +- **LongMemEval**: Tests memory retention across multi-session conversations +- **BEAM**: Beyond a Million Tokens - comprehensive long-term memory evaluation across 10 memory abilities + +## Development Harness + +The development harness script makes it easy to run Honcho locally with a Docker database. ## Overview diff --git a/tests/bench/beam.py b/tests/bench/beam.py new file mode 100644 index 00000000..ed5575cf --- /dev/null +++ b/tests/bench/beam.py @@ -0,0 +1,1095 @@ +""" +Honcho BEAM Benchmark Test Runner + +A script that executes BEAM (Beyond a Million Tokens) benchmark tests against a running Honcho instance. +This script: +1. Loads BEAM conversations and probing questions from the data directory +2. Creates a workspace for each conversation +3. Ingests conversation turns as messages between user/assistant peers +4. Waits for the deriver queue to process everything +5. Executes probing questions across 10 memory ability categories +6. Evaluates responses using nugget-based LLM judging + +## BEAM Overview + +BEAM evaluates long-term memory capabilities across ten distinct memory abilities: +1. Abstention - Determines if models avoid answering without evidence +2. Contradiction Resolution - Detects inconsistencies across distant dialogue turns +3. Event Ordering - Assesses sequence recognition of evolving information +4. Information Extraction - Measures factual recall from lengthy histories +5. Instruction Following - Tests sustained adherence to user constraints +6. Knowledge Update - Evaluates fact revision when new information emerges +7. Multi-Session Reasoning - Probes inference integrating evidence across non-adjacent segments +8. Preference Following - Captures personalized, adaptive responses +9. Summarization - Tests content compression and abstraction +10. Temporal Reasoning - Examines explicit and implicit time-relation understanding + +## To use + +0. Set up env: +``` +uv sync +source .venv/bin/activate +``` +NOTE: you may create a .env file in this directory to customize honcho config. + +1. Run the test harness: +``` +python -m tests.bench.harness +``` + +2. Run this file with the 100K dataset: +``` +python -m tests.bench.beam --context-length 100K +``` + +Optional arguments: +``` +--context-length: Context length subset to test (100K, 500K, 1M, 10M) (default: 100K) +--conversation-ids: Comma-separated list of conversation IDs to test (default: all in context length) +--anthropic-api-key: Anthropic API key for response judging (can be set in .env as LLM_ANTHROPIC_API_KEY) +--timeout: Timeout for deriver queue to empty in seconds (default: 10 minutes) +--base-api-port: Base port for Honcho API instances (default: 8000) +--pool-size: Number of Honcho instances in the pool (default: 1) +--batch-size: Number of conversations to run concurrently in each batch (default: 1) +--json-output: Path to write JSON summary results for analytics +--cleanup-workspace: Delete workspace after executing each conversation (default: False) +--use-get-context: Use get_context + judge LLM instead of dialectic .chat endpoint (default: False) +``` + +## Other notes +- Judge is Claude Sonnet 4.5 +- Evaluation follows the paper's nugget-based methodology with 0/0.5/1 scoring +- Event ordering uses Kendall tau-b coefficient +""" + +import argparse +import asyncio +import json +import logging +import os +import time +from datetime import datetime +from pathlib import Path +from typing import Any + +import tiktoken +from anthropic import AsyncAnthropic +from anthropic.types import MessageParam +from dotenv import load_dotenv +from honcho import AsyncHoncho +from honcho.async_client.session import SessionPeerConfig +from honcho_core.types.workspaces.sessions.message_create_param import ( + MessageCreateParam, +) +from typing_extensions import TypedDict + +try: + from scipy.stats import ( + kendalltau, # type: ignore # pyright: ignore[reportUnknownVariableType] + ) +except ImportError: + kendalltau = None # type: ignore + +from src.config import settings +from src.utils.metrics_collector import MetricsCollector + +load_dotenv() + + +class QuestionResult(TypedDict): + """Type definition for question evaluation results.""" + + question: str + answer: str | None + actual_response: str + memory_ability: str + rubric: list[str] + nugget_scores: list[dict[str, Any]] | None + score: float + passed: bool + reasoning: str + + +class ConversationResult(TypedDict): + """Type definition for conversation execution results.""" + + conversation_id: str + context_length: str + workspace_id: str + total_turns: int + total_messages: int + question_results: list[QuestionResult] + ability_scores: dict[str, float] + overall_score: float + error: str | None + start_time: float + end_time: float + duration_seconds: float + + +class BEAMRunner: + """ + Executes BEAM benchmark tests against a Honcho instance. + """ + + def __init__( + self, + data_dir: Path, + base_api_port: int = 8000, + pool_size: int = 1, + anthropic_api_key: str | None = None, + timeout_seconds: int | None = None, + cleanup_workspace: bool = False, + use_get_context: bool = False, + ): + """ + Initialize the BEAM test runner. + + Args: + data_dir: Path to the BEAM data directory + base_api_port: Base port for Honcho API instances (default: 8000) + pool_size: Number of Honcho instances in the pool (default: 1) + anthropic_api_key: Anthropic API key for judging responses + timeout_seconds: Timeout for deriver queue in seconds + cleanup_workspace: If True, delete workspace after executing conversation + use_get_context: If True, use get_context + judge LLM instead of dialectic .chat endpoint + """ + self.data_dir: Path = data_dir + self.base_api_port: int = base_api_port + self.pool_size: int = pool_size + self.anthropic_api_key: str | None = anthropic_api_key + self.timeout_seconds: int = ( + timeout_seconds if timeout_seconds is not None else 10000 + ) + self.cleanup_workspace: bool = cleanup_workspace + self.use_get_context: bool = use_get_context + + # Initialize metrics collector + self.metrics_collector: MetricsCollector = MetricsCollector() + self.metrics_collector.start_collection( + f"beam_{datetime.now().strftime('%Y%m%d_%H%M%S')}" + ) + + # Configure logging + logging.basicConfig( + level=logging.WARNING, format="%(asctime)s - %(levelname)s - %(message)s" + ) + self.logger: logging.Logger = logging.getLogger(__name__) + + # Suppress HTTP request logs from the Honcho SDK + logging.getLogger("httpx").setLevel(logging.ERROR) + logging.getLogger("httpcore").setLevel(logging.ERROR) + + if self.anthropic_api_key: + self.anthropic_client: AsyncAnthropic = AsyncAnthropic( + api_key=self.anthropic_api_key + ) + else: + api_key = os.getenv("LLM_ANTHROPIC_API_KEY") + if not api_key: + raise ValueError("LLM_ANTHROPIC_API_KEY is not set") + self.anthropic_client = AsyncAnthropic(api_key=api_key) + + def get_honcho_url_for_index(self, conversation_index: int) -> str: + """ + Get the Honcho URL for a given conversation index using round-robin distribution. + + Args: + conversation_index: Index of the conversation + + Returns: + URL of the Honcho instance to use for this conversation + """ + instance_id = conversation_index % self.pool_size + port = self.base_api_port + instance_id + return f"http://localhost:{port}" + + def _format_duration(self, total_seconds: float) -> str: + """Format a duration in seconds into a human-readable string.""" + minutes = int(total_seconds // 60) + if minutes > 0: + seconds_rounded = int(round(total_seconds - minutes * 60)) + if seconds_rounded == 60: + minutes += 1 + seconds_rounded = 0 + return f"{minutes}m{seconds_rounded:02d}s" + return f"{total_seconds:.2f}s" + + def _calculate_tokens(self, text: str) -> int: + """Calculate tokens for a given text.""" + tokenizer = tiktoken.get_encoding("cl100k_base") + try: + return len( + tokenizer.encode( + text, + disallowed_special=( + tokenizer.special_tokens_set - {"<|endoftext|>"} + ), + ) + ) + except Exception: + return len(text) // 4 + + def load_conversation( + self, context_length: str, conversation_id: str + ) -> dict[str, Any]: + """ + Load a BEAM conversation from the data directory. + + Args: + context_length: Context length (100K, 500K, 1M, 10M) + conversation_id: Conversation ID + + Returns: + Dictionary containing conversation data and probing questions + """ + conv_dir = self.data_dir / context_length / conversation_id + + # Load chat data + chat_file = conv_dir / "chat.json" + with open(chat_file) as f: + chat_data = json.load(f) + + # Load probing questions + questions_file = conv_dir / "probing_questions" / "probing_questions.json" + with open(questions_file) as f: + questions_data = json.load(f) + + return {"chat": chat_data, "questions": questions_data} + + def list_conversations(self, context_length: str) -> list[str]: + """ + List all conversation IDs for a given context length. + + Args: + context_length: Context length (100K, 500K, 1M, 10M) + + Returns: + List of conversation ID strings + """ + context_dir = self.data_dir / context_length + return [ + d.name + for d in sorted(context_dir.iterdir()) + if d.is_dir() and d.name.isdigit() + ] + + async def create_honcho_client( + self, workspace_id: str, honcho_url: str + ) -> AsyncHoncho: + """ + Create a Honcho client for a specific workspace. + + Args: + workspace_id: Workspace ID + honcho_url: URL of the Honcho instance + + Returns: + AsyncHoncho client instance + """ + return AsyncHoncho( + environment="local", + workspace_id=workspace_id, + base_url=honcho_url, + ) + + async def wait_for_deriver_queue_empty( + self, honcho_client: AsyncHoncho, session_id: str | None = None + ) -> bool: + """Wait for the deriver queue to be empty.""" + start_time = time.time() + while True: + try: + status = await honcho_client.get_deriver_status(session_id=session_id) + except Exception: + await asyncio.sleep(1) + elapsed_time = time.time() - start_time + if elapsed_time >= self.timeout_seconds: + return False + continue + + if status.pending_work_units == 0 and status.in_progress_work_units == 0: + return True + + elapsed_time = time.time() - start_time + if elapsed_time >= self.timeout_seconds: + return False + await asyncio.sleep(1) + + async def judge_nugget_based( + self, + question: str, + rubric: list[str], + actual_response: str, + memory_ability: str, + ) -> dict[str, Any]: + """ + Use an LLM to judge a response using nugget-based evaluation. + + Args: + question: The question asked + rubric: List of nuggets (atomic criteria) to check + actual_response: Actual response from Honcho + memory_ability: The memory ability being tested + + Returns: + Judgment result with nugget scores and overall score + """ + try: + # Build the nugget evaluation prompt + nuggets_formatted = "\n".join( + [f"{i + 1}. {nugget}" for i, nugget in enumerate(rubric)] + ) + + system_prompt = f"""You are an expert judge evaluating AI responses for the {memory_ability} memory ability in the BEAM benchmark. + +Your task is to evaluate whether the AI's response satisfies each atomic criterion (nugget) from the rubric. + +SCORING INSTRUCTIONS: +For each nugget, assign a score: +- 1.0: The response fully satisfies this criterion +- 0.5: The response partially satisfies this criterion +- 0.0: The response does not satisfy this criterion + +Be strict but fair in your evaluation. Focus on whether the response contains the required information or demonstrates the required behavior. + +Always respond with valid JSON in this exact format: +{{ + "nugget_scores": [ + {{"nugget_index": 1, "score": 1.0, "reasoning": "brief explanation"}}, + {{"nugget_index": 2, "score": 0.5, "reasoning": "brief explanation"}} + ], + "overall_score": 0.75, + "overall_reasoning": "brief summary of the evaluation" +}}""" + + user_prompt = f"""Question: "{question}" + +Rubric (atomic criteria to check): +{nuggets_formatted} + +Actual Response: "{actual_response}" + +Evaluate the response against each nugget in the rubric. Provide a score for each nugget and calculate the overall score as the average of all nugget scores.""" + + response = await self.anthropic_client.messages.create( + model="claude-sonnet-4-5", + max_tokens=2000, + temperature=0.0, + system=system_prompt, + messages=[ + { + "role": "user", + "content": user_prompt, + } + ], + ) + + if not response.content: + raise ValueError("Anthropic returned empty response") + + content_block = response.content[0] + judgment_text = getattr(content_block, "text", None) + if judgment_text is None: + raise ValueError( + f"No text content in response block: {type(content_block)}" + ) + + # Extract JSON from the response if it's wrapped in markdown + if "```json" in judgment_text: + json_start = judgment_text.find("```json") + 7 + json_end = judgment_text.find("```", json_start) + judgment_text = judgment_text[json_start:json_end].strip() + elif "```" in judgment_text: + json_start = judgment_text.find("```") + 3 + json_end = judgment_text.find("```", json_start) + judgment_text = judgment_text[json_start:json_end].strip() + + judgment = json.loads(judgment_text) + return judgment + + except Exception as e: + self.logger.error(f"Error judging response: {e}") + # Fallback to simple 0 score + return { + "nugget_scores": [ + {"nugget_index": i + 1, "score": 0.0, "reasoning": f"Error: {e}"} + for i in range(len(rubric)) + ], + "overall_score": 0.0, + "overall_reasoning": f"Evaluation failed due to error: {e}", + } + + async def judge_event_ordering( + self, question: str, rubric: list[str], actual_response: str + ) -> dict[str, Any]: + """ + Judge event ordering questions using Kendall tau-b coefficient. + + Args: + question: The question asked + rubric: List of expected events in correct order + actual_response: Actual response from Honcho + + Returns: + Judgment with Kendall tau-b score + """ + try: + # First, extract the events mentioned in the response + system_prompt = """You are an expert at extracting ordered lists of events or items from text. + +Your task is to extract the ordered list of events/items mentioned in a response. + +Return valid JSON in this format: +{ + "extracted_events": ["first event", "second event", "third event"] +}""" + + user_prompt = f"""Question: "{question}" + +Response: "{actual_response}" + +Extract the ordered list of events or items mentioned in the response. Preserve the order as stated in the response.""" + + response = await self.anthropic_client.messages.create( + model="claude-sonnet-4-5", + max_tokens=1000, + temperature=0.0, + system=system_prompt, + messages=[ + { + "role": "user", + "content": user_prompt, + } + ], + ) + + if not response.content: + raise ValueError("Anthropic returned empty response") + + content_block = response.content[0] + extraction_text = getattr(content_block, "text", None) + if extraction_text is None: + raise ValueError( + f"No text content in response block: {type(content_block)}" + ) + + # Extract JSON + if "```json" in extraction_text: + json_start = extraction_text.find("```json") + 7 + json_end = extraction_text.find("```", json_start) + extraction_text = extraction_text[json_start:json_end].strip() + elif "```" in extraction_text: + json_start = extraction_text.find("```") + 3 + json_end = extraction_text.find("```", json_start) + extraction_text = extraction_text[json_start:json_end].strip() + + extracted = json.loads(extraction_text) + extracted_events = extracted.get("extracted_events", []) + + # Now compute alignment and Kendall tau-b + # Match extracted events to rubric events using LLM equivalence + alignment = await self._align_events(rubric, extracted_events) + + # Compute Kendall tau-b + tau: float + if kendalltau is None: + self.logger.warning( + "scipy not installed, cannot compute Kendall tau-b. Install with: uv pip install scipy" + ) + tau = 0.0 + elif len(alignment) < 2: + tau = 0.0 + else: + # Create rank lists + expected_ranks = list(range(len(alignment))) + actual_ranks = [alignment[i] for i in range(len(alignment))] + result_tuple: Any = kendalltau(expected_ranks, actual_ranks) + # kendalltau returns a tuple, first element is the tau coefficient + tau_value: Any = result_tuple[0] + # Handle the return type properly - convert to float + try: + tau = float(tau_value) + if tau != tau: # Check for NaN + tau = 0.0 + except (TypeError, ValueError): + tau = 0.0 + + return { + "kendall_tau_b": tau, + "extracted_events": extracted_events, + "alignment": alignment, + "overall_score": (tau + 1) / 2, # Normalize to [0, 1] + "overall_reasoning": f"Kendall tau-b coefficient: {tau:.3f}. Extracted {len(extracted_events)} events from response.", + } + + except Exception as e: + self.logger.error(f"Error in event ordering evaluation: {e}") + return { + "kendall_tau_b": 0.0, + "extracted_events": [], + "alignment": [], + "overall_score": 0.0, + "overall_reasoning": f"Evaluation failed due to error: {e}", + } + + async def _align_events( + self, expected_events: list[str], extracted_events: list[str] + ) -> list[int]: + """ + Align extracted events with expected events using LLM equivalence detection. + + Returns a list of indices mapping extracted events to expected events. + """ + # For each extracted event, find the best match in expected events + alignment: list[int] = [] + for extracted in extracted_events: + best_match_idx: int = -1 + for i, expected in enumerate(expected_events): + # Use simple string matching for now (can be enhanced with LLM) + if ( + expected.lower() in extracted.lower() + or extracted.lower() in expected.lower() + ): + best_match_idx = i + break + if best_match_idx >= 0: + alignment.append(best_match_idx) + return alignment + + async def execute_conversation( + self, context_length: str, conversation_id: str, honcho_url: str + ) -> ConversationResult: + """ + Execute BEAM benchmark for a single conversation. + + Args: + context_length: Context length (100K, 500K, 1M, 10M) + conversation_id: Conversation ID + honcho_url: URL of the Honcho instance to use + + Returns: + Conversation execution results + """ + start_time = time.time() + + print(f"\n{'=' * 80}") + print( + f"Executing BEAM conversation {conversation_id} ({context_length} context)" + ) + print(f"{'=' * 80}") + + # Create workspace for this conversation + workspace_id = f"beam_{context_length}_{conversation_id}" + honcho_client = await self.create_honcho_client(workspace_id, honcho_url) + + result: ConversationResult = { + "conversation_id": conversation_id, + "context_length": context_length, + "workspace_id": workspace_id, + "total_turns": 0, + "total_messages": 0, + "question_results": [], + "ability_scores": {}, + "overall_score": 0.0, + "error": None, + "start_time": start_time, + "end_time": 0.0, + "duration_seconds": 0.0, + } + + try: + # Load conversation data + conv_data = self.load_conversation(context_length, conversation_id) + chat_data = conv_data["chat"] + questions_data = conv_data["questions"] + + # Create peers + user_peer = await honcho_client.peer(id="user") + assistant_peer = await honcho_client.peer(id="assistant") + + # Create session for this conversation + session_id = f"{workspace_id}_session" + session = await honcho_client.session(id=session_id) + + # Configure peer observation - observe the user peer + await session.add_peers( + [ + ( + user_peer, + SessionPeerConfig(observe_me=True, observe_others=False), + ), + ( + assistant_peer, + SessionPeerConfig(observe_me=False, observe_others=False), + ), + ] + ) + + # Ingest conversation turns + print(f"[{workspace_id}] Ingesting conversation turns...") + messages: list[MessageCreateParam] = [] + + for batch in chat_data: + for turn_group in batch.get("turns", []): + for turn in turn_group: + role = turn["role"] + content = turn["content"] + result["total_turns"] += 1 + + # Split message if it exceeds 25000 characters + if len(content) > 25000: + chunks = [ + content[i : i + 25000] + for i in range(0, len(content), 25000) + ] + for chunk in chunks: + if role == "user": + messages.append(user_peer.message(chunk)) + elif role == "assistant": + messages.append(assistant_peer.message(chunk)) + else: + if role == "user": + messages.append(user_peer.message(content)) + elif role == "assistant": + messages.append(assistant_peer.message(content)) + + result["total_messages"] = len(messages) + + # Add messages in batches of 100 + for i in range(0, len(messages), 100): + batch = messages[i : i + 100] + await session.add_messages(batch) + + print( + f"[{workspace_id}] Ingested {result['total_messages']} messages. Waiting for deriver queue..." + ) + + # Wait for deriver queue to empty + await asyncio.sleep(1) + queue_empty = await self.wait_for_deriver_queue_empty(honcho_client) + if not queue_empty: + result["error"] = "Deriver queue timeout" + result["end_time"] = time.time() + result["duration_seconds"] = result["end_time"] - result["start_time"] + return result + + print(f"[{workspace_id}] Deriver queue empty. Executing questions...") + + # Execute questions for each memory ability + for ability, questions in questions_data.items(): + print( + f"\n[{workspace_id}] Testing {ability} ({len(questions)} questions)" + ) + + for q_idx, q_data in enumerate(questions): + question = q_data["question"] + rubric = q_data.get("rubric", []) + answer = ( + q_data.get("answer") + or q_data.get("ideal_response") + or q_data.get("ideal_answer") + ) + + print(f" Q{q_idx + 1}: {question[:100]}...") + + # Execute question using dialectic + if self.use_get_context: + context = await session.get_context( + summary=True, + peer_target="user", + last_user_message=question, + ) + context_messages = context.to_anthropic(assistant="assistant") + context_messages.append({"role": "user", "content": question}) + + from typing import cast + + response = await self.anthropic_client.messages.create( + model="claude-sonnet-4-5", + max_tokens=2048, + messages=cast(list[MessageParam], context_messages), + ) + + if not response.content: + actual_response = "" + else: + content_block = response.content[0] + actual_response = getattr(content_block, "text", "") + else: + actual_response = await user_peer.chat(question) + actual_response = ( + actual_response if isinstance(actual_response, str) else "" + ) + + # Judge response based on memory ability + if ability == "event_ordering": + judgment = await self.judge_event_ordering( + question, rubric, actual_response + ) + nugget_scores = None + else: + judgment = await self.judge_nugget_based( + question, rubric, actual_response, ability + ) + nugget_scores = judgment.get("nugget_scores") + + score = judgment.get("overall_score", 0.0) + reasoning = judgment.get("overall_reasoning", "") + + question_result: QuestionResult = { + "question": question, + "answer": answer, + "actual_response": actual_response, + "memory_ability": ability, + "rubric": rubric, + "nugget_scores": nugget_scores, + "score": score, + "passed": score >= 0.5, + "reasoning": reasoning, + } + + result["question_results"].append(question_result) + + status = "PASS" if score >= 0.5 else "FAIL" + print(f" Score: {score:.2f} [{status}]") + + # Calculate ability scores + ability_totals: dict[str, list[float]] = {} + for qr in result["question_results"]: + ability = qr["memory_ability"] + if ability not in ability_totals: + ability_totals[ability] = [] + ability_totals[ability].append(qr["score"]) + + for ability, scores in ability_totals.items(): + result["ability_scores"][ability] = sum(scores) / len(scores) + + # Calculate overall score + if result["ability_scores"]: + result["overall_score"] = sum(result["ability_scores"].values()) / len( + result["ability_scores"] + ) + + # Cleanup workspace if requested + if self.cleanup_workspace: + try: + await honcho_client.delete_workspace(workspace_id) + print(f"[{workspace_id}] Cleaned up workspace") + except Exception as e: + print(f"Failed to delete workspace: {e}") + + result["end_time"] = time.time() + result["duration_seconds"] = result["end_time"] - result["start_time"] + + print( + f"\n[{workspace_id}] Completed in {self._format_duration(result['duration_seconds'])}" + ) + print(f"Overall Score: {result['overall_score']:.3f}") + + except Exception as e: + self.logger.error(f"Error executing conversation {conversation_id}: {e}") + result["error"] = str(e) + result["end_time"] = time.time() + result["duration_seconds"] = result["end_time"] - result["start_time"] + + return result + + async def run_conversations( + self, + context_length: str, + conversation_ids: list[str], + batch_size: int = 1, + ) -> tuple[list[ConversationResult], float]: + """ + Run multiple conversations from the BEAM benchmark. + + Args: + context_length: Context length (100K, 500K, 1M, 10M) + conversation_ids: List of conversation IDs to run + batch_size: Number of conversations to run concurrently in each batch + + Returns: + Tuple of (list of conversation results, total duration) + """ + print( + f"Running {len(conversation_ids)} conversations from {context_length} context length" + ) + if self.pool_size > 1: + print( + f"Distributing conversations across {self.pool_size} Honcho instances" + ) + + overall_start = time.time() + all_results: list[ConversationResult] = [] + + for i in range(0, len(conversation_ids), batch_size): + batch = conversation_ids[i : i + batch_size] + batch_num = (i // batch_size) + 1 + total_batches = (len(conversation_ids) + batch_size - 1) // batch_size + + print(f"\n{'=' * 80}") + print( + f"Processing batch {batch_num}/{total_batches} ({len(batch)} conversations)" + ) + print(f"{'=' * 80}") + + # Run conversations in current batch concurrently + batch_results: list[ConversationResult] = await asyncio.gather( + *[ + self.execute_conversation( + context_length, conv_id, self.get_honcho_url_for_index(i + idx) + ) + for idx, conv_id in enumerate(batch) + ] + ) + + all_results.extend(batch_results) + + overall_end = time.time() + overall_duration = overall_end - overall_start + + # Finalize metrics collection + self.metrics_collector.finalize_collection() + + return all_results, overall_duration + + def print_summary( + self, results: list[ConversationResult], total_elapsed_seconds: float + ) -> None: + """Print a summary of all test results.""" + print(f"\n{'=' * 80}") + print("BEAM BENCHMARK EXECUTION SUMMARY") + print(f"{'=' * 80}") + + total_conversations = len(results) + total_questions = sum(len(r["question_results"]) for r in results) + + print(f"Total Conversations: {total_conversations}") + print(f"Total Questions: {total_questions}") + print(f"Total Test Time: {self._format_duration(total_elapsed_seconds)}") + + # Calculate average scores by ability + ability_scores: dict[str, list[float]] = {} + for result in results: + for ability, score in result["ability_scores"].items(): + if ability not in ability_scores: + ability_scores[ability] = [] + ability_scores[ability].append(score) + + print("\nAverage Scores by Memory Ability:") + for ability, scores in sorted(ability_scores.items()): + avg_score = sum(scores) / len(scores) + print(f" {ability:30s}: {avg_score:.3f}") + + # Overall average + overall_scores = [r["overall_score"] for r in results] + overall_avg = ( + sum(overall_scores) / len(overall_scores) if overall_scores else 0.0 + ) + print(f"\n{'Overall Average Score':30s}: {overall_avg:.3f}") + + print(f"{'=' * 80}") + + def generate_json_summary( + self, + results: list[ConversationResult], + context_length: str, + total_elapsed_seconds: float, + output_file: Path, + ) -> None: + """Generate a comprehensive JSON summary of test results.""" + # Calculate summary statistics + total_conversations = len(results) + total_questions = sum(len(r["question_results"]) for r in results) + + # Calculate average scores by ability + ability_scores: dict[str, list[float]] = {} + for result in results: + for ability, score in result["ability_scores"].items(): + if ability not in ability_scores: + ability_scores[ability] = [] + ability_scores[ability].append(score) + + ability_averages = { + ability: sum(scores) / len(scores) + for ability, scores in ability_scores.items() + } + + # Overall average + overall_scores = [r["overall_score"] for r in results] + overall_avg = ( + sum(overall_scores) / len(overall_scores) if overall_scores else 0.0 + ) + + summary = { + "metadata": { + "context_length": context_length, + "execution_timestamp": datetime.now().isoformat(), + "runner_version": "1.0.0", + "base_api_port": self.base_api_port, + "pool_size": self.pool_size, + "timeout_seconds": self.timeout_seconds, + "deriver_settings": settings.DERIVER.model_dump(), + "dialectic_settings": settings.DIALECTIC.model_dump(), + }, + "summary_statistics": { + "total_conversations": total_conversations, + "total_questions": total_questions, + "overall_average_score": overall_avg, + "ability_averages": ability_averages, + }, + "timing": { + "total_duration_seconds": total_elapsed_seconds, + }, + "detailed_results": results, + } + + output_file.parent.mkdir(parents=True, exist_ok=True) + with open(output_file, "w") as f: + json.dump(summary, f, indent=2, default=str) + print(f"\nJSON summary written to: {output_file}") + + +async def main() -> int: + """Main entry point for the BEAM test runner.""" + parser = argparse.ArgumentParser( + description="Run BEAM benchmark tests against a Honcho instance", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + + parser.add_argument( + "--context-length", + type=str, + default="100K", + choices=["100K", "500K", "1M", "10M"], + help="Context length subset to test (default: 100K)", + ) + + parser.add_argument( + "--conversation-ids", + type=str, + help="Comma-separated list of conversation IDs to test (default: all)", + ) + + parser.add_argument( + "--base-api-port", + type=int, + default=8000, + help="Base port for Honcho API instances (default: 8000)", + ) + + parser.add_argument( + "--pool-size", + type=int, + default=1, + help="Number of Honcho instances in the pool (default: 1)", + ) + + parser.add_argument( + "--anthropic-api-key", + type=str, + help="Anthropic API key for response judging (optional)", + ) + + parser.add_argument( + "--timeout", + type=int, + default=None, + help="Timeout for deriver queue to empty in seconds (default: 10 minutes)", + ) + + parser.add_argument( + "--batch-size", + type=int, + default=1, + help="Number of conversations to run concurrently in each batch (default: 1)", + ) + + parser.add_argument( + "--json-output", + type=Path, + help="Path to write JSON summary results for analytics (optional)", + ) + + parser.add_argument( + "--cleanup-workspace", + action="store_true", + help="Delete workspace after executing each conversation (default: False)", + ) + + parser.add_argument( + "--use-get-context", + action="store_true", + help="Use get_context + judge LLM instead of dialectic .chat endpoint (default: False)", + ) + + args = parser.parse_args() + + # Setup data directory + data_dir = Path(__file__).parent / "beam_data" + if not data_dir.exists(): + print(f"Error: BEAM data directory not found at {data_dir}") + return 1 + + # Create runner + runner = BEAMRunner( + data_dir=data_dir, + base_api_port=args.base_api_port, + pool_size=args.pool_size, + anthropic_api_key=args.anthropic_api_key, + timeout_seconds=args.timeout, + cleanup_workspace=args.cleanup_workspace, + use_get_context=args.use_get_context, + ) + + try: + # Determine which conversations to run + if args.conversation_ids: + conversation_ids = args.conversation_ids.split(",") + else: + conversation_ids = runner.list_conversations(args.context_length) + + # Run conversations + results, total_elapsed = await runner.run_conversations( + args.context_length, conversation_ids, args.batch_size + ) + + runner.print_summary(results, total_elapsed) + + # Generate JSON output + if args.json_output: + output_file = args.json_output + else: + output_file = Path( + f"tests/bench/eval_results/beam_{args.context_length}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json" + ) + + runner.generate_json_summary( + results, args.context_length, total_elapsed, output_file + ) + + # Export metrics + metrics_output = Path( + f"tests/bench/perf_metrics/beam_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json" + ) + runner.metrics_collector.export_to_json(metrics_output) + runner.metrics_collector.cleanup_collection() + + return 0 + + except KeyboardInterrupt: + print("\nTest execution interrupted by user") + return 1 + except Exception as e: + print(f"Error running tests: {e}") + import traceback + + traceback.print_exc() + return 1 + + +if __name__ == "__main__": + exit_code = asyncio.run(main()) + exit(exit_code) diff --git a/tests/bench/harness.py b/tests/bench/harness.py index 1e3f1a2c..cedb68c1 100755 --- a/tests/bench/harness.py +++ b/tests/bench/harness.py @@ -137,7 +137,8 @@ class HonchoHarness: # Change to the temp directory and start the database service result = subprocess.run( [ - "docker-compose", + "docker", + "compose", "-f", str(self.docker_compose_file), "-p", @@ -166,7 +167,8 @@ class HonchoHarness: # Change to the temp directory and start the redis service result = subprocess.run( [ - "docker-compose", + "docker", + "compose", "-f", str(self.docker_compose_file), "-p", From a88799ca3eb38cdd8e8553b6633f36a97b4191f9 Mon Sep 17 00:00:00 2001 From: vintro Date: Wed, 19 Nov 2025 09:47:49 -0500 Subject: [PATCH 2/6] fix: update code to handle 10M --- tests/bench/beam.py | 74 +++++++++++++++++++++++++++++++++------------ 1 file changed, 55 insertions(+), 19 deletions(-) diff --git a/tests/bench/beam.py b/tests/bench/beam.py index ed5575cf..fc2ef499 100644 --- a/tests/bench/beam.py +++ b/tests/bench/beam.py @@ -631,29 +631,65 @@ Extract the ordered list of events or items mentioned in the response. Preserve print(f"[{workspace_id}] Ingesting conversation turns...") messages: list[MessageCreateParam] = [] + # Handle different data structures for 10M vs other sizes for batch in chat_data: - for turn_group in batch.get("turns", []): - for turn in turn_group: - role = turn["role"] - content = turn["content"] - result["total_turns"] += 1 + # Check if this is a 10M conversation with plan-based structure + if any(key.startswith("plan-") for key in batch): + # 10M structure: { "plan-1": [...], "plan-2": [...], ... } + for plan_name, plan_batches in batch.items(): + if not plan_name.startswith("plan-"): + continue + for plan_batch in plan_batches: + for turn_group in plan_batch.get("turns", []): + for turn in turn_group: + role = turn["role"] + content = turn["content"] + result["total_turns"] += 1 - # Split message if it exceeds 25000 characters - if len(content) > 25000: - chunks = [ - content[i : i + 25000] - for i in range(0, len(content), 25000) - ] - for chunk in chunks: + # Split message if it exceeds 25000 characters + if len(content) > 25000: + chunks = [ + content[i : i + 25000] + for i in range(0, len(content), 25000) + ] + for chunk in chunks: + if role == "user": + messages.append(user_peer.message(chunk)) + elif role == "assistant": + messages.append( + assistant_peer.message(chunk) + ) + else: + if role == "user": + messages.append(user_peer.message(content)) + elif role == "assistant": + messages.append( + assistant_peer.message(content) + ) + else: + # Standard structure for 100K, 500K, 1M + for turn_group in batch.get("turns", []): + for turn in turn_group: + role = turn["role"] + content = turn["content"] + result["total_turns"] += 1 + + # Split message if it exceeds 25000 characters + if len(content) > 25000: + chunks = [ + content[i : i + 25000] + for i in range(0, len(content), 25000) + ] + for chunk in chunks: + if role == "user": + messages.append(user_peer.message(chunk)) + elif role == "assistant": + messages.append(assistant_peer.message(chunk)) + else: if role == "user": - messages.append(user_peer.message(chunk)) + messages.append(user_peer.message(content)) elif role == "assistant": - messages.append(assistant_peer.message(chunk)) - else: - if role == "user": - messages.append(user_peer.message(content)) - elif role == "assistant": - messages.append(assistant_peer.message(content)) + messages.append(assistant_peer.message(content)) result["total_messages"] = len(messages) From c8b0aa9cf2ac997a8c307f1662fb64408bd66fde Mon Sep 17 00:00:00 2001 From: vintro Date: Wed, 19 Nov 2025 13:43:42 -0500 Subject: [PATCH 3/6] fix: timeout transparency --- tests/bench/beam.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/bench/beam.py b/tests/bench/beam.py index fc2ef499..4129dae9 100644 --- a/tests/bench/beam.py +++ b/tests/bench/beam.py @@ -654,7 +654,9 @@ Extract the ordered list of events or items mentioned in the response. Preserve ] for chunk in chunks: if role == "user": - messages.append(user_peer.message(chunk)) + messages.append( + user_peer.message(chunk) + ) elif role == "assistant": messages.append( assistant_peer.message(chunk) @@ -709,6 +711,12 @@ Extract the ordered list of events or items mentioned in the response. Preserve result["error"] = "Deriver queue timeout" result["end_time"] = time.time() result["duration_seconds"] = result["end_time"] - result["start_time"] + print( + f"\n[{workspace_id}] ERROR: Deriver queue timeout after {self.timeout_seconds}s" + ) + print( + f"[{workspace_id}] Failed to complete in {self._format_duration(result['duration_seconds'])}" + ) return result print(f"[{workspace_id}] Deriver queue empty. Executing questions...") From 1616304b4049cf1808669b7ff40562cf155921b3 Mon Sep 17 00:00:00 2001 From: Benjamin McCormick Date: Wed, 19 Nov 2025 17:56:12 -0500 Subject: [PATCH 4/6] chore: fix up beam script --- pyproject.toml | 1 + tests/bench/.gitignore | 1 + tests/bench/beam.py | 131 +++++++++++++++++++++----------------- uv.lock | 139 ++++++++++++++++++++++++++++++++++++++++- 4 files changed, 214 insertions(+), 58 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index da2400ac..06f4e6b3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,6 +50,7 @@ dev-dependencies = [ "pytest-cov>=6.2.1", "honcho-ai", "fakeredis>=2.32.0", + "scipy>=1.15.3", ] [tool.uv.workspace] diff --git a/tests/bench/.gitignore b/tests/bench/.gitignore index e8d34dc6..4b8fc430 100644 --- a/tests/bench/.gitignore +++ b/tests/bench/.gitignore @@ -1,3 +1,4 @@ longmemeval_data eval_results perf_metrics +beam_data diff --git a/tests/bench/beam.py b/tests/bench/beam.py index 4129dae9..ae322c9b 100644 --- a/tests/bench/beam.py +++ b/tests/bench/beam.py @@ -71,26 +71,20 @@ import os import time from datetime import datetime from pathlib import Path -from typing import Any +from typing import Any, cast import tiktoken from anthropic import AsyncAnthropic -from anthropic.types import MessageParam +from anthropic.types import MessageParam, ToolParam from dotenv import load_dotenv from honcho import AsyncHoncho from honcho.async_client.session import SessionPeerConfig from honcho_core.types.workspaces.sessions.message_create_param import ( MessageCreateParam, ) +from scipy.stats import kendalltau # pyright: ignore[reportUnknownVariableType] from typing_extensions import TypedDict -try: - from scipy.stats import ( - kendalltau, # type: ignore # pyright: ignore[reportUnknownVariableType] - ) -except ImportError: - kendalltau = None # type: ignore - from src.config import settings from src.utils.metrics_collector import MetricsCollector @@ -140,7 +134,7 @@ class BEAMRunner: pool_size: int = 1, anthropic_api_key: str | None = None, timeout_seconds: int | None = None, - cleanup_workspace: bool = False, + cleanup_workspace: bool = True, use_get_context: bool = False, ): """ @@ -354,15 +348,7 @@ For each nugget, assign a score: Be strict but fair in your evaluation. Focus on whether the response contains the required information or demonstrates the required behavior. -Always respond with valid JSON in this exact format: -{{ - "nugget_scores": [ - {{"nugget_index": 1, "score": 1.0, "reasoning": "brief explanation"}}, - {{"nugget_index": 2, "score": 0.5, "reasoning": "brief explanation"}} - ], - "overall_score": 0.75, - "overall_reasoning": "brief summary of the evaluation" -}}""" +Use the `evaluate_response` tool to submit your evaluation.""" user_prompt = f"""Question: "{question}" @@ -373,6 +359,31 @@ Actual Response: "{actual_response}" Evaluate the response against each nugget in the rubric. Provide a score for each nugget and calculate the overall score as the average of all nugget scores.""" + tool_definition: ToolParam = { + "name": "evaluate_response", + "description": "Submit the evaluation results for the response based on the rubric.", + "input_schema": { + "type": "object", + "properties": { + "nugget_scores": { + "type": "array", + "items": { + "type": "object", + "properties": { + "nugget_index": {"type": "integer"}, + "score": {"type": "number"}, + "reasoning": {"type": "string"}, + }, + "required": ["nugget_index", "score", "reasoning"], + }, + }, + "overall_score": {"type": "number"}, + "overall_reasoning": {"type": "string"}, + }, + "required": ["nugget_scores", "overall_score", "overall_reasoning"], + }, + } + response = await self.anthropic_client.messages.create( model="claude-sonnet-4-5", max_tokens=2000, @@ -384,30 +395,26 @@ Evaluate the response against each nugget in the rubric. Provide a score for eac "content": user_prompt, } ], + tools=[tool_definition], + tool_choice={"type": "tool", "name": "evaluate_response"}, ) if not response.content: raise ValueError("Anthropic returned empty response") - content_block = response.content[0] - judgment_text = getattr(content_block, "text", None) - if judgment_text is None: - raise ValueError( - f"No text content in response block: {type(content_block)}" - ) + # Find the tool use block + tool_use_block = next( + (block for block in response.content if block.type == "tool_use"), None + ) - # Extract JSON from the response if it's wrapped in markdown - if "```json" in judgment_text: - json_start = judgment_text.find("```json") + 7 - json_end = judgment_text.find("```", json_start) - judgment_text = judgment_text[json_start:json_end].strip() - elif "```" in judgment_text: - json_start = judgment_text.find("```") + 3 - json_end = judgment_text.find("```", json_start) - judgment_text = judgment_text[json_start:json_end].strip() + if not tool_use_block: + raise ValueError("No tool use block found in response") - judgment = json.loads(judgment_text) - return judgment + judgment: object = tool_use_block.input + if not isinstance(judgment, dict): + raise ValueError(f"Tool input is not a dictionary: {type(judgment)}") + + return cast(dict[str, Any], judgment) except Exception as e: self.logger.error(f"Error judging response: {e}") @@ -441,10 +448,7 @@ Evaluate the response against each nugget in the rubric. Provide a score for eac Your task is to extract the ordered list of events/items mentioned in a response. -Return valid JSON in this format: -{ - "extracted_events": ["first event", "second event", "third event"] -}""" +Use the `extract_ordered_events` tool to submit the extracted list.""" user_prompt = f"""Question: "{question}" @@ -452,6 +456,21 @@ Response: "{actual_response}" Extract the ordered list of events or items mentioned in the response. Preserve the order as stated in the response.""" + tool_definition: ToolParam = { + "name": "extract_ordered_events", + "description": "Submit the ordered list of events extracted from the response.", + "input_schema": { + "type": "object", + "properties": { + "extracted_events": { + "type": "array", + "items": {"type": "string"}, + }, + }, + "required": ["extracted_events"], + }, + } + response = await self.anthropic_client.messages.create( model="claude-sonnet-4-5", max_tokens=1000, @@ -463,30 +482,28 @@ Extract the ordered list of events or items mentioned in the response. Preserve "content": user_prompt, } ], + tools=[tool_definition], + tool_choice={"type": "tool", "name": "extract_ordered_events"}, ) if not response.content: raise ValueError("Anthropic returned empty response") - content_block = response.content[0] - extraction_text = getattr(content_block, "text", None) - if extraction_text is None: - raise ValueError( - f"No text content in response block: {type(content_block)}" - ) + # Find the tool use block + tool_use_block = next( + (block for block in response.content if block.type == "tool_use"), None + ) - # Extract JSON - if "```json" in extraction_text: - json_start = extraction_text.find("```json") + 7 - json_end = extraction_text.find("```", json_start) - extraction_text = extraction_text[json_start:json_end].strip() - elif "```" in extraction_text: - json_start = extraction_text.find("```") + 3 - json_end = extraction_text.find("```", json_start) - extraction_text = extraction_text[json_start:json_end].strip() + if not tool_use_block: + raise ValueError("No tool use block found in response") - extracted = json.loads(extraction_text) - extracted_events = extracted.get("extracted_events", []) + extracted = tool_use_block.input + if not isinstance(extracted, dict): + raise ValueError(f"Tool input is not a dictionary: {type(extracted)}") + + extracted_dict = cast(dict[str, Any], extracted) + raw_events: list[str] = extracted_dict.get("extracted_events", []) + extracted_events = [str(e) for e in raw_events] # Now compute alignment and Kendall tau-b # Match extracted events to rubric events using LLM equivalence diff --git a/uv.lock b/uv.lock index 01ae81c3..0b347b5f 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.10" resolution-markers = [ "python_full_version >= '3.13'", @@ -755,6 +755,8 @@ dev = [ { name = "pytest-asyncio" }, { name = "pytest-cov" }, { name = "ruff" }, + { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "scipy", version = "1.16.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "sqlalchemy-utils" }, ] @@ -802,6 +804,7 @@ dev = [ { name = "pytest-asyncio", specifier = ">=0.23.7" }, { name = "pytest-cov", specifier = ">=6.2.1" }, { name = "ruff", specifier = ">=0.11.2" }, + { name = "scipy", specifier = ">=1.15.3" }, { name = "sqlalchemy-utils", specifier = ">=0.41.2" }, ] @@ -2274,6 +2277,140 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4c/9b/0b8aa09817b63e78d94b4977f18b1fcaead3165a5ee49251c5d5c245bb2d/ruff-0.12.7-py3-none-win_arm64.whl", hash = "sha256:dfce05101dbd11833a0776716d5d1578641b7fddb537fe7fa956ab85d1769b69", size = 11982083, upload-time = "2025-07-29T22:32:33.881Z" }, ] +[[package]] +name = "scipy" +version = "1.15.3" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.11'", +] +dependencies = [ + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0f/37/6964b830433e654ec7485e45a00fc9a27cf868d622838f6b6d9c5ec0d532/scipy-1.15.3.tar.gz", hash = "sha256:eae3cf522bc7df64b42cad3925c876e1b0b6c35c1337c93e12c0f366f55b0eaf", size = 59419214, upload-time = "2025-05-08T16:13:05.955Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/2f/4966032c5f8cc7e6a60f1b2e0ad686293b9474b65246b0c642e3ef3badd0/scipy-1.15.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:a345928c86d535060c9c2b25e71e87c39ab2f22fc96e9636bd74d1dbf9de448c", size = 38702770, upload-time = "2025-05-08T16:04:20.849Z" }, + { url = "https://files.pythonhosted.org/packages/a0/6e/0c3bf90fae0e910c274db43304ebe25a6b391327f3f10b5dcc638c090795/scipy-1.15.3-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:ad3432cb0f9ed87477a8d97f03b763fd1d57709f1bbde3c9369b1dff5503b253", size = 30094511, upload-time = "2025-05-08T16:04:27.103Z" }, + { url = "https://files.pythonhosted.org/packages/ea/b1/4deb37252311c1acff7f101f6453f0440794f51b6eacb1aad4459a134081/scipy-1.15.3-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:aef683a9ae6eb00728a542b796f52a5477b78252edede72b8327a886ab63293f", size = 22368151, upload-time = "2025-05-08T16:04:31.731Z" }, + { url = "https://files.pythonhosted.org/packages/38/7d/f457626e3cd3c29b3a49ca115a304cebb8cc6f31b04678f03b216899d3c6/scipy-1.15.3-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:1c832e1bd78dea67d5c16f786681b28dd695a8cb1fb90af2e27580d3d0967e92", size = 25121732, upload-time = "2025-05-08T16:04:36.596Z" }, + { url = "https://files.pythonhosted.org/packages/db/0a/92b1de4a7adc7a15dcf5bddc6e191f6f29ee663b30511ce20467ef9b82e4/scipy-1.15.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:263961f658ce2165bbd7b99fa5135195c3a12d9bef045345016b8b50c315cb82", size = 35547617, upload-time = "2025-05-08T16:04:43.546Z" }, + { url = "https://files.pythonhosted.org/packages/8e/6d/41991e503e51fc1134502694c5fa7a1671501a17ffa12716a4a9151af3df/scipy-1.15.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e2abc762b0811e09a0d3258abee2d98e0c703eee49464ce0069590846f31d40", size = 37662964, upload-time = "2025-05-08T16:04:49.431Z" }, + { url = "https://files.pythonhosted.org/packages/25/e1/3df8f83cb15f3500478c889be8fb18700813b95e9e087328230b98d547ff/scipy-1.15.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ed7284b21a7a0c8f1b6e5977ac05396c0d008b89e05498c8b7e8f4a1423bba0e", size = 37238749, upload-time = "2025-05-08T16:04:55.215Z" }, + { url = "https://files.pythonhosted.org/packages/93/3e/b3257cf446f2a3533ed7809757039016b74cd6f38271de91682aa844cfc5/scipy-1.15.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5380741e53df2c566f4d234b100a484b420af85deb39ea35a1cc1be84ff53a5c", size = 40022383, upload-time = "2025-05-08T16:05:01.914Z" }, + { url = "https://files.pythonhosted.org/packages/d1/84/55bc4881973d3f79b479a5a2e2df61c8c9a04fcb986a213ac9c02cfb659b/scipy-1.15.3-cp310-cp310-win_amd64.whl", hash = "sha256:9d61e97b186a57350f6d6fd72640f9e99d5a4a2b8fbf4b9ee9a841eab327dc13", size = 41259201, upload-time = "2025-05-08T16:05:08.166Z" }, + { url = "https://files.pythonhosted.org/packages/96/ab/5cc9f80f28f6a7dff646c5756e559823614a42b1939d86dd0ed550470210/scipy-1.15.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:993439ce220d25e3696d1b23b233dd010169b62f6456488567e830654ee37a6b", size = 38714255, upload-time = "2025-05-08T16:05:14.596Z" }, + { url = "https://files.pythonhosted.org/packages/4a/4a/66ba30abe5ad1a3ad15bfb0b59d22174012e8056ff448cb1644deccbfed2/scipy-1.15.3-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:34716e281f181a02341ddeaad584205bd2fd3c242063bd3423d61ac259ca7eba", size = 30111035, upload-time = "2025-05-08T16:05:20.152Z" }, + { url = "https://files.pythonhosted.org/packages/4b/fa/a7e5b95afd80d24313307f03624acc65801846fa75599034f8ceb9e2cbf6/scipy-1.15.3-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3b0334816afb8b91dab859281b1b9786934392aa3d527cd847e41bb6f45bee65", size = 22384499, upload-time = "2025-05-08T16:05:24.494Z" }, + { url = "https://files.pythonhosted.org/packages/17/99/f3aaddccf3588bb4aea70ba35328c204cadd89517a1612ecfda5b2dd9d7a/scipy-1.15.3-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:6db907c7368e3092e24919b5e31c76998b0ce1684d51a90943cb0ed1b4ffd6c1", size = 25152602, upload-time = "2025-05-08T16:05:29.313Z" }, + { url = "https://files.pythonhosted.org/packages/56/c5/1032cdb565f146109212153339f9cb8b993701e9fe56b1c97699eee12586/scipy-1.15.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:721d6b4ef5dc82ca8968c25b111e307083d7ca9091bc38163fb89243e85e3889", size = 35503415, upload-time = "2025-05-08T16:05:34.699Z" }, + { url = "https://files.pythonhosted.org/packages/bd/37/89f19c8c05505d0601ed5650156e50eb881ae3918786c8fd7262b4ee66d3/scipy-1.15.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39cb9c62e471b1bb3750066ecc3a3f3052b37751c7c3dfd0fd7e48900ed52982", size = 37652622, upload-time = "2025-05-08T16:05:40.762Z" }, + { url = "https://files.pythonhosted.org/packages/7e/31/be59513aa9695519b18e1851bb9e487de66f2d31f835201f1b42f5d4d475/scipy-1.15.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:795c46999bae845966368a3c013e0e00947932d68e235702b5c3f6ea799aa8c9", size = 37244796, upload-time = "2025-05-08T16:05:48.119Z" }, + { url = "https://files.pythonhosted.org/packages/10/c0/4f5f3eeccc235632aab79b27a74a9130c6c35df358129f7ac8b29f562ac7/scipy-1.15.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:18aaacb735ab38b38db42cb01f6b92a2d0d4b6aabefeb07f02849e47f8fb3594", size = 40047684, upload-time = "2025-05-08T16:05:54.22Z" }, + { url = "https://files.pythonhosted.org/packages/ab/a7/0ddaf514ce8a8714f6ed243a2b391b41dbb65251affe21ee3077ec45ea9a/scipy-1.15.3-cp311-cp311-win_amd64.whl", hash = "sha256:ae48a786a28412d744c62fd7816a4118ef97e5be0bee968ce8f0a2fba7acf3bb", size = 41246504, upload-time = "2025-05-08T16:06:00.437Z" }, + { url = "https://files.pythonhosted.org/packages/37/4b/683aa044c4162e10ed7a7ea30527f2cbd92e6999c10a8ed8edb253836e9c/scipy-1.15.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6ac6310fdbfb7aa6612408bd2f07295bcbd3fda00d2d702178434751fe48e019", size = 38766735, upload-time = "2025-05-08T16:06:06.471Z" }, + { url = "https://files.pythonhosted.org/packages/7b/7e/f30be3d03de07f25dc0ec926d1681fed5c732d759ac8f51079708c79e680/scipy-1.15.3-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:185cd3d6d05ca4b44a8f1595af87f9c372bb6acf9c808e99aa3e9aa03bd98cf6", size = 30173284, upload-time = "2025-05-08T16:06:11.686Z" }, + { url = "https://files.pythonhosted.org/packages/07/9c/0ddb0d0abdabe0d181c1793db51f02cd59e4901da6f9f7848e1f96759f0d/scipy-1.15.3-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:05dc6abcd105e1a29f95eada46d4a3f251743cfd7d3ae8ddb4088047f24ea477", size = 22446958, upload-time = "2025-05-08T16:06:15.97Z" }, + { url = "https://files.pythonhosted.org/packages/af/43/0bce905a965f36c58ff80d8bea33f1f9351b05fad4beaad4eae34699b7a1/scipy-1.15.3-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:06efcba926324df1696931a57a176c80848ccd67ce6ad020c810736bfd58eb1c", size = 25242454, upload-time = "2025-05-08T16:06:20.394Z" }, + { url = "https://files.pythonhosted.org/packages/56/30/a6f08f84ee5b7b28b4c597aca4cbe545535c39fe911845a96414700b64ba/scipy-1.15.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c05045d8b9bfd807ee1b9f38761993297b10b245f012b11b13b91ba8945f7e45", size = 35210199, upload-time = "2025-05-08T16:06:26.159Z" }, + { url = "https://files.pythonhosted.org/packages/0b/1f/03f52c282437a168ee2c7c14a1a0d0781a9a4a8962d84ac05c06b4c5b555/scipy-1.15.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:271e3713e645149ea5ea3e97b57fdab61ce61333f97cfae392c28ba786f9bb49", size = 37309455, upload-time = "2025-05-08T16:06:32.778Z" }, + { url = "https://files.pythonhosted.org/packages/89/b1/fbb53137f42c4bf630b1ffdfc2151a62d1d1b903b249f030d2b1c0280af8/scipy-1.15.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6cfd56fc1a8e53f6e89ba3a7a7251f7396412d655bca2aa5611c8ec9a6784a1e", size = 36885140, upload-time = "2025-05-08T16:06:39.249Z" }, + { url = "https://files.pythonhosted.org/packages/2e/2e/025e39e339f5090df1ff266d021892694dbb7e63568edcfe43f892fa381d/scipy-1.15.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0ff17c0bb1cb32952c09217d8d1eed9b53d1463e5f1dd6052c7857f83127d539", size = 39710549, upload-time = "2025-05-08T16:06:45.729Z" }, + { url = "https://files.pythonhosted.org/packages/e6/eb/3bf6ea8ab7f1503dca3a10df2e4b9c3f6b3316df07f6c0ded94b281c7101/scipy-1.15.3-cp312-cp312-win_amd64.whl", hash = "sha256:52092bc0472cfd17df49ff17e70624345efece4e1a12b23783a1ac59a1b728ed", size = 40966184, upload-time = "2025-05-08T16:06:52.623Z" }, + { url = "https://files.pythonhosted.org/packages/73/18/ec27848c9baae6e0d6573eda6e01a602e5649ee72c27c3a8aad673ebecfd/scipy-1.15.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:2c620736bcc334782e24d173c0fdbb7590a0a436d2fdf39310a8902505008759", size = 38728256, upload-time = "2025-05-08T16:06:58.696Z" }, + { url = "https://files.pythonhosted.org/packages/74/cd/1aef2184948728b4b6e21267d53b3339762c285a46a274ebb7863c9e4742/scipy-1.15.3-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:7e11270a000969409d37ed399585ee530b9ef6aa99d50c019de4cb01e8e54e62", size = 30109540, upload-time = "2025-05-08T16:07:04.209Z" }, + { url = "https://files.pythonhosted.org/packages/5b/d8/59e452c0a255ec352bd0a833537a3bc1bfb679944c4938ab375b0a6b3a3e/scipy-1.15.3-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:8c9ed3ba2c8a2ce098163a9bdb26f891746d02136995df25227a20e71c396ebb", size = 22383115, upload-time = "2025-05-08T16:07:08.998Z" }, + { url = "https://files.pythonhosted.org/packages/08/f5/456f56bbbfccf696263b47095291040655e3cbaf05d063bdc7c7517f32ac/scipy-1.15.3-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:0bdd905264c0c9cfa74a4772cdb2070171790381a5c4d312c973382fc6eaf730", size = 25163884, upload-time = "2025-05-08T16:07:14.091Z" }, + { url = "https://files.pythonhosted.org/packages/a2/66/a9618b6a435a0f0c0b8a6d0a2efb32d4ec5a85f023c2b79d39512040355b/scipy-1.15.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79167bba085c31f38603e11a267d862957cbb3ce018d8b38f79ac043bc92d825", size = 35174018, upload-time = "2025-05-08T16:07:19.427Z" }, + { url = "https://files.pythonhosted.org/packages/b5/09/c5b6734a50ad4882432b6bb7c02baf757f5b2f256041da5df242e2d7e6b6/scipy-1.15.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c9deabd6d547aee2c9a81dee6cc96c6d7e9a9b1953f74850c179f91fdc729cb7", size = 37269716, upload-time = "2025-05-08T16:07:25.712Z" }, + { url = "https://files.pythonhosted.org/packages/77/0a/eac00ff741f23bcabd352731ed9b8995a0a60ef57f5fd788d611d43d69a1/scipy-1.15.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:dde4fc32993071ac0c7dd2d82569e544f0bdaff66269cb475e0f369adad13f11", size = 36872342, upload-time = "2025-05-08T16:07:31.468Z" }, + { url = "https://files.pythonhosted.org/packages/fe/54/4379be86dd74b6ad81551689107360d9a3e18f24d20767a2d5b9253a3f0a/scipy-1.15.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f77f853d584e72e874d87357ad70f44b437331507d1c311457bed8ed2b956126", size = 39670869, upload-time = "2025-05-08T16:07:38.002Z" }, + { url = "https://files.pythonhosted.org/packages/87/2e/892ad2862ba54f084ffe8cc4a22667eaf9c2bcec6d2bff1d15713c6c0703/scipy-1.15.3-cp313-cp313-win_amd64.whl", hash = "sha256:b90ab29d0c37ec9bf55424c064312930ca5f4bde15ee8619ee44e69319aab163", size = 40988851, upload-time = "2025-05-08T16:08:33.671Z" }, + { url = "https://files.pythonhosted.org/packages/1b/e9/7a879c137f7e55b30d75d90ce3eb468197646bc7b443ac036ae3fe109055/scipy-1.15.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3ac07623267feb3ae308487c260ac684b32ea35fd81e12845039952f558047b8", size = 38863011, upload-time = "2025-05-08T16:07:44.039Z" }, + { url = "https://files.pythonhosted.org/packages/51/d1/226a806bbd69f62ce5ef5f3ffadc35286e9fbc802f606a07eb83bf2359de/scipy-1.15.3-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:6487aa99c2a3d509a5227d9a5e889ff05830a06b2ce08ec30df6d79db5fcd5c5", size = 30266407, upload-time = "2025-05-08T16:07:49.891Z" }, + { url = "https://files.pythonhosted.org/packages/e5/9b/f32d1d6093ab9eeabbd839b0f7619c62e46cc4b7b6dbf05b6e615bbd4400/scipy-1.15.3-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:50f9e62461c95d933d5c5ef4a1f2ebf9a2b4e83b0db374cb3f1de104d935922e", size = 22540030, upload-time = "2025-05-08T16:07:54.121Z" }, + { url = "https://files.pythonhosted.org/packages/e7/29/c278f699b095c1a884f29fda126340fcc201461ee8bfea5c8bdb1c7c958b/scipy-1.15.3-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:14ed70039d182f411ffc74789a16df3835e05dc469b898233a245cdfd7f162cb", size = 25218709, upload-time = "2025-05-08T16:07:58.506Z" }, + { url = "https://files.pythonhosted.org/packages/24/18/9e5374b617aba742a990581373cd6b68a2945d65cc588482749ef2e64467/scipy-1.15.3-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a769105537aa07a69468a0eefcd121be52006db61cdd8cac8a0e68980bbb723", size = 34809045, upload-time = "2025-05-08T16:08:03.929Z" }, + { url = "https://files.pythonhosted.org/packages/e1/fe/9c4361e7ba2927074360856db6135ef4904d505e9b3afbbcb073c4008328/scipy-1.15.3-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9db984639887e3dffb3928d118145ffe40eff2fa40cb241a306ec57c219ebbbb", size = 36703062, upload-time = "2025-05-08T16:08:09.558Z" }, + { url = "https://files.pythonhosted.org/packages/b7/8e/038ccfe29d272b30086b25a4960f757f97122cb2ec42e62b460d02fe98e9/scipy-1.15.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:40e54d5c7e7ebf1aa596c374c49fa3135f04648a0caabcb66c52884b943f02b4", size = 36393132, upload-time = "2025-05-08T16:08:15.34Z" }, + { url = "https://files.pythonhosted.org/packages/10/7e/5c12285452970be5bdbe8352c619250b97ebf7917d7a9a9e96b8a8140f17/scipy-1.15.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:5e721fed53187e71d0ccf382b6bf977644c533e506c4d33c3fb24de89f5c3ed5", size = 38979503, upload-time = "2025-05-08T16:08:21.513Z" }, + { url = "https://files.pythonhosted.org/packages/81/06/0a5e5349474e1cbc5757975b21bd4fad0e72ebf138c5592f191646154e06/scipy-1.15.3-cp313-cp313t-win_amd64.whl", hash = "sha256:76ad1fb5f8752eabf0fa02e4cc0336b4e8f021e2d5f061ed37d6d264db35e3ca", size = 40308097, upload-time = "2025-05-08T16:08:27.627Z" }, +] + +[[package]] +name = "scipy" +version = "1.16.3" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.13'", + "python_full_version >= '3.11' and python_full_version < '3.13'", +] +dependencies = [ + { name = "numpy", version = "2.3.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0a/ca/d8ace4f98322d01abcd52d381134344bf7b431eba7ed8b42bdea5a3c2ac9/scipy-1.16.3.tar.gz", hash = "sha256:01e87659402762f43bd2fee13370553a17ada367d42e7487800bf2916535aecb", size = 30597883, upload-time = "2025-10-28T17:38:54.068Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9b/5f/6f37d7439de1455ce9c5a556b8d1db0979f03a796c030bafdf08d35b7bf9/scipy-1.16.3-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:40be6cf99e68b6c4321e9f8782e7d5ff8265af28ef2cd56e9c9b2638fa08ad97", size = 36630881, upload-time = "2025-10-28T17:31:47.104Z" }, + { url = "https://files.pythonhosted.org/packages/7c/89/d70e9f628749b7e4db2aa4cd89735502ff3f08f7b9b27d2e799485987cd9/scipy-1.16.3-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:8be1ca9170fcb6223cc7c27f4305d680ded114a1567c0bd2bfcbf947d1b17511", size = 28941012, upload-time = "2025-10-28T17:31:53.411Z" }, + { url = "https://files.pythonhosted.org/packages/a8/a8/0e7a9a6872a923505dbdf6bb93451edcac120363131c19013044a1e7cb0c/scipy-1.16.3-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:bea0a62734d20d67608660f69dcda23e7f90fb4ca20974ab80b6ed40df87a005", size = 20931935, upload-time = "2025-10-28T17:31:57.361Z" }, + { url = "https://files.pythonhosted.org/packages/bd/c7/020fb72bd79ad798e4dbe53938543ecb96b3a9ac3fe274b7189e23e27353/scipy-1.16.3-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:2a207a6ce9c24f1951241f4693ede2d393f59c07abc159b2cb2be980820e01fb", size = 23534466, upload-time = "2025-10-28T17:32:01.875Z" }, + { url = "https://files.pythonhosted.org/packages/be/a0/668c4609ce6dbf2f948e167836ccaf897f95fb63fa231c87da7558a374cd/scipy-1.16.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:532fb5ad6a87e9e9cd9c959b106b73145a03f04c7d57ea3e6f6bb60b86ab0876", size = 33593618, upload-time = "2025-10-28T17:32:06.902Z" }, + { url = "https://files.pythonhosted.org/packages/ca/6e/8942461cf2636cdae083e3eb72622a7fbbfa5cf559c7d13ab250a5dbdc01/scipy-1.16.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0151a0749efeaaab78711c78422d413c583b8cdd2011a3c1d6c794938ee9fdb2", size = 35899798, upload-time = "2025-10-28T17:32:12.665Z" }, + { url = "https://files.pythonhosted.org/packages/79/e8/d0f33590364cdbd67f28ce79368b373889faa4ee959588beddf6daef9abe/scipy-1.16.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b7180967113560cca57418a7bc719e30366b47959dd845a93206fbed693c867e", size = 36226154, upload-time = "2025-10-28T17:32:17.961Z" }, + { url = "https://files.pythonhosted.org/packages/39/c1/1903de608c0c924a1749c590064e65810f8046e437aba6be365abc4f7557/scipy-1.16.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:deb3841c925eeddb6afc1e4e4a45e418d19ec7b87c5df177695224078e8ec733", size = 38878540, upload-time = "2025-10-28T17:32:23.907Z" }, + { url = "https://files.pythonhosted.org/packages/f1/d0/22ec7036ba0b0a35bccb7f25ab407382ed34af0b111475eb301c16f8a2e5/scipy-1.16.3-cp311-cp311-win_amd64.whl", hash = "sha256:53c3844d527213631e886621df5695d35e4f6a75f620dca412bcd292f6b87d78", size = 38722107, upload-time = "2025-10-28T17:32:29.921Z" }, + { url = "https://files.pythonhosted.org/packages/7b/60/8a00e5a524bb3bf8898db1650d350f50e6cffb9d7a491c561dc9826c7515/scipy-1.16.3-cp311-cp311-win_arm64.whl", hash = "sha256:9452781bd879b14b6f055b26643703551320aa8d79ae064a71df55c00286a184", size = 25506272, upload-time = "2025-10-28T17:32:34.577Z" }, + { url = "https://files.pythonhosted.org/packages/40/41/5bf55c3f386b1643812f3a5674edf74b26184378ef0f3e7c7a09a7e2ca7f/scipy-1.16.3-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:81fc5827606858cf71446a5e98715ba0e11f0dbc83d71c7409d05486592a45d6", size = 36659043, upload-time = "2025-10-28T17:32:40.285Z" }, + { url = "https://files.pythonhosted.org/packages/1e/0f/65582071948cfc45d43e9870bf7ca5f0e0684e165d7c9ef4e50d783073eb/scipy-1.16.3-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:c97176013d404c7346bf57874eaac5187d969293bf40497140b0a2b2b7482e07", size = 28898986, upload-time = "2025-10-28T17:32:45.325Z" }, + { url = "https://files.pythonhosted.org/packages/96/5e/36bf3f0ac298187d1ceadde9051177d6a4fe4d507e8f59067dc9dd39e650/scipy-1.16.3-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:2b71d93c8a9936046866acebc915e2af2e292b883ed6e2cbe5c34beb094b82d9", size = 20889814, upload-time = "2025-10-28T17:32:49.277Z" }, + { url = "https://files.pythonhosted.org/packages/80/35/178d9d0c35394d5d5211bbff7ac4f2986c5488b59506fef9e1de13ea28d3/scipy-1.16.3-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:3d4a07a8e785d80289dfe66b7c27d8634a773020742ec7187b85ccc4b0e7b686", size = 23565795, upload-time = "2025-10-28T17:32:53.337Z" }, + { url = "https://files.pythonhosted.org/packages/fa/46/d1146ff536d034d02f83c8afc3c4bab2eddb634624d6529a8512f3afc9da/scipy-1.16.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0553371015692a898e1aa858fed67a3576c34edefa6b7ebdb4e9dde49ce5c203", size = 33349476, upload-time = "2025-10-28T17:32:58.353Z" }, + { url = "https://files.pythonhosted.org/packages/79/2e/415119c9ab3e62249e18c2b082c07aff907a273741b3f8160414b0e9193c/scipy-1.16.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:72d1717fd3b5e6ec747327ce9bda32d5463f472c9dce9f54499e81fbd50245a1", size = 35676692, upload-time = "2025-10-28T17:33:03.88Z" }, + { url = "https://files.pythonhosted.org/packages/27/82/df26e44da78bf8d2aeaf7566082260cfa15955a5a6e96e6a29935b64132f/scipy-1.16.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1fb2472e72e24d1530debe6ae078db70fb1605350c88a3d14bc401d6306dbffe", size = 36019345, upload-time = "2025-10-28T17:33:09.773Z" }, + { url = "https://files.pythonhosted.org/packages/82/31/006cbb4b648ba379a95c87262c2855cd0d09453e500937f78b30f02fa1cd/scipy-1.16.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c5192722cffe15f9329a3948c4b1db789fbb1f05c97899187dcf009b283aea70", size = 38678975, upload-time = "2025-10-28T17:33:15.809Z" }, + { url = "https://files.pythonhosted.org/packages/c2/7f/acbd28c97e990b421af7d6d6cd416358c9c293fc958b8529e0bd5d2a2a19/scipy-1.16.3-cp312-cp312-win_amd64.whl", hash = "sha256:56edc65510d1331dae01ef9b658d428e33ed48b4f77b1d51caf479a0253f96dc", size = 38555926, upload-time = "2025-10-28T17:33:21.388Z" }, + { url = "https://files.pythonhosted.org/packages/ce/69/c5c7807fd007dad4f48e0a5f2153038dc96e8725d3345b9ee31b2b7bed46/scipy-1.16.3-cp312-cp312-win_arm64.whl", hash = "sha256:a8a26c78ef223d3e30920ef759e25625a0ecdd0d60e5a8818b7513c3e5384cf2", size = 25463014, upload-time = "2025-10-28T17:33:25.975Z" }, + { url = "https://files.pythonhosted.org/packages/72/f1/57e8327ab1508272029e27eeef34f2302ffc156b69e7e233e906c2a5c379/scipy-1.16.3-cp313-cp313-macosx_10_14_x86_64.whl", hash = "sha256:d2ec56337675e61b312179a1ad124f5f570c00f920cc75e1000025451b88241c", size = 36617856, upload-time = "2025-10-28T17:33:31.375Z" }, + { url = "https://files.pythonhosted.org/packages/44/13/7e63cfba8a7452eb756306aa2fd9b37a29a323b672b964b4fdeded9a3f21/scipy-1.16.3-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:16b8bc35a4cc24db80a0ec836a9286d0e31b2503cb2fd7ff7fb0e0374a97081d", size = 28874306, upload-time = "2025-10-28T17:33:36.516Z" }, + { url = "https://files.pythonhosted.org/packages/15/65/3a9400efd0228a176e6ec3454b1fa998fbbb5a8defa1672c3f65706987db/scipy-1.16.3-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:5803c5fadd29de0cf27fa08ccbfe7a9e5d741bf63e4ab1085437266f12460ff9", size = 20865371, upload-time = "2025-10-28T17:33:42.094Z" }, + { url = "https://files.pythonhosted.org/packages/33/d7/eda09adf009a9fb81827194d4dd02d2e4bc752cef16737cc4ef065234031/scipy-1.16.3-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:b81c27fc41954319a943d43b20e07c40bdcd3ff7cf013f4fb86286faefe546c4", size = 23524877, upload-time = "2025-10-28T17:33:48.483Z" }, + { url = "https://files.pythonhosted.org/packages/7d/6b/3f911e1ebc364cb81320223a3422aab7d26c9c7973109a9cd0f27c64c6c0/scipy-1.16.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0c3b4dd3d9b08dbce0f3440032c52e9e2ab9f96ade2d3943313dfe51a7056959", size = 33342103, upload-time = "2025-10-28T17:33:56.495Z" }, + { url = "https://files.pythonhosted.org/packages/21/f6/4bfb5695d8941e5c570a04d9fcd0d36bce7511b7d78e6e75c8f9791f82d0/scipy-1.16.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7dc1360c06535ea6116a2220f760ae572db9f661aba2d88074fe30ec2aa1ff88", size = 35697297, upload-time = "2025-10-28T17:34:04.722Z" }, + { url = "https://files.pythonhosted.org/packages/04/e1/6496dadbc80d8d896ff72511ecfe2316b50313bfc3ebf07a3f580f08bd8c/scipy-1.16.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:663b8d66a8748051c3ee9c96465fb417509315b99c71550fda2591d7dd634234", size = 36021756, upload-time = "2025-10-28T17:34:13.482Z" }, + { url = "https://files.pythonhosted.org/packages/fe/bd/a8c7799e0136b987bda3e1b23d155bcb31aec68a4a472554df5f0937eef7/scipy-1.16.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eab43fae33a0c39006a88096cd7b4f4ef545ea0447d250d5ac18202d40b6611d", size = 38696566, upload-time = "2025-10-28T17:34:22.384Z" }, + { url = "https://files.pythonhosted.org/packages/cd/01/1204382461fcbfeb05b6161b594f4007e78b6eba9b375382f79153172b4d/scipy-1.16.3-cp313-cp313-win_amd64.whl", hash = "sha256:062246acacbe9f8210de8e751b16fc37458213f124bef161a5a02c7a39284304", size = 38529877, upload-time = "2025-10-28T17:35:51.076Z" }, + { url = "https://files.pythonhosted.org/packages/7f/14/9d9fbcaa1260a94f4bb5b64ba9213ceb5d03cd88841fe9fd1ffd47a45b73/scipy-1.16.3-cp313-cp313-win_arm64.whl", hash = "sha256:50a3dbf286dbc7d84f176f9a1574c705f277cb6565069f88f60db9eafdbe3ee2", size = 25455366, upload-time = "2025-10-28T17:35:59.014Z" }, + { url = "https://files.pythonhosted.org/packages/e2/a3/9ec205bd49f42d45d77f1730dbad9ccf146244c1647605cf834b3a8c4f36/scipy-1.16.3-cp313-cp313t-macosx_10_14_x86_64.whl", hash = "sha256:fb4b29f4cf8cc5a8d628bc8d8e26d12d7278cd1f219f22698a378c3d67db5e4b", size = 37027931, upload-time = "2025-10-28T17:34:31.451Z" }, + { url = "https://files.pythonhosted.org/packages/25/06/ca9fd1f3a4589cbd825b1447e5db3a8ebb969c1eaf22c8579bd286f51b6d/scipy-1.16.3-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:8d09d72dc92742988b0e7750bddb8060b0c7079606c0d24a8cc8e9c9c11f9079", size = 29400081, upload-time = "2025-10-28T17:34:39.087Z" }, + { url = "https://files.pythonhosted.org/packages/6a/56/933e68210d92657d93fb0e381683bc0e53a965048d7358ff5fbf9e6a1b17/scipy-1.16.3-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:03192a35e661470197556de24e7cb1330d84b35b94ead65c46ad6f16f6b28f2a", size = 21391244, upload-time = "2025-10-28T17:34:45.234Z" }, + { url = "https://files.pythonhosted.org/packages/a8/7e/779845db03dc1418e215726329674b40576879b91814568757ff0014ad65/scipy-1.16.3-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:57d01cb6f85e34f0946b33caa66e892aae072b64b034183f3d87c4025802a119", size = 23929753, upload-time = "2025-10-28T17:34:51.793Z" }, + { url = "https://files.pythonhosted.org/packages/4c/4b/f756cf8161d5365dcdef9e5f460ab226c068211030a175d2fc7f3f41ca64/scipy-1.16.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:96491a6a54e995f00a28a3c3badfff58fd093bf26cd5fb34a2188c8c756a3a2c", size = 33496912, upload-time = "2025-10-28T17:34:59.8Z" }, + { url = "https://files.pythonhosted.org/packages/09/b5/222b1e49a58668f23839ca1542a6322bb095ab8d6590d4f71723869a6c2c/scipy-1.16.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cd13e354df9938598af2be05822c323e97132d5e6306b83a3b4ee6724c6e522e", size = 35802371, upload-time = "2025-10-28T17:35:08.173Z" }, + { url = "https://files.pythonhosted.org/packages/c1/8d/5964ef68bb31829bde27611f8c9deeac13764589fe74a75390242b64ca44/scipy-1.16.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:63d3cdacb8a824a295191a723ee5e4ea7768ca5ca5f2838532d9f2e2b3ce2135", size = 36190477, upload-time = "2025-10-28T17:35:16.7Z" }, + { url = "https://files.pythonhosted.org/packages/ab/f2/b31d75cb9b5fa4dd39a0a931ee9b33e7f6f36f23be5ef560bf72e0f92f32/scipy-1.16.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:e7efa2681ea410b10dde31a52b18b0154d66f2485328830e45fdf183af5aefc6", size = 38796678, upload-time = "2025-10-28T17:35:26.354Z" }, + { url = "https://files.pythonhosted.org/packages/b4/1e/b3723d8ff64ab548c38d87055483714fefe6ee20e0189b62352b5e015bb1/scipy-1.16.3-cp313-cp313t-win_amd64.whl", hash = "sha256:2d1ae2cf0c350e7705168ff2429962a89ad90c2d49d1dd300686d8b2a5af22fc", size = 38640178, upload-time = "2025-10-28T17:35:35.304Z" }, + { url = "https://files.pythonhosted.org/packages/8e/f3/d854ff38789aca9b0cc23008d607ced9de4f7ab14fa1ca4329f86b3758ca/scipy-1.16.3-cp313-cp313t-win_arm64.whl", hash = "sha256:0c623a54f7b79dd88ef56da19bc2873afec9673a48f3b85b18e4d402bdd29a5a", size = 25803246, upload-time = "2025-10-28T17:35:42.155Z" }, + { url = "https://files.pythonhosted.org/packages/99/f6/99b10fd70f2d864c1e29a28bbcaa0c6340f9d8518396542d9ea3b4aaae15/scipy-1.16.3-cp314-cp314-macosx_10_14_x86_64.whl", hash = "sha256:875555ce62743e1d54f06cdf22c1e0bc47b91130ac40fe5d783b6dfa114beeb6", size = 36606469, upload-time = "2025-10-28T17:36:08.741Z" }, + { url = "https://files.pythonhosted.org/packages/4d/74/043b54f2319f48ea940dd025779fa28ee360e6b95acb7cd188fad4391c6b/scipy-1.16.3-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:bb61878c18a470021fb515a843dc7a76961a8daceaaaa8bad1332f1bf4b54657", size = 28872043, upload-time = "2025-10-28T17:36:16.599Z" }, + { url = "https://files.pythonhosted.org/packages/4d/e1/24b7e50cc1c4ee6ffbcb1f27fe9f4c8b40e7911675f6d2d20955f41c6348/scipy-1.16.3-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:f2622206f5559784fa5c4b53a950c3c7c1cf3e84ca1b9c4b6c03f062f289ca26", size = 20862952, upload-time = "2025-10-28T17:36:22.966Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3a/3e8c01a4d742b730df368e063787c6808597ccb38636ed821d10b39ca51b/scipy-1.16.3-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:7f68154688c515cdb541a31ef8eb66d8cd1050605be9dcd74199cbd22ac739bc", size = 23508512, upload-time = "2025-10-28T17:36:29.731Z" }, + { url = "https://files.pythonhosted.org/packages/1f/60/c45a12b98ad591536bfe5330cb3cfe1850d7570259303563b1721564d458/scipy-1.16.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8b3c820ddb80029fe9f43d61b81d8b488d3ef8ca010d15122b152db77dc94c22", size = 33413639, upload-time = "2025-10-28T17:36:37.982Z" }, + { url = "https://files.pythonhosted.org/packages/71/bc/35957d88645476307e4839712642896689df442f3e53b0fa016ecf8a3357/scipy-1.16.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d3837938ae715fc0fe3c39c0202de3a8853aff22ca66781ddc2ade7554b7e2cc", size = 35704729, upload-time = "2025-10-28T17:36:46.547Z" }, + { url = "https://files.pythonhosted.org/packages/3b/15/89105e659041b1ca11c386e9995aefacd513a78493656e57789f9d9eab61/scipy-1.16.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:aadd23f98f9cb069b3bd64ddc900c4d277778242e961751f77a8cb5c4b946fb0", size = 36086251, upload-time = "2025-10-28T17:36:55.161Z" }, + { url = "https://files.pythonhosted.org/packages/1a/87/c0ea673ac9c6cc50b3da2196d860273bc7389aa69b64efa8493bdd25b093/scipy-1.16.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:b7c5f1bda1354d6a19bc6af73a649f8285ca63ac6b52e64e658a5a11d4d69800", size = 38716681, upload-time = "2025-10-28T17:37:04.1Z" }, + { url = "https://files.pythonhosted.org/packages/91/06/837893227b043fb9b0d13e4bd7586982d8136cb249ffb3492930dab905b8/scipy-1.16.3-cp314-cp314-win_amd64.whl", hash = "sha256:e5d42a9472e7579e473879a1990327830493a7047506d58d73fc429b84c1d49d", size = 39358423, upload-time = "2025-10-28T17:38:20.005Z" }, + { url = "https://files.pythonhosted.org/packages/95/03/28bce0355e4d34a7c034727505a02d19548549e190bedd13a721e35380b7/scipy-1.16.3-cp314-cp314-win_arm64.whl", hash = "sha256:6020470b9d00245926f2d5bb93b119ca0340f0d564eb6fbaad843eaebf9d690f", size = 26135027, upload-time = "2025-10-28T17:38:24.966Z" }, + { url = "https://files.pythonhosted.org/packages/b2/6f/69f1e2b682efe9de8fe9f91040f0cd32f13cfccba690512ba4c582b0bc29/scipy-1.16.3-cp314-cp314t-macosx_10_14_x86_64.whl", hash = "sha256:e1d27cbcb4602680a49d787d90664fa4974063ac9d4134813332a8c53dbe667c", size = 37028379, upload-time = "2025-10-28T17:37:14.061Z" }, + { url = "https://files.pythonhosted.org/packages/7c/2d/e826f31624a5ebbab1cd93d30fd74349914753076ed0593e1d56a98c4fb4/scipy-1.16.3-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:9b9c9c07b6d56a35777a1b4cc8966118fb16cfd8daf6743867d17d36cfad2d40", size = 29400052, upload-time = "2025-10-28T17:37:21.709Z" }, + { url = "https://files.pythonhosted.org/packages/69/27/d24feb80155f41fd1f156bf144e7e049b4e2b9dd06261a242905e3bc7a03/scipy-1.16.3-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:3a4c460301fb2cffb7f88528f30b3127742cff583603aa7dc964a52c463b385d", size = 21391183, upload-time = "2025-10-28T17:37:29.559Z" }, + { url = "https://files.pythonhosted.org/packages/f8/d3/1b229e433074c5738a24277eca520a2319aac7465eea7310ea6ae0e98ae2/scipy-1.16.3-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:f667a4542cc8917af1db06366d3f78a5c8e83badd56409f94d1eac8d8d9133fa", size = 23930174, upload-time = "2025-10-28T17:37:36.306Z" }, + { url = "https://files.pythonhosted.org/packages/16/9d/d9e148b0ec680c0f042581a2be79a28a7ab66c0c4946697f9e7553ead337/scipy-1.16.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f379b54b77a597aa7ee5e697df0d66903e41b9c85a6dd7946159e356319158e8", size = 33497852, upload-time = "2025-10-28T17:37:42.228Z" }, + { url = "https://files.pythonhosted.org/packages/2f/22/4e5f7561e4f98b7bea63cf3fd7934bff1e3182e9f1626b089a679914d5c8/scipy-1.16.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4aff59800a3b7f786b70bfd6ab551001cb553244988d7d6b8299cb1ea653b353", size = 35798595, upload-time = "2025-10-28T17:37:48.102Z" }, + { url = "https://files.pythonhosted.org/packages/83/42/6644d714c179429fc7196857866f219fef25238319b650bb32dde7bf7a48/scipy-1.16.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:da7763f55885045036fabcebd80144b757d3db06ab0861415d1c3b7c69042146", size = 36186269, upload-time = "2025-10-28T17:37:53.72Z" }, + { url = "https://files.pythonhosted.org/packages/ac/70/64b4d7ca92f9cf2e6fc6aaa2eecf80bb9b6b985043a9583f32f8177ea122/scipy-1.16.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ffa6eea95283b2b8079b821dc11f50a17d0571c92b43e2b5b12764dc5f9b285d", size = 38802779, upload-time = "2025-10-28T17:37:59.393Z" }, + { url = "https://files.pythonhosted.org/packages/61/82/8d0e39f62764cce5ffd5284131e109f07cf8955aef9ab8ed4e3aa5e30539/scipy-1.16.3-cp314-cp314t-win_amd64.whl", hash = "sha256:d9f48cafc7ce94cf9b15c6bffdc443a81a27bf7075cf2dcd5c8b40f85d10c4e7", size = 39471128, upload-time = "2025-10-28T17:38:05.259Z" }, + { url = "https://files.pythonhosted.org/packages/64/47/a494741db7280eae6dc033510c319e34d42dd41b7ac0c7ead39354d1a2b5/scipy-1.16.3-cp314-cp314t-win_arm64.whl", hash = "sha256:21d9d6b197227a12dcbf9633320a4e34c6b0e51c57268df255a0942983bac562", size = 26464127, upload-time = "2025-10-28T17:38:11.34Z" }, +] + [[package]] name = "sentry-sdk" version = "2.34.1" From f333e64f12c870c8ff6e7caa452b23480b077373 Mon Sep 17 00:00:00 2001 From: Benjamin McCormick Date: Thu, 20 Nov 2025 17:19:14 -0500 Subject: [PATCH 5/6] chore: tuning beam bench --- tests/bench/beam.py | 170 ++++++++++++++++++++++++----------------- tests/bench/harness.py | 66 ++++++++++++++-- 2 files changed, 158 insertions(+), 78 deletions(-) diff --git a/tests/bench/beam.py b/tests/bench/beam.py index ae322c9b..5d693b23 100644 --- a/tests/bench/beam.py +++ b/tests/bench/beam.py @@ -53,7 +53,7 @@ Optional arguments: --pool-size: Number of Honcho instances in the pool (default: 1) --batch-size: Number of conversations to run concurrently in each batch (default: 1) --json-output: Path to write JSON summary results for analytics ---cleanup-workspace: Delete workspace after executing each conversation (default: False) +--cleanup-workspace: Delete workspace after executing each conversation (default: True) --use-get-context: Use get_context + judge LLM instead of dialectic .chat endpoint (default: False) ``` @@ -575,6 +575,86 @@ Extract the ordered list of events or items mentioned in the response. Preserve alignment.append(best_match_idx) return alignment + async def _process_single_question( + self, + session: Any, + user_peer: Any, + ability: str, + q_idx: int, + q_data: dict[str, Any], + semaphore: asyncio.Semaphore, + ) -> QuestionResult: + """Process a single BEAM question.""" + async with semaphore: + question = q_data["question"] + rubric = q_data.get("rubric", []) + answer = ( + q_data.get("answer") + or q_data.get("ideal_response") + or q_data.get("ideal_answer") + ) + + print(f" [{ability}] Q{q_idx + 1}: {question[:100]}...") + + # Execute question using dialectic + if self.use_get_context: + context = await session.get_context( + summary=True, + peer_target="user", + last_user_message=question, + ) + context_messages = context.to_anthropic(assistant="assistant") + context_messages.append({"role": "user", "content": question}) + + response = await self.anthropic_client.messages.create( + model="claude-sonnet-4-5", + max_tokens=2048, + messages=cast(list[MessageParam], context_messages), + ) + + if not response.content: + actual_response = "" + else: + content_block = response.content[0] + actual_response = getattr(content_block, "text", "") + else: + actual_response = await user_peer.chat(question) + actual_response = ( + actual_response if isinstance(actual_response, str) else "" + ) + + # Judge response based on memory ability + if ability == "event_ordering": + judgment = await self.judge_event_ordering( + question, rubric, actual_response + ) + nugget_scores = None + else: + judgment = await self.judge_nugget_based( + question, rubric, actual_response, ability + ) + nugget_scores = judgment.get("nugget_scores") + + score = judgment.get("overall_score", 0.0) + reasoning = judgment.get("overall_reasoning", "") + + question_result: QuestionResult = { + "question": question, + "answer": answer, + "actual_response": actual_response, + "memory_ability": ability, + "rubric": rubric, + "nugget_scores": nugget_scores, + "score": score, + "passed": score >= 0.5, + "reasoning": reasoning, + } + + status = "PASS" if score >= 0.5 else "FAIL" + print(f" [{ability}] Q{q_idx + 1} Score: {score:.2f} [{status}]") + + return question_result + async def execute_conversation( self, context_length: str, conversation_id: str, honcho_url: str ) -> ConversationResult: @@ -739,82 +819,28 @@ Extract the ordered list of events or items mentioned in the response. Preserve print(f"[{workspace_id}] Deriver queue empty. Executing questions...") # Execute questions for each memory ability + question_tasks: list[Any] = [] + semaphore = asyncio.Semaphore(5) + for ability, questions in questions_data.items(): print( - f"\n[{workspace_id}] Testing {ability} ({len(questions)} questions)" + f"\n[{workspace_id}] Queuing {ability} ({len(questions)} questions)" ) for q_idx, q_data in enumerate(questions): - question = q_data["question"] - rubric = q_data.get("rubric", []) - answer = ( - q_data.get("answer") - or q_data.get("ideal_response") - or q_data.get("ideal_answer") + question_tasks.append( + self._process_single_question( + session, + user_peer, + ability, + q_idx, + q_data, + semaphore, + ) ) - print(f" Q{q_idx + 1}: {question[:100]}...") - - # Execute question using dialectic - if self.use_get_context: - context = await session.get_context( - summary=True, - peer_target="user", - last_user_message=question, - ) - context_messages = context.to_anthropic(assistant="assistant") - context_messages.append({"role": "user", "content": question}) - - from typing import cast - - response = await self.anthropic_client.messages.create( - model="claude-sonnet-4-5", - max_tokens=2048, - messages=cast(list[MessageParam], context_messages), - ) - - if not response.content: - actual_response = "" - else: - content_block = response.content[0] - actual_response = getattr(content_block, "text", "") - else: - actual_response = await user_peer.chat(question) - actual_response = ( - actual_response if isinstance(actual_response, str) else "" - ) - - # Judge response based on memory ability - if ability == "event_ordering": - judgment = await self.judge_event_ordering( - question, rubric, actual_response - ) - nugget_scores = None - else: - judgment = await self.judge_nugget_based( - question, rubric, actual_response, ability - ) - nugget_scores = judgment.get("nugget_scores") - - score = judgment.get("overall_score", 0.0) - reasoning = judgment.get("overall_reasoning", "") - - question_result: QuestionResult = { - "question": question, - "answer": answer, - "actual_response": actual_response, - "memory_ability": ability, - "rubric": rubric, - "nugget_scores": nugget_scores, - "score": score, - "passed": score >= 0.5, - "reasoning": reasoning, - } - - result["question_results"].append(question_result) - - status = "PASS" if score >= 0.5 else "FAIL" - print(f" Score: {score:.2f} [{status}]") + results = await asyncio.gather(*question_tasks) + result["question_results"] = list(results) # Calculate ability scores ability_totals: dict[str, list[float]] = {} @@ -1064,7 +1090,7 @@ async def main() -> int: parser.add_argument( "--batch-size", type=int, - default=1, + default=10, help="Number of conversations to run concurrently in each batch (default: 1)", ) @@ -1077,7 +1103,7 @@ async def main() -> int: parser.add_argument( "--cleanup-workspace", action="store_true", - help="Delete workspace after executing each conversation (default: False)", + help="Delete workspace after executing each conversation (default: True)", ) parser.add_argument( diff --git a/tests/bench/harness.py b/tests/bench/harness.py index cedb68c1..fc2da6cd 100755 --- a/tests/bench/harness.py +++ b/tests/bench/harness.py @@ -68,19 +68,55 @@ class HonchoHarness: # Update the database port compose_data["services"]["database"]["ports"] = [f"{self.db_port}:5432"] + # Increase shared memory size + compose_data["services"]["database"]["shm_size"] = "4gb" + + # Configure Postgres for larger workloads + cmd = ["postgres"] + params = { + "max_connections": "800", + "max_wal_size": "8GB", + "min_wal_size": "2GB", + "checkpoint_timeout": "15min", + "maintenance_work_mem": "1GB", + "work_mem": "64MB", + } + for k, v in params.items(): + cmd.extend(["-c", f"{k}={v}"]) + compose_data["services"]["database"]["command"] = cmd + # Update the Redis port + # TODO: Make this configurable if running multiple instances compose_data["services"]["redis"]["ports"] = ["6379:6379"] # Add a unique project name to avoid conflicts compose_data["name"] = f"honcho_harness_{self.db_port}" - # remove init.sql mount since we use provision_db.py for setup + # Configure named volumes for better performance and cleanup + compose_data.setdefault("volumes", {}) + db_vol_name = f"pgdata_{self.db_port}" + redis_vol_name = f"redis_data_{self.db_port}" + compose_data["volumes"][db_vol_name] = {} + compose_data["volumes"][redis_vol_name] = {} + + # Update database volumes - use named volume instead of bind mount + new_db_volumes: list[str] = [] if "volumes" in compose_data["services"]["database"]: - compose_data["services"]["database"]["volumes"] = [ - vol - for vol in compose_data["services"]["database"]["volumes"] - if "init.sql" not in vol - ] + for vol in compose_data["services"]["database"]["volumes"]: + # Filter out init.sql (handled by provision script) and existing data mounts + if "init.sql" not in vol and ":/var/lib/postgresql/data/" not in vol: + new_db_volumes.append(vol) + new_db_volumes.append(f"{db_vol_name}:/var/lib/postgresql/data/") + compose_data["services"]["database"]["volumes"] = new_db_volumes + + # Update redis volumes - use named volume + new_redis_volumes: list[str] = [] + if "volumes" in compose_data["services"]["redis"]: + for vol in compose_data["services"]["redis"]["volumes"]: + if ":/data" not in vol: + new_redis_volumes.append(vol) + new_redis_volumes.append(f"{redis_vol_name}:/data") + compose_data["services"]["redis"]["volumes"] = new_redis_volumes # Create temporary file self.temp_dir = Path(tempfile.mkdtemp(prefix="honcho_harness_")) @@ -134,6 +170,24 @@ class HonchoHarness: """ print(f"Starting PostgreSQL database on port {self.db_port}...") + # Ensure clean state by removing any existing containers/volumes + # This handles cases where a previous run was interrupted + subprocess.run( + [ + "docker", + "compose", + "-f", + str(self.docker_compose_file), + "-p", + f"honcho_harness_{self.db_port}", + "down", + "--volumes", + "--remove-orphans", + ], + cwd=self.temp_dir, + capture_output=True, + ) + # Change to the temp directory and start the database service result = subprocess.run( [ From 298dbbdb8547d8c3f7a8278c804261bbd071c1a6 Mon Sep 17 00:00:00 2001 From: Benjamin McCormick Date: Thu, 20 Nov 2025 17:56:27 -0500 Subject: [PATCH 6/6] chore: coderabbit review --- tests/bench/beam.py | 10 +++++----- tests/bench/harness.py | 21 +++++++++++++++++++-- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/tests/bench/beam.py b/tests/bench/beam.py index 5d693b23..acc726fe 100644 --- a/tests/bench/beam.py +++ b/tests/bench/beam.py @@ -48,7 +48,7 @@ Optional arguments: --context-length: Context length subset to test (100K, 500K, 1M, 10M) (default: 100K) --conversation-ids: Comma-separated list of conversation IDs to test (default: all in context length) --anthropic-api-key: Anthropic API key for response judging (can be set in .env as LLM_ANTHROPIC_API_KEY) ---timeout: Timeout for deriver queue to empty in seconds (default: 10 minutes) + --timeout: Timeout for deriver queue to empty in seconds (default: 10 minutes (600s)) --base-api-port: Base port for Honcho API instances (default: 8000) --pool-size: Number of Honcho instances in the pool (default: 1) --batch-size: Number of conversations to run concurrently in each batch (default: 1) @@ -154,7 +154,7 @@ class BEAMRunner: self.pool_size: int = pool_size self.anthropic_api_key: str | None = anthropic_api_key self.timeout_seconds: int = ( - timeout_seconds if timeout_seconds is not None else 10000 + timeout_seconds if timeout_seconds is not None else 600 ) self.cleanup_workspace: bool = cleanup_workspace self.use_get_context: bool = use_get_context @@ -507,7 +507,7 @@ Extract the ordered list of events or items mentioned in the response. Preserve # Now compute alignment and Kendall tau-b # Match extracted events to rubric events using LLM equivalence - alignment = await self._align_events(rubric, extracted_events) + alignment = self._align_events(rubric, extracted_events) # Compute Kendall tau-b tau: float @@ -551,7 +551,7 @@ Extract the ordered list of events or items mentioned in the response. Preserve "overall_reasoning": f"Evaluation failed due to error: {e}", } - async def _align_events( + def _align_events( self, expected_events: list[str], extracted_events: list[str] ) -> list[int]: """ @@ -1084,7 +1084,7 @@ async def main() -> int: "--timeout", type=int, default=None, - help="Timeout for deriver queue to empty in seconds (default: 10 minutes)", + help="Timeout for deriver queue to empty in seconds (default: 10 minutes (600s))", ) parser.add_argument( diff --git a/tests/bench/harness.py b/tests/bench/harness.py index fc2da6cd..1a568023 100755 --- a/tests/bench/harness.py +++ b/tests/bench/harness.py @@ -218,6 +218,23 @@ class HonchoHarness: """ print("Starting Redis cache server on port 6379...") + # Ensure clean state by removing any existing containers/volumes + subprocess.run( + [ + "docker", + "compose", + "-f", + str(self.docker_compose_file), + "-p", + f"honcho_harness_{self.db_port}", + "down", + "--volumes", + "--remove-orphans", + ], + cwd=self.temp_dir, + capture_output=True, + ) + # Change to the temp directory and start the redis service result = subprocess.run( [ @@ -699,7 +716,8 @@ except Exception as e: # More aggressive cleanup - remove containers, volumes, and orphaned containers subprocess.run( [ - "docker-compose", + "docker", + "compose", "-f", str(self.docker_compose_file), "-p", @@ -710,7 +728,6 @@ except Exception as e: ], cwd=self.temp_dir, capture_output=True, - text=True, ) # Also try to remove any containers that might still be running