mirror of https://github.com/razor-ai/soup.git
67 lines
2.2 KiB
Python
67 lines
2.2 KiB
Python
"""Shared text-extraction helpers for v0.55.0 eval-design + canary-discovery.
|
|
|
|
Extracted out of `eval_design.py` so `canary_discovery.py` no longer
|
|
imports private symbols across modules (code-review LOW fix — removes
|
|
hidden coupling).
|
|
|
|
Pure functions — no I/O, no torch.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from collections.abc import Mapping, Sequence
|
|
from typing import List
|
|
|
|
# Small stop-words list used by the TF-IDF salience clustering; just
|
|
# enough to filter the dominant function-word tokens.
|
|
STOPWORDS = frozenset(
|
|
{
|
|
"a", "an", "and", "are", "as", "at", "be", "but", "by", "do", "for",
|
|
"from", "have", "i", "in", "is", "it", "of", "on", "or", "that", "the",
|
|
"this", "to", "was", "were", "will", "with", "you",
|
|
}
|
|
)
|
|
|
|
|
|
def row_text(row: Mapping[str, object]) -> str:
|
|
"""Best-effort text extraction from the *output* side of a dataset row.
|
|
|
|
Soup datasets normalise to ``{"messages": [...]}``; this helper
|
|
pulls the assistant turn(s) when present, falling back to common
|
|
SFT fields. Returns an empty string on missing data.
|
|
"""
|
|
if not isinstance(row, Mapping):
|
|
return ""
|
|
messages = row.get("messages")
|
|
if isinstance(messages, Sequence) and not isinstance(messages, (str, bytes)):
|
|
chunks: List[str] = []
|
|
for msg in messages:
|
|
if not isinstance(msg, Mapping):
|
|
continue
|
|
if msg.get("role") == "assistant":
|
|
content = msg.get("content")
|
|
if isinstance(content, str):
|
|
chunks.append(content)
|
|
if chunks:
|
|
return "\n".join(chunks)
|
|
for key in ("output", "completion", "chosen", "response", "answer", "text"):
|
|
val = row.get(key)
|
|
if isinstance(val, str) and val:
|
|
return val
|
|
return ""
|
|
|
|
|
|
def tokenize(text: str) -> List[str]:
|
|
"""Tokenise to lowercase alphanumeric runs, filtering stop-words.
|
|
|
|
Returns an empty list for empty / non-string input.
|
|
"""
|
|
if not text:
|
|
return []
|
|
return [
|
|
token
|
|
for token in re.findall(r"[A-Za-z][A-Za-z0-9_-]{1,30}", text.lower())
|
|
if token not in STOPWORDS and len(token) > 2
|
|
]
|