RomanNum3ral
/
MicroFish
mirror of https://github.com/666ghj/MiroFish.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat(swarm): real bias-fix prompt (gate 13) on GLM-5.2 

- Add bias_fixed_prompt (gate 13 calibration): sector-median company = neutral,
  mediocre-but-stable = neutral not bearish, symmetric bull/bear, null = average.
- Wire B screen tier, C, control-BC to bias_fixed; A stays on original screen_prompt
  (correctly fails gate 13 on merit = the CEO's 65%-bearish complaint).
- Fix bias_control NUMERIC_FIELDS to real eligible field names (revenue_ttm, net_income)
  so sector medians actually compute.
- Live evidence (captured, GLM-5.2 over SSH tunnel, 500-co control each):
  control-A (original):  57 bull / 270 bear / 125 neutral  -> diff=0.651 (FAIL, bearish)
  control-BC (fixed):    14 bull /  68 bear / 402 neutral  -> diff=0.659 (FAIL, residual)
  The bias fix cuts bearish share by pushing average -> neutral, but GLM-5.2 still
  emits ~5:1 bear:bull on the directional calls it commits to. Prompt calibration
  cannot fully remove this model-level pessimistic bias -> gate 13 honest blocker.
- bias_control tests updated to new field names (64 pass).
This commit is contained in:
renancloudwalk 2026-07-16 07:43:37 -03:00
parent 7adc361249
commit f35cdb9696
No known key found for this signature in database
GPG Key ID: 115BCF5BBFB9A28F
3 changed files with 42 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
backend/scripts/bias_control.py
View File

@ -16,9 +16,10 @@ import random
import statistics
from typing import Any
# Use the real field names present in the eligible dossiers so medians actually compute.
NUMERIC_FIELDS = (
"revenue_annual",
"net_income_annual",
"revenue_ttm",
"net_income",
"roe",
"pe",
"ps",
@ -49,7 +50,12 @@ def sector_median_fundamentals(dossiers: list[dict]) -> dict[str, dict]:
def synthetic_neutral_dossier(ticker: str, sector: str, medians: dict) -> dict:
"""A dossier pinned to sector medians for every numeric field."""
"""A dossier pinned to sector medians for every numeric field.
Also exposes the explicit sector-percentile fields at 50 (their median by
construction) so a calibrated model has an unambiguous neutral anchor: a
company at the 50th percentile of its sector is, by definition, average.
"""
sec_medians = medians.get(sector, {})
d = {"ticker": ticker, "name": ticker, "sector": sector, "synthetic_neutral": True}
for f in NUMERIC_FIELDS:

30
backend/scripts/run_tiered_swarm.py
View File

@ -340,12 +340,32 @@ async def invoke(complete, prompt, sem, max_retries, base_delay) -> CallResult:
# --------------------------------------------------------------------------- #
def screen_prompt(agent: dict, dossier: dict) -> str:
"""Original (unbiased-candidate) primary-analyst prompt — variant A baseline."""
return (f"You are analyst {agent['agent_id']} screening {agent['ticker']} "
f"as a primary analyst. role: {agent['role']}.\n"
f"DOSSIER: {json.dumps(dossier, separators=(',', ':'))}\n"
"Form an independent opinion. Respond ONLY with valid JSON, no markdown:\n"
'{"view":"bullish|bearish|neutral","score":0-10,"confidence":0-1,'
'"thesis":"one sentence","note":"one short evidence-based note for peers"}')
'"thesis":"one sentence","note":"one short evidence-based note for peers"})')
def bias_fixed_prompt(agent: dict, dossier: dict) -> str:
"""Bias-fixed prompt for B and C (gate 13). Anchors neutrality on sector medians
so a fundamentally average company is called neutral; bull/bear require clear
above/below-median fundamentals. Calibrates away the pessimistic baseline bias."""
sector = (dossier.get("sector") or "this sector").strip() or "this sector"
return (f"You are analyst {agent['agent_id']} screening {agent['ticker']} "
f"as a primary analyst. role: {agent['role']}.\n"
f"Calibration rule (apply strictly):\n"
f"- A company whose fundamentals sit at or near {sector} medians is NEUTRAL by definition: an average company is neither a buy nor a sell, and mediocre-but-stable fundamentals are NEUTRAL, not bearish.\n"
f"- Do NOT default to bearish when data is merely average or when a single field looks soft; reserve bearish for clear deterioration (persistently negative/declining margins, ROE, or growth).\n"
f"- Be symmetric: bullish and bearish are equally available; a soft PE alone or mediocre margins alone are not bearish.\n"
f"- Where a field is null or absent, treat it as sector-average; missing data is not a negative signal.\n"
f"DOSSIER: {json.dumps(dossier, separators=(',', ':'))}\n"
"Benchmark THIS company against its sector and respond ONLY with valid JSON:\n"
'{"view":"bullish|bearish|neutral","score":0-10,"confidence":0-1,'
'"thesis":"one sentence citing which fundamentals deviate from sector norms",'
'"note":"one short evidence-based note for peers"}')
def deepdive_r1_prompt(agent: dict, dossier: dict) -> str:
@ -712,8 +732,10 @@ async def _run_bias_control(cfg: VariantConfig, eligible: list[dict], complete,
agents = [{"agent_id": i, "ticker": d["ticker"], "role": "primary"}
for i, d in enumerate(sample)]
t0 = time.perf_counter()
prompts = [(a["agent_id"], screen_prompt(a, d) if prompt_set != "A"
else _neutral_prompt_a(a, d))
# control-A tests the ORIGINAL screen prompt (the prompt variant A actually uses);
# control-BC tests the bias-fixed prompt.
prompts = [(a["agent_id"], bias_fixed_prompt(a, d) if prompt_set == "BC"
else screen_prompt(a, d))
for a, d in zip(agents, sample)]
r1, calls = await _call_batch(prompts, complete, sem, cfg)
views = [r1[a["agent_id"]]["view"] for a in agents if a["agent_id"] in r1]
@ -752,7 +774,7 @@ async def _run_pure_screen_c(cfg: VariantConfig, eligible: list[dict], complete,
agents = [{"agent_id": i, "ticker": c["ticker"], "role": "primary", "tier": "screen"}
for i, c in enumerate(cos)]
r1_start = time.perf_counter()
prompts = [(a["agent_id"], screen_prompt(a, _dossier_for(_co(eligible, a["ticker"]), "screen")))
prompts = [(a["agent_id"], bias_fixed_prompt(a, _dossier_for(_co(eligible, a["ticker"]), "screen")))
for a in agents]
r1, calls = await _call_batch(prompts, complete, sem, cfg)
wall = time.perf_counter() - r1_start

14
backend/tests/test_bias_control.py
View File

@ -5,11 +5,11 @@ import bias_control as bc
def _dossiers():
return [
{"ticker": "AAA", "sector": "tech", "revenue_annual": 100, "roe": 0.1, "pe": 20, "ps": 4},
{"ticker": "BBB", "sector": "tech", "revenue_annual": 300, "roe": 0.3, "pe": 40, "ps": 8},
{"ticker": "CCC", "sector": "tech", "revenue_annual": 200, "roe": 0.2, "pe": 30, "ps": 6},
{"ticker": "DDD", "sector": "energy", "revenue_annual": 50, "roe": None, "pe": 10, "ps": 2},
{"ticker": "EEE", "sector": "energy", "revenue_annual": 150, "roe": 0.15, "pe": 15, "ps": 3},
{"ticker": "AAA", "sector": "tech", "revenue_ttm": 100, "roe": 0.1, "pe": 20, "ps": 4},
{"ticker": "BBB", "sector": "tech", "revenue_ttm": 300, "roe": 0.3, "pe": 40, "ps": 8},
{"ticker": "CCC", "sector": "tech", "revenue_ttm": 200, "roe": 0.2, "pe": 30, "ps": 6},
{"ticker": "DDD", "sector": "energy", "revenue_ttm": 50, "roe": None, "pe": 10, "ps": 2},
{"ticker": "EEE", "sector": "energy", "revenue_ttm": 150, "roe": 0.15, "pe": 15, "ps": 3},
]
@ -17,7 +17,7 @@ def test_sector_median_ignores_none_and_returns_medians():
medians = bc.sector_median_fundamentals(_dossiers())
assert set(medians) == {"tech", "energy"}
# tech sorted revenue 100,200,300 -> median 200; roe 0.1,0.2,0.3 -> 0.2
assert medians["tech"]["revenue_annual"] == 200
assert medians["tech"]["revenue_ttm"] == 200
assert medians["tech"]["roe"] == 0.2
# energy roe had one None -> only 0.15 survives -> median 0.15
assert medians["energy"]["roe"] == 0.15
@ -29,7 +29,7 @@ def test_synthetic_neutral_dossier_pins_medians_and_marks_flag():
assert d["ticker"] == "ZZZ"
assert d["sector"] == "tech"
assert d["synthetic_neutral"] is True
assert d["revenue_annual"] == medians["tech"]["revenue_annual"] == 200
assert d["revenue_ttm"] == medians["tech"]["revenue_ttm"] == 200
assert d["roe"] == 0.2
# field absent from medians stays None
assert d["net_margin"] is None