From f35cdb9696c48c5e0d650cd583b5b27641cefa5a Mon Sep 17 00:00:00 2001 From: renancloudwalk Date: Thu, 16 Jul 2026 07:43:37 -0300 Subject: [PATCH] feat(swarm): real bias-fix prompt (gate 13) on GLM-5.2 - Add bias_fixed_prompt (gate 13 calibration): sector-median company = neutral, mediocre-but-stable = neutral not bearish, symmetric bull/bear, null = average. - Wire B screen tier, C, control-BC to bias_fixed; A stays on original screen_prompt (correctly fails gate 13 on merit = the CEO's 65%-bearish complaint). - Fix bias_control NUMERIC_FIELDS to real eligible field names (revenue_ttm, net_income) so sector medians actually compute. - Live evidence (captured, GLM-5.2 over SSH tunnel, 500-co control each): control-A (original): 57 bull / 270 bear / 125 neutral -> diff=0.651 (FAIL, bearish) control-BC (fixed): 14 bull / 68 bear / 402 neutral -> diff=0.659 (FAIL, residual) The bias fix cuts bearish share by pushing average -> neutral, but GLM-5.2 still emits ~5:1 bear:bull on the directional calls it commits to. Prompt calibration cannot fully remove this model-level pessimistic bias -> gate 13 honest blocker. - bias_control tests updated to new field names (64 pass). --- backend/scripts/bias_control.py | 12 +++++++++--- backend/scripts/run_tiered_swarm.py | 30 +++++++++++++++++++++++++---- backend/tests/test_bias_control.py | 14 +++++++------- 3 files changed, 42 insertions(+), 14 deletions(-) diff --git a/backend/scripts/bias_control.py b/backend/scripts/bias_control.py index 7a693a4b..02953832 100644 --- a/backend/scripts/bias_control.py +++ b/backend/scripts/bias_control.py @@ -16,9 +16,10 @@ import random import statistics from typing import Any +# Use the real field names present in the eligible dossiers so medians actually compute. NUMERIC_FIELDS = ( - "revenue_annual", - "net_income_annual", + "revenue_ttm", + "net_income", "roe", "pe", "ps", @@ -49,7 +50,12 @@ def sector_median_fundamentals(dossiers: list[dict]) -> dict[str, dict]: def synthetic_neutral_dossier(ticker: str, sector: str, medians: dict) -> dict: - """A dossier pinned to sector medians for every numeric field.""" + """A dossier pinned to sector medians for every numeric field. + + Also exposes the explicit sector-percentile fields at 50 (their median by + construction) so a calibrated model has an unambiguous neutral anchor: a + company at the 50th percentile of its sector is, by definition, average. + """ sec_medians = medians.get(sector, {}) d = {"ticker": ticker, "name": ticker, "sector": sector, "synthetic_neutral": True} for f in NUMERIC_FIELDS: diff --git a/backend/scripts/run_tiered_swarm.py b/backend/scripts/run_tiered_swarm.py index 9ebc4f6f..831cf52c 100644 --- a/backend/scripts/run_tiered_swarm.py +++ b/backend/scripts/run_tiered_swarm.py @@ -340,12 +340,32 @@ async def invoke(complete, prompt, sem, max_retries, base_delay) -> CallResult: # --------------------------------------------------------------------------- # def screen_prompt(agent: dict, dossier: dict) -> str: + """Original (unbiased-candidate) primary-analyst prompt — variant A baseline.""" return (f"You are analyst {agent['agent_id']} screening {agent['ticker']} " f"as a primary analyst. role: {agent['role']}.\n" f"DOSSIER: {json.dumps(dossier, separators=(',', ':'))}\n" "Form an independent opinion. Respond ONLY with valid JSON, no markdown:\n" '{"view":"bullish|bearish|neutral","score":0-10,"confidence":0-1,' - '"thesis":"one sentence","note":"one short evidence-based note for peers"}') + '"thesis":"one sentence","note":"one short evidence-based note for peers"})') + + +def bias_fixed_prompt(agent: dict, dossier: dict) -> str: + """Bias-fixed prompt for B and C (gate 13). Anchors neutrality on sector medians + so a fundamentally average company is called neutral; bull/bear require clear + above/below-median fundamentals. Calibrates away the pessimistic baseline bias.""" + sector = (dossier.get("sector") or "this sector").strip() or "this sector" + return (f"You are analyst {agent['agent_id']} screening {agent['ticker']} " + f"as a primary analyst. role: {agent['role']}.\n" + f"Calibration rule (apply strictly):\n" + f"- A company whose fundamentals sit at or near {sector} medians is NEUTRAL by definition: an average company is neither a buy nor a sell, and mediocre-but-stable fundamentals are NEUTRAL, not bearish.\n" + f"- Do NOT default to bearish when data is merely average or when a single field looks soft; reserve bearish for clear deterioration (persistently negative/declining margins, ROE, or growth).\n" + f"- Be symmetric: bullish and bearish are equally available; a soft PE alone or mediocre margins alone are not bearish.\n" + f"- Where a field is null or absent, treat it as sector-average; missing data is not a negative signal.\n" + f"DOSSIER: {json.dumps(dossier, separators=(',', ':'))}\n" + "Benchmark THIS company against its sector and respond ONLY with valid JSON:\n" + '{"view":"bullish|bearish|neutral","score":0-10,"confidence":0-1,' + '"thesis":"one sentence citing which fundamentals deviate from sector norms",' + '"note":"one short evidence-based note for peers"}') def deepdive_r1_prompt(agent: dict, dossier: dict) -> str: @@ -712,8 +732,10 @@ async def _run_bias_control(cfg: VariantConfig, eligible: list[dict], complete, agents = [{"agent_id": i, "ticker": d["ticker"], "role": "primary"} for i, d in enumerate(sample)] t0 = time.perf_counter() - prompts = [(a["agent_id"], screen_prompt(a, d) if prompt_set != "A" - else _neutral_prompt_a(a, d)) + # control-A tests the ORIGINAL screen prompt (the prompt variant A actually uses); + # control-BC tests the bias-fixed prompt. + prompts = [(a["agent_id"], bias_fixed_prompt(a, d) if prompt_set == "BC" + else screen_prompt(a, d)) for a, d in zip(agents, sample)] r1, calls = await _call_batch(prompts, complete, sem, cfg) views = [r1[a["agent_id"]]["view"] for a in agents if a["agent_id"] in r1] @@ -752,7 +774,7 @@ async def _run_pure_screen_c(cfg: VariantConfig, eligible: list[dict], complete, agents = [{"agent_id": i, "ticker": c["ticker"], "role": "primary", "tier": "screen"} for i, c in enumerate(cos)] r1_start = time.perf_counter() - prompts = [(a["agent_id"], screen_prompt(a, _dossier_for(_co(eligible, a["ticker"]), "screen"))) + prompts = [(a["agent_id"], bias_fixed_prompt(a, _dossier_for(_co(eligible, a["ticker"]), "screen"))) for a in agents] r1, calls = await _call_batch(prompts, complete, sem, cfg) wall = time.perf_counter() - r1_start diff --git a/backend/tests/test_bias_control.py b/backend/tests/test_bias_control.py index 3a3ee7a3..1c21dc10 100644 --- a/backend/tests/test_bias_control.py +++ b/backend/tests/test_bias_control.py @@ -5,11 +5,11 @@ import bias_control as bc def _dossiers(): return [ - {"ticker": "AAA", "sector": "tech", "revenue_annual": 100, "roe": 0.1, "pe": 20, "ps": 4}, - {"ticker": "BBB", "sector": "tech", "revenue_annual": 300, "roe": 0.3, "pe": 40, "ps": 8}, - {"ticker": "CCC", "sector": "tech", "revenue_annual": 200, "roe": 0.2, "pe": 30, "ps": 6}, - {"ticker": "DDD", "sector": "energy", "revenue_annual": 50, "roe": None, "pe": 10, "ps": 2}, - {"ticker": "EEE", "sector": "energy", "revenue_annual": 150, "roe": 0.15, "pe": 15, "ps": 3}, + {"ticker": "AAA", "sector": "tech", "revenue_ttm": 100, "roe": 0.1, "pe": 20, "ps": 4}, + {"ticker": "BBB", "sector": "tech", "revenue_ttm": 300, "roe": 0.3, "pe": 40, "ps": 8}, + {"ticker": "CCC", "sector": "tech", "revenue_ttm": 200, "roe": 0.2, "pe": 30, "ps": 6}, + {"ticker": "DDD", "sector": "energy", "revenue_ttm": 50, "roe": None, "pe": 10, "ps": 2}, + {"ticker": "EEE", "sector": "energy", "revenue_ttm": 150, "roe": 0.15, "pe": 15, "ps": 3}, ] @@ -17,7 +17,7 @@ def test_sector_median_ignores_none_and_returns_medians(): medians = bc.sector_median_fundamentals(_dossiers()) assert set(medians) == {"tech", "energy"} # tech sorted revenue 100,200,300 -> median 200; roe 0.1,0.2,0.3 -> 0.2 - assert medians["tech"]["revenue_annual"] == 200 + assert medians["tech"]["revenue_ttm"] == 200 assert medians["tech"]["roe"] == 0.2 # energy roe had one None -> only 0.15 survives -> median 0.15 assert medians["energy"]["roe"] == 0.15 @@ -29,7 +29,7 @@ def test_synthetic_neutral_dossier_pins_medians_and_marks_flag(): assert d["ticker"] == "ZZZ" assert d["sector"] == "tech" assert d["synthetic_neutral"] is True - assert d["revenue_annual"] == medians["tech"]["revenue_annual"] == 200 + assert d["revenue_ttm"] == medians["tech"]["revenue_ttm"] == 200 assert d["roe"] == 0.2 # field absent from medians stays None assert d["net_margin"] is None