fix(deriver): delimit examples and forbid example leakage
Wrap the minimal deriver EXAMPLES block in <examples> tags and add an explicit negative instruction: the examples are fabricated format illustrations only, and every conclusion must be grounded in the <messages> block. Soft mitigation for example-content leakage; real fixes (sources_indices + SFT / structural enforcement) are follow-ups.
This commit is contained in:
parent
444897975c
commit
0c104161dd
|
|
@ -71,10 +71,14 @@ RULES:
|
|||
- Extract ALL observations from the target peer's messages, using others as context.
|
||||
- Contextualize each observation sufficiently (e.g. "Ann is nervous about the job interview at the pharmacy" not just "Ann is nervous")
|
||||
|
||||
<examples>
|
||||
These examples are fabricated illustrations of the output format. Never emit a conclusion for which content comes from these examples. Every conclusion must be supported by the <messages> block only.
|
||||
|
||||
EXAMPLES (using `alice` as the target peer id):
|
||||
- EXPLICIT: "I just turned 25" → "alice is 25 years old"
|
||||
- EXPLICIT: "I took my dog for a walk in NYC" → "alice has a dog", "alice walked her dog in NYC"
|
||||
- EXPLICIT: "I've lived in NYC for six years" → "alice lives in NYC", "alice has lived in NYC for six years"
|
||||
</examples>
|
||||
|
||||
{custom_instructions_section}
|
||||
|
||||
|
|
|
|||
|
|
@ -30,6 +30,29 @@ def test_minimal_deriver_prompt_omits_custom_instructions_when_absent() -> None:
|
|||
assert "CUSTOM INSTRUCTIONS:" not in prompt
|
||||
|
||||
|
||||
def test_minimal_deriver_prompt_delimits_examples_and_forbids_example_leakage() -> None:
|
||||
prompt = minimal_deriver_prompt(
|
||||
peer_id="alice",
|
||||
messages="alice: hello",
|
||||
custom_instructions=None,
|
||||
)
|
||||
|
||||
assert "<examples>" in prompt
|
||||
assert "</examples>" in prompt
|
||||
assert "<messages>" in prompt
|
||||
assert "</messages>" in prompt
|
||||
assert "These examples are fabricated illustrations of the output format." in prompt
|
||||
assert (
|
||||
"Never emit a conclusion for which content comes from these examples." in prompt
|
||||
)
|
||||
assert "Every conclusion must be supported by the <messages> block only." in prompt
|
||||
examples_start = prompt.index("<examples>")
|
||||
examples_end = prompt.index("</examples>")
|
||||
examples_block = prompt[examples_start:examples_end]
|
||||
assert "alice is 25 years old" in examples_block
|
||||
assert "alice has a dog" in examples_block
|
||||
|
||||
|
||||
def test_estimate_deriver_prompt_tokens_increases_with_custom_instructions() -> None:
|
||||
base_tokens = estimate_minimal_deriver_prompt_tokens()
|
||||
custom_tokens = estimate_deriver_prompt_tokens(
|
||||
|
|
|
|||
Loading…
Reference in New Issue