fix: normalize profile fields at construction
This commit is contained in:
666ghj
parent
41fc932b62
commit
9f80ad66fd
|
|
@ -33,15 +33,20 @@ def _coerce_to_str(value: Any) -> str:
|
|||
Handles dict, list, and other non-string types that may be returned
|
||||
by LLM JSON parsing.
|
||||
"""
|
||||
if value is None:
|
||||
return ""
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
if isinstance(value, dict):
|
||||
for key in ('text', 'value', 'description', 'content', 'summary', 'name'):
|
||||
if key in value and isinstance(value[key], str):
|
||||
return value[key]
|
||||
if key in value:
|
||||
candidate = _coerce_to_str(value[key])
|
||||
if candidate:
|
||||
return candidate
|
||||
return json.dumps(value, ensure_ascii=False)
|
||||
if isinstance(value, list):
|
||||
if isinstance(value, (list, tuple)):
|
||||
str_items = [_coerce_to_str(item) for item in value]
|
||||
str_items = [item for item in str_items if item]
|
||||
return ', '.join(str_items)
|
||||
return str(value)
|
||||
|
||||
|
|
@ -51,21 +56,20 @@ def _coerce_to_str_list(value: Any) -> List[str]:
|
|||
|
||||
Handles nested structures that may be returned by LLM JSON parsing.
|
||||
"""
|
||||
if isinstance(value, list):
|
||||
result = []
|
||||
if value is None:
|
||||
return []
|
||||
if isinstance(value, (list, tuple)):
|
||||
result: List[str] = []
|
||||
for item in value:
|
||||
if isinstance(item, str):
|
||||
result.append(item)
|
||||
elif isinstance(item, dict):
|
||||
result.append(_coerce_to_str(item))
|
||||
if isinstance(item, (list, tuple)):
|
||||
result.extend(_coerce_to_str_list(item))
|
||||
else:
|
||||
result.append(str(item))
|
||||
text = _coerce_to_str(item)
|
||||
if text:
|
||||
result.append(text)
|
||||
return result
|
||||
if isinstance(value, str):
|
||||
return [value]
|
||||
if isinstance(value, dict):
|
||||
return [_coerce_to_str(value)]
|
||||
return []
|
||||
text = _coerce_to_str(value)
|
||||
return [text] if text else []
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
@ -101,13 +105,15 @@ class OasisAgentProfile:
|
|||
created_at: str = field(default_factory=lambda: datetime.now().strftime("%Y-%m-%d"))
|
||||
|
||||
def __post_init__(self):
|
||||
"""Normalize field types to guard against structured LLM outputs
|
||||
(e.g. dict/list instead of plain strings)."""
|
||||
self.bio = _coerce_to_str(self.bio)
|
||||
self.persona = _coerce_to_str(self.persona)
|
||||
self.country = _coerce_to_str(self.country) if self.country is not None else None
|
||||
self.profession = _coerce_to_str(self.profession) if self.profession is not None else None
|
||||
self.gender = _coerce_to_str(self.gender) if self.gender is not None else None
|
||||
"""Normalize structured LLM fields once at the profile boundary."""
|
||||
self.bio = _coerce_to_str(self.bio) or self.name
|
||||
self.persona = _coerce_to_str(self.persona) or (
|
||||
f"{self.name} is a participant in social discussions."
|
||||
)
|
||||
self.country = _coerce_to_str(self.country) or None
|
||||
self.profession = _coerce_to_str(self.profession) or None
|
||||
self.gender = _coerce_to_str(self.gender) or None
|
||||
self.mbti = _coerce_to_str(self.mbti) or None
|
||||
self.interested_topics = _coerce_to_str_list(self.interested_topics)
|
||||
|
||||
def to_reddit_format(self) -> Dict[str, Any]:
|
||||
|
|
@ -603,15 +609,6 @@ class OasisProfileGenerator:
|
|||
try:
|
||||
result = json.loads(content)
|
||||
|
||||
# Normalize types from LLM output
|
||||
for str_field in ('bio', 'persona', 'country', 'profession'):
|
||||
if str_field in result and result[str_field] is not None:
|
||||
result[str_field] = _coerce_to_str(result[str_field])
|
||||
if 'interested_topics' in result:
|
||||
result['interested_topics'] = _coerce_to_str_list(
|
||||
result['interested_topics']
|
||||
)
|
||||
|
||||
# 验证必需字段
|
||||
if "bio" not in result or not result["bio"]:
|
||||
result["bio"] = entity_summary[:200] if entity_summary else f"{entity_type}: {entity_name}"
|
||||
|
|
@ -1159,19 +1156,15 @@ class OasisProfileGenerator:
|
|||
|
||||
# 写入数据行
|
||||
for idx, profile in enumerate(profiles):
|
||||
# Defensive coercion in case __post_init__ was bypassed
|
||||
bio = _coerce_to_str(profile.bio) if profile.bio else profile.name
|
||||
persona = _coerce_to_str(profile.persona) if profile.persona else ''
|
||||
|
||||
# user_char: 完整人设（bio + persona），用于LLM系统提示
|
||||
user_char = bio
|
||||
if persona and persona != bio:
|
||||
user_char = f"{bio} {persona}"
|
||||
user_char = profile.bio
|
||||
if profile.persona and profile.persona != profile.bio:
|
||||
user_char = f"{profile.bio} {profile.persona}"
|
||||
# 处理换行符（CSV中用空格替代）
|
||||
user_char = user_char.replace('\n', ' ').replace('\r', ' ')
|
||||
|
||||
# description: 简短简介，用于外部显示
|
||||
description = bio.replace('\n', ' ').replace('\r', ' ')
|
||||
description = profile.bio.replace('\n', ' ').replace('\r', ' ')
|
||||
|
||||
row = [
|
||||
idx, # user_id: 从0开始的顺序ID
|
||||
|
|
@ -1229,40 +1222,27 @@ class OasisProfileGenerator:
|
|||
"""
|
||||
data = []
|
||||
for idx, profile in enumerate(profiles):
|
||||
# Defensive coercion in case __post_init__ was bypassed
|
||||
bio = _coerce_to_str(profile.bio) if profile.bio else f"{profile.name}"
|
||||
persona = _coerce_to_str(profile.persona) if profile.persona else (
|
||||
f"{profile.name} is a participant in social discussions."
|
||||
)
|
||||
country = _coerce_to_str(profile.country) if profile.country else "中国"
|
||||
profession = _coerce_to_str(profile.profession) if profile.profession else None
|
||||
interested_topics = (
|
||||
_coerce_to_str_list(profile.interested_topics)
|
||||
if profile.interested_topics
|
||||
else None
|
||||
)
|
||||
|
||||
# 使用与 to_reddit_format() 一致的格式
|
||||
item = {
|
||||
"user_id": profile.user_id if profile.user_id is not None else idx, # 关键：必须包含 user_id
|
||||
"username": profile.user_name,
|
||||
"name": profile.name,
|
||||
"bio": bio[:150],
|
||||
"persona": persona,
|
||||
"bio": profile.bio[:150],
|
||||
"persona": profile.persona,
|
||||
"karma": profile.karma if profile.karma else 1000,
|
||||
"created_at": profile.created_at,
|
||||
# OASIS必需字段 - 确保都有默认值
|
||||
"age": profile.age if profile.age else 30,
|
||||
"gender": self._normalize_gender(profile.gender),
|
||||
"mbti": profile.mbti if profile.mbti else "ISTJ",
|
||||
"country": country,
|
||||
"country": profile.country if profile.country else "中国",
|
||||
}
|
||||
|
||||
# 可选字段
|
||||
if profession:
|
||||
item["profession"] = profession
|
||||
if interested_topics:
|
||||
item["interested_topics"] = interested_topics
|
||||
if profile.profession:
|
||||
item["profession"] = profile.profession
|
||||
if profile.interested_topics:
|
||||
item["interested_topics"] = profile.interested_topics
|
||||
|
||||
data.append(item)
|
||||
|
||||
|
|
@ -1281,4 +1261,3 @@ class OasisProfileGenerator:
|
|||
"""[已废弃] 请使用 save_profiles() 方法"""
|
||||
logger.warning("save_profiles_to_json已废弃，请使用save_profiles方法")
|
||||
self.save_profiles(profiles, file_path, platform)
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,76 @@
|
|||
import csv
|
||||
import json
|
||||
|
||||
from app.services.oasis_profile_generator import (
|
||||
OasisAgentProfile,
|
||||
OasisProfileGenerator,
|
||||
_coerce_to_str,
|
||||
_coerce_to_str_list,
|
||||
)
|
||||
|
||||
|
||||
def test_text_coercion_handles_none_nested_objects_and_lists():
|
||||
assert _coerce_to_str(None) == ""
|
||||
assert _coerce_to_str({"text": {"value": "中文"}}) == "中文"
|
||||
assert _coerce_to_str([None, {"description": "alpha"}, ["beta"]]) == "alpha, beta"
|
||||
assert _coerce_to_str({"unexpected": None}) == '{"unexpected": null}'
|
||||
|
||||
|
||||
def test_list_coercion_flattens_nested_values_and_drops_missing_items():
|
||||
assert _coerce_to_str_list(
|
||||
["AI", ["policy", None], {"name": "society"}, 4]
|
||||
) == ["AI", "policy", "society", "4"]
|
||||
assert _coerce_to_str_list(None) == []
|
||||
|
||||
|
||||
def test_profile_construction_is_the_single_normalization_boundary():
|
||||
profile = OasisAgentProfile(
|
||||
user_id=1,
|
||||
user_name="agent",
|
||||
name="Agent Name",
|
||||
bio=None,
|
||||
persona={"text": {"value": "详细人设"}},
|
||||
gender=["female"],
|
||||
mbti={"value": "INTJ"},
|
||||
country={"name": "中国"},
|
||||
profession={"description": "研究员"},
|
||||
interested_topics=["AI", ["政策", None], {"name": "社会"}],
|
||||
)
|
||||
|
||||
assert profile.bio == "Agent Name"
|
||||
assert profile.persona == "详细人设"
|
||||
assert profile.gender == "female"
|
||||
assert profile.mbti == "INTJ"
|
||||
assert profile.country == "中国"
|
||||
assert profile.profession == "研究员"
|
||||
assert profile.interested_topics == ["AI", "政策", "社会"]
|
||||
assert "None" not in json.dumps(profile.to_dict(), ensure_ascii=False)
|
||||
|
||||
|
||||
def test_normalized_profile_serializes_to_twitter_and_reddit(tmp_path):
|
||||
profile = OasisAgentProfile(
|
||||
user_id=1,
|
||||
user_name="agent",
|
||||
name="Agent Name",
|
||||
bio={"summary": "公开简介"},
|
||||
persona=["详细", "人设"],
|
||||
mbti={"text": "ENFP"},
|
||||
interested_topics=["AI", ["政策"]],
|
||||
)
|
||||
generator = object.__new__(OasisProfileGenerator)
|
||||
twitter_path = tmp_path / "twitter_profiles.csv"
|
||||
reddit_path = tmp_path / "reddit_profiles.json"
|
||||
|
||||
generator._save_twitter_csv([profile], str(twitter_path))
|
||||
generator._save_reddit_json([profile], str(reddit_path))
|
||||
|
||||
with twitter_path.open(encoding="utf-8") as handle:
|
||||
twitter = next(csv.DictReader(handle))
|
||||
reddit = json.loads(reddit_path.read_text(encoding="utf-8"))[0]
|
||||
|
||||
assert twitter["description"] == "公开简介"
|
||||
assert twitter["user_char"] == "公开简介 详细, 人设"
|
||||
assert reddit["bio"] == "公开简介"
|
||||
assert reddit["persona"] == "详细, 人设"
|
||||
assert reddit["mbti"] == "ENFP"
|
||||
assert reddit["interested_topics"] == ["AI", "政策"]
|
||||
Loading…
Reference in New Issue