From 41fc932b627f8cec78d6581b8e57caf7d19326f4 Mon Sep 17 00:00:00 2001 From: ygh1254 Date: Thu, 12 Mar 2026 13:52:40 +0900 Subject: [PATCH 1/3] fix: normalize structured LLM profile fields before serialization --- .../app/services/oasis_profile_generator.py | 99 ++++++++++++++++--- 1 file changed, 88 insertions(+), 11 deletions(-) diff --git a/backend/app/services/oasis_profile_generator.py b/backend/app/services/oasis_profile_generator.py index 5847bcd1..c8fdf60e 100644 --- a/backend/app/services/oasis_profile_generator.py +++ b/backend/app/services/oasis_profile_generator.py @@ -27,6 +27,47 @@ from .zep_entity_reader import EntityNode, ZepEntityReader logger = get_logger('mirofish.oasis_profile') +def _coerce_to_str(value: Any) -> str: + """Coerce a value to a plain string. + + Handles dict, list, and other non-string types that may be returned + by LLM JSON parsing. + """ + if isinstance(value, str): + return value + if isinstance(value, dict): + for key in ('text', 'value', 'description', 'content', 'summary', 'name'): + if key in value and isinstance(value[key], str): + return value[key] + return json.dumps(value, ensure_ascii=False) + if isinstance(value, list): + str_items = [_coerce_to_str(item) for item in value] + return ', '.join(str_items) + return str(value) + + +def _coerce_to_str_list(value: Any) -> List[str]: + """Coerce a value to a list of strings. + + Handles nested structures that may be returned by LLM JSON parsing. + """ + if isinstance(value, list): + result = [] + for item in value: + if isinstance(item, str): + result.append(item) + elif isinstance(item, dict): + result.append(_coerce_to_str(item)) + else: + result.append(str(item)) + return result + if isinstance(value, str): + return [value] + if isinstance(value, dict): + return [_coerce_to_str(value)] + return [] + + @dataclass class OasisAgentProfile: """OASIS Agent Profile数据结构""" @@ -59,6 +100,16 @@ class OasisAgentProfile: created_at: str = field(default_factory=lambda: datetime.now().strftime("%Y-%m-%d")) + def __post_init__(self): + """Normalize field types to guard against structured LLM outputs + (e.g. dict/list instead of plain strings).""" + self.bio = _coerce_to_str(self.bio) + self.persona = _coerce_to_str(self.persona) + self.country = _coerce_to_str(self.country) if self.country is not None else None + self.profession = _coerce_to_str(self.profession) if self.profession is not None else None + self.gender = _coerce_to_str(self.gender) if self.gender is not None else None + self.interested_topics = _coerce_to_str_list(self.interested_topics) + def to_reddit_format(self) -> Dict[str, Any]: """转换为Reddit平台格式""" profile = { @@ -552,6 +603,15 @@ class OasisProfileGenerator: try: result = json.loads(content) + # Normalize types from LLM output + for str_field in ('bio', 'persona', 'country', 'profession'): + if str_field in result and result[str_field] is not None: + result[str_field] = _coerce_to_str(result[str_field]) + if 'interested_topics' in result: + result['interested_topics'] = _coerce_to_str_list( + result['interested_topics'] + ) + # 验证必需字段 if "bio" not in result or not result["bio"]: result["bio"] = entity_summary[:200] if entity_summary else f"{entity_type}: {entity_name}" @@ -1099,15 +1159,19 @@ class OasisProfileGenerator: # 写入数据行 for idx, profile in enumerate(profiles): + # Defensive coercion in case __post_init__ was bypassed + bio = _coerce_to_str(profile.bio) if profile.bio else profile.name + persona = _coerce_to_str(profile.persona) if profile.persona else '' + # user_char: 完整人设（bio + persona），用于LLM系统提示 - user_char = profile.bio - if profile.persona and profile.persona != profile.bio: - user_char = f"{profile.bio} {profile.persona}" + user_char = bio + if persona and persona != bio: + user_char = f"{bio} {persona}" # 处理换行符（CSV中用空格替代） user_char = user_char.replace('\n', ' ').replace('\r', ' ') # description: 简短简介，用于外部显示 - description = profile.bio.replace('\n', ' ').replace('\r', ' ') + description = bio.replace('\n', ' ').replace('\r', ' ') row = [ idx, # user_id: 从0开始的顺序ID @@ -1165,27 +1229,40 @@ class OasisProfileGenerator: """ data = [] for idx, profile in enumerate(profiles): + # Defensive coercion in case __post_init__ was bypassed + bio = _coerce_to_str(profile.bio) if profile.bio else f"{profile.name}" + persona = _coerce_to_str(profile.persona) if profile.persona else ( + f"{profile.name} is a participant in social discussions." + ) + country = _coerce_to_str(profile.country) if profile.country else "中国" + profession = _coerce_to_str(profile.profession) if profile.profession else None + interested_topics = ( + _coerce_to_str_list(profile.interested_topics) + if profile.interested_topics + else None + ) + # 使用与 to_reddit_format() 一致的格式 item = { "user_id": profile.user_id if profile.user_id is not None else idx, # 关键：必须包含 user_id "username": profile.user_name, "name": profile.name, - "bio": profile.bio[:150] if profile.bio else f"{profile.name}", - "persona": profile.persona or f"{profile.name} is a participant in social discussions.", + "bio": bio[:150], + "persona": persona, "karma": profile.karma if profile.karma else 1000, "created_at": profile.created_at, # OASIS必需字段 - 确保都有默认值 "age": profile.age if profile.age else 30, "gender": self._normalize_gender(profile.gender), "mbti": profile.mbti if profile.mbti else "ISTJ", - "country": profile.country if profile.country else "中国", + "country": country, } # 可选字段 - if profile.profession: - item["profession"] = profile.profession - if profile.interested_topics: - item["interested_topics"] = profile.interested_topics + if profession: + item["profession"] = profession + if interested_topics: + item["interested_topics"] = interested_topics data.append(item) From 9f80ad66fdb562346ad9d2278da3c3cdc0aee47b Mon Sep 17 00:00:00 2001 From: 666ghj <670939375@qq.com> Date: Wed, 22 Jul 2026 01:36:18 +0800 Subject: [PATCH 2/3] fix: normalize profile fields at construction --- .../app/services/oasis_profile_generator.py | 99 ++++++++----------- .../tests/test_profile_field_normalization.py | 76 ++++++++++++++ 2 files changed, 115 insertions(+), 60 deletions(-) create mode 100644 backend/tests/test_profile_field_normalization.py diff --git a/backend/app/services/oasis_profile_generator.py b/backend/app/services/oasis_profile_generator.py index c8fdf60e..168f4b14 100644 --- a/backend/app/services/oasis_profile_generator.py +++ b/backend/app/services/oasis_profile_generator.py @@ -33,15 +33,20 @@ def _coerce_to_str(value: Any) -> str: Handles dict, list, and other non-string types that may be returned by LLM JSON parsing. """ + if value is None: + return "" if isinstance(value, str): return value if isinstance(value, dict): for key in ('text', 'value', 'description', 'content', 'summary', 'name'): - if key in value and isinstance(value[key], str): - return value[key] + if key in value: + candidate = _coerce_to_str(value[key]) + if candidate: + return candidate return json.dumps(value, ensure_ascii=False) - if isinstance(value, list): + if isinstance(value, (list, tuple)): str_items = [_coerce_to_str(item) for item in value] + str_items = [item for item in str_items if item] return ', '.join(str_items) return str(value) @@ -51,21 +56,20 @@ def _coerce_to_str_list(value: Any) -> List[str]: Handles nested structures that may be returned by LLM JSON parsing. """ - if isinstance(value, list): - result = [] + if value is None: + return [] + if isinstance(value, (list, tuple)): + result: List[str] = [] for item in value: - if isinstance(item, str): - result.append(item) - elif isinstance(item, dict): - result.append(_coerce_to_str(item)) + if isinstance(item, (list, tuple)): + result.extend(_coerce_to_str_list(item)) else: - result.append(str(item)) + text = _coerce_to_str(item) + if text: + result.append(text) return result - if isinstance(value, str): - return [value] - if isinstance(value, dict): - return [_coerce_to_str(value)] - return [] + text = _coerce_to_str(value) + return [text] if text else [] @dataclass @@ -101,13 +105,15 @@ class OasisAgentProfile: created_at: str = field(default_factory=lambda: datetime.now().strftime("%Y-%m-%d")) def __post_init__(self): - """Normalize field types to guard against structured LLM outputs - (e.g. dict/list instead of plain strings).""" - self.bio = _coerce_to_str(self.bio) - self.persona = _coerce_to_str(self.persona) - self.country = _coerce_to_str(self.country) if self.country is not None else None - self.profession = _coerce_to_str(self.profession) if self.profession is not None else None - self.gender = _coerce_to_str(self.gender) if self.gender is not None else None + """Normalize structured LLM fields once at the profile boundary.""" + self.bio = _coerce_to_str(self.bio) or self.name + self.persona = _coerce_to_str(self.persona) or ( + f"{self.name} is a participant in social discussions." + ) + self.country = _coerce_to_str(self.country) or None + self.profession = _coerce_to_str(self.profession) or None + self.gender = _coerce_to_str(self.gender) or None + self.mbti = _coerce_to_str(self.mbti) or None self.interested_topics = _coerce_to_str_list(self.interested_topics) def to_reddit_format(self) -> Dict[str, Any]: @@ -603,15 +609,6 @@ class OasisProfileGenerator: try: result = json.loads(content) - # Normalize types from LLM output - for str_field in ('bio', 'persona', 'country', 'profession'): - if str_field in result and result[str_field] is not None: - result[str_field] = _coerce_to_str(result[str_field]) - if 'interested_topics' in result: - result['interested_topics'] = _coerce_to_str_list( - result['interested_topics'] - ) - # 验证必需字段 if "bio" not in result or not result["bio"]: result["bio"] = entity_summary[:200] if entity_summary else f"{entity_type}: {entity_name}" @@ -1159,19 +1156,15 @@ class OasisProfileGenerator: # 写入数据行 for idx, profile in enumerate(profiles): - # Defensive coercion in case __post_init__ was bypassed - bio = _coerce_to_str(profile.bio) if profile.bio else profile.name - persona = _coerce_to_str(profile.persona) if profile.persona else '' - # user_char: 完整人设（bio + persona），用于LLM系统提示 - user_char = bio - if persona and persona != bio: - user_char = f"{bio} {persona}" + user_char = profile.bio + if profile.persona and profile.persona != profile.bio: + user_char = f"{profile.bio} {profile.persona}" # 处理换行符（CSV中用空格替代） user_char = user_char.replace('\n', ' ').replace('\r', ' ') # description: 简短简介，用于外部显示 - description = bio.replace('\n', ' ').replace('\r', ' ') + description = profile.bio.replace('\n', ' ').replace('\r', ' ') row = [ idx, # user_id: 从0开始的顺序ID @@ -1229,40 +1222,27 @@ class OasisProfileGenerator: """ data = [] for idx, profile in enumerate(profiles): - # Defensive coercion in case __post_init__ was bypassed - bio = _coerce_to_str(profile.bio) if profile.bio else f"{profile.name}" - persona = _coerce_to_str(profile.persona) if profile.persona else ( - f"{profile.name} is a participant in social discussions." - ) - country = _coerce_to_str(profile.country) if profile.country else "中国" - profession = _coerce_to_str(profile.profession) if profile.profession else None - interested_topics = ( - _coerce_to_str_list(profile.interested_topics) - if profile.interested_topics - else None - ) - # 使用与 to_reddit_format() 一致的格式 item = { "user_id": profile.user_id if profile.user_id is not None else idx, # 关键：必须包含 user_id "username": profile.user_name, "name": profile.name, - "bio": bio[:150], - "persona": persona, + "bio": profile.bio[:150], + "persona": profile.persona, "karma": profile.karma if profile.karma else 1000, "created_at": profile.created_at, # OASIS必需字段 - 确保都有默认值 "age": profile.age if profile.age else 30, "gender": self._normalize_gender(profile.gender), "mbti": profile.mbti if profile.mbti else "ISTJ", - "country": country, + "country": profile.country if profile.country else "中国", } # 可选字段 - if profession: - item["profession"] = profession - if interested_topics: - item["interested_topics"] = interested_topics + if profile.profession: + item["profession"] = profile.profession + if profile.interested_topics: + item["interested_topics"] = profile.interested_topics data.append(item) @@ -1281,4 +1261,3 @@ class OasisProfileGenerator: """[已废弃] 请使用 save_profiles() 方法""" logger.warning("save_profiles_to_json已废弃，请使用save_profiles方法") self.save_profiles(profiles, file_path, platform) - diff --git a/backend/tests/test_profile_field_normalization.py b/backend/tests/test_profile_field_normalization.py new file mode 100644 index 00000000..0714410b --- /dev/null +++ b/backend/tests/test_profile_field_normalization.py @@ -0,0 +1,76 @@ +import csv +import json + +from app.services.oasis_profile_generator import ( + OasisAgentProfile, + OasisProfileGenerator, + _coerce_to_str, + _coerce_to_str_list, +) + + +def test_text_coercion_handles_none_nested_objects_and_lists(): + assert _coerce_to_str(None) == "" + assert _coerce_to_str({"text": {"value": "中文"}}) == "中文" + assert _coerce_to_str([None, {"description": "alpha"}, ["beta"]]) == "alpha, beta" + assert _coerce_to_str({"unexpected": None}) == '{"unexpected": null}' + + +def test_list_coercion_flattens_nested_values_and_drops_missing_items(): + assert _coerce_to_str_list( + ["AI", ["policy", None], {"name": "society"}, 4] + ) == ["AI", "policy", "society", "4"] + assert _coerce_to_str_list(None) == [] + + +def test_profile_construction_is_the_single_normalization_boundary(): + profile = OasisAgentProfile( + user_id=1, + user_name="agent", + name="Agent Name", + bio=None, + persona={"text": {"value": "详细人设"}}, + gender=["female"], + mbti={"value": "INTJ"}, + country={"name": "中国"}, + profession={"description": "研究员"}, + interested_topics=["AI", ["政策", None], {"name": "社会"}], + ) + + assert profile.bio == "Agent Name" + assert profile.persona == "详细人设" + assert profile.gender == "female" + assert profile.mbti == "INTJ" + assert profile.country == "中国" + assert profile.profession == "研究员" + assert profile.interested_topics == ["AI", "政策", "社会"] + assert "None" not in json.dumps(profile.to_dict(), ensure_ascii=False) + + +def test_normalized_profile_serializes_to_twitter_and_reddit(tmp_path): + profile = OasisAgentProfile( + user_id=1, + user_name="agent", + name="Agent Name", + bio={"summary": "公开简介"}, + persona=["详细", "人设"], + mbti={"text": "ENFP"}, + interested_topics=["AI", ["政策"]], + ) + generator = object.__new__(OasisProfileGenerator) + twitter_path = tmp_path / "twitter_profiles.csv" + reddit_path = tmp_path / "reddit_profiles.json" + + generator._save_twitter_csv([profile], str(twitter_path)) + generator._save_reddit_json([profile], str(reddit_path)) + + with twitter_path.open(encoding="utf-8") as handle: + twitter = next(csv.DictReader(handle)) + reddit = json.loads(reddit_path.read_text(encoding="utf-8"))[0] + + assert twitter["description"] == "公开简介" + assert twitter["user_char"] == "公开简介 详细, 人设" + assert reddit["bio"] == "公开简介" + assert reddit["persona"] == "详细, 人设" + assert reddit["mbti"] == "ENFP" + assert reddit["interested_topics"] == ["AI", "政策"] From c415387995d48fbf415dc0250cf8c935467a5dd5 Mon Sep 17 00:00:00 2001 From: 666ghj <670939375@qq.com> Date: Wed, 22 Jul 2026 19:50:04 +0800 Subject: [PATCH 3/3] style: remove trailing whitespace from profile model --- backend/app/services/oasis_profile_generator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/app/services/oasis_profile_generator.py b/backend/app/services/oasis_profile_generator.py index 168f4b14..add3a24f 100644 --- a/backend/app/services/oasis_profile_generator.py +++ b/backend/app/services/oasis_profile_generator.py @@ -81,7 +81,7 @@ class OasisAgentProfile: name: str bio: str persona: str - + # 可选字段 - Reddit风格 karma: int = 1000 @@ -115,7 +115,7 @@ class OasisAgentProfile: self.gender = _coerce_to_str(self.gender) or None self.mbti = _coerce_to_str(self.mbti) or None self.interested_topics = _coerce_to_str_list(self.interested_topics) - + def to_reddit_format(self) -> Dict[str, Any]: """转换为Reddit平台格式""" profile = {