RomanNum3ral
/
MicroFish
mirror of https://github.com/666ghj/MiroFish.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Refactor: replace Zep with Graphiti+FalkorDB (FalkorDriver adapter, M3 LLM, deterministic embedder) + add e2e test, docker-compose, e2e one-shot, deploy notes

This commit is contained in:
ENI 2026-06-09 11:28:32 +00:00
parent 96096ea0ff
commit 1b62a3c5b8
65 changed files with 8541 additions and 3077 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

83
Dockerfile
View File

@ -1,29 +1,78 @@
FROM python:3.11
# MiroFish production Dockerfile for agent.profikid.nl deployment
# - Builds the Vue frontend into static files
# - Serves the static bundle via nginx on :3000
# - Reverse-proxies /api/* to the Flask backend on :5001 (same container, localhost)
# - Single exposed port for Traefik
# - Memory graph: Graphiti (Python lib) talks to FalkorDB on the shared network
# 安装 Node.js （满足 >=18）及必要工具
FROM python:3.11-slim
ENV DEBIAN_FRONTEND=noninteractive \
PYTHONUNBUFFERED=1 \
PIP_NO_CACHE_DIR=1 \
VITE_API_BASE_URL=/api \
HF_HUB_OFFLINE=0 \
SENTENCE_TRANSFORMERS_HOME=/root/.cache/huggingface
# Install Node 20 (for building the frontend), nginx (for serving it), and
# build deps for the Python wheels (sentence-transformers pulls numpy/torch).
RUN apt-get update \
&& apt-get install -y --no-install-recommends nodejs npm \
&& rm -rf /var/lib/apt/lists/*
&& apt-get install -y --no-install-recommends \
nodejs \
npm \
nginx \
curl \
ca-certificates \
git \
&& rm -rf /var/lib/apt/lists/*
# 从 uv 官方镜像复制 uv
# Install uv for fast Python package management
COPY --from=ghcr.io/astral-sh/uv:0.9.26 /uv /uvx /bin/
WORKDIR /app
# 先复制依赖描述文件以利用缓存
COPY package.json package-lock.json ./
# ---- 1) Install Python deps (backend) ----
# Copy the local path-dep (camel-oasis fork) BEFORE uv sync, so the path can resolve.
# Note: we don't COPY uv.lock here — the first build has to generate it. Once a
# successful build has happened, the lock will be in the source tree and a
# subsequent `uv sync --frozen` will be reproducible.
COPY backend/vendor ./backend/vendor
COPY backend/pyproject.toml ./backend/
WORKDIR /app/backend
RUN uv sync
# ---- 2) Install Node deps + build frontend ----
WORKDIR /app
# Copy locales first because the Vite build's `@locales` alias points to ../locales
COPY locales/ ./locales/
COPY frontend/package.json frontend/package-lock.json ./frontend/
COPY backend/pyproject.toml backend/uv.lock ./backend/
RUN cd frontend && npm ci --no-audit --no-fund
# 安装依赖（Node + Python）
RUN npm ci \
&& npm ci --prefix frontend \
&& cd backend && uv sync --frozen
# Copy full frontend source and build the production bundle
COPY frontend/ ./frontend/
RUN cd frontend && npm run build
# 复制项目源码
COPY . .
# ---- 3) Copy the rest of the app (backend source) ----
COPY backend/ ./backend/
EXPOSE 3000 5001
# ---- 4) Pre-download the sentence-transformers embedding model ----
# Doing it at build time means the container is ready to serve on first boot
# (no 100MB download the first time the user runs a graph build).
WORKDIR /app/backend
RUN uv run python -c "from sentence_transformers import SentenceTransformer; \
SentenceTransformer('sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2')" \
|| (echo 'Failed to pre-download embedding model — build will fall back to download on first use.' && exit 0)
# 同时启动前后端（开发模式）
CMD ["npm", "run", "dev"]
# ---- 5) nginx config + start script ----
COPY nginx.conf /etc/nginx/nginx.conf
COPY start.sh /app/start.sh
RUN chmod +x /app/start.sh
# Runtime dirs
RUN mkdir -p /app/backend/uploads /app/logs /var/lib/nginx /var/log/nginx
EXPOSE 3000
CMD ["/app/start.sh"]

12
backend/app/api/graph.py
View File

@ -285,7 +285,7 @@ def build_graph():
# 检查配置
errors = []
if not Config.ZEP_API_KEY:
if not Config.FALKORDB_HOST:
errors.append(t('api.zepApiKeyMissing'))
if errors:
logger.error(f"配置错误: {errors}")
@ -387,7 +387,7 @@ def build_graph():
)
# 创建图谱构建服务
builder = GraphBuilderService(api_key=Config.ZEP_API_KEY)
builder = GraphBuilderService(api_key=None)
# 分块
task_manager.update_task(
@ -572,13 +572,13 @@ def get_graph_data(graph_id: str):
获取图谱数据（节点和边）
"""
try:
if not Config.ZEP_API_KEY:
if not Config.FALKORDB_HOST:
return jsonify({
"success": False,
"error": t('api.zepApiKeyMissing')
}), 500
builder = GraphBuilderService(api_key=Config.ZEP_API_KEY)
builder = GraphBuilderService(api_key=None)
graph_data = builder.get_graph_data(graph_id)
return jsonify({
@ -600,13 +600,13 @@ def delete_graph(graph_id: str):
删除Zep图谱
"""
try:
if not Config.ZEP_API_KEY:
if not Config.FALKORDB_HOST:
return jsonify({
"success": False,
"error": t('api.zepApiKeyMissing')
}), 500
builder = GraphBuilderService(api_key=Config.ZEP_API_KEY)
builder = GraphBuilderService(api_key=None)
builder.delete_graph(graph_id)
return jsonify({

6
backend/app/api/simulation.py
View File

@ -57,7 +57,7 @@ def get_graph_entities(graph_id: str):
enrich: 是否获取相关边信息（默认true）
"""
try:
if not Config.ZEP_API_KEY:
if not Config.FALKORDB_HOST:
return jsonify({
"success": False,
"error": t('api.zepApiKeyMissing')
@ -94,7 +94,7 @@ def get_graph_entities(graph_id: str):
def get_entity_detail(graph_id: str, entity_uuid: str):
"""获取单个实体的详细信息"""
try:
if not Config.ZEP_API_KEY:
if not Config.FALKORDB_HOST:
return jsonify({
"success": False,
"error": t('api.zepApiKeyMissing')
@ -127,7 +127,7 @@ def get_entity_detail(graph_id: str, entity_uuid: str):
def get_entities_by_type(graph_id: str, entity_type: str):
"""获取指定类型的所有实体"""
try:
if not Config.ZEP_API_KEY:
if not Config.FALKORDB_HOST:
return jsonify({
"success": False,
"error": t('api.zepApiKeyMissing')

19
backend/app/config.py
View File

@ -32,8 +32,17 @@ class Config:
LLM_BASE_URL = os.environ.get('LLM_BASE_URL', 'https://api.openai.com/v1')
LLM_MODEL_NAME = os.environ.get('LLM_MODEL_NAME', 'gpt-4o-mini')
# Zep配置
ZEP_API_KEY = os.environ.get('ZEP_API_KEY')
# FalkorDB配置（replaces Zep）
FALKORDB_HOST = os.environ.get('FALKORDB_HOST', 'localhost')
FALKORDB_PORT = int(os.environ.get('FALKORDB_PORT', '6379'))
FALKORDB_USERNAME = os.environ.get('FALKORDB_USERNAME', None) or None
FALKORDB_PASSWORD = os.environ.get('FALKORDB_PASSWORD', None) or None
# 嵌入模型：本地 sentence-transformers（pre-downloaded in image）
EMBEDDING_MODEL = os.environ.get(
'EMBEDDING_MODEL',
'sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2',
)
# 文件上传配置
MAX_CONTENT_LENGTH = 50 * 1024 * 1024 # 50MB
@ -69,7 +78,7 @@ class Config:
errors: list[str] = []
if not cls.LLM_API_KEY:
errors.append("LLM_API_KEY 未配置")
if not cls.ZEP_API_KEY:
errors.append("ZEP_API_KEY 未配置")
# FalkorDB host defaults to localhost; only fail if explicitly empty
if not cls.FALKORDB_HOST:
errors.append("FALKORDB_HOST 未配置")
return errors

138
backend/app/services/graph_builder.py
View File

@ -10,13 +10,31 @@ import threading
from typing import Dict, Any, List, Optional, Callable
from dataclasses import dataclass
from zep_cloud.client import Zep
from zep_cloud import EpisodeData, EntityEdgeSourceTarget
# Zep types (EpisodeData, EntityEdgeSourceTarget, EntityModel/EdgeModel from
# external_clients.ontology) are still referenced for ontology dynamic Pydantic
# classes. When zep-cloud is uninstalled, we provide stubs so module import
# succeeds and the Zep `client` attribute on this service is just the
# GraphitiAdapter (see __init__ below).
try:
from zep_cloud.client import Zep
from zep_cloud import EpisodeData, EntityEdgeSourceTarget
except ImportError:
class Zep:
def __init__(self, *a, **kw): pass
class EpisodeData:
def __init__(self, data, type="text"):
self.data = data
self.type = type
class EntityEdgeSourceTarget:
def __init__(self, source, target):
self.source = source
self.target = target
from ..config import Config
from ..models.task import TaskManager, TaskStatus
from ..utils.zep_paging import fetch_all_nodes, fetch_all_edges
from .text_processor import TextProcessor
from .graphiti_service import get_graphiti_adapter
from ..utils.locale import t, get_locale, set_locale
@ -44,11 +62,10 @@ class GraphBuilderService:
"""
def __init__(self, api_key: Optional[str] = None):
self.api_key = api_key or Config.ZEP_API_KEY
if not self.api_key:
raise ValueError("ZEP_API_KEY 未配置")
self.client = Zep(api_key=self.api_key)
# Zep's per-project graph becomes Graphiti's group_id (a partition).
# The single shared GraphitiAdapter handles all projects.
self.api_key = api_key # kept for signature compat; no longer required
self.client = get_graphiti_adapter()
self.task_manager = TaskManager()
def build_graph_async(
@ -191,13 +208,14 @@ class GraphBuilderService:
self.task_manager.fail_task(task_id, error_msg)
def create_graph(self, name: str) -> str:
"""创建Zep图谱（公开方法）"""
"""创建图谱（公开方法）"""
graph_id = f"mirofish_{uuid.uuid4().hex[:16]}"
self.client.graph.create(
# Graphiti creates the partition implicitly on first add_episode; this
# call just reserves the group_id.
self.client.create_graph(
graph_id=graph_id,
name=name,
description="MiroFish Social Simulation Graph"
description="MiroFish Social Simulation Graph",
)
return graph_id
@ -207,7 +225,23 @@ class GraphBuilderService:
import warnings
from typing import Optional
from pydantic import Field
from zep_cloud.external_clients.ontology import EntityModel, EntityText, EdgeModel
# Base classes for the dynamic Pydantic ontology types. Try Zep's
# EntityModel/EdgeModel first (preserves a tiny bit of compatibility if
# zep-cloud is somehow still in the env), else fall back to Graphiti's
# EntityNode/EntityEdge which is what we actually use.
try:
from zep_cloud.external_clients.ontology import EntityModel, EntityText, EdgeModel
except ImportError:
try:
from graphiti_core.nodes import EntityNode as _GTEntity
from graphiti_core.edges import EntityEdge as _GTEdge
EntityModel = _GTEntity # type: ignore[assignment,misc]
EdgeModel = _GTEdge # type: ignore[assignment,misc]
EntityText = str # type: ignore[assignment,misc]
except ImportError:
from pydantic import BaseModel as EntityModel # type: ignore[assignment,misc]
from pydantic import BaseModel as EdgeModel # type: ignore[assignment,misc]
EntityText = str # type: ignore[assignment,misc]
# 抑制 Pydantic v2 关于 Field(default=None) 的警告
# 这是 Zep SDK 要求的用法，警告来自动态类创建，可以安全忽略
@ -283,13 +317,14 @@ class GraphBuilderService:
if source_targets:
edge_definitions[name] = (edge_class, source_targets)
# 调用Zep API设置本体
# Graphiti accepts the original JSON ontology directly. The Pydantic
# class construction above was a Zep-specific quirk; we can skip it
# for Graphiti and just forward the input. Keep the code path the same
# shape (Pydantic EntityModel/EdgeModel classes still constructed and
# exposed to introspection code), but the actual set_ontology call now
# uses the adapter with the source-of-truth JSON.
if entity_types or edge_definitions:
self.client.graph.set_ontology(
graph_ids=[graph_id],
entities=entity_types if entity_types else None,
edges=edge_definitions if edge_definitions else None,
)
self.client.set_ontology(graph_id=graph_id, ontology=ontology)
def add_text_batches(
self,
@ -320,13 +355,13 @@ class GraphBuilderService:
for chunk in batch_chunks
]
# 发送到Zep
# 发送到图谱（Graphiti via adapter — returns after sync extraction）
try:
batch_result = self.client.graph.add_batch(
batch_result = self.client.add_batch(
graph_id=graph_id,
episodes=episodes
episodes=episodes,
)
# 收集返回的 episode uuid
if batch_result and isinstance(batch_result, list):
for ep in batch_result:
@ -334,9 +369,6 @@ class GraphBuilderService:
if ep_uuid:
episode_uuids.append(ep_uuid)
# 避免请求过快
time.sleep(1)
except Exception as e:
if progress_callback:
progress_callback(t('progress.batchFailed', batch=batch_num, error=str(e)), 0)
@ -350,55 +382,21 @@ class GraphBuilderService:
progress_callback: Optional[Callable] = None,
timeout: int = 600
):
"""等待所有 episode 处理完成（通过查询每个 episode 的 processed 状态）"""
"""等待所有 episode 处理完成。
Graphiti's add_episode is synchronous (extraction completes before the
call returns), so all episodes returned by add_batch are already done.
We still surface progress messages and respect the timeout for the
outer loop contract, but we never actually need to poll.
"""
if not episode_uuids:
if progress_callback:
progress_callback(t('progress.noEpisodesWait'), 1.0)
return
start_time = time.time()
pending_episodes = set(episode_uuids)
completed_count = 0
total_episodes = len(episode_uuids)
if progress_callback:
progress_callback(t('progress.waitingEpisodes', count=total_episodes), 0)
while pending_episodes:
if time.time() - start_time > timeout:
if progress_callback:
progress_callback(
t('progress.episodesTimeout', completed=completed_count, total=total_episodes),
completed_count / total_episodes
)
break
# 检查每个 episode 的处理状态
for ep_uuid in list(pending_episodes):
try:
episode = self.client.graph.episode.get(uuid_=ep_uuid)
is_processed = getattr(episode, 'processed', False)
if is_processed:
pending_episodes.remove(ep_uuid)
completed_count += 1
except Exception as e:
# 忽略单个查询错误，继续
pass
elapsed = int(time.time() - start_time)
if progress_callback:
progress_callback(
t('progress.zepProcessing', completed=completed_count, total=total_episodes, pending=len(pending_episodes), elapsed=elapsed),
completed_count / total_episodes if total_episodes > 0 else 0
)
if pending_episodes:
time.sleep(3) # 每3秒检查一次
if progress_callback:
progress_callback(t('progress.processingComplete', completed=completed_count, total=total_episodes), 1.0)
progress_callback(t('progress.waitingEpisodes', count=len(episode_uuids)), 0)
progress_callback(t('progress.processingComplete', completed=len(episode_uuids), total=len(episode_uuids)), 1.0)
def _get_graph_info(self, graph_id: str) -> GraphInfo:
"""获取图谱信息"""
@ -502,5 +500,5 @@ class GraphBuilderService:
def delete_graph(self, graph_id: str):
"""删除图谱"""
self.client.graph.delete(graph_id=graph_id)
self.client.delete_graph(graph_id=graph_id)

1155
backend/app/services/graphiti_service.py Normal file
View File

File diff suppressed because it is too large Load Diff

31
backend/app/services/oasis_profile_generator.py
View File

@ -16,7 +16,13 @@ from dataclasses import dataclass, field
from datetime import datetime
from openai import OpenAI
from zep_cloud.client import Zep
try:
from zep_cloud.client import Zep # noqa: F811
except ImportError:
class Zep: # type: ignore[no-redef]
def __init__(self, *a, **kw): pass
class graph:
def search(self, **kw): raise NotImplementedError("zep-cloud not installed; use graphiti_service")
from ..config import Config
from ..utils.logger import get_logger
@ -198,16 +204,11 @@ class OasisProfileGenerator:
base_url=self.base_url
)
# Zep客户端用于检索丰富上下文
self.zep_api_key = zep_api_key or Config.ZEP_API_KEY
self.zep_client = None
# Graphiti adapter (replaces Zep) for retrieval-enrichment
self.zep_api_key = zep_api_key # kept for signature compat
from .graphiti_service import get_graphiti_adapter
self.zep_client = get_graphiti_adapter()
self.graph_id = graph_id
if self.zep_api_key:
try:
self.zep_client = Zep(api_key=self.zep_api_key)
except Exception as e:
logger.warning(f"Zep客户端初始化失败: {e}")
def generate_profile_from_entity(
self,
@ -324,12 +325,11 @@ class OasisProfileGenerator:
for attempt in range(max_retries):
try:
return self.zep_client.graph.search(
query=comprehensive_query,
return self.zep_client.search(
graph_id=self.graph_id,
query=comprehensive_query,
limit=30,
scope="edges",
reranker="rrf"
)
except Exception as e:
last_exception = e
@ -349,12 +349,11 @@ class OasisProfileGenerator:
for attempt in range(max_retries):
try:
return self.zep_client.graph.search(
return self.zep_client.search(
graph_id=self.graph_id or "",
query=comprehensive_query,
graph_id=self.graph_id,
limit=20,
scope="nodes",
reranker="rrf"
)
except Exception as e:
last_exception = e

28
backend/app/services/zep_entity_reader.py
View File

@ -7,7 +7,17 @@ import time
from typing import Dict, Any, List, Optional, Set, Callable, TypeVar
from dataclasses import dataclass, field
from zep_cloud.client import Zep
try:
from zep_cloud.client import Zep
except ImportError:
class Zep: # type: ignore[no-redef]
def __init__(self, *a, **kw): pass
class graph:
class node:
@staticmethod
def get_entity_edges(**kw): raise NotImplementedError("zep-cloud not installed; use graphiti_service")
@staticmethod
def get(**kw): raise NotImplementedError("zep-cloud not installed; use graphiti_service")
from ..config import Config
from ..utils.logger import get_logger
@ -79,11 +89,9 @@ class ZepEntityReader:
"""
def __init__(self, api_key: Optional[str] = None):
self.api_key = api_key or Config.ZEP_API_KEY
if not self.api_key:
raise ValueError("ZEP_API_KEY 未配置")
self.client = Zep(api_key=self.api_key)
self.api_key = api_key # kept for signature compat; no longer required
from .graphiti_service import get_graphiti_adapter
self.client = get_graphiti_adapter()
def _call_with_retry(
self,
@ -190,9 +198,9 @@ class ZepEntityReader:
边列表
"""
try:
# 使用重试机制调用Zep API
# Graphiti via adapter
edges = self._call_with_retry(
func=lambda: self.client.graph.node.get_entity_edges(node_uuid=node_uuid),
func=lambda: self.client.get_node_edges(node_uuid=node_uuid),
operation_name=f"获取节点边(node={node_uuid[:8]}...)"
)
@ -346,9 +354,9 @@ class ZepEntityReader:
EntityNode或None
"""
try:
# 使用重试机制获取节点
# Graphiti via adapter
node = self._call_with_retry(
func=lambda: self.client.graph.node.get(uuid_=entity_uuid),
func=lambda: self.client.get_node(node_uuid=entity_uuid),
operation_name=f"获取节点详情(uuid={entity_uuid[:8]}...)"
)

24
backend/app/services/zep_graph_memory_updater.py
View File

@ -12,7 +12,13 @@ from dataclasses import dataclass
from datetime import datetime
from queue import Queue, Empty
from zep_cloud.client import Zep
try:
from zep_cloud.client import Zep # noqa: F811
except ImportError:
class Zep: # type: ignore[no-redef]
def __init__(self, *a, **kw): pass
class graph:
def add(self, **kw): raise NotImplementedError("zep-cloud not installed; use graphiti_service")
from ..config import Config
from ..utils.logger import get_logger
@ -238,12 +244,10 @@ class ZepGraphMemoryUpdater:
api_key: Zep API Key（可选，默认从配置读取）
"""
self.graph_id = graph_id
self.api_key = api_key or Config.ZEP_API_KEY
if not self.api_key:
raise ValueError("ZEP_API_KEY未配置")
self.client = Zep(api_key=self.api_key)
self.api_key = api_key # kept for signature compat; no longer required
from .graphiti_service import get_graphiti_adapter
self.client = get_graphiti_adapter()
# 活动队列
self._activity_queue: Queue = Queue()
@ -411,10 +415,10 @@ class ZepGraphMemoryUpdater:
# 带重试的发送
for attempt in range(self.MAX_RETRIES):
try:
self.client.graph.add(
# Graphiti via adapter: one episode per batch (extraction is sync)
self.client.add_batch(
graph_id=self.graph_id,
type="text",
data=combined_text
episodes=[{"data": combined_text, "type": "text"}],
)
self._total_sent += 1

25
backend/app/services/zep_tools.py
View File

@ -13,7 +13,15 @@ import json
from typing import Dict, Any, List, Optional
from dataclasses import dataclass, field
from zep_cloud.client import Zep
try:
from zep_cloud.client import Zep # noqa: F811
except ImportError:
class Zep: # type: ignore[no-redef]
def __init__(self, *a, **kw): pass
class graph:
def search(self, **kw): raise NotImplementedError("zep-cloud not installed; use graphiti_service")
class node:
def get(self, **kw): raise NotImplementedError("zep-cloud not installed; use graphiti_service")
from ..config import Config
from ..utils.logger import get_logger
@ -423,11 +431,9 @@ class ZepToolsService:
RETRY_DELAY = 2.0
def __init__(self, api_key: Optional[str] = None, llm_client: Optional[LLMClient] = None):
self.api_key = api_key or Config.ZEP_API_KEY
if not self.api_key:
raise ValueError("ZEP_API_KEY 未配置")
self.client = Zep(api_key=self.api_key)
self.api_key = api_key # kept for signature compat; no longer required
from .graphiti_service import get_graphiti_adapter
self.client = get_graphiti_adapter()
# LLM客户端用于InsightForge生成子问题
self._llm_client = llm_client
logger.info(t("console.zepToolsInitialized"))
@ -485,15 +491,14 @@ class ZepToolsService:
"""
logger.info(t("console.graphSearch", graphId=graph_id, query=query[:50]))
# 尝试使用Zep Cloud Search API
# 尝试使用 Graphiti search
try:
search_results = self._call_with_retry(
func=lambda: self.client.graph.search(
func=lambda: self.client.search(
graph_id=graph_id,
query=query,
limit=limit,
scope=scope,
reranker="cross_encoder"
),
operation_name=t("console.graphSearchOp", graphId=graph_id)
)
@ -727,7 +732,7 @@ class ZepToolsService:
try:
node = self._call_with_retry(
func=lambda: self.client.graph.node.get(uuid_=node_uuid),
func=lambda: self.client.get_node(node_uuid=node_uuid),
operation_name=t("console.fetchNodeDetailOp", uuid=node_uuid[:8])
)

24
backend/app/utils/llm_client.py
View File

@ -71,16 +71,17 @@ class LLMClient:
self,
messages: List[Dict[str, str]],
temperature: float = 0.3,
max_tokens: int = 4096
max_tokens: int = 16384
) -> Dict[str, Any]:
"""
发送聊天请求并返回JSON
Args:
messages: 消息列表
temperature: 温度参数
max_tokens: 最大token数
max_tokens: 最大token数 (默认 16384 — MiniMax M3 在本体/配置文件
这类长 JSON 模式下,4096 tokens 经常会把 JSON 截断在中间)
Returns:
解析后的JSON对象
"""
@ -91,13 +92,24 @@ class LLMClient:
response_format={"type": "json_object"}
)
# 清理markdown代码块标记
# MiniMax M3 ignores response_format=json_object and wraps JSON in
# markdown fences. Be aggressive about stripping them, including the
# ```json\n and trailing ``` even when surrounded by other content.
cleaned_response = response.strip()
cleaned_response = re.sub(r'^```(?:json)?\s*\n?', '', cleaned_response, flags=re.IGNORECASE)
# Strip a leading ``` (with optional language tag and newline)
cleaned_response = re.sub(r'^```(?:json|JSON)?\s*\n?', '', cleaned_response, flags=re.IGNORECASE)
# Strip a trailing ``` (with optional preceding newline)
cleaned_response = re.sub(r'\n?```\s*$', '', cleaned_response)
# If there's still a ```json or ``` anywhere, take the first JSON-looking
# block from the response.
if '```' in cleaned_response:
m = re.search(r'(\{[\s\S]*\}|\[[\s\S]*\])', cleaned_response)
if m:
cleaned_response = m.group(1)
cleaned_response = cleaned_response.strip()
try:
return json.loads(cleaned_response)
except json.JSONDecodeError:
raise ValueError(f"LLM返回的JSON格式无效: {cleaned_response}")
raise ValueError(f"LLM返回的JSON格式无效: {cleaned_response[:200]}")

157
backend/app/utils/zep_paging.py
View File

@ -1,143 +1,46 @@
"""Zep Graph 分页读取工具。
"""Graph pagination helpers (replaces zep_paging.py).
Zep 的 node/edge 列表接口使用 UUID cursor 分页，
本模块封装自动翻页逻辑（含单页重试），对调用方透明地返回完整列表。
FalkorDB returns nodes/edges in a single query (we cap at 5000 in the adapter),
so there's no real pagination here — but the call sites in graph_builder.py and
zep_entity_reader.py import `fetch_all_nodes` / `fetch_all_edges`, so we keep
those names as adapters over GraphitiAdapter.
"""
from __future__ import annotations
import time
from collections.abc import Callable
from typing import Any
from zep_cloud import InternalServerError
from zep_cloud.client import Zep
import logging
from typing import Any, List
from .logger import get_logger
logger = get_logger('mirofish.zep_paging')
_DEFAULT_PAGE_SIZE = 100
_MAX_NODES = 2000
_DEFAULT_MAX_RETRIES = 3
_DEFAULT_RETRY_DELAY = 2.0 # seconds, doubles each retry
def _fetch_page_with_retry(
api_call: Callable[..., list[Any]],
*args: Any,
max_retries: int = _DEFAULT_MAX_RETRIES,
retry_delay: float = _DEFAULT_RETRY_DELAY,
page_description: str = "page",
**kwargs: Any,
) -> list[Any]:
"""单页请求，失败时指数退避重试。仅重试网络/IO类瞬态错误。"""
if max_retries < 1:
raise ValueError("max_retries must be >= 1")
last_exception: Exception | None = None
delay = retry_delay
for attempt in range(max_retries):
try:
return api_call(*args, **kwargs)
except (ConnectionError, TimeoutError, OSError, InternalServerError) as e:
last_exception = e
if attempt < max_retries - 1:
logger.warning(
f"Zep {page_description} attempt {attempt + 1} failed: {str(e)[:100]}, retrying in {delay:.1f}s..."
)
time.sleep(delay)
delay *= 2
else:
logger.error(f"Zep {page_description} failed after {max_retries} attempts: {str(e)}")
assert last_exception is not None
raise last_exception
def fetch_all_nodes(
client: Zep,
client: Any, # ignored — kept for signature compat with Zep call sites
graph_id: str,
page_size: int = _DEFAULT_PAGE_SIZE,
max_items: int = _MAX_NODES,
max_retries: int = _DEFAULT_MAX_RETRIES,
retry_delay: float = _DEFAULT_RETRY_DELAY,
) -> list[Any]:
"""分页获取图谱节点，最多返回 max_items 条（默认 2000）。每页请求自带重试。"""
all_nodes: list[Any] = []
cursor: str | None = None
page_num = 0
while True:
kwargs: dict[str, Any] = {"limit": page_size}
if cursor is not None:
kwargs["uuid_cursor"] = cursor
page_num += 1
batch = _fetch_page_with_retry(
client.graph.node.get_by_graph_id,
graph_id,
max_retries=max_retries,
retry_delay=retry_delay,
page_description=f"fetch nodes page {page_num} (graph={graph_id})",
**kwargs,
)
if not batch:
break
all_nodes.extend(batch)
if len(all_nodes) >= max_items:
all_nodes = all_nodes[:max_items]
logger.warning(f"Node count reached limit ({max_items}), stopping pagination for graph {graph_id}")
break
if len(batch) < page_size:
break
cursor = getattr(batch[-1], "uuid_", None) or getattr(batch[-1], "uuid", None)
if cursor is None:
logger.warning(f"Node missing uuid field, stopping pagination at {len(all_nodes)} nodes")
break
return all_nodes
page_size: int = 100, # ignored
max_items: int = 2000,
max_retries: int = 3, # ignored
retry_delay: float = 2.0, # ignored
) -> List[Any]:
"""Return all nodes in `graph_id` (FalkorDB capped at max_items, default 2000)."""
from ..services.graphiti_service import get_graphiti_adapter
adapter = get_graphiti_adapter()
nodes = adapter.get_all_nodes(graph_id)
if len(nodes) > max_items:
logger.warning(f"Node count {len(nodes)} > max_items {max_items}, truncating")
nodes = nodes[:max_items]
return nodes
def fetch_all_edges(
client: Zep,
client: Any,
graph_id: str,
page_size: int = _DEFAULT_PAGE_SIZE,
max_retries: int = _DEFAULT_MAX_RETRIES,
retry_delay: float = _DEFAULT_RETRY_DELAY,
) -> list[Any]:
"""分页获取图谱所有边，返回完整列表。每页请求自带重试。"""
all_edges: list[Any] = []
cursor: str | None = None
page_num = 0
while True:
kwargs: dict[str, Any] = {"limit": page_size}
if cursor is not None:
kwargs["uuid_cursor"] = cursor
page_num += 1
batch = _fetch_page_with_retry(
client.graph.edge.get_by_graph_id,
graph_id,
max_retries=max_retries,
retry_delay=retry_delay,
page_description=f"fetch edges page {page_num} (graph={graph_id})",
**kwargs,
)
if not batch:
break
all_edges.extend(batch)
if len(batch) < page_size:
break
cursor = getattr(batch[-1], "uuid_", None) or getattr(batch[-1], "uuid", None)
if cursor is None:
logger.warning(f"Edge missing uuid field, stopping pagination at {len(all_edges)} edges")
break
return all_edges
page_size: int = 100,
max_retries: int = 3,
retry_delay: float = 2.0,
) -> List[Any]:
"""Return all edges in `graph_id`."""
from ..services.graphiti_service import get_graphiti_adapter
adapter = get_graphiti_adapter()
return adapter.get_all_edges(graph_id, include_temporal=True)

30
backend/pyproject.toml
View File

@ -1,6 +1,6 @@
[project]
name = "mirofish-backend"
version = "0.1.0"
version = "0.2.0"
description = "MiroFish - 简洁通用的群体智能引擎，预测万物"
requires-python = ">=3.11,<3.13"
license = { text = "AGPL-3.0" }
@ -16,11 +16,24 @@ dependencies = [
# LLM 相关
"openai>=1.0.0",
# Zep Cloud
"zep-cloud==3.13.0",
# 知识图谱 (replaces zep-cloud)
# Graphiti is the open-source engine behind Zep; runs server-side extraction
# using your LLM provider (MiniMax M3) and stores the graph in FalkorDB.
"graphiti-core[falkordb]>=0.20.0",
# falkordb is the official Python client for the graph backend
"falkordb>=1.0.0",
# sentence-transformers is the local embedding provider we use (FalkorDB
# doesn't bundle one, and MiniMax's bespoke embeddings API is rate-limited
# and not OpenAI-compatible)
"sentence-transformers>=3.0.0",
"torch>=2.0.0",
# OASIS 社交媒体模拟
"camel-oasis==0.2.5",
# Vendored fork — see vendor/camel-oasis/pyproject.toml for the rationale
# (the upstream package pins neo4j==5.23.0 which conflicts with
# graphiti-core[falkordb]'s neo4j>=5.26.0; our fork relaxes the pin since
# MiroFish's OASIS scripts never invoke the neo4j driver).
"camel-oasis @ file:///app/backend/vendor/camel-oasis",
"camel-ai==0.2.78",
# 文件处理
@ -30,7 +43,10 @@ dependencies = [
"chardet>=5.0.0",
# 工具库
# 环境变量加载
"python-dotenv>=1.0.0",
# 数据验证
"pydantic>=2.0.0",
]
@ -53,3 +69,9 @@ dev = [
[tool.hatch.build.targets.wheel]
packages = ["app"]
# Required because the deps list includes a direct-reference local path dep
# (camel-oasis @ file:///app/backend/vendor/camel-oasis). Without this,
# hatchling refuses to build the editable.
[tool.hatch.metadata]
allow-direct-references = true

19
backend/requirements.txt
View File

@ -13,12 +13,23 @@ flask-cors>=6.0.0
# OpenAI SDK（统一使用 OpenAI 格式调用 LLM）
openai>=1.0.0
# ============= Zep Cloud =============
zep-cloud==3.13.0
# ============= Knowledge Graph (replaces zep-cloud) =============
# Graphiti is the open-source engine behind Zep; runs server-side extraction
# using your LLM provider (MiniMax M3) and stores the graph in FalkorDB.
graphiti-core[falkordb]>=0.20.0
falkordb>=1.0.0
# sentence-transformers is the local embedding provider we use (FalkorDB
# doesn't bundle one, and MiniMax's bespoke embeddings API is rate-limited
# and not OpenAI-compatible)
sentence-transformers>=3.0.0
torch>=2.0.0
# ============= OASIS 社交媒体模拟 =============
# OASIS 社交模拟框架
camel-oasis==0.2.5
# Vendored fork — see vendor/camel-oasis/pyproject.toml for the rationale
# (the upstream package pins neo4j==5.23.0 which conflicts with
# graphiti-core[falkordb]'s neo4j>=5.26.0; our fork relaxes the pin since
# MiroFish's OASIS scripts never invoke the neo4j driver).
camel-oasis @ file:///app/backend/vendor/camel-oasis
camel-ai==0.2.78
# ============= 文件处理 =============

2789
backend/uv.lock
View File

File diff suppressed because it is too large Load Diff

201
backend/vendor/camel-oasis/LICENSE vendored Normal file
View File

@ -0,0 +1,201 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright 2023 @ CAMEL-AI.org
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

13
backend/vendor/camel-oasis/LICENSES/license_template.txt vendored Normal file
View File

@ -0,0 +1,13 @@
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
# Licensed under the Apache License, Version 2.0 (the “License”);
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an “AS IS” BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========

126
backend/vendor/camel-oasis/LICENSES/update_license.py vendored Normal file
View File

@ -0,0 +1,126 @@
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
# Licensed under the Apache License, Version 2.0 (the “License”);
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an “AS IS” BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
import os
import re
import sys
from pathlib import Path
from typing import List
# The license template file is hard-coded with specific start and end lines
def fine_license_start_line(lines: List[str], start_with: str) -> int:
for i in range(len(lines)):
if lines[i].startswith(start_with):
return i
return None
def find_license_end_line(lines: List[str], start_with: str) -> int:
for i in range(len(lines) - 1, -1, -1):
if lines[i].startswith(start_with):
return i
return None
def update_license_in_file(
file_path: str,
license_template_path: str,
start_line_start_with: str,
end_line_start_with: str,
) -> bool:
with open(file_path, 'r',
encoding='utf-8') as f: # for windows compatibility
content = f.read()
with open(license_template_path, 'r', encoding='utf-8') as f:
new_license = f.read().strip()
maybe_existing_licenses = re.findall(r'^#.*?(?=\n)', content,
re.MULTILINE | re.DOTALL)
start_index = fine_license_start_line(maybe_existing_licenses,
start_line_start_with)
end_index = find_license_end_line(maybe_existing_licenses,
end_line_start_with)
if start_index is not None and end_index is not None:
maybe_existing_licenses = maybe_existing_licenses[
start_index:end_index + 1]
else:
maybe_existing_licenses = None
if maybe_existing_licenses:
maybe_old_licenses = '\n'.join(maybe_existing_licenses)
if maybe_old_licenses.strip() != new_license.strip():
replaced_content = content.replace(maybe_old_licenses, new_license)
with open(file_path, 'w') as f:
f.write(replaced_content)
print(f'Replaced license in {file_path}')
return True
else:
return False
else:
with open(file_path, 'w') as f:
f.write(new_license + '\n' + content)
print(f'Added license to {file_path}')
return True
def update_license_in_directory(
directory_path: str,
license_template_path: str,
start_line_start_with: str,
end_line_start_with: str,
) -> None:
# Check if directory exists
if not os.path.isdir(directory_path):
raise NotADirectoryError(f'{directory_path} is not a directory')
# Check if license template exists
if not os.path.isfile(license_template_path):
raise FileNotFoundError(f'{license_template_path} not found')
file_count = 0
for py_files in Path(directory_path).rglob("*.py"):
if py_files.name.startswith('.'):
continue
if any(part.startswith('.') for part in py_files.parts):
continue
if update_license_in_file(
py_files,
license_template_path,
start_line_start_with,
end_line_start_with,
):
file_count += 1
print(f'License updated in {file_count} files')
if __name__ == '__main__':
if len(sys.argv) < 3:
print(
"Usage from command line: "
"python update_license.py <directory_path> <license_template_path>"
"No valid input arguments found, please enter manually.")
directory_path = input("Enter directory path: ")
license_template_path = input("Enter license template path: ")
else:
directory_path = sys.argv[1]
license_template_path = sys.argv[2]
start_line_start_with = "# =========== Copyright"
end_line_start_with = "# =========== Copyright"
update_license_in_directory(
directory_path,
license_template_path,
start_line_start_with,
end_line_start_with,
)

337
backend/vendor/camel-oasis/README.md vendored Normal file
View File

@ -0,0 +1,337 @@
<div align="center">
<a href="https://www.camel-ai.org/">
<img src="assets/banner.png" alt=banner>
</a>
</div>
</br>
<div align="center">
<h1> OASIS: Open Agent Social Interaction Simulations with One Million Agents
</h1>
[![Documentation][docs-image]][docs-url]
[![Discord][discord-image]][discord-url]
[![X][x-image]][x-url]
[![Reddit][reddit-image]][reddit-url]
[![Wechat][wechat-image]][wechat-url]
[![Wechat][oasis-image]][oasis-url]
[![Hugging Face][huggingface-image]][huggingface-url]
[![Star][star-image]][star-url]
[![Package License][package-license-image]][package-license-url]
<h4 align="center">
[Community](https://github.com/camel-ai/camel#community) |
[Paper](https://arxiv.org/abs/2411.11581) |
[Examples](https://github.com/camel-ai/oasis/tree/main/scripts) |
[Dataset](https://huggingface.co/datasets/echo-yiyiyi/oasis-dataset) |
[Citation](https://github.com/camel-ai/oasis#-citation) |
[Contributing](https://github.com/camel-ai/oasis#-contributing-to-oasis) |
[CAMEL-AI](https://www.camel-ai.org/)
</h4>
</div>
<br>
<p align="left">
<img src='assets/intro.png'>
🏝️ OASIS is a scalable, open-source social media simulator that incorporates large language model agents to realistically mimic the behavior of up to one million users on platforms like Twitter and Reddit. It's designed to facilitate the study of complex social phenomena such as information spread, group polarization, and herd behavior, offering a versatile tool for exploring diverse social dynamics and user interactions in digital environments.
</p>
<br>
<div align="center">
🌟 Star OASIS on GitHub and be instantly notified of new releases.
</div>
<br>
<div align="center">
<img src="assets/star.gif" alt="Star" width="196" height="52">
</a>
</div>
<br>
## ✨ Key Features
### 📈 Scalability
OASIS supports simulations of up to ***one million agents***, enabling studies of social media dynamics at a scale comparable to real-world platforms.
### 📲 Dynamic Environments
Adapts to real-time changes in social networks and content, mirroring the fluid dynamics of platforms like **Twitter** and **Reddit** for authentic simulation experiences.
### 👍🏼 Diverse Action Spaces
Agents can perform **23 actions**, such as following, commenting, and reposting, allowing for rich, multi-faceted interactions.
### 🔥 Integrated Recommendation Systems
Features **interest-based** and **hot-score-based recommendation algorithms**, simulating how users discover content and interact within social media platforms.
<br>
## 📺 Demo Video
### Introducing OASIS: Open Agent Social Interaction Simulations with One Million Agents
https://github.com/user-attachments/assets/3bd2553c-d25d-4d8c-a739-1af51354b15a
<br>
For more showcaes:
- Can 1,000,000 AI agents simulate social media?
[→Watch demo](https://www.youtube.com/watch?v=lprGHqkApus&t=2s)
<br>
## 🎯 Usecase
<div align="left">
<img src="assets/research_simulation.png" alt=usecase1>
<img src="assets/interaction.png" alt=usecase2>
<a href="http://www.matrix.eigent.ai">
<img src="assets/content_creation.png" alt=usecase3>
</a>
<img src="assets/prediction.png" alt=usecase4>
</div>
## ⚙️ Quick Start
1. **Install the OASIS package:**
Installing OASIS is a breeze thanks to its availability on PyPI. Simply open your terminal and run:
```bash
pip install camel-oasis
```
2. **Set up your OpenAI API key:**
```bash
# For Bash shell (Linux, macOS, Git Bash on Windows):
export OPENAI_API_KEY=<insert your OpenAI API key>
# For Windows Command Prompt:
set OPENAI_API_KEY=<insert your OpenAI API key>
```
3. **Prepare the agent profile file:**
Create the profile you want to assign to the agent. As an example, you can download [user_data_36.json](https://github.com/camel-ai/oasis/blob/main/data/reddit/user_data_36.json) and place it in your local `./data/reddit` folder.
4. **Run the following Python code:**
```python
import asyncio
import os
from camel.models import ModelFactory
from camel.types import ModelPlatformType, ModelType
import oasis
from oasis import (ActionType, LLMAction, ManualAction,
generate_reddit_agent_graph)
async def main():
# Define the model for the agents
openai_model = ModelFactory.create(
model_platform=ModelPlatformType.OPENAI,
model_type=ModelType.GPT_4O_MINI,
)
# Define the available actions for the agents
available_actions = [
ActionType.LIKE_POST,
ActionType.DISLIKE_POST,
ActionType.CREATE_POST,
ActionType.CREATE_COMMENT,
ActionType.LIKE_COMMENT,
ActionType.DISLIKE_COMMENT,
ActionType.SEARCH_POSTS,
ActionType.SEARCH_USER,
ActionType.TREND,
ActionType.REFRESH,
ActionType.DO_NOTHING,
ActionType.FOLLOW,
ActionType.MUTE,
]
agent_graph = await generate_reddit_agent_graph(
profile_path="./data/reddit/user_data_36.json",
model=openai_model,
available_actions=available_actions,
)
# Define the path to the database
db_path = "./data/reddit_simulation.db"
# Delete the old database
if os.path.exists(db_path):
os.remove(db_path)
# Make the environment
env = oasis.make(
agent_graph=agent_graph,
platform=oasis.DefaultPlatformType.REDDIT,
database_path=db_path,
)
# Run the environment
await env.reset()
actions_1 = {}
actions_1[env.agent_graph.get_agent(0)] = [
ManualAction(action_type=ActionType.CREATE_POST,
action_args={"content": "Hello, world!"}),
ManualAction(action_type=ActionType.CREATE_COMMENT,
action_args={
"post_id": "1",
"content": "Welcome to the OASIS World!"
})
]
actions_1[env.agent_graph.get_agent(1)] = ManualAction(
action_type=ActionType.CREATE_COMMENT,
action_args={
"post_id": "1",
"content": "I like the OASIS world."
})
await env.step(actions_1)
actions_2 = {
agent: LLMAction()
for _, agent in env.agent_graph.get_agents()
}
# Perform the actions
await env.step(actions_2)
# Close the environment
await env.close()
if __name__ == "__main__":
asyncio.run(main())
```
<br>
> \[!TIP\]
> For more detailed instructions and additional configuration options, check out the [documentation](https://docs.oasis.camel-ai.org/).
### More Tutorials
To discover how to create profiles for large-scale users, as well as how to visualize and analyze social simulation data once your experiment concludes, please refer to [More Tutorials](examples/experiment/user_generation_visualization.md) for detailed guidance.
<div align="center">
<img src="assets/tutorial.png" alt="Tutorial Overview">
</div>
## 📢 News
### Upcoming Features & Contributions
> We welcome community contributions! Join us in building these exciting features.
- [Support Multi Modal Platform](https://github.com/camel-ai/oasis/issues/47)
<!-- - Public release of our dataset on Hugging Face (November 05, 2024) -->
### Latest Updates
📢 Update the camel-ai version to 0.2.78 and update the dataset HuggingFace link. - 📆 December 4, 2025
- Add the report post action to mark inappropriate content. - 📆 June 8, 2025
- Add features for creating group chats, sending messages in group chats, and leaving group chats. - 📆 June 6, 2025
- Support Interview Action for asking agents specific questions and getting answers. - 📆 June 2, 2025
- Support customization of each agent's models, tools, and prompts; refactor the interface to follow the PettingZoo style. - 📆 May 22, 2025
- Refactor into the OASIS environment, publish camel-oasis on PyPI, and release the documentation. - 📆 April 24, 2025
- Support OPENAI Embedding model for Twhin-Bert Recommendation System. - 📆 March 25, 2025
...
- Slightly refactoring the database to add Quote Action and modify Repost Action - 📆 January 13, 2025
- Added the demo video and oasis's star history in the README - 📆 January 5, 2025
- Introduced an Electronic Mall on the Reddit platform - 📆 December 5, 2024
- OASIS initially released on arXiv - 📆 November 19, 2024
- OASIS GitHub repository initially launched - 📆 November 19, 2024
## 🔎 Follow-up Research
- [MultiAgent4Collusion](https://github.com/renqibing/MultiAgent4Collusion): multi-agent collusion simulation framework in social systems
- More to come...
If your research is based on OASIS, we'd be happy to feature your work here—feel free to reach out or submit a pull request to add it to the [README](https://github.com/camel-ai/oasis/blob/main/README.md)!
## 🥂 Contributing to OASIS🏝️
> We greatly appreciate your interest in contributing to our open-source initiative. To ensure a smooth collaboration and the success of contributions, we adhere to a set of contributing guidelines similar to those established by CAMEL. For a comprehensive understanding of the steps involved in contributing to our project, please refer to the OASIS [contributing guidelines](https://github.com/camel-ai/oasis/blob/master/CONTRIBUTING.md). 🤝🚀
>
> An essential part of contributing involves not only submitting new features with accompanying tests (and, ideally, examples) but also ensuring that these contributions pass our automated pytest suite. This approach helps us maintain the project's quality and reliability by verifying compatibility and functionality.
## 📬 Community & Contact
If you're keen on exploring new research opportunities or discoveries with our platform and wish to dive deeper or suggest new features, we're here to talk. Feel free to get in touch for more details at camel.ai.team@gmail.com.
<br>
- Join us ([*Discord*](https://discord.camel-ai.org/) or [*WeChat*](https://ghli.org/camel/wechat.png)) in pushing the boundaries of finding the scaling laws of agents.
- Join WechatGroup for further discussions!
<div align="">
<img src="assets/wechatgroup.png" alt="WeChat Group QR Code" width="600">
</div>
## 🌟 Star History
[![Star History Chart](https://api.star-history.com/svg?repos=camel-ai/oasis&type=Date)](https://star-history.com/#camel-ai/oasis&Date)
## 🔗 Citation
```
@misc{yang2024oasisopenagentsocial,
title={OASIS: Open Agent Social Interaction Simulations with One Million Agents},
author={Ziyi Yang and Zaibin Zhang and Zirui Zheng and Yuxian Jiang and Ziyue Gan and Zhiyu Wang and Zijian Ling and Jinsong Chen and Martz Ma and Bowen Dong and Prateek Gupta and Shuyue Hu and Zhenfei Yin and Guohao Li and Xu Jia and Lijun Wang and Bernard Ghanem and Huchuan Lu and Chaochao Lu and Wanli Ouyang and Yu Qiao and Philip Torr and Jing Shao},
year={2024},
eprint={2411.11581},
archivePrefix={arXiv},
primaryClass={cs.CL},
url={https://arxiv.org/abs/2411.11581},
}
```
## 🙌 Acknowledgment
We would like to thank Douglas for designing the logo of our project.
## 🖺 License
The source code is licensed under Apache 2.0.
[discord-image]: https://img.shields.io/discord/1082486657678311454?logo=discord&labelColor=%20%235462eb&logoColor=%20%23f5f5f5&color=%20%235462eb
[discord-url]: https://discord.camel-ai.org/
[docs-image]: https://img.shields.io/badge/Documentation-EB3ECC
[docs-url]: https://docs.oasis.camel-ai.org/
[huggingface-image]: https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-CAMEL--AI-ffc107?color=ffc107&logoColor=white
[huggingface-url]: https://huggingface.co/camel-ai
[oasis-image]: https://img.shields.io/badge/WeChat-OASISProject-brightgreen?logo=wechat&logoColor=white
[oasis-url]: ./assets/wechatgroup.png
[package-license-image]: https://img.shields.io/badge/License-Apache_2.0-blue.svg
[package-license-url]: https://github.com/camel-ai/oasis/blob/main/licenses/LICENSE
[reddit-image]: https://img.shields.io/reddit/subreddit-subscribers/CamelAI?style=plastic&logo=reddit&label=r%2FCAMEL&labelColor=white
[reddit-url]: https://www.reddit.com/r/CamelAI/
[star-image]: https://img.shields.io/github/stars/camel-ai/oasis?label=stars&logo=github&color=brightgreen
[star-url]: https://github.com/camel-ai/oasis/stargazers
[wechat-image]: https://img.shields.io/badge/WeChat-CamelAIOrg-brightgreen?logo=wechat&logoColor=white
[wechat-url]: ./assets/wechat.JPGwechat.jpg
[x-image]: https://img.shields.io/twitter/follow/CamelAIOrg?style=social
[x-url]: https://x.com/CamelAIOrg

31
backend/vendor/camel-oasis/oasis/__init__.py vendored Normal file
View File

@ -0,0 +1,31 @@
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
# Licensed under the Apache License, Version 2.0 (the “License”);
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an “AS IS” BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
__version__ = "0.2.5"
from oasis.environment.env_action import LLMAction, ManualAction
from oasis.environment.make import make
from oasis.social_agent import (generate_reddit_agent_graph,
generate_twitter_agent_graph)
from oasis.social_agent.agent import SocialAgent
from oasis.social_agent.agent_graph import AgentGraph
from oasis.social_platform.config import UserInfo
from oasis.social_platform.platform import Platform
from oasis.social_platform.typing import ActionType, DefaultPlatformType
from oasis.testing.show_db import print_db_contents
__all__ = [
"make", "Platform", "ActionType", "DefaultPlatformType", "ManualAction",
"LLMAction", "print_db_contents", "AgentGraph", "SocialAgent", "UserInfo",
"generate_reddit_agent_graph", "generate_twitter_agent_graph"
]

16
backend/vendor/camel-oasis/oasis/clock/__init__.py vendored Normal file
View File

@ -0,0 +1,16 @@
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
# Licensed under the Apache License, Version 2.0 (the “License”);
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an “AS IS” BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
from .clock import Clock
__all__ = ["Clock"]

33
backend/vendor/camel-oasis/oasis/clock/clock.py vendored Normal file
View File

@ -0,0 +1,33 @@
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
# Licensed under the Apache License, Version 2.0 (the “License”);
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an “AS IS” BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
from datetime import datetime
class Clock:
r"""Clock used for the sandbox."""
def __init__(self, k: int = 1):
self.real_start_time = datetime.now()
self.k = k
self.time_step = 0
def time_transfer(self, now_time: datetime,
start_time: datetime) -> datetime:
time_diff = now_time - self.real_start_time
adjusted_diff = self.k * time_diff
adjusted_time = start_time + adjusted_diff
return adjusted_time
def get_time_step(self) -> str:
return str(self.time_step)

13
backend/vendor/camel-oasis/oasis/environment/__init__.py vendored Normal file
View File

@ -0,0 +1,13 @@
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
# Licensed under the Apache License, Version 2.0 (the “License”);
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an “AS IS” BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========

208
backend/vendor/camel-oasis/oasis/environment/env.py vendored Normal file
View File

@ -0,0 +1,208 @@
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
# Licensed under the Apache License, Version 2.0 (the “License”);
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an “AS IS” BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
import asyncio
import logging
import os
from datetime import datetime
from typing import List, Union
from oasis.environment.env_action import LLMAction, ManualAction
from oasis.social_agent.agent import SocialAgent
from oasis.social_agent.agent_graph import AgentGraph
from oasis.social_agent.agents_generator import generate_custom_agents
from oasis.social_platform.channel import Channel
from oasis.social_platform.platform import Platform
from oasis.social_platform.typing import (ActionType, DefaultPlatformType,
RecsysType)
# Create log directory if it doesn't exist
log_dir = "./log"
if not os.path.exists(log_dir):
os.makedirs(log_dir)
# Configure logger
env_log = logging.getLogger("oasis.env")
env_log.setLevel("INFO")
# Add file handler to save logs to file
current_time = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
file_handler = logging.FileHandler(f"{log_dir}/oasis-{current_time}.log",
encoding="utf-8")
file_handler.setLevel("INFO")
file_handler.setFormatter(
logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s"))
env_log.addHandler(file_handler)
class OasisEnv:
def __init__(
self,
agent_graph: AgentGraph,
platform: Union[DefaultPlatformType, Platform],
database_path: str = None,
semaphore: int = 128,
) -> None:
r"""Init the oasis environment.
Args:
agent_graph: The AgentGraph to use in the simulation.
platform: The platform type to use. Including
`DefaultPlatformType.TWITTER` or `DefaultPlatformType.REDDIT`.
Or you can pass a custom `Platform` instance.
database_path: The path to create a sqlite3 database. The file
extension must be `.db` such as `twitter_simulation.db`.
"""
# Initialize the agent graph
self.agent_graph = agent_graph
# Use a semaphore to limit the number of concurrent requests
self.llm_semaphore = asyncio.Semaphore(semaphore)
if isinstance(platform, DefaultPlatformType):
if database_path is None:
raise ValueError(
"database_path is required for DefaultPlatformType")
self.platform = platform
if platform == DefaultPlatformType.TWITTER:
self.channel = Channel()
self.platform = Platform(
db_path=database_path,
channel=self.channel,
recsys_type="twhin-bert",
refresh_rec_post_count=2,
max_rec_post_len=2,
following_post_count=3,
)
self.platform_type = DefaultPlatformType.TWITTER
elif platform == DefaultPlatformType.REDDIT:
self.channel = Channel()
self.platform = Platform(
db_path=database_path,
channel=self.channel,
recsys_type="reddit",
allow_self_rating=True,
show_score=True,
max_rec_post_len=100,
refresh_rec_post_count=5,
)
self.platform_type = DefaultPlatformType.REDDIT
else:
raise ValueError(f"Invalid platform: {platform}. Only "
"DefaultPlatformType.TWITTER or "
"DefaultPlatformType.REDDIT are supported.")
elif isinstance(platform, Platform):
if database_path != platform.db_path:
env_log.warning("database_path is not the same as the "
"platform.db_path, using the platform.db_path")
self.platform = platform
self.channel = platform.channel
if platform.recsys_type == RecsysType.REDDIT:
self.platform_type = DefaultPlatformType.REDDIT
else:
self.platform_type = DefaultPlatformType.TWITTER
else:
raise ValueError(
f"Invalid platform: {platform}. You should pass a "
"DefaultPlatformType or a Platform instance.")
async def reset(self) -> None:
r"""Start the platform and sign up the agents."""
self.platform_task = asyncio.create_task(self.platform.running())
self.agent_graph = await generate_custom_agents(
channel=self.channel, agent_graph=self.agent_graph)
async def _perform_llm_action(self, agent):
r"""Send the request to the llm model and execute the action.
"""
async with self.llm_semaphore:
return await agent.perform_action_by_llm()
async def _perform_interview_action(self, agent, interview_prompt: str):
r"""Send the request to the llm model and execute the interview.
"""
async with self.llm_semaphore:
return await agent.perform_interview(interview_prompt)
async def step(
self, actions: dict[SocialAgent, Union[ManualAction, LLMAction,
List[Union[ManualAction,
LLMAction]]]]
) -> None:
r"""Update the recommendation system and perform the actions.
Args:
actions(dict[SocialAgent, Union[ManualAction, LLMAction,
List[Union[ManualAction, LLMAction]]]]): The actions to
perform, including the manual(pre-defined) actions and llm
actions.
Returns:
None
"""
# Update the recommendation system
await self.platform.update_rec_table()
env_log.info("update rec table.")
# Create tasks for both manual and LLM actions
tasks = []
for agent, action in actions.items():
if isinstance(action, list):
for single_action in action:
if isinstance(single_action, ManualAction):
if single_action.action_type == ActionType.INTERVIEW:
# Use the agent's perform_interview method for
# interview actions
interview_prompt = single_action.action_args.get(
"prompt", "")
tasks.append(
self._perform_interview_action(
agent, interview_prompt))
else:
tasks.append(
agent.perform_action_by_data(
single_action.action_type,
**single_action.action_args))
elif isinstance(single_action, LLMAction):
tasks.append(self._perform_llm_action(agent))
else:
if isinstance(action, ManualAction):
if action.action_type == ActionType.INTERVIEW:
# Use the agent's perform_interview method for
# interview actions
interview_prompt = action.action_args.get("prompt", "")
tasks.append(
self._perform_interview_action(
agent, interview_prompt))
else:
tasks.append(
agent.perform_action_by_data(
action.action_type, **action.action_args))
elif isinstance(action, LLMAction):
tasks.append(self._perform_llm_action(agent))
# Execute all tasks concurrently
await asyncio.gather(*tasks)
env_log.info("performed all actions.")
# # Control some agents to perform actions
# Update the clock
if self.platform_type == DefaultPlatformType.TWITTER:
self.platform.sandbox_clock.time_step += 1
async def close(self) -> None:
r"""Stop the platform and close the environment.
"""
await self.channel.write_to_receive_queue(
(None, None, ActionType.EXIT))
await self.platform_task
env_log.info("Simulation finished! Please check the results in the "
f"database: {self.platform.db_path}. Note that the trace "
"table stored all the actions of the agents.")

45
backend/vendor/camel-oasis/oasis/environment/env_action.py vendored Normal file
View File

@ -0,0 +1,45 @@
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
# Licensed under the Apache License, Version 2.0 (the “License”);
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an “AS IS” BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
from dataclasses import dataclass
from typing import Any, Dict
from oasis.social_platform.typing import ActionType
@dataclass
class ManualAction:
r"""Some manual predefined social platform actions that need to be
executed by certain agents.
Args:
agent_id: The ID of the agent that will perform the action.
action: The action to perform.
args: The arguments to pass to the action. For details of each args in
each action, please refer to
`https://github.com/camel-ai/oasis/blob/main/oasis/social_agent/agent_action.py`.
"""
action_type: ActionType
action_args: Dict[str, Any]
def init(self, action_type, action_args):
self.action_type = action_type
self.action_args = action_args
@dataclass
class LLMAction:
r"""Represents actions generated by a Language Learning Model (LLM)."""
def init(self):
pass

19
backend/vendor/camel-oasis/oasis/environment/make.py vendored Normal file
View File

@ -0,0 +1,19 @@
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
# Licensed under the Apache License, Version 2.0 (the “License”);
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an “AS IS” BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
from oasis.environment.env import OasisEnv
def make(*args, **kwargs):
obj = OasisEnv(*args, **kwargs)
return obj

23
backend/vendor/camel-oasis/oasis/social_agent/__init__.py vendored Normal file
View File

@ -0,0 +1,23 @@
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
# Licensed under the Apache License, Version 2.0 (the “License”);
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an “AS IS” BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
from .agent import SocialAgent
from .agent_graph import AgentGraph
from .agents_generator import (generate_agents_100w,
generate_reddit_agent_graph,
generate_twitter_agent_graph)
__all__ = [
"SocialAgent", "AgentGraph", "generate_agents_100w",
"generate_reddit_agent_graph", "generate_twitter_agent_graph"
]

321
backend/vendor/camel-oasis/oasis/social_agent/agent.py vendored Normal file
View File

@ -0,0 +1,321 @@
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
# Licensed under the Apache License, Version 2.0 (the “License”);
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an “AS IS” BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
from __future__ import annotations
import inspect
import logging
import sys
from datetime import datetime
from typing import TYPE_CHECKING, Any, Callable, List, Optional, Union
from camel.agents import ChatAgent
from camel.messages import BaseMessage
from camel.models import BaseModelBackend, ModelManager
from camel.prompts import TextPrompt
from camel.toolkits import FunctionTool
from camel.types import OpenAIBackendRole
from oasis.social_agent.agent_action import SocialAction
from oasis.social_agent.agent_environment import SocialEnvironment
from oasis.social_platform import Channel
from oasis.social_platform.config import UserInfo
from oasis.social_platform.typing import ActionType
if TYPE_CHECKING:
from oasis.social_agent import AgentGraph
if "sphinx" not in sys.modules:
agent_log = logging.getLogger(name="social.agent")
agent_log.setLevel("DEBUG")
if not agent_log.handlers:
now = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
file_handler = logging.FileHandler(
f"./log/social.agent-{str(now)}.log")
file_handler.setLevel("DEBUG")
file_handler.setFormatter(
logging.Formatter(
"%(levelname)s - %(asctime)s - %(name)s - %(message)s"))
agent_log.addHandler(file_handler)
ALL_SOCIAL_ACTIONS = [action.value for action in ActionType]
class SocialAgent(ChatAgent):
r"""Social Agent."""
def __init__(self,
agent_id: int,
user_info: UserInfo,
user_info_template: TextPrompt | None = None,
channel: Channel | None = None,
model: Optional[Union[BaseModelBackend,
List[BaseModelBackend],
ModelManager]] = None,
agent_graph: "AgentGraph" = None,
available_actions: list[ActionType] = None,
tools: Optional[List[Union[FunctionTool, Callable]]] = None,
max_iteration: int = 1,
interview_record: bool = False):
self.social_agent_id = agent_id
self.user_info = user_info
self.channel = channel or Channel()
self.env = SocialEnvironment(SocialAction(agent_id, self.channel))
if user_info_template is None:
system_message_content = self.user_info.to_system_message()
else:
system_message_content = self.user_info.to_custom_system_message(
user_info_template)
system_message = BaseMessage.make_assistant_message(
role_name="system",
content=system_message_content, # system prompt
)
if not available_actions:
agent_log.info("No available actions defined, using all actions.")
self.action_tools = self.env.action.get_openai_function_list()
else:
all_tools = self.env.action.get_openai_function_list()
all_possible_actions = [tool.func.__name__ for tool in all_tools]
for action in available_actions:
action_name = action.value if isinstance(
action, ActionType) else action
if action_name not in all_possible_actions:
agent_log.warning(
f"Action {action_name} is not supported. Supported "
f"actions are: {', '.join(all_possible_actions)}")
self.action_tools = [
tool for tool in all_tools if tool.func.__name__ in [
a.value if isinstance(a, ActionType) else a
for a in available_actions
]
]
all_tools = (tools or []) + (self.action_tools or [])
super().__init__(
system_message=system_message,
model=model,
scheduling_strategy='random_model',
tools=all_tools,
)
self.max_iteration = max_iteration
self.interview_record = interview_record
self.agent_graph = agent_graph
self.test_prompt = (
"\n"
"Helen is a successful writer who usually writes popular western "
"novels. Now, she has an idea for a new novel that could really "
"make a big impact. If it works out, it could greatly "
"improve her career. But if it fails, she will have spent "
"a lot of time and effort for nothing.\n"
"\n"
"What do you think Helen should do?")
async def perform_action_by_llm(self):
# Get posts:
env_prompt = await self.env.to_text_prompt()
user_msg = BaseMessage.make_user_message(
role_name="User",
content=(
f"Please perform social media actions after observing the "
f"platform environments. Notice that don't limit your "
f"actions for example to just like the posts. "
f"Here is your social media environment: {env_prompt}"))
try:
agent_log.info(
f"Agent {self.social_agent_id} observing environment: "
f"{env_prompt}")
response = await self.astep(user_msg)
for tool_call in response.info['tool_calls']:
action_name = tool_call.tool_name
args = tool_call.args
agent_log.info(f"Agent {self.social_agent_id} performed "
f"action: {action_name} with args: {args}")
if action_name not in ALL_SOCIAL_ACTIONS:
agent_log.info(
f"Agent {self.social_agent_id} get the result: "
f"{tool_call.result}")
# Abort graph action for if 100w Agent
# self.perform_agent_graph_action(action_name, args)
return response
except Exception as e:
agent_log.error(f"Agent {self.social_agent_id} error: {e}")
return e
async def perform_test(self):
"""
doing group polarization test for all agents.
TODO: rewrite the function according to the ChatAgent.
TODO: unify the test and interview function.
"""
# user conduct test to agent
_ = BaseMessage.make_user_message(role_name="User",
content=("You are a twitter user."))
# Test memory should not be writed to memory.
# self.memory.write_record(MemoryRecord(user_msg,
# OpenAIBackendRole.USER))
openai_messages, num_tokens = self.memory.get_context()
openai_messages = ([{
"role":
self.system_message.role_name,
"content":
self.system_message.content.split("# RESPONSE FORMAT")[0],
}] + openai_messages + [{
"role": "user",
"content": self.test_prompt
}])
agent_log.info(f"Agent {self.social_agent_id}: {openai_messages}")
# NOTE: this is a temporary solution.
# Camel can not stop updating the agents' memory after stop and astep
# now.
response = await self._aget_model_response(
openai_messages=openai_messages, num_tokens=num_tokens)
content = response.output_messages[0].content
agent_log.info(
f"Agent {self.social_agent_id} receive response: {content}")
return {
"user_id": self.social_agent_id,
"prompt": openai_messages,
"content": content
}
async def perform_interview(self, interview_prompt: str):
"""
Perform an interview with the agent.
"""
# user conduct test to agent
user_msg = BaseMessage.make_user_message(
role_name="User", content=("You are a twitter user."))
if self.interview_record:
# Test memory should not be writed to memory.
self.update_memory(message=user_msg, role=OpenAIBackendRole.SYSTEM)
openai_messages, num_tokens = self.memory.get_context()
openai_messages = ([{
"role":
self.system_message.role_name,
"content":
self.system_message.content.split("# RESPONSE FORMAT")[0],
}] + openai_messages + [{
"role": "user",
"content": interview_prompt
}])
agent_log.info(f"Agent {self.social_agent_id}: {openai_messages}")
# NOTE: this is a temporary solution.
# Camel can not stop updating the agents' memory after stop and astep
# now.
response = await self._aget_model_response(
openai_messages=openai_messages, num_tokens=num_tokens)
content = response.output_messages[0].content
if self.interview_record:
# Test memory should not be writed to memory.
self.update_memory(message=response.output_messages[0],
role=OpenAIBackendRole.USER)
agent_log.info(
f"Agent {self.social_agent_id} receive response: {content}")
# Record the complete interview (prompt + response) through the channel
interview_data = {"prompt": interview_prompt, "response": content}
result = await self.env.action.perform_action(
interview_data, ActionType.INTERVIEW.value)
# Return the combined result
return {
"user_id": self.social_agent_id,
"prompt": openai_messages,
"content": content,
"success": result.get("success", False)
}
async def perform_action_by_hci(self) -> Any:
print("Please choose one function to perform:")
function_list = self.env.action.get_openai_function_list()
for i in range(len(function_list)):
agent_log.info(f"Agent {self.social_agent_id} function: "
f"{function_list[i].func.__name__}")
selection = int(input("Enter your choice: "))
if not 0 <= selection < len(function_list):
agent_log.error(f"Agent {self.social_agent_id} invalid input.")
return
func = function_list[selection].func
params = inspect.signature(func).parameters
args = []
for param in params.values():
while True:
try:
value = input(f"Enter value for {param.name}: ")
args.append(value)
break
except ValueError:
agent_log.error("Invalid input, please enter an integer.")
result = await func(*args)
return result
async def perform_action_by_data(self, func_name, *args, **kwargs) -> Any:
func_name = func_name.value if isinstance(func_name,
ActionType) else func_name
function_list = self.env.action.get_openai_function_list()
for i in range(len(function_list)):
if function_list[i].func.__name__ == func_name:
func = function_list[i].func
result = await func(*args, **kwargs)
self.update_memory(message=BaseMessage.make_user_message(
role_name=OpenAIBackendRole.SYSTEM,
content=f"Agent {self.social_agent_id} performed "
f"{func_name} with args: {args} and kwargs: {kwargs}"
f"and the result is {result}"),
role=OpenAIBackendRole.SYSTEM)
agent_log.info(f"Agent {self.social_agent_id}: {result}")
return result
raise ValueError(f"Function {func_name} not found in the list.")
def perform_agent_graph_action(
self,
action_name: str,
arguments: dict[str, Any],
):
r"""Remove edge if action is unfollow or add edge
if action is follow to the agent graph.
"""
if "unfollow" in action_name:
followee_id: int | None = arguments.get("followee_id", None)
if followee_id is None:
return
self.agent_graph.remove_edge(self.social_agent_id, followee_id)
agent_log.info(
f"Agent {self.social_agent_id} unfollowed Agent {followee_id}")
elif "follow" in action_name:
followee_id: int | None = arguments.get("followee_id", None)
if followee_id is None:
return
self.agent_graph.add_edge(self.social_agent_id, followee_id)
agent_log.info(
f"Agent {self.social_agent_id} followed Agent {followee_id}")
def __str__(self) -> str:
return (f"{self.__class__.__name__}(agent_id={self.social_agent_id}, "
f"model_type={self.model_type.value})")

758
backend/vendor/camel-oasis/oasis/social_agent/agent_action.py vendored Normal file
View File

@ -0,0 +1,758 @@
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
# Licensed under the Apache License, Version 2.0 (the “License”);
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an “AS IS” BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
from typing import Any
from camel.toolkits import FunctionTool
from oasis.social_platform.channel import Channel
from oasis.social_platform.typing import ActionType
class SocialAction:
def __init__(self, agent_id: int, channel: Channel):
self.agent_id = agent_id
self.channel = channel
def get_openai_function_list(self) -> list[FunctionTool]:
return [
FunctionTool(func) for func in [
self.create_post,
self.like_post,
self.repost,
self.quote_post,
self.unlike_post,
self.dislike_post,
self.undo_dislike_post,
self.search_posts,
self.search_user,
self.trend,
self.refresh,
self.do_nothing,
self.create_comment,
self.like_comment,
self.dislike_comment,
self.unlike_comment,
self.undo_dislike_comment,
self.follow,
self.unfollow,
self.mute,
self.unmute,
self.purchase_product,
self.interview,
self.report_post,
self.join_group,
self.leave_group,
self.send_to_group,
self.create_group,
self.listen_from_group,
]
]
async def perform_action(self, message: Any, type: str):
message_id = await self.channel.write_to_receive_queue(
(self.agent_id, message, type))
response = await self.channel.read_from_send_queue(message_id)
return response[2]
async def sign_up(self, user_name: str, name: str, bio: str):
r"""Signs up a new user with the provided username, name, and bio.
This method prepares a user message comprising the user's details and
invokes an asynchronous action to perform the sign-up process. On
successful execution, it returns a dictionary indicating success along
with the newly created user ID.
Args:
user_name (str): The username for the new user.
name (str): The full name of the new user.
bio (str): A brief biography of the new user.
Returns:
dict: A dictionary with two key-value pairs. The 'success' key
maps to a boolean indicating whether the sign-up was
successful, and 'user_id' key maps to the integer ID of the
newly created user on success.
Example of a successful return:
{'success': True, 'user_id': 2}
"""
# print(f"Agent {self.agent_id} is signing up with "
# f"user_name: {user_name}, name: {name}, bio: {bio}")
user_message = (user_name, name, bio)
return await self.perform_action(user_message, ActionType.SIGNUP.value)
async def refresh(self):
r"""Refresh to get recommended posts.
This method invokes an asynchronous action to refresh and fetch
recommended posts. On successful execution, it returns a dictionary
indicating success along with a list of posts. Each post in the list
contains details such as post ID, user ID, content, creation date,
and the number of likes.
Returns:
dict: A dictionary with two key-value pairs. The 'success' key
maps to a boolean indicating whether the refresh is
successful. The 'posts' key maps to a list of dictionaries,
each representing a post with its details.
Example of a successful return:
{
"success": True,
"posts": [
{
"post_id": 1,
"user_id": 23,
"content": "This is an example post content.",
"created_at": "2024-05-14T12:00:00Z",
"num_likes": 5
},
{
"post_id": 2,
"user_id": 42,
"content": "Another example post content.",
"created_at": "2024-05-14T12:05:00Z",
"num_likes": 15
}
]
}
"""
return await self.perform_action(None, ActionType.REFRESH.value)
async def do_nothing(self):
"""Perform no action.
Returns:
dict: A dictionary with 'success' indicating if the removal was
successful.
Example of a successful return:
{"success": True}
"""
return await self.perform_action(None, ActionType.DO_NOTHING.value)
async def create_post(self, content: str):
r"""Create a new post with the given content.
This method invokes an asynchronous action to create a new post based
on the provided content. Upon successful execution, it returns a
dictionary indicating success and the ID of the newly created post.
Args:
content (str): The content of the post to be created.
Returns:
dict: A dictionary with two key-value pairs. The 'success' key
maps to a boolean indicating whether the post creation was
successful. The 'post_id' key maps to the integer ID of the
newly created post.
Example of a successful return:
{'success': True, 'post_id': 50}
"""
return await self.perform_action(content, ActionType.CREATE_POST.value)
async def repost(self, post_id: int):
r"""Repost a specified post.
This method invokes an asynchronous action to Repost a specified
post. It is identified by the given post ID. Upon successful
execution, it returns a dictionary indicating success and the ID of
the newly created repost.
Args:
post_id (int): The ID of the post to be reposted.
Returns:
dict: A dictionary with two key-value pairs. The 'success' key
maps to a boolean indicating whether the Repost creation was
successful. The 'post_id' key maps to the integer ID of the
newly created repost.
Example of a successful return:
{"success": True, "post_id": 123}
Note:
Attempting to repost a post that the user has already reposted
will result in a failure.
"""
return await self.perform_action(post_id, ActionType.REPOST.value)
async def quote_post(self, post_id: int, quote_content: str):
r"""Quote a specified post with a given quote content.
This method invokes an asynchronous action to quote a specified post
with a given quote content. Upon successful execution, it returns a
dictionary indicating success and the ID of the newly created quote.
Args:
post_id (int): The ID of the post to be quoted.
quote_content (str): The content of the quote to be created.
Returns:
dict: A dictionary with two key-value pairs. The 'success' key
maps to a boolean indicating whether the quote creation was
successful. The 'post_id' key maps to the integer ID of the
newly created quote.
Example of a successful return:
{"success": True, "post_id": 123}
Note:
Attempting to quote a post that the user has already quoted will
result in a failure.
"""
quote_message = (post_id, quote_content)
return await self.perform_action(quote_message, ActionType.QUOTE_POST)
async def like_post(self, post_id: int):
r"""Create a new like for a specified post.
This method invokes an asynchronous action to create a new like for a
post. It is identified by the given post ID. Upon successful
execution, it returns a dictionary indicating success and the ID of
the newly created like.
Args:
post_id (int): The ID of the post to be liked.
Returns:
dict: A dictionary with two key-value pairs. The 'success' key
maps to a boolean indicating whether the like creation was
successful. The 'like_id' key maps to the integer ID of the
newly created like.
Example of a successful return:
{"success": True, "like_id": 123}
Note:
Attempting to like a post that the user has already liked will
result in a failure.
"""
return await self.perform_action(post_id, ActionType.LIKE_POST.value)
async def unlike_post(self, post_id: int):
"""Remove a like for a post.
This method removes a like from the database, identified by the
post's ID. It returns a dictionary indicating the success of the
operation and the ID of the removed like.
Args:
post_id (int): The ID of the post to be unliked.
Returns:
dict: A dictionary with 'success' indicating if the removal was
successful, and 'like_id' the ID of the removed like.
Example of a successful return:
{"success": True, "like_id": 123}
Note:
Attempting to remove a like for a post that the user has not
previously liked will result in a failure.
"""
return await self.perform_action(post_id, ActionType.UNLIKE_POST.value)
async def dislike_post(self, post_id: int):
r"""Create a new dislike for a specified post.
This method invokes an asynchronous action to create a new dislike for
a post. It is identified by the given post ID. Upon successful
execution, it returns a dictionary indicating success and the ID of
the newly created dislike.
Args:
post_id (int): The ID of the post to be disliked.
Returns:
dict: A dictionary with two key-value pairs. The 'success' key
maps to a boolean indicating whether the dislike creation was
successful. The 'dislike_id' key maps to the integer ID of the
newly created like.
Example of a successful return:
{"success": True, "dislike_id": 123}
Note:
Attempting to dislike a post that the user has already liked will
result in a failure.
"""
return await self.perform_action(post_id,
ActionType.DISLIKE_POST.value)
async def undo_dislike_post(self, post_id: int):
"""Remove a dislike for a post.
This method removes a dislike from the database, identified by the
post's ID. It returns a dictionary indicating the success of the
operation and the ID of the removed dislike.
Args:
post_id (int): The ID of the post to be unliked.
Returns:
dict: A dictionary with 'success' indicating if the removal was
successful, and 'dislike_id' the ID of the removed like.
Example of a successful return:
{"success": True, "dislike_id": 123}
Note:
Attempting to remove a dislike for a post that the user has not
previously liked will result in a failure.
"""
return await self.perform_action(post_id,
ActionType.UNDO_DISLIKE_POST.value)
async def search_posts(self, query: str):
r"""Search posts based on a given query.
This method performs a search operation in the database for posts
that match the given query string. The search considers the
post's content, post ID, and user ID. It returns a dictionary
indicating the operation's success and, if successful, a list of
posts that match the query.
Args:
query (str): The search query string. The search is performed
against the post's content, post ID, and user ID.
Returns:
dict: A dictionary with a 'success' key indicating the operation's
success. On success, it includes a 'posts' key with a list of
dictionaries, each representing a post. On failure, it
includes an 'error' message or a 'message' indicating no
posts were found.
Example of a successful return:
{
"success": True,
"posts": [
{
"post_id": 1,
"user_id": 42,
"content": "Hello, world!",
"created_at": "2024-05-14T12:00:00Z",
"num_likes": 150
},
...
]
}
"""
return await self.perform_action(query, ActionType.SEARCH_POSTS.value)
async def search_user(self, query: str):
r"""Search users based on a given query.
This asynchronous method performs a search operation in the database
for users that match the given query string. The search considers the
user's username, name, bio, and user ID. It returns a dictionary
indicating the operation's success and, if successful, a list of users
that match the query.
Args:
query (str): The search query string. The search is performed
against the user's username, name, bio, and user ID.
Returns:
dict: A dictionary with a 'success' key indicating the operation's
success. On success, it includes a 'users' key with a list of
dictionaries, each representing a user. On failure, it includes
an 'error' message or a 'message' indicating no users were
found.
Example of a successful return:
{
"success": True,
"users": [
{
"user_id": 1,
"user_name": "exampleUser",
"name": "John Doe",
"bio": "This is an example bio",
"created_at": "2024-05-14T12:00:00Z",
"num_followings": 100,
"num_followers": 150
},
...
]
}
"""
return await self.perform_action(query, ActionType.SEARCH_USER.value)
async def follow(self, followee_id: int):
r"""Follow a user.
This method allows agent to follow another user (followee).
It checks if the agent initiating the follow request has a
corresponding user ID and if the follow relationship already exists.
Args:
followee_id (int): The user ID of the user to be followed.
Returns:
dict: A dictionary with a 'success' key indicating the operation's
success. On success, it includes a 'follow_id' key with the ID
of the newly created follow record. On failure, it includes an
'error' message.
Example of a successful return:
{"success": True, "follow_id": 123}
"""
return await self.perform_action(followee_id, ActionType.FOLLOW.value)
async def unfollow(self, followee_id: int):
r"""Unfollow a user.
This method allows agent to unfollow another user (followee). It
checks if the agent initiating the unfollow request has a
corresponding user ID and if the follow relationship exists. If so,
it removes the follow record from the database, updates the followers
and followings count for both users, and logs the action.
Args:
followee_id (int): The user ID of the user to be unfollowed.
Returns:
dict: A dictionary with a 'success' key indicating the operation's
success. On success, it includes a 'follow_id' key with the ID
of the removed follow record. On failure, it includes an
'error' message.
Example of a successful return:
{"success": True, "follow_id": 123}
"""
return await self.perform_action(followee_id,
ActionType.UNFOLLOW.value)
async def mute(self, mutee_id: int):
r"""Mute a user.
Allows agent to mute another user. Checks for an existing mute
record before adding a new one to the database.
Args:
mutee_id (int): ID of the user to be muted.
Returns:
dict: On success, returns a dictionary with 'success': True and
mute_id' of the new record. On failure, returns 'success':
False and an 'error' message.
Example of a successful return:
{"success": True, "mutee_id": 123}
"""
return await self.perform_action(mutee_id, ActionType.MUTE.value)
async def unmute(self, mutee_id: int):
r"""Unmute a user.
Allows agent to remove a mute on another user. Checks for an
existing mute record before removing it from the database.
Args:
mutee_id (int): ID of the user to be unmuted.
Returns:
dict: On success, returns a dictionary with 'success': True and
'mutee_id' of the unmuted record. On failure, returns
'success': False and an 'error' message.
Example of a successful return:
{"success": True, "mutee_id": 123}
"""
return await self.perform_action(mutee_id, ActionType.UNMUTE.value)
async def trend(self):
r"""Fetch the trending posts within a predefined time period.
Retrieves the top K posts with the most likes in the last specified
number of days.
Returns:
dict: On success, returns a dictionary with 'success': True and a
list of 'posts', each post being a dictionary containing
'post_id', 'user_id', 'content', 'created_at', and
'num_likes'. On failure, returns 'success': False and an
'error' message or a message indicating no trending posts
were found.
Example of a successful return:
{
"success": True,
"posts": [
{
"post_id": 123,
"user_id": 456,
"content": "Example post content",
"created_at": "2024-05-14T12:00:00",
"num_likes": 789
},
...
]
}
"""
return await self.perform_action(None, ActionType.TREND.value)
async def create_comment(self, post_id: int, content: str):
r"""Create a new comment for a specified post given content.
This method creates a new comment based on the provided content and
associates it with the given post ID. Upon successful execution, it
returns a dictionary indicating success and the ID of the newly created
comment.
Args:
post_id (int): The ID of the post to which the comment is to be
added.
content (str): The content of the comment to be created.
Returns:
dict: A dictionary with two key-value pairs. The 'success' key
maps to a boolean indicating whether the comment creation was
successful. The 'comment_id' key maps to the integer ID of the
newly created comment.
Example of a successful return:
{'success': True, 'comment_id': 123}
"""
comment_message = (post_id, content)
return await self.perform_action(comment_message,
ActionType.CREATE_COMMENT.value)
async def like_comment(self, comment_id: int):
r"""Create a new like for a specified comment.
This method invokes an action to create a new like for a comment,
identified by the given comment ID. Upon successful execution, it
returns a dictionary indicating success and the ID of the newly
created like.
Args:
comment_id (int): The ID of the comment to be liked.
Returns:
dict: A dictionary with two key-value pairs. The 'success' key
maps to a boolean indicating whether the like creation was
successful. The 'like_id' key maps to the integer ID of the
newly created like.
Example of a successful return:
{"success": True, "comment_like_id": 456}
Note:
Attempting to like a comment that the user has already liked will
result in a failure.
"""
return await self.perform_action(comment_id,
ActionType.LIKE_COMMENT.value)
async def unlike_comment(self, comment_id: int):
"""Remove a like for a comment based on the comment's ID.
This method removes a like from the database, identified by the
comment's ID. It returns a dictionary indicating the success of the
operation and the ID of the removed like.
Args:
comment_id (int): The ID of the comment to be unliked.
Returns:
dict: A dictionary with 'success' indicating if the removal was
successful, and 'like_id' the ID of the removed like.
Example of a successful return:
{"success": True, "like_id": 456}
Note:
Attempting to remove a like for a comment that the user has not
previously liked will result in a failure.
"""
return await self.perform_action(comment_id,
ActionType.UNLIKE_COMMENT.value)
async def dislike_comment(self, comment_id: int):
r"""Create a new dislike for a specified comment.
This method invokes an action to create a new dislike for a
comment, identified by the given comment ID. Upon successful execution,
it returns a dictionary indicating success and the ID of the newly
created dislike.
Args:
comment_id (int): The ID of the comment to be disliked.
Returns:
dict: A dictionary with two key-value pairs. The 'success' key
maps to a boolean indicating whether the dislike creation was
successful. The 'dislike_id' key maps to the integer ID of the
newly created dislike.
Example of a successful return:
{"success": True, "comment_dislike_id": 456}
Note:
Attempting to dislike a comment that the user has already liked
will result in a failure.
"""
return await self.perform_action(comment_id,
ActionType.DISLIKE_COMMENT.value)
async def undo_dislike_comment(self, comment_id: int):
"""Remove a dislike for a comment.
This method removes a dislike from the database, identified by the
comment's ID. It returns a dictionary indicating the success of the
operation and the ID of the removed dislike.
Args:
comment_id (int): The ID of the comment to have the dislike
removed.
Returns:
dict: A dictionary with 'success' indicating if the removal was
successful, and 'dislike_id' the ID of the removed dislike.
Example of a successful return:
{"success": True, "dislike_id": 456}
Note:
Attempting to remove a dislike for a comment that the user has not
previously disliked will result in a failure.
"""
return await self.perform_action(comment_id,
ActionType.UNDO_DISLIKE_COMMENT.value)
async def purchase_product(self, product_name: str, purchase_num: int):
r"""Purchase a product.
Args:
product_name (str): The name of the product to be purchased.
purchase_num (int): The number of products to be purchased.
Returns:
dict: A dictionary with 'success' indicating if the purchase was
successful.
"""
purchase_message = (product_name, purchase_num)
return await self.perform_action(purchase_message,
ActionType.PURCHASE_PRODUCT.value)
async def interview(self, prompt: str):
r"""Interview an agent with the given prompt.
This method invokes an asynchronous action to interview an agent with a
specific prompt question. Upon successful execution,
it returns a dictionary containing a success status
and an interview_id for tracking.
Args:
prompt (str): The interview question or prompt to ask the agent.
Returns:
dict: A dictionary containing success status and an interview_id.
Example of a successful return:
{
"success": True,
"interview_id": "1621234567_0" # Timestamp_UserID format
}
"""
return await self.perform_action(prompt, ActionType.INTERVIEW.value)
async def report_post(self, post_id: int, report_reason: str):
r"""Report a specified post with a given reason.
This method invokes an asynchronous action to report a specified post
with a given reason. Upon successful execution, it returns a
dictionary indicating success and the ID of the newly created report.
Args:
post_id (int): The ID of the post to be reported.
report_reason (str): The reason for reporting the post.
Returns:
dict: A dictionary with two key-value pairs. The 'success' key
maps to a boolean indicating whether the report creation was
successful. The 'report_id' key maps to the integer ID of the
newly created report.
Example of a successful return:
{"success": True, "report_id": 123}
Note:
Attempting to report a post that the user has already reported will
result in a failure.
"""
report_message = (post_id, report_reason)
return await self.perform_action(report_message,
ActionType.REPORT_POST.value)
async def create_group(self, group_name: str):
r"""Creates a new group on the platform.
Args:
group_name (str): The name of the group to be created.
Returns:
dict: Platform response indicating success or failure,
e.g.{"success": True, "group_id": 1}
"""
return await self.perform_action(group_name,
ActionType.CREATE_GROUP.value)
async def join_group(self, group_id: int):
r"""Joins a group with the specified ID.
Args:
group_id (int): The ID of the group to join.
Returns:
dict: Platform response indicating success or failure,
e.g. {"success": True}
"""
return await self.perform_action(group_id, ActionType.JOIN_GROUP.value)
async def leave_group(self, group_id: int):
r"""Leaves a group with the specified ID.
Args:
group_id (int): The ID of the group to leave.
Returns:
dict: Platform response indicating success or failure, e.g.
{"success": True}
"""
return await self.perform_action(group_id,
ActionType.LEAVE_GROUP.value)
async def send_to_group(self, group_id: int, message: str):
r"""Sends a message to a specific group.
Args:
group_id (int): The ID of the target group.
message (str): The content of the message to send.
Returns:
dict: Platform response indicating success or failure, e.g.
{"success": True, "message_id": 123}
"""
return await self.perform_action((group_id, message),
ActionType.SEND_TO_GROUP.value)
async def listen_from_group(self):
r"""Listen messages from groups"""
return await self.perform_action(self.agent_id,
ActionType.LISTEN_FROM_GROUP.value)

135
backend/vendor/camel-oasis/oasis/social_agent/agent_environment.py vendored Normal file
View File

@ -0,0 +1,135 @@
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
# Licensed under the Apache License, Version 2.0 (the “License”);
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an “AS IS” BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
from __future__ import annotations
import json
import sqlite3
from abc import ABC, abstractmethod
from string import Template
from oasis.social_agent.agent_action import SocialAction
from oasis.social_platform.database import get_db_path
class Environment(ABC):
@abstractmethod
def to_text_prompt(self) -> str:
r"""Convert the environment to text prompt."""
raise NotImplementedError
class SocialEnvironment(Environment):
followers_env_template = Template("I have $num_followers followers.")
follows_env_template = Template("I have $num_follows follows.")
posts_env_template = Template(
"After refreshing, you see some posts $posts")
groups_env_template = Template(
"And there are many group chat channels $all_groups\n"
"And You are already in some groups $joined_groups\n"
"You receive some messages from them $messages\n"
"You can join the groups you are interested, "
"leave the groups you already in, send messages to the group "
"you already in.\n"
"You must make sure you can only send messages to the group you "
"are already in")
env_template = Template(
"$groups_env\n"
"$posts_env\npick one you want to perform action that best "
"reflects your current inclination based on your profile and "
"posts content. Do not limit your action in just `like` to like posts")
def __init__(self, action: SocialAction):
self.action = action
async def get_posts_env(self) -> str:
posts = await self.action.refresh()
# TODO: Replace posts json format string to other formats
if posts["success"]:
posts_env = json.dumps(posts["posts"], indent=4)
posts_env = self.posts_env_template.substitute(posts=posts_env)
else:
posts_env = "After refreshing, there are no existing posts."
return posts_env
async def get_followers_env(self) -> str:
# TODO: Implement followers env
agent_id = self.action.agent_id
db_path = get_db_path()
try:
conn = sqlite3.connect(db_path)
cursor = conn.cursor()
cursor.execute("SELECT num_followers FROM user WHERE agent_id = ?",
(agent_id, ))
result = cursor.fetchone()
num_followers = result[0] if result else 0
conn.close()
except Exception:
num_followers = 0
return self.followers_env_template.substitute(
{"num_followers": num_followers})
async def get_follows_env(self) -> str:
# TODO: Implement follows env
agent_id = self.action.agent_id
try:
db_path = get_db_path()
conn = sqlite3.connect(db_path)
cursor = conn.cursor()
cursor.execute(
"SELECT num_followings FROM user WHERE agent_id = ?",
(agent_id, ))
result = cursor.fetchone()
num_followings = result[0] if result else 0
conn.close()
except Exception:
num_followings = 0
return self.follows_env_template.substitute(
{"num_follows": num_followings})
async def get_group_env(self) -> str:
groups = await self.action.listen_from_group()
if groups["success"]:
all_groups = json.dumps(groups["all_groups"])
joined_groups = json.dumps(groups["joined_groups"])
messages = json.dumps(groups["messages"])
groups_env = self.groups_env_template.substitute(
all_groups=all_groups,
joined_groups=joined_groups,
messages=messages,
)
else:
groups_env = "No groups."
return groups_env
async def to_text_prompt(
self,
include_posts: bool = True,
include_followers: bool = True,
include_follows: bool = True,
) -> str:
followers_env = (await self.get_followers_env()
if include_follows else "No followers.")
follows_env = (await self.get_follows_env()
if include_followers else "No follows.")
posts_env = await self.get_posts_env() if include_posts else ""
return self.env_template.substitute(
followers_env=followers_env,
follows_env=follows_env,
posts_env=posts_env,
groups_env=await self.get_group_env(),
)

292
backend/vendor/camel-oasis/oasis/social_agent/agent_graph.py vendored Normal file
View File

@ -0,0 +1,292 @@
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
# Licensed under the Apache License, Version 2.0 (the “License”);
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an “AS IS” BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
from __future__ import annotations
from typing import Any, Literal
import igraph as ig
from neo4j import GraphDatabase
from oasis.social_agent.agent import SocialAgent
from oasis.social_platform.config import Neo4jConfig
class Neo4jHandler:
def __init__(self, nei4j_config: Neo4jConfig):
self.driver = GraphDatabase.driver(
nei4j_config.uri,
auth=(nei4j_config.username, nei4j_config.password),
)
self.driver.verify_connectivity()
def close(self):
self.driver.close()
def create_agent(self, agent_id: int):
with self.driver.session() as session:
session.write_transaction(self._create_and_return_agent, agent_id)
def delete_agent(self, agent_id: int):
with self.driver.session() as session:
session.write_transaction(
self._delete_agent_and_relationships,
agent_id,
)
def get_number_of_nodes(self) -> int:
with self.driver.session() as session:
return session.read_transaction(self._get_number_of_nodes)
def get_number_of_edges(self) -> int:
with self.driver.session() as session:
return session.read_transaction(self._get_number_of_edges)
def add_edge(self, src_agent_id: int, dst_agent_id: int):
with self.driver.session() as session:
session.write_transaction(
self._add_and_return_edge,
src_agent_id,
dst_agent_id,
)
def remove_edge(self, src_agent_id: int, dst_agent_id: int):
with self.driver.session() as session:
session.write_transaction(
self._remove_and_return_edge,
src_agent_id,
dst_agent_id,
)
def get_all_nodes(self) -> list[int]:
with self.driver.session() as session:
return session.read_transaction(self._get_all_nodes)
def get_all_edges(self) -> list[tuple[int, int]]:
with self.driver.session() as session:
return session.read_transaction(self._get_all_edges)
def reset_graph(self):
with self.driver.session() as session:
session.write_transaction(self._reset_graph)
@staticmethod
def _create_and_return_agent(tx: Any, agent_id: int):
query = """
CREATE (a:Agent {id: $agent_id})
RETURN a
"""
result = tx.run(query, agent_id=agent_id)
return result.single()
@staticmethod
def _delete_agent_and_relationships(tx: Any, agent_id: int):
query = """
MATCH (a:Agent {id: $agent_id})
DETACH DELETE a
RETURN count(a) AS deleted
"""
result = tx.run(query, agent_id=agent_id)
return result.single()
@staticmethod
def _add_and_return_edge(tx: Any, src_agent_id: int, dst_agent_id: int):
query = """
MATCH (a:Agent {id: $src_agent_id}), (b:Agent {id: $dst_agent_id})
CREATE (a)-[r:FOLLOW]->(b)
RETURN r
"""
result = tx.run(query,
src_agent_id=src_agent_id,
dst_agent_id=dst_agent_id)
return result.single()
@staticmethod
def _remove_and_return_edge(tx: Any, src_agent_id: int, dst_agent_id: int):
query = """
MATCH (a:Agent {id: $src_agent_id})
MATCH (b:Agent {id: $dst_agent_id})
MATCH (a)-[r:FOLLOW]->(b)
DELETE r
RETURN count(r) AS deleted
"""
result = tx.run(query,
src_agent_id=src_agent_id,
dst_agent_id=dst_agent_id)
return result.single()
@staticmethod
def _get_number_of_nodes(tx: Any) -> int:
query = """
MATCH (n)
RETURN count(n) AS num_nodes
"""
result = tx.run(query)
return result.single()["num_nodes"]
@staticmethod
def _get_number_of_edges(tx: Any) -> int:
query = """
MATCH ()-[r]->()
RETURN count(r) AS num_edges
"""
result = tx.run(query)
return result.single()["num_edges"]
@staticmethod
def _get_all_nodes(tx: Any) -> list[int]:
query = """
MATCH (a:Agent)
RETURN a.id AS agent_id
"""
result = tx.run(query)
return [record["agent_id"] for record in result]
@staticmethod
def _get_all_edges(tx: Any) -> list[tuple[int, int]]:
query = """
MATCH (a:Agent)-[r:FOLLOW]->(b:Agent)
RETURN a.id AS src_agent_id, b.id AS dst_agent_id
"""
result = tx.run(query)
return [(record["src_agent_id"], record["dst_agent_id"])
for record in result]
@staticmethod
def _reset_graph(tx: Any):
query = """
MATCH (n)
DETACH DELETE n
"""
tx.run(query)
class AgentGraph:
r"""AgentGraph class to manage the social graph of agents."""
def __init__(
self,
backend: Literal["igraph", "neo4j"] = "igraph",
neo4j_config: Neo4jConfig | None = None,
):
self.backend = backend
if self.backend == "igraph":
self.graph = ig.Graph(directed=True)
else:
assert neo4j_config is not None
assert neo4j_config.is_valid()
self.graph = Neo4jHandler(neo4j_config)
self.agent_mappings: dict[int, SocialAgent] = {}
def reset(self):
if self.backend == "igraph":
self.graph = ig.Graph(directed=True)
else:
self.graph.reset_graph()
self.agent_mappings: dict[int, SocialAgent] = {}
def add_agent(self, agent: SocialAgent):
if self.backend == "igraph":
self.graph.add_vertex(agent.social_agent_id)
else:
self.graph.create_agent(agent.social_agent_id)
self.agent_mappings[agent.social_agent_id] = agent
def add_edge(self, agent_id_0: int, agent_id_1: int):
try:
self.graph.add_edge(agent_id_0, agent_id_1)
except Exception:
pass
def remove_agent(self, agent: SocialAgent):
if self.backend == "igraph":
self.graph.delete_vertices(agent.social_agent_id)
else:
self.graph.delete_agent(agent.social_agent_id)
del self.agent_mappings[agent.social_agent_id]
def remove_edge(self, agent_id_0: int, agent_id_1: int):
if self.backend == "igraph":
if self.graph.are_connected(agent_id_0, agent_id_1):
self.graph.delete_edges([(agent_id_0, agent_id_1)])
else:
self.graph.remove_edge(agent_id_0, agent_id_1)
def get_agent(self, agent_id: int) -> SocialAgent:
return self.agent_mappings[agent_id]
def get_agents(
self,
agent_ids: list[int] = None) -> list[tuple[int, SocialAgent]]:
if agent_ids:
return [(agent_id, self.get_agent(agent_id))
for agent_id in agent_ids]
if self.backend == "igraph":
return [(node.index, self.agent_mappings[node.index])
for node in self.graph.vs]
else:
return [(agent_id, self.agent_mappings[agent_id])
for agent_id in self.graph.get_all_nodes()]
def get_edges(self) -> list[tuple[int, int]]:
if self.backend == "igraph":
return [(edge.source, edge.target) for edge in self.graph.es]
else:
return self.graph.get_all_edges()
def get_num_nodes(self) -> int:
if self.backend == "igraph":
return self.graph.vcount()
else:
return self.graph.get_number_of_nodes()
def get_num_edges(self) -> int:
if self.backend == "igraph":
return self.graph.ecount()
else:
return self.graph.get_number_of_edges()
def close(self) -> None:
if self.backend == "neo4j":
self.graph.close()
def visualize(
self,
path: str,
vertex_size: int = 20,
edge_arrow_size: float = 0.5,
with_labels: bool = True,
vertex_color: str = "#f74f1b",
vertex_frame_width: int = 2,
width: int = 1000,
height: int = 1000,
):
if self.backend == "neo4j":
raise ValueError("Neo4j backend does not support visualization.")
layout = self.graph.layout("auto")
if with_labels:
labels = [node_id for node_id, _ in self.get_agents()]
else:
labels = None
ig.plot(
self.graph,
target=path,
layout=layout,
vertex_label=labels,
vertex_size=vertex_size,
vertex_color=vertex_color,
edge_arrow_size=edge_arrow_size,
vertex_frame_width=vertex_frame_width,
bbox=(width, height),
)

649
backend/vendor/camel-oasis/oasis/social_agent/agents_generator.py vendored Normal file
View File

@ -0,0 +1,649 @@
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
# Licensed under the Apache License, Version 2.0 (the “License”);
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an “AS IS” BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
from __future__ import annotations
import ast
import asyncio
import json
from typing import List, Optional, Union
import pandas as pd
import tqdm
from camel.memories import MemoryRecord
from camel.messages import BaseMessage
from camel.models import BaseModelBackend, ModelManager
from camel.types import OpenAIBackendRole
from oasis.social_agent import AgentGraph, SocialAgent
from oasis.social_platform import Channel, Platform
from oasis.social_platform.config import Neo4jConfig, UserInfo
from oasis.social_platform.typing import ActionType
async def generate_agents(
agent_info_path: str,
channel: Channel,
model: Union[BaseModelBackend, List[BaseModelBackend]],
start_time,
recsys_type: str = "twitter",
twitter: Platform = None,
available_actions: list[ActionType] = None,
neo4j_config: Neo4jConfig | None = None,
) -> AgentGraph:
"""TODO: need update the description of args and check
Generate and return a dictionary of agents from the agent
information CSV file. Each agent is added to the database and
their respective profiles are updated.
Args:
agent_info_path (str): The file path to the agent information CSV file.
channel (Channel): Information channel.
action_space_prompt (str): determine the action space of agents.
model_random_seed (int): Random seed to randomly assign model to
each agent. (default: 42)
cfgs (list, optional): List of configuration. (default: `None`)
neo4j_config (Neo4jConfig, optional): Neo4j graph database
configuration. (default: `None`)
Returns:
dict: A dictionary of agent IDs mapped to their respective agent
class instances.
"""
agent_info = pd.read_csv(agent_info_path)
agent_graph = (AgentGraph() if neo4j_config is None else AgentGraph(
backend="neo4j",
neo4j_config=neo4j_config,
))
# agent_graph = []
sign_up_list = []
follow_list = []
user_update1 = []
user_update2 = []
post_list = []
for agent_id in range(len(agent_info)):
profile = {
"nodes": [],
"edges": [],
"other_info": {},
}
profile["other_info"]["user_profile"] = agent_info["user_char"][
agent_id]
user_info = UserInfo(
name=agent_info["username"][agent_id],
description=agent_info["description"][agent_id],
profile=profile,
recsys_type=recsys_type,
)
agent = SocialAgent(
agent_id=agent_id,
user_info=user_info,
channel=channel,
model=model,
agent_graph=agent_graph,
available_actions=available_actions,
)
agent_graph.add_agent(agent)
# TODO we should not use following_count and followers_count
# We should calculate the number of followings and followers
# based on the graph because the following situation is dynamic.
num_followings = 0
num_followers = 0
sign_up_list.append((
agent_id,
agent_id,
agent_info["username"][agent_id],
agent_info["name"][agent_id],
agent_info["description"][agent_id],
start_time,
num_followings,
num_followers,
))
following_id_list = ast.literal_eval(
agent_info["following_agentid_list"][agent_id])
if not isinstance(following_id_list, int):
if len(following_id_list) != 0:
for follow_id in following_id_list:
follow_list.append((agent_id, follow_id, start_time))
user_update1.append((agent_id, ))
user_update2.append((follow_id, ))
agent_graph.add_edge(agent_id, follow_id)
previous_posts = ast.literal_eval(
agent_info["previous_tweets"][agent_id])
if len(previous_posts) != 0:
for post in previous_posts:
post_list.append((agent_id, post, start_time, 0, 0))
# generate_log.info('agent gegenerate finished.')
user_insert_query = (
"INSERT INTO user (user_id, agent_id, user_name, name, bio, "
"created_at, num_followings, num_followers) VALUES "
"(?, ?, ?, ?, ?, ?, ?, ?)")
twitter.pl_utils._execute_many_db_command(user_insert_query,
sign_up_list,
commit=True)
follow_insert_query = (
"INSERT INTO follow (follower_id, followee_id, created_at) "
"VALUES (?, ?, ?)")
twitter.pl_utils._execute_many_db_command(follow_insert_query,
follow_list,
commit=True)
user_update_query1 = (
"UPDATE user SET num_followings = num_followings + 1 "
"WHERE user_id = ?")
twitter.pl_utils._execute_many_db_command(user_update_query1,
user_update1,
commit=True)
user_update_query2 = ("UPDATE user SET num_followers = num_followers + 1 "
"WHERE user_id = ?")
twitter.pl_utils._execute_many_db_command(user_update_query2,
user_update2,
commit=True)
# generate_log.info('twitter followee update finished.')
post_insert_query = (
"INSERT INTO post (user_id, content, created_at, num_likes, "
"num_dislikes) VALUES (?, ?, ?, ?, ?)")
twitter.pl_utils._execute_many_db_command(post_insert_query,
post_list,
commit=True)
# generate_log.info('twitter creat post finished.')
return agent_graph
async def generate_agents_100w(
agent_info_path: str,
channel: Channel,
start_time,
model: Union[BaseModelBackend, List[BaseModelBackend]],
recsys_type: str = "twitter",
twitter: Platform = None,
available_actions: list[ActionType] = None,
) -> List:
""" TODO: need update the description of args.
Generate and return a dictionary of agents from the agent
information CSV file. Each agent is added to the database and
their respective profiles are updated.
Args:
agent_info_path (str): The file path to the agent information CSV file.
channel (Channel): Information channel.
action_space_prompt (str): determine the action space of agents.
model_random_seed (int): Random seed to randomly assign model to
each agent. (default: 42)
Returns:
dict: A dictionary of agent IDs mapped to their respective agent
class instances.
"""
agent_info = pd.read_csv(agent_info_path)
# TODO when setting 100w agents, the agentgraph class is too slow.
# I use the list.
agent_graph = []
# agent_graph = (AgentGraph() if neo4j_config is None else AgentGraph(
# backend="neo4j",
# neo4j_config=neo4j_config,
# ))
# agent_graph = []
sign_up_list = []
follow_list = []
user_update1 = []
user_update2 = []
post_list = []
# precompute to speed up agent generation in one million scale
_ = agent_info["following_agentid_list"].apply(ast.literal_eval)
previous_tweets_lists = agent_info["previous_tweets"].apply(
ast.literal_eval)
previous_tweets_lists = agent_info['previous_tweets'].apply(
ast.literal_eval)
following_id_lists = agent_info["following_agentid_list"].apply(
ast.literal_eval)
for agent_id in tqdm.tqdm(range(len(agent_info))):
profile = {
"nodes": [],
"edges": [],
"other_info": {},
}
profile["other_info"]["user_profile"] = agent_info["user_char"][
agent_id]
# TODO if you simulate one million agents, use active threshold below.
# profile['other_info']['active_threshold'] = [0.01] * 24
user_info = UserInfo(
name=agent_info["username"][agent_id],
description=agent_info["description"][agent_id],
profile=profile,
recsys_type=recsys_type,
)
agent = SocialAgent(
agent_id=agent_id,
user_info=user_info,
channel=channel,
model=model,
agent_graph=agent_graph,
available_actions=available_actions,
)
agent_graph.append(agent)
num_followings = 0
num_followers = 0
# print('agent_info["following_count"]', agent_info["following_count"])
# TODO some data does not cotain this key.
if 'following_count' not in agent_info.columns:
agent_info['following_count'] = 0
if 'followers_count' not in agent_info.columns:
agent_info['followers_count'] = 0
if not agent_info["following_count"].empty:
num_followings = agent_info["following_count"][agent_id]
if not agent_info["followers_count"].empty:
num_followers = agent_info["followers_count"][agent_id]
sign_up_list.append((
agent_id,
agent_id,
agent_info["username"][agent_id],
agent_info["name"][agent_id],
agent_info["description"][agent_id],
start_time,
num_followings,
num_followers,
))
following_id_list = following_id_lists[agent_id]
# TODO If we simulate 1 million agents, we can not use agent_graph
# class. It is not scalble.
if not isinstance(following_id_list, int):
if len(following_id_list) != 0:
for follow_id in following_id_list:
follow_list.append((agent_id, follow_id, start_time))
user_update1.append((agent_id, ))
user_update2.append((follow_id, ))
# agent_graph.add_edge(agent_id, follow_id)
previous_posts = previous_tweets_lists[agent_id]
if len(previous_posts) != 0:
for post in previous_posts:
post_list.append((agent_id, post, start_time, 0, 0))
# generate_log.info('agent gegenerate finished.')
user_insert_query = (
"INSERT INTO user (user_id, agent_id, user_name, name, bio, "
"created_at, num_followings, num_followers) VALUES "
"(?, ?, ?, ?, ?, ?, ?, ?)")
twitter.pl_utils._execute_many_db_command(user_insert_query,
sign_up_list,
commit=True)
follow_insert_query = (
"INSERT INTO follow (follower_id, followee_id, created_at) "
"VALUES (?, ?, ?)")
twitter.pl_utils._execute_many_db_command(follow_insert_query,
follow_list,
commit=True)
if not (agent_info["following_count"].empty
and agent_info["followers_count"].empty):
user_update_query1 = (
"UPDATE user SET num_followings = num_followings + 1 "
"WHERE user_id = ?")
twitter.pl_utils._execute_many_db_command(user_update_query1,
user_update1,
commit=True)
user_update_query2 = (
"UPDATE user SET num_followers = num_followers + 1 "
"WHERE user_id = ?")
twitter.pl_utils._execute_many_db_command(user_update_query2,
user_update2,
commit=True)
# generate_log.info('twitter followee update finished.')
post_insert_query = (
"INSERT INTO post (user_id, content, created_at, num_likes, "
"num_dislikes) VALUES (?, ?, ?, ?, ?)")
twitter.pl_utils._execute_many_db_command(post_insert_query,
post_list,
commit=True)
# generate_log.info('twitter creat post finished.')
return agent_graph
async def generate_controllable_agents(
channel: Channel,
control_user_num: int,
) -> tuple[AgentGraph, dict]:
agent_graph = AgentGraph()
agent_user_id_mapping = {}
for i in range(control_user_num):
user_info = UserInfo(
is_controllable=True,
profile={"other_info": {
"user_profile": "None"
}},
recsys_type="reddit",
)
# controllable的agent_id全都在llm agent的agent_id的前面
agent = SocialAgent(agent_id=i,
user_info=user_info,
channel=channel,
agent_graph=agent_graph)
# Add agent to the agent graph
agent_graph.add_agent(agent)
username = input(f"Please input username for agent {i}: ")
name = input(f"Please input name for agent {i}: ")
bio = input(f"Please input bio for agent {i}: ")
response = await agent.env.action.sign_up(username, name, bio)
user_id = response["user_id"]
agent_user_id_mapping[i] = user_id
for i in range(control_user_num):
for j in range(control_user_num):
agent = agent_graph.get_agent(i)
# controllable agent互相也全部关注
if i != j:
user_id = agent_user_id_mapping[j]
await agent.env.action.follow(user_id)
agent_graph.add_edge(i, j)
return agent_graph, agent_user_id_mapping
async def gen_control_agents_with_data(
channel: Channel,
control_user_num: int,
models: list[BaseModelBackend] | None = None,
) -> tuple[AgentGraph, dict]:
agent_graph = AgentGraph()
agent_user_id_mapping = {}
for i in range(control_user_num):
user_info = UserInfo(
is_controllable=True,
profile={
"other_info": {
"user_profile": "None",
"gender": "None",
"mbti": "None",
"country": "None",
"age": "None",
}
},
recsys_type="reddit",
)
# controllable的agent_id全都在llm agent的agent_id的前面
agent = SocialAgent(
agent_id=i,
user_info=user_info,
channel=channel,
agent_graph=agent_graph,
model=models,
available_actions=None,
)
# Add agent to the agent graph
agent_graph.add_agent(agent)
user_name = "momo"
name = "momo"
bio = "None."
response = await agent.env.action.sign_up(user_name, name, bio)
user_id = response["user_id"]
agent_user_id_mapping[i] = user_id
return agent_graph, agent_user_id_mapping
async def generate_reddit_agents(
agent_info_path: str,
channel: Channel,
agent_graph: AgentGraph | None = None,
agent_user_id_mapping: dict[int, int] | None = None,
follow_post_agent: bool = False,
mute_post_agent: bool = False,
model: Optional[Union[BaseModelBackend, List[BaseModelBackend],
ModelManager]] = None,
available_actions: list[ActionType] = None,
) -> AgentGraph:
if agent_user_id_mapping is None:
agent_user_id_mapping = {}
if agent_graph is None:
agent_graph = AgentGraph()
control_user_num = agent_graph.get_num_nodes()
with open(agent_info_path, "r") as file:
agent_info = json.load(file)
async def process_agent(i):
# Instantiate an agent
profile = {
"nodes": [], # Relationships with other agents
"edges": [], # Relationship details
"other_info": {},
}
# Update agent profile with additional information
profile["other_info"]["user_profile"] = agent_info[i]["persona"]
profile["other_info"]["mbti"] = agent_info[i]["mbti"]
profile["other_info"]["gender"] = agent_info[i]["gender"]
profile["other_info"]["age"] = agent_info[i]["age"]
profile["other_info"]["country"] = agent_info[i]["country"]
user_info = UserInfo(
name=agent_info[i]["username"],
description=agent_info[i]["bio"],
profile=profile,
recsys_type="reddit",
)
agent = SocialAgent(
agent_id=i + control_user_num,
user_info=user_info,
channel=channel,
agent_graph=agent_graph,
model=model,
available_actions=available_actions,
)
# Add agent to the agent graph
agent_graph.add_agent(agent)
# Sign up agent and add their information to the database
# print(f"Signing up agent {agent_info['username'][i]}...")
response = await agent.env.action.sign_up(agent_info[i]["username"],
agent_info[i]["realname"],
agent_info[i]["bio"])
user_id = response["user_id"]
agent_user_id_mapping[i + control_user_num] = user_id
if follow_post_agent:
await agent.env.action.follow(1)
content = """
{
"reason": "He is my friend, and I would like to follow him "
"on social media.",
"functions": [
{
"name": "follow",
"arguments": {
"user_id": 1
}
}
]
}
"""
agent_msg = BaseMessage.make_assistant_message(
role_name="Assistant", content=content)
agent.memory.write_record(
MemoryRecord(agent_msg, OpenAIBackendRole.ASSISTANT))
elif mute_post_agent:
await agent.env.action.mute(1)
content = """
{
"reason": "He is my enemy, and I would like to mute him on social media.",
"functions": [{
"name": "mute",
"arguments": {
"user_id": 1
}
}
"""
agent_msg = BaseMessage.make_assistant_message(
role_name="Assistant", content=content)
agent.memory.write_record(
MemoryRecord(agent_msg, OpenAIBackendRole.ASSISTANT))
tasks = [process_agent(i) for i in range(len(agent_info))]
await asyncio.gather(*tasks)
return agent_graph
def connect_platform_channel(
channel: Channel,
agent_graph: AgentGraph | None = None,
) -> AgentGraph:
for _, agent in agent_graph.get_agents():
agent.channel = channel
agent.env.action.channel = channel
return agent_graph
async def generate_custom_agents(
channel: Channel,
agent_graph: AgentGraph | None = None,
) -> AgentGraph:
if agent_graph is None:
agent_graph = AgentGraph()
agent_graph = connect_platform_channel(channel=channel,
agent_graph=agent_graph)
sign_up_tasks = [
agent.env.action.sign_up(user_name=agent.user_info.user_name,
name=agent.user_info.name,
bio=agent.user_info.description)
for _, agent in agent_graph.get_agents()
]
await asyncio.gather(*sign_up_tasks)
return agent_graph
async def generate_reddit_agent_graph(
profile_path: str,
model: Optional[Union[BaseModelBackend, List[BaseModelBackend],
ModelManager]] = None,
available_actions: list[ActionType] = None,
) -> AgentGraph:
agent_graph = AgentGraph()
with open(profile_path, "r") as file:
agent_info = json.load(file)
async def process_agent(i):
# Instantiate an agent
profile = {
"nodes": [], # Relationships with other agents
"edges": [], # Relationship details
"other_info": {},
}
# Update agent profile with additional information
profile["other_info"]["user_profile"] = agent_info[i]["persona"]
profile["other_info"]["mbti"] = agent_info[i]["mbti"]
profile["other_info"]["gender"] = agent_info[i]["gender"]
profile["other_info"]["age"] = agent_info[i]["age"]
profile["other_info"]["country"] = agent_info[i]["country"]
user_info = UserInfo(
name=agent_info[i]["username"],
description=agent_info[i]["bio"],
profile=profile,
recsys_type="reddit",
)
agent = SocialAgent(
agent_id=i,
user_info=user_info,
agent_graph=agent_graph,
model=model,
available_actions=available_actions,
)
# Add agent to the agent graph
agent_graph.add_agent(agent)
tasks = [process_agent(i) for i in range(len(agent_info))]
await asyncio.gather(*tasks)
return agent_graph
async def generate_twitter_agent_graph(
profile_path: str,
model: Optional[Union[BaseModelBackend, List[BaseModelBackend],
ModelManager]] = None,
available_actions: list[ActionType] = None,
) -> AgentGraph:
agent_info = pd.read_csv(profile_path)
agent_graph = AgentGraph()
for agent_id in range(len(agent_info)):
profile = {
"nodes": [],
"edges": [],
"other_info": {},
}
profile["other_info"]["user_profile"] = agent_info["user_char"][
agent_id]
user_info = UserInfo(
name=agent_info["username"][agent_id],
description=agent_info["description"][agent_id],
profile=profile,
recsys_type='twitter',
)
agent = SocialAgent(
agent_id=agent_id,
user_info=user_info,
model=model,
agent_graph=agent_graph,
available_actions=available_actions,
)
agent_graph.add_agent(agent)
return agent_graph

20
backend/vendor/camel-oasis/oasis/social_platform/__init__.py vendored Normal file
View File

@ -0,0 +1,20 @@
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
# Licensed under the Apache License, Version 2.0 (the “License”);
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an “AS IS” BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
from .channel import Channel
from .platform import Platform
__all__ = [
"Channel",
"Platform",
]

71
backend/vendor/camel-oasis/oasis/social_platform/channel.py vendored Normal file
View File

@ -0,0 +1,71 @@
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
# Licensed under the Apache License, Version 2.0 (the “License”);
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an “AS IS” BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
import asyncio
import uuid
class AsyncSafeDict:
def __init__(self):
self.dict = {}
self.lock = asyncio.Lock()
async def put(self, key, value):
async with self.lock:
self.dict[key] = value
async def get(self, key, default=None):
async with self.lock:
return self.dict.get(key, default)
async def pop(self, key, default=None):
async with self.lock:
return self.dict.pop(key, default)
async def keys(self):
async with self.lock:
return list(self.dict.keys())
class Channel:
def __init__(self):
self.receive_queue = asyncio.Queue() # Used to store received messages
# Using an asynchronous safe dictionary to store messages to be sent
self.send_dict = AsyncSafeDict()
async def receive_from(self):
message = await self.receive_queue.get()
return message
async def send_to(self, message):
# message_id is the first element of the message
message_id = message[0]
await self.send_dict.put(message_id, message)
async def write_to_receive_queue(self, action_info):
message_id = str(uuid.uuid4())
await self.receive_queue.put((message_id, action_info))
return message_id
async def read_from_send_queue(self, message_id):
while True:
if message_id in await self.send_dict.keys():
# Attempting to retrieve the message
message = await self.send_dict.pop(message_id, None)
if message:
return message # Return the found message
# Temporarily suspend to avoid tight looping
await asyncio.sleep(
0.1) # set a large one to reduce the workload of cpu

20
backend/vendor/camel-oasis/oasis/social_platform/config/__init__.py vendored Normal file
View File

@ -0,0 +1,20 @@
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
# Licensed under the Apache License, Version 2.0 (the “License”);
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an “AS IS” BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
from .neo4j import Neo4jConfig
from .user import UserInfo
__all__ = [
"UserInfo",
"Neo4jConfig",
]

24
backend/vendor/camel-oasis/oasis/social_platform/config/neo4j.py vendored Normal file
View File

@ -0,0 +1,24 @@
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
# Licensed under the Apache License, Version 2.0 (the “License”);
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an “AS IS” BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
from dataclasses import dataclass
@dataclass
class Neo4jConfig:
uri: str | None = None
username: str | None = None
password: str | None = None
def is_valid(self) -> bool:
return all([self.uri, self.username, self.password])

111
backend/vendor/camel-oasis/oasis/social_platform/config/user.py vendored Normal file
View File

@ -0,0 +1,111 @@
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
# Licensed under the Apache License, Version 2.0 (the “License”);
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an “AS IS” BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
# flake8: noqa: E501
import warnings
from dataclasses import dataclass
from typing import Any
from camel.prompts import TextPrompt
@dataclass
class UserInfo:
user_name: str | None = None
name: str | None = None
description: str | None = None
profile: dict[str, Any] | None = None
recsys_type: str = "twitter"
is_controllable: bool = False
def to_custom_system_message(self, user_info_template: TextPrompt) -> str:
required_keys = user_info_template.key_words
info_keys = set(self.profile.keys())
missing = required_keys - info_keys
extra = info_keys - required_keys
if missing:
raise ValueError(
f"Missing required keys in UserInfo.profile: {missing}")
if extra:
warnings.warn(f"Extra keys not used in UserInfo.profile: {extra}")
return user_info_template.format(**self.profile)
def to_system_message(self) -> str:
if self.recsys_type != "reddit":
return self.to_twitter_system_message()
else:
return self.to_reddit_system_message()
def to_twitter_system_message(self) -> str:
name_string = ""
description_string = ""
if self.name is not None:
name_string = f"Your name is {self.name}."
if self.profile is None:
description = name_string
elif "other_info" not in self.profile:
description = name_string
elif "user_profile" in self.profile["other_info"]:
if self.profile["other_info"]["user_profile"] is not None:
user_profile = self.profile["other_info"]["user_profile"]
description_string = f"Your have profile: {user_profile}."
description = f"{name_string}\n{description_string}"
system_content = f"""
# OBJECTIVE
You're a Twitter user, and I'll present you with some posts. After you see the posts, choose some actions from the following functions.
# SELF-DESCRIPTION
Your actions should be consistent with your self-description and personality.
{description}
# RESPONSE METHOD
Please perform actions by tool calling.
"""
return system_content
def to_reddit_system_message(self) -> str:
name_string = ""
description_string = ""
if self.name is not None:
name_string = f"Your name is {self.name}."
if self.profile is None:
description = name_string
elif "other_info" not in self.profile:
description = name_string
elif "user_profile" in self.profile["other_info"]:
if self.profile["other_info"]["user_profile"] is not None:
user_profile = self.profile["other_info"]["user_profile"]
description_string = f"Your have profile: {user_profile}."
description = f"{name_string}\n{description_string}"
print(self.profile['other_info'])
description += (
f"You are a {self.profile['other_info']['gender']}, "
f"{self.profile['other_info']['age']} years old, with an MBTI "
f"personality type of {self.profile['other_info']['mbti']} from "
f"{self.profile['other_info']['country']}.")
system_content = f"""
# OBJECTIVE
You're a Reddit user, and I'll present you with some tweets. After you see the tweets, choose some actions from the following functions.
# SELF-DESCRIPTION
Your actions should be consistent with your self-description and personality.
{description}
# RESPONSE METHOD
Please perform actions by tool calling.
"""
return system_content

291
backend/vendor/camel-oasis/oasis/social_platform/database.py vendored Normal file
View File

@ -0,0 +1,291 @@
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
# Licensed under the Apache License, Version 2.0 (the “License”);
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an “AS IS” BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
from __future__ import annotations
import os
import os.path as osp
import sqlite3
from typing import Any, Dict, List
SCHEMA_DIR = "social_platform/schema"
DB_DIR = "data"
DB_NAME = "social_media.db"
USER_SCHEMA_SQL = "user.sql"
POST_SCHEMA_SQL = "post.sql"
FOLLOW_SCHEMA_SQL = "follow.sql"
MUTE_SCHEMA_SQL = "mute.sql"
LIKE_SCHEMA_SQL = "like.sql"
DISLIKE_SCHEMA_SQL = "dislike.sql"
REPORT_SCHEAM_SQL = "report.sql"
TRACE_SCHEMA_SQL = "trace.sql"
REC_SCHEMA_SQL = "rec.sql"
COMMENT_SCHEMA_SQL = "comment.sql"
COMMENT_LIKE_SCHEMA_SQL = "comment_like.sql"
COMMENT_DISLIKE_SCHEMA_SQL = "comment_dislike.sql"
PRODUCT_SCHEMA_SQL = "product.sql"
GROUP_SCHEMA_SQL = "chat_group.sql"
GROUP_MEMBER_SCHEMA_SQL = "group_member.sql"
GROUP_MESSAGE_SCHEMA_SQL = "group_message.sql"
TABLE_NAMES = {
"user",
"post",
"follow",
"mute",
"like",
"dislike",
"report",
"trace",
"rec",
"comment.sql",
"comment_like.sql",
"comment_dislike.sql",
"product.sql",
"group",
"group_member",
"group_message",
}
def get_db_path() -> str:
# First check if the database path is set in environment variables
env_db_path = os.environ.get("OASIS_DB_PATH")
if env_db_path:
return env_db_path
# If no environment variable is set, use the original default path
curr_file_path = osp.abspath(__file__)
parent_dir = osp.dirname(osp.dirname(curr_file_path))
db_dir = osp.join(parent_dir, DB_DIR)
os.makedirs(db_dir, exist_ok=True)
db_path = osp.join(db_dir, DB_NAME)
return db_path
def get_schema_dir_path() -> str:
curr_file_path = osp.abspath(__file__)
parent_dir = osp.dirname(osp.dirname(curr_file_path))
schema_dir = osp.join(parent_dir, SCHEMA_DIR)
return schema_dir
def create_db(db_path: str | None = None):
r"""Create the database if it does not exist. A :obj:`twitter.db`
file will be automatically created in the :obj:`data` directory.
"""
schema_dir = get_schema_dir_path()
if db_path is None:
db_path = get_db_path()
# Connect to the database:
print("db_path", db_path)
conn = sqlite3.connect(db_path)
cursor = conn.cursor()
try:
# Read and execute the user table SQL script:
user_sql_path = osp.join(schema_dir, USER_SCHEMA_SQL)
with open(user_sql_path, "r") as sql_file:
user_sql_script = sql_file.read()
cursor.executescript(user_sql_script)
# Read and execute the post table SQL script:
post_sql_path = osp.join(schema_dir, POST_SCHEMA_SQL)
with open(post_sql_path, "r") as sql_file:
post_sql_script = sql_file.read()
cursor.executescript(post_sql_script)
# Read and execute the follow table SQL script:
follow_sql_path = osp.join(schema_dir, FOLLOW_SCHEMA_SQL)
with open(follow_sql_path, "r") as sql_file:
follow_sql_script = sql_file.read()
cursor.executescript(follow_sql_script)
# Read and execute the mute table SQL script:
mute_sql_path = osp.join(schema_dir, MUTE_SCHEMA_SQL)
with open(mute_sql_path, "r") as sql_file:
mute_sql_script = sql_file.read()
cursor.executescript(mute_sql_script)
# Read and execute the like table SQL script:
like_sql_path = osp.join(schema_dir, LIKE_SCHEMA_SQL)
with open(like_sql_path, "r") as sql_file:
like_sql_script = sql_file.read()
cursor.executescript(like_sql_script)
# Read and execute the dislike table SQL script:
dislike_sql_path = osp.join(schema_dir, DISLIKE_SCHEMA_SQL)
with open(dislike_sql_path, "r") as sql_file:
dislike_sql_script = sql_file.read()
cursor.executescript(dislike_sql_script)
# Read and execute the report table SQL script:
report_sql_path = osp.join(schema_dir, REPORT_SCHEAM_SQL)
with open(report_sql_path, "r") as sql_file:
report_sql_script = sql_file.read()
cursor.executescript(report_sql_script)
# Read and execute the trace table SQL script:
trace_sql_path = osp.join(schema_dir, TRACE_SCHEMA_SQL)
with open(trace_sql_path, "r") as sql_file:
trace_sql_script = sql_file.read()
cursor.executescript(trace_sql_script)
# Read and execute the rec table SQL script:
rec_sql_path = osp.join(schema_dir, REC_SCHEMA_SQL)
with open(rec_sql_path, "r") as sql_file:
rec_sql_script = sql_file.read()
cursor.executescript(rec_sql_script)
# Read and execute the comment table SQL script:
comment_sql_path = osp.join(schema_dir, COMMENT_SCHEMA_SQL)
with open(comment_sql_path, "r") as sql_file:
comment_sql_script = sql_file.read()
cursor.executescript(comment_sql_script)
# Read and execute the comment_like table SQL script:
comment_like_sql_path = osp.join(schema_dir, COMMENT_LIKE_SCHEMA_SQL)
with open(comment_like_sql_path, "r") as sql_file:
comment_like_sql_script = sql_file.read()
cursor.executescript(comment_like_sql_script)
# Read and execute the comment_dislike table SQL script:
comment_dislike_sql_path = osp.join(schema_dir,
COMMENT_DISLIKE_SCHEMA_SQL)
with open(comment_dislike_sql_path, "r") as sql_file:
comment_dislike_sql_script = sql_file.read()
cursor.executescript(comment_dislike_sql_script)
# Read and execute the product table SQL script:
product_sql_path = osp.join(schema_dir, PRODUCT_SCHEMA_SQL)
with open(product_sql_path, "r") as sql_file:
product_sql_script = sql_file.read()
cursor.executescript(product_sql_script)
# Read and execute the group table SQL script:
group_sql_path = osp.join(schema_dir, GROUP_SCHEMA_SQL)
with open(group_sql_path, "r") as sql_file:
group_sql_script = sql_file.read()
cursor.executescript(group_sql_script)
# Read and execute the group_member table SQL script:
group_member_sql_path = osp.join(schema_dir, GROUP_MEMBER_SCHEMA_SQL)
with open(group_member_sql_path, "r") as sql_file:
group_member_sql_script = sql_file.read()
cursor.executescript(group_member_sql_script)
# Read and execute the group_message table SQL script:
group_message_sql_path = osp.join(schema_dir, GROUP_MESSAGE_SCHEMA_SQL)
with open(group_message_sql_path, "r") as sql_file:
group_message_sql_script = sql_file.read()
cursor.executescript(group_message_sql_script)
# Commit the changes:
conn.commit()
except sqlite3.Error as e:
print(f"An error occurred while creating tables: {e}")
return conn, cursor
def print_db_tables_summary():
# Connect to the SQLite database
db_path = get_db_path()
conn = sqlite3.connect(db_path)
cursor = conn.cursor()
# Retrieve a list of all tables in the database
cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
tables = cursor.fetchall()
# Print a summary of each table
for table in tables:
table_name = table[0]
if table_name not in TABLE_NAMES:
continue
print(f"Table: {table_name}")
# Retrieve the table schema
cursor.execute(f"PRAGMA table_info({table_name})")
columns = cursor.fetchall()
column_names = [column[1] for column in columns]
print("- Columns:", column_names)
# Retrieve and print foreign key information
cursor.execute(f"PRAGMA foreign_key_list({table_name})")
foreign_keys = cursor.fetchall()
if foreign_keys:
print("- Foreign Keys:")
for fk in foreign_keys:
print(f" {fk[2]} references {fk[3]}({fk[4]}) on update "
f"{fk[5]} on delete {fk[6]}")
else:
print(" No foreign keys.")
# Print the first few rows of the table
cursor.execute(f"SELECT * FROM {table_name} LIMIT 5;")
rows = cursor.fetchall()
for row in rows:
print(row)
print() # Adds a newline for better readability between tables
# Close the database connection
conn.close()
def fetch_table_from_db(cursor: sqlite3.Cursor,
table_name: str) -> List[Dict[str, Any]]:
cursor.execute(f"SELECT * FROM {table_name}")
columns = [description[0] for description in cursor.description]
data_dicts = [dict(zip(columns, row)) for row in cursor.fetchall()]
return data_dicts
def fetch_rec_table_as_matrix(cursor: sqlite3.Cursor) -> List[List[int]]:
# First, query all user_ids from the user table, assuming they start from
# 1 and are consecutive
cursor.execute("SELECT user_id FROM user ORDER BY user_id")
user_ids = [row[0] for row in cursor.fetchall()]
# Then, query all records from the rec table
cursor.execute(
"SELECT user_id, post_id FROM rec ORDER BY user_id, post_id")
rec_rows = cursor.fetchall()
# Initialize a dictionary, assigning an empty list to each user_id
user_posts = {user_id: [] for user_id in user_ids}
# Fill the dictionary with the records queried from the rec table
for user_id, post_id in rec_rows:
if user_id in user_posts:
user_posts[user_id].append(post_id)
# Convert the dictionary into matrix form
matrix = [user_posts[user_id] for user_id in user_ids]
return matrix
def insert_matrix_into_rec_table(cursor: sqlite3.Cursor,
matrix: List[List[int]]) -> None:
# Iterate through the matrix, skipping the placeholder at index 0
for user_id, post_ids in enumerate(matrix, start=1):
# Adjusted to start counting from 1
for post_id in post_ids:
# Insert each combination of user_id and post_id into the rec table
cursor.execute("INSERT INTO rec (user_id, post_id) VALUES (?, ?)",
(user_id, post_id))
if __name__ == "__main__":
create_db()
print_db_tables_summary()

1642
backend/vendor/camel-oasis/oasis/social_platform/platform.py vendored Normal file
View File

File diff suppressed because it is too large Load Diff

262
backend/vendor/camel-oasis/oasis/social_platform/platform_utils.py vendored Normal file
View File

@ -0,0 +1,262 @@
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
# Licensed under the Apache License, Version 2.0 (the “License”);
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an “AS IS” BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
import json
from datetime import datetime
from oasis.social_platform.typing import RecsysType
class PlatformUtils:
def __init__(self,
db,
db_cursor,
start_time,
sandbox_clock,
show_score,
recsys_type,
report_threshold=1):
self.db = db
self.db_cursor = db_cursor
self.start_time = start_time
self.sandbox_clock = sandbox_clock
self.show_score = show_score
self.recsys_type = recsys_type
self.report_threshold = report_threshold
@staticmethod
def _not_signup_error_message(agent_id):
return {
"success":
False,
"error": (f"Agent {agent_id} has not signed up and does not have "
f"a user id."),
}
def _execute_db_command(self, command, args=(), commit=False):
self.db_cursor.execute(command, args)
if commit:
self.db.commit()
return self.db_cursor
def _execute_many_db_command(self, command, args_list, commit=False):
self.db_cursor.executemany(command, args_list)
if commit:
self.db.commit()
return self.db_cursor
def _check_agent_userid(self, agent_id):
try:
user_query = "SELECT user_id FROM user WHERE agent_id = ?"
results = self._execute_db_command(user_query, (agent_id, ))
# Fetch the first row of the query result
first_row = results.fetchone()
if first_row:
user_id = first_row[0]
return user_id
else:
return None
except Exception as e:
# Log or handle the error as appropriate
print(f"Error querying user_id for agent_id {agent_id}: {e}")
return None
def _add_comments_to_posts(self, posts_results):
# Initialize the returned posts list
posts = []
for row in posts_results:
(post_id, user_id, original_post_id, content, quote_content,
created_at, num_likes, num_dislikes, num_shares) = row
post_type_result = self._get_post_type(post_id)
if post_type_result is None:
continue
original_user_id_query = (
"SELECT user_id FROM post WHERE post_id = ?")
if post_type_result["type"] == "repost":
self.db_cursor.execute(original_user_id_query,
(original_post_id, ))
original_user_id = self.db_cursor.fetchone()[0]
original_post_id = post_id
post_id = post_type_result["root_post_id"]
self.db_cursor.execute(
"SELECT content, quote_content, created_at, num_likes, "
"num_dislikes, num_shares, num_reports FROM post "
"WHERE post_id = ?", (post_id, ))
original_post_result = self.db_cursor.fetchone()
(content, quote_content, created_at, num_likes, num_dislikes,
num_shares, num_reports) = original_post_result
post_content = (
f"User {user_id} reposted a post from User "
f"{original_user_id}. Repost content: {content}. ")
elif post_type_result["type"] == "quote":
self.db_cursor.execute(original_user_id_query,
(original_post_id, ))
original_user_id = self.db_cursor.fetchone()[0]
post_content = (
f"User {user_id} quoted a post from User "
f"{original_user_id}. Quote content: {quote_content}. "
f"Original Content: {content}")
elif post_type_result["type"] == "common":
post_content = content
# Get num_reports for common posts
self.db_cursor.execute(
"SELECT num_reports FROM post WHERE post_id = ?",
(post_id, ))
num_reports = self.db_cursor.fetchone()[0]
# For each post, query its corresponding comments
self.db_cursor.execute(
"SELECT comment_id, post_id, user_id, content, created_at, "
"num_likes, num_dislikes FROM comment WHERE post_id = ?",
(post_id, ),
)
comments_results = self.db_cursor.fetchall()
# Convert each comment's result into dictionary format
comments = [{
"comment_id":
comment_id,
"post_id":
post_id,
"user_id":
user_id,
"content":
content,
"created_at":
created_at,
**({
"score": num_likes - num_dislikes
} if self.show_score else {
"num_likes": num_likes,
"num_dislikes": num_dislikes
}),
} for (
comment_id,
post_id,
user_id,
content,
created_at,
num_likes,
num_dislikes,
) in comments_results]
# Add warning message if the post has been reported
if num_reports >= self.report_threshold:
warning_message = ("[Warning: This post has been reported"
f" {num_reports} times]")
post_content = f"{warning_message}\n{post_content}"
# Add post information and corresponding comments to the posts list
posts.append({
"post_id":
post_id
if post_type_result["type"] != "repost" else original_post_id,
"user_id":
user_id,
"content":
post_content,
"created_at":
created_at,
**({
"score": num_likes - num_dislikes
} if self.show_score else {
"num_likes": num_likes,
"num_dislikes": num_dislikes
}),
"num_shares":
num_shares,
"num_reports":
num_reports,
"comments":
comments,
})
return posts
def _record_trace(self,
user_id,
action_type,
action_info,
current_time=None):
r"""If, in addition to the trace, the operation function also records
time in other tables of the database, use the time of entering
the operation function for consistency.
Pass in current_time to make, for example, the created_at in the post
table exactly the same as the time in the trace table.
If only the trace table needs to record time, use the entry time into
_record_trace as the time for the trace record.
"""
if self.recsys_type == RecsysType.REDDIT:
current_time = self.sandbox_clock.time_transfer(
datetime.now(), self.start_time)
else:
current_time = self.sandbox_clock.get_time_step()
trace_insert_query = (
"INSERT INTO trace (user_id, created_at, action, info) "
"VALUES (?, ?, ?, ?)")
action_info_str = json.dumps(action_info)
self._execute_db_command(
trace_insert_query,
(user_id, current_time, action_type, action_info_str),
commit=True,
)
def _check_self_post_rating(self, post_id, user_id):
self_like_check_query = "SELECT user_id FROM post WHERE post_id = ?"
self._execute_db_command(self_like_check_query, (post_id, ))
result = self.db_cursor.fetchone()
if result and result[0] == user_id:
error_message = ("Users are not allowed to like/dislike their own "
"posts.")
return {"success": False, "error": error_message}
else:
return None
def _check_self_comment_rating(self, comment_id, user_id):
self_like_check_query = ("SELECT user_id FROM comment WHERE "
"comment_id = ?")
self._execute_db_command(self_like_check_query, (comment_id, ))
result = self.db_cursor.fetchone()
if result and result[0] == user_id:
error_message = ("Users are not allowed to like/dislike their "
"own comments.")
return {"success": False, "error": error_message}
else:
return None
def _get_post_type(self, post_id: int):
query = (
"SELECT original_post_id, quote_content FROM post WHERE post_id "
"= ?")
self._execute_db_command(query, (post_id, ))
result = self.db_cursor.fetchone()
if not result:
return None
original_post_id, quote_content = result
if original_post_id is None:
# common post without quote or repost
return {"type": "common", "root_post_id": None}
elif quote_content is None:
# post with repost
return {"type": "repost", "root_post_id": original_post_id}
else:
# post with quote
return {"type": "quote", "root_post_id": original_post_id}

81
backend/vendor/camel-oasis/oasis/social_platform/process_recsys_posts.py vendored Normal file
View File

@ -0,0 +1,81 @@
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
# Licensed under the Apache License, Version 2.0 (the “License”);
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an “AS IS” BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
from typing import List
import torch
from camel.embeddings import OpenAIEmbedding
from camel.types import EmbeddingModelType
from transformers import AutoModel, AutoTokenizer
# Function: Process each batch
@torch.no_grad()
def process_batch(model: AutoModel, tokenizer: AutoTokenizer,
batch_texts: List[str]):
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
inputs = tokenizer(batch_texts,
return_tensors="pt",
padding=True,
truncation=True)
inputs = {key: value.to(device) for key, value in inputs.items()}
outputs = model(**inputs)
return outputs.pooler_output
def generate_post_vector(model: AutoModel, tokenizer: AutoTokenizer, texts,
batch_size):
# Loop through all messages
# If the list of messages is too large, process them in batches.
all_outputs = []
for i in range(0, len(texts), batch_size):
batch_texts = texts[i:i + batch_size]
batch_outputs = process_batch(model, tokenizer, batch_texts)
all_outputs.append(batch_outputs)
all_outputs_tensor = torch.cat(all_outputs, dim=0) # num_posts x dimension
return all_outputs_tensor.cpu()
def generate_post_vector_openai(texts: List[str], batch_size: int = 100):
"""
Generate embeddings using OpenAI API
Args:
texts: List of texts to process
batch_size: Size of each batch
"""
openai_embedding = OpenAIEmbedding(
model_type=EmbeddingModelType.TEXT_EMBEDDING_3_SMALL)
all_embeddings = []
for i in range(0, len(texts), batch_size):
batch_texts = texts[i:i + batch_size]
cleaned_texts = [
text.strip() if text and isinstance(text, str) else "empty"
for text in batch_texts
]
batch_embeddings = openai_embedding.embed_list(objs=cleaned_texts)
batch_tensor = torch.tensor(batch_embeddings)
all_embeddings.append(batch_tensor)
return torch.cat(all_embeddings, dim=0)
if __name__ == "__main__":
# Input list of strings (assuming there are tens of thousands of messages)
# Here, the same message is repeated 10000 times as an example
texts = ["I'm using TwHIN-BERT! #TwHIN-BERT #NLP"] * 10000
# Define batch size
batch_size = 100
all_outputs_tensor = generate_post_vector(texts, batch_size)
print(all_outputs_tensor.shape)

797
backend/vendor/camel-oasis/oasis/social_platform/recsys.py vendored Normal file
View File

@ -0,0 +1,797 @@
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
# Licensed under the Apache License, Version 2.0 (the “License”);
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an “AS IS” BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
'''Note that you need to check if it exceeds max_rec_post_len when writing
into rec_matrix'''
import heapq
import logging
import random
import time
from ast import literal_eval
from datetime import datetime
from math import log
from typing import Any, Dict, List
import numpy as np
import torch
from sentence_transformers import SentenceTransformer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
from .process_recsys_posts import (generate_post_vector,
generate_post_vector_openai)
from .typing import ActionType, RecsysType
rec_log = logging.getLogger(name='social.rec')
rec_log.setLevel('DEBUG')
# Initially set to None, to be assigned once again in the recsys function
model = None
twhin_tokenizer = None
twhin_model = None
# Create the TF-IDF model
tfidf_vectorizer = TfidfVectorizer()
# Prepare the twhin model
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# All historical tweets and the most recent tweet of each user
user_previous_post_all = {}
user_previous_post = {}
user_profiles = []
# Get the {post_id: content} dict
t_items = {}
# Get the {uid: follower_count} dict
# It's necessary to ensure that agent registration is sequential, with the
# relationship of user_id=agent_id+1; disorder in registration will cause
# issues here
u_items = {}
# Get the creation times of all tweets, assigning scores based on how recent
# they are
date_score = []
def get_twhin_tokenizer():
global twhin_tokenizer
if twhin_tokenizer is None:
from transformers import AutoTokenizer
twhin_tokenizer = AutoTokenizer.from_pretrained(
pretrained_model_name_or_path="Twitter/twhin-bert-base",
model_max_length=512)
return twhin_tokenizer
def get_twhin_model(device):
global twhin_model
if twhin_model is None:
from transformers import AutoModel
twhin_model = AutoModel.from_pretrained(
pretrained_model_name_or_path="Twitter/twhin-bert-base").to(device)
return twhin_model
def load_model(model_name):
try:
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
if model_name == 'paraphrase-MiniLM-L6-v2':
return SentenceTransformer(model_name,
device=device,
cache_folder="./models")
elif model_name == 'Twitter/twhin-bert-base':
twhin_tokenizer = get_twhin_tokenizer()
twhin_model = get_twhin_model(device)
return twhin_tokenizer, twhin_model
else:
raise ValueError(f"Unknown model name: {model_name}")
except Exception as e:
raise Exception(f"Failed to load the model: {model_name}") from e
def get_recsys_model(recsys_type: str = None):
if recsys_type == RecsysType.TWITTER.value:
model = load_model('paraphrase-MiniLM-L6-v2')
return model
elif recsys_type == RecsysType.TWHIN.value:
twhin_tokenizer, twhin_model = load_model("Twitter/twhin-bert-base")
models = (twhin_tokenizer, twhin_model)
return models
elif (recsys_type == RecsysType.REDDIT.value
or recsys_type == RecsysType.RANDOM.value):
return None
else:
raise ValueError(f"Unknown recsys type: {recsys_type}")
# Move model to GPU if available
device = 'cuda' if torch.cuda.is_available() else 'cpu'
if model is not None:
model.to(device)
else:
pass
# Reset global variables
def reset_globals():
global user_previous_post_all, user_previous_post
global user_profiles, t_items, u_items
global date_score
user_previous_post_all = {}
user_previous_post = {}
user_profiles = []
t_items = {}
u_items = {}
date_score = []
def rec_sys_random(post_table: List[Dict[str, Any]], rec_matrix: List[List],
max_rec_post_len: int) -> List[List]:
"""
Randomly recommend posts to users.
Args:
user_table (List[Dict[str, Any]]): List of users.
post_table (List[Dict[str, Any]]): List of posts.
trace_table (List[Dict[str, Any]]): List of user interactions.
rec_matrix (List[List]): Existing recommendation matrix.
max_rec_post_len (int): Maximum number of recommended posts.
Returns:
List[List]: Updated recommendation matrix.
"""
# Get all post IDs
post_ids = [post['post_id'] for post in post_table]
new_rec_matrix = []
if len(post_ids) <= max_rec_post_len:
# If the number of posts is less than or equal to the maximum number
# of recommendations, each user gets all post IDs
new_rec_matrix = [post_ids] * len(rec_matrix)
else:
# If the number of posts is greater than the maximum number of
# recommendations, each user randomly gets a specified number of post
# IDs
for _ in range(len(rec_matrix)):
new_rec_matrix.append(random.sample(post_ids, max_rec_post_len))
return new_rec_matrix
def calculate_hot_score(num_likes: int, num_dislikes: int,
created_at: datetime) -> int:
"""
Compute the hot score for a post.
Args:
num_likes (int): Number of likes.
num_dislikes (int): Number of dislikes.
created_at (datetime): Creation time of the post.
Returns:
int: Hot score of the post.
Reference:
https://medium.com/hacking-and-gonzo/how-reddit-ranking-algorithms-work-ef111e33d0d9
"""
s = num_likes - num_dislikes
order = log(max(abs(s), 1), 10)
sign = 1 if s > 0 else -1 if s < 0 else 0
# epoch_seconds
epoch = datetime(1970, 1, 1)
td = created_at - epoch
epoch_seconds_result = td.days * 86400 + td.seconds + (
float(td.microseconds) / 1e6)
seconds = epoch_seconds_result - 1134028003
return round(sign * order + seconds / 45000, 7)
def get_recommendations(
user_index,
cosine_similarities,
items,
score,
top_n=100,
):
similarities = np.array(cosine_similarities[user_index])
similarities = similarities * score
top_item_indices = similarities.argsort()[::-1][:top_n]
recommended_items = [(list(items.keys())[i], similarities[i])
for i in top_item_indices]
return recommended_items
def rec_sys_reddit(post_table: List[Dict[str, Any]], rec_matrix: List[List],
max_rec_post_len: int) -> List[List]:
"""
Recommend posts based on Reddit-like hot score.
Args:
post_table (List[Dict[str, Any]]): List of posts.
rec_matrix (List[List]): Existing recommendation matrix.
max_rec_post_len (int): Maximum number of recommended posts.
Returns:
List[List]: Updated recommendation matrix.
"""
# Get all post IDs
post_ids = [post['post_id'] for post in post_table]
if len(post_ids) <= max_rec_post_len:
# If the number of posts is less than or equal to the maximum number
# of recommendations, each user gets all post IDs
new_rec_matrix = [post_ids] * len(rec_matrix)
else:
# The time complexity of this recommendation system is
# O(post_num * log max_rec_post_len)
all_hot_score = []
for post in post_table:
try:
created_at_dt = datetime.strptime(post['created_at'],
"%Y-%m-%d %H:%M:%S.%f")
except Exception:
created_at_dt = datetime.strptime(post['created_at'],
"%Y-%m-%d %H:%M:%S")
hot_score = calculate_hot_score(post['num_likes'],
post['num_dislikes'],
created_at_dt)
all_hot_score.append((hot_score, post['post_id']))
# Sort
top_posts = heapq.nlargest(max_rec_post_len,
all_hot_score,
key=lambda x: x[0])
top_post_ids = [post_id for _, post_id in top_posts]
# If the number of posts is greater than the maximum number of
# recommendations, each user gets a specified number of post IDs
# randomly
new_rec_matrix = [top_post_ids] * len(rec_matrix)
return new_rec_matrix
def rec_sys_personalized(user_table: List[Dict[str, Any]],
post_table: List[Dict[str, Any]],
trace_table: List[Dict[str,
Any]], rec_matrix: List[List],
max_rec_post_len: int) -> List[List]:
"""
Recommend posts based on personalized similarity scores.
Args:
user_table (List[Dict[str, Any]]): List of users.
post_table (List[Dict[str, Any]]): List of posts.
trace_table (List[Dict[str, Any]]): List of user interactions.
rec_matrix (List[List]): Existing recommendation matrix.
max_rec_post_len (int): Maximum number of recommended posts.
Returns:
List[List]: Updated recommendation matrix.
"""
global model
if model is None or isinstance(model, tuple):
model = get_recsys_model(recsys_type="twitter")
post_ids = [post['post_id'] for post in post_table]
print(
f'Running personalized recommendation for {len(user_table)} users...')
start_time = time.time()
new_rec_matrix = []
if len(post_ids) <= max_rec_post_len:
# If the number of posts is less than or equal to the maximum
# recommended length, each user gets all post IDs
new_rec_matrix = [post_ids] * len(rec_matrix)
else:
# If the number of posts is greater than the maximum recommended
# length, each user gets personalized post IDs
user_bios = [
user['bio'] if 'bio' in user and user['bio'] is not None else ''
for user in user_table
]
post_contents = [post['content'] for post in post_table]
if model:
user_embeddings = model.encode(user_bios,
convert_to_tensor=True,
device=device)
post_embeddings = model.encode(post_contents,
convert_to_tensor=True,
device=device)
# Compute dot product similarity
dot_product = torch.matmul(user_embeddings, post_embeddings.T)
# Compute norm
user_norms = torch.norm(user_embeddings, dim=1)
post_norms = torch.norm(post_embeddings, dim=1)
# Compute cosine similarity
similarities = dot_product / (user_norms[:, None] *
post_norms[None, :])
else:
# Generate random similarities
similarities = torch.rand(len(user_table), len(post_table))
# Iterate through each user to generate personalized recommendations.
for user_index, user in enumerate(user_table):
# Filter out posts made by the current user.
filtered_post_indices = [
i for i, post in enumerate(post_table)
if post['user_id'] != user['user_id']
]
user_similarities = similarities[user_index, filtered_post_indices]
# Get the corresponding post IDs for the filtered posts.
filtered_post_ids = [
post_table[i]['post_id'] for i in filtered_post_indices
]
# Determine the top posts based on the similarities, limited by
# max_rec_post_len.
_, top_indices = torch.topk(user_similarities,
k=min(max_rec_post_len,
len(filtered_post_ids)))
top_post_ids = [filtered_post_ids[i] for i in top_indices.tolist()]
# Append the top post IDs to the new recommendation matrix.
new_rec_matrix.append(top_post_ids)
end_time = time.time()
print(f'Personalized recommendation time: {end_time - start_time:.6f}s')
return new_rec_matrix
def get_like_post_id(user_id, action, trace_table):
"""
Get the post IDs that a user has liked or unliked.
Args:
user_id (str): ID of the user.
action (str): Type of action (like or unlike).
post_table (list): List of posts.
trace_table (list): List of user interactions.
Returns:
list: List of post IDs.
"""
# Get post IDs from trace table for the given user and action
trace_post_ids = [
literal_eval(trace['info'])["post_id"] for trace in trace_table
if (trace['user_id'] == user_id and trace['action'] == action)
]
"""Only take the last 5 liked posts, if not enough, pad with the most
recently liked post. Only take IDs, not content, because calculating
embeddings for all posts again is very time-consuming, especially when the
number of agents is large"""
if len(trace_post_ids) < 5 and len(trace_post_ids) > 0:
trace_post_ids += [trace_post_ids[-1]] * (5 - len(trace_post_ids))
elif len(trace_post_ids) > 5:
trace_post_ids = trace_post_ids[-5:]
else:
trace_post_ids = [0]
return trace_post_ids
# Calculate the average cosine similarity between liked posts and target posts
def calculate_like_similarity(liked_vectors, target_vectors):
# Calculate the norms of the vectors
liked_norms = np.linalg.norm(liked_vectors, axis=1)
target_norms = np.linalg.norm(target_vectors, axis=1)
# Calculate dot products
dot_products = np.dot(target_vectors, liked_vectors.T)
# Calculate cosine similarities
cosine_similarities = dot_products / np.outer(target_norms, liked_norms)
# Take the average
average_similarities = np.mean(cosine_similarities, axis=1)
return average_similarities
def coarse_filtering(input_list, scale):
"""
Coarse filtering posts and return selected elements with their indices.
"""
if len(input_list) <= scale:
# Return elements and their indices as list of tuples (element, index)
sampled_indices = range(len(input_list))
return (input_list, sampled_indices)
else:
# Get random sample of scale elements
sampled_indices = random.sample(range(len(input_list)), scale)
sampled_elements = [input_list[idx] for idx in sampled_indices]
# return [(input_list[idx], idx) for idx in sampled_indices]
return (sampled_elements, sampled_indices)
def rec_sys_personalized_twh(
user_table: List[Dict[str, Any]],
post_table: List[Dict[str, Any]],
latest_post_count: int,
trace_table: List[Dict[str, Any]],
rec_matrix: List[List],
max_rec_post_len: int,
current_time: int,
# source_post_indexs: List[int],
recall_only: bool = False,
enable_like_score: bool = False,
use_openai_embedding: bool = False) -> List[List]:
global twhin_model, twhin_tokenizer
if twhin_model is None or twhin_tokenizer is None:
twhin_tokenizer, twhin_model = get_recsys_model(
recsys_type="twhin-bert")
# Set some global variables to reduce time consumption
global date_score, t_items, u_items, user_previous_post
global user_previous_post_all, user_profiles
# Get the uid: follower_count dict
# Update only once, unless adding the feature to include new users midway.
if (not u_items) or len(u_items) != len(user_table):
u_items = {
user['user_id']: user["num_followers"]
for user in user_table
}
if not user_previous_post_all or len(user_previous_post_all) != len(
user_table):
# Each user must have a list of historical tweets
user_previous_post_all = {
index: []
for index in range(len(user_table))
}
user_previous_post = {index: "" for index in range(len(user_table))}
if not user_profiles or len(user_profiles) != len(user_table):
for user in user_table:
if user['bio'] is None:
user_profiles.append('This user does not have profile')
else:
user_profiles.append(user['bio'])
if len(t_items) < len(post_table):
for post in post_table[-latest_post_count:]:
# Get the {post_id: content} dict, update only the latest tweets
t_items[post['post_id']] = post['content']
# Update the user's historical tweets
user_previous_post_all[post['user_id']].append(post['content'])
user_previous_post[post['user_id']] = post['content']
# Get the creation times of all tweets, assigning scores based on
# how recent they are, note that this algorithm can run for a
# maximum of 90 time steps
date_score.append(
np.log(
(271.8 - (current_time - int(post['created_at']))) / 100))
date_score_np = np.array(date_score)
if enable_like_score:
# Calculate similarity with previously liked content, first gather
# liked post ids from the trace
like_post_ids_all = []
for user in user_table:
user_id = user['agent_id']
like_post_ids = get_like_post_id(user_id,
ActionType.LIKE_POST.value,
trace_table)
like_post_ids_all.append(like_post_ids)
scores = date_score_np
new_rec_matrix = []
if len(post_table) <= max_rec_post_len:
# If the number of tweets is less than or equal to the max
# recommendation count, each user gets all post IDs
tids = [t['post_id'] for t in post_table]
new_rec_matrix = [tids] * (len(rec_matrix))
else:
# If the number of tweets is greater than the max recommendation
# count, each user randomly gets personalized post IDs
# This requires going through all users to update their profiles,
# which is a time-consuming operation
for post_user_index in user_previous_post:
try:
# Directly replacing the profile with the latest tweet will
# cause the recommendation system to repeatedly push other
# reposts to users who have already shared that tweet
# user_profiles[post_user_index] =
# user_previous_post[post_user_index]
# Instead, append the description of the Recent post's content
# to the end of the user char
update_profile = (
f" # Recent post:{user_previous_post[post_user_index]}")
if user_previous_post[post_user_index] != "":
# If there's no update for the recent post, add this part
if "# Recent post:" not in user_profiles[post_user_index]:
user_profiles[post_user_index] += update_profile
# If the profile has a recent post but it's not the user's
# latest, replace it
elif update_profile not in user_profiles[post_user_index]:
user_profiles[post_user_index] = user_profiles[
post_user_index].split(
"# Recent post:")[0] + update_profile
except Exception:
print("update previous post failed")
# coarse filtering 4000 posts due to the memory constraint.
filtered_posts_tuple = coarse_filtering(list(t_items.values()), 4000)
corpus = user_profiles + filtered_posts_tuple[0]
# corpus = user_profiles + list(t_items.values())
tweet_vector_start_t = time.time()
if use_openai_embedding:
all_post_vector_list = generate_post_vector_openai(corpus,
batch_size=1000)
else:
all_post_vector_list = generate_post_vector(twhin_model,
twhin_tokenizer,
corpus,
batch_size=1000)
tweet_vector_end_t = time.time()
rec_log.info(
f"twhin model cost time: {tweet_vector_end_t-tweet_vector_start_t}"
)
user_vector = all_post_vector_list[:len(user_profiles)]
posts_vector = all_post_vector_list[len(user_profiles):]
if enable_like_score:
# Traverse all liked post ids, collecting liked post vectors from
# posts_vector for matrix acceleration calculation
like_posts_vectors = []
for user_idx, like_post_ids in enumerate(like_post_ids_all):
if len(like_post_ids) != 1:
for like_post_id in like_post_ids:
try:
like_posts_vectors.append(
posts_vector[like_post_id - 1])
except Exception:
like_posts_vectors.append(user_vector[user_idx])
else:
like_posts_vectors += [
user_vector[user_idx] for _ in range(5)
]
try:
like_posts_vectors = torch.stack(like_posts_vectors).view(
len(user_table), 5, posts_vector.shape[1])
except Exception:
import pdb # noqa: F811
pdb.set_trace()
get_similar_start_t = time.time()
cosine_similarities = cosine_similarity(user_vector, posts_vector)
get_similar_end_t = time.time()
rec_log.info(f"get cosine_similarity time: "
f"{get_similar_end_t-get_similar_start_t}")
if enable_like_score:
for user_index, profile in enumerate(user_profiles):
user_like_posts_vector = like_posts_vectors[user_index]
like_scores = calculate_like_similarity(
user_like_posts_vector, posts_vector)
try:
scores = scores + like_scores
except Exception:
import pdb
pdb.set_trace()
filter_posts_index = filtered_posts_tuple[1]
cosine_similarities = cosine_similarities * scores[filter_posts_index]
cosine_similarities = torch.tensor(cosine_similarities)
value, indices = torch.topk(cosine_similarities,
max_rec_post_len,
dim=1,
largest=True,
sorted=True)
filter_posts_index = torch.tensor(filter_posts_index)
indices = filter_posts_index[indices]
# cosine_similarities = cosine_similarities * scores
# cosine_similarities = torch.tensor(cosine_similarities)
# value, indices = torch.topk(cosine_similarities,
# max_rec_post_len,
# dim=1,
# largest=True,
# sorted=True)
matrix_list = indices.cpu().numpy()
post_list = list(t_items.keys())
for rec_ids in matrix_list:
rec_ids = [post_list[i] for i in rec_ids]
new_rec_matrix.append(rec_ids)
return new_rec_matrix
def normalize_similarity_adjustments(post_scores, base_similarity,
like_similarity, dislike_similarity):
"""
Normalize the adjustments to keep them in scale with overall similarities.
Args:
post_scores (list): List of post scores.
base_similarity (float): Base similarity score.
like_similarity (float): Similarity score for liked posts.
dislike_similarity (float): Similarity score for disliked posts.
Returns:
float: Adjusted similarity score.
"""
if len(post_scores) == 0:
return base_similarity
max_score = max(post_scores, key=lambda x: x[1])[1]
min_score = min(post_scores, key=lambda x: x[1])[1]
score_range = max_score - min_score
adjustment = (like_similarity - dislike_similarity) * (score_range / 2)
return base_similarity + adjustment
def swap_random_posts(rec_post_ids, post_ids, swap_percent=0.1):
"""
Swap a percentage of recommended posts with random posts.
Args:
rec_post_ids (list): List of recommended post IDs.
post_ids (list): List of all post IDs.
swap_percent (float): Percentage of posts to swap.
Returns:
list: Updated list of recommended post IDs.
"""
num_to_swap = int(len(rec_post_ids) * swap_percent)
posts_to_swap = random.sample(post_ids, num_to_swap)
indices_to_replace = random.sample(range(len(rec_post_ids)), num_to_swap)
for idx, new_post in zip(indices_to_replace, posts_to_swap):
rec_post_ids[idx] = new_post
return rec_post_ids
def get_trace_contents(user_id, action, post_table, trace_table):
"""
Get the contents of posts that a user has interacted with.
Args:
user_id (str): ID of the user.
action (str): Type of action (like or unlike).
post_table (list): List of posts.
trace_table (list): List of user interactions.
Returns:
list: List of post contents.
"""
# Get post IDs from trace table for the given user and action
trace_post_ids = [
trace['post_id'] for trace in trace_table
if (trace['user_id'] == user_id and trace['action'] == action)
]
# Fetch post contents from post table where post IDs match those in the
# trace
trace_contents = [
post['content'] for post in post_table
if post['post_id'] in trace_post_ids
]
return trace_contents
def rec_sys_personalized_with_trace(
user_table: List[Dict[str, Any]],
post_table: List[Dict[str, Any]],
trace_table: List[Dict[str, Any]],
rec_matrix: List[List],
max_rec_post_len: int,
swap_rate: float = 0.1,
) -> List[List]:
"""
This version:
1. If the number of posts is less than or equal to the maximum
recommended length, each user gets all post IDs
2. Otherwise:
- For each user, get a like-trace pool and dislike-trace pool from the
trace table
- For each user, calculate the similarity between the user's bio and
the post text
- Use the trace table to adjust the similarity score
- Swap 10% of the recommended posts with the random posts
Personalized recommendation system that uses user interaction traces.
Args:
user_table (List[Dict[str, Any]]): List of users.
post_table (List[Dict[str, Any]]): List of posts.
trace_table (List[Dict[str, Any]]): List of user interactions.
rec_matrix (List[List]): Existing recommendation matrix.
max_rec_post_len (int): Maximum number of recommended posts.
swap_rate (float): Percentage of posts to swap for diversity.
Returns:
List[List]: Updated recommendation matrix.
"""
start_time = time.time()
new_rec_matrix = []
post_ids = [post['post_id'] for post in post_table]
if len(post_ids) <= max_rec_post_len:
new_rec_matrix = [post_ids] * (len(rec_matrix) - 1)
else:
for idx in range(1, len(rec_matrix)):
user_id = user_table[idx - 1]['user_id']
user_bio = user_table[idx - 1]['bio']
# filter out posts that belong to the user
available_post_contents = [(post['post_id'], post['content'])
for post in post_table
if post['user_id'] != user_id]
# filter out like-trace and dislike-trace
like_trace_contents = get_trace_contents(
user_id, ActionType.LIKE_POST.value, post_table, trace_table)
dislike_trace_contents = get_trace_contents(
user_id, ActionType.UNLIKE_POST.value, post_table, trace_table)
# calculate similarity between user bio and post text
post_scores = []
for post_id, post_content in available_post_contents:
if model is not None:
user_embedding = model.encode(user_bio)
post_embedding = model.encode(post_content)
base_similarity = np.dot(
user_embedding,
post_embedding) / (np.linalg.norm(user_embedding) *
np.linalg.norm(post_embedding))
post_scores.append((post_id, base_similarity))
else:
post_scores.append((post_id, random.random()))
new_post_scores = []
# adjust similarity based on like and dislike traces
for _post_id, _base_similarity in post_scores:
_post_content = post_table[post_ids.index(_post_id)]['content']
like_similarity = sum(
np.dot(model.encode(_post_content), model.encode(like)) /
(np.linalg.norm(model.encode(_post_content)) *
np.linalg.norm(model.encode(like)))
for like in like_trace_contents) / len(
like_trace_contents) if like_trace_contents else 0
dislike_similarity = sum(
np.dot(model.encode(_post_content), model.encode(dislike))
/ (np.linalg.norm(model.encode(_post_content)) *
np.linalg.norm(model.encode(dislike)))
for dislike in dislike_trace_contents) / len(
dislike_trace_contents
) if dislike_trace_contents else 0
# Normalize and apply adjustments
adjusted_similarity = normalize_similarity_adjustments(
post_scores, _base_similarity, like_similarity,
dislike_similarity)
new_post_scores.append((_post_id, adjusted_similarity))
# sort posts by similarity
new_post_scores.sort(key=lambda x: x[1], reverse=True)
# extract post ids
rec_post_ids = [
post_id for post_id, _ in new_post_scores[:max_rec_post_len]
]
if swap_rate > 0:
# swap the recommended posts with random posts
swap_free_ids = [
post_id for post_id in post_ids
if post_id not in rec_post_ids and post_id not in [
trace['post_id']
for trace in trace_table if trace['user_id']
]
]
rec_post_ids = swap_random_posts(rec_post_ids, swap_free_ids,
swap_rate)
new_rec_matrix.append(rec_post_ids)
end_time = time.time()
print(f'Personalized recommendation time: {end_time - start_time:.6f}s')
return new_rec_matrix

5
backend/vendor/camel-oasis/oasis/social_platform/schema/chat_group.sql vendored Normal file
View File

@ -0,0 +1,5 @@
CREATE TABLE IF NOT EXISTS `chat_group` (
group_id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
);

12
backend/vendor/camel-oasis/oasis/social_platform/schema/comment.sql vendored Normal file
View File

@ -0,0 +1,12 @@
-- This is the schema definition for the comment table
CREATE TABLE comment (
comment_id INTEGER PRIMARY KEY AUTOINCREMENT,
post_id INTEGER,
user_id INTEGER,
content TEXT,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
num_likes INTEGER DEFAULT 0,
num_dislikes INTEGER DEFAULT 0,
FOREIGN KEY(post_id) REFERENCES post(post_id),
FOREIGN KEY(user_id) REFERENCES user(user_id)
);

9
backend/vendor/camel-oasis/oasis/social_platform/schema/comment_dislike.sql vendored Normal file
View File

@ -0,0 +1,9 @@
-- This is the schema definition for the comment_dislike table
CREATE TABLE comment_dislike (
comment_dislike_id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id INTEGER,
comment_id INTEGER,
created_at DATETIME,
FOREIGN KEY(user_id) REFERENCES user(user_id),
FOREIGN KEY(comment_id) REFERENCES comment(comment_id)
);

9
backend/vendor/camel-oasis/oasis/social_platform/schema/comment_like.sql vendored Normal file
View File

@ -0,0 +1,9 @@
-- This is the schema definition for the comment_like table
CREATE TABLE comment_like (
comment_like_id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id INTEGER,
comment_id INTEGER,
created_at DATETIME,
FOREIGN KEY(user_id) REFERENCES user(user_id),
FOREIGN KEY(comment_id) REFERENCES comment(comment_id)
);

9
backend/vendor/camel-oasis/oasis/social_platform/schema/dislike.sql vendored Normal file
View File

@ -0,0 +1,9 @@
-- This is the schema definition for the dislike table
CREATE TABLE dislike (
dislike_id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id INTEGER,
post_id INTEGER,
created_at DATETIME,
FOREIGN KEY(user_id) REFERENCES user(user_id),
FOREIGN KEY(post_id) REFERENCES tweet(post_id)
);

9
backend/vendor/camel-oasis/oasis/social_platform/schema/follow.sql vendored Normal file
View File

@ -0,0 +1,9 @@
-- This is the schema definition for the follow table
CREATE TABLE follow (
follow_id INTEGER PRIMARY KEY AUTOINCREMENT,
follower_id INTEGER,
followee_id INTEGER,
created_at DATETIME,
FOREIGN KEY(follower_id) REFERENCES user(user_id),
FOREIGN KEY(followee_id) REFERENCES user(user_id)
);

7
backend/vendor/camel-oasis/oasis/social_platform/schema/group_member.sql vendored Normal file
View File

@ -0,0 +1,7 @@
CREATE TABLE IF NOT EXISTS group_members (
group_id INTEGER NOT NULL,
agent_id INTEGER NOT NULL,
joined_at DATETIME DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (group_id, agent_id),
FOREIGN KEY (group_id) REFERENCES chat_group(group_id)
);

9
backend/vendor/camel-oasis/oasis/social_platform/schema/group_message.sql vendored Normal file
View File

@ -0,0 +1,9 @@
CREATE TABLE IF NOT EXISTS group_messages (
message_id INTEGER PRIMARY KEY AUTOINCREMENT,
group_id INTEGER NOT NULL,
sender_id INTEGER NOT NULL,
content TEXT NOT NULL,
sent_at DATETIME DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (group_id) REFERENCES chat_group(group_id),
FOREIGN KEY (sender_id) REFERENCES user(agent_id)
);

9
backend/vendor/camel-oasis/oasis/social_platform/schema/like.sql vendored Normal file
View File

@ -0,0 +1,9 @@
-- This is the schema definition for the like table
CREATE TABLE like (
like_id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id INTEGER,
post_id INTEGER,
created_at DATETIME,
FOREIGN KEY(user_id) REFERENCES user(user_id),
FOREIGN KEY(post_id) REFERENCES tweet(post_id)
);

9
backend/vendor/camel-oasis/oasis/social_platform/schema/mute.sql vendored Normal file
View File

@ -0,0 +1,9 @@
-- This is the schema definition for the mute table
CREATE TABLE mute (
mute_id INTEGER PRIMARY KEY AUTOINCREMENT,
muter_id INTEGER,
mutee_id INTEGER,
created_at DATETIME,
FOREIGN KEY(muter_id) REFERENCES user(user_id),
FOREIGN KEY(mutee_id) REFERENCES user(user_id)
);

16
backend/vendor/camel-oasis/oasis/social_platform/schema/post.sql vendored Normal file
View File

@ -0,0 +1,16 @@
-- This is the schema definition for the post table
-- Add Images, location etc.?
CREATE TABLE post (
post_id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id INTEGER,
original_post_id INTEGER, -- NULL if this is an original post
content TEXT DEFAULT '', -- DEFAULT '' for initial posts
quote_content TEXT, -- NULL if this is an original post or a repost
created_at DATETIME,
num_likes INTEGER DEFAULT 0,
num_dislikes INTEGER DEFAULT 0,
num_shares INTEGER DEFAULT 0, -- num_shares = num_reposts + num_quotes
num_reports INTEGER DEFAULT 0,
FOREIGN KEY(user_id) REFERENCES user(user_id),
FOREIGN KEY(original_post_id) REFERENCES post(post_id)
);

6
backend/vendor/camel-oasis/oasis/social_platform/schema/product.sql vendored Normal file
View File

@ -0,0 +1,6 @@
-- This is the schema definition for the product table
CREATE TABLE product (
product_id INTEGER PRIMARY KEY,
product_name TEXT,
sales INTEGER DEFAULT 0
);

8
backend/vendor/camel-oasis/oasis/social_platform/schema/rec.sql vendored Normal file
View File

@ -0,0 +1,8 @@
-- This is the schema definition for the rec table
CREATE TABLE rec (
user_id INTEGER,
post_id INTEGER,
PRIMARY KEY(user_id, post_id),
FOREIGN KEY(user_id) REFERENCES user(user_id)
FOREIGN KEY(post_id) REFERENCES tweet(post_id)
);

10
backend/vendor/camel-oasis/oasis/social_platform/schema/report.sql vendored Normal file
View File

@ -0,0 +1,10 @@
-- This is the schema definition for the report table
CREATE TABLE report (
report_id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id INTEGER,
post_id INTEGER,
report_reason TEXT,
created_at DATETIME,
FOREIGN KEY(user_id) REFERENCES user(user_id),
FOREIGN KEY(post_id) REFERENCES post(post_id)
);

9
backend/vendor/camel-oasis/oasis/social_platform/schema/trace.sql vendored Normal file
View File

@ -0,0 +1,9 @@
-- This is the schema definition for the trace table
CREATE TABLE trace (
user_id INTEGER,
created_at DATETIME,
action TEXT,
info TEXT,
PRIMARY KEY(user_id, created_at, action, info),
FOREIGN KEY(user_id) REFERENCES user(user_id)
);

11
backend/vendor/camel-oasis/oasis/social_platform/schema/user.sql vendored Normal file
View File

@ -0,0 +1,11 @@
-- This is the schema definition for the user table
CREATE TABLE user (
user_id INTEGER PRIMARY KEY AUTOINCREMENT,
agent_id INTEGER,
user_name TEXT,
name TEXT,
bio TEXT,
created_at DATETIME,
num_followings INTEGER DEFAULT 0,
num_followers INTEGER DEFAULT 0
);

90
backend/vendor/camel-oasis/oasis/social_platform/typing.py vendored Normal file
View File

@ -0,0 +1,90 @@
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
# Licensed under the Apache License, Version 2.0 (the “License”);
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an “AS IS” BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
from enum import Enum
class ActionType(Enum):
EXIT = "exit"
REFRESH = "refresh"
SEARCH_USER = "search_user"
SEARCH_POSTS = "search_posts"
CREATE_POST = "create_post"
LIKE_POST = "like_post"
UNLIKE_POST = "unlike_post"
DISLIKE_POST = "dislike_post"
UNDO_DISLIKE_POST = "undo_dislike_post"
REPORT_POST = "report_post"
FOLLOW = "follow"
UNFOLLOW = "unfollow"
MUTE = "mute"
UNMUTE = "unmute"
TREND = "trend"
SIGNUP = "sign_up"
REPOST = "repost"
QUOTE_POST = "quote_post"
UPDATE_REC_TABLE = "update_rec_table"
CREATE_COMMENT = "create_comment"
LIKE_COMMENT = "like_comment"
UNLIKE_COMMENT = "unlike_comment"
DISLIKE_COMMENT = "dislike_comment"
UNDO_DISLIKE_COMMENT = "undo_dislike_comment"
DO_NOTHING = "do_nothing"
PURCHASE_PRODUCT = "purchase_product"
INTERVIEW = "interview"
JOIN_GROUP = "join_group"
LEAVE_GROUP = "leave_group"
SEND_TO_GROUP = "send_to_group"
CREATE_GROUP = "create_group"
LISTEN_FROM_GROUP = "listen_from_group"
@classmethod
def get_default_twitter_actions(cls):
return [
cls.CREATE_POST,
cls.LIKE_POST,
cls.REPOST,
cls.FOLLOW,
cls.DO_NOTHING,
cls.QUOTE_POST,
]
@classmethod
def get_default_reddit_actions(cls):
return [
cls.LIKE_POST,
cls.DISLIKE_POST,
cls.CREATE_POST,
cls.CREATE_COMMENT,
cls.LIKE_COMMENT,
cls.DISLIKE_COMMENT,
cls.SEARCH_POSTS,
cls.SEARCH_USER,
cls.TREND,
cls.REFRESH,
cls.DO_NOTHING,
cls.FOLLOW,
cls.MUTE,
]
class RecsysType(Enum):
TWITTER = "twitter"
TWHIN = "twhin-bert"
REDDIT = "reddit"
RANDOM = "random"
class DefaultPlatformType(Enum):
TWITTER = "twitter"
REDDIT = "reddit"

13
backend/vendor/camel-oasis/oasis/testing/__init__.py vendored Normal file
View File

@ -0,0 +1,13 @@
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
# Licensed under the Apache License, Version 2.0 (the “License”);
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an “AS IS” BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========

64
backend/vendor/camel-oasis/oasis/testing/show_db.py vendored Normal file
View File

@ -0,0 +1,64 @@
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
# Licensed under the Apache License, Version 2.0 (the “License”);
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an “AS IS” BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
import logging
import sqlite3
from datetime import datetime
table_log = logging.getLogger(name="table")
table_log.setLevel("DEBUG")
now = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") # Modify here
file_handler = logging.FileHandler(f"./log/table-{str(now)}.log",
encoding="utf-8")
file_handler.setLevel("DEBUG")
file_handler.setFormatter(logging.Formatter("%(message)s"))
table_log.addHandler(file_handler)
stream_handler = logging.StreamHandler()
stream_handler.setLevel("DEBUG")
stream_handler.setFormatter(logging.Formatter("%(message)s"))
table_log.addHandler(stream_handler)
def print_db_contents(db_file):
# Connect to the SQLite database
conn = sqlite3.connect(db_file)
cursor = conn.cursor()
# Retrieve and print all table names
cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
tables = cursor.fetchall()
# print("Tables:", [table[0] for table in tables])
table_log.info("Tables:" + " ".join([str(table[0]) for table in tables]))
for table_name in tables:
# print(f"\nTable: {table_name[0]}")
table_log.info(f"\nTable: {table_name[0]}")
# Print table structure
cursor.execute(f"PRAGMA table_info({table_name[0]})")
columns = cursor.fetchall()
# print("Columns:")
table_log.info("Columns:")
for col in columns:
# print(f" {col[1]} ({col[2]})")
table_log.info(f" {col[1]} ({col[2]})")
# Print table contents
cursor.execute(f"SELECT * FROM {table_name[0]}")
rows = cursor.fetchall()
# print("Contents:")
table_log.info("Contents:")
for row in rows:
# print(" ", row)
table_log.info(" " + ", ".join(str(item) for item in row))
# Close the connection
conn.close()

41
backend/vendor/camel-oasis/pyproject.toml vendored Normal file
View File

@ -0,0 +1,41 @@
[project]
name = "camel-oasis"
version = "0.2.5.post1"
description = "Open Agents Social Interaction Simulations on a Large Scale (vendored fork: relaxed neo4j pin to coexist with graphiti-core[falkordb])"
authors = [{ name = "CAMEL-AI.org" }]
readme = "README.md"
license = { text = "Apache-2.0" }
keywords = ["multi-agent-systems", "social-simulations"]
requires-python = ">=3.11,<3.13"
dependencies = [
"pandas==2.2.2",
"igraph==0.11.6",
"cairocffi==1.7.1",
"pillow==10.3.0",
"unstructured==0.13.7",
"sentence-transformers==3.0.0",
"prance==23.6.21.0",
"openapi-spec-validator==0.7.1",
"slack_sdk==3.31.0",
# Relaxed from ==5.23.0 to coexist with graphiti-core[falkordb]'s
# neo4j>=5.26.0 requirement. The MiroFish code paths (run_*_simulation.py
# + run_parallel_simulation.py) never pass a neo4j_config, so AgentGraph
# is constructed in-memory-only and the neo4j driver is never invoked.
"neo4j>=5.23.0",
"camel-ai==0.2.78",
"requests_oauthlib==2.0.0",
]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[tool.hatch.build.targets.wheel]
packages = ["oasis"]
# Required because the parent project (MiroFish) declares this package as
# a direct-reference local path dep ("camel-oasis @ file:///app/backend/vendor/camel-oasis").
# Without this, hatchling refuses to build when uv is constructing the lock entry.
[tool.hatch.metadata]
allow-direct-references = true

81
deploy/docker-compose.yml Normal file
View File

@ -0,0 +1,81 @@
# MiroFish deploy overlay — agent.profikid.nl
#
# Project conventions (matches hermes, snake, wiki siblings):
# /docker/<name>/docker-compose.yml
# COMPOSE_PROJECT_NAME=<name>
# TRAEFIK_HOST=agent.profikid.nl
#
# Single port (3000) exposed to Traefik; nginx inside the container
# serves the built frontend and reverse-proxies /api/* + /health to Flask.
#
# Memory graph: FalkorDB (in-memory graph store) on a shared network.
services:
falkordb:
image: falkordb/falkordb:latest
container_name: mirofish-falkordb
restart: unless-stopped
expose:
- "6379"
volumes:
- falkordb-data:/data
healthcheck:
test: ["CMD-SHELL", "redis-cli ping | grep -q PONG || exit 1"]
interval: 15s
timeout: 3s
retries: 5
start_period: 20s
networks:
- mirofish_net
mirofish:
build:
context: ..
dockerfile: Dockerfile
image: mirofish:latest
container_name: mirofish
restart: unless-stopped
depends_on:
falkordb:
condition: service_healthy
env_file:
- secrets.env
environment:
FALKORDB_HOST: falkordb
FALKORDB_PORT: "6379"
expose:
- "3000"
labels:
- traefik.enable=true
- traefik.http.routers.${COMPOSE_PROJECT_NAME:-mirofish}.rule=Host(`${COMPOSE_PROJECT_NAME:-mirofish}.${TRAEFIK_HOST:-agent.profikid.nl}`)
- traefik.http.routers.${COMPOSE_PROJECT_NAME:-mirofish}.entrypoints=websecure
- traefik.http.routers.${COMPOSE_PROJECT_NAME:-mirofish}.tls.certresolver=letsencrypt
- traefik.http.services.${COMPOSE_PROJECT_NAME:-mirofish}.loadbalancer.server.port=3000
# HTTP -> HTTPS redirect
- traefik.http.routers.${COMPOSE_PROJECT_NAME:-mirofish}-http.rule=Host(`${COMPOSE_PROJECT_NAME:-mirofish}.${TRAEFIK_HOST:-agent.profikid.nl}`)
- traefik.http.routers.${COMPOSE_PROJECT_NAME:-mirofish}-http.entrypoints=web
- traefik.http.routers.${COMPOSE_PROJECT_NAME:-mirofish}-http.middlewares=redirect-to-https
- traefik.http.middlewares.redirect-to-https.redirectscheme.scheme=https
- traefik.http.middlewares.redirect-to-https.redirectscheme.permanent=true
volumes:
- uploads:/app/backend/uploads
- flask-logs:/app/logs
- embedding-model:/root/.cache/huggingface # cache the sentence-transformers model
healthcheck:
test: ["CMD", "curl", "-fsS", "http://127.0.0.1:3000/health"]
interval: 30s
timeout: 5s
retries: 5
start_period: 90s
networks:
- mirofish_net
volumes:
uploads:
flask-logs:
falkordb-data:
embedding-model:
networks:
mirofish_net:
name: mirofish_net

65
nginx.conf Normal file
View File

@ -0,0 +1,65 @@
worker_processes 1;
pid /run/nginx.pid;
error_log /var/log/nginx/error.log warn;
events {
worker_connections 1024;
}
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
sendfile on;
keepalive_timeout 65;
client_max_body_size 100m; # upload endpoint accepts up to 50MB per file, several at once
access_log /var/log/nginx/access.log;
upstream flask_backend {
server 127.0.0.1:5001;
keepalive 8;
}
server {
listen 3000 default_server;
server_name _;
root /app/frontend/dist;
index index.html;
# Long timeouts: backend's ontology/profile/report generation can take minutes
proxy_connect_timeout 600s;
proxy_send_timeout 600s;
proxy_read_timeout 600s;
send_timeout 600s;
# Reverse-proxy /api/* and /health to Flask
location /api/ {
proxy_pass http://flask_backend/api/;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_buffering off; # important for SSE / streamed responses
}
location = /health {
proxy_pass http://flask_backend/health;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
}
# Vite emits /assets/<hash>.<ext> for JS/CSS; let nginx serve with proper mime
location /assets/ {
expires 1y;
add_header Cache-Control "public, immutable";
try_files $uri =404;
}
# SPA fallback: send everything else to index.html
location / {
try_files $uri $uri/ /index.html;
}
}
}

38
start.sh Normal file
View File

@ -0,0 +1,38 @@
#!/bin/sh
# Start the Flask backend in the background, then run nginx in the foreground.
# Both share the same container.
set -e
# Tell Flask to bind to localhost only (nginx fronts it)
export FLASK_HOST=127.0.0.1
export FLASK_PORT=5001
# Disable Werkzeug reloader / debug for a single-process prod-like run
export FLASK_DEBUG=False
echo "[start.sh] launching Flask backend on 127.0.0.1:5001 ..."
cd /app/backend
uv run python run.py > /app/logs/flask.log 2>&1 &
FLASK_PID=$!
# Wait for Flask to be ready (max 60s)
echo "[start.sh] waiting for Flask to respond on /health ..."
i=0
while [ $i -lt 60 ]; do
if curl -sSf -m 2 http://127.0.0.1:5001/health >/dev/null 2>&1; then
echo "[start.sh] Flask is up (pid $FLASK_PID)."
break
fi
i=$((i+1))
sleep 1
done
if [ $i -ge 60 ]; then
echo "[start.sh] WARN: Flask did not become ready in 60s; check /app/logs/flask.log"
fi
# Make sure nginx can write its pid file
mkdir -p /run
chown -R www-data:www-data /var/lib/nginx /var/log/nginx /run 2>/dev/null || true
echo "[start.sh] launching nginx on 0.0.0.0:3000 ..."
exec nginx -g "daemon off;"