mirror of https://github.com/aliasrobotics/cai.git
Merge branch 'main' of https://github.com/openai/openai-agents-python
This commit is contained in:
Víctor Mayoral Vilches
commit
09c2beefdc
|
|
@ -17,7 +17,10 @@ jobs:
|
|||
stale-issue-label: "stale"
|
||||
stale-issue-message: "This issue is stale because it has been open for 7 days with no activity."
|
||||
close-issue-message: "This issue was closed because it has been inactive for 3 days since being marked as stale."
|
||||
days-before-pr-stale: -1
|
||||
days-before-pr-close: -1
|
||||
any-of-labels: 'question,needs-more-info'
|
||||
any-of-issue-labels: 'question,needs-more-info'
|
||||
days-before-pr-stale: 10
|
||||
days-before-pr-close: 7
|
||||
stale-pr-label: "stale"
|
||||
stale-pr-message: "This PR is stale because it has been open for 10 days with no activity."
|
||||
close-pr-message: "This PR was closed because it has been inactive for 7 days since being marked as stale."
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
|
|
|||
|
|
@ -50,8 +50,8 @@ jobs:
|
|||
enable-cache: true
|
||||
- name: Install dependencies
|
||||
run: make sync
|
||||
- name: Run tests
|
||||
run: make tests
|
||||
- name: Run tests with coverage
|
||||
run: make coverage
|
||||
|
||||
build-docs:
|
||||
runs-on: ubuntu-latest
|
||||
|
|
|
|||
|
|
@ -135,7 +135,7 @@ dmypy.json
|
|||
cython_debug/
|
||||
|
||||
# PyCharm
|
||||
#.idea/
|
||||
.idea/
|
||||
|
||||
# Ruff stuff:
|
||||
.ruff_cache/
|
||||
|
|
|
|||
18
Makefile
18
Makefile
|
|
@ -5,6 +5,7 @@ sync:
|
|||
.PHONY: format
|
||||
format:
|
||||
uv run ruff format
|
||||
uv run ruff check --fix
|
||||
|
||||
.PHONY: lint
|
||||
lint:
|
||||
|
|
@ -18,6 +19,21 @@ mypy:
|
|||
tests:
|
||||
uv run pytest
|
||||
|
||||
.PHONY: coverage
|
||||
coverage:
|
||||
|
||||
uv run coverage run -m pytest
|
||||
uv run coverage xml -o coverage.xml
|
||||
uv run coverage report -m --fail-under=95
|
||||
|
||||
.PHONY: snapshots-fix
|
||||
snapshots-fix:
|
||||
uv run pytest --inline-snapshot=fix
|
||||
|
||||
.PHONY: snapshots-create
|
||||
snapshots-create:
|
||||
uv run pytest --inline-snapshot=create
|
||||
|
||||
.PHONY: old_version_tests
|
||||
old_version_tests:
|
||||
UV_PROJECT_ENVIRONMENT=.venv_39 uv run --python 3.9 -m pytest
|
||||
|
|
@ -34,4 +50,6 @@ serve-docs:
|
|||
.PHONY: deploy-docs
|
||||
deploy-docs:
|
||||
uv run mkdocs gh-deploy --force --verbose
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ The OpenAI Agents SDK is a lightweight yet powerful framework for building multi
|
|||
### Core concepts:
|
||||
|
||||
1. [**Agents**](https://openai.github.io/openai-agents-python/agents): LLMs configured with instructions, tools, guardrails, and handoffs
|
||||
2. [**Handoffs**](https://openai.github.io/openai-agents-python/handoffs/): Allow agents to transfer control to other agents for specific tasks
|
||||
2. [**Handoffs**](https://openai.github.io/openai-agents-python/handoffs/): A specialized tool call used by the Agents SDK for transferring control between agents
|
||||
3. [**Guardrails**](https://openai.github.io/openai-agents-python/guardrails/): Configurable safety checks for input and output validation
|
||||
4. [**Tracing**](https://openai.github.io/openai-agents-python/tracing/): Built-in tracking of agent runs, allowing you to view, debug and optimize your workflows
|
||||
|
||||
|
|
@ -30,6 +30,8 @@ source env/bin/activate
|
|||
pip install openai-agents
|
||||
```
|
||||
|
||||
For voice support, install with the optional `voice` group: `pip install 'openai-agents[voice]'`.
|
||||
|
||||
## Hello world example
|
||||
|
||||
```python
|
||||
|
|
@ -142,7 +144,7 @@ The Agents SDK is designed to be highly flexible, allowing you to model a wide r
|
|||
|
||||
## Tracing
|
||||
|
||||
The Agents SDK automatically traces your agent runs, making it easy to track and debug the behavior of your agents. Tracing is extensible by design, supporting custom spans and a wide variety of external destinations, including [Logfire](https://logfire.pydantic.dev/docs/integrations/llms/openai/#openai-agents), [AgentOps](https://docs.agentops.ai/v1/integrations/agentssdk), [Braintrust](https://braintrust.dev/docs/guides/traces/integrations#openai-agents-sdk), [Scorecard](https://docs.scorecard.io/docs/documentation/features/tracing#openai-agents-sdk-integration), and [Keywords AI](https://docs.keywordsai.co/integration/development-frameworks/openai-agent). For more details about how to customize or disable tracing, see [Tracing](http://openai.github.io/openai-agents-python/tracing).
|
||||
The Agents SDK automatically traces your agent runs, making it easy to track and debug the behavior of your agents. Tracing is extensible by design, supporting custom spans and a wide variety of external destinations, including [Logfire](https://logfire.pydantic.dev/docs/integrations/llms/openai/#openai-agents), [AgentOps](https://docs.agentops.ai/v1/integrations/agentssdk), [Braintrust](https://braintrust.dev/docs/guides/traces/integrations#openai-agents-sdk), [Scorecard](https://docs.scorecard.io/docs/documentation/features/tracing#openai-agents-sdk-integration), and [Keywords AI](https://docs.keywordsai.co/integration/development-frameworks/openai-agent). For more details about how to customize or disable tracing, see [Tracing](http://openai.github.io/openai-agents-python/tracing), which also includes a larger list of [external tracing processors](http://openai.github.io/openai-agents-python/tracing/#external-tracing-processors-list).
|
||||
|
||||
## Development (only needed if you need to edit the SDK/examples)
|
||||
|
||||
|
|
|
|||
|
|
@ -130,3 +130,16 @@ robot_agent = pirate_agent.clone(
|
|||
instructions="Write like a robot",
|
||||
)
|
||||
```
|
||||
|
||||
## Forcing tool use
|
||||
|
||||
Supplying a list of tools doesn't always mean the LLM will use a tool. You can force tool use by setting [`ModelSettings.tool_choice`][agents.model_settings.ModelSettings.tool_choice]. Valid values are:
|
||||
|
||||
1. `auto`, which allows the LLM to decide whether or not to use a tool.
|
||||
2. `required`, which requires the LLM to use a tool (but it can intelligently decide which tool).
|
||||
3. `none`, which requires the LLM to _not_ use a tool.
|
||||
4. Setting a specific string e.g. `my_tool`, which requires the LLM to use that specific tool.
|
||||
|
||||
!!! note
|
||||
|
||||
If requiring tool use, you should consider setting [`Agent.tool_use_behavior`] to stop the Agent from running when a tool output is produced. Otherwise, the Agent might run in an infinite loop, where the LLM produces a tool call , and the tool result is sent to the LLM, and this infinite loops because the LLM is always forced to use a tool.
|
||||
|
|
|
|||
|
|
@ -41,14 +41,14 @@ async def fetch_user_age(wrapper: RunContextWrapper[UserInfo]) -> str: # (2)!
|
|||
return f"User {wrapper.context.name} is 47 years old"
|
||||
|
||||
async def main():
|
||||
user_info = UserInfo(name="John", uid=123) # (3)!
|
||||
user_info = UserInfo(name="John", uid=123)
|
||||
|
||||
agent = Agent[UserInfo]( # (4)!
|
||||
agent = Agent[UserInfo]( # (3)!
|
||||
name="Assistant",
|
||||
tools=[fetch_user_age],
|
||||
)
|
||||
|
||||
result = await Runner.run(
|
||||
result = await Runner.run( # (4)!
|
||||
starting_agent=agent,
|
||||
input="What is the age of the user?",
|
||||
context=user_info,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,36 @@
|
|||
# Examples
|
||||
|
||||
Check out a variety of sample implementations of the SDK in the examples section of the [repo](https://github.com/openai/openai-agents-python/tree/main/examples). The examples are organized into several categories that demonstrate different patterns and capabilities.
|
||||
|
||||
|
||||
## Categories
|
||||
|
||||
- **agent_patterns:**
|
||||
Examples in this category illustrate common agent design patterns, such as
|
||||
|
||||
- Deterministic workflows
|
||||
- Agents as tools
|
||||
- Parallel agent execution
|
||||
|
||||
- **basic:**
|
||||
These examples showcase foundational capabilities of the SDK, such as
|
||||
|
||||
- Dynamic system prompts
|
||||
- Streaming outputs
|
||||
- Lifecycle events
|
||||
|
||||
- **tool examples:**
|
||||
Learn how to implement OAI hosted tools such as web search and file search,
|
||||
and integrate them into your agents.
|
||||
|
||||
- **model providers:**
|
||||
Explore how to use non-OpenAI models with the SDK.
|
||||
|
||||
- **handoffs:**
|
||||
See practical examples of agent handoffs.
|
||||
|
||||
- **customer_service** and **research_bot:**
|
||||
Two more built-out examples that illustrate real-world applications
|
||||
|
||||
- **customer_service**: Example customer service system for an airline.
|
||||
- **research_bot**: Simple deep research clone.
|
||||
|
|
@ -29,7 +29,7 @@ Output guardrails run in 3 steps:
|
|||
|
||||
!!! Note
|
||||
|
||||
Output guardrails are intended to run on the final agent input, so an agent's guardrails only run if the agent is the *last* agent. Similar to the input guardrails, we do this because guardrails tend to be related to the actual Agent - you'd run different guardrails for different agents, so colocating the code is useful for readability.
|
||||
Output guardrails are intended to run on the final agent output, so an agent's guardrails only run if the agent is the *last* agent. Similar to the input guardrails, we do this because guardrails tend to be related to the actual Agent - you'd run different guardrails for different agents, so colocating the code is useful for readability.
|
||||
|
||||
## Tripwires
|
||||
|
||||
|
|
@ -111,8 +111,8 @@ class MessageOutput(BaseModel): # (1)!
|
|||
response: str
|
||||
|
||||
class MathOutput(BaseModel): # (2)!
|
||||
is_math: bool
|
||||
reasoning: str
|
||||
is_math: bool
|
||||
|
||||
guardrail_agent = Agent(
|
||||
name="Guardrail check",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,3 @@
|
|||
# `Events`
|
||||
|
||||
::: agents.voice.events
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
# `Exceptions`
|
||||
|
||||
::: agents.voice.exceptions
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
# `Input`
|
||||
|
||||
::: agents.voice.input
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
# `Model`
|
||||
|
||||
::: agents.voice.model
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
# `OpenAIVoiceModelProvider`
|
||||
|
||||
::: agents.voice.models.openai_model_provider
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
# `OpenAI STT`
|
||||
|
||||
::: agents.voice.models.openai_stt
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
# `OpenAI TTS`
|
||||
|
||||
::: agents.voice.models.openai_tts
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
# `Pipeline`
|
||||
|
||||
::: agents.voice.pipeline
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
# `Pipeline Config`
|
||||
|
||||
::: agents.voice.pipeline_config
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
# `Result`
|
||||
|
||||
::: agents.voice.result
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
# `Utils`
|
||||
|
||||
::: agents.voice.utils
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
# `Workflow`
|
||||
|
||||
::: agents.voice.workflow
|
||||
|
|
@ -9,6 +9,8 @@ The Agents SDK includes built-in tracing, collecting a comprehensive record of e
|
|||
1. You can globally disable tracing by setting the env var `OPENAI_AGENTS_DISABLE_TRACING=1`
|
||||
2. You can disable tracing for a single run by setting [`agents.run.RunConfig.tracing_disabled`][] to `True`
|
||||
|
||||
***For organizations operating under a Zero Data Retention (ZDR) policy using OpenAI's APIs, tracing is unavailable.***
|
||||
|
||||
## Traces and spans
|
||||
|
||||
- **Traces** represent a single end-to-end operation of a "workflow". They're composed of Spans. Traces have the following properties:
|
||||
|
|
@ -33,6 +35,9 @@ By default, the SDK traces the following:
|
|||
- Function tool calls are each wrapped in `function_span()`
|
||||
- Guardrails are wrapped in `guardrail_span()`
|
||||
- Handoffs are wrapped in `handoff_span()`
|
||||
- Audio inputs (speech-to-text) are wrapped in a `transcription_span()`
|
||||
- Audio outputs (text-to-speech) are wrapped in a `speech_span()`
|
||||
- Related audio spans may be parented under a `speech_group_span()`
|
||||
|
||||
By default, the trace is named "Agent trace". You can set this name if you use `trace`, or you can can configure the name and other properties with the [`RunConfig`][agents.run.RunConfig].
|
||||
|
||||
|
|
@ -74,7 +79,11 @@ Spans are automatically part of the current trace, and are nested under the near
|
|||
|
||||
## Sensitive data
|
||||
|
||||
Some spans track potentially sensitive data. For example, the `generation_span()` stores the inputs/outputs of the LLM generation, and `function_span()` stores the inputs/outputs of function calls. These may contain sensitive data, so you can disable capturing that data via [`RunConfig.trace_include_sensitive_data`][agents.run.RunConfig.trace_include_sensitive_data].
|
||||
Certain spans may capture potentially sensitive data.
|
||||
|
||||
The `generation_span()` stores the inputs/outputs of the LLM generation, and `function_span()` stores the inputs/outputs of function calls. These may contain sensitive data, so you can disable capturing that data via [`RunConfig.trace_include_sensitive_data`][agents.run.RunConfig.trace_include_sensitive_data].
|
||||
|
||||
Similarly, Audio spans include base64-encoded PCM data for input and output audio by default. You can disable capturing this audio data by configuring [`VoicePipelineConfig.trace_include_sensitive_audio_data`][agents.voice.pipeline_config.VoicePipelineConfig.trace_include_sensitive_audio_data].
|
||||
|
||||
## Custom tracing processors
|
||||
|
||||
|
|
@ -88,10 +97,17 @@ To customize this default setup, to send traces to alternative or additional bac
|
|||
1. [`add_trace_processor()`][agents.tracing.add_trace_processor] lets you add an **additional** trace processor that will receive traces and spans as they are ready. This lets you do your own processing in addition to sending traces to OpenAI's backend.
|
||||
2. [`set_trace_processors()`][agents.tracing.set_trace_processors] lets you **replace** the default processors with your own trace processors. This means traces will not be sent to the OpenAI backend unless you include a `TracingProcessor` that does so.
|
||||
|
||||
External trace processors include:
|
||||
## External tracing processors list
|
||||
|
||||
- [Weights & Biases](https://weave-docs.wandb.ai/guides/integrations/openai_agents)
|
||||
- [Arize-Phoenix](https://docs.arize.com/phoenix/tracing/integrations-tracing/openai-agents-sdk)
|
||||
- [MLflow](https://mlflow.org/docs/latest/tracing/integrations/openai-agent)
|
||||
- [Braintrust](https://braintrust.dev/docs/guides/traces/integrations#openai-agents-sdk)
|
||||
- [Pydantic Logfire](https://logfire.pydantic.dev/docs/integrations/llms/openai/#openai-agents)
|
||||
- [AgentOps](https://docs.agentops.ai/v1/integrations/agentssdk)
|
||||
- [Scorecard](https://docs.scorecard.io/docs/documentation/features/tracing#openai-agents-sdk-integration))
|
||||
- [Scorecard](https://docs.scorecard.io/docs/documentation/features/tracing#openai-agents-sdk-integration)
|
||||
- [Keywords AI](https://docs.keywordsai.co/integration/development-frameworks/openai-agent)
|
||||
- [LangSmith](https://docs.smith.langchain.com/observability/how_to_guides/trace_with_openai_agents_sdk)
|
||||
- [Maxim AI](https://www.getmaxim.ai/docs/observe/integrations/openai-agents-sdk)
|
||||
- [Comet Opik](https://www.comet.com/docs/opik/tracing/integrations/openai_agents)
|
||||
- [Langfuse](https://langfuse.com/docs/integrations/openaiagentssdk/openai-agents)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,75 @@
|
|||
# Pipelines and workflows
|
||||
|
||||
[`VoicePipeline`][agents.voice.pipeline.VoicePipeline] is a class that makes it easy to turn your agentic workflows into a voice app. You pass in a workflow to run, and the pipeline takes care of transcribing input audio, detecting when the audio ends, calling your workflow at the right time, and turning the workflow output back into audio.
|
||||
|
||||
```mermaid
|
||||
graph LR
|
||||
%% Input
|
||||
A["🎤 Audio Input"]
|
||||
|
||||
%% Voice Pipeline
|
||||
subgraph Voice_Pipeline [Voice Pipeline]
|
||||
direction TB
|
||||
B["Transcribe (speech-to-text)"]
|
||||
C["Your Code"]:::highlight
|
||||
D["Text-to-speech"]
|
||||
B --> C --> D
|
||||
end
|
||||
|
||||
%% Output
|
||||
E["🎧 Audio Output"]
|
||||
|
||||
%% Flow
|
||||
A --> Voice_Pipeline
|
||||
Voice_Pipeline --> E
|
||||
|
||||
%% Custom styling
|
||||
classDef highlight fill:#ffcc66,stroke:#333,stroke-width:1px,font-weight:700;
|
||||
|
||||
```
|
||||
|
||||
## Configuring a pipeline
|
||||
|
||||
When you create a pipeline, you can set a few things:
|
||||
|
||||
1. The [`workflow`][agents.voice.workflow.VoiceWorkflowBase], which is the code that runs each time new audio is transcribed.
|
||||
2. The [`speech-to-text`][agents.voice.model.STTModel] and [`text-to-speech`][agents.voice.model.TTSModel] models used
|
||||
3. The [`config`][agents.voice.pipeline_config.VoicePipelineConfig], which lets you configure things like:
|
||||
- A model provider, which can map model names to models
|
||||
- Tracing, including whether to disable tracing, whether audio files are uploaded, the workflow name, trace IDs etc.
|
||||
- Settings on the TTS and STT models, like the prompt, language and data types used.
|
||||
|
||||
## Running a pipeline
|
||||
|
||||
You can run a pipeline via the [`run()`][agents.voice.pipeline.VoicePipeline.run] method, which lets you pass in audio input in two forms:
|
||||
|
||||
1. [`AudioInput`][agents.voice.input.AudioInput] is used when you have a full audio transcript, and just want to produce a result for it. This is useful in cases where you don't need to detect when a speaker is done speaking; for example, when you have pre-recorded audio or in push-to-talk apps where it's clear when the user is done speaking.
|
||||
2. [`StreamedAudioInput`][agents.voice.input.StreamedAudioInput] is used when you might need to detect when a user is done speaking. It allows you to push audio chunks as they are detected, and the voice pipeline will automatically run the agent workflow at the right time, via a process called "activity detection".
|
||||
|
||||
## Results
|
||||
|
||||
The result of a voice pipeline run is a [`StreamedAudioResult`][agents.voice.result.StreamedAudioResult]. This is an object that lets you stream events as they occur. There are a few kinds of [`VoiceStreamEvent`][agents.voice.events.VoiceStreamEvent], including:
|
||||
|
||||
1. [`VoiceStreamEventAudio`][agents.voice.events.VoiceStreamEventAudio], which contains a chunk of audio.
|
||||
2. [`VoiceStreamEventLifecycle`][agents.voice.events.VoiceStreamEventLifecycle], which informs you of lifecycle events like a turn starting or ending.
|
||||
3. [`VoiceStreamEventError`][agents.voice.events.VoiceStreamEventError], is an error event.
|
||||
|
||||
```python
|
||||
|
||||
result = await pipeline.run(input)
|
||||
|
||||
async for event in result.stream():
|
||||
if event.type == "voice_stream_event_audio":
|
||||
# play audio
|
||||
elif event.type == "voice_stream_event_lifecycle":
|
||||
# lifecycle
|
||||
elif event.type == "voice_stream_event_error"
|
||||
# error
|
||||
...
|
||||
```
|
||||
|
||||
## Best practices
|
||||
|
||||
### Interruptions
|
||||
|
||||
The Agents SDK currently does not support any built-in interruptions support for [`StreamedAudioInput`][agents.voice.input.StreamedAudioInput]. Instead for every detected turn it will trigger a separate run of your workflow. If you want to handle interruptions inside your application you can listen to the [`VoiceStreamEventLifecycle`][agents.voice.events.VoiceStreamEventLifecycle] events. `turn_started` will indicate that a new turn was transcribed and processing is beginning. `turn_ended` will trigger after all the audio was dispatched for a respective turn. You could use these events to mute the microphone of the speaker when the model starts a turn and unmute it after you flushed all the related audio for a turn.
|
||||
|
|
@ -0,0 +1,194 @@
|
|||
# Quickstart
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Make sure you've followed the base [quickstart instructions](../quickstart.md) for the Agents SDK, and set up a virtual environment. Then, install the optional voice dependencies from the SDK:
|
||||
|
||||
```bash
|
||||
pip install 'openai-agents[voice]'
|
||||
```
|
||||
|
||||
## Concepts
|
||||
|
||||
The main concept to know about is a [`VoicePipeline`][agents.voice.pipeline.VoicePipeline], which is a 3 step process:
|
||||
|
||||
1. Run a speech-to-text model to turn audio into text.
|
||||
2. Run your code, which is usually an agentic workflow, to produce a result.
|
||||
3. Run a text-to-speech model to turn the result text back into audio.
|
||||
|
||||
```mermaid
|
||||
graph LR
|
||||
%% Input
|
||||
A["🎤 Audio Input"]
|
||||
|
||||
%% Voice Pipeline
|
||||
subgraph Voice_Pipeline [Voice Pipeline]
|
||||
direction TB
|
||||
B["Transcribe (speech-to-text)"]
|
||||
C["Your Code"]:::highlight
|
||||
D["Text-to-speech"]
|
||||
B --> C --> D
|
||||
end
|
||||
|
||||
%% Output
|
||||
E["🎧 Audio Output"]
|
||||
|
||||
%% Flow
|
||||
A --> Voice_Pipeline
|
||||
Voice_Pipeline --> E
|
||||
|
||||
%% Custom styling
|
||||
classDef highlight fill:#ffcc66,stroke:#333,stroke-width:1px,font-weight:700;
|
||||
|
||||
```
|
||||
|
||||
## Agents
|
||||
|
||||
First, let's set up some Agents. This should feel familiar to you if you've built any agents with this SDK. We'll have a couple of Agents, a handoff, and a tool.
|
||||
|
||||
```python
|
||||
import asyncio
|
||||
import random
|
||||
|
||||
from agents import (
|
||||
Agent,
|
||||
function_tool,
|
||||
)
|
||||
from agents.extensions.handoff_prompt import prompt_with_handoff_instructions
|
||||
|
||||
|
||||
|
||||
@function_tool
|
||||
def get_weather(city: str) -> str:
|
||||
"""Get the weather for a given city."""
|
||||
print(f"[debug] get_weather called with city: {city}")
|
||||
choices = ["sunny", "cloudy", "rainy", "snowy"]
|
||||
return f"The weather in {city} is {random.choice(choices)}."
|
||||
|
||||
|
||||
spanish_agent = Agent(
|
||||
name="Spanish",
|
||||
handoff_description="A spanish speaking agent.",
|
||||
instructions=prompt_with_handoff_instructions(
|
||||
"You're speaking to a human, so be polite and concise. Speak in Spanish.",
|
||||
),
|
||||
model="gpt-4o-mini",
|
||||
)
|
||||
|
||||
agent = Agent(
|
||||
name="Assistant",
|
||||
instructions=prompt_with_handoff_instructions(
|
||||
"You're speaking to a human, so be polite and concise. If the user speaks in Spanish, handoff to the spanish agent.",
|
||||
),
|
||||
model="gpt-4o-mini",
|
||||
handoffs=[spanish_agent],
|
||||
tools=[get_weather],
|
||||
)
|
||||
```
|
||||
|
||||
## Voice pipeline
|
||||
|
||||
We'll set up a simple voice pipeline, using [`SingleAgentVoiceWorkflow`][agents.voice.workflow.SingleAgentVoiceWorkflow] as the workflow.
|
||||
|
||||
```python
|
||||
from agents.voice import SingleAgentVoiceWorkflow, VoicePipeline
|
||||
pipeline = VoicePipeline(workflow=SingleAgentVoiceWorkflow(agent))
|
||||
```
|
||||
|
||||
## Run the pipeline
|
||||
|
||||
```python
|
||||
import numpy as np
|
||||
import sounddevice as sd
|
||||
from agents.voice import AudioInput
|
||||
|
||||
# For simplicity, we'll just create 3 seconds of silence
|
||||
# In reality, you'd get microphone data
|
||||
buffer = np.zeros(24000 * 3, dtype=np.int16)
|
||||
audio_input = AudioInput(buffer=buffer)
|
||||
|
||||
result = await pipeline.run(audio_input)
|
||||
|
||||
# Create an audio player using `sounddevice`
|
||||
player = sd.OutputStream(samplerate=24000, channels=1, dtype=np.int16)
|
||||
player.start()
|
||||
|
||||
# Play the audio stream as it comes in
|
||||
async for event in result.stream():
|
||||
if event.type == "voice_stream_event_audio":
|
||||
player.write(event.data)
|
||||
|
||||
```
|
||||
|
||||
## Put it all together
|
||||
|
||||
```python
|
||||
import asyncio
|
||||
import random
|
||||
|
||||
import numpy as np
|
||||
import sounddevice as sd
|
||||
|
||||
from agents import (
|
||||
Agent,
|
||||
function_tool,
|
||||
set_tracing_disabled,
|
||||
)
|
||||
from agents.voice import (
|
||||
AudioInput,
|
||||
SingleAgentVoiceWorkflow,
|
||||
VoicePipeline,
|
||||
)
|
||||
from agents.extensions.handoff_prompt import prompt_with_handoff_instructions
|
||||
|
||||
|
||||
@function_tool
|
||||
def get_weather(city: str) -> str:
|
||||
"""Get the weather for a given city."""
|
||||
print(f"[debug] get_weather called with city: {city}")
|
||||
choices = ["sunny", "cloudy", "rainy", "snowy"]
|
||||
return f"The weather in {city} is {random.choice(choices)}."
|
||||
|
||||
|
||||
spanish_agent = Agent(
|
||||
name="Spanish",
|
||||
handoff_description="A spanish speaking agent.",
|
||||
instructions=prompt_with_handoff_instructions(
|
||||
"You're speaking to a human, so be polite and concise. Speak in Spanish.",
|
||||
),
|
||||
model="gpt-4o-mini",
|
||||
)
|
||||
|
||||
agent = Agent(
|
||||
name="Assistant",
|
||||
instructions=prompt_with_handoff_instructions(
|
||||
"You're speaking to a human, so be polite and concise. If the user speaks in Spanish, handoff to the spanish agent.",
|
||||
),
|
||||
model="gpt-4o-mini",
|
||||
handoffs=[spanish_agent],
|
||||
tools=[get_weather],
|
||||
)
|
||||
|
||||
|
||||
async def main():
|
||||
pipeline = VoicePipeline(workflow=SingleAgentVoiceWorkflow(agent))
|
||||
buffer = np.zeros(24000 * 3, dtype=np.int16)
|
||||
audio_input = AudioInput(buffer=buffer)
|
||||
|
||||
result = await pipeline.run(audio_input)
|
||||
|
||||
# Create an audio player using `sounddevice`
|
||||
player = sd.OutputStream(samplerate=24000, channels=1, dtype=np.int16)
|
||||
player.start()
|
||||
|
||||
# Play the audio stream as it comes in
|
||||
async for event in result.stream():
|
||||
if event.type == "voice_stream_event_audio":
|
||||
player.write(event.data)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
```
|
||||
|
||||
If you run this example, the agent will speak to you! Check out the example in [examples/voice/static](https://github.com/openai/openai-agents-python/tree/main/examples/voice/static) to see a demo where you can speak to the agent yourself.
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
# Tracing
|
||||
|
||||
Just like the way [agents are traced](../tracing.md), voice pipelines are also automatically traced.
|
||||
|
||||
You can read the tracing doc above for basic tracing information, but you can additionally configure tracing of a pipeline via [`VoicePipelineConfig`][agents.voice.pipeline_config.VoicePipelineConfig].
|
||||
|
||||
Key tracing related fields are:
|
||||
|
||||
- [`tracing_disabled`][agents.voice.pipeline_config.VoicePipelineConfig.tracing_disabled]: controls whether tracing is disabled. By default, tracing is enabled.
|
||||
- [`trace_include_sensitive_data`][agents.voice.pipeline_config.VoicePipelineConfig.trace_include_sensitive_data]: controls whether traces include potentially sensitive data, like audio transcripts. This is specifically for the voice pipeline, and not for anything that goes on inside your Workflow.
|
||||
- [`trace_include_sensitive_audio_data`][agents.voice.pipeline_config.VoicePipelineConfig.trace_include_sensitive_audio_data]: controls whether traces include audio data.
|
||||
- [`workflow_name`][agents.voice.pipeline_config.VoicePipelineConfig.workflow_name]: The name of the trace workflow.
|
||||
- [`group_id`][agents.voice.pipeline_config.VoicePipelineConfig.group_id]: The `group_id` of the trace, which lets you link multiple traces.
|
||||
- [`trace_metadata`][agents.voice.pipeline_config.VoicePipelineConfig.tracing_disabled]: Additional metadata to include with the trace.
|
||||
|
|
@ -0,0 +1,99 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from agents import (
|
||||
Agent,
|
||||
FunctionToolResult,
|
||||
ModelSettings,
|
||||
RunContextWrapper,
|
||||
Runner,
|
||||
ToolsToFinalOutputFunction,
|
||||
ToolsToFinalOutputResult,
|
||||
function_tool,
|
||||
)
|
||||
|
||||
"""
|
||||
This example shows how to force the agent to use a tool. It uses `ModelSettings(tool_choice="required")`
|
||||
to force the agent to use any tool.
|
||||
|
||||
You can run it with 3 options:
|
||||
1. `default`: The default behavior, which is to send the tool output to the LLM. In this case,
|
||||
`tool_choice` is not set, because otherwise it would result in an infinite loop - the LLM would
|
||||
call the tool, the tool would run and send the results to the LLM, and that would repeat
|
||||
(because the model is forced to use a tool every time.)
|
||||
2. `first_tool_result`: The first tool result is used as the final output.
|
||||
3. `custom`: A custom tool use behavior function is used. The custom function receives all the tool
|
||||
results, and chooses to use the first tool result to generate the final output.
|
||||
|
||||
Usage:
|
||||
python examples/agent_patterns/forcing_tool_use.py -t default
|
||||
python examples/agent_patterns/forcing_tool_use.py -t first_tool
|
||||
python examples/agent_patterns/forcing_tool_use.py -t custom
|
||||
"""
|
||||
|
||||
|
||||
class Weather(BaseModel):
|
||||
city: str
|
||||
temperature_range: str
|
||||
conditions: str
|
||||
|
||||
|
||||
@function_tool
|
||||
def get_weather(city: str) -> Weather:
|
||||
print("[debug] get_weather called")
|
||||
return Weather(city=city, temperature_range="14-20C", conditions="Sunny with wind")
|
||||
|
||||
|
||||
async def custom_tool_use_behavior(
|
||||
context: RunContextWrapper[Any], results: list[FunctionToolResult]
|
||||
) -> ToolsToFinalOutputResult:
|
||||
weather: Weather = results[0].output
|
||||
return ToolsToFinalOutputResult(
|
||||
is_final_output=True, final_output=f"{weather.city} is {weather.conditions}."
|
||||
)
|
||||
|
||||
|
||||
async def main(tool_use_behavior: Literal["default", "first_tool", "custom"] = "default"):
|
||||
if tool_use_behavior == "default":
|
||||
behavior: Literal["run_llm_again", "stop_on_first_tool"] | ToolsToFinalOutputFunction = (
|
||||
"run_llm_again"
|
||||
)
|
||||
elif tool_use_behavior == "first_tool":
|
||||
behavior = "stop_on_first_tool"
|
||||
elif tool_use_behavior == "custom":
|
||||
behavior = custom_tool_use_behavior
|
||||
|
||||
agent = Agent(
|
||||
name="Weather agent",
|
||||
instructions="You are a helpful agent.",
|
||||
tools=[get_weather],
|
||||
tool_use_behavior=behavior,
|
||||
model_settings=ModelSettings(
|
||||
tool_choice="required" if tool_use_behavior != "default" else None
|
||||
),
|
||||
)
|
||||
|
||||
result = await Runner.run(agent, input="What's the weather in Tokyo?")
|
||||
print(result.final_output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"-t",
|
||||
"--tool-use-behavior",
|
||||
type=str,
|
||||
required=True,
|
||||
choices=["default", "first_tool", "custom"],
|
||||
help="The behavior to use for tool use. Default will cause tool outputs to be sent to the model. "
|
||||
"first_tool_result will cause the first tool result to be used as the final output. "
|
||||
"custom will use a custom tool use behavior function.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
asyncio.run(main(args.tool_use_behavior))
|
||||
|
|
@ -30,8 +30,8 @@ If the guardrail trips, we'll respond with a refusal message.
|
|||
|
||||
### 1. An agent-based guardrail that is triggered if the user is asking to do math homework
|
||||
class MathHomeworkOutput(BaseModel):
|
||||
is_math_homework: bool
|
||||
reasoning: str
|
||||
is_math_homework: bool
|
||||
|
||||
|
||||
guardrail_agent = Agent(
|
||||
|
|
|
|||
|
|
@ -23,8 +23,8 @@ story_outline_generator = Agent(
|
|||
|
||||
@dataclass
|
||||
class EvaluationFeedback:
|
||||
score: Literal["pass", "needs_improvement", "fail"]
|
||||
feedback: str
|
||||
score: Literal["pass", "needs_improvement", "fail"]
|
||||
|
||||
|
||||
evaluator = Agent[None](
|
||||
|
|
|
|||
|
|
@ -74,7 +74,7 @@ multiply_agent = Agent(
|
|||
|
||||
start_agent = Agent(
|
||||
name="Start Agent",
|
||||
instructions="Generate a random number. If it's even, stop. If it's odd, hand off to the multipler agent.",
|
||||
instructions="Generate a random number. If it's even, stop. If it's odd, hand off to the multiply agent.",
|
||||
tools=[random_number],
|
||||
output_type=FinalResult,
|
||||
handoffs=[multiply_agent],
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ from agents import Agent, Runner
|
|||
agent = Agent(name="Assistant", instructions="You are a helpful assistant")
|
||||
|
||||
# Intended for Jupyter notebooks where there's an existing event loop
|
||||
result = await Runner.run(agent, "Write a haiku about recursion in programming.") # type: ignore[top-level-await] # noqa: F704
|
||||
result = await Runner.run(agent, "Write a haiku about recursion in programming.") # type: ignore[top-level-await] # noqa: F704
|
||||
print(result.final_output)
|
||||
|
||||
# Code within code loops,
|
||||
|
|
|
|||
|
|
@ -79,7 +79,7 @@ multiply_agent = Agent(
|
|||
|
||||
start_agent = Agent(
|
||||
name="Start Agent",
|
||||
instructions="Generate a random number. If it's even, stop. If it's odd, hand off to the multipler agent.",
|
||||
instructions="Generate a random number. If it's even, stop. If it's odd, hand off to the multiplier agent.",
|
||||
tools=[random_number],
|
||||
output_type=FinalResult,
|
||||
handoffs=[multiply_agent],
|
||||
|
|
|
|||
|
|
@ -0,0 +1,34 @@
|
|||
import asyncio
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from agents import Agent, Runner, function_tool
|
||||
|
||||
|
||||
class Weather(BaseModel):
|
||||
city: str
|
||||
temperature_range: str
|
||||
conditions: str
|
||||
|
||||
|
||||
@function_tool
|
||||
def get_weather(city: str) -> Weather:
|
||||
print("[debug] get_weather called")
|
||||
return Weather(city=city, temperature_range="14-20C", conditions="Sunny with wind.")
|
||||
|
||||
|
||||
agent = Agent(
|
||||
name="Hello world",
|
||||
instructions="You are a helpful agent.",
|
||||
tools=[get_weather],
|
||||
)
|
||||
|
||||
|
||||
async def main():
|
||||
result = await Runner.run(agent, input="What's the weather in Tokyo?")
|
||||
print(result.final_output)
|
||||
# The weather in Tokyo is sunny.
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
# Financial Research Agent Example
|
||||
|
||||
This example shows how you might compose a richer financial research agent using the Agents SDK. The pattern is similar to the `research_bot` example, but with more specialized sub‑agents and a verification step.
|
||||
|
||||
The flow is:
|
||||
|
||||
1. **Planning**: A planner agent turns the end user’s request into a list of search terms relevant to financial analysis – recent news, earnings calls, corporate filings, industry commentary, etc.
|
||||
2. **Search**: A search agent uses the built‑in `WebSearchTool` to retrieve terse summaries for each search term. (You could also add `FileSearchTool` if you have indexed PDFs or 10‑Ks.)
|
||||
3. **Sub‑analysts**: Additional agents (e.g. a fundamentals analyst and a risk analyst) are exposed as tools so the writer can call them inline and incorporate their outputs.
|
||||
4. **Writing**: A senior writer agent brings together the search snippets and any sub‑analyst summaries into a long‑form markdown report plus a short executive summary.
|
||||
5. **Verification**: A final verifier agent audits the report for obvious inconsistencies or missing sourcing.
|
||||
|
||||
You can run the example with:
|
||||
|
||||
```bash
|
||||
python -m examples.financial_research_agent.main
|
||||
```
|
||||
|
||||
and enter a query like:
|
||||
|
||||
```
|
||||
Write up an analysis of Apple Inc.'s most recent quarter.
|
||||
```
|
||||
|
||||
### Starter prompt
|
||||
|
||||
The writer agent is seeded with instructions similar to:
|
||||
|
||||
```
|
||||
You are a senior financial analyst. You will be provided with the original query
|
||||
and a set of raw search summaries. Your job is to synthesize these into a
|
||||
long‑form markdown report (at least several paragraphs) with a short executive
|
||||
summary. You also have access to tools like `fundamentals_analysis` and
|
||||
`risk_analysis` to get short specialist write‑ups if you want to incorporate them.
|
||||
Add a few follow‑up questions for further research.
|
||||
```
|
||||
|
||||
You can tweak these prompts and sub‑agents to suit your own data sources and preferred report structure.
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
from pydantic import BaseModel
|
||||
|
||||
from agents import Agent
|
||||
|
||||
# A sub‑agent focused on analyzing a company's fundamentals.
|
||||
FINANCIALS_PROMPT = (
|
||||
"You are a financial analyst focused on company fundamentals such as revenue, "
|
||||
"profit, margins and growth trajectory. Given a collection of web (and optional file) "
|
||||
"search results about a company, write a concise analysis of its recent financial "
|
||||
"performance. Pull out key metrics or quotes. Keep it under 2 paragraphs."
|
||||
)
|
||||
|
||||
|
||||
class AnalysisSummary(BaseModel):
|
||||
summary: str
|
||||
"""Short text summary for this aspect of the analysis."""
|
||||
|
||||
|
||||
financials_agent = Agent(
|
||||
name="FundamentalsAnalystAgent",
|
||||
instructions=FINANCIALS_PROMPT,
|
||||
output_type=AnalysisSummary,
|
||||
)
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
from pydantic import BaseModel
|
||||
|
||||
from agents import Agent
|
||||
|
||||
# Generate a plan of searches to ground the financial analysis.
|
||||
# For a given financial question or company, we want to search for
|
||||
# recent news, official filings, analyst commentary, and other
|
||||
# relevant background.
|
||||
PROMPT = (
|
||||
"You are a financial research planner. Given a request for financial analysis, "
|
||||
"produce a set of web searches to gather the context needed. Aim for recent "
|
||||
"headlines, earnings calls or 10‑K snippets, analyst commentary, and industry background. "
|
||||
"Output between 5 and 15 search terms to query for."
|
||||
)
|
||||
|
||||
|
||||
class FinancialSearchItem(BaseModel):
|
||||
reason: str
|
||||
"""Your reasoning for why this search is relevant."""
|
||||
|
||||
query: str
|
||||
"""The search term to feed into a web (or file) search."""
|
||||
|
||||
|
||||
class FinancialSearchPlan(BaseModel):
|
||||
searches: list[FinancialSearchItem]
|
||||
"""A list of searches to perform."""
|
||||
|
||||
|
||||
planner_agent = Agent(
|
||||
name="FinancialPlannerAgent",
|
||||
instructions=PROMPT,
|
||||
model="o3-mini",
|
||||
output_type=FinancialSearchPlan,
|
||||
)
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
from pydantic import BaseModel
|
||||
|
||||
from agents import Agent
|
||||
|
||||
# A sub‑agent specializing in identifying risk factors or concerns.
|
||||
RISK_PROMPT = (
|
||||
"You are a risk analyst looking for potential red flags in a company's outlook. "
|
||||
"Given background research, produce a short analysis of risks such as competitive threats, "
|
||||
"regulatory issues, supply chain problems, or slowing growth. Keep it under 2 paragraphs."
|
||||
)
|
||||
|
||||
|
||||
class AnalysisSummary(BaseModel):
|
||||
summary: str
|
||||
"""Short text summary for this aspect of the analysis."""
|
||||
|
||||
|
||||
risk_agent = Agent(
|
||||
name="RiskAnalystAgent",
|
||||
instructions=RISK_PROMPT,
|
||||
output_type=AnalysisSummary,
|
||||
)
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
from agents import Agent, WebSearchTool
|
||||
from agents.model_settings import ModelSettings
|
||||
|
||||
# Given a search term, use web search to pull back a brief summary.
|
||||
# Summaries should be concise but capture the main financial points.
|
||||
INSTRUCTIONS = (
|
||||
"You are a research assistant specializing in financial topics. "
|
||||
"Given a search term, use web search to retrieve up‑to‑date context and "
|
||||
"produce a short summary of at most 300 words. Focus on key numbers, events, "
|
||||
"or quotes that will be useful to a financial analyst."
|
||||
)
|
||||
|
||||
search_agent = Agent(
|
||||
name="FinancialSearchAgent",
|
||||
instructions=INSTRUCTIONS,
|
||||
tools=[WebSearchTool()],
|
||||
model_settings=ModelSettings(tool_choice="required"),
|
||||
)
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
from pydantic import BaseModel
|
||||
|
||||
from agents import Agent
|
||||
|
||||
# Agent to sanity‑check a synthesized report for consistency and recall.
|
||||
# This can be used to flag potential gaps or obvious mistakes.
|
||||
VERIFIER_PROMPT = (
|
||||
"You are a meticulous auditor. You have been handed a financial analysis report. "
|
||||
"Your job is to verify the report is internally consistent, clearly sourced, and makes "
|
||||
"no unsupported claims. Point out any issues or uncertainties."
|
||||
)
|
||||
|
||||
|
||||
class VerificationResult(BaseModel):
|
||||
verified: bool
|
||||
"""Whether the report seems coherent and plausible."""
|
||||
|
||||
issues: str
|
||||
"""If not verified, describe the main issues or concerns."""
|
||||
|
||||
|
||||
verifier_agent = Agent(
|
||||
name="VerificationAgent",
|
||||
instructions=VERIFIER_PROMPT,
|
||||
model="gpt-4o",
|
||||
output_type=VerificationResult,
|
||||
)
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
from pydantic import BaseModel
|
||||
|
||||
from agents import Agent
|
||||
|
||||
# Writer agent brings together the raw search results and optionally calls out
|
||||
# to sub‑analyst tools for specialized commentary, then returns a cohesive markdown report.
|
||||
WRITER_PROMPT = (
|
||||
"You are a senior financial analyst. You will be provided with the original query and "
|
||||
"a set of raw search summaries. Your task is to synthesize these into a long‑form markdown "
|
||||
"report (at least several paragraphs) including a short executive summary and follow‑up "
|
||||
"questions. If needed, you can call the available analysis tools (e.g. fundamentals_analysis, "
|
||||
"risk_analysis) to get short specialist write‑ups to incorporate."
|
||||
)
|
||||
|
||||
|
||||
class FinancialReportData(BaseModel):
|
||||
short_summary: str
|
||||
"""A short 2‑3 sentence executive summary."""
|
||||
|
||||
markdown_report: str
|
||||
"""The full markdown report."""
|
||||
|
||||
follow_up_questions: list[str]
|
||||
"""Suggested follow‑up questions for further research."""
|
||||
|
||||
|
||||
# Note: We will attach handoffs to specialist analyst agents at runtime in the manager.
|
||||
# This shows how an agent can use handoffs to delegate to specialized subagents.
|
||||
writer_agent = Agent(
|
||||
name="FinancialWriterAgent",
|
||||
instructions=WRITER_PROMPT,
|
||||
model="gpt-4.5-preview-2025-02-27",
|
||||
output_type=FinancialReportData,
|
||||
)
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
import asyncio
|
||||
|
||||
from .manager import FinancialResearchManager
|
||||
|
||||
|
||||
# Entrypoint for the financial bot example.
|
||||
# Run this as `python -m examples.financial_bot.main` and enter a
|
||||
# financial research query, for example:
|
||||
# "Write up an analysis of Apple Inc.'s most recent quarter."
|
||||
async def main() -> None:
|
||||
query = input("Enter a financial research query: ")
|
||||
mgr = FinancialResearchManager()
|
||||
await mgr.run(query)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,135 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from collections.abc import Sequence
|
||||
|
||||
from rich.console import Console
|
||||
|
||||
from agents import Runner, RunResult, custom_span, gen_trace_id, trace
|
||||
|
||||
from .agents.financials_agent import financials_agent
|
||||
from .agents.planner_agent import FinancialSearchItem, FinancialSearchPlan, planner_agent
|
||||
from .agents.risk_agent import risk_agent
|
||||
from .agents.search_agent import search_agent
|
||||
from .agents.verifier_agent import VerificationResult, verifier_agent
|
||||
from .agents.writer_agent import FinancialReportData, writer_agent
|
||||
from .printer import Printer
|
||||
|
||||
|
||||
async def _summary_extractor(run_result: RunResult) -> str:
|
||||
"""Custom output extractor for sub‑agents that return an AnalysisSummary."""
|
||||
# The financial/risk analyst agents emit an AnalysisSummary with a `summary` field.
|
||||
# We want the tool call to return just that summary text so the writer can drop it inline.
|
||||
return str(run_result.final_output.summary)
|
||||
|
||||
|
||||
class FinancialResearchManager:
|
||||
"""
|
||||
Orchestrates the full flow: planning, searching, sub‑analysis, writing, and verification.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.console = Console()
|
||||
self.printer = Printer(self.console)
|
||||
|
||||
async def run(self, query: str) -> None:
|
||||
trace_id = gen_trace_id()
|
||||
with trace("Financial research trace", trace_id=trace_id):
|
||||
self.printer.update_item(
|
||||
"trace_id",
|
||||
f"View trace: https://platform.openai.com/traces/{trace_id}",
|
||||
is_done=True,
|
||||
hide_checkmark=True,
|
||||
)
|
||||
self.printer.update_item("start", "Starting financial research...", is_done=True)
|
||||
search_plan = await self._plan_searches(query)
|
||||
search_results = await self._perform_searches(search_plan)
|
||||
report = await self._write_report(query, search_results)
|
||||
verification = await self._verify_report(report)
|
||||
|
||||
final_report = f"Report summary\n\n{report.short_summary}"
|
||||
self.printer.update_item("final_report", final_report, is_done=True)
|
||||
|
||||
self.printer.end()
|
||||
|
||||
# Print to stdout
|
||||
print("\n\n=====REPORT=====\n\n")
|
||||
print(f"Report:\n{report.markdown_report}")
|
||||
print("\n\n=====FOLLOW UP QUESTIONS=====\n\n")
|
||||
print("\n".join(report.follow_up_questions))
|
||||
print("\n\n=====VERIFICATION=====\n\n")
|
||||
print(verification)
|
||||
|
||||
async def _plan_searches(self, query: str) -> FinancialSearchPlan:
|
||||
self.printer.update_item("planning", "Planning searches...")
|
||||
result = await Runner.run(planner_agent, f"Query: {query}")
|
||||
self.printer.update_item(
|
||||
"planning",
|
||||
f"Will perform {len(result.final_output.searches)} searches",
|
||||
is_done=True,
|
||||
)
|
||||
return result.final_output_as(FinancialSearchPlan)
|
||||
|
||||
async def _perform_searches(self, search_plan: FinancialSearchPlan) -> Sequence[str]:
|
||||
with custom_span("Search the web"):
|
||||
self.printer.update_item("searching", "Searching...")
|
||||
tasks = [asyncio.create_task(self._search(item)) for item in search_plan.searches]
|
||||
results: list[str] = []
|
||||
num_completed = 0
|
||||
for task in asyncio.as_completed(tasks):
|
||||
result = await task
|
||||
if result is not None:
|
||||
results.append(result)
|
||||
num_completed += 1
|
||||
self.printer.update_item(
|
||||
"searching", f"Searching... {num_completed}/{len(tasks)} completed"
|
||||
)
|
||||
self.printer.mark_item_done("searching")
|
||||
return results
|
||||
|
||||
async def _search(self, item: FinancialSearchItem) -> str | None:
|
||||
input_data = f"Search term: {item.query}\nReason: {item.reason}"
|
||||
try:
|
||||
result = await Runner.run(search_agent, input_data)
|
||||
return str(result.final_output)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
async def _write_report(self, query: str, search_results: Sequence[str]) -> FinancialReportData:
|
||||
# Expose the specialist analysts as tools so the writer can invoke them inline
|
||||
# and still produce the final FinancialReportData output.
|
||||
fundamentals_tool = financials_agent.as_tool(
|
||||
tool_name="fundamentals_analysis",
|
||||
tool_description="Use to get a short write‑up of key financial metrics",
|
||||
custom_output_extractor=_summary_extractor,
|
||||
)
|
||||
risk_tool = risk_agent.as_tool(
|
||||
tool_name="risk_analysis",
|
||||
tool_description="Use to get a short write‑up of potential red flags",
|
||||
custom_output_extractor=_summary_extractor,
|
||||
)
|
||||
writer_with_tools = writer_agent.clone(tools=[fundamentals_tool, risk_tool])
|
||||
self.printer.update_item("writing", "Thinking about report...")
|
||||
input_data = f"Original query: {query}\nSummarized search results: {search_results}"
|
||||
result = Runner.run_streamed(writer_with_tools, input_data)
|
||||
update_messages = [
|
||||
"Planning report structure...",
|
||||
"Writing sections...",
|
||||
"Finalizing report...",
|
||||
]
|
||||
last_update = time.time()
|
||||
next_message = 0
|
||||
async for _ in result.stream_events():
|
||||
if time.time() - last_update > 5 and next_message < len(update_messages):
|
||||
self.printer.update_item("writing", update_messages[next_message])
|
||||
next_message += 1
|
||||
last_update = time.time()
|
||||
self.printer.mark_item_done("writing")
|
||||
return result.final_output_as(FinancialReportData)
|
||||
|
||||
async def _verify_report(self, report: FinancialReportData) -> VerificationResult:
|
||||
self.printer.update_item("verifying", "Verifying report...")
|
||||
result = await Runner.run(verifier_agent, report.markdown_report)
|
||||
self.printer.mark_item_done("verifying")
|
||||
return result.final_output_as(VerificationResult)
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
from typing import Any
|
||||
|
||||
from rich.console import Console, Group
|
||||
from rich.live import Live
|
||||
from rich.spinner import Spinner
|
||||
|
||||
|
||||
class Printer:
|
||||
"""
|
||||
Simple wrapper to stream status updates. Used by the financial bot
|
||||
manager as it orchestrates planning, search and writing.
|
||||
"""
|
||||
|
||||
def __init__(self, console: Console) -> None:
|
||||
self.live = Live(console=console)
|
||||
self.items: dict[str, tuple[str, bool]] = {}
|
||||
self.hide_done_ids: set[str] = set()
|
||||
self.live.start()
|
||||
|
||||
def end(self) -> None:
|
||||
self.live.stop()
|
||||
|
||||
def hide_done_checkmark(self, item_id: str) -> None:
|
||||
self.hide_done_ids.add(item_id)
|
||||
|
||||
def update_item(
|
||||
self, item_id: str, content: str, is_done: bool = False, hide_checkmark: bool = False
|
||||
) -> None:
|
||||
self.items[item_id] = (content, is_done)
|
||||
if hide_checkmark:
|
||||
self.hide_done_ids.add(item_id)
|
||||
self.flush()
|
||||
|
||||
def mark_item_done(self, item_id: str) -> None:
|
||||
self.items[item_id] = (self.items[item_id][0], True)
|
||||
self.flush()
|
||||
|
||||
def flush(self) -> None:
|
||||
renderables: list[Any] = []
|
||||
for item_id, (content, is_done) in self.items.items():
|
||||
if is_done:
|
||||
prefix = "✅ " if item_id not in self.hide_done_ids else ""
|
||||
renderables.append(prefix + content)
|
||||
else:
|
||||
renderables.append(Spinner("dots", text=content))
|
||||
self.live.update(Group(*renderables))
|
||||
|
|
@ -60,9 +60,9 @@ async def main():
|
|||
|
||||
print("Step 1 done")
|
||||
|
||||
# 2. Ask it to square a number
|
||||
# 2. Ask it to generate a number
|
||||
result = await Runner.run(
|
||||
second_agent,
|
||||
first_agent,
|
||||
input=result.to_input_list()
|
||||
+ [{"content": "Can you generate a random number between 0 and 100?", "role": "user"}],
|
||||
)
|
||||
|
|
|
|||
|
|
@ -60,9 +60,9 @@ async def main():
|
|||
|
||||
print("Step 1 done")
|
||||
|
||||
# 2. Ask it to square a number
|
||||
# 2. Ask it to generate a number
|
||||
result = await Runner.run(
|
||||
second_agent,
|
||||
first_agent,
|
||||
input=result.to_input_list()
|
||||
+ [{"content": "Can you generate a random number between 0 and 100?", "role": "user"}],
|
||||
)
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ from agents.model_settings import ModelSettings
|
|||
INSTRUCTIONS = (
|
||||
"You are a research assistant. Given a search term, you search the web for that term and"
|
||||
"produce a concise summary of the results. The summary must 2-3 paragraphs and less than 300"
|
||||
"words. Capture the main points. Write succintly, no need to have complete sentences or good"
|
||||
"words. Capture the main points. Write succinctly, no need to have complete sentences or good"
|
||||
"grammar. This will be consumed by someone synthesizing a report, so its vital you capture the"
|
||||
"essence and ignore any fluff. Do not include any additional commentary other than the summary"
|
||||
"itself."
|
||||
|
|
|
|||
|
|
@ -0,0 +1,26 @@
|
|||
# Static voice demo
|
||||
|
||||
This demo operates by capturing a recording, then running a voice pipeline on it.
|
||||
|
||||
Run via:
|
||||
|
||||
```
|
||||
python -m examples.voice.static.main
|
||||
```
|
||||
|
||||
## How it works
|
||||
|
||||
1. We create a `VoicePipeline`, setup with a custom workflow. The workflow runs an Agent, but it also has some custom responses if you say the secret word.
|
||||
2. When you speak, audio is forwarded to the voice pipeline. When you stop speaking, the agent runs.
|
||||
3. The pipeline is run with the audio, which causes it to:
|
||||
1. Transcribe the audio
|
||||
2. Feed the transcription to the workflow, which runs the agent.
|
||||
3. Stream the output of the agent to a text-to-speech model.
|
||||
4. Play the audio.
|
||||
|
||||
Some suggested examples to try:
|
||||
|
||||
- Tell me a joke (_the assistant tells you a joke_)
|
||||
- What's the weather in Tokyo? (_will call the `get_weather` tool and then speak_)
|
||||
- Hola, como estas? (_will handoff to the spanish agent_)
|
||||
- Tell me about dogs. (_will respond with the hardcoded "you guessed the secret word" message_)
|
||||
|
|
@ -0,0 +1,83 @@
|
|||
import asyncio
|
||||
import random
|
||||
|
||||
from agents import Agent, function_tool
|
||||
from agents.extensions.handoff_prompt import prompt_with_handoff_instructions
|
||||
from agents.voice import (
|
||||
AudioInput,
|
||||
SingleAgentVoiceWorkflow,
|
||||
SingleAgentWorkflowCallbacks,
|
||||
VoicePipeline,
|
||||
)
|
||||
|
||||
from .util import AudioPlayer, record_audio
|
||||
|
||||
"""
|
||||
This is a simple example that uses a recorded audio buffer. Run it via:
|
||||
`python -m examples.voice.static.main`
|
||||
|
||||
1. You can record an audio clip in the terminal.
|
||||
2. The pipeline automatically transcribes the audio.
|
||||
3. The agent workflow is a simple one that starts at the Assistant agent.
|
||||
4. The output of the agent is streamed to the audio player.
|
||||
|
||||
Try examples like:
|
||||
- Tell me a joke (will respond with a joke)
|
||||
- What's the weather in Tokyo? (will call the `get_weather` tool and then speak)
|
||||
- Hola, como estas? (will handoff to the spanish agent)
|
||||
"""
|
||||
|
||||
|
||||
@function_tool
|
||||
def get_weather(city: str) -> str:
|
||||
"""Get the weather for a given city."""
|
||||
print(f"[debug] get_weather called with city: {city}")
|
||||
choices = ["sunny", "cloudy", "rainy", "snowy"]
|
||||
return f"The weather in {city} is {random.choice(choices)}."
|
||||
|
||||
|
||||
spanish_agent = Agent(
|
||||
name="Spanish",
|
||||
handoff_description="A spanish speaking agent.",
|
||||
instructions=prompt_with_handoff_instructions(
|
||||
"You're speaking to a human, so be polite and concise. Speak in Spanish.",
|
||||
),
|
||||
model="gpt-4o-mini",
|
||||
)
|
||||
|
||||
agent = Agent(
|
||||
name="Assistant",
|
||||
instructions=prompt_with_handoff_instructions(
|
||||
"You're speaking to a human, so be polite and concise. If the user speaks in Spanish, handoff to the spanish agent.",
|
||||
),
|
||||
model="gpt-4o-mini",
|
||||
handoffs=[spanish_agent],
|
||||
tools=[get_weather],
|
||||
)
|
||||
|
||||
|
||||
class WorkflowCallbacks(SingleAgentWorkflowCallbacks):
|
||||
def on_run(self, workflow: SingleAgentVoiceWorkflow, transcription: str) -> None:
|
||||
print(f"[debug] on_run called with transcription: {transcription}")
|
||||
|
||||
|
||||
async def main():
|
||||
pipeline = VoicePipeline(
|
||||
workflow=SingleAgentVoiceWorkflow(agent, callbacks=WorkflowCallbacks())
|
||||
)
|
||||
|
||||
audio_input = AudioInput(buffer=record_audio())
|
||||
|
||||
result = await pipeline.run(audio_input)
|
||||
|
||||
with AudioPlayer() as player:
|
||||
async for event in result.stream():
|
||||
if event.type == "voice_stream_event_audio":
|
||||
player.add_audio(event.data)
|
||||
print("Received audio")
|
||||
elif event.type == "voice_stream_event_lifecycle":
|
||||
print(f"Received lifecycle event: {event.event}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
import curses
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
import numpy.typing as npt
|
||||
import sounddevice as sd
|
||||
|
||||
|
||||
def _record_audio(screen: curses.window) -> npt.NDArray[np.float32]:
|
||||
screen.nodelay(True) # Non-blocking input
|
||||
screen.clear()
|
||||
screen.addstr(
|
||||
"Press <spacebar> to start recording. Press <spacebar> again to stop recording.\n"
|
||||
)
|
||||
screen.refresh()
|
||||
|
||||
recording = False
|
||||
audio_buffer: list[npt.NDArray[np.float32]] = []
|
||||
|
||||
def _audio_callback(indata, frames, time_info, status):
|
||||
if status:
|
||||
screen.addstr(f"Status: {status}\n")
|
||||
screen.refresh()
|
||||
if recording:
|
||||
audio_buffer.append(indata.copy())
|
||||
|
||||
# Open the audio stream with the callback.
|
||||
with sd.InputStream(samplerate=24000, channels=1, dtype=np.float32, callback=_audio_callback):
|
||||
while True:
|
||||
key = screen.getch()
|
||||
if key == ord(" "):
|
||||
recording = not recording
|
||||
if recording:
|
||||
screen.addstr("Recording started...\n")
|
||||
else:
|
||||
screen.addstr("Recording stopped.\n")
|
||||
break
|
||||
screen.refresh()
|
||||
time.sleep(0.01)
|
||||
|
||||
# Combine recorded audio chunks.
|
||||
if audio_buffer:
|
||||
audio_data = np.concatenate(audio_buffer, axis=0)
|
||||
else:
|
||||
audio_data = np.empty((0,), dtype=np.float32)
|
||||
|
||||
return audio_data
|
||||
|
||||
|
||||
def record_audio():
|
||||
# Using curses to record audio in a way that:
|
||||
# - doesn't require accessibility permissions on macos
|
||||
# - doesn't block the terminal
|
||||
audio_data = curses.wrapper(_record_audio)
|
||||
return audio_data
|
||||
|
||||
|
||||
class AudioPlayer:
|
||||
def __enter__(self):
|
||||
self.stream = sd.OutputStream(samplerate=24000, channels=1, dtype=np.int16)
|
||||
self.stream.start()
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
self.stream.close()
|
||||
|
||||
def add_audio(self, audio_data: npt.NDArray[np.int16]):
|
||||
self.stream.write(audio_data)
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
# Streamed voice demo
|
||||
|
||||
This is an interactive demo, where you can talk to an Agent conversationally. It uses the voice pipeline's built in turn detection feature, so if you stop speaking the Agent responds.
|
||||
|
||||
Run via:
|
||||
|
||||
```
|
||||
python -m examples.voice.streamed.main
|
||||
```
|
||||
|
||||
## How it works
|
||||
|
||||
1. We create a `VoicePipeline`, setup with a `SingleAgentVoiceWorkflow`. This is a workflow that starts at an Assistant agent, has tools and handoffs.
|
||||
2. Audio input is captured from the terminal.
|
||||
3. The pipeline is run with the recorded audio, which causes it to:
|
||||
1. Transcribe the audio
|
||||
2. Feed the transcription to the workflow, which runs the agent.
|
||||
3. Stream the output of the agent to a text-to-speech model.
|
||||
4. Play the audio.
|
||||
|
||||
Some suggested examples to try:
|
||||
|
||||
- Tell me a joke (_the assistant tells you a joke_)
|
||||
- What's the weather in Tokyo? (_will call the `get_weather` tool and then speak_)
|
||||
- Hola, como estas? (_will handoff to the spanish agent_)
|
||||
|
|
@ -0,0 +1,233 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import numpy as np
|
||||
import sounddevice as sd
|
||||
from textual import events
|
||||
from textual.app import App, ComposeResult
|
||||
from textual.containers import Container
|
||||
from textual.reactive import reactive
|
||||
from textual.widgets import Button, RichLog, Static
|
||||
from typing_extensions import override
|
||||
|
||||
from agents.voice import StreamedAudioInput, VoicePipeline
|
||||
|
||||
# Import MyWorkflow class - handle both module and package use cases
|
||||
if TYPE_CHECKING:
|
||||
# For type checking, use the relative import
|
||||
from .my_workflow import MyWorkflow
|
||||
else:
|
||||
# At runtime, try both import styles
|
||||
try:
|
||||
# Try relative import first (when used as a package)
|
||||
from .my_workflow import MyWorkflow
|
||||
except ImportError:
|
||||
# Fall back to direct import (when run as a script)
|
||||
from my_workflow import MyWorkflow
|
||||
|
||||
CHUNK_LENGTH_S = 0.05 # 100ms
|
||||
SAMPLE_RATE = 24000
|
||||
FORMAT = np.int16
|
||||
CHANNELS = 1
|
||||
|
||||
|
||||
class Header(Static):
|
||||
"""A header widget."""
|
||||
|
||||
session_id = reactive("")
|
||||
|
||||
@override
|
||||
def render(self) -> str:
|
||||
return "Speak to the agent. When you stop speaking, it will respond."
|
||||
|
||||
|
||||
class AudioStatusIndicator(Static):
|
||||
"""A widget that shows the current audio recording status."""
|
||||
|
||||
is_recording = reactive(False)
|
||||
|
||||
@override
|
||||
def render(self) -> str:
|
||||
status = (
|
||||
"🔴 Recording... (Press K to stop)"
|
||||
if self.is_recording
|
||||
else "⚪ Press K to start recording (Q to quit)"
|
||||
)
|
||||
return status
|
||||
|
||||
|
||||
class RealtimeApp(App[None]):
|
||||
CSS = """
|
||||
Screen {
|
||||
background: #1a1b26; /* Dark blue-grey background */
|
||||
}
|
||||
|
||||
Container {
|
||||
border: double rgb(91, 164, 91);
|
||||
}
|
||||
|
||||
Horizontal {
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
#input-container {
|
||||
height: 5; /* Explicit height for input container */
|
||||
margin: 1 1;
|
||||
padding: 1 2;
|
||||
}
|
||||
|
||||
Input {
|
||||
width: 80%;
|
||||
height: 3; /* Explicit height for input */
|
||||
}
|
||||
|
||||
Button {
|
||||
width: 20%;
|
||||
height: 3; /* Explicit height for button */
|
||||
}
|
||||
|
||||
#bottom-pane {
|
||||
width: 100%;
|
||||
height: 82%; /* Reduced to make room for session display */
|
||||
border: round rgb(205, 133, 63);
|
||||
content-align: center middle;
|
||||
}
|
||||
|
||||
#status-indicator {
|
||||
height: 3;
|
||||
content-align: center middle;
|
||||
background: #2a2b36;
|
||||
border: solid rgb(91, 164, 91);
|
||||
margin: 1 1;
|
||||
}
|
||||
|
||||
#session-display {
|
||||
height: 3;
|
||||
content-align: center middle;
|
||||
background: #2a2b36;
|
||||
border: solid rgb(91, 164, 91);
|
||||
margin: 1 1;
|
||||
}
|
||||
|
||||
Static {
|
||||
color: white;
|
||||
}
|
||||
"""
|
||||
|
||||
should_send_audio: asyncio.Event
|
||||
audio_player: sd.OutputStream
|
||||
last_audio_item_id: str | None
|
||||
connected: asyncio.Event
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.last_audio_item_id = None
|
||||
self.should_send_audio = asyncio.Event()
|
||||
self.connected = asyncio.Event()
|
||||
self.pipeline = VoicePipeline(
|
||||
workflow=MyWorkflow(secret_word="dog", on_start=self._on_transcription)
|
||||
)
|
||||
self._audio_input = StreamedAudioInput()
|
||||
self.audio_player = sd.OutputStream(
|
||||
samplerate=SAMPLE_RATE,
|
||||
channels=CHANNELS,
|
||||
dtype=FORMAT,
|
||||
)
|
||||
|
||||
def _on_transcription(self, transcription: str) -> None:
|
||||
try:
|
||||
self.query_one("#bottom-pane", RichLog).write(f"Transcription: {transcription}")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@override
|
||||
def compose(self) -> ComposeResult:
|
||||
"""Create child widgets for the app."""
|
||||
with Container():
|
||||
yield Header(id="session-display")
|
||||
yield AudioStatusIndicator(id="status-indicator")
|
||||
yield RichLog(id="bottom-pane", wrap=True, highlight=True, markup=True)
|
||||
|
||||
async def on_mount(self) -> None:
|
||||
self.run_worker(self.start_voice_pipeline())
|
||||
self.run_worker(self.send_mic_audio())
|
||||
|
||||
async def start_voice_pipeline(self) -> None:
|
||||
try:
|
||||
self.audio_player.start()
|
||||
self.result = await self.pipeline.run(self._audio_input)
|
||||
|
||||
async for event in self.result.stream():
|
||||
bottom_pane = self.query_one("#bottom-pane", RichLog)
|
||||
if event.type == "voice_stream_event_audio":
|
||||
self.audio_player.write(event.data)
|
||||
bottom_pane.write(
|
||||
f"Received audio: {len(event.data) if event.data is not None else '0'} bytes"
|
||||
)
|
||||
elif event.type == "voice_stream_event_lifecycle":
|
||||
bottom_pane.write(f"Lifecycle event: {event.event}")
|
||||
except Exception as e:
|
||||
bottom_pane = self.query_one("#bottom-pane", RichLog)
|
||||
bottom_pane.write(f"Error: {e}")
|
||||
finally:
|
||||
self.audio_player.close()
|
||||
|
||||
async def send_mic_audio(self) -> None:
|
||||
device_info = sd.query_devices()
|
||||
print(device_info)
|
||||
|
||||
read_size = int(SAMPLE_RATE * 0.02)
|
||||
|
||||
stream = sd.InputStream(
|
||||
channels=CHANNELS,
|
||||
samplerate=SAMPLE_RATE,
|
||||
dtype="int16",
|
||||
)
|
||||
stream.start()
|
||||
|
||||
status_indicator = self.query_one(AudioStatusIndicator)
|
||||
|
||||
try:
|
||||
while True:
|
||||
if stream.read_available < read_size:
|
||||
await asyncio.sleep(0)
|
||||
continue
|
||||
|
||||
await self.should_send_audio.wait()
|
||||
status_indicator.is_recording = True
|
||||
|
||||
data, _ = stream.read(read_size)
|
||||
|
||||
await self._audio_input.add_audio(data)
|
||||
await asyncio.sleep(0)
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
finally:
|
||||
stream.stop()
|
||||
stream.close()
|
||||
|
||||
async def on_key(self, event: events.Key) -> None:
|
||||
"""Handle key press events."""
|
||||
if event.key == "enter":
|
||||
self.query_one(Button).press()
|
||||
return
|
||||
|
||||
if event.key == "q":
|
||||
self.exit()
|
||||
return
|
||||
|
||||
if event.key == "k":
|
||||
status_indicator = self.query_one(AudioStatusIndicator)
|
||||
if status_indicator.is_recording:
|
||||
self.should_send_audio.clear()
|
||||
status_indicator.is_recording = False
|
||||
else:
|
||||
self.should_send_audio.set()
|
||||
status_indicator.is_recording = True
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
app = RealtimeApp()
|
||||
app.run()
|
||||
|
|
@ -0,0 +1,81 @@
|
|||
import random
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Callable
|
||||
|
||||
from agents import Agent, Runner, TResponseInputItem, function_tool
|
||||
from agents.extensions.handoff_prompt import prompt_with_handoff_instructions
|
||||
from agents.voice import VoiceWorkflowBase, VoiceWorkflowHelper
|
||||
|
||||
|
||||
@function_tool
|
||||
def get_weather(city: str) -> str:
|
||||
"""Get the weather for a given city."""
|
||||
print(f"[debug] get_weather called with city: {city}")
|
||||
choices = ["sunny", "cloudy", "rainy", "snowy"]
|
||||
return f"The weather in {city} is {random.choice(choices)}."
|
||||
|
||||
|
||||
spanish_agent = Agent(
|
||||
name="Spanish",
|
||||
handoff_description="A spanish speaking agent.",
|
||||
instructions=prompt_with_handoff_instructions(
|
||||
"You're speaking to a human, so be polite and concise. Speak in Spanish.",
|
||||
),
|
||||
model="gpt-4o-mini",
|
||||
)
|
||||
|
||||
agent = Agent(
|
||||
name="Assistant",
|
||||
instructions=prompt_with_handoff_instructions(
|
||||
"You're speaking to a human, so be polite and concise. If the user speaks in Spanish, handoff to the spanish agent.",
|
||||
),
|
||||
model="gpt-4o-mini",
|
||||
handoffs=[spanish_agent],
|
||||
tools=[get_weather],
|
||||
)
|
||||
|
||||
|
||||
class MyWorkflow(VoiceWorkflowBase):
|
||||
def __init__(self, secret_word: str, on_start: Callable[[str], None]):
|
||||
"""
|
||||
Args:
|
||||
secret_word: The secret word to guess.
|
||||
on_start: A callback that is called when the workflow starts. The transcription
|
||||
is passed in as an argument.
|
||||
"""
|
||||
self._input_history: list[TResponseInputItem] = []
|
||||
self._current_agent = agent
|
||||
self._secret_word = secret_word.lower()
|
||||
self._on_start = on_start
|
||||
|
||||
async def run(self, transcription: str) -> AsyncIterator[str]:
|
||||
self._on_start(transcription)
|
||||
|
||||
# Add the transcription to the input history
|
||||
self._input_history.append(
|
||||
{
|
||||
"role": "user",
|
||||
"content": transcription,
|
||||
}
|
||||
)
|
||||
|
||||
# If the user guessed the secret word, do alternate logic
|
||||
if self._secret_word in transcription.lower():
|
||||
yield "You guessed the secret word!"
|
||||
self._input_history.append(
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "You guessed the secret word!",
|
||||
}
|
||||
)
|
||||
return
|
||||
|
||||
# Otherwise, run the agent
|
||||
result = Runner.run_streamed(self._current_agent, self._input_history)
|
||||
|
||||
async for chunk in VoiceWorkflowHelper.stream_text_from(result):
|
||||
yield chunk
|
||||
|
||||
# Update the input history and current agent
|
||||
self._input_history = result.to_input_list()
|
||||
self._current_agent = result.last_agent
|
||||
234
mkdocs.yml
234
mkdocs.yml
|
|
@ -1,121 +1,143 @@
|
|||
site_name: OpenAI Agents SDK
|
||||
theme:
|
||||
name: material
|
||||
features:
|
||||
# Allows copying code blocks
|
||||
- content.code.copy
|
||||
# Allows selecting code blocks
|
||||
- content.code.select
|
||||
# Shows the current path in the sidebar
|
||||
- navigation.path
|
||||
# Shows sections in the sidebar
|
||||
- navigation.sections
|
||||
# Shows sections expanded by default
|
||||
- navigation.expand
|
||||
# Enables annotations in code blocks
|
||||
- content.code.annotate
|
||||
palette:
|
||||
primary: black
|
||||
logo: assets/logo.svg
|
||||
favicon: images/favicon-platform.svg
|
||||
name: material
|
||||
features:
|
||||
# Allows copying code blocks
|
||||
- content.code.copy
|
||||
# Allows selecting code blocks
|
||||
- content.code.select
|
||||
# Shows the current path in the sidebar
|
||||
- navigation.path
|
||||
# Shows sections in the sidebar
|
||||
- navigation.sections
|
||||
# Shows sections expanded by default
|
||||
- navigation.expand
|
||||
# Enables annotations in code blocks
|
||||
- content.code.annotate
|
||||
palette:
|
||||
primary: black
|
||||
logo: assets/logo.svg
|
||||
favicon: images/favicon-platform.svg
|
||||
nav:
|
||||
- Intro: index.md
|
||||
- Quickstart: quickstart.md
|
||||
- Documentation:
|
||||
- agents.md
|
||||
- running_agents.md
|
||||
- results.md
|
||||
- streaming.md
|
||||
- tools.md
|
||||
- handoffs.md
|
||||
- tracing.md
|
||||
- context.md
|
||||
- guardrails.md
|
||||
- multi_agent.md
|
||||
- models.md
|
||||
- config.md
|
||||
- API Reference:
|
||||
- Agents:
|
||||
- ref/index.md
|
||||
- ref/agent.md
|
||||
- ref/run.md
|
||||
- ref/tool.md
|
||||
- ref/result.md
|
||||
- ref/stream_events.md
|
||||
- ref/handoffs.md
|
||||
- ref/lifecycle.md
|
||||
- ref/items.md
|
||||
- ref/run_context.md
|
||||
- ref/usage.md
|
||||
- ref/exceptions.md
|
||||
- ref/guardrail.md
|
||||
- ref/model_settings.md
|
||||
- ref/agent_output.md
|
||||
- ref/function_schema.md
|
||||
- ref/models/interface.md
|
||||
- ref/models/openai_chatcompletions.md
|
||||
- ref/models/openai_responses.md
|
||||
- Tracing:
|
||||
- ref/tracing/index.md
|
||||
- ref/tracing/create.md
|
||||
- ref/tracing/traces.md
|
||||
- ref/tracing/spans.md
|
||||
- ref/tracing/processor_interface.md
|
||||
- ref/tracing/processors.md
|
||||
- ref/tracing/scope.md
|
||||
- ref/tracing/setup.md
|
||||
- ref/tracing/span_data.md
|
||||
- ref/tracing/util.md
|
||||
- Extensions:
|
||||
- ref/extensions/handoff_filters.md
|
||||
- ref/extensions/handoff_prompt.md
|
||||
- Intro: index.md
|
||||
- Quickstart: quickstart.md
|
||||
- Examples: examples.md
|
||||
- Documentation:
|
||||
- agents.md
|
||||
- running_agents.md
|
||||
- results.md
|
||||
- streaming.md
|
||||
- tools.md
|
||||
- handoffs.md
|
||||
- tracing.md
|
||||
- context.md
|
||||
- guardrails.md
|
||||
- multi_agent.md
|
||||
- models.md
|
||||
- config.md
|
||||
- Voice agents:
|
||||
- voice/quickstart.md
|
||||
- voice/pipeline.md
|
||||
- voice/tracing.md
|
||||
- API Reference:
|
||||
- Agents:
|
||||
- ref/index.md
|
||||
- ref/agent.md
|
||||
- ref/run.md
|
||||
- ref/tool.md
|
||||
- ref/result.md
|
||||
- ref/stream_events.md
|
||||
- ref/handoffs.md
|
||||
- ref/lifecycle.md
|
||||
- ref/items.md
|
||||
- ref/run_context.md
|
||||
- ref/usage.md
|
||||
- ref/exceptions.md
|
||||
- ref/guardrail.md
|
||||
- ref/model_settings.md
|
||||
- ref/agent_output.md
|
||||
- ref/function_schema.md
|
||||
- ref/models/interface.md
|
||||
- ref/models/openai_chatcompletions.md
|
||||
- ref/models/openai_responses.md
|
||||
- Tracing:
|
||||
- ref/tracing/index.md
|
||||
- ref/tracing/create.md
|
||||
- ref/tracing/traces.md
|
||||
- ref/tracing/spans.md
|
||||
- ref/tracing/processor_interface.md
|
||||
- ref/tracing/processors.md
|
||||
- ref/tracing/scope.md
|
||||
- ref/tracing/setup.md
|
||||
- ref/tracing/span_data.md
|
||||
- ref/tracing/util.md
|
||||
- Voice:
|
||||
- ref/voice/pipeline.md
|
||||
- ref/voice/workflow.md
|
||||
- ref/voice/input.md
|
||||
- ref/voice/result.md
|
||||
- ref/voice/pipeline_config.md
|
||||
- ref/voice/events.md
|
||||
- ref/voice/exceptions.md
|
||||
- ref/voice/model.md
|
||||
- ref/voice/utils.md
|
||||
- ref/voice/models/openai_provider.md
|
||||
- ref/voice/models/openai_stt.md
|
||||
- ref/voice/models/openai_tts.md
|
||||
- Extensions:
|
||||
- ref/extensions/handoff_filters.md
|
||||
- ref/extensions/handoff_prompt.md
|
||||
|
||||
plugins:
|
||||
- search
|
||||
- mkdocstrings:
|
||||
handlers:
|
||||
python:
|
||||
paths: ["src/agents"]
|
||||
selection:
|
||||
docstring_style: google
|
||||
options:
|
||||
# Shows links to other members in signatures
|
||||
signature_crossrefs: true
|
||||
# Orders members by source order, rather than alphabetical
|
||||
members_order: source
|
||||
# Puts the signature on a separate line from the member name
|
||||
separate_signature: true
|
||||
# Shows type annotations in signatures
|
||||
show_signature_annotations: true
|
||||
# Makes the font sizes nicer
|
||||
heading_level: 3
|
||||
- search
|
||||
- mkdocstrings:
|
||||
handlers:
|
||||
python:
|
||||
paths: ["src/agents"]
|
||||
selection:
|
||||
docstring_style: google
|
||||
options:
|
||||
# Shows links to other members in signatures
|
||||
signature_crossrefs: true
|
||||
# Orders members by source order, rather than alphabetical
|
||||
members_order: source
|
||||
# Puts the signature on a separate line from the member name
|
||||
separate_signature: true
|
||||
# Shows type annotations in signatures
|
||||
show_signature_annotations: true
|
||||
# Makes the font sizes nicer
|
||||
heading_level: 3
|
||||
|
||||
extra:
|
||||
# Remove material generation message in footer
|
||||
generator: false
|
||||
# Remove material generation message in footer
|
||||
generator: false
|
||||
|
||||
markdown_extensions:
|
||||
- admonition
|
||||
- pymdownx.details
|
||||
- pymdownx.superfences
|
||||
- attr_list
|
||||
- md_in_html
|
||||
- pymdownx.highlight:
|
||||
anchor_linenums: true
|
||||
line_spans: __span
|
||||
pygments_lang_class: true
|
||||
- pymdownx.inlinehilite
|
||||
- pymdownx.snippets
|
||||
- pymdownx.superfences
|
||||
- pymdownx.superfences:
|
||||
custom_fences:
|
||||
- name: mermaid
|
||||
class: mermaid
|
||||
format: !!python/name:pymdownx.superfences.fence_code_format
|
||||
- admonition
|
||||
- pymdownx.details
|
||||
- attr_list
|
||||
- md_in_html
|
||||
- pymdownx.highlight:
|
||||
anchor_linenums: true
|
||||
line_spans: __span
|
||||
pygments_lang_class: true
|
||||
- pymdownx.inlinehilite
|
||||
- pymdownx.snippets
|
||||
- pymdownx.superfences
|
||||
|
||||
validation:
|
||||
omitted_files: warn
|
||||
absolute_links: warn
|
||||
unrecognized_links: warn
|
||||
anchors: warn
|
||||
omitted_files: warn
|
||||
absolute_links: warn
|
||||
unrecognized_links: warn
|
||||
anchors: warn
|
||||
|
||||
extra_css:
|
||||
- stylesheets/extra.css
|
||||
- stylesheets/extra.css
|
||||
|
||||
watch:
|
||||
- "src/agents"
|
||||
- "src/agents"
|
||||
|
|
|
|||
|
|
@ -1,15 +1,13 @@
|
|||
[project]
|
||||
name = "openai-agents"
|
||||
version = "0.0.4"
|
||||
version = "0.0.6"
|
||||
description = "OpenAI Agents SDK"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.9"
|
||||
license = "MIT"
|
||||
authors = [
|
||||
{ name = "OpenAI", email = "support@openai.com" },
|
||||
]
|
||||
authors = [{ name = "OpenAI", email = "support@openai.com" }]
|
||||
dependencies = [
|
||||
"openai>=1.66.2",
|
||||
"openai>=1.66.5",
|
||||
"pydantic>=2.10, <3",
|
||||
"griffe>=1.5.6, <2",
|
||||
"typing-extensions>=4.12.2, <5",
|
||||
|
|
@ -27,13 +25,16 @@ classifiers = [
|
|||
"Intended Audience :: Developers",
|
||||
"Operating System :: OS Independent",
|
||||
"Topic :: Software Development :: Libraries :: Python Modules",
|
||||
"License :: OSI Approved :: MIT License"
|
||||
"License :: OSI Approved :: MIT License",
|
||||
]
|
||||
|
||||
[project.urls]
|
||||
Homepage = "https://github.com/openai/openai-agents-python"
|
||||
Repository = "https://github.com/openai/openai-agents-python"
|
||||
|
||||
[project.optional-dependencies]
|
||||
voice = ["numpy>=2.2.0, <3; python_version>='3.10'", "websockets>=15.0, <16"]
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"mypy",
|
||||
|
|
@ -47,6 +48,13 @@ dev = [
|
|||
"mkdocstrings[python]>=0.28.0",
|
||||
"coverage>=7.6.12",
|
||||
"playwright==1.50.0",
|
||||
"inline-snapshot>=0.20.7",
|
||||
"pynput",
|
||||
"types-pynput",
|
||||
"sounddevice",
|
||||
"pynput",
|
||||
"textual",
|
||||
"websockets",
|
||||
]
|
||||
[tool.uv.workspace]
|
||||
members = ["agents"]
|
||||
|
|
@ -73,8 +81,8 @@ select = [
|
|||
"F", # pyflakes
|
||||
"I", # isort
|
||||
"B", # flake8-bugbear
|
||||
"C4", # flake8-comprehensions
|
||||
"UP", # pyupgrade
|
||||
"C4", # flake8-comprehensions
|
||||
"UP", # pyupgrade
|
||||
]
|
||||
isort = { combine-as-imports = true, known-first-party = ["agents"] }
|
||||
|
||||
|
|
@ -90,11 +98,12 @@ disallow_incomplete_defs = false
|
|||
disallow_untyped_defs = false
|
||||
disallow_untyped_calls = false
|
||||
|
||||
[[tool.mypy.overrides]]
|
||||
module = "sounddevice.*"
|
||||
ignore_missing_imports = true
|
||||
|
||||
[tool.coverage.run]
|
||||
source = [
|
||||
"tests",
|
||||
"src/agents",
|
||||
]
|
||||
source = ["tests", "src/agents"]
|
||||
|
||||
[tool.coverage.report]
|
||||
show_missing = true
|
||||
|
|
@ -108,7 +117,7 @@ exclude_also = [
|
|||
]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
asyncio_mode = "auto"
|
||||
asyncio_mode = "auto"
|
||||
asyncio_default_fixture_loop_scope = "session"
|
||||
filterwarnings = [
|
||||
# This is a warning that is expected to happen: we have an async filter that raises an exception
|
||||
|
|
@ -116,4 +125,7 @@ filterwarnings = [
|
|||
]
|
||||
markers = [
|
||||
"allow_call_model_methods: mark test as allowing calls to real model implementations",
|
||||
]
|
||||
]
|
||||
|
||||
[tool.inline-snapshot]
|
||||
format-command = "ruff format --stdin-filename {filename}"
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ from typing import Literal
|
|||
from openai import AsyncOpenAI
|
||||
|
||||
from . import _config
|
||||
from .agent import Agent
|
||||
from .agent import Agent, ToolsToFinalOutputFunction, ToolsToFinalOutputResult
|
||||
from .agent_output import AgentOutputSchema
|
||||
from .computer import AsyncComputer, Button, Computer, Environment
|
||||
from .exceptions import (
|
||||
|
|
@ -57,6 +57,7 @@ from .tool import (
|
|||
ComputerTool,
|
||||
FileSearchTool,
|
||||
FunctionTool,
|
||||
FunctionToolResult,
|
||||
Tool,
|
||||
WebSearchTool,
|
||||
default_tool_error_function,
|
||||
|
|
@ -72,7 +73,11 @@ from .tracing import (
|
|||
Span,
|
||||
SpanData,
|
||||
SpanError,
|
||||
SpeechGroupSpanData,
|
||||
SpeechSpanData,
|
||||
Trace,
|
||||
TracingProcessor,
|
||||
TranscriptionSpanData,
|
||||
add_trace_processor,
|
||||
agent_span,
|
||||
custom_span,
|
||||
|
|
@ -87,7 +92,10 @@ from .tracing import (
|
|||
set_trace_processors,
|
||||
set_tracing_disabled,
|
||||
set_tracing_export_api_key,
|
||||
speech_group_span,
|
||||
speech_span,
|
||||
trace,
|
||||
transcription_span,
|
||||
)
|
||||
from .usage import Usage
|
||||
|
||||
|
|
@ -136,6 +144,8 @@ def enable_verbose_stdout_logging():
|
|||
|
||||
__all__ = [
|
||||
"Agent",
|
||||
"ToolsToFinalOutputFunction",
|
||||
"ToolsToFinalOutputResult",
|
||||
"Runner",
|
||||
"Model",
|
||||
"ModelProvider",
|
||||
|
|
@ -189,6 +199,7 @@ __all__ = [
|
|||
"AgentUpdatedStreamEvent",
|
||||
"StreamEvent",
|
||||
"FunctionTool",
|
||||
"FunctionToolResult",
|
||||
"ComputerTool",
|
||||
"FileSearchTool",
|
||||
"Tool",
|
||||
|
|
@ -206,8 +217,12 @@ __all__ = [
|
|||
"handoff_span",
|
||||
"set_trace_processors",
|
||||
"set_tracing_disabled",
|
||||
"speech_group_span",
|
||||
"transcription_span",
|
||||
"speech_span",
|
||||
"trace",
|
||||
"Trace",
|
||||
"TracingProcessor",
|
||||
"SpanError",
|
||||
"Span",
|
||||
"SpanData",
|
||||
|
|
@ -217,6 +232,9 @@ __all__ = [
|
|||
"GenerationSpanData",
|
||||
"GuardrailSpanData",
|
||||
"HandoffSpanData",
|
||||
"SpeechGroupSpanData",
|
||||
"SpeechSpanData",
|
||||
"TranscriptionSpanData",
|
||||
"set_default_openai_key",
|
||||
"set_default_openai_client",
|
||||
"set_default_openai_api",
|
||||
|
|
|
|||
|
|
@ -1,8 +1,10 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import inspect
|
||||
from collections.abc import Awaitable
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
from openai.types.responses import (
|
||||
ResponseComputerToolCall,
|
||||
|
|
@ -25,8 +27,7 @@ from openai.types.responses.response_computer_tool_call import (
|
|||
from openai.types.responses.response_input_param import ComputerCallOutput
|
||||
from openai.types.responses.response_reasoning_item import ResponseReasoningItem
|
||||
|
||||
from . import _utils
|
||||
from .agent import Agent
|
||||
from .agent import Agent, ToolsToFinalOutputResult
|
||||
from .agent_output import AgentOutputSchema
|
||||
from .computer import AsyncComputer, Computer
|
||||
from .exceptions import AgentsException, ModelBehaviorError, UserError
|
||||
|
|
@ -49,7 +50,7 @@ from .logger import logger
|
|||
from .models.interface import ModelTracing
|
||||
from .run_context import RunContextWrapper, TContext
|
||||
from .stream_events import RunItemStreamEvent, StreamEvent
|
||||
from .tool import ComputerTool, FunctionTool
|
||||
from .tool import ComputerTool, FunctionTool, FunctionToolResult
|
||||
from .tracing import (
|
||||
SpanError,
|
||||
Trace,
|
||||
|
|
@ -59,6 +60,7 @@ from .tracing import (
|
|||
handoff_span,
|
||||
trace,
|
||||
)
|
||||
from .util import _coro, _error_tracing
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .run import RunConfig
|
||||
|
|
@ -70,6 +72,8 @@ class QueueCompleteSentinel:
|
|||
|
||||
QUEUE_COMPLETE_SENTINEL = QueueCompleteSentinel()
|
||||
|
||||
_NOT_FINAL_OUTPUT = ToolsToFinalOutputResult(is_final_output=False, final_output=None)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolRunHandoff:
|
||||
|
|
@ -199,7 +203,7 @@ class RunImpl:
|
|||
config=run_config,
|
||||
),
|
||||
)
|
||||
new_step_items.extend(function_results)
|
||||
new_step_items.extend([result.run_item for result in function_results])
|
||||
new_step_items.extend(computer_results)
|
||||
|
||||
# Second, check if there are any handoffs
|
||||
|
|
@ -216,6 +220,36 @@ class RunImpl:
|
|||
run_config=run_config,
|
||||
)
|
||||
|
||||
# Third, we'll check if the tool use should result in a final output
|
||||
check_tool_use = await cls._check_for_final_output_from_tools(
|
||||
agent=agent,
|
||||
tool_results=function_results,
|
||||
context_wrapper=context_wrapper,
|
||||
config=run_config,
|
||||
)
|
||||
|
||||
if check_tool_use.is_final_output:
|
||||
# If the output type is str, then let's just stringify it
|
||||
if not agent.output_type or agent.output_type is str:
|
||||
check_tool_use.final_output = str(check_tool_use.final_output)
|
||||
|
||||
if check_tool_use.final_output is None:
|
||||
logger.error(
|
||||
"Model returned a final output of None. Not raising an error because we assume"
|
||||
"you know what you're doing."
|
||||
)
|
||||
|
||||
return await cls.execute_final_output(
|
||||
agent=agent,
|
||||
original_input=original_input,
|
||||
new_response=new_response,
|
||||
pre_step_items=pre_step_items,
|
||||
new_step_items=new_step_items,
|
||||
final_output=check_tool_use.final_output,
|
||||
hooks=hooks,
|
||||
context_wrapper=context_wrapper,
|
||||
)
|
||||
|
||||
# Now we can check if the model also produced a final output
|
||||
message_items = [item for item in new_step_items if isinstance(item, MessageOutputItem)]
|
||||
|
||||
|
|
@ -293,7 +327,7 @@ class RunImpl:
|
|||
elif isinstance(output, ResponseComputerToolCall):
|
||||
items.append(ToolCallItem(raw_item=output, agent=agent))
|
||||
if not computer_tool:
|
||||
_utils.attach_error_to_current_span(
|
||||
_error_tracing.attach_error_to_current_span(
|
||||
SpanError(
|
||||
message="Computer tool not found",
|
||||
data={},
|
||||
|
|
@ -324,7 +358,7 @@ class RunImpl:
|
|||
# Regular function tool call
|
||||
else:
|
||||
if output.name not in function_map:
|
||||
_utils.attach_error_to_current_span(
|
||||
_error_tracing.attach_error_to_current_span(
|
||||
SpanError(
|
||||
message="Tool not found",
|
||||
data={"tool_name": output.name},
|
||||
|
|
@ -355,10 +389,10 @@ class RunImpl:
|
|||
hooks: RunHooks[TContext],
|
||||
context_wrapper: RunContextWrapper[TContext],
|
||||
config: RunConfig,
|
||||
) -> list[RunItem]:
|
||||
) -> list[FunctionToolResult]:
|
||||
async def run_single_tool(
|
||||
func_tool: FunctionTool, tool_call: ResponseFunctionToolCall
|
||||
) -> str:
|
||||
) -> Any:
|
||||
with function_span(func_tool.name) as span_fn:
|
||||
if config.trace_include_sensitive_data:
|
||||
span_fn.span_data.input = tool_call.arguments
|
||||
|
|
@ -368,7 +402,7 @@ class RunImpl:
|
|||
(
|
||||
agent.hooks.on_tool_start(context_wrapper, agent, func_tool)
|
||||
if agent.hooks
|
||||
else _utils.noop_coroutine()
|
||||
else _coro.noop_coroutine()
|
||||
),
|
||||
func_tool.on_invoke_tool(context_wrapper, tool_call.arguments),
|
||||
)
|
||||
|
|
@ -378,11 +412,11 @@ class RunImpl:
|
|||
(
|
||||
agent.hooks.on_tool_end(context_wrapper, agent, func_tool, result)
|
||||
if agent.hooks
|
||||
else _utils.noop_coroutine()
|
||||
else _coro.noop_coroutine()
|
||||
),
|
||||
)
|
||||
except Exception as e:
|
||||
_utils.attach_error_to_current_span(
|
||||
_error_tracing.attach_error_to_current_span(
|
||||
SpanError(
|
||||
message="Error running tool",
|
||||
data={"tool_name": func_tool.name, "error": str(e)},
|
||||
|
|
@ -404,10 +438,14 @@ class RunImpl:
|
|||
results = await asyncio.gather(*tasks)
|
||||
|
||||
return [
|
||||
ToolCallOutputItem(
|
||||
output=str(result),
|
||||
raw_item=ItemHelpers.tool_call_output_item(tool_run.tool_call, str(result)),
|
||||
agent=agent,
|
||||
FunctionToolResult(
|
||||
tool=tool_run.function_tool,
|
||||
output=result,
|
||||
run_item=ToolCallOutputItem(
|
||||
output=result,
|
||||
raw_item=ItemHelpers.tool_call_output_item(tool_run.tool_call, str(result)),
|
||||
agent=agent,
|
||||
),
|
||||
)
|
||||
for tool_run, result in zip(tool_runs, results)
|
||||
]
|
||||
|
|
@ -502,7 +540,7 @@ class RunImpl:
|
|||
source=agent,
|
||||
)
|
||||
if agent.hooks
|
||||
else _utils.noop_coroutine()
|
||||
else _coro.noop_coroutine()
|
||||
),
|
||||
)
|
||||
|
||||
|
|
@ -520,7 +558,7 @@ class RunImpl:
|
|||
new_items=tuple(new_step_items),
|
||||
)
|
||||
if not callable(input_filter):
|
||||
_utils.attach_error_to_span(
|
||||
_error_tracing.attach_error_to_span(
|
||||
span_handoff,
|
||||
SpanError(
|
||||
message="Invalid input filter",
|
||||
|
|
@ -530,7 +568,7 @@ class RunImpl:
|
|||
raise UserError(f"Invalid input filter: {input_filter}")
|
||||
filtered = input_filter(handoff_input_data)
|
||||
if not isinstance(filtered, HandoffInputData):
|
||||
_utils.attach_error_to_span(
|
||||
_error_tracing.attach_error_to_span(
|
||||
span_handoff,
|
||||
SpanError(
|
||||
message="Invalid input filter result",
|
||||
|
|
@ -591,7 +629,7 @@ class RunImpl:
|
|||
hooks.on_agent_end(context_wrapper, agent, final_output),
|
||||
agent.hooks.on_end(context_wrapper, agent, final_output)
|
||||
if agent.hooks
|
||||
else _utils.noop_coroutine(),
|
||||
else _coro.noop_coroutine(),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
|
|
@ -646,6 +684,47 @@ class RunImpl:
|
|||
if event:
|
||||
queue.put_nowait(event)
|
||||
|
||||
@classmethod
|
||||
async def _check_for_final_output_from_tools(
|
||||
cls,
|
||||
*,
|
||||
agent: Agent[TContext],
|
||||
tool_results: list[FunctionToolResult],
|
||||
context_wrapper: RunContextWrapper[TContext],
|
||||
config: RunConfig,
|
||||
) -> ToolsToFinalOutputResult:
|
||||
"""Returns (i, final_output)."""
|
||||
if not tool_results:
|
||||
return _NOT_FINAL_OUTPUT
|
||||
|
||||
if agent.tool_use_behavior == "run_llm_again":
|
||||
return _NOT_FINAL_OUTPUT
|
||||
elif agent.tool_use_behavior == "stop_on_first_tool":
|
||||
return ToolsToFinalOutputResult(
|
||||
is_final_output=True, final_output=tool_results[0].output
|
||||
)
|
||||
elif isinstance(agent.tool_use_behavior, dict):
|
||||
names = agent.tool_use_behavior.get("stop_at_tool_names", [])
|
||||
for tool_result in tool_results:
|
||||
if tool_result.tool.name in names:
|
||||
return ToolsToFinalOutputResult(
|
||||
is_final_output=True, final_output=tool_result.output
|
||||
)
|
||||
return ToolsToFinalOutputResult(is_final_output=False, final_output=None)
|
||||
elif callable(agent.tool_use_behavior):
|
||||
if inspect.iscoroutinefunction(agent.tool_use_behavior):
|
||||
return await cast(
|
||||
Awaitable[ToolsToFinalOutputResult],
|
||||
agent.tool_use_behavior(context_wrapper, tool_results),
|
||||
)
|
||||
else:
|
||||
return cast(
|
||||
ToolsToFinalOutputResult, agent.tool_use_behavior(context_wrapper, tool_results)
|
||||
)
|
||||
|
||||
logger.error(f"Invalid tool_use_behavior: {agent.tool_use_behavior}")
|
||||
raise UserError(f"Invalid tool_use_behavior: {agent.tool_use_behavior}")
|
||||
|
||||
|
||||
class TraceCtxManager:
|
||||
"""Creates a trace only if there is no current trace, and manages the trace lifecycle."""
|
||||
|
|
@ -706,7 +785,7 @@ class ComputerAction:
|
|||
(
|
||||
agent.hooks.on_tool_start(context_wrapper, agent, action.computer_tool)
|
||||
if agent.hooks
|
||||
else _utils.noop_coroutine()
|
||||
else _coro.noop_coroutine()
|
||||
),
|
||||
output_func,
|
||||
)
|
||||
|
|
@ -716,7 +795,7 @@ class ComputerAction:
|
|||
(
|
||||
agent.hooks.on_tool_end(context_wrapper, agent, action.computer_tool, output)
|
||||
if agent.hooks
|
||||
else _utils.noop_coroutine()
|
||||
else _coro.noop_coroutine()
|
||||
),
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,61 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Awaitable
|
||||
from typing import Any, Literal, Union
|
||||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
from typing_extensions import TypeVar
|
||||
|
||||
from .exceptions import ModelBehaviorError
|
||||
from .logger import logger
|
||||
from .tracing import Span, SpanError, get_current_span
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
MaybeAwaitable = Union[Awaitable[T], T]
|
||||
|
||||
|
||||
def transform_string_function_style(name: str) -> str:
|
||||
# Replace spaces with underscores
|
||||
name = name.replace(" ", "_")
|
||||
|
||||
# Replace non-alphanumeric characters with underscores
|
||||
name = re.sub(r"[^a-zA-Z0-9]", "_", name)
|
||||
|
||||
return name.lower()
|
||||
|
||||
|
||||
def validate_json(json_str: str, type_adapter: TypeAdapter[T], partial: bool) -> T:
|
||||
partial_setting: bool | Literal["off", "on", "trailing-strings"] = (
|
||||
"trailing-strings" if partial else False
|
||||
)
|
||||
try:
|
||||
validated = type_adapter.validate_json(json_str, experimental_allow_partial=partial_setting)
|
||||
return validated
|
||||
except ValidationError as e:
|
||||
attach_error_to_current_span(
|
||||
SpanError(
|
||||
message="Invalid JSON provided",
|
||||
data={},
|
||||
)
|
||||
)
|
||||
raise ModelBehaviorError(
|
||||
f"Invalid JSON when parsing {json_str} for {type_adapter}; {e}"
|
||||
) from e
|
||||
|
||||
|
||||
def attach_error_to_span(span: Span[Any], error: SpanError) -> None:
|
||||
span.set_error(error)
|
||||
|
||||
|
||||
def attach_error_to_current_span(error: SpanError) -> None:
|
||||
span = get_current_span()
|
||||
if span:
|
||||
attach_error_to_span(span, error)
|
||||
else:
|
||||
logger.warning(f"No span to add error {error} to")
|
||||
|
||||
|
||||
async def noop_coroutine() -> None:
|
||||
pass
|
||||
|
|
@ -4,10 +4,10 @@ import dataclasses
|
|||
import inspect
|
||||
from collections.abc import Awaitable
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, Any, Callable, Generic, cast
|
||||
from typing import TYPE_CHECKING, Any, Callable, Generic, Literal, cast
|
||||
|
||||
from typing_extensions import TypeAlias, TypedDict
|
||||
|
||||
from . import _utils
|
||||
from ._utils import MaybeAwaitable
|
||||
from .guardrail import InputGuardrail, OutputGuardrail
|
||||
from .handoffs import Handoff
|
||||
from .items import ItemHelpers
|
||||
|
|
@ -15,20 +15,49 @@ from .logger import logger
|
|||
from .model_settings import ModelSettings
|
||||
from .models.interface import Model
|
||||
from .run_context import RunContextWrapper, TContext
|
||||
from .tool import Tool, function_tool
|
||||
from .tool import FunctionToolResult, Tool, function_tool
|
||||
from .util import _transforms
|
||||
from .util._types import MaybeAwaitable
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .lifecycle import AgentHooks
|
||||
from .result import RunResult
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolsToFinalOutputResult:
|
||||
is_final_output: bool
|
||||
"""Whether this is the final output. If False, the LLM will run again and receive the tool call
|
||||
output.
|
||||
"""
|
||||
|
||||
final_output: Any | None = None
|
||||
"""The final output. Can be None if `is_final_output` is False, otherwise must match the
|
||||
`output_type` of the agent.
|
||||
"""
|
||||
|
||||
|
||||
ToolsToFinalOutputFunction: TypeAlias = Callable[
|
||||
[RunContextWrapper[TContext], list[FunctionToolResult]],
|
||||
MaybeAwaitable[ToolsToFinalOutputResult],
|
||||
]
|
||||
"""A function that takes a run context and a list of tool results, and returns a
|
||||
`ToolToFinalOutputResult`.
|
||||
"""
|
||||
|
||||
|
||||
class StopAtTools(TypedDict):
|
||||
stop_at_tool_names: list[str]
|
||||
"""A list of tool names, any of which will stop the agent from running further."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class Agent(Generic[TContext]):
|
||||
"""An agent is an AI model configured with instructions, tools, guardrails, handoffs and more.
|
||||
|
||||
We strongly recommend passing `instructions`, which is the "system prompt" for the agent. In
|
||||
addition, you can pass `description`, which is a human-readable description of the agent, used
|
||||
when the agent is used inside tools/handoffs.
|
||||
addition, you can pass `handoff_description`, which is a human-readable description of the
|
||||
agent, used when the agent is used inside tools/handoffs.
|
||||
|
||||
Agents are generic on the context type. The context is a (mutable) object you create. It is
|
||||
passed to tool functions, handoffs, guardrails, etc.
|
||||
|
|
@ -95,6 +124,25 @@ class Agent(Generic[TContext]):
|
|||
"""A class that receives callbacks on various lifecycle events for this agent.
|
||||
"""
|
||||
|
||||
tool_use_behavior: (
|
||||
Literal["run_llm_again", "stop_on_first_tool"] | StopAtTools | ToolsToFinalOutputFunction
|
||||
) = "run_llm_again"
|
||||
"""This lets you configure how tool use is handled.
|
||||
- "run_llm_again": The default behavior. Tools are run, and then the LLM receives the results
|
||||
and gets to respond.
|
||||
- "stop_on_first_tool": The output of the first tool call is used as the final output. This
|
||||
means that the LLM does not process the result of the tool call.
|
||||
- A list of tool names: The agent will stop running if any of the tools in the list are called.
|
||||
The final output will be the output of the first matching tool call. The LLM does not
|
||||
process the result of the tool call.
|
||||
- A function: If you pass a function, it will be called with the run context and the list of
|
||||
tool results. It must return a `ToolToFinalOutputResult`, which determines whether the tool
|
||||
calls result in a final output.
|
||||
|
||||
NOTE: This configuration is specific to FunctionTools. Hosted tools, such as file search,
|
||||
web search, etc are always processed by the LLM.
|
||||
"""
|
||||
|
||||
def clone(self, **kwargs: Any) -> Agent[TContext]:
|
||||
"""Make a copy of the agent, with the given arguments changed. For example, you could do:
|
||||
```
|
||||
|
|
@ -126,7 +174,7 @@ class Agent(Generic[TContext]):
|
|||
"""
|
||||
|
||||
@function_tool(
|
||||
name_override=tool_name or _utils.transform_string_function_style(self.name),
|
||||
name_override=tool_name or _transforms.transform_string_function_style(self.name),
|
||||
description_override=tool_description or "",
|
||||
)
|
||||
async def run_agent(context: RunContextWrapper, input: str) -> str:
|
||||
|
|
|
|||
|
|
@ -4,10 +4,10 @@ from typing import Any
|
|||
from pydantic import BaseModel, TypeAdapter
|
||||
from typing_extensions import TypedDict, get_args, get_origin
|
||||
|
||||
from . import _utils
|
||||
from .exceptions import ModelBehaviorError, UserError
|
||||
from .strict_schema import ensure_strict_json_schema
|
||||
from .tracing import SpanError
|
||||
from .util import _error_tracing, _json
|
||||
|
||||
_WRAPPER_DICT_KEY = "response"
|
||||
|
||||
|
|
@ -87,10 +87,10 @@ class AgentOutputSchema:
|
|||
"""Validate a JSON string against the output type. Returns the validated object, or raises
|
||||
a `ModelBehaviorError` if the JSON is invalid.
|
||||
"""
|
||||
validated = _utils.validate_json(json_str, self._type_adapter, partial)
|
||||
validated = _json.validate_json(json_str, self._type_adapter, partial)
|
||||
if self._is_wrapped:
|
||||
if not isinstance(validated, dict):
|
||||
_utils.attach_error_to_current_span(
|
||||
_error_tracing.attach_error_to_current_span(
|
||||
SpanError(
|
||||
message="Invalid JSON",
|
||||
data={"details": f"Expected a dict, got {type(validated)}"},
|
||||
|
|
@ -101,7 +101,7 @@ class AgentOutputSchema:
|
|||
)
|
||||
|
||||
if _WRAPPER_DICT_KEY not in validated:
|
||||
_utils.attach_error_to_current_span(
|
||||
_error_tracing.attach_error_to_current_span(
|
||||
SpanError(
|
||||
message="Invalid JSON",
|
||||
data={"details": f"Could not find key {_WRAPPER_DICT_KEY} in JSON"},
|
||||
|
|
|
|||
|
|
@ -33,6 +33,9 @@ class FuncSchema:
|
|||
"""The signature of the function."""
|
||||
takes_context: bool = False
|
||||
"""Whether the function takes a RunContextWrapper argument (must be the first argument)."""
|
||||
strict_json_schema: bool = True
|
||||
"""Whether the JSON schema is in strict mode. We **strongly** recommend setting this to True,
|
||||
as it increases the likelihood of correct JSON input."""
|
||||
|
||||
def to_call_args(self, data: BaseModel) -> tuple[list[Any], dict[str, Any]]:
|
||||
"""
|
||||
|
|
@ -337,4 +340,5 @@ def function_schema(
|
|||
params_json_schema=json_schema,
|
||||
signature=sig,
|
||||
takes_context=takes_context,
|
||||
strict_json_schema=strict_json_schema,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -7,10 +7,10 @@ from typing import TYPE_CHECKING, Any, Callable, Generic, Union, overload
|
|||
|
||||
from typing_extensions import TypeVar
|
||||
|
||||
from ._utils import MaybeAwaitable
|
||||
from .exceptions import UserError
|
||||
from .items import TResponseInputItem
|
||||
from .run_context import RunContextWrapper, TContext
|
||||
from .util._types import MaybeAwaitable
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .agent import Agent
|
||||
|
|
|
|||
|
|
@ -8,12 +8,12 @@ from typing import TYPE_CHECKING, Any, Callable, Generic, cast, overload
|
|||
from pydantic import TypeAdapter
|
||||
from typing_extensions import TypeAlias, TypeVar
|
||||
|
||||
from . import _utils
|
||||
from .exceptions import ModelBehaviorError, UserError
|
||||
from .items import RunItem, TResponseInputItem
|
||||
from .run_context import RunContextWrapper, TContext
|
||||
from .strict_schema import ensure_strict_json_schema
|
||||
from .tracing.spans import SpanError
|
||||
from .util import _error_tracing, _json, _transforms
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .agent import Agent
|
||||
|
|
@ -104,7 +104,7 @@ class Handoff(Generic[TContext]):
|
|||
|
||||
@classmethod
|
||||
def default_tool_name(cls, agent: Agent[Any]) -> str:
|
||||
return _utils.transform_string_function_style(f"transfer_to_{agent.name}")
|
||||
return _transforms.transform_string_function_style(f"transfer_to_{agent.name}")
|
||||
|
||||
@classmethod
|
||||
def default_tool_description(cls, agent: Agent[Any]) -> str:
|
||||
|
|
@ -192,7 +192,7 @@ def handoff(
|
|||
) -> Agent[Any]:
|
||||
if input_type is not None and type_adapter is not None:
|
||||
if input_json is None:
|
||||
_utils.attach_error_to_current_span(
|
||||
_error_tracing.attach_error_to_current_span(
|
||||
SpanError(
|
||||
message="Handoff function expected non-null input, but got None",
|
||||
data={"details": "input_json is None"},
|
||||
|
|
@ -200,7 +200,7 @@ def handoff(
|
|||
)
|
||||
raise ModelBehaviorError("Handoff function expected non-null input, but got None")
|
||||
|
||||
validated_input = _utils.validate_json(
|
||||
validated_input = _json.validate_json(
|
||||
json_str=input_json,
|
||||
type_adapter=type_adapter,
|
||||
partial=False,
|
||||
|
|
|
|||
|
|
@ -129,8 +129,10 @@ class ToolCallOutputItem(RunItemBase[Union[FunctionCallOutput, ComputerCallOutpu
|
|||
raw_item: FunctionCallOutput | ComputerCallOutput
|
||||
"""The raw item from the model."""
|
||||
|
||||
output: str
|
||||
"""The output of the tool call."""
|
||||
output: Any
|
||||
"""The output of the tool call. This is whatever the tool call returned; the `raw_item`
|
||||
contains a string representation of the output.
|
||||
"""
|
||||
|
||||
type: Literal["tool_call_output_item"] = "tool_call_output_item"
|
||||
|
||||
|
|
|
|||
|
|
@ -54,7 +54,7 @@ from openai.types.responses import (
|
|||
ResponseUsage,
|
||||
)
|
||||
from openai.types.responses.response_input_param import FunctionCallOutput, ItemReference, Message
|
||||
from openai.types.responses.response_usage import OutputTokensDetails
|
||||
from openai.types.responses.response_usage import InputTokensDetails, OutputTokensDetails
|
||||
|
||||
from .. import _debug
|
||||
from ..agent_output import AgentOutputSchema
|
||||
|
|
@ -420,6 +420,11 @@ class OpenAIChatCompletionsModel(Model):
|
|||
and usage.completion_tokens_details.reasoning_tokens
|
||||
else 0
|
||||
),
|
||||
input_tokens_details=InputTokensDetails(
|
||||
cached_tokens=usage.prompt_tokens_details.cached_tokens
|
||||
if usage.prompt_tokens_details and usage.prompt_tokens_details.cached_tokens
|
||||
else 0
|
||||
),
|
||||
)
|
||||
if usage
|
||||
else None
|
||||
|
|
@ -752,7 +757,7 @@ class _Converter:
|
|||
elif isinstance(c, dict) and c.get("type") == "input_file":
|
||||
raise UserError(f"File uploads are not supported for chat completions {c}")
|
||||
else:
|
||||
raise UserError(f"Unknonw content: {c}")
|
||||
raise UserError(f"Unknown content: {c}")
|
||||
return out
|
||||
|
||||
@classmethod
|
||||
|
|
|
|||
|
|
@ -34,6 +34,19 @@ class OpenAIProvider(ModelProvider):
|
|||
project: str | None = None,
|
||||
use_responses: bool | None = None,
|
||||
) -> None:
|
||||
"""Create a new OpenAI provider.
|
||||
|
||||
Args:
|
||||
api_key: The API key to use for the OpenAI client. If not provided, we will use the
|
||||
default API key.
|
||||
base_url: The base URL to use for the OpenAI client. If not provided, we will use the
|
||||
default base URL.
|
||||
openai_client: An optional OpenAI client to use. If not provided, we will create a new
|
||||
OpenAI client using the api_key and base_url.
|
||||
organization: The organization to use for the OpenAI client.
|
||||
project: The project to use for the OpenAI client.
|
||||
use_responses: Whether to use the OpenAI responses API.
|
||||
"""
|
||||
if openai_client is not None:
|
||||
assert api_key is None and base_url is None, (
|
||||
"Don't provide api_key or base_url if you provide openai_client"
|
||||
|
|
|
|||
|
|
@ -83,7 +83,7 @@ class OpenAIResponsesModel(Model):
|
|||
)
|
||||
|
||||
if _debug.DONT_LOG_MODEL_DATA:
|
||||
logger.debug("LLM responsed")
|
||||
logger.debug("LLM responded")
|
||||
else:
|
||||
logger.debug(
|
||||
"LLM resp:\n"
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
|
||||
|
|
@ -17,6 +17,7 @@ from .items import ItemHelpers, ModelResponse, RunItem, TResponseInputItem
|
|||
from .logger import logger
|
||||
from .stream_events import StreamEvent
|
||||
from .tracing import Trace
|
||||
from .util._pretty_print import pretty_print_result, pretty_print_run_result_streaming
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ._run_impl import QueueCompleteSentinel
|
||||
|
|
@ -89,6 +90,9 @@ class RunResult(RunResultBase):
|
|||
"""The last agent that was run."""
|
||||
return self._last_agent
|
||||
|
||||
def __str__(self) -> str:
|
||||
return pretty_print_result(self)
|
||||
|
||||
|
||||
@dataclass
|
||||
class RunResultStreaming(RunResultBase):
|
||||
|
|
@ -216,3 +220,6 @@ class RunResultStreaming(RunResultBase):
|
|||
|
||||
if self._output_guardrails_task and not self._output_guardrails_task.done():
|
||||
self._output_guardrails_task.cancel()
|
||||
|
||||
def __str__(self) -> str:
|
||||
return pretty_print_run_result_streaming(self)
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ from typing import Any, cast
|
|||
|
||||
from openai.types.responses import ResponseCompletedEvent
|
||||
|
||||
from . import Model, _utils
|
||||
from ._run_impl import (
|
||||
NextStepFinalOutput,
|
||||
NextStepHandoff,
|
||||
|
|
@ -33,7 +32,7 @@ from .items import ItemHelpers, ModelResponse, RunItem, TResponseInputItem
|
|||
from .lifecycle import RunHooks
|
||||
from .logger import logger
|
||||
from .model_settings import ModelSettings
|
||||
from .models.interface import ModelProvider
|
||||
from .models.interface import Model, ModelProvider
|
||||
from .models.openai_provider import OpenAIProvider
|
||||
from .result import RunResult, RunResultStreaming
|
||||
from .run_context import RunContextWrapper, TContext
|
||||
|
|
@ -41,6 +40,7 @@ from .stream_events import AgentUpdatedStreamEvent, RawResponsesStreamEvent
|
|||
from .tracing import Span, SpanError, agent_span, get_current_trace, trace
|
||||
from .tracing.span_data import AgentSpanData
|
||||
from .usage import Usage
|
||||
from .util import _coro, _error_tracing
|
||||
|
||||
DEFAULT_MAX_TURNS = 10
|
||||
|
||||
|
|
@ -193,7 +193,7 @@ class Runner:
|
|||
|
||||
current_turn += 1
|
||||
if current_turn > max_turns:
|
||||
_utils.attach_error_to_span(
|
||||
_error_tracing.attach_error_to_span(
|
||||
current_span,
|
||||
SpanError(
|
||||
message="Max turns exceeded",
|
||||
|
|
@ -447,7 +447,7 @@ class Runner:
|
|||
for done in asyncio.as_completed(guardrail_tasks):
|
||||
result = await done
|
||||
if result.output.tripwire_triggered:
|
||||
_utils.attach_error_to_span(
|
||||
_error_tracing.attach_error_to_span(
|
||||
parent_span,
|
||||
SpanError(
|
||||
message="Guardrail tripwire triggered",
|
||||
|
|
@ -511,7 +511,7 @@ class Runner:
|
|||
streamed_result.current_turn = current_turn
|
||||
|
||||
if current_turn > max_turns:
|
||||
_utils.attach_error_to_span(
|
||||
_error_tracing.attach_error_to_span(
|
||||
current_span,
|
||||
SpanError(
|
||||
message="Max turns exceeded",
|
||||
|
|
@ -583,7 +583,7 @@ class Runner:
|
|||
pass
|
||||
except Exception as e:
|
||||
if current_span:
|
||||
_utils.attach_error_to_span(
|
||||
_error_tracing.attach_error_to_span(
|
||||
current_span,
|
||||
SpanError(
|
||||
message="Error in agent run",
|
||||
|
|
@ -615,7 +615,7 @@ class Runner:
|
|||
(
|
||||
agent.hooks.on_start(context_wrapper, agent)
|
||||
if agent.hooks
|
||||
else _utils.noop_coroutine()
|
||||
else _coro.noop_coroutine()
|
||||
),
|
||||
)
|
||||
|
||||
|
|
@ -705,7 +705,7 @@ class Runner:
|
|||
(
|
||||
agent.hooks.on_start(context_wrapper, agent)
|
||||
if agent.hooks
|
||||
else _utils.noop_coroutine()
|
||||
else _coro.noop_coroutine()
|
||||
),
|
||||
)
|
||||
|
||||
|
|
@ -796,7 +796,7 @@ class Runner:
|
|||
# Cancel all guardrail tasks if a tripwire is triggered.
|
||||
for t in guardrail_tasks:
|
||||
t.cancel()
|
||||
_utils.attach_error_to_current_span(
|
||||
_error_tracing.attach_error_to_current_span(
|
||||
SpanError(
|
||||
message="Guardrail tripwire triggered",
|
||||
data={"guardrail": result.guardrail.get_name()},
|
||||
|
|
@ -834,7 +834,7 @@ class Runner:
|
|||
# Cancel all guardrail tasks if a tripwire is triggered.
|
||||
for t in guardrail_tasks:
|
||||
t.cancel()
|
||||
_utils.attach_error_to_current_span(
|
||||
_error_tracing.attach_error_to_current_span(
|
||||
SpanError(
|
||||
message="Guardrail tripwire triggered",
|
||||
data={"guardrail": result.guardrail.get_name()},
|
||||
|
|
|
|||
|
|
@ -11,14 +11,16 @@ from openai.types.responses.web_search_tool_param import UserLocation
|
|||
from pydantic import ValidationError
|
||||
from typing_extensions import Concatenate, ParamSpec
|
||||
|
||||
from . import _debug, _utils
|
||||
from ._utils import MaybeAwaitable
|
||||
from . import _debug
|
||||
from .computer import AsyncComputer, Computer
|
||||
from .exceptions import ModelBehaviorError
|
||||
from .function_schema import DocstringStyle, function_schema
|
||||
from .items import RunItem
|
||||
from .logger import logger
|
||||
from .run_context import RunContextWrapper
|
||||
from .tracing import SpanError
|
||||
from .util import _error_tracing
|
||||
from .util._types import MaybeAwaitable
|
||||
|
||||
ToolParams = ParamSpec("ToolParams")
|
||||
|
||||
|
|
@ -28,6 +30,18 @@ ToolFunctionWithContext = Callable[Concatenate[RunContextWrapper[Any], ToolParam
|
|||
ToolFunction = Union[ToolFunctionWithoutContext[ToolParams], ToolFunctionWithContext[ToolParams]]
|
||||
|
||||
|
||||
@dataclass
|
||||
class FunctionToolResult:
|
||||
tool: FunctionTool
|
||||
"""The tool that was run."""
|
||||
|
||||
output: Any
|
||||
"""The output of the tool."""
|
||||
|
||||
run_item: RunItem
|
||||
"""The run item that was produced as a result of the tool call."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class FunctionTool:
|
||||
"""A tool that wraps a function. In most cases, you should use the `function_tool` helpers to
|
||||
|
|
@ -43,15 +57,15 @@ class FunctionTool:
|
|||
params_json_schema: dict[str, Any]
|
||||
"""The JSON schema for the tool's parameters."""
|
||||
|
||||
on_invoke_tool: Callable[[RunContextWrapper[Any], str], Awaitable[str]]
|
||||
on_invoke_tool: Callable[[RunContextWrapper[Any], str], Awaitable[Any]]
|
||||
"""A function that invokes the tool with the given context and parameters. The params passed
|
||||
are:
|
||||
1. The tool run context.
|
||||
2. The arguments from the LLM, as a JSON string.
|
||||
|
||||
You must return a string representation of the tool output. In case of errors, you can either
|
||||
raise an Exception (which will cause the run to fail) or return a string error message (which
|
||||
will be sent back to the LLM).
|
||||
You must return a string representation of the tool output, or something we can call `str()` on.
|
||||
In case of errors, you can either raise an Exception (which will cause the run to fail) or
|
||||
return a string error message (which will be sent back to the LLM).
|
||||
"""
|
||||
|
||||
strict_json_schema: bool = True
|
||||
|
|
@ -137,6 +151,7 @@ def function_tool(
|
|||
docstring_style: DocstringStyle | None = None,
|
||||
use_docstring_info: bool = True,
|
||||
failure_error_function: ToolErrorFunction | None = None,
|
||||
strict_mode: bool = True,
|
||||
) -> FunctionTool:
|
||||
"""Overload for usage as @function_tool (no parentheses)."""
|
||||
...
|
||||
|
|
@ -150,6 +165,7 @@ def function_tool(
|
|||
docstring_style: DocstringStyle | None = None,
|
||||
use_docstring_info: bool = True,
|
||||
failure_error_function: ToolErrorFunction | None = None,
|
||||
strict_mode: bool = True,
|
||||
) -> Callable[[ToolFunction[...]], FunctionTool]:
|
||||
"""Overload for usage as @function_tool(...)."""
|
||||
...
|
||||
|
|
@ -163,6 +179,7 @@ def function_tool(
|
|||
docstring_style: DocstringStyle | None = None,
|
||||
use_docstring_info: bool = True,
|
||||
failure_error_function: ToolErrorFunction | None = default_tool_error_function,
|
||||
strict_mode: bool = True,
|
||||
) -> FunctionTool | Callable[[ToolFunction[...]], FunctionTool]:
|
||||
"""
|
||||
Decorator to create a FunctionTool from a function. By default, we will:
|
||||
|
|
@ -186,6 +203,11 @@ def function_tool(
|
|||
failure_error_function: If provided, use this function to generate an error message when
|
||||
the tool call fails. The error message is sent to the LLM. If you pass None, then no
|
||||
error message will be sent and instead an Exception will be raised.
|
||||
strict_mode: Whether to enable strict mode for the tool's JSON schema. We *strongly*
|
||||
recommend setting this to True, as it increases the likelihood of correct JSON input.
|
||||
If False, it allows non-strict JSON schemas. For example, if a parameter has a default
|
||||
value, it will be optional, additional properties are allowed, etc. See here for more:
|
||||
https://platform.openai.com/docs/guides/structured-outputs?api-mode=responses#supported-schemas
|
||||
"""
|
||||
|
||||
def _create_function_tool(the_func: ToolFunction[...]) -> FunctionTool:
|
||||
|
|
@ -195,9 +217,10 @@ def function_tool(
|
|||
description_override=description_override,
|
||||
docstring_style=docstring_style,
|
||||
use_docstring_info=use_docstring_info,
|
||||
strict_json_schema=strict_mode,
|
||||
)
|
||||
|
||||
async def _on_invoke_tool_impl(ctx: RunContextWrapper[Any], input: str) -> str:
|
||||
async def _on_invoke_tool_impl(ctx: RunContextWrapper[Any], input: str) -> Any:
|
||||
try:
|
||||
json_data: dict[str, Any] = json.loads(input) if input else {}
|
||||
except Exception as e:
|
||||
|
|
@ -244,9 +267,9 @@ def function_tool(
|
|||
else:
|
||||
logger.debug(f"Tool {schema.name} returned {result}")
|
||||
|
||||
return str(result)
|
||||
return result
|
||||
|
||||
async def _on_invoke_tool(ctx: RunContextWrapper[Any], input: str) -> str:
|
||||
async def _on_invoke_tool(ctx: RunContextWrapper[Any], input: str) -> Any:
|
||||
try:
|
||||
return await _on_invoke_tool_impl(ctx, input)
|
||||
except Exception as e:
|
||||
|
|
@ -257,7 +280,7 @@ def function_tool(
|
|||
if inspect.isawaitable(result):
|
||||
return await result
|
||||
|
||||
_utils.attach_error_to_current_span(
|
||||
_error_tracing.attach_error_to_current_span(
|
||||
SpanError(
|
||||
message="Error running tool (non-fatal)",
|
||||
data={
|
||||
|
|
@ -273,6 +296,7 @@ def function_tool(
|
|||
description=schema.description or "",
|
||||
params_json_schema=schema.params_json_schema,
|
||||
on_invoke_tool=_on_invoke_tool,
|
||||
strict_json_schema=strict_mode,
|
||||
)
|
||||
|
||||
# If func is actually a callable, we were used as @function_tool with no parentheses
|
||||
|
|
|
|||
|
|
@ -10,7 +10,10 @@ from .create import (
|
|||
guardrail_span,
|
||||
handoff_span,
|
||||
response_span,
|
||||
speech_group_span,
|
||||
speech_span,
|
||||
trace,
|
||||
transcription_span,
|
||||
)
|
||||
from .processor_interface import TracingProcessor
|
||||
from .processors import default_exporter, default_processor
|
||||
|
|
@ -24,6 +27,9 @@ from .span_data import (
|
|||
HandoffSpanData,
|
||||
ResponseSpanData,
|
||||
SpanData,
|
||||
SpeechGroupSpanData,
|
||||
SpeechSpanData,
|
||||
TranscriptionSpanData,
|
||||
)
|
||||
from .spans import Span, SpanError
|
||||
from .traces import Trace
|
||||
|
|
@ -54,9 +60,15 @@ __all__ = [
|
|||
"GuardrailSpanData",
|
||||
"HandoffSpanData",
|
||||
"ResponseSpanData",
|
||||
"SpeechGroupSpanData",
|
||||
"SpeechSpanData",
|
||||
"TranscriptionSpanData",
|
||||
"TracingProcessor",
|
||||
"gen_trace_id",
|
||||
"gen_span_id",
|
||||
"speech_group_span",
|
||||
"speech_span",
|
||||
"transcription_span",
|
||||
]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -13,6 +13,9 @@ from .span_data import (
|
|||
GuardrailSpanData,
|
||||
HandoffSpanData,
|
||||
ResponseSpanData,
|
||||
SpeechGroupSpanData,
|
||||
SpeechSpanData,
|
||||
TranscriptionSpanData,
|
||||
)
|
||||
from .spans import Span
|
||||
from .traces import Trace
|
||||
|
|
@ -181,7 +184,11 @@ def generation_span(
|
|||
"""
|
||||
return GLOBAL_TRACE_PROVIDER.create_span(
|
||||
span_data=GenerationSpanData(
|
||||
input=input, output=output, model=model, model_config=model_config, usage=usage
|
||||
input=input,
|
||||
output=output,
|
||||
model=model,
|
||||
model_config=model_config,
|
||||
usage=usage,
|
||||
),
|
||||
span_id=span_id,
|
||||
parent=parent,
|
||||
|
|
@ -304,3 +311,116 @@ def guardrail_span(
|
|||
parent=parent,
|
||||
disabled=disabled,
|
||||
)
|
||||
|
||||
|
||||
def transcription_span(
|
||||
model: str | None = None,
|
||||
input: str | None = None,
|
||||
input_format: str | None = "pcm",
|
||||
output: str | None = None,
|
||||
model_config: Mapping[str, Any] | None = None,
|
||||
span_id: str | None = None,
|
||||
parent: Trace | Span[Any] | None = None,
|
||||
disabled: bool = False,
|
||||
) -> Span[TranscriptionSpanData]:
|
||||
"""Create a new transcription span. The span will not be started automatically, you should
|
||||
either do `with transcription_span() ...` or call `span.start()` + `span.finish()` manually.
|
||||
|
||||
Args:
|
||||
model: The name of the model used for the speech-to-text.
|
||||
input: The audio input of the speech-to-text transcription, as a base64 encoded string of
|
||||
audio bytes.
|
||||
input_format: The format of the audio input (defaults to "pcm").
|
||||
output: The output of the speech-to-text transcription.
|
||||
model_config: The model configuration (hyperparameters) used.
|
||||
span_id: The ID of the span. Optional. If not provided, we will generate an ID. We
|
||||
recommend using `util.gen_span_id()` to generate a span ID, to guarantee that IDs are
|
||||
correctly formatted.
|
||||
parent: The parent span or trace. If not provided, we will automatically use the current
|
||||
trace/span as the parent.
|
||||
disabled: If True, we will return a Span but the Span will not be recorded.
|
||||
|
||||
Returns:
|
||||
The newly created speech-to-text span.
|
||||
"""
|
||||
return GLOBAL_TRACE_PROVIDER.create_span(
|
||||
span_data=TranscriptionSpanData(
|
||||
input=input,
|
||||
input_format=input_format,
|
||||
output=output,
|
||||
model=model,
|
||||
model_config=model_config,
|
||||
),
|
||||
span_id=span_id,
|
||||
parent=parent,
|
||||
disabled=disabled,
|
||||
)
|
||||
|
||||
|
||||
def speech_span(
|
||||
model: str | None = None,
|
||||
input: str | None = None,
|
||||
output: str | None = None,
|
||||
output_format: str | None = "pcm",
|
||||
model_config: Mapping[str, Any] | None = None,
|
||||
first_content_at: str | None = None,
|
||||
span_id: str | None = None,
|
||||
parent: Trace | Span[Any] | None = None,
|
||||
disabled: bool = False,
|
||||
) -> Span[SpeechSpanData]:
|
||||
"""Create a new speech span. The span will not be started automatically, you should either do
|
||||
`with speech_span() ...` or call `span.start()` + `span.finish()` manually.
|
||||
|
||||
Args:
|
||||
model: The name of the model used for the text-to-speech.
|
||||
input: The text input of the text-to-speech.
|
||||
output: The audio output of the text-to-speech as base64 encoded string of PCM audio bytes.
|
||||
output_format: The format of the audio output (defaults to "pcm").
|
||||
model_config: The model configuration (hyperparameters) used.
|
||||
first_content_at: The time of the first byte of the audio output.
|
||||
span_id: The ID of the span. Optional. If not provided, we will generate an ID. We
|
||||
recommend using `util.gen_span_id()` to generate a span ID, to guarantee that IDs are
|
||||
correctly formatted.
|
||||
parent: The parent span or trace. If not provided, we will automatically use the current
|
||||
trace/span as the parent.
|
||||
disabled: If True, we will return a Span but the Span will not be recorded.
|
||||
"""
|
||||
return GLOBAL_TRACE_PROVIDER.create_span(
|
||||
span_data=SpeechSpanData(
|
||||
model=model,
|
||||
input=input,
|
||||
output=output,
|
||||
output_format=output_format,
|
||||
model_config=model_config,
|
||||
first_content_at=first_content_at,
|
||||
),
|
||||
span_id=span_id,
|
||||
parent=parent,
|
||||
disabled=disabled,
|
||||
)
|
||||
|
||||
|
||||
def speech_group_span(
|
||||
input: str | None = None,
|
||||
span_id: str | None = None,
|
||||
parent: Trace | Span[Any] | None = None,
|
||||
disabled: bool = False,
|
||||
) -> Span[SpeechGroupSpanData]:
|
||||
"""Create a new speech group span. The span will not be started automatically, you should
|
||||
either do `with speech_group_span() ...` or call `span.start()` + `span.finish()` manually.
|
||||
|
||||
Args:
|
||||
input: The input text used for the speech request.
|
||||
span_id: The ID of the span. Optional. If not provided, we will generate an ID. We
|
||||
recommend using `util.gen_span_id()` to generate a span ID, to guarantee that IDs are
|
||||
correctly formatted.
|
||||
parent: The parent span or trace. If not provided, we will automatically use the current
|
||||
trace/span as the parent.
|
||||
disabled: If True, we will return a Span but the Span will not be recorded.
|
||||
"""
|
||||
return GLOBAL_TRACE_PROVIDER.create_span(
|
||||
span_data=SpeechGroupSpanData(input=input),
|
||||
span_id=span_id,
|
||||
parent=parent,
|
||||
disabled=disabled,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import queue
|
|||
import random
|
||||
import threading
|
||||
import time
|
||||
from functools import cached_property
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
|
@ -50,9 +51,9 @@ class BackendSpanExporter(TracingExporter):
|
|||
base_delay: Base delay (in seconds) for the first backoff.
|
||||
max_delay: Maximum delay (in seconds) for backoff growth.
|
||||
"""
|
||||
self.api_key = api_key or os.environ.get("OPENAI_API_KEY")
|
||||
self.organization = organization or os.environ.get("OPENAI_ORG_ID")
|
||||
self.project = project or os.environ.get("OPENAI_PROJECT_ID")
|
||||
self._api_key = api_key
|
||||
self._organization = organization
|
||||
self._project = project
|
||||
self.endpoint = endpoint
|
||||
self.max_retries = max_retries
|
||||
self.base_delay = base_delay
|
||||
|
|
@ -68,8 +69,22 @@ class BackendSpanExporter(TracingExporter):
|
|||
api_key: The OpenAI API key to use. This is the same key used by the OpenAI Python
|
||||
client.
|
||||
"""
|
||||
# We're specifically setting the underlying cached property as well
|
||||
self._api_key = api_key
|
||||
self.api_key = api_key
|
||||
|
||||
@cached_property
|
||||
def api_key(self):
|
||||
return self._api_key or os.environ.get("OPENAI_API_KEY")
|
||||
|
||||
@cached_property
|
||||
def organization(self):
|
||||
return self._organization or os.environ.get("OPENAI_ORG_ID")
|
||||
|
||||
@cached_property
|
||||
def project(self):
|
||||
return self._project or os.environ.get("OPENAI_PROJECT_ID")
|
||||
|
||||
def export(self, items: list[Trace | Span[Any]]) -> None:
|
||||
if not items:
|
||||
return
|
||||
|
|
@ -102,18 +117,22 @@ class BackendSpanExporter(TracingExporter):
|
|||
|
||||
# If the response is a client error (4xx), we wont retry
|
||||
if 400 <= response.status_code < 500:
|
||||
logger.error(f"Tracing client error {response.status_code}: {response.text}")
|
||||
logger.error(
|
||||
f"[non-fatal] Tracing client error {response.status_code}: {response.text}"
|
||||
)
|
||||
return
|
||||
|
||||
# For 5xx or other unexpected codes, treat it as transient and retry
|
||||
logger.warning(f"Server error {response.status_code}, retrying.")
|
||||
logger.warning(
|
||||
f"[non-fatal] Tracing: server error {response.status_code}, retrying."
|
||||
)
|
||||
except httpx.RequestError as exc:
|
||||
# Network or other I/O error, we'll retry
|
||||
logger.warning(f"Request failed: {exc}")
|
||||
logger.warning(f"[non-fatal] Tracing: request failed: {exc}")
|
||||
|
||||
# If we reach here, we need to retry or give up
|
||||
if attempt >= self.max_retries:
|
||||
logger.error("Max retries reached, giving up on this batch.")
|
||||
logger.error("[non-fatal] Tracing: max retries reached, giving up on this batch.")
|
||||
return
|
||||
|
||||
# Exponential backoff + jitter
|
||||
|
|
|
|||
|
|
@ -51,7 +51,7 @@ class AgentSpanData(SpanData):
|
|||
class FunctionSpanData(SpanData):
|
||||
__slots__ = ("name", "input", "output")
|
||||
|
||||
def __init__(self, name: str, input: str | None, output: str | None):
|
||||
def __init__(self, name: str, input: str | None, output: Any | None):
|
||||
self.name = name
|
||||
self.input = input
|
||||
self.output = output
|
||||
|
|
@ -65,7 +65,7 @@ class FunctionSpanData(SpanData):
|
|||
"type": self.type,
|
||||
"name": self.name,
|
||||
"input": self.input,
|
||||
"output": self.output,
|
||||
"output": str(self.output) if self.output else None,
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -186,3 +186,99 @@ class GuardrailSpanData(SpanData):
|
|||
"name": self.name,
|
||||
"triggered": self.triggered,
|
||||
}
|
||||
|
||||
|
||||
class TranscriptionSpanData(SpanData):
|
||||
__slots__ = (
|
||||
"input",
|
||||
"output",
|
||||
"model",
|
||||
"model_config",
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
input: str | None = None,
|
||||
input_format: str | None = "pcm",
|
||||
output: str | None = None,
|
||||
model: str | None = None,
|
||||
model_config: Mapping[str, Any] | None = None,
|
||||
):
|
||||
self.input = input
|
||||
self.input_format = input_format
|
||||
self.output = output
|
||||
self.model = model
|
||||
self.model_config = model_config
|
||||
|
||||
@property
|
||||
def type(self) -> str:
|
||||
return "transcription"
|
||||
|
||||
def export(self) -> dict[str, Any]:
|
||||
return {
|
||||
"type": self.type,
|
||||
"input": {
|
||||
"data": self.input or "",
|
||||
"format": self.input_format,
|
||||
},
|
||||
"output": self.output,
|
||||
"model": self.model,
|
||||
"model_config": self.model_config,
|
||||
}
|
||||
|
||||
|
||||
class SpeechSpanData(SpanData):
|
||||
__slots__ = ("input", "output", "model", "model_config", "first_byte_at")
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
input: str | None = None,
|
||||
output: str | None = None,
|
||||
output_format: str | None = "pcm",
|
||||
model: str | None = None,
|
||||
model_config: Mapping[str, Any] | None = None,
|
||||
first_content_at: str | None = None,
|
||||
):
|
||||
self.input = input
|
||||
self.output = output
|
||||
self.output_format = output_format
|
||||
self.model = model
|
||||
self.model_config = model_config
|
||||
self.first_content_at = first_content_at
|
||||
|
||||
@property
|
||||
def type(self) -> str:
|
||||
return "speech"
|
||||
|
||||
def export(self) -> dict[str, Any]:
|
||||
return {
|
||||
"type": self.type,
|
||||
"input": self.input,
|
||||
"output": {
|
||||
"data": self.output or "",
|
||||
"format": self.output_format,
|
||||
},
|
||||
"model": self.model,
|
||||
"model_config": self.model_config,
|
||||
"first_content_at": self.first_content_at,
|
||||
}
|
||||
|
||||
|
||||
class SpeechGroupSpanData(SpanData):
|
||||
__slots__ = "input"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
input: str | None = None,
|
||||
):
|
||||
self.input = input
|
||||
|
||||
@property
|
||||
def type(self) -> str:
|
||||
return "speech-group"
|
||||
|
||||
def export(self) -> dict[str, Any]:
|
||||
return {
|
||||
"type": self.type,
|
||||
"input": self.input,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,3 +15,8 @@ def gen_trace_id() -> str:
|
|||
def gen_span_id() -> str:
|
||||
"""Generates a new span ID."""
|
||||
return f"span_{uuid.uuid4().hex[:24]}"
|
||||
|
||||
|
||||
def gen_group_id() -> str:
|
||||
"""Generates a new group ID."""
|
||||
return f"group_{uuid.uuid4().hex[:24]}"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,2 @@
|
|||
async def noop_coroutine() -> None:
|
||||
pass
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
from typing import Any
|
||||
|
||||
from ..logger import logger
|
||||
from ..tracing import Span, SpanError, get_current_span
|
||||
|
||||
|
||||
def attach_error_to_span(span: Span[Any], error: SpanError) -> None:
|
||||
span.set_error(error)
|
||||
|
||||
|
||||
def attach_error_to_current_span(error: SpanError) -> None:
|
||||
span = get_current_span()
|
||||
if span:
|
||||
attach_error_to_span(span, error)
|
||||
else:
|
||||
logger.warning(f"No span to add error {error} to")
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
from typing_extensions import TypeVar
|
||||
|
||||
from ..exceptions import ModelBehaviorError
|
||||
from ..tracing import SpanError
|
||||
from ._error_tracing import attach_error_to_current_span
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
def validate_json(json_str: str, type_adapter: TypeAdapter[T], partial: bool) -> T:
|
||||
partial_setting: bool | Literal["off", "on", "trailing-strings"] = (
|
||||
"trailing-strings" if partial else False
|
||||
)
|
||||
try:
|
||||
validated = type_adapter.validate_json(json_str, experimental_allow_partial=partial_setting)
|
||||
return validated
|
||||
except ValidationError as e:
|
||||
attach_error_to_current_span(
|
||||
SpanError(
|
||||
message="Invalid JSON provided",
|
||||
data={},
|
||||
)
|
||||
)
|
||||
raise ModelBehaviorError(
|
||||
f"Invalid JSON when parsing {json_str} for {type_adapter}; {e}"
|
||||
) from e
|
||||
|
|
@ -0,0 +1,56 @@
|
|||
from typing import TYPE_CHECKING
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..result import RunResult, RunResultBase, RunResultStreaming
|
||||
|
||||
|
||||
def _indent(text: str, indent_level: int) -> str:
|
||||
indent_string = " " * indent_level
|
||||
return "\n".join(f"{indent_string}{line}" for line in text.splitlines())
|
||||
|
||||
|
||||
def _final_output_str(result: "RunResultBase") -> str:
|
||||
if result.final_output is None:
|
||||
return "None"
|
||||
elif isinstance(result.final_output, str):
|
||||
return result.final_output
|
||||
elif isinstance(result.final_output, BaseModel):
|
||||
return result.final_output.model_dump_json(indent=2)
|
||||
else:
|
||||
return str(result.final_output)
|
||||
|
||||
|
||||
def pretty_print_result(result: "RunResult") -> str:
|
||||
output = "RunResult:"
|
||||
output += f'\n- Last agent: Agent(name="{result.last_agent.name}", ...)'
|
||||
output += (
|
||||
f"\n- Final output ({type(result.final_output).__name__}):\n"
|
||||
f"{_indent(_final_output_str(result), 2)}"
|
||||
)
|
||||
output += f"\n- {len(result.new_items)} new item(s)"
|
||||
output += f"\n- {len(result.raw_responses)} raw response(s)"
|
||||
output += f"\n- {len(result.input_guardrail_results)} input guardrail result(s)"
|
||||
output += f"\n- {len(result.output_guardrail_results)} output guardrail result(s)"
|
||||
output += "\n(See `RunResult` for more details)"
|
||||
|
||||
return output
|
||||
|
||||
|
||||
def pretty_print_run_result_streaming(result: "RunResultStreaming") -> str:
|
||||
output = "RunResultStreaming:"
|
||||
output += f'\n- Current agent: Agent(name="{result.current_agent.name}", ...)'
|
||||
output += f"\n- Current turn: {result.current_turn}"
|
||||
output += f"\n- Max turns: {result.max_turns}"
|
||||
output += f"\n- Is complete: {result.is_complete}"
|
||||
output += (
|
||||
f"\n- Final output ({type(result.final_output).__name__}):\n"
|
||||
f"{_indent(_final_output_str(result), 2)}"
|
||||
)
|
||||
output += f"\n- {len(result.new_items)} new item(s)"
|
||||
output += f"\n- {len(result.raw_responses)} raw response(s)"
|
||||
output += f"\n- {len(result.input_guardrail_results)} input guardrail result(s)"
|
||||
output += f"\n- {len(result.output_guardrail_results)} output guardrail result(s)"
|
||||
output += "\n(See `RunResultStreaming` for more details)"
|
||||
return output
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
import re
|
||||
|
||||
|
||||
def transform_string_function_style(name: str) -> str:
|
||||
# Replace spaces with underscores
|
||||
name = name.replace(" ", "_")
|
||||
|
||||
# Replace non-alphanumeric characters with underscores
|
||||
name = re.sub(r"[^a-zA-Z0-9]", "_", name)
|
||||
|
||||
return name.lower()
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
from collections.abc import Awaitable
|
||||
from typing import Union
|
||||
|
||||
from typing_extensions import TypeVar
|
||||
|
||||
T = TypeVar("T")
|
||||
MaybeAwaitable = Union[Awaitable[T], T]
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
from .events import VoiceStreamEvent, VoiceStreamEventAudio, VoiceStreamEventLifecycle
|
||||
from .exceptions import STTWebsocketConnectionError
|
||||
from .input import AudioInput, StreamedAudioInput
|
||||
from .model import (
|
||||
StreamedTranscriptionSession,
|
||||
STTModel,
|
||||
STTModelSettings,
|
||||
TTSModel,
|
||||
TTSModelSettings,
|
||||
VoiceModelProvider,
|
||||
)
|
||||
from .models.openai_model_provider import OpenAIVoiceModelProvider
|
||||
from .models.openai_stt import OpenAISTTModel, OpenAISTTTranscriptionSession
|
||||
from .models.openai_tts import OpenAITTSModel
|
||||
from .pipeline import VoicePipeline
|
||||
from .pipeline_config import VoicePipelineConfig
|
||||
from .result import StreamedAudioResult
|
||||
from .utils import get_sentence_based_splitter
|
||||
from .workflow import (
|
||||
SingleAgentVoiceWorkflow,
|
||||
SingleAgentWorkflowCallbacks,
|
||||
VoiceWorkflowBase,
|
||||
VoiceWorkflowHelper,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"AudioInput",
|
||||
"StreamedAudioInput",
|
||||
"STTModel",
|
||||
"STTModelSettings",
|
||||
"TTSModel",
|
||||
"TTSModelSettings",
|
||||
"VoiceModelProvider",
|
||||
"StreamedAudioResult",
|
||||
"SingleAgentVoiceWorkflow",
|
||||
"OpenAIVoiceModelProvider",
|
||||
"OpenAISTTModel",
|
||||
"OpenAITTSModel",
|
||||
"VoiceStreamEventAudio",
|
||||
"VoiceStreamEventLifecycle",
|
||||
"VoiceStreamEvent",
|
||||
"VoicePipeline",
|
||||
"VoicePipelineConfig",
|
||||
"get_sentence_based_splitter",
|
||||
"VoiceWorkflowHelper",
|
||||
"VoiceWorkflowBase",
|
||||
"SingleAgentWorkflowCallbacks",
|
||||
"StreamedTranscriptionSession",
|
||||
"OpenAISTTTranscriptionSession",
|
||||
"STTWebsocketConnectionError",
|
||||
]
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Literal, Union
|
||||
|
||||
from typing_extensions import TypeAlias
|
||||
|
||||
from .imports import np, npt
|
||||
|
||||
|
||||
@dataclass
|
||||
class VoiceStreamEventAudio:
|
||||
"""Streaming event from the VoicePipeline"""
|
||||
|
||||
data: npt.NDArray[np.int16 | np.float32] | None
|
||||
"""The audio data."""
|
||||
|
||||
type: Literal["voice_stream_event_audio"] = "voice_stream_event_audio"
|
||||
"""The type of event."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class VoiceStreamEventLifecycle:
|
||||
"""Streaming event from the VoicePipeline"""
|
||||
|
||||
event: Literal["turn_started", "turn_ended", "session_ended"]
|
||||
"""The event that occurred."""
|
||||
|
||||
type: Literal["voice_stream_event_lifecycle"] = "voice_stream_event_lifecycle"
|
||||
"""The type of event."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class VoiceStreamEventError:
|
||||
"""Streaming event from the VoicePipeline"""
|
||||
|
||||
error: Exception
|
||||
"""The error that occurred."""
|
||||
|
||||
type: Literal["voice_stream_event_error"] = "voice_stream_event_error"
|
||||
"""The type of event."""
|
||||
|
||||
|
||||
VoiceStreamEvent: TypeAlias = Union[
|
||||
VoiceStreamEventAudio, VoiceStreamEventLifecycle, VoiceStreamEventError
|
||||
]
|
||||
"""An event from the `VoicePipeline`, streamed via `StreamedAudioResult.stream()`."""
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
from ..exceptions import AgentsException
|
||||
|
||||
|
||||
class STTWebsocketConnectionError(AgentsException):
|
||||
"""Exception raised when the STT websocket connection fails."""
|
||||
|
||||
def __init__(self, message: str):
|
||||
self.message = message
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
try:
|
||||
import numpy as np
|
||||
import numpy.typing as npt
|
||||
import websockets
|
||||
except ImportError as _e:
|
||||
raise ImportError(
|
||||
"`numpy` + `websockets` are required to use voice. You can install them via the optional "
|
||||
"dependency group: `pip install 'openai-agents[voice]'`."
|
||||
) from _e
|
||||
|
||||
__all__ = ["np", "npt", "websockets"]
|
||||
|
|
@ -0,0 +1,88 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import io
|
||||
import wave
|
||||
from dataclasses import dataclass
|
||||
|
||||
from ..exceptions import UserError
|
||||
from .imports import np, npt
|
||||
|
||||
DEFAULT_SAMPLE_RATE = 24000
|
||||
|
||||
|
||||
def _buffer_to_audio_file(
|
||||
buffer: npt.NDArray[np.int16 | np.float32],
|
||||
frame_rate: int = DEFAULT_SAMPLE_RATE,
|
||||
sample_width: int = 2,
|
||||
channels: int = 1,
|
||||
) -> tuple[str, io.BytesIO, str]:
|
||||
if buffer.dtype == np.float32:
|
||||
# convert to int16
|
||||
buffer = np.clip(buffer, -1.0, 1.0)
|
||||
buffer = (buffer * 32767).astype(np.int16)
|
||||
elif buffer.dtype != np.int16:
|
||||
raise UserError("Buffer must be a numpy array of int16 or float32")
|
||||
|
||||
audio_file = io.BytesIO()
|
||||
with wave.open(audio_file, "w") as wav_file:
|
||||
wav_file.setnchannels(channels)
|
||||
wav_file.setsampwidth(sample_width)
|
||||
wav_file.setframerate(frame_rate)
|
||||
wav_file.writeframes(buffer.tobytes())
|
||||
audio_file.seek(0)
|
||||
|
||||
# (filename, bytes, content_type)
|
||||
return ("audio.wav", audio_file, "audio/wav")
|
||||
|
||||
|
||||
@dataclass
|
||||
class AudioInput:
|
||||
"""Static audio to be used as input for the VoicePipeline."""
|
||||
|
||||
buffer: npt.NDArray[np.int16 | np.float32]
|
||||
"""
|
||||
A buffer containing the audio data for the agent. Must be a numpy array of int16 or float32.
|
||||
"""
|
||||
|
||||
frame_rate: int = DEFAULT_SAMPLE_RATE
|
||||
"""The sample rate of the audio data. Defaults to 24000."""
|
||||
|
||||
sample_width: int = 2
|
||||
"""The sample width of the audio data. Defaults to 2."""
|
||||
|
||||
channels: int = 1
|
||||
"""The number of channels in the audio data. Defaults to 1."""
|
||||
|
||||
def to_audio_file(self) -> tuple[str, io.BytesIO, str]:
|
||||
"""Returns a tuple of (filename, bytes, content_type)"""
|
||||
return _buffer_to_audio_file(self.buffer, self.frame_rate, self.sample_width, self.channels)
|
||||
|
||||
def to_base64(self) -> str:
|
||||
"""Returns the audio data as a base64 encoded string."""
|
||||
if self.buffer.dtype == np.float32:
|
||||
# convert to int16
|
||||
self.buffer = np.clip(self.buffer, -1.0, 1.0)
|
||||
self.buffer = (self.buffer * 32767).astype(np.int16)
|
||||
elif self.buffer.dtype != np.int16:
|
||||
raise UserError("Buffer must be a numpy array of int16 or float32")
|
||||
|
||||
return base64.b64encode(self.buffer.tobytes()).decode("utf-8")
|
||||
|
||||
|
||||
class StreamedAudioInput:
|
||||
"""Audio input represented as a stream of audio data. You can pass this to the `VoicePipeline`
|
||||
and then push audio data into the queue using the `add_audio` method.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.queue: asyncio.Queue[npt.NDArray[np.int16 | np.float32]] = asyncio.Queue()
|
||||
|
||||
async def add_audio(self, audio: npt.NDArray[np.int16 | np.float32]):
|
||||
"""Adds more audio data to the stream.
|
||||
|
||||
Args:
|
||||
audio: The audio data to add. Must be a numpy array of int16 or float32.
|
||||
"""
|
||||
await self.queue.put(audio)
|
||||
|
|
@ -0,0 +1,193 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import abc
|
||||
from collections.abc import AsyncIterator
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Callable, Literal
|
||||
|
||||
from .imports import np, npt
|
||||
from .input import AudioInput, StreamedAudioInput
|
||||
from .utils import get_sentence_based_splitter
|
||||
|
||||
DEFAULT_TTS_INSTRUCTIONS = (
|
||||
"You will receive partial sentences. Do not complete the sentence, just read out the text."
|
||||
)
|
||||
DEFAULT_TTS_BUFFER_SIZE = 120
|
||||
|
||||
|
||||
@dataclass
|
||||
class TTSModelSettings:
|
||||
"""Settings for a TTS model."""
|
||||
|
||||
voice: (
|
||||
Literal["alloy", "ash", "coral", "echo", "fable", "onyx", "nova", "sage", "shimmer"] | None
|
||||
) = None
|
||||
"""
|
||||
The voice to use for the TTS model. If not provided, the default voice for the respective model
|
||||
will be used.
|
||||
"""
|
||||
|
||||
buffer_size: int = 120
|
||||
"""The minimal size of the chunks of audio data that are being streamed out."""
|
||||
|
||||
dtype: npt.DTypeLike = np.int16
|
||||
"""The data type for the audio data to be returned in."""
|
||||
|
||||
transform_data: (
|
||||
Callable[[npt.NDArray[np.int16 | np.float32]], npt.NDArray[np.int16 | np.float32]] | None
|
||||
) = None
|
||||
"""
|
||||
A function to transform the data from the TTS model. This is useful if you want the resulting
|
||||
audio stream to have the data in a specific shape already.
|
||||
"""
|
||||
|
||||
instructions: str = (
|
||||
"You will receive partial sentences. Do not complete the sentence just read out the text."
|
||||
)
|
||||
"""
|
||||
The instructions to use for the TTS model. This is useful if you want to control the tone of the
|
||||
audio output.
|
||||
"""
|
||||
|
||||
text_splitter: Callable[[str], tuple[str, str]] = get_sentence_based_splitter()
|
||||
"""
|
||||
A function to split the text into chunks. This is useful if you want to split the text into
|
||||
chunks before sending it to the TTS model rather than waiting for the whole text to be
|
||||
processed.
|
||||
"""
|
||||
|
||||
speed: float | None = None
|
||||
"""The speed with which the TTS model will read the text. Between 0.25 and 4.0."""
|
||||
|
||||
|
||||
class TTSModel(abc.ABC):
|
||||
"""A text-to-speech model that can convert text into audio output."""
|
||||
|
||||
@property
|
||||
@abc.abstractmethod
|
||||
def model_name(self) -> str:
|
||||
"""The name of the TTS model."""
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
def run(self, text: str, settings: TTSModelSettings) -> AsyncIterator[bytes]:
|
||||
"""Given a text string, produces a stream of audio bytes, in PCM format.
|
||||
|
||||
Args:
|
||||
text: The text to convert to audio.
|
||||
|
||||
Returns:
|
||||
An async iterator of audio bytes, in PCM format.
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class StreamedTranscriptionSession(abc.ABC):
|
||||
"""A streamed transcription of audio input."""
|
||||
|
||||
@abc.abstractmethod
|
||||
def transcribe_turns(self) -> AsyncIterator[str]:
|
||||
"""Yields a stream of text transcriptions. Each transcription is a turn in the conversation.
|
||||
|
||||
This method is expected to return only after `close()` is called.
|
||||
"""
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
async def close(self) -> None:
|
||||
"""Closes the session."""
|
||||
pass
|
||||
|
||||
|
||||
@dataclass
|
||||
class STTModelSettings:
|
||||
"""Settings for a speech-to-text model."""
|
||||
|
||||
prompt: str | None = None
|
||||
"""Instructions for the model to follow."""
|
||||
|
||||
language: str | None = None
|
||||
"""The language of the audio input."""
|
||||
|
||||
temperature: float | None = None
|
||||
"""The temperature of the model."""
|
||||
|
||||
turn_detection: dict[str, Any] | None = None
|
||||
"""The turn detection settings for the model when using streamed audio input."""
|
||||
|
||||
|
||||
class STTModel(abc.ABC):
|
||||
"""A speech-to-text model that can convert audio input into text."""
|
||||
|
||||
@property
|
||||
@abc.abstractmethod
|
||||
def model_name(self) -> str:
|
||||
"""The name of the STT model."""
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
async def transcribe(
|
||||
self,
|
||||
input: AudioInput,
|
||||
settings: STTModelSettings,
|
||||
trace_include_sensitive_data: bool,
|
||||
trace_include_sensitive_audio_data: bool,
|
||||
) -> str:
|
||||
"""Given an audio input, produces a text transcription.
|
||||
|
||||
Args:
|
||||
input: The audio input to transcribe.
|
||||
settings: The settings to use for the transcription.
|
||||
trace_include_sensitive_data: Whether to include sensitive data in traces.
|
||||
trace_include_sensitive_audio_data: Whether to include sensitive audio data in traces.
|
||||
|
||||
Returns:
|
||||
The text transcription of the audio input.
|
||||
"""
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
async def create_session(
|
||||
self,
|
||||
input: StreamedAudioInput,
|
||||
settings: STTModelSettings,
|
||||
trace_include_sensitive_data: bool,
|
||||
trace_include_sensitive_audio_data: bool,
|
||||
) -> StreamedTranscriptionSession:
|
||||
"""Creates a new transcription session, which you can push audio to, and receive a stream
|
||||
of text transcriptions.
|
||||
|
||||
Args:
|
||||
input: The audio input to transcribe.
|
||||
settings: The settings to use for the transcription.
|
||||
trace_include_sensitive_data: Whether to include sensitive data in traces.
|
||||
trace_include_sensitive_audio_data: Whether to include sensitive audio data in traces.
|
||||
|
||||
Returns:
|
||||
A new transcription session.
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class VoiceModelProvider(abc.ABC):
|
||||
"""The base interface for a voice model provider.
|
||||
|
||||
A model provider is responsible for creating speech-to-text and text-to-speech models, given a
|
||||
name.
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def get_stt_model(self, model_name: str | None) -> STTModel:
|
||||
"""Get a speech-to-text model by name.
|
||||
|
||||
Args:
|
||||
model_name: The name of the model to get.
|
||||
|
||||
Returns:
|
||||
The speech-to-text model.
|
||||
"""
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
def get_tts_model(self, model_name: str | None) -> TTSModel:
|
||||
"""Get a text-to-speech model by name."""
|
||||
|
|
@ -0,0 +1,97 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import httpx
|
||||
from openai import AsyncOpenAI, DefaultAsyncHttpxClient
|
||||
|
||||
from ...models import _openai_shared
|
||||
from ..model import STTModel, TTSModel, VoiceModelProvider
|
||||
from .openai_stt import OpenAISTTModel
|
||||
from .openai_tts import OpenAITTSModel
|
||||
|
||||
_http_client: httpx.AsyncClient | None = None
|
||||
|
||||
|
||||
# If we create a new httpx client for each request, that would mean no sharing of connection pools,
|
||||
# which would mean worse latency and resource usage. So, we share the client across requests.
|
||||
def shared_http_client() -> httpx.AsyncClient:
|
||||
global _http_client
|
||||
if _http_client is None:
|
||||
_http_client = DefaultAsyncHttpxClient()
|
||||
return _http_client
|
||||
|
||||
|
||||
DEFAULT_STT_MODEL = "gpt-4o-transcribe"
|
||||
DEFAULT_TTS_MODEL = "gpt-4o-mini-tts"
|
||||
|
||||
|
||||
class OpenAIVoiceModelProvider(VoiceModelProvider):
|
||||
"""A voice model provider that uses OpenAI models."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
api_key: str | None = None,
|
||||
base_url: str | None = None,
|
||||
openai_client: AsyncOpenAI | None = None,
|
||||
organization: str | None = None,
|
||||
project: str | None = None,
|
||||
) -> None:
|
||||
"""Create a new OpenAI voice model provider.
|
||||
|
||||
Args:
|
||||
api_key: The API key to use for the OpenAI client. If not provided, we will use the
|
||||
default API key.
|
||||
base_url: The base URL to use for the OpenAI client. If not provided, we will use the
|
||||
default base URL.
|
||||
openai_client: An optional OpenAI client to use. If not provided, we will create a new
|
||||
OpenAI client using the api_key and base_url.
|
||||
organization: The organization to use for the OpenAI client.
|
||||
project: The project to use for the OpenAI client.
|
||||
"""
|
||||
if openai_client is not None:
|
||||
assert api_key is None and base_url is None, (
|
||||
"Don't provide api_key or base_url if you provide openai_client"
|
||||
)
|
||||
self._client: AsyncOpenAI | None = openai_client
|
||||
else:
|
||||
self._client = None
|
||||
self._stored_api_key = api_key
|
||||
self._stored_base_url = base_url
|
||||
self._stored_organization = organization
|
||||
self._stored_project = project
|
||||
|
||||
# We lazy load the client in case you never actually use OpenAIProvider(). Otherwise
|
||||
# AsyncOpenAI() raises an error if you don't have an API key set.
|
||||
def _get_client(self) -> AsyncOpenAI:
|
||||
if self._client is None:
|
||||
self._client = _openai_shared.get_default_openai_client() or AsyncOpenAI(
|
||||
api_key=self._stored_api_key or _openai_shared.get_default_openai_key(),
|
||||
base_url=self._stored_base_url,
|
||||
organization=self._stored_organization,
|
||||
project=self._stored_project,
|
||||
http_client=shared_http_client(),
|
||||
)
|
||||
|
||||
return self._client
|
||||
|
||||
def get_stt_model(self, model_name: str | None) -> STTModel:
|
||||
"""Get a speech-to-text model by name.
|
||||
|
||||
Args:
|
||||
model_name: The name of the model to get.
|
||||
|
||||
Returns:
|
||||
The speech-to-text model.
|
||||
"""
|
||||
return OpenAISTTModel(model_name or DEFAULT_STT_MODEL, self._get_client())
|
||||
|
||||
def get_tts_model(self, model_name: str | None) -> TTSModel:
|
||||
"""Get a text-to-speech model by name.
|
||||
|
||||
Args:
|
||||
model_name: The name of the model to get.
|
||||
|
||||
Returns:
|
||||
The text-to-speech model.
|
||||
"""
|
||||
return OpenAITTSModel(model_name or DEFAULT_TTS_MODEL, self._get_client())
|
||||
|
|
@ -0,0 +1,457 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
import time
|
||||
from collections.abc import AsyncIterator
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, cast
|
||||
|
||||
from openai import AsyncOpenAI
|
||||
|
||||
from agents.exceptions import AgentsException
|
||||
|
||||
from ... import _debug
|
||||
from ...logger import logger
|
||||
from ...tracing import Span, SpanError, TranscriptionSpanData, transcription_span
|
||||
from ..exceptions import STTWebsocketConnectionError
|
||||
from ..imports import np, npt, websockets
|
||||
from ..input import AudioInput, StreamedAudioInput
|
||||
from ..model import StreamedTranscriptionSession, STTModel, STTModelSettings
|
||||
|
||||
EVENT_INACTIVITY_TIMEOUT = 1000 # Timeout for inactivity in event processing
|
||||
SESSION_CREATION_TIMEOUT = 10 # Timeout waiting for session.created event
|
||||
SESSION_UPDATE_TIMEOUT = 10 # Timeout waiting for session.updated event
|
||||
|
||||
DEFAULT_TURN_DETECTION = {"type": "semantic_vad"}
|
||||
|
||||
|
||||
@dataclass
|
||||
class ErrorSentinel:
|
||||
error: Exception
|
||||
|
||||
|
||||
class SessionCompleteSentinel:
|
||||
pass
|
||||
|
||||
|
||||
class WebsocketDoneSentinel:
|
||||
pass
|
||||
|
||||
|
||||
def _audio_to_base64(audio_data: list[npt.NDArray[np.int16 | np.float32]]) -> str:
|
||||
concatenated_audio = np.concatenate(audio_data)
|
||||
if concatenated_audio.dtype == np.float32:
|
||||
# convert to int16
|
||||
concatenated_audio = np.clip(concatenated_audio, -1.0, 1.0)
|
||||
concatenated_audio = (concatenated_audio * 32767).astype(np.int16)
|
||||
audio_bytes = concatenated_audio.tobytes()
|
||||
return base64.b64encode(audio_bytes).decode("utf-8")
|
||||
|
||||
|
||||
async def _wait_for_event(
|
||||
event_queue: asyncio.Queue[dict[str, Any]], expected_types: list[str], timeout: float
|
||||
):
|
||||
"""
|
||||
Wait for an event from event_queue whose type is in expected_types within the specified timeout.
|
||||
"""
|
||||
start_time = time.time()
|
||||
while True:
|
||||
remaining = timeout - (time.time() - start_time)
|
||||
if remaining <= 0:
|
||||
raise TimeoutError(f"Timeout waiting for event(s): {expected_types}")
|
||||
evt = await asyncio.wait_for(event_queue.get(), timeout=remaining)
|
||||
evt_type = evt.get("type", "")
|
||||
if evt_type in expected_types:
|
||||
return evt
|
||||
elif evt_type == "error":
|
||||
raise Exception(f"Error event: {evt.get('error')}")
|
||||
|
||||
|
||||
class OpenAISTTTranscriptionSession(StreamedTranscriptionSession):
|
||||
"""A transcription session for OpenAI's STT model."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
input: StreamedAudioInput,
|
||||
client: AsyncOpenAI,
|
||||
model: str,
|
||||
settings: STTModelSettings,
|
||||
trace_include_sensitive_data: bool,
|
||||
trace_include_sensitive_audio_data: bool,
|
||||
):
|
||||
self.connected: bool = False
|
||||
self._client = client
|
||||
self._model = model
|
||||
self._settings = settings
|
||||
self._turn_detection = settings.turn_detection or DEFAULT_TURN_DETECTION
|
||||
self._trace_include_sensitive_data = trace_include_sensitive_data
|
||||
self._trace_include_sensitive_audio_data = trace_include_sensitive_audio_data
|
||||
|
||||
self._input_queue: asyncio.Queue[npt.NDArray[np.int16 | np.float32]] = input.queue
|
||||
self._output_queue: asyncio.Queue[str | ErrorSentinel | SessionCompleteSentinel] = (
|
||||
asyncio.Queue()
|
||||
)
|
||||
self._websocket: websockets.ClientConnection | None = None
|
||||
self._event_queue: asyncio.Queue[dict[str, Any] | WebsocketDoneSentinel] = asyncio.Queue()
|
||||
self._state_queue: asyncio.Queue[dict[str, Any]] = asyncio.Queue()
|
||||
self._turn_audio_buffer: list[npt.NDArray[np.int16 | np.float32]] = []
|
||||
self._tracing_span: Span[TranscriptionSpanData] | None = None
|
||||
|
||||
# tasks
|
||||
self._listener_task: asyncio.Task[Any] | None = None
|
||||
self._process_events_task: asyncio.Task[Any] | None = None
|
||||
self._stream_audio_task: asyncio.Task[Any] | None = None
|
||||
self._connection_task: asyncio.Task[Any] | None = None
|
||||
self._stored_exception: Exception | None = None
|
||||
|
||||
def _start_turn(self) -> None:
|
||||
self._tracing_span = transcription_span(
|
||||
model=self._model,
|
||||
model_config={
|
||||
"temperature": self._settings.temperature,
|
||||
"language": self._settings.language,
|
||||
"prompt": self._settings.prompt,
|
||||
"turn_detection": self._turn_detection,
|
||||
},
|
||||
)
|
||||
self._tracing_span.start()
|
||||
|
||||
def _end_turn(self, _transcript: str) -> None:
|
||||
if len(_transcript) < 1:
|
||||
return
|
||||
|
||||
if self._tracing_span:
|
||||
if self._trace_include_sensitive_audio_data:
|
||||
self._tracing_span.span_data.input = _audio_to_base64(self._turn_audio_buffer)
|
||||
|
||||
self._tracing_span.span_data.input_format = "pcm"
|
||||
|
||||
if self._trace_include_sensitive_data:
|
||||
self._tracing_span.span_data.output = _transcript
|
||||
|
||||
self._tracing_span.finish()
|
||||
self._turn_audio_buffer = []
|
||||
self._tracing_span = None
|
||||
|
||||
async def _event_listener(self) -> None:
|
||||
assert self._websocket is not None, "Websocket not initialized"
|
||||
|
||||
async for message in self._websocket:
|
||||
try:
|
||||
event = json.loads(message)
|
||||
|
||||
if event.get("type") == "error":
|
||||
raise STTWebsocketConnectionError(f"Error event: {event.get('error')}")
|
||||
|
||||
if event.get("type") in [
|
||||
"session.updated",
|
||||
"transcription_session.updated",
|
||||
"session.created",
|
||||
"transcription_session.created",
|
||||
]:
|
||||
await self._state_queue.put(event)
|
||||
|
||||
await self._event_queue.put(event)
|
||||
except Exception as e:
|
||||
await self._output_queue.put(ErrorSentinel(e))
|
||||
raise STTWebsocketConnectionError("Error parsing events") from e
|
||||
await self._event_queue.put(WebsocketDoneSentinel())
|
||||
|
||||
async def _configure_session(self) -> None:
|
||||
assert self._websocket is not None, "Websocket not initialized"
|
||||
await self._websocket.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "transcription_session.update",
|
||||
"session": {
|
||||
"input_audio_format": "pcm16",
|
||||
"input_audio_transcription": {"model": self._model},
|
||||
"turn_detection": self._turn_detection,
|
||||
},
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
async def _setup_connection(self, ws: websockets.ClientConnection) -> None:
|
||||
self._websocket = ws
|
||||
self._listener_task = asyncio.create_task(self._event_listener())
|
||||
|
||||
try:
|
||||
event = await _wait_for_event(
|
||||
self._state_queue,
|
||||
["session.created", "transcription_session.created"],
|
||||
SESSION_CREATION_TIMEOUT,
|
||||
)
|
||||
except TimeoutError as e:
|
||||
wrapped_err = STTWebsocketConnectionError(
|
||||
"Timeout waiting for transcription_session.created event"
|
||||
)
|
||||
await self._output_queue.put(ErrorSentinel(wrapped_err))
|
||||
raise wrapped_err from e
|
||||
except Exception as e:
|
||||
await self._output_queue.put(ErrorSentinel(e))
|
||||
raise e
|
||||
|
||||
await self._configure_session()
|
||||
|
||||
try:
|
||||
event = await _wait_for_event(
|
||||
self._state_queue,
|
||||
["session.updated", "transcription_session.updated"],
|
||||
SESSION_UPDATE_TIMEOUT,
|
||||
)
|
||||
if _debug.DONT_LOG_MODEL_DATA:
|
||||
logger.debug("Session updated")
|
||||
else:
|
||||
logger.debug(f"Session updated: {event}")
|
||||
except TimeoutError as e:
|
||||
wrapped_err = STTWebsocketConnectionError(
|
||||
"Timeout waiting for transcription_session.updated event"
|
||||
)
|
||||
await self._output_queue.put(ErrorSentinel(wrapped_err))
|
||||
raise wrapped_err from e
|
||||
except Exception as e:
|
||||
await self._output_queue.put(ErrorSentinel(e))
|
||||
raise
|
||||
|
||||
async def _handle_events(self) -> None:
|
||||
while True:
|
||||
try:
|
||||
event = await asyncio.wait_for(
|
||||
self._event_queue.get(), timeout=EVENT_INACTIVITY_TIMEOUT
|
||||
)
|
||||
if isinstance(event, WebsocketDoneSentinel):
|
||||
# processed all events and websocket is done
|
||||
break
|
||||
|
||||
event_type = event.get("type", "unknown")
|
||||
if event_type == "conversation.item.input_audio_transcription.completed":
|
||||
transcript = cast(str, event.get("transcript", ""))
|
||||
if len(transcript) > 0:
|
||||
self._end_turn(transcript)
|
||||
self._start_turn()
|
||||
await self._output_queue.put(transcript)
|
||||
await asyncio.sleep(0) # yield control
|
||||
except asyncio.TimeoutError:
|
||||
# No new events for a while. Assume the session is done.
|
||||
break
|
||||
except Exception as e:
|
||||
await self._output_queue.put(ErrorSentinel(e))
|
||||
raise e
|
||||
await self._output_queue.put(SessionCompleteSentinel())
|
||||
|
||||
async def _stream_audio(
|
||||
self, audio_queue: asyncio.Queue[npt.NDArray[np.int16 | np.float32]]
|
||||
) -> None:
|
||||
assert self._websocket is not None, "Websocket not initialized"
|
||||
self._start_turn()
|
||||
while True:
|
||||
buffer = await audio_queue.get()
|
||||
if buffer is None:
|
||||
break
|
||||
|
||||
self._turn_audio_buffer.append(buffer)
|
||||
try:
|
||||
await self._websocket.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "input_audio_buffer.append",
|
||||
"audio": base64.b64encode(buffer.tobytes()).decode("utf-8"),
|
||||
}
|
||||
)
|
||||
)
|
||||
except websockets.ConnectionClosed:
|
||||
break
|
||||
except Exception as e:
|
||||
await self._output_queue.put(ErrorSentinel(e))
|
||||
raise e
|
||||
|
||||
await asyncio.sleep(0) # yield control
|
||||
|
||||
async def _process_websocket_connection(self) -> None:
|
||||
try:
|
||||
async with websockets.connect(
|
||||
"wss://api.openai.com/v1/realtime?intent=transcription",
|
||||
additional_headers={
|
||||
"Authorization": f"Bearer {self._client.api_key}",
|
||||
"OpenAI-Beta": "realtime=v1",
|
||||
"OpenAI-Log-Session": "1",
|
||||
},
|
||||
) as ws:
|
||||
await self._setup_connection(ws)
|
||||
self._process_events_task = asyncio.create_task(self._handle_events())
|
||||
self._stream_audio_task = asyncio.create_task(self._stream_audio(self._input_queue))
|
||||
self.connected = True
|
||||
if self._listener_task:
|
||||
await self._listener_task
|
||||
else:
|
||||
logger.error("Listener task not initialized")
|
||||
raise AgentsException("Listener task not initialized")
|
||||
except Exception as e:
|
||||
await self._output_queue.put(ErrorSentinel(e))
|
||||
raise e
|
||||
|
||||
def _check_errors(self) -> None:
|
||||
if self._connection_task and self._connection_task.done():
|
||||
exc = self._connection_task.exception()
|
||||
if exc and isinstance(exc, Exception):
|
||||
self._stored_exception = exc
|
||||
|
||||
if self._process_events_task and self._process_events_task.done():
|
||||
exc = self._process_events_task.exception()
|
||||
if exc and isinstance(exc, Exception):
|
||||
self._stored_exception = exc
|
||||
|
||||
if self._stream_audio_task and self._stream_audio_task.done():
|
||||
exc = self._stream_audio_task.exception()
|
||||
if exc and isinstance(exc, Exception):
|
||||
self._stored_exception = exc
|
||||
|
||||
if self._listener_task and self._listener_task.done():
|
||||
exc = self._listener_task.exception()
|
||||
if exc and isinstance(exc, Exception):
|
||||
self._stored_exception = exc
|
||||
|
||||
def _cleanup_tasks(self) -> None:
|
||||
if self._listener_task and not self._listener_task.done():
|
||||
self._listener_task.cancel()
|
||||
|
||||
if self._process_events_task and not self._process_events_task.done():
|
||||
self._process_events_task.cancel()
|
||||
|
||||
if self._stream_audio_task and not self._stream_audio_task.done():
|
||||
self._stream_audio_task.cancel()
|
||||
|
||||
if self._connection_task and not self._connection_task.done():
|
||||
self._connection_task.cancel()
|
||||
|
||||
async def transcribe_turns(self) -> AsyncIterator[str]:
|
||||
self._connection_task = asyncio.create_task(self._process_websocket_connection())
|
||||
|
||||
while True:
|
||||
try:
|
||||
turn = await self._output_queue.get()
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
|
||||
if (
|
||||
turn is None
|
||||
or isinstance(turn, ErrorSentinel)
|
||||
or isinstance(turn, SessionCompleteSentinel)
|
||||
):
|
||||
self._output_queue.task_done()
|
||||
break
|
||||
yield turn
|
||||
self._output_queue.task_done()
|
||||
|
||||
if self._tracing_span:
|
||||
self._end_turn("")
|
||||
|
||||
if self._websocket:
|
||||
await self._websocket.close()
|
||||
|
||||
self._check_errors()
|
||||
if self._stored_exception:
|
||||
raise self._stored_exception
|
||||
|
||||
async def close(self) -> None:
|
||||
if self._websocket:
|
||||
await self._websocket.close()
|
||||
|
||||
self._cleanup_tasks()
|
||||
|
||||
|
||||
class OpenAISTTModel(STTModel):
|
||||
"""A speech-to-text model for OpenAI."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model: str,
|
||||
openai_client: AsyncOpenAI,
|
||||
):
|
||||
"""Create a new OpenAI speech-to-text model.
|
||||
|
||||
Args:
|
||||
model: The name of the model to use.
|
||||
openai_client: The OpenAI client to use.
|
||||
"""
|
||||
self.model = model
|
||||
self._client = openai_client
|
||||
|
||||
@property
|
||||
def model_name(self) -> str:
|
||||
return self.model
|
||||
|
||||
def _non_null_or_not_given(self, value: Any) -> Any:
|
||||
return value if value is not None else None # NOT_GIVEN
|
||||
|
||||
async def transcribe(
|
||||
self,
|
||||
input: AudioInput,
|
||||
settings: STTModelSettings,
|
||||
trace_include_sensitive_data: bool,
|
||||
trace_include_sensitive_audio_data: bool,
|
||||
) -> str:
|
||||
"""Transcribe an audio input.
|
||||
|
||||
Args:
|
||||
input: The audio input to transcribe.
|
||||
settings: The settings to use for the transcription.
|
||||
|
||||
Returns:
|
||||
The transcribed text.
|
||||
"""
|
||||
with transcription_span(
|
||||
model=self.model,
|
||||
input=input.to_base64() if trace_include_sensitive_audio_data else "",
|
||||
input_format="pcm",
|
||||
model_config={
|
||||
"temperature": self._non_null_or_not_given(settings.temperature),
|
||||
"language": self._non_null_or_not_given(settings.language),
|
||||
"prompt": self._non_null_or_not_given(settings.prompt),
|
||||
},
|
||||
) as span:
|
||||
try:
|
||||
response = await self._client.audio.transcriptions.create(
|
||||
model=self.model,
|
||||
file=input.to_audio_file(),
|
||||
prompt=self._non_null_or_not_given(settings.prompt),
|
||||
language=self._non_null_or_not_given(settings.language),
|
||||
temperature=self._non_null_or_not_given(settings.temperature),
|
||||
)
|
||||
if trace_include_sensitive_data:
|
||||
span.span_data.output = response.text
|
||||
return response.text
|
||||
except Exception as e:
|
||||
span.span_data.output = ""
|
||||
span.set_error(SpanError(message=str(e), data={}))
|
||||
raise e
|
||||
|
||||
async def create_session(
|
||||
self,
|
||||
input: StreamedAudioInput,
|
||||
settings: STTModelSettings,
|
||||
trace_include_sensitive_data: bool,
|
||||
trace_include_sensitive_audio_data: bool,
|
||||
) -> StreamedTranscriptionSession:
|
||||
"""Create a new transcription session.
|
||||
|
||||
Args:
|
||||
input: The audio input to transcribe.
|
||||
settings: The settings to use for the transcription.
|
||||
trace_include_sensitive_data: Whether to include sensitive data in traces.
|
||||
trace_include_sensitive_audio_data: Whether to include sensitive audio data in traces.
|
||||
|
||||
Returns:
|
||||
A new transcription session.
|
||||
"""
|
||||
return OpenAISTTTranscriptionSession(
|
||||
input,
|
||||
self._client,
|
||||
self.model,
|
||||
settings,
|
||||
trace_include_sensitive_data,
|
||||
trace_include_sensitive_audio_data,
|
||||
)
|
||||
|
|
@ -0,0 +1,54 @@
|
|||
from collections.abc import AsyncIterator
|
||||
from typing import Literal
|
||||
|
||||
from openai import AsyncOpenAI
|
||||
|
||||
from ..model import TTSModel, TTSModelSettings
|
||||
|
||||
DEFAULT_VOICE: Literal["ash"] = "ash"
|
||||
|
||||
|
||||
class OpenAITTSModel(TTSModel):
|
||||
"""A text-to-speech model for OpenAI."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model: str,
|
||||
openai_client: AsyncOpenAI,
|
||||
):
|
||||
"""Create a new OpenAI text-to-speech model.
|
||||
|
||||
Args:
|
||||
model: The name of the model to use.
|
||||
openai_client: The OpenAI client to use.
|
||||
"""
|
||||
self.model = model
|
||||
self._client = openai_client
|
||||
|
||||
@property
|
||||
def model_name(self) -> str:
|
||||
return self.model
|
||||
|
||||
async def run(self, text: str, settings: TTSModelSettings) -> AsyncIterator[bytes]:
|
||||
"""Run the text-to-speech model.
|
||||
|
||||
Args:
|
||||
text: The text to convert to speech.
|
||||
settings: The settings to use for the text-to-speech model.
|
||||
|
||||
Returns:
|
||||
An iterator of audio chunks.
|
||||
"""
|
||||
response = self._client.audio.speech.with_streaming_response.create(
|
||||
model=self.model,
|
||||
voice=settings.voice or DEFAULT_VOICE,
|
||||
input=text,
|
||||
response_format="pcm",
|
||||
extra_body={
|
||||
"instructions": settings.instructions,
|
||||
},
|
||||
)
|
||||
|
||||
async with response as stream:
|
||||
async for chunk in stream.iter_bytes(chunk_size=1024):
|
||||
yield chunk
|
||||
|
|
@ -0,0 +1,151 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
from .._run_impl import TraceCtxManager
|
||||
from ..exceptions import UserError
|
||||
from ..logger import logger
|
||||
from .input import AudioInput, StreamedAudioInput
|
||||
from .model import STTModel, TTSModel
|
||||
from .pipeline_config import VoicePipelineConfig
|
||||
from .result import StreamedAudioResult
|
||||
from .workflow import VoiceWorkflowBase
|
||||
|
||||
|
||||
class VoicePipeline:
|
||||
"""An opinionated voice agent pipeline. It works in three steps:
|
||||
1. Transcribe audio input into text.
|
||||
2. Run the provided `workflow`, which produces a sequence of text responses.
|
||||
3. Convert the text responses into streaming audio output.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
workflow: VoiceWorkflowBase,
|
||||
stt_model: STTModel | str | None = None,
|
||||
tts_model: TTSModel | str | None = None,
|
||||
config: VoicePipelineConfig | None = None,
|
||||
):
|
||||
"""Create a new voice pipeline.
|
||||
|
||||
Args:
|
||||
workflow: The workflow to run. See `VoiceWorkflowBase`.
|
||||
stt_model: The speech-to-text model to use. If not provided, a default OpenAI
|
||||
model will be used.
|
||||
tts_model: The text-to-speech model to use. If not provided, a default OpenAI
|
||||
model will be used.
|
||||
config: The pipeline configuration. If not provided, a default configuration will be
|
||||
used.
|
||||
"""
|
||||
self.workflow = workflow
|
||||
self.stt_model = stt_model if isinstance(stt_model, STTModel) else None
|
||||
self.tts_model = tts_model if isinstance(tts_model, TTSModel) else None
|
||||
self._stt_model_name = stt_model if isinstance(stt_model, str) else None
|
||||
self._tts_model_name = tts_model if isinstance(tts_model, str) else None
|
||||
self.config = config or VoicePipelineConfig()
|
||||
|
||||
async def run(self, audio_input: AudioInput | StreamedAudioInput) -> StreamedAudioResult:
|
||||
"""Run the voice pipeline.
|
||||
|
||||
Args:
|
||||
audio_input: The audio input to process. This can either be an `AudioInput` instance,
|
||||
which is a single static buffer, or a `StreamedAudioInput` instance, which is a
|
||||
stream of audio data that you can append to.
|
||||
|
||||
Returns:
|
||||
A `StreamedAudioResult` instance. You can use this object to stream audio events and
|
||||
play them out.
|
||||
"""
|
||||
if isinstance(audio_input, AudioInput):
|
||||
return await self._run_single_turn(audio_input)
|
||||
elif isinstance(audio_input, StreamedAudioInput):
|
||||
return await self._run_multi_turn(audio_input)
|
||||
else:
|
||||
raise UserError(f"Unsupported audio input type: {type(audio_input)}")
|
||||
|
||||
def _get_tts_model(self) -> TTSModel:
|
||||
if not self.tts_model:
|
||||
self.tts_model = self.config.model_provider.get_tts_model(self._tts_model_name)
|
||||
return self.tts_model
|
||||
|
||||
def _get_stt_model(self) -> STTModel:
|
||||
if not self.stt_model:
|
||||
self.stt_model = self.config.model_provider.get_stt_model(self._stt_model_name)
|
||||
return self.stt_model
|
||||
|
||||
async def _process_audio_input(self, audio_input: AudioInput) -> str:
|
||||
model = self._get_stt_model()
|
||||
return await model.transcribe(
|
||||
audio_input,
|
||||
self.config.stt_settings,
|
||||
self.config.trace_include_sensitive_data,
|
||||
self.config.trace_include_sensitive_audio_data,
|
||||
)
|
||||
|
||||
async def _run_single_turn(self, audio_input: AudioInput) -> StreamedAudioResult:
|
||||
# Since this is single turn, we can use the TraceCtxManager to manage starting/ending the
|
||||
# trace
|
||||
with TraceCtxManager(
|
||||
workflow_name=self.config.workflow_name or "Voice Agent",
|
||||
trace_id=None, # Automatically generated
|
||||
group_id=self.config.group_id,
|
||||
metadata=self.config.trace_metadata,
|
||||
disabled=self.config.tracing_disabled,
|
||||
):
|
||||
input_text = await self._process_audio_input(audio_input)
|
||||
|
||||
output = StreamedAudioResult(
|
||||
self._get_tts_model(), self.config.tts_settings, self.config
|
||||
)
|
||||
|
||||
async def stream_events():
|
||||
try:
|
||||
async for text_event in self.workflow.run(input_text):
|
||||
await output._add_text(text_event)
|
||||
await output._turn_done()
|
||||
await output._done()
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing single turn: {e}")
|
||||
await output._add_error(e)
|
||||
raise e
|
||||
|
||||
output._set_task(asyncio.create_task(stream_events()))
|
||||
return output
|
||||
|
||||
async def _run_multi_turn(self, audio_input: StreamedAudioInput) -> StreamedAudioResult:
|
||||
with TraceCtxManager(
|
||||
workflow_name=self.config.workflow_name or "Voice Agent",
|
||||
trace_id=None,
|
||||
group_id=self.config.group_id,
|
||||
metadata=self.config.trace_metadata,
|
||||
disabled=self.config.tracing_disabled,
|
||||
):
|
||||
output = StreamedAudioResult(
|
||||
self._get_tts_model(), self.config.tts_settings, self.config
|
||||
)
|
||||
|
||||
transcription_session = await self._get_stt_model().create_session(
|
||||
audio_input,
|
||||
self.config.stt_settings,
|
||||
self.config.trace_include_sensitive_data,
|
||||
self.config.trace_include_sensitive_audio_data,
|
||||
)
|
||||
|
||||
async def process_turns():
|
||||
try:
|
||||
async for input_text in transcription_session.transcribe_turns():
|
||||
result = self.workflow.run(input_text)
|
||||
async for text_event in result:
|
||||
await output._add_text(text_event)
|
||||
await output._turn_done()
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing turns: {e}")
|
||||
await output._add_error(e)
|
||||
raise e
|
||||
finally:
|
||||
await transcription_session.close()
|
||||
await output._done()
|
||||
|
||||
output._set_task(asyncio.create_task(process_turns()))
|
||||
return output
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from ..tracing.util import gen_group_id
|
||||
from .model import STTModelSettings, TTSModelSettings, VoiceModelProvider
|
||||
from .models.openai_model_provider import OpenAIVoiceModelProvider
|
||||
|
||||
|
||||
@dataclass
|
||||
class VoicePipelineConfig:
|
||||
"""Configuration for a `VoicePipeline`."""
|
||||
|
||||
model_provider: VoiceModelProvider = field(default_factory=OpenAIVoiceModelProvider)
|
||||
"""The voice model provider to use for the pipeline. Defaults to OpenAI."""
|
||||
|
||||
tracing_disabled: bool = False
|
||||
"""Whether to disable tracing of the pipeline. Defaults to `False`."""
|
||||
|
||||
trace_include_sensitive_data: bool = True
|
||||
"""Whether to include sensitive data in traces. Defaults to `True`. This is specifically for the
|
||||
voice pipeline, and not for anything that goes on inside your Workflow."""
|
||||
|
||||
trace_include_sensitive_audio_data: bool = True
|
||||
"""Whether to include audio data in traces. Defaults to `True`."""
|
||||
|
||||
workflow_name: str = "Voice Agent"
|
||||
"""The name of the workflow to use for tracing. Defaults to `Voice Agent`."""
|
||||
|
||||
group_id: str = field(default_factory=gen_group_id)
|
||||
"""
|
||||
A grouping identifier to use for tracing, to link multiple traces from the same conversation
|
||||
or process. If not provided, we will create a random group ID.
|
||||
"""
|
||||
|
||||
trace_metadata: dict[str, Any] | None = None
|
||||
"""
|
||||
An optional dictionary of additional metadata to include with the trace.
|
||||
"""
|
||||
|
||||
stt_settings: STTModelSettings = field(default_factory=STTModelSettings)
|
||||
"""The settings to use for the STT model."""
|
||||
|
||||
tts_settings: TTSModelSettings = field(default_factory=TTSModelSettings)
|
||||
"""The settings to use for the TTS model."""
|
||||
|
|
@ -0,0 +1,287 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any
|
||||
|
||||
from ..exceptions import UserError
|
||||
from ..logger import logger
|
||||
from ..tracing import Span, SpeechGroupSpanData, speech_group_span, speech_span
|
||||
from ..tracing.util import time_iso
|
||||
from .events import (
|
||||
VoiceStreamEvent,
|
||||
VoiceStreamEventAudio,
|
||||
VoiceStreamEventError,
|
||||
VoiceStreamEventLifecycle,
|
||||
)
|
||||
from .imports import np, npt
|
||||
from .model import TTSModel, TTSModelSettings
|
||||
from .pipeline_config import VoicePipelineConfig
|
||||
|
||||
|
||||
def _audio_to_base64(audio_data: list[bytes]) -> str:
|
||||
joined_audio_data = b"".join(audio_data)
|
||||
return base64.b64encode(joined_audio_data).decode("utf-8")
|
||||
|
||||
|
||||
class StreamedAudioResult:
|
||||
"""The output of a `VoicePipeline`. Streams events and audio data as they're generated."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tts_model: TTSModel,
|
||||
tts_settings: TTSModelSettings,
|
||||
voice_pipeline_config: VoicePipelineConfig,
|
||||
):
|
||||
"""Create a new `StreamedAudioResult` instance.
|
||||
|
||||
Args:
|
||||
tts_model: The TTS model to use.
|
||||
tts_settings: The TTS settings to use.
|
||||
voice_pipeline_config: The voice pipeline config to use.
|
||||
"""
|
||||
self.tts_model = tts_model
|
||||
self.tts_settings = tts_settings
|
||||
self.total_output_text = ""
|
||||
self.instructions = tts_settings.instructions
|
||||
self.text_generation_task: asyncio.Task[Any] | None = None
|
||||
|
||||
self._voice_pipeline_config = voice_pipeline_config
|
||||
self._text_buffer = ""
|
||||
self._turn_text_buffer = ""
|
||||
self._queue: asyncio.Queue[VoiceStreamEvent] = asyncio.Queue()
|
||||
self._tasks: list[asyncio.Task[Any]] = []
|
||||
self._ordered_tasks: list[
|
||||
asyncio.Queue[VoiceStreamEvent | None]
|
||||
] = [] # New: list to hold local queues for each text segment
|
||||
self._dispatcher_task: asyncio.Task[Any] | None = (
|
||||
None # Task to dispatch audio chunks in order
|
||||
)
|
||||
|
||||
self._done_processing = False
|
||||
self._buffer_size = tts_settings.buffer_size
|
||||
self._started_processing_turn = False
|
||||
self._first_byte_received = False
|
||||
self._generation_start_time: str | None = None
|
||||
self._completed_session = False
|
||||
self._stored_exception: BaseException | None = None
|
||||
self._tracing_span: Span[SpeechGroupSpanData] | None = None
|
||||
|
||||
async def _start_turn(self):
|
||||
if self._started_processing_turn:
|
||||
return
|
||||
|
||||
self._tracing_span = speech_group_span()
|
||||
self._tracing_span.start()
|
||||
self._started_processing_turn = True
|
||||
self._first_byte_received = False
|
||||
self._generation_start_time = time_iso()
|
||||
await self._queue.put(VoiceStreamEventLifecycle(event="turn_started"))
|
||||
|
||||
def _set_task(self, task: asyncio.Task[Any]):
|
||||
self.text_generation_task = task
|
||||
|
||||
async def _add_error(self, error: Exception):
|
||||
await self._queue.put(VoiceStreamEventError(error))
|
||||
|
||||
def _transform_audio_buffer(
|
||||
self, buffer: list[bytes], output_dtype: npt.DTypeLike
|
||||
) -> npt.NDArray[np.int16 | np.float32]:
|
||||
np_array = np.frombuffer(b"".join(buffer), dtype=np.int16)
|
||||
|
||||
if output_dtype == np.int16:
|
||||
return np_array
|
||||
elif output_dtype == np.float32:
|
||||
return (np_array.astype(np.float32) / 32767.0).reshape(-1, 1)
|
||||
else:
|
||||
raise UserError("Invalid output dtype")
|
||||
|
||||
async def _stream_audio(
|
||||
self,
|
||||
text: str,
|
||||
local_queue: asyncio.Queue[VoiceStreamEvent | None],
|
||||
finish_turn: bool = False,
|
||||
):
|
||||
with speech_span(
|
||||
model=self.tts_model.model_name,
|
||||
input=text if self._voice_pipeline_config.trace_include_sensitive_data else "",
|
||||
model_config={
|
||||
"voice": self.tts_settings.voice,
|
||||
"instructions": self.instructions,
|
||||
"speed": self.tts_settings.speed,
|
||||
},
|
||||
output_format="pcm",
|
||||
parent=self._tracing_span,
|
||||
) as tts_span:
|
||||
try:
|
||||
first_byte_received = False
|
||||
buffer: list[bytes] = []
|
||||
full_audio_data: list[bytes] = []
|
||||
|
||||
async for chunk in self.tts_model.run(text, self.tts_settings):
|
||||
if not first_byte_received:
|
||||
first_byte_received = True
|
||||
tts_span.span_data.first_content_at = time_iso()
|
||||
|
||||
if chunk:
|
||||
buffer.append(chunk)
|
||||
full_audio_data.append(chunk)
|
||||
if len(buffer) >= self._buffer_size:
|
||||
audio_np = self._transform_audio_buffer(buffer, self.tts_settings.dtype)
|
||||
if self.tts_settings.transform_data:
|
||||
audio_np = self.tts_settings.transform_data(audio_np)
|
||||
await local_queue.put(
|
||||
VoiceStreamEventAudio(data=audio_np)
|
||||
) # Use local queue
|
||||
buffer = []
|
||||
if buffer:
|
||||
audio_np = self._transform_audio_buffer(buffer, self.tts_settings.dtype)
|
||||
if self.tts_settings.transform_data:
|
||||
audio_np = self.tts_settings.transform_data(audio_np)
|
||||
await local_queue.put(VoiceStreamEventAudio(data=audio_np)) # Use local queue
|
||||
|
||||
if self._voice_pipeline_config.trace_include_sensitive_audio_data:
|
||||
tts_span.span_data.output = _audio_to_base64(full_audio_data)
|
||||
else:
|
||||
tts_span.span_data.output = ""
|
||||
|
||||
if finish_turn:
|
||||
await local_queue.put(VoiceStreamEventLifecycle(event="turn_ended"))
|
||||
else:
|
||||
await local_queue.put(None) # Signal completion for this segment
|
||||
except Exception as e:
|
||||
tts_span.set_error(
|
||||
{
|
||||
"message": str(e),
|
||||
"data": {
|
||||
"text": text
|
||||
if self._voice_pipeline_config.trace_include_sensitive_data
|
||||
else "",
|
||||
},
|
||||
}
|
||||
)
|
||||
logger.error(f"Error streaming audio: {e}")
|
||||
|
||||
# Signal completion for whole session because of error
|
||||
await local_queue.put(VoiceStreamEventLifecycle(event="session_ended"))
|
||||
raise e
|
||||
|
||||
async def _add_text(self, text: str):
|
||||
await self._start_turn()
|
||||
|
||||
self._text_buffer += text
|
||||
self.total_output_text += text
|
||||
self._turn_text_buffer += text
|
||||
|
||||
combined_sentences, self._text_buffer = self.tts_settings.text_splitter(self._text_buffer)
|
||||
|
||||
if len(combined_sentences) >= 20:
|
||||
local_queue: asyncio.Queue[VoiceStreamEvent | None] = asyncio.Queue()
|
||||
self._ordered_tasks.append(local_queue)
|
||||
self._tasks.append(
|
||||
asyncio.create_task(self._stream_audio(combined_sentences, local_queue))
|
||||
)
|
||||
if self._dispatcher_task is None:
|
||||
self._dispatcher_task = asyncio.create_task(self._dispatch_audio())
|
||||
|
||||
async def _turn_done(self):
|
||||
if self._text_buffer:
|
||||
local_queue: asyncio.Queue[VoiceStreamEvent | None] = asyncio.Queue()
|
||||
self._ordered_tasks.append(local_queue) # Append the local queue for the final segment
|
||||
self._tasks.append(
|
||||
asyncio.create_task(
|
||||
self._stream_audio(self._text_buffer, local_queue, finish_turn=True)
|
||||
)
|
||||
)
|
||||
self._text_buffer = ""
|
||||
self._done_processing = True
|
||||
if self._dispatcher_task is None:
|
||||
self._dispatcher_task = asyncio.create_task(self._dispatch_audio())
|
||||
await asyncio.gather(*self._tasks)
|
||||
|
||||
def _finish_turn(self):
|
||||
if self._tracing_span:
|
||||
if self._voice_pipeline_config.trace_include_sensitive_data:
|
||||
self._tracing_span.span_data.input = self._turn_text_buffer
|
||||
else:
|
||||
self._tracing_span.span_data.input = ""
|
||||
|
||||
self._tracing_span.finish()
|
||||
self._tracing_span = None
|
||||
self._turn_text_buffer = ""
|
||||
self._started_processing_turn = False
|
||||
|
||||
async def _done(self):
|
||||
self._completed_session = True
|
||||
await self._wait_for_completion()
|
||||
|
||||
async def _dispatch_audio(self):
|
||||
# Dispatch audio chunks from each segment in the order they were added
|
||||
while True:
|
||||
if len(self._ordered_tasks) == 0:
|
||||
if self._completed_session:
|
||||
break
|
||||
await asyncio.sleep(0)
|
||||
continue
|
||||
local_queue = self._ordered_tasks.pop(0)
|
||||
while True:
|
||||
chunk = await local_queue.get()
|
||||
if chunk is None:
|
||||
break
|
||||
await self._queue.put(chunk)
|
||||
if isinstance(chunk, VoiceStreamEventLifecycle):
|
||||
local_queue.task_done()
|
||||
if chunk.event == "turn_ended":
|
||||
self._finish_turn()
|
||||
break
|
||||
await self._queue.put(VoiceStreamEventLifecycle(event="session_ended"))
|
||||
|
||||
async def _wait_for_completion(self):
|
||||
tasks: list[asyncio.Task[Any]] = self._tasks
|
||||
if self._dispatcher_task is not None:
|
||||
tasks.append(self._dispatcher_task)
|
||||
await asyncio.gather(*tasks)
|
||||
|
||||
def _cleanup_tasks(self):
|
||||
self._finish_turn()
|
||||
|
||||
for task in self._tasks:
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
|
||||
if self._dispatcher_task and not self._dispatcher_task.done():
|
||||
self._dispatcher_task.cancel()
|
||||
|
||||
if self.text_generation_task and not self.text_generation_task.done():
|
||||
self.text_generation_task.cancel()
|
||||
|
||||
def _check_errors(self):
|
||||
for task in self._tasks:
|
||||
if task.done():
|
||||
if task.exception():
|
||||
self._stored_exception = task.exception()
|
||||
break
|
||||
|
||||
async def stream(self) -> AsyncIterator[VoiceStreamEvent]:
|
||||
"""Stream the events and audio data as they're generated."""
|
||||
while True:
|
||||
try:
|
||||
event = await self._queue.get()
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
if isinstance(event, VoiceStreamEventError):
|
||||
self._stored_exception = event.error
|
||||
logger.error(f"Error processing output: {event.error}")
|
||||
break
|
||||
if event is None:
|
||||
break
|
||||
yield event
|
||||
if event.type == "voice_stream_event_lifecycle" and event.event == "session_ended":
|
||||
break
|
||||
|
||||
self._check_errors()
|
||||
self._cleanup_tasks()
|
||||
|
||||
if self._stored_exception:
|
||||
raise self._stored_exception
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
import re
|
||||
from typing import Callable
|
||||
|
||||
|
||||
def get_sentence_based_splitter(
|
||||
min_sentence_length: int = 20,
|
||||
) -> Callable[[str], tuple[str, str]]:
|
||||
"""Returns a function that splits text into chunks based on sentence boundaries.
|
||||
|
||||
Args:
|
||||
min_sentence_length: The minimum length of a sentence to be included in a chunk.
|
||||
|
||||
Returns:
|
||||
A function that splits text into chunks based on sentence boundaries.
|
||||
"""
|
||||
|
||||
def sentence_based_text_splitter(text_buffer: str) -> tuple[str, str]:
|
||||
"""
|
||||
A function to split the text into chunks. This is useful if you want to split the text into
|
||||
chunks before sending it to the TTS model rather than waiting for the whole text to be
|
||||
processed.
|
||||
|
||||
Args:
|
||||
text_buffer: The text to split.
|
||||
|
||||
Returns:
|
||||
A tuple of the text to process and the remaining text buffer.
|
||||
"""
|
||||
sentences = re.split(r"(?<=[.!?])\s+", text_buffer.strip())
|
||||
if len(sentences) >= 1:
|
||||
combined_sentences = " ".join(sentences[:-1])
|
||||
if len(combined_sentences) >= min_sentence_length:
|
||||
remaining_text_buffer = sentences[-1]
|
||||
return combined_sentences, remaining_text_buffer
|
||||
return "", text_buffer
|
||||
|
||||
return sentence_based_text_splitter
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue