Skip to content

Commit

Permalink
Dev/realtime tts (#355)
Browse files Browse the repository at this point in the history 
* support non audio

* add input transcript switch

* fix connection

* fix

* fix

* fix weather check failure
  • Loading branch information
@tomasliu-agora
tomasliu-agora authored Oct 22, 2024
1 parent 1c2c235 commit 41d1a26
Show file tree
Hide file tree
Showing 6 changed files with 415 additions and 28 deletions.
293 changes: 293 additions & 0 deletions agents/property.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2860,6 +2860,299 @@
]
}
]
},
{
"name": "va.openai.v2v.fish",
"auto_start": false,
"nodes": [
{
"type": "extension",
"extension_group": "rtc",
"addon": "agora_rtc",
"name": "agora_rtc",
"property": {
"app_id": "${env:AGORA_APP_ID}",
"token": "",
"channel": "ten_agent_test",
"stream_id": 1234,
"remote_stream_id": 123,
"subscribe_audio": true,
"publish_audio": true,
"publish_data": true,
"subscribe_audio_sample_rate": 24000,
"enable_agora_asr": true,
"agora_asr_vendor_name": "microsoft",
"agora_asr_language": "en-US",
"agora_asr_vendor_key": "${env:AZURE_STT_KEY}",
"agora_asr_vendor_region": "${env:AZURE_STT_REGION}",
"agora_asr_session_control_file_path": "session_control.conf"
}
},
{
"type": "extension",
"extension_group": "llm",
"addon": "openai_v2v_python",
"name": "openai_v2v_python",
"property": {
"api_key": "${env:OPENAI_REALTIME_API_KEY}",
"temperature": 0.9,
"model": "gpt-4o-realtime-preview",
"max_tokens": 2048,
"audio_out": false,
"input_transcript": false,
"language": "en-US",
"server_vad": true,
"dump": true,
"history": 10
}
},
{
"type": "extension",
"extension_group": "tts",
"addon": "fish_audio_tts",
"name": "fish_audio_tts",
"property": {
"api_key": "${env:FISH_AUDIO_TTS_KEY}",
"model_id": "d8639b5cc95548f5afbcfe22d3ba5ce5",
"optimize_streaming_latency": true,
"request_timeout_seconds": 30,
"base_url": "https://api.fish.audio"
}
},
{
"type": "extension",
"extension_group": "transcriber",
"addon": "message_collector",
"name": "message_collector"
},
{
"type": "extension",
"extension_group": "tools",
"addon": "weatherapi_tool_python",
"name": "weatherapi_tool_python",
"property": {
"api_key": "${env:WEATHERAPI_API_KEY}"
}
},
{
"type": "extension",
"extension_group": "tools",
"addon": "bingsearch_tool_python",
"name": "bingsearch_tool_python",
"property": {
"api_key": "${env:BING_API_KEY}"
}
}
],
"connections": [
{
"extension_group": "rtc",
"extension": "agora_rtc",
"audio_frame": [
{
"name": "pcm_frame",
"dest": [
{
"extension_group": "llm",
"extension": "openai_v2v_python"
}
]
}
],
"data": [
{
"name": "text_data",
"dest": [
{
"extension_group": "transcriber",
"extension": "message_collector"
}
]
}
]
},
{
"extension_group": "tools",
"extension": "weatherapi_tool_python",
"cmd": [
{
"name": "tool_register",
"dest": [
{
"extension_group": "llm",
"extension": "openai_v2v_python"
}
]
}
]
},
{
"extension_group": "tools",
"extension": "bingsearch_tool_python",
"cmd": [
{
"name": "tool_register",
"dest": [
{
"extension_group": "llm",
"extension": "openai_v2v_python"
}
]
}
]
},
{
"extension_group": "llm",
"extension": "openai_v2v_python",
"data": [
{
"name": "text_data",
"dest": [
{
"extension_group": "transcriber",
"extension": "message_collector"
},
{
"extension_group": "tts",
"extension": "fish_audio_tts"
}
]
}
],
"cmd": [
{
"name": "flush",
"dest": [
{
"extension_group": "tts",
"extension": "fish_audio_tts"
}
]
},
{
"name": "tool_call_get_current_weather",
"dest": [
{
"extension_group": "tools",
"extension": "weatherapi_tool_python",
"msg_conversion": {
"type": "per_property",
"keep_original": true,
"rules": [
{
"path": "_ten.name",
"conversion_mode": "fixed_value",
"value": "tool_call"
}
]
}
}
]
},
{
"name": "tool_call_get_past_weather",
"dest": [
{
"extension_group": "tools",
"extension": "weatherapi_tool_python",
"msg_conversion": {
"type": "per_property",
"keep_original": true,
"rules": [
{
"path": "_ten.name",
"conversion_mode": "fixed_value",
"value": "tool_call"
}
]
}
}
]
},
{
"name": "tool_call_get_future_weather",
"dest": [
{
"extension_group": "tools",
"extension": "weatherapi_tool_python",
"msg_conversion": {
"type": "per_property",
"keep_original": true,
"rules": [
{
"path": "_ten.name",
"conversion_mode": "fixed_value",
"value": "tool_call"
}
]
}
}
]
},
{
"name": "tool_call_bing_search",
"dest": [
{
"extension_group": "tools",
"extension": "bingsearch_tool_python",
"msg_conversion": {
"type": "per_property",
"keep_original": true,
"rules": [
{
"path": "_ten.name",
"conversion_mode": "fixed_value",
"value": "tool_call"
}
]
}
}
]
}
]
},
{
"extension_group": "tts",
"extension": "fish_audio_tts",
"audio_frame": [
{
"name": "pcm_frame",
"dest": [
{
"extension_group": "rtc",
"extension": "agora_rtc"
}
]
}
],
"cmd": [
{
"name": "flush",
"dest": [
{
"extension_group": "rtc",
"extension": "agora_rtc"
}
]
}
]
},
{
"extension_group": "transcriber",
"extension": "message_collector",
"data": [
{
"name": "data",
"dest": [
{
"extension_group": "rtc",
"extension": "agora_rtc"
}
]
}
]
}
]
}
]
}
Expand Down
6 changes: 5 additions & 1 deletion agents/ten_packages/extension/openai_v2v_python/conf.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,9 @@ def __init__(
temperature: float = 0.5,
max_tokens: int = 1024,
voice: Voices = Voices.Alloy,
server_vad:bool=True
server_vad: bool = True,
audio_out: bool = True,
input_transcript: bool = True
):
self.base_uri = base_uri
self.api_key = api_key
Expand All @@ -39,6 +41,8 @@ def __init__(
self.max_tokens = max_tokens
self.voice = voice
self.server_vad = server_vad
self.audio_out = audio_out
self.input_transcript = input_transcript

def build_ctx(self) -> dict:
return {
Expand Down
Loading

0 comments on commit 41d1a26

Please sign in to comment.