wehub/microsoft--semantic-kernel
1
0
Fork 0
Code Issues Pull Requests Actions 3 Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution
CodeQL / Analyze (csharp) (push) Has been cancelled
Details
CodeQL / Analyze (python) (push) Has been cancelled
Details

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 13:21:23 +08:00
commit b957a53def
5423 changed files with 863745 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/samples/demos/call_automation/.env.example
+8
View File
@@ -0,0 +1,8 @@
ACS_CONNECTION_STRING=
CALLBACK_URI_HOST=
AZURE_OPENAI_ENDPOINT=
AZURE_OPENAI_REALTIME_DEPLOYMENT_NAME=
AZURE_OPENAI_API_VERSION=
AZURE_OPENAI_API_KEY=
python/samples/demos/call_automation/call_automation.py
Executable
+291
View File
@@ -0,0 +1,291 @@
# Copyright (c) Microsoft. All rights reserved.
####################################################################
# Sample Quart webapp with that connects to Azure OpenAI #
# Make sure to install `uv`, see: #
# https://docs.astral.sh/uv/getting-started/installation/ #
# and rename .env.example to .env and fill in the values. #
# Follow the guidance in README.md for more info. #
# To run the app, use: #
# `uv run --env-file .env call_automation.py` #
####################################################################
#
# /// script
# requires-python = ">=3.12"
# dependencies = [
# "Quart",
# "azure-eventgrid",
# "azure-communication-callautomation==1.4.0b1",
# "semantic-kernel",
# ]
# ///
import asyncio
import base64
import logging
import os
import uuid
from datetime import datetime
from random import randint
from urllib.parse import urlencode, urlparse, urlunparse
from azure.communication.callautomation import (
AudioFormat,
MediaStreamingAudioChannelType,
MediaStreamingContentType,
MediaStreamingOptions,
MediaStreamingTransportType,
)
from azure.communication.callautomation.aio import CallAutomationClient
from azure.eventgrid import EventGridEvent, SystemEventNames
from azure.identity import AzureCliCredential
from numpy import ndarray
from quart import Quart, Response, json, request, websocket
from semantic_kernel import Kernel
from semantic_kernel.connectors.ai import FunctionChoiceBehavior
from semantic_kernel.connectors.ai.open_ai import (
AzureRealtimeExecutionSettings,
AzureRealtimeWebsocket,
)
from semantic_kernel.connectors.ai.open_ai.services._open_ai_realtime import ListenEvents
from semantic_kernel.connectors.ai.realtime_client_base import RealtimeClientBase
from semantic_kernel.contents import AudioContent, RealtimeAudioEvent
from semantic_kernel.functions import kernel_function
# Callback events URI to handle callback events.
CALLBACK_URI_HOST = os.environ["CALLBACK_URI_HOST"]
CALLBACK_EVENTS_URI = CALLBACK_URI_HOST + "/api/callbacks"
acs_client = CallAutomationClient.from_connection_string(os.environ["ACS_CONNECTION_STRING"])
app = Quart(__name__)
# region: Semantic Kernel
kernel = Kernel()
class HelperPlugin:
"""Helper plugin for the Semantic Kernel."""
@kernel_function
def get_weather(self, location: str) -> str:
"""Get the weather for a location."""
app.logger.info(f"@ Getting weather for {location}")
weather_conditions = ("sunny", "hot", "cloudy", "raining", "freezing", "snowing")
weather = weather_conditions[randint(0, len(weather_conditions) - 1)] # nosec
return f"The weather in {location} is {weather}."
@kernel_function
def get_date_time(self) -> str:
"""Get the current date and time."""
app.logger.info("@ Getting current datetime")
return f"The current date and time is {datetime.now().isoformat()}."
@kernel_function
async def goodbye(self):
"""When the user is done, say goodbye and then call this function."""
app.logger.info("@ Goodbye has been called!")
global call_connection_id
await acs_client.get_call_connection(call_connection_id).hang_up(is_for_everyone=True)
kernel.add_plugin(plugin=HelperPlugin(), plugin_name="helpers", description="Helper functions for the realtime client.")
# region: Handlers for audio and data streams
async def from_realtime_to_acs(audio: ndarray):
"""Function that forwards the audio from the model to the websocket of the ACS client."""
app.logger.debug("Audio received from the model, sending to ACS client")
await websocket.send(
json.dumps({"kind": "AudioData", "audioData": {"data": base64.b64encode(audio.tobytes()).decode("utf-8")}})
)
async def from_acs_to_realtime(client: RealtimeClientBase):
"""Function that forwards the audio from the ACS client to the model."""
while True:
try:
# Receive data from the ACS client
stream_data = await websocket.receive()
data = json.loads(stream_data)
if data["kind"] == "AudioData":
# send it to the Realtime service
await client.send(
event=RealtimeAudioEvent(
audio=AudioContent(data=data["audioData"]["data"], data_format="base64", inner_content=data),
)
)
except Exception:
app.logger.info("Websocket connection closed.")
break
async def handle_realtime_messages(client: RealtimeClientBase):
"""Function that handles the messages from the Realtime service.
This function only handles the non-audio messages.
Audio is done through the callback so that it is faster and smoother.
"""
async for event in client.receive(audio_output_callback=from_realtime_to_acs):
match event.service_type:
case ListenEvents.SESSION_CREATED:
app.logger.info("Session Created Message")
app.logger.debug(f" Session Id: {event.service_event.session.id}")
case ListenEvents.ERROR:
app.logger.error(f" Error: {event.service_event.error}")
case ListenEvents.INPUT_AUDIO_BUFFER_CLEARED:
app.logger.info("Input Audio Buffer Cleared Message")
case ListenEvents.INPUT_AUDIO_BUFFER_SPEECH_STARTED:
app.logger.debug(f"Voice activity detection started at {event.service_event.audio_start_ms} [ms]")
await websocket.send(json.dumps({"Kind": "StopAudio", "AudioData": None, "StopAudio": {}}))
case ListenEvents.CONVERSATION_ITEM_INPUT_AUDIO_TRANSCRIPTION_COMPLETED:
app.logger.info(f" User:-- {event.service_event.transcript}")
case ListenEvents.CONVERSATION_ITEM_INPUT_AUDIO_TRANSCRIPTION_FAILED:
app.logger.error(f" Error: {event.service_event.error}")
case ListenEvents.RESPONSE_DONE:
app.logger.info("Response Done Message")
app.logger.debug(f" Response Id: {event.service_event.response.id}")
if event.service_event.response.status_details:
app.logger.debug(
f" Status Details: {event.service_event.response.status_details.model_dump_json()}"
)
case ListenEvents.RESPONSE_AUDIO_TRANSCRIPT_DONE:
app.logger.info(f" AI:-- {event.service_event.transcript}")
# region: Routes
# WebSocket.
@app.websocket("/ws")
async def ws():
app.logger.info("Client connected to WebSocket")
client = AzureRealtimeWebsocket(credential=AzureCliCredential())
settings = AzureRealtimeExecutionSettings(
instructions="""You are a chat bot. Your name is Mosscap and
you have one goal: figure out what people need.
Your full name, should you need to know it, is
Splendid Speckled Mosscap. You communicate
effectively, but you tend to answer with long
flowery prose.""",
turn_detection={"type": "server_vad"},
voice="shimmer",
input_audio_format="pcm16",
output_audio_format="pcm16",
input_audio_transcription={"model": "whisper-1"},
function_choice_behavior=FunctionChoiceBehavior.Auto(),
)
# create the realtime client session
async with client(settings=settings, create_response=True, kernel=kernel):
# start handling the messages from the realtime client
# and allow the callback to be used to forward the audio to the acs client
receive_task = asyncio.create_task(handle_realtime_messages(client))
# receive messages from the ACS client and send them to the realtime client
await from_acs_to_realtime(client)
receive_task.cancel()
@app.route("/api/incomingCall", methods=["POST"])
async def incoming_call_handler() -> Response:
for event_dict in await request.json:
event = EventGridEvent.from_dict(event_dict)
match event.event_type:
case SystemEventNames.EventGridSubscriptionValidationEventName:
app.logger.info("Validating subscription")
validation_code = event.data["validationCode"]
validation_response = {"validationResponse": validation_code}
return Response(response=json.dumps(validation_response), status=200)
case SystemEventNames.AcsIncomingCallEventName:
app.logger.debug("Incoming call received: data=%s", event.data)
caller_id = (
event.data["from"]["phoneNumber"]["value"]
if event.data["from"]["kind"] == "phoneNumber"
else event.data["from"]["rawId"]
)
app.logger.info("incoming call handler caller id: %s", caller_id)
incoming_call_context = event.data["incomingCallContext"]
guid = uuid.uuid4()
query_parameters = urlencode({"callerId": caller_id})
callback_uri = f"{CALLBACK_EVENTS_URI}/{guid}?{query_parameters}"
parsed_url = urlparse(CALLBACK_EVENTS_URI)
websocket_url = urlunparse(("wss", parsed_url.netloc, "/ws", "", "", ""))
app.logger.debug("callback url: %s", callback_uri)
app.logger.debug("websocket url: %s", websocket_url)
answer_call_result = await acs_client.answer_call(
incoming_call_context=incoming_call_context,
operation_context="incomingCall",
callback_url=callback_uri,
media_streaming=MediaStreamingOptions(
transport_url=websocket_url,
transport_type=MediaStreamingTransportType.WEBSOCKET,
content_type=MediaStreamingContentType.AUDIO,
audio_channel_type=MediaStreamingAudioChannelType.MIXED,
start_media_streaming=True,
enable_bidirectional=True,
audio_format=AudioFormat.PCM24_K_MONO,
),
)
app.logger.info(f"Answered call for connection id: {answer_call_result.call_connection_id}")
case _:
app.logger.debug("Event type not handled: %s", event.event_type)
app.logger.debug("Event data: %s", event.data)
return Response(status=200)
return Response(status=200)
@app.route("/api/callbacks/<contextId>", methods=["POST"])
async def callbacks(contextId):
for event in await request.json:
# Parsing callback events
global call_connection_id
event_data = event["data"]
call_connection_id = event_data["callConnectionId"]
app.logger.debug(
f"Received Event:-> {event['type']}, Correlation Id:-> {event_data['correlationId']}, CallConnectionId:-> {call_connection_id}" # noqa: E501
)
match event["type"]:
case "Microsoft.Communication.CallConnected":
call_connection_properties = await acs_client.get_call_connection(
call_connection_id
).get_call_properties()
media_streaming_subscription = call_connection_properties.media_streaming_subscription
app.logger.info(f"MediaStreamingSubscription:--> {media_streaming_subscription}")
app.logger.info(f"Received CallConnected event for connection id: {call_connection_id}")
app.logger.debug("CORRELATION ID:--> %s", event_data["correlationId"])
app.logger.debug("CALL CONNECTION ID:--> %s", event_data["callConnectionId"])
case "Microsoft.Communication.MediaStreamingStarted" | "Microsoft.Communication.MediaStreamingStopped":
app.logger.debug(
f"Media streaming content type:--> {event_data['mediaStreamingUpdate']['contentType']}"
)
app.logger.debug(
f"Media streaming status:--> {event_data['mediaStreamingUpdate']['mediaStreamingStatus']}"
)
app.logger.debug(
f"Media streaming status details:--> {event_data['mediaStreamingUpdate']['mediaStreamingStatusDetails']}" # noqa: E501
)
case "Microsoft.Communication.MediaStreamingFailed":
app.logger.warning(
f"Code:->{event_data['resultInformation']['code']}, Subcode:-> {event_data['resultInformation']['subCode']}" # noqa: E501
)
app.logger.warning(f"Message:->{event_data['resultInformation']['message']}")
case "Microsoft.Communication.CallDisconnected":
app.logger.debug(f"Call disconnected for connection id: {call_connection_id}")
return Response(status=200)
@app.route("/")
def home():
return "Hello SKxACS CallAutomation!"
# region: Main
if __name__ == "__main__":
app.logger.setLevel(logging.INFO)
app.run(port=8080)
python/samples/demos/call_automation/readme.md
+92
View File
@@ -0,0 +1,92 @@
# Call Automation - Quick Start Sample
This is a sample application. It highlights an integration of Azure Communication Services with Semantic Kernel, using the Azure OpenAI Service to enable intelligent conversational agents.
Original code for this sample can be found [here](https://github.com/Azure-Samples/communication-services-python-quickstarts/tree/main/callautomation-openai-sample).
## Prerequisites
- An Azure account with an active subscription. [Create an account for free](https://azure.microsoft.com/free/?WT.mc_id=A261C142F).
- A deployed Communication Services resource. [Create a Communication Services resource](https://docs.microsoft.com/azure/communication-services/quickstarts/create-communication-resource).
- A [phone number](https://learn.microsoft.com/en-us/azure/communication-services/quickstarts/telephony/get-phone-number) in your Azure Communication Services resource that can get inbound calls. NOTE: although trial phone numbers are available in free subscriptions, they will not work properly. You must have a paid phone number.
- [Python](https://www.python.org/downloads/) 3.9 or above.
- An Azure OpenAI Resource and Deployed Model. See [instructions](https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/create-resource?pivots=web-portal).
- Install `uv`, see [the uv docs](https://docs.astral.sh/uv/getting-started/installation/).
## To run the app
1. Open an instance of PowerShell, Windows Terminal, Command Prompt or equivalent and navigate to the directory that you would like to clone the sample to.
2. git clone `https://github.com/microsoft/semantic-kernel.git`.
3. Navigate to `python/samples/demos/call_automation` folder
### Setup and host your Azure DevTunnel
[Azure DevTunnels](https://learn.microsoft.com/en-us/azure/developer/dev-tunnels/overview) is an Azure service that enables you to share local web services hosted on the internet. Use the commands below to connect your local development environment to the public internet. This creates a tunnel with a persistent endpoint URL and which allows anonymous access. We will then use this endpoint to notify your application of calling events from the ACS Call Automation service.
```bash
devtunnel create --allow-anonymous
devtunnel port create -p 8080
devtunnel host
```
### Configuring application
Copy the `.env.example` file to `.env` and update the following values:
1. `ACS_CONNECTION_STRING`: Azure Communication Service resource's connection string.
2. `CALLBACK_URI_HOST`: Base url of the app. (For local development use the dev tunnel url from the step above). This URI is in the form of `https://<unique-id>-8080.XXXX.devtunnels.ms`.
3. `AZURE_OPENAI_ENDPOINT`: Azure Open AI service endpoint
4. `AZURE_OPENAI_REALTIME_DEPLOYMENT_NAME`: Azure Open AI deployment name
5. `AZURE_OPENAI_API_VERSION`: Azure Open AI API version, this should be one that includes the realtime api, for instance '2024-10-01-preview'. This API version is the currently the only supported version.
6. `AZURE_OPENAI_API_KEY`: Azure Open AI API key, optionally, you can also use Entra Auth.
## Run the app
1. Navigate to `call_automation` folder and do one of the following to start the main application:
- run `call_automation.py` in debug mode from your IDE (VSCode will load your .env variables into the environment automatically, other IDE's might need an extra step).
- execute `uv run --env-file .env call_automation.py` directly in your terminal (this uses `uv`, which will then install the requirements in a temporary virtual environment, see [uv docs](https://docs.astral.sh/uv/guides/scripts) for more info).
2. Browser should pop up with a simple page. If not navigate it to `http://localhost:8080/` or your dev tunnel url.
3. Register an EventGrid Webhook for the IncomingCall(`https://<devtunnelurl>/api/incomingCall`) event that points to your devtunnel URI. Instructions [here](https://learn.microsoft.com/en-us/azure/communication-services/concepts/call-automation/incoming-call-notification).
- To register the event, navigate to your ACS resource in the Azure Portal (follow the Microsoft Learn Docs if you prefer to use the CLI).
- On the left menu bar click "Events."
- Click on "+Event Subscription."
- Provide a unique name for the event subscription details, for example, "IncomingCallWebhook"
- Leave the "Event Schema" as "Event Grid Schema"
- Provide a unique "System Topic Name"
- For the "Event Types" select "Incoming Call"
- For the "Endpoint Details" select "Webhook" from the drop down
- Once "Webhook" is selected, you will need to configure the URI for the incoming call webhook, as mentioned above: `https://<devtunnelurl>/api/incomingCall`.
- **Important**: before clicking on "Create" to create the event subscription, the `call_automation.py` script must be running, as well as your devtunnel. ACS sends a verification payload to the app to make sure that the communication is configured properly. The event subscription will not succeed in the portal without the script running. If you see an error, this is most likely the root cause.
Once that's completed you should have a running application. The way to test this is to place a call to your ACS phone number and talk to your intelligent agent!
In the terminal you should see all sorts of logs from both ACS and Semantic Kernel. Successful logs can look like the following:
```bash
[YYYY-MM-DD HH:MM:SS,mmm] INFO in call_automation: incoming call handler caller id: +14255551234
[2YYYY-MM-DD HH:MM:SS,mmm] INFO in call_automation: callback url: https://<devtunnelurl>/api/callbacks/<guid>?callerId=%2B14257059063
[YYYY-MM-DD HH:MM:SS,mmm] INFO in call_automation: websocket url: wss://<devtunnelurl>/ws
[YYYY-MM-DD HH:MM:SS,mmm] INFO in call_automation: Answered call for connection id: <guid>>
[YYYY-MM-DD HH:MM:SS,mmm] [28438] [INFO] 127.0.0.1:55481 POST /api/incomingCall 1.1 200 - 595851
[YYYY-MM-DD HH:MM:SS,mmm] INFO in call_automation: Client connected to WebSocket
[YYYY-MM-DD HH:MM:SS,mmm] [28438] [INFO] 127.0.0.1:55483 GET /ws 1.1 101 - 1262939
Session Created Message
Session Id: sess_BG0CBnM6CMEGSbpsK5CiC
[YYYY-MM-DD HH:MM:SS,mmm] INFO in call_automation: Received Event:-> Microsoft.Communication.ParticipantsUpdated, Correlation Id:-> <guid>, CallConnectionId:-> <guid>
[YYYY-MM-DD HH:MM:SS,mmm] [28438] [INFO] 127.0.0.1:55486 POST /api/callbacks/1629809d 1.1 200 - 1642
[YYYY-MM-DD HH:MM:SS,mmm] INFO in call_automation: Received Event:-> Microsoft.Communication.CallConnected, Correlation Id:-> <guid>, CallConnectionId:-> <guid>
[YYYY-MM-DD HH:MM:SS,mmm] INFO in call_automation: Received Event:-> Microsoft.Communication.MediaStreamingStarted, Correlation Id:-> <guid>, CallConnectionId:-> <guid>
[YYYY-MM-DD HH:MM:SS,mmm] INFO in call_automation: Media streaming content type:--> Audio
[YYYY-MM-DD HH:MM:SS,mmm] INFO in call_automation: Media streaming status:--> mediaStreamingStarted
[YYYY-MM-DD HH:MM:SS,mmm] INFO in call_automation: Media streaming status details:--> subscriptionStarted
[YYYY-MM-DD HH:MM:SS,mmm] [28438] [INFO] 127.0.0.1:55488 POST /api/callbacks/1629809d 1.1 200 - 1490
[YYYY-MM-DD HH:MM:SS,mmm] INFO in call_automation: MediaStreamingSubscription:--> {'additional_properties': {}, 'id': '50a8ca48', 'state': 'active', 'subscribed_content_types': ['audio']}
[YYYY-MM-DD HH:MM:SS,mmm] INFO in call_automation: Received CallConnected event for connection id: <guid>
[YYYY-MM-DD HH:MM:SS,mmm] INFO in call_automation: CORRELATION ID:--> <guid>
[YYYY-MM-DD HH:MM:SS,mmm] INFO in call_automation: CALL CONNECTION ID:--> <guid>
[YYYY-MM-DD HH:MM:SS,mmm] [28438] [INFO] 127.0.0.1:55487 POST /api/callbacks/1629809d 1.1 200 - 137171
AI:-- Ah, greetings, dear traveler of the world! In this moment, amidst the tapestry of time and space, you seek a glimpse of the weather that graces a particular place. Pray, tell me, to which location do you wish to turn your gaze, so I might summon the winds and the skies to unfold their secrets for you?
Response Done Message
Response Id: resp_BG0CBedSOZgrlo82IlxMC
```