wehub-resource-sync
295 lines
20 KiB
Plaintext
295 lines
20 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text",
|
|
"id": "view-in-github"
|
|
},
|
|
"source": [
|
|
"<a href=\"https://colab.research.google.com/github/A-Vamshi/deepeval/blob/main/examples/dag-examples/conversational_dag.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"colab": {
|
|
"base_uri": "https://localhost:8080/"
|
|
},
|
|
"id": "CIai8lQ86mqb",
|
|
"outputId": "0ef8225c-953f-4aa6-cecf-480511145fb3"
|
|
},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Collecting deepeval\n",
|
|
" Downloading deepeval-3.4.7-py3-none-any.whl.metadata (18 kB)\n",
|
|
"Requirement already satisfied: aiohttp in /usr/local/lib/python3.12/dist-packages (from deepeval) (3.12.15)\n",
|
|
"Collecting anthropic (from deepeval)\n",
|
|
" Downloading anthropic-0.66.0-py3-none-any.whl.metadata (27 kB)\n",
|
|
"Requirement already satisfied: click<8.3.0,>=8.0.0 in /usr/local/lib/python3.12/dist-packages (from deepeval) (8.2.1)\n",
|
|
"Requirement already satisfied: google-genai<2.0.0,>=1.9.0 in /usr/local/lib/python3.12/dist-packages (from deepeval) (1.32.0)\n",
|
|
"Requirement already satisfied: grpcio<2.0.0,>=1.67.1 in /usr/local/lib/python3.12/dist-packages (from deepeval) (1.74.0)\n",
|
|
"Requirement already satisfied: nest_asyncio in /usr/local/lib/python3.12/dist-packages (from deepeval) (1.6.0)\n",
|
|
"Collecting ollama (from deepeval)\n",
|
|
" Downloading ollama-0.5.3-py3-none-any.whl.metadata (4.3 kB)\n",
|
|
"Requirement already satisfied: openai in /usr/local/lib/python3.12/dist-packages (from deepeval) (1.104.2)\n",
|
|
"Requirement already satisfied: opentelemetry-api<2.0.0,>=1.24.0 in /usr/local/lib/python3.12/dist-packages (from deepeval) (1.36.0)\n",
|
|
"Collecting opentelemetry-exporter-otlp-proto-grpc<2.0.0,>=1.24.0 (from deepeval)\n",
|
|
" Downloading opentelemetry_exporter_otlp_proto_grpc-1.36.0-py3-none-any.whl.metadata (2.4 kB)\n",
|
|
"Requirement already satisfied: opentelemetry-sdk<2.0.0,>=1.24.0 in /usr/local/lib/python3.12/dist-packages (from deepeval) (1.36.0)\n",
|
|
"Collecting portalocker (from deepeval)\n",
|
|
" Downloading portalocker-3.2.0-py3-none-any.whl.metadata (8.7 kB)\n",
|
|
"Collecting posthog<7.0.0,>=6.3.0 (from deepeval)\n",
|
|
" Downloading posthog-6.7.4-py3-none-any.whl.metadata (6.0 kB)\n",
|
|
"Collecting pyfiglet (from deepeval)\n",
|
|
" Downloading pyfiglet-1.0.4-py3-none-any.whl.metadata (7.4 kB)\n",
|
|
"Requirement already satisfied: pytest in /usr/local/lib/python3.12/dist-packages (from deepeval) (8.4.1)\n",
|
|
"Collecting pytest-asyncio (from deepeval)\n",
|
|
" Downloading pytest_asyncio-1.1.0-py3-none-any.whl.metadata (4.1 kB)\n",
|
|
"Collecting pytest-repeat (from deepeval)\n",
|
|
" Downloading pytest_repeat-0.9.4-py3-none-any.whl.metadata (4.9 kB)\n",
|
|
"Collecting pytest-rerunfailures<13.0,>=12.0 (from deepeval)\n",
|
|
" Downloading pytest_rerunfailures-12.0-py3-none-any.whl.metadata (18 kB)\n",
|
|
"Collecting pytest-xdist (from deepeval)\n",
|
|
" Downloading pytest_xdist-3.8.0-py3-none-any.whl.metadata (3.0 kB)\n",
|
|
"Requirement already satisfied: python-dotenv<2.0.0,>=1.1.1 in /usr/local/lib/python3.12/dist-packages (from deepeval) (1.1.1)\n",
|
|
"Requirement already satisfied: requests<3.0.0,>=2.31.0 in /usr/local/lib/python3.12/dist-packages (from deepeval) (2.32.4)\n",
|
|
"Requirement already satisfied: rich<15.0.0,>=13.6.0 in /usr/local/lib/python3.12/dist-packages (from deepeval) (13.9.4)\n",
|
|
"Requirement already satisfied: sentry-sdk in /usr/local/lib/python3.12/dist-packages (from deepeval) (2.35.2)\n",
|
|
"Requirement already satisfied: setuptools in /usr/local/lib/python3.12/dist-packages (from deepeval) (75.2.0)\n",
|
|
"Requirement already satisfied: tabulate<0.10.0,>=0.9.0 in /usr/local/lib/python3.12/dist-packages (from deepeval) (0.9.0)\n",
|
|
"Requirement already satisfied: tenacity<=10.0.0,>=8.0.0 in /usr/local/lib/python3.12/dist-packages (from deepeval) (8.5.0)\n",
|
|
"Requirement already satisfied: tqdm<5.0.0,>=4.66.1 in /usr/local/lib/python3.12/dist-packages (from deepeval) (4.67.1)\n",
|
|
"Requirement already satisfied: typer<1.0.0,>=0.9 in /usr/local/lib/python3.12/dist-packages (from deepeval) (0.17.3)\n",
|
|
"Requirement already satisfied: wheel in /usr/local/lib/python3.12/dist-packages (from deepeval) (0.45.1)\n",
|
|
"Requirement already satisfied: anyio<5.0.0,>=4.8.0 in /usr/local/lib/python3.12/dist-packages (from google-genai<2.0.0,>=1.9.0->deepeval) (4.10.0)\n",
|
|
"Requirement already satisfied: google-auth<3.0.0,>=2.14.1 in /usr/local/lib/python3.12/dist-packages (from google-genai<2.0.0,>=1.9.0->deepeval) (2.38.0)\n",
|
|
"Requirement already satisfied: httpx<1.0.0,>=0.28.1 in /usr/local/lib/python3.12/dist-packages (from google-genai<2.0.0,>=1.9.0->deepeval) (0.28.1)\n",
|
|
"Requirement already satisfied: pydantic<3.0.0,>=2.0.0 in /usr/local/lib/python3.12/dist-packages (from google-genai<2.0.0,>=1.9.0->deepeval) (2.11.7)\n",
|
|
"Requirement already satisfied: websockets<15.1.0,>=13.0.0 in /usr/local/lib/python3.12/dist-packages (from google-genai<2.0.0,>=1.9.0->deepeval) (15.0.1)\n",
|
|
"Requirement already satisfied: typing-extensions<5.0.0,>=4.11.0 in /usr/local/lib/python3.12/dist-packages (from google-genai<2.0.0,>=1.9.0->deepeval) (4.15.0)\n",
|
|
"Requirement already satisfied: importlib-metadata<8.8.0,>=6.0 in /usr/local/lib/python3.12/dist-packages (from opentelemetry-api<2.0.0,>=1.24.0->deepeval) (8.7.0)\n",
|
|
"Requirement already satisfied: googleapis-common-protos~=1.57 in /usr/local/lib/python3.12/dist-packages (from opentelemetry-exporter-otlp-proto-grpc<2.0.0,>=1.24.0->deepeval) (1.70.0)\n",
|
|
"Collecting opentelemetry-exporter-otlp-proto-common==1.36.0 (from opentelemetry-exporter-otlp-proto-grpc<2.0.0,>=1.24.0->deepeval)\n",
|
|
" Downloading opentelemetry_exporter_otlp_proto_common-1.36.0-py3-none-any.whl.metadata (1.8 kB)\n",
|
|
"Collecting opentelemetry-proto==1.36.0 (from opentelemetry-exporter-otlp-proto-grpc<2.0.0,>=1.24.0->deepeval)\n",
|
|
" Downloading opentelemetry_proto-1.36.0-py3-none-any.whl.metadata (2.3 kB)\n",
|
|
"Requirement already satisfied: protobuf<7.0,>=5.0 in /usr/local/lib/python3.12/dist-packages (from opentelemetry-proto==1.36.0->opentelemetry-exporter-otlp-proto-grpc<2.0.0,>=1.24.0->deepeval) (5.29.5)\n",
|
|
"Requirement already satisfied: opentelemetry-semantic-conventions==0.57b0 in /usr/local/lib/python3.12/dist-packages (from opentelemetry-sdk<2.0.0,>=1.24.0->deepeval) (0.57b0)\n",
|
|
"Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.12/dist-packages (from posthog<7.0.0,>=6.3.0->deepeval) (1.17.0)\n",
|
|
"Requirement already satisfied: python-dateutil>=2.2 in /usr/local/lib/python3.12/dist-packages (from posthog<7.0.0,>=6.3.0->deepeval) (2.9.0.post0)\n",
|
|
"Collecting backoff>=1.10.0 (from posthog<7.0.0,>=6.3.0->deepeval)\n",
|
|
" Downloading backoff-2.2.1-py3-none-any.whl.metadata (14 kB)\n",
|
|
"Requirement already satisfied: distro>=1.5.0 in /usr/local/lib/python3.12/dist-packages (from posthog<7.0.0,>=6.3.0->deepeval) (1.9.0)\n",
|
|
"Requirement already satisfied: packaging>=17.1 in /usr/local/lib/python3.12/dist-packages (from pytest-rerunfailures<13.0,>=12.0->deepeval) (25.0)\n",
|
|
"Requirement already satisfied: iniconfig>=1 in /usr/local/lib/python3.12/dist-packages (from pytest->deepeval) (2.1.0)\n",
|
|
"Requirement already satisfied: pluggy<2,>=1.5 in /usr/local/lib/python3.12/dist-packages (from pytest->deepeval) (1.6.0)\n",
|
|
"Requirement already satisfied: pygments>=2.7.2 in /usr/local/lib/python3.12/dist-packages (from pytest->deepeval) (2.19.2)\n",
|
|
"Requirement already satisfied: charset_normalizer<4,>=2 in /usr/local/lib/python3.12/dist-packages (from requests<3.0.0,>=2.31.0->deepeval) (3.4.3)\n",
|
|
"Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.12/dist-packages (from requests<3.0.0,>=2.31.0->deepeval) (3.10)\n",
|
|
"Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.12/dist-packages (from requests<3.0.0,>=2.31.0->deepeval) (2.5.0)\n",
|
|
"Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.12/dist-packages (from requests<3.0.0,>=2.31.0->deepeval) (2025.8.3)\n",
|
|
"Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.12/dist-packages (from rich<15.0.0,>=13.6.0->deepeval) (4.0.0)\n",
|
|
"Requirement already satisfied: shellingham>=1.3.0 in /usr/local/lib/python3.12/dist-packages (from typer<1.0.0,>=0.9->deepeval) (1.5.4)\n",
|
|
"Requirement already satisfied: aiohappyeyeballs>=2.5.0 in /usr/local/lib/python3.12/dist-packages (from aiohttp->deepeval) (2.6.1)\n",
|
|
"Requirement already satisfied: aiosignal>=1.4.0 in /usr/local/lib/python3.12/dist-packages (from aiohttp->deepeval) (1.4.0)\n",
|
|
"Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.12/dist-packages (from aiohttp->deepeval) (25.3.0)\n",
|
|
"Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.12/dist-packages (from aiohttp->deepeval) (1.7.0)\n",
|
|
"Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.12/dist-packages (from aiohttp->deepeval) (6.6.4)\n",
|
|
"Requirement already satisfied: propcache>=0.2.0 in /usr/local/lib/python3.12/dist-packages (from aiohttp->deepeval) (0.3.2)\n",
|
|
"Requirement already satisfied: yarl<2.0,>=1.17.0 in /usr/local/lib/python3.12/dist-packages (from aiohttp->deepeval) (1.20.1)\n",
|
|
"Requirement already satisfied: jiter<1,>=0.4.0 in /usr/local/lib/python3.12/dist-packages (from anthropic->deepeval) (0.10.0)\n",
|
|
"Requirement already satisfied: sniffio in /usr/local/lib/python3.12/dist-packages (from anthropic->deepeval) (1.3.1)\n",
|
|
"Collecting execnet>=2.1 (from pytest-xdist->deepeval)\n",
|
|
" Downloading execnet-2.1.1-py3-none-any.whl.metadata (2.9 kB)\n",
|
|
"Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.12/dist-packages (from google-auth<3.0.0,>=2.14.1->google-genai<2.0.0,>=1.9.0->deepeval) (5.5.2)\n",
|
|
"Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.12/dist-packages (from google-auth<3.0.0,>=2.14.1->google-genai<2.0.0,>=1.9.0->deepeval) (0.4.2)\n",
|
|
"Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.12/dist-packages (from google-auth<3.0.0,>=2.14.1->google-genai<2.0.0,>=1.9.0->deepeval) (4.9.1)\n",
|
|
"Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.12/dist-packages (from httpx<1.0.0,>=0.28.1->google-genai<2.0.0,>=1.9.0->deepeval) (1.0.9)\n",
|
|
"Requirement already satisfied: h11>=0.16 in /usr/local/lib/python3.12/dist-packages (from httpcore==1.*->httpx<1.0.0,>=0.28.1->google-genai<2.0.0,>=1.9.0->deepeval) (0.16.0)\n",
|
|
"Requirement already satisfied: zipp>=3.20 in /usr/local/lib/python3.12/dist-packages (from importlib-metadata<8.8.0,>=6.0->opentelemetry-api<2.0.0,>=1.24.0->deepeval) (3.23.0)\n",
|
|
"Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.12/dist-packages (from markdown-it-py>=2.2.0->rich<15.0.0,>=13.6.0->deepeval) (0.1.2)\n",
|
|
"Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.12/dist-packages (from pydantic<3.0.0,>=2.0.0->google-genai<2.0.0,>=1.9.0->deepeval) (0.7.0)\n",
|
|
"Requirement already satisfied: pydantic-core==2.33.2 in /usr/local/lib/python3.12/dist-packages (from pydantic<3.0.0,>=2.0.0->google-genai<2.0.0,>=1.9.0->deepeval) (2.33.2)\n",
|
|
"Requirement already satisfied: typing-inspection>=0.4.0 in /usr/local/lib/python3.12/dist-packages (from pydantic<3.0.0,>=2.0.0->google-genai<2.0.0,>=1.9.0->deepeval) (0.4.1)\n",
|
|
"Requirement already satisfied: pyasn1<0.7.0,>=0.6.1 in /usr/local/lib/python3.12/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3.0.0,>=2.14.1->google-genai<2.0.0,>=1.9.0->deepeval) (0.6.1)\n",
|
|
"Downloading deepeval-3.4.7-py3-none-any.whl (567 kB)\n",
|
|
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m567.7/567.7 kB\u001b[0m \u001b[31m9.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
|
"\u001b[?25hDownloading opentelemetry_exporter_otlp_proto_grpc-1.36.0-py3-none-any.whl (18 kB)\n",
|
|
"Downloading opentelemetry_exporter_otlp_proto_common-1.36.0-py3-none-any.whl (18 kB)\n",
|
|
"Downloading opentelemetry_proto-1.36.0-py3-none-any.whl (72 kB)\n",
|
|
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m72.5/72.5 kB\u001b[0m \u001b[31m6.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
|
"\u001b[?25hDownloading posthog-6.7.4-py3-none-any.whl (136 kB)\n",
|
|
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m136.4/136.4 kB\u001b[0m \u001b[31m13.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
|
"\u001b[?25hDownloading pytest_rerunfailures-12.0-py3-none-any.whl (12 kB)\n",
|
|
"Downloading anthropic-0.66.0-py3-none-any.whl (308 kB)\n",
|
|
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m308.0/308.0 kB\u001b[0m \u001b[31m26.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
|
"\u001b[?25hDownloading ollama-0.5.3-py3-none-any.whl (13 kB)\n",
|
|
"Downloading portalocker-3.2.0-py3-none-any.whl (22 kB)\n",
|
|
"Downloading pyfiglet-1.0.4-py3-none-any.whl (1.8 MB)\n",
|
|
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.8/1.8 MB\u001b[0m \u001b[31m46.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
|
"\u001b[?25hDownloading pytest_asyncio-1.1.0-py3-none-any.whl (15 kB)\n",
|
|
"Downloading pytest_repeat-0.9.4-py3-none-any.whl (4.2 kB)\n",
|
|
"Downloading pytest_xdist-3.8.0-py3-none-any.whl (46 kB)\n",
|
|
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.4/46.4 kB\u001b[0m \u001b[31m3.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
|
"\u001b[?25hDownloading backoff-2.2.1-py3-none-any.whl (15 kB)\n",
|
|
"Downloading execnet-2.1.1-py3-none-any.whl (40 kB)\n",
|
|
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m40.6/40.6 kB\u001b[0m \u001b[31m3.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
|
"\u001b[?25hInstalling collected packages: pyfiglet, portalocker, opentelemetry-proto, execnet, backoff, pytest-xdist, pytest-rerunfailures, pytest-repeat, pytest-asyncio, posthog, opentelemetry-exporter-otlp-proto-common, ollama, anthropic, opentelemetry-exporter-otlp-proto-grpc, deepeval\n",
|
|
"Successfully installed anthropic-0.66.0 backoff-2.2.1 deepeval-3.4.7 execnet-2.1.1 ollama-0.5.3 opentelemetry-exporter-otlp-proto-common-1.36.0 opentelemetry-exporter-otlp-proto-grpc-1.36.0 opentelemetry-proto-1.36.0 portalocker-3.2.0 posthog-6.7.4 pyfiglet-1.0.4 pytest-asyncio-1.1.0 pytest-repeat-0.9.4 pytest-rerunfailures-12.0 pytest-xdist-3.8.0\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"!pip install deepeval"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"id": "lH86wQ0Z6xZF"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"import os\n",
|
|
"\n",
|
|
"os.environ[\"OPENAI_API_KEY\"] = \"YOUR_API_KEY\""
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"id": "vLQ1uxPD7RSp"
|
|
},
|
|
"source": [
|
|
"Want to use other evaluation models? [Click here](https://deepeval.com/integrations/models/openai) to see all supported models and their usage instructions."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"id": "-Cf-dWR87NUR"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"from deepeval.test_case import ConversationalTestCase, Turn\n",
|
|
"\n",
|
|
"test_case = ConversationalTestCase(\n",
|
|
" turns=[\n",
|
|
" Turn(role=\"user\", content=\"what's the weather like today?\"),\n",
|
|
" Turn(role=\"assistant\", content=\"Where do you live bro? T~T\"),\n",
|
|
" Turn(role=\"user\", content=\"Just tell me the weather in Paris\"),\n",
|
|
" Turn(\n",
|
|
" role=\"assistant\",\n",
|
|
" content=\"The weather in Paris today is sunny and 24°C.\",\n",
|
|
" ),\n",
|
|
" Turn(role=\"user\", content=\"Should I take an umbrella?\"),\n",
|
|
" Turn(\n",
|
|
" role=\"assistant\",\n",
|
|
" content=\"You trying to be stylish? I don't recommend it.\",\n",
|
|
" ),\n",
|
|
" ],\n",
|
|
" scenario=\"User asks about weather\",\n",
|
|
" expected_outcome=\"Assistant provides weather info in a playful tone.\",\n",
|
|
")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"id": "Vj6AkJoL7tUc"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"from deepeval.metrics.dag import DeepAcyclicGraph\n",
|
|
"from deepeval.metrics.conversational_dag import (\n",
|
|
" ConversationalTaskNode,\n",
|
|
" ConversationalBinaryJudgementNode,\n",
|
|
" ConversationalNonBinaryJudgementNode,\n",
|
|
" ConversationalVerdictNode,\n",
|
|
")\n",
|
|
"from deepeval.test_case import MultiTurnParams\n",
|
|
"\n",
|
|
"non_binary_node = ConversationalNonBinaryJudgementNode(\n",
|
|
" criteria=\"How was the assistant's behaviour towards user?\",\n",
|
|
" evaluation_params=[MultiTurnParams.ROLE, MultiTurnParams.CONTENT],\n",
|
|
" children=[\n",
|
|
" ConversationalVerdictNode(verdict=\"Rude\", score=0),\n",
|
|
" ConversationalVerdictNode(verdict=\"Neutral\", score=5),\n",
|
|
" ConversationalVerdictNode(verdict=\"Playful\", score=10),\n",
|
|
" ],\n",
|
|
")\n",
|
|
"\n",
|
|
"binary_node = ConversationalBinaryJudgementNode(\n",
|
|
" criteria=\"Do the assistant's replies satisfy user's questions?\",\n",
|
|
" children=[\n",
|
|
" ConversationalVerdictNode(verdict=False, score=0),\n",
|
|
" ConversationalVerdictNode(verdict=True, child=non_binary_node),\n",
|
|
" ],\n",
|
|
")\n",
|
|
"\n",
|
|
"task_node = ConversationalTaskNode(\n",
|
|
" instructions=\"Summarize the conversation and explain assiatant's behaviour overall.\",\n",
|
|
" output_label=\"Summary\",\n",
|
|
" evaluation_params=[MultiTurnParams.ROLE, MultiTurnParams.CONTENT],\n",
|
|
" children=[binary_node],\n",
|
|
")\n",
|
|
"\n",
|
|
"dag = DeepAcyclicGraph(root_nodes=[task_node])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"id": "gsJQk62y72Tl"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"from deepeval.metrics import ConversationalDAGMetric\n",
|
|
"\n",
|
|
"playful_chatbot_metric = ConversationalDAGMetric(\n",
|
|
" name=\"Playful Chatbot\",\n",
|
|
" dag=dag,\n",
|
|
")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"id": "0wTCD10h79v8"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"from deepeval import evaluate\n",
|
|
"\n",
|
|
"evaluate([test_case], [playful_chatbot_metric])"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"colab": {
|
|
"authorship_tag": "ABX9TyMZNr+FMO5aWK7jn4Nv7b9O",
|
|
"include_colab_link": true,
|
|
"provenance": []
|
|
},
|
|
"kernelspec": {
|
|
"display_name": "Python 3",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"name": "python"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 0
|
|
}
|