wehub-resource-sync
6b7e6b44f1
Python Build and Type Check / python-ci (ubuntu-latest, 3.11) (push) Has been cancelled
Python Build and Type Check / python-ci (ubuntu-latest, 3.13) (push) Has been cancelled
Python Build and Type Check / python-ci (windows-latest, 3.11) (push) Has been cancelled
Python Build and Type Check / python-ci (windows-latest, 3.13) (push) Has been cancelled
Python Integration Tests / python-ci (ubuntu-latest, 3.13) (push) Has been cancelled
Python Integration Tests / python-ci (windows-latest, 3.13) (push) Has been cancelled
Python Notebook Tests / python-ci (ubuntu-latest, 3.13) (push) Has been cancelled
Python Notebook Tests / python-ci (windows-latest, 3.13) (push) Has been cancelled
Python Smoke Tests / python-ci (ubuntu-latest, 3.13) (push) Has been cancelled
Python Smoke Tests / python-ci (windows-latest, 3.13) (push) Has been cancelled
Python Unit Tests / python-ci (ubuntu-latest, 3.13) (push) Has been cancelled
Python Unit Tests / python-ci (windows-latest, 3.13) (push) Has been cancelled
gh-pages / build (push) Has been cancelled
Python Publish (pypi) / Upload release to PyPI (push) Has been cancelled
Spellcheck / spellcheck (push) Has been cancelled
596 lines
24 KiB
Plaintext
596 lines
24 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "4dc68732",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Metrics\n",
|
|
"\n",
|
|
"Metrics are automatically tracked for completion and embedding calls.\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 1,
|
|
"id": "868deb65",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Metrics for: azure/gpt-4o\n",
|
|
"{\n",
|
|
" \"attempted_request_count\": 1,\n",
|
|
" \"successful_response_count\": 1,\n",
|
|
" \"failed_response_count\": 0,\n",
|
|
" \"failure_rate\": 0.0,\n",
|
|
" \"requests_with_retries\": 0,\n",
|
|
" \"retries\": 0,\n",
|
|
" \"retry_rate\": 0.0,\n",
|
|
" \"compute_duration_seconds\": 3.4281113147735596,\n",
|
|
" \"compute_duration_per_response_seconds\": 3.4281113147735596,\n",
|
|
" \"cache_hit_rate\": 0.0,\n",
|
|
" \"streaming_responses\": 0,\n",
|
|
" \"responses_with_tokens\": 1,\n",
|
|
" \"prompt_tokens\": 14,\n",
|
|
" \"completion_tokens\": 8,\n",
|
|
" \"total_tokens\": 22,\n",
|
|
" \"tokens_per_response\": 22.0,\n",
|
|
" \"responses_with_cost\": 1,\n",
|
|
" \"input_cost\": 3.5000000000000004e-05,\n",
|
|
" \"output_cost\": 8e-05,\n",
|
|
" \"total_cost\": 0.000115,\n",
|
|
" \"cost_per_response\": 0.000115\n",
|
|
"}\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"# Copyright (c) 2024 Microsoft Corporation.\n",
|
|
"# Licensed under the MIT License\n",
|
|
"\n",
|
|
"import json\n",
|
|
"import os\n",
|
|
"\n",
|
|
"from dotenv import load_dotenv\n",
|
|
"from graphrag_llm.completion import LLMCompletion, create_completion\n",
|
|
"from graphrag_llm.config import AuthMethod, ModelConfig\n",
|
|
"\n",
|
|
"load_dotenv()\n",
|
|
"\n",
|
|
"api_key = os.getenv(\"GRAPHRAG_API_KEY\")\n",
|
|
"model_config = ModelConfig(\n",
|
|
" model_provider=\"azure\",\n",
|
|
" model=os.getenv(\"GRAPHRAG_MODEL\", \"gpt-4o\"),\n",
|
|
" azure_deployment_name=os.getenv(\"GRAPHRAG_MODEL\", \"gpt-4o\"),\n",
|
|
" api_base=os.getenv(\"GRAPHRAG_API_BASE\"),\n",
|
|
" api_version=os.getenv(\"GRAPHRAG_API_VERSION\", \"2025-04-01-preview\"),\n",
|
|
" api_key=api_key,\n",
|
|
" auth_method=AuthMethod.AzureManagedIdentity if not api_key else AuthMethod.ApiKey,\n",
|
|
")\n",
|
|
"llm_completion: LLMCompletion = create_completion(model_config)\n",
|
|
"\n",
|
|
"response = llm_completion.completion(\n",
|
|
" messages=\"What is the capital of France?\",\n",
|
|
")\n",
|
|
"\n",
|
|
"print(f\"Metrics for: {llm_completion.metrics_store.id}\")\n",
|
|
"print(json.dumps(llm_completion.metrics_store.get_metrics(), indent=2))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "dd9e7e19",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Disable Metrics\n",
|
|
"\n",
|
|
"Set `metrics` to `None` in the `ModelConfig` to disable metrics.\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 2,
|
|
"id": "44ab5fcd",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Metrics for: \n",
|
|
"{}\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"model_config.metrics = None\n",
|
|
"llm_completion_no_metrics: LLMCompletion = create_completion(model_config)\n",
|
|
"\n",
|
|
"response = llm_completion_no_metrics.completion(\n",
|
|
" messages=\"What is the capital of France?\",\n",
|
|
")\n",
|
|
"\n",
|
|
"# Now .metrics_store should be a NoOpMetricsStore\n",
|
|
"print(f\"Metrics for: {llm_completion_no_metrics.metrics_store.id}\")\n",
|
|
"print(json.dumps(llm_completion_no_metrics.metrics_store.get_metrics(), indent=2))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "f38a5a44",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Automatic Metrics Logging\n",
|
|
"\n",
|
|
"Metrics foreach instantiated model are automatically logged on process exit. To see this, update the log level to info.\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 3,
|
|
"id": "16b71da8",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"\u001b[92m22:45:27 - LiteLLM:INFO\u001b[0m: utils.py:3373 - \n",
|
|
"LiteLLM completion() model= gpt-4o; provider = azure\n",
|
|
"INFO:LiteLLM:\n",
|
|
"LiteLLM completion() model= gpt-4o; provider = azure\n",
|
|
"\u001b[92m22:45:27 - LiteLLM:INFO\u001b[0m: utils.py:1286 - Wrapper: Completed Call, calling success_handler\n",
|
|
"INFO:LiteLLM:Wrapper: Completed Call, calling success_handler\n",
|
|
"INFO:graphrag_llm.metrics.log_metrics_writer:Metrics for azure/gpt-4o: {\n",
|
|
" \"attempted_request_count\": 1,\n",
|
|
" \"successful_response_count\": 1,\n",
|
|
" \"failed_response_count\": 0,\n",
|
|
" \"failure_rate\": 0.0,\n",
|
|
" \"requests_with_retries\": 0,\n",
|
|
" \"retries\": 0,\n",
|
|
" \"retry_rate\": 0.0,\n",
|
|
" \"compute_duration_seconds\": 0.3004579544067383,\n",
|
|
" \"compute_duration_per_response_seconds\": 0.3004579544067383,\n",
|
|
" \"cache_hit_rate\": 0.0,\n",
|
|
" \"streaming_responses\": 0,\n",
|
|
" \"responses_with_tokens\": 1,\n",
|
|
" \"prompt_tokens\": 14,\n",
|
|
" \"completion_tokens\": 8,\n",
|
|
" \"total_tokens\": 22,\n",
|
|
" \"tokens_per_response\": 22.0,\n",
|
|
" \"responses_with_cost\": 1,\n",
|
|
" \"input_cost\": 3.5000000000000004e-05,\n",
|
|
" \"output_cost\": 8e-05,\n",
|
|
" \"total_cost\": 0.000115,\n",
|
|
" \"cost_per_response\": 0.000115\n",
|
|
"}\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"import logging\n",
|
|
"\n",
|
|
"logging.basicConfig(level=logging.INFO)\n",
|
|
"\n",
|
|
"llm_completion.metrics_store.clear_metrics()\n",
|
|
"response = llm_completion.completion(\n",
|
|
" messages=\"What is the capital of France?\",\n",
|
|
")\n",
|
|
"\n",
|
|
"# NOTE: Call _on_exit_ to simulate application exit since\n",
|
|
"# the notebook process does not exit and the llm_completion\n",
|
|
"# object is not garbage collected.\n",
|
|
"# This should not be called in normal python scripts.\n",
|
|
"llm_completion.metrics_store._on_exit_() # type: ignore"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "7d97bd8c",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Save Metrics to a File\n",
|
|
"\n",
|
|
"Instead of logging on exit, metrics can automatically be saved to a file on exit by using a `MetricsWriter.File` metrics writer.\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 4,
|
|
"id": "4c16806a",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"INFO:azure.identity._credentials.environment:No environment configuration found.\n",
|
|
"INFO:azure.identity._credentials.managed_identity:ManagedIdentityCredential will use IMDS\n",
|
|
"\u001b[92m22:45:27 - LiteLLM:INFO\u001b[0m: utils.py:3373 - \n",
|
|
"LiteLLM completion() model= gpt-4o; provider = azure\n",
|
|
"INFO:LiteLLM:\n",
|
|
"LiteLLM completion() model= gpt-4o; provider = azure\n",
|
|
"\u001b[92m22:45:28 - LiteLLM:INFO\u001b[0m: utils.py:1286 - Wrapper: Completed Call, calling success_handler\n",
|
|
"INFO:LiteLLM:Wrapper: Completed Call, calling success_handler\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Contents of metrics\\20260111_211007.jsonl:\n",
|
|
"{\"id\": \"azure/gpt-4o\", \"metrics\": {\"attempted_request_count\": 1, \"successful_response_count\": 1, \"failed_response_count\": 0, \"failure_rate\": 0.0, \"requests_with_retries\": 0, \"retries\": 0, \"retry_rate\": 0.0, \"compute_duration_seconds\": 0.6868698596954346, \"compute_duration_per_response_seconds\": 0.6868698596954346, \"streaming_responses\": 0, \"responses_with_tokens\": 1, \"prompt_tokens\": 14, \"completion_tokens\": 8, \"total_tokens\": 22, \"tokens_per_response\": 22.0, \"responses_with_cost\": 1, \"input_cost\": 3.5000000000000004e-05, \"output_cost\": 8e-05, \"total_cost\": 0.000115, \"cost_per_response\": 0.000115}}\n",
|
|
"\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"from pathlib import Path\n",
|
|
"\n",
|
|
"from graphrag_llm.config import MetricsConfig, MetricsWriterType\n",
|
|
"\n",
|
|
"model_config.metrics = MetricsConfig(\n",
|
|
" writer=MetricsWriterType.File,\n",
|
|
" base_dir=\"./metrics\", # Default\n",
|
|
")\n",
|
|
"llm_completion: LLMCompletion = create_completion(model_config)\n",
|
|
"\n",
|
|
"response = llm_completion.completion(\n",
|
|
" messages=\"What is the capital of France?\",\n",
|
|
")\n",
|
|
"\n",
|
|
"# NOTE: Call _on_exit_ to simulate application exit since\n",
|
|
"# the notebook process does not exit and the llm_completion\n",
|
|
"# object is not garbage collected.\n",
|
|
"# This should not be called in normal python scripts.\n",
|
|
"llm_completion.metrics_store._on_exit_() # type: ignore\n",
|
|
"\n",
|
|
"metrics_dir = Path(\"./metrics\")\n",
|
|
"for metric_file in metrics_dir.glob(\"*.jsonl\"):\n",
|
|
" print(f\"Contents of {metric_file}:\")\n",
|
|
" print(metric_file.read_text())\n",
|
|
" break # Just print one file for brevity"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "9076af04",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Default Metrics\n",
|
|
"\n",
|
|
"- `attempted_request_count`: Number of network requests made, not including retries.\n",
|
|
"- `successful_response_count`: Number of successful responses.\n",
|
|
"- `failed_response_count`: Number of network requests that threw errors and could not be resolved even after retries. `successful_response_count + failed_response_count` should equal `attempted_request_count` unless the job or process was killed early.\n",
|
|
"- `failure_rate`: `failed_response_count / attempted_request_count`.\n",
|
|
"- `requests_with_retries`: Number of original requests that had to go through a retry loop.\n",
|
|
"- `retries`: Number of network requests that were retries.\n",
|
|
"- `retry_rate`: `retries / (retries + attempted_request_count)`\n",
|
|
"- `compute_duration_seconds`: Total number of seconds to complete all non-streaming network requests.\n",
|
|
"- `compute_duration_per_response_seconds`: `compute_duration_seconds / successful non-streaming responses`\n",
|
|
"- `runtime_duration_seconds`: Only present if using the batching utilities. The batching utilities run multiple completions/embedding in parallel so `runtime_duration_seconds` is the actual runtime duration. Comparing this with `compute_duration_seconds` indicates how much time was saved using the batching utilities vs if all network requests ran in series.\n",
|
|
"- `cached_responses`: Number of cached responses. Only present if using a cache. When a response is cached so are the corresponding metrics. When a response is retrieved from the cache the corresponding metrics are also retrieved from the cache and provided in the overall metrics so metrics like `compute_duration_seconds`, `input_cost`, `output_cost`, etc include cached rsponses metrics. This is helpful when having to resume stopped jobs or rerunning failed jobs. At the end of the job the metrics indicate how long and costly the job would have been when running off a fresh cache/no cache. The `cached_responses` only indicates how many network requests were skipped and retrieved from cache.\n",
|
|
"- `streaming_responses`: Number of requests using the `stream=True` parameter. Many metrics such as token counts and costs are not tracked for streaming requests as that would require analyzing the stream to completion within the middleware stack and preventing the ability to build true streaming interfaces with `graphrag-llm`\n",
|
|
"- `responses_with_tokens`: Number of responses in which token counts were obtained. Typically this should equal `successful_response_count - streaming_responses`.\n",
|
|
"- `prompt_tokens`: Total number of prompt tokens used accross all successful non-streaming network requests.\n",
|
|
"- `completion_tokens`: Total number of completion tokens accress all succesful non-streaming network requests.\n",
|
|
"- `total_tokens`: `prompt_tokens + completion_tokens`\n",
|
|
"- `tokens_per_response`: `total_tokens / responses_with_tokens`\n",
|
|
"- `responses_with_cost`: Number of responses in which costs were calculated. typically this should equal `successful_response_count - streaming_responses`.\n",
|
|
"- `input_cost`: Cost of the input tokens accross all successful non-streaming network requests.\n",
|
|
"- `output_cost`: Cost of the output tokens accross all successful non-streaming network requests.\n",
|
|
"- `total_cost`: `input_cost + output_cost`\n",
|
|
"- `cost_per_response`: `total_cost / responses_with_cost`.\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "2749473e",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Custom Model Costs\n",
|
|
"\n",
|
|
"The default metrics include costs for prompt tokens and completion tokens. These are calculated using a registry of known models and associated costs managed by litellm: https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json\n",
|
|
"\n",
|
|
"One can register custom model costs if using a custom model that is not in the registry or one that differs from the known/default cost.\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "7a47f496",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"\u001b[92m22:45:28 - LiteLLM:INFO\u001b[0m: utils.py:3373 - \n",
|
|
"LiteLLM completion() model= gpt-4o; provider = azure\n",
|
|
"INFO:LiteLLM:\n",
|
|
"LiteLLM completion() model= gpt-4o; provider = azure\n",
|
|
"\u001b[92m22:45:28 - LiteLLM:INFO\u001b[0m: utils.py:1286 - Wrapper: Completed Call, calling success_handler\n",
|
|
"INFO:LiteLLM:Wrapper: Completed Call, calling success_handler\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"{\n",
|
|
" \"attempted_request_count\": 1,\n",
|
|
" \"successful_response_count\": 1,\n",
|
|
" \"failed_response_count\": 0,\n",
|
|
" \"failure_rate\": 0.0,\n",
|
|
" \"requests_with_retries\": 0,\n",
|
|
" \"retries\": 0,\n",
|
|
" \"retry_rate\": 0.0,\n",
|
|
" \"compute_duration_seconds\": 0.3090023994445801,\n",
|
|
" \"compute_duration_per_response_seconds\": 0.3090023994445801,\n",
|
|
" \"cache_hit_rate\": 0.0,\n",
|
|
" \"streaming_responses\": 0,\n",
|
|
" \"responses_with_tokens\": 1,\n",
|
|
" \"prompt_tokens\": 14,\n",
|
|
" \"completion_tokens\": 8,\n",
|
|
" \"total_tokens\": 22,\n",
|
|
" \"tokens_per_response\": 22.0,\n",
|
|
" \"responses_with_cost\": 1,\n",
|
|
" \"input_cost\": 14000,\n",
|
|
" \"output_cost\": 40000,\n",
|
|
" \"total_cost\": 54000,\n",
|
|
" \"cost_per_response\": 54000.0\n",
|
|
"}\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"from graphrag_llm.model_cost_registry import model_cost_registry\n",
|
|
"\n",
|
|
"model_cost_registry.register_model_costs(\n",
|
|
" model=\"azure/gpt-4o\", # This should use format \"{model_provider}/{model_name}\" and not the azure deployment name\n",
|
|
" costs={\n",
|
|
" # Expensive model\n",
|
|
" \"input_cost_per_token\": 1000,\n",
|
|
" \"output_cost_per_token\": 5000,\n",
|
|
" },\n",
|
|
")\n",
|
|
"\n",
|
|
"llm_completion.metrics_store.clear_metrics()\n",
|
|
"response = llm_completion.completion(\n",
|
|
" messages=\"What is the capital of France?\",\n",
|
|
")\n",
|
|
"\n",
|
|
"print(json.dumps(llm_completion.metrics_store.get_metrics(), indent=2))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "4132a718",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Custom Metrics Processor\n",
|
|
"\n",
|
|
"It is possible to register a custom metrics processor if one needs to track metrics not already tracked.\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 6,
|
|
"id": "f68ed4bb",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"INFO:azure.identity._credentials.environment:No environment configuration found.\n",
|
|
"INFO:azure.identity._credentials.managed_identity:ManagedIdentityCredential will use IMDS\n",
|
|
"\u001b[92m22:45:28 - LiteLLM:INFO\u001b[0m: utils.py:3373 - \n",
|
|
"LiteLLM completion() model= gpt-4o; provider = azure\n",
|
|
"INFO:LiteLLM:\n",
|
|
"LiteLLM completion() model= gpt-4o; provider = azure\n",
|
|
"\u001b[92m22:45:28 - LiteLLM:INFO\u001b[0m: utils.py:1286 - Wrapper: Completed Call, calling success_handler\n",
|
|
"INFO:LiteLLM:Wrapper: Completed Call, calling success_handler\n",
|
|
"\u001b[92m22:45:28 - LiteLLM:INFO\u001b[0m: utils.py:3373 - \n",
|
|
"LiteLLM completion() model= gpt-4o; provider = azure\n",
|
|
"INFO:LiteLLM:\n",
|
|
"LiteLLM completion() model= gpt-4o; provider = azure\n",
|
|
"\u001b[92m22:45:29 - LiteLLM:INFO\u001b[0m: utils.py:1286 - Wrapper: Completed Call, calling success_handler\n",
|
|
"INFO:LiteLLM:Wrapper: Completed Call, calling success_handler\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Metrics for: azure/gpt-4o\n",
|
|
"{\n",
|
|
" \"attempted_request_count\": 2,\n",
|
|
" \"successful_response_count\": 2,\n",
|
|
" \"failed_response_count\": 0,\n",
|
|
" \"failure_rate\": 0.0,\n",
|
|
" \"requests_with_retries\": 0,\n",
|
|
" \"retries\": 0,\n",
|
|
" \"retry_rate\": 0.0,\n",
|
|
" \"compute_duration_seconds\": 0.6117346286773682,\n",
|
|
" \"compute_duration_per_response_seconds\": 0.3058673143386841,\n",
|
|
" \"cache_hit_rate\": 0.0,\n",
|
|
" \"streaming_responses\": 0,\n",
|
|
" \"responses_with_tokens\": 2,\n",
|
|
" \"prompt_tokens\": 28,\n",
|
|
" \"completion_tokens\": 16,\n",
|
|
" \"total_tokens\": 44,\n",
|
|
" \"tokens_per_response\": 22.0,\n",
|
|
" \"responses_with_cost\": 2,\n",
|
|
" \"input_cost\": 28000,\n",
|
|
" \"output_cost\": 80000,\n",
|
|
" \"total_cost\": 108000,\n",
|
|
" \"cost_per_response\": 54000.0,\n",
|
|
" \"responses_with_temperature\": 1,\n",
|
|
" \"temperature_rate\": 0.5\n",
|
|
"}\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"import json\n",
|
|
"import os\n",
|
|
"from collections.abc import AsyncIterator, Iterator\n",
|
|
"from typing import Any\n",
|
|
"\n",
|
|
"from dotenv import load_dotenv\n",
|
|
"from graphrag_llm.completion import LLMCompletion, create_completion\n",
|
|
"from graphrag_llm.config import MetricsConfig, MetricsWriterType, ModelConfig\n",
|
|
"from graphrag_llm.metrics import metrics_aggregator, register_metrics_processor\n",
|
|
"from graphrag_llm.metrics.default_metrics_processor import DefaultMetricsProcessor\n",
|
|
"from graphrag_llm.types import (\n",
|
|
" LLMCompletionChunk,\n",
|
|
" LLMCompletionResponse,\n",
|
|
" LLMEmbeddingResponse,\n",
|
|
" Metrics,\n",
|
|
")\n",
|
|
"\n",
|
|
"load_dotenv()\n",
|
|
"\n",
|
|
"\n",
|
|
"class MyCustomMetricsProcessor(DefaultMetricsProcessor):\n",
|
|
" \"\"\"Custom metrics processor.\n",
|
|
"\n",
|
|
" Inheriting from DefaultMetricsProcessor to add to the default metrics being\n",
|
|
" tracked instead of implementing the interface from scratch.\n",
|
|
"\n",
|
|
" Metrics = dict[str, float]. The metrics passed to process_metrics method\n",
|
|
" represent the metrics for a single request. Typically, you will count/flag\n",
|
|
" metrics of interest per request and then aggregate them in the metrics_aggregator.\n",
|
|
" \"\"\"\n",
|
|
"\n",
|
|
" def __init__(self, some_custom_option: str, **kwargs: Any) -> None:\n",
|
|
" \"\"\"Initialize the custom metrics processor.\"\"\"\n",
|
|
" super().__init__(**kwargs)\n",
|
|
" self._some_custom_option = some_custom_option # Not actually used\n",
|
|
"\n",
|
|
" def process_metrics(\n",
|
|
" self,\n",
|
|
" *,\n",
|
|
" model_config: ModelConfig,\n",
|
|
" metrics: Metrics,\n",
|
|
" input_args: dict[str, Any],\n",
|
|
" response: LLMCompletionResponse\n",
|
|
" | Iterator[LLMCompletionChunk]\n",
|
|
" | AsyncIterator[LLMCompletionChunk]\n",
|
|
" | LLMEmbeddingResponse,\n",
|
|
" ) -> None:\n",
|
|
" \"\"\"On top of the default metrics, track if temperature argument was used.\n",
|
|
"\n",
|
|
" Expected to mutate the metrics dict in place with metrics you want to track.\n",
|
|
"\n",
|
|
" process_metrics is only called for successful requests and will be passed in the response\n",
|
|
" from either a completion or embedding call.\n",
|
|
"\n",
|
|
" Args\n",
|
|
" ----\n",
|
|
" model_config: ModelConfig\n",
|
|
" The model config used for the request.\n",
|
|
" metrics: Metrics\n",
|
|
" The metrics dict to be mutated in place.\n",
|
|
" input_args: dict[str, Any]\n",
|
|
" The input arguments passed to completion or embedding.\n",
|
|
" response: LLMChatCompletion | Iterator[LLMChatCompletionChunk] | LLMEmbeddingResponse\n",
|
|
" Either a completion or embedding response from the LLM.\n",
|
|
" \"\"\"\n",
|
|
" # Track default metrics first\n",
|
|
" super().process_metrics(\n",
|
|
" model_config=model_config,\n",
|
|
" metrics=metrics,\n",
|
|
" input_args=input_args,\n",
|
|
" response=response,\n",
|
|
" )\n",
|
|
"\n",
|
|
" metrics[\"responses_with_temperature\"] = 1 if \"temperature\" in input_args else 0\n",
|
|
"\n",
|
|
"\n",
|
|
"# Register custom metrics processor\n",
|
|
"register_metrics_processor(\n",
|
|
" processor_type=\"custom_with_temperature\",\n",
|
|
" processor_initializer=MyCustomMetricsProcessor,\n",
|
|
")\n",
|
|
"\n",
|
|
"\n",
|
|
"# Custom aggregator to calculate temperature usage rate\n",
|
|
"def _temperature_rate(metrics: \"Metrics\") -> None:\n",
|
|
" \"\"\"Calculate temperature usage rate.\n",
|
|
"\n",
|
|
" Custom aggregate function to track the usage rate of temperature parameter.\n",
|
|
"\n",
|
|
" Here, metrics represents the aggregated metrics for the current model.\n",
|
|
" \"\"\"\n",
|
|
" responses = metrics.get(\"successful_response_count\", 0)\n",
|
|
" temperature_responses = metrics.get(\"responses_with_temperature\", 0)\n",
|
|
" if responses > 0:\n",
|
|
" metrics[\"temperature_rate\"] = temperature_responses / responses\n",
|
|
" else:\n",
|
|
" metrics[\"temperature_rate\"] = 0.0\n",
|
|
"\n",
|
|
"\n",
|
|
"# Register custom aggregator\n",
|
|
"metrics_aggregator.register(\"temperature_rate\", _temperature_rate)\n",
|
|
"\n",
|
|
"api_key = os.getenv(\"GRAPHRAG_API_KEY\")\n",
|
|
"model_config = ModelConfig(\n",
|
|
" model_provider=\"azure\",\n",
|
|
" model=os.getenv(\"GRAPHRAG_MODEL\", \"gpt-4o\"),\n",
|
|
" azure_deployment_name=os.getenv(\"GRAPHRAG_MODEL\", \"gpt-4o\"),\n",
|
|
" api_base=os.getenv(\"GRAPHRAG_API_BASE\"),\n",
|
|
" api_version=os.getenv(\"GRAPHRAG_API_VERSION\", \"2025-04-01-preview\"),\n",
|
|
" api_key=api_key,\n",
|
|
" auth_method=AuthMethod.AzureManagedIdentity if not api_key else AuthMethod.ApiKey,\n",
|
|
" metrics=MetricsConfig(\n",
|
|
" # Use the custom metrics processor registered above\n",
|
|
" type=\"custom_with_temperature\",\n",
|
|
" some_custom_option=\"example_option_value\", # type: ignore\n",
|
|
" writer=MetricsWriterType.File,\n",
|
|
" base_dir=\"./metrics\", # Default\n",
|
|
" ),\n",
|
|
")\n",
|
|
"llm_completion: LLMCompletion = create_completion(model_config)\n",
|
|
"\n",
|
|
"response = llm_completion.completion(\n",
|
|
" messages=\"What is the capital of France?\",\n",
|
|
")\n",
|
|
"\n",
|
|
"response_with_temperature = llm_completion.completion(\n",
|
|
" messages=\"What is the capital of France?\",\n",
|
|
" temperature=0.7,\n",
|
|
")\n",
|
|
"\n",
|
|
"print(f\"Metrics for: {llm_completion.metrics_store.id}\")\n",
|
|
"print(json.dumps(llm_completion.metrics_store.get_metrics(), indent=2))"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.11.9"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|