chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,78 @@
|
||||
# Semantic Kernel Model-as-a-Service Sample
|
||||
|
||||
This sample contains a script to run multiple models against the popular [**Measuring Massive Multitask Language Understanding** (MMLU)](https://en.wikipedia.org/wiki/MMLU) dataset and produces results for benchmarking.
|
||||
|
||||
You can use this script as a starting point if you are planning to do the followings:
|
||||
1. You are developing a new dataset or augmenting an existing dataset for benchmarking Large Language Models.
|
||||
2. You would like to reproduce results from academic papers with existing datasets and models available on Azure AI Studio, such as the Phi series of models, or larger models like the Llama series and the Mistral large model. You can find model availabilities [here](https://learn.microsoft.com/en-us/azure/ai-studio/how-to/deploy-models-serverless-availability).
|
||||
|
||||
## Dataset
|
||||
|
||||
In this sample, we will be using the MMLU dataset hosted on HuggingFace.
|
||||
|
||||
To gain access to the dataset from HuggingFace, you will need a HuggingFace access token. Follow the steps [here](https://huggingface.co/docs/hub/security-tokens) to create one. You will be asked to provide the token when you run the sample.
|
||||
|
||||
The MMLU dataset has many subsets, organized by subjects. You can load whichever subjects you are interested in. Add or remove subjects by modifying the following line in the script:
|
||||
```Python
|
||||
datasets = load_mmlu_dataset(
|
||||
[
|
||||
"college_computer_science",
|
||||
"astronomy",
|
||||
"college_biology",
|
||||
"college_chemistry",
|
||||
"elementary_mathematics",
|
||||
# Add more subjects here.
|
||||
# See here for a full list of subjects: https://huggingface.co/datasets/cais/mmlu/viewer
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
## Models
|
||||
|
||||
This sample by default assumes three models: [Llama3-8b](https://learn.microsoft.com/en-us/azure/ai-studio/how-to/deploy-models-llama?tabs=llama-three#deploy-meta-llama-models-as-a-serverless-api), [Phi3-mini](https://learn.microsoft.com/en-us/azure/ai-studio/how-to/deploy-models-phi-3?tabs=phi-3-mini#deploy-phi-3-models-as-serverless-apis), and [Phi3-small](https://learn.microsoft.com/en-us/azure/ai-studio/how-to/deploy-models-phi-3?tabs=phi-3-small#deploy-phi-3-models-as-serverless-apis). However, you are free to add or remove models as long as it's available in the [model catalog](https://learn.microsoft.com/en-us/azure/ai-studio/how-to/model-catalog-overview).
|
||||
|
||||
Add a new model by adding a new AI service to the kernel in the script:
|
||||
```Python
|
||||
def setup_kernel():
|
||||
"""Set up the kernel."""
|
||||
...
|
||||
kernel.add_service(
|
||||
AzureAIInferenceChatCompletion(
|
||||
ai_model_id="",
|
||||
api_key="",
|
||||
endpoint="",
|
||||
)
|
||||
)
|
||||
...
|
||||
```
|
||||
The new service will automatically get picked up to run against the dataset.
|
||||
|
||||
The default models are selected based on the benchmark results reported on page 6 of the paper [**Phi-3 Technical Report:
|
||||
A Highly Capable Language Model Locally on Your Phone**](https://arxiv.org/pdf/2404.14219). In theory, Phi3-small will perform better than Phi3-mini, which will perform better than Llama3-8b. You should see the same result when you run this sample, though the numbers will not be the same, because this sample employs zero-shot learning whereas the report employed 5-shot learning.
|
||||
|
||||
Follow the steps [here](https://learn.microsoft.com/en-us/azure/ai-studio/how-to/deploy-models-serverless?tabs=azure-ai-studio) to deploy models of your choice.
|
||||
|
||||
> We intentionally use zero-shot so that you will have the opportunity to tune the prompt to get accuracies closer to what the report shows. You can tune the prompt in [helpers.py](helpers.py).
|
||||
|
||||
## Running the sample
|
||||
1. Deploy the required models. Follow the steps [here](https://learn.microsoft.com/en-us/azure/ai-studio/how-to/deploy-models-serverless?tabs=azure-ai-studio).
|
||||
2. Fill in the API keys and endpoints in the script.
|
||||
3. Open a terminal and activate your virtual environment for the Semantic Kernel project.
|
||||
4. Run `pip install datasets` to install the HuggingFace `datasets` module.
|
||||
5. Run `python mmlu_model_eval.py`
|
||||
|
||||
> If you are using VS code, you can simply select the interpreter in your virtual environment and click the run icon on the top right corner of the file panel when you focus on the script file.
|
||||
|
||||
## Results
|
||||
After the sample finishes running, you will see outputs similar to the following:
|
||||
```
|
||||
Finished evaluating college_biology.
|
||||
Accuracy of Llama3-8b: 75.00%.
|
||||
Accuracy of Phi3-mini: 81.25%.
|
||||
Accuracy of Phi3-small: 93.75%.
|
||||
...
|
||||
Overall results:
|
||||
Overall Accuracy of Llama3-8b: 51.09%.
|
||||
Overall Accuracy of Phi3-mini: 55.43%.
|
||||
Overall Accuracy of Phi3-small: 66.30%.
|
||||
```
|
||||
@@ -0,0 +1,32 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
|
||||
def formatted_system_message(subject: str):
|
||||
"""Return a formatted system message."""
|
||||
return f"""
|
||||
You are an expert in {subject}. You answer multiple choice questions on this topic.
|
||||
"""
|
||||
|
||||
|
||||
def formatted_question(question: str, answer_a: str, answer_b: str, answer_c: str, answer_d: str):
|
||||
"""Return a formatted question."""
|
||||
return f"""
|
||||
Question: {question}
|
||||
|
||||
Which of the following answers is correct?
|
||||
|
||||
A. {answer_a}
|
||||
B. {answer_b}
|
||||
C. {answer_c}
|
||||
D. {answer_d}
|
||||
|
||||
State ONLY the letter corresponding to the correct answer without any additional text.
|
||||
"""
|
||||
|
||||
|
||||
def expected_answer_to_letter(answer: str):
|
||||
"""Return the letter corresponding to the expected answer.
|
||||
|
||||
The dataset contains numbers as answers, this function converts them to letters.
|
||||
"""
|
||||
return ["A", "B", "C", "D"][int(answer)]
|
||||
@@ -0,0 +1,189 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import asyncio
|
||||
from typing import Annotated
|
||||
|
||||
from datasets import Dataset, DatasetDict, load_dataset
|
||||
from huggingface_hub import login
|
||||
from tqdm import tqdm
|
||||
|
||||
from samples.concepts.model_as_a_service.helpers import (
|
||||
expected_answer_to_letter,
|
||||
formatted_question,
|
||||
formatted_system_message,
|
||||
)
|
||||
from semantic_kernel.connectors.ai.azure_ai_inference import AzureAIInferenceChatCompletion
|
||||
from semantic_kernel.connectors.ai.chat_completion_client_base import ChatCompletionClientBase
|
||||
from semantic_kernel.contents.chat_history import ChatHistory
|
||||
from semantic_kernel.functions.kernel_arguments import KernelArguments
|
||||
from semantic_kernel.functions.kernel_function_decorator import kernel_function
|
||||
from semantic_kernel.kernel import Kernel
|
||||
|
||||
|
||||
def setup_kernel():
|
||||
"""Set up the kernel with AI services."""
|
||||
kernel = Kernel()
|
||||
|
||||
# Add multiple AI services to the kernel
|
||||
kernel.add_service(
|
||||
AzureAIInferenceChatCompletion(
|
||||
ai_model_id="Llama3-8b",
|
||||
api_key="",
|
||||
endpoint="",
|
||||
)
|
||||
)
|
||||
kernel.add_service(
|
||||
AzureAIInferenceChatCompletion(
|
||||
ai_model_id="Phi3-mini",
|
||||
api_key="",
|
||||
endpoint="",
|
||||
)
|
||||
)
|
||||
kernel.add_service(
|
||||
AzureAIInferenceChatCompletion(
|
||||
ai_model_id="Phi3-small",
|
||||
api_key="",
|
||||
endpoint="",
|
||||
)
|
||||
)
|
||||
|
||||
# Add the plugin to the kernel
|
||||
kernel.add_plugin(MMLUPlugin(), "MMLUPlugin")
|
||||
|
||||
return kernel
|
||||
|
||||
|
||||
def load_mmlu_dataset(subjects: list[str]) -> dict[str, Dataset]:
|
||||
"""Load the MMLU dataset for given subjects.
|
||||
|
||||
Args:
|
||||
subjects (list[str]): List of subjects to load the dataset for.
|
||||
|
||||
Returns:
|
||||
dict[str, Dataset]: A dictionary with subjects as keys and corresponding datasets as values.
|
||||
"""
|
||||
login()
|
||||
|
||||
datasets = {}
|
||||
number_of_samples = 0
|
||||
for subject in subjects:
|
||||
ds: DatasetDict = load_dataset("cais/mmlu", name=subject)
|
||||
validation_ds: Dataset = ds["validation"]
|
||||
datasets[subject] = validation_ds
|
||||
print(f"Loaded MMLU validation dataset for {subject}. This dataset has {validation_ds.num_rows} examples.")
|
||||
|
||||
number_of_samples += validation_ds.num_rows
|
||||
|
||||
print(f"Loaded {len(subjects)} datasets with a total of {number_of_samples} examples.")
|
||||
|
||||
return datasets
|
||||
|
||||
|
||||
class MMLUPlugin:
|
||||
"""A plugin for evaluating the MMLU dataset."""
|
||||
|
||||
@kernel_function(name="evaluate", description="Run a sample and return if the answer was correct.")
|
||||
async def evaluate(
|
||||
self,
|
||||
sample: Annotated[dict, "The sample"],
|
||||
subject: Annotated[str, "The subject of the sample"],
|
||||
kernel: Annotated[Kernel, "The kernel"],
|
||||
service_id: Annotated[str, "The service id"],
|
||||
) -> Annotated[bool, "Whether the answer was correct"]:
|
||||
"""Run a sample and return if the answer was correct.
|
||||
|
||||
Args:
|
||||
sample (dict): The sample containing the question and the correct answer.
|
||||
subject (str): The subject of the sample.
|
||||
kernel (Kernel): The kernel.
|
||||
service_id (str): The service id.
|
||||
|
||||
Returns:
|
||||
bool: Whether the answer was correct.
|
||||
"""
|
||||
# Initialize chat history with a system message
|
||||
chat_history = ChatHistory(system_message=formatted_system_message(subject))
|
||||
|
||||
# Add the user message to the chat history
|
||||
chat_history.add_user_message(
|
||||
formatted_question(
|
||||
sample["question"],
|
||||
sample["choices"][0],
|
||||
sample["choices"][1],
|
||||
sample["choices"][2],
|
||||
sample["choices"][3],
|
||||
),
|
||||
)
|
||||
|
||||
# Determine the correct answer
|
||||
correct_answer = expected_answer_to_letter(sample["answer"])
|
||||
|
||||
# Get the chat response from the AI service
|
||||
response = await kernel.get_service(service_id).get_chat_message_content(
|
||||
chat_history,
|
||||
settings=kernel.get_prompt_execution_settings_from_service_id(service_id),
|
||||
)
|
||||
|
||||
if not response:
|
||||
return False
|
||||
|
||||
# Compare the AI response with the correct answer
|
||||
return response.content.strip() == correct_answer
|
||||
|
||||
|
||||
async def main():
|
||||
# Load the MMLU dataset for specified subjects
|
||||
datasets = load_mmlu_dataset([
|
||||
"college_computer_science",
|
||||
"astronomy",
|
||||
"college_biology",
|
||||
"college_chemistry",
|
||||
"elementary_mathematics",
|
||||
# Add more subjects here.
|
||||
# See here for a full list of subjects: https://huggingface.co/datasets/cais/mmlu/viewer
|
||||
])
|
||||
|
||||
# Set up the kernel with AI services
|
||||
kernel = setup_kernel()
|
||||
ai_services = kernel.get_services_by_type(ChatCompletionClientBase).keys()
|
||||
|
||||
# Calculate total number of samples
|
||||
totals = sum([datasets[subject].num_rows for subject in datasets])
|
||||
|
||||
# Initialize counters for correct answers by each AI service
|
||||
total_corrects = {ai_service: 0.0 for ai_service in ai_services}
|
||||
for subject in datasets:
|
||||
corrects = {ai_service: 0.0 for ai_service in ai_services}
|
||||
print(f"Evaluating {subject}...")
|
||||
|
||||
# Evaluate each sample in the dataset
|
||||
for sample in tqdm(datasets[subject]):
|
||||
for ai_service in ai_services:
|
||||
kernel_arguments = KernelArguments(
|
||||
sample=sample,
|
||||
subject=subject,
|
||||
kernel=kernel,
|
||||
service_id=ai_service,
|
||||
)
|
||||
result = await kernel.invoke(
|
||||
plugin_name="MMLUPlugin", function_name="evaluate", arguments=kernel_arguments
|
||||
)
|
||||
|
||||
if result.value is True:
|
||||
corrects[ai_service] += 1
|
||||
|
||||
print(f"Finished evaluating {subject}.")
|
||||
|
||||
# Print accuracy for each AI service
|
||||
for ai_service in ai_services:
|
||||
total_corrects[ai_service] += corrects[ai_service]
|
||||
print(f"Accuracy of {ai_service}: {corrects[ai_service] / datasets[subject].num_rows * 100:.2f}%.")
|
||||
|
||||
# Print overall results
|
||||
print("Overall results:")
|
||||
for ai_service in ai_services:
|
||||
print(f"Overall Accuracy of {ai_service}: {total_corrects[ai_service] / totals * 100:.2f}%.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user