chore: import upstream snapshot with attribution

2026-07-13 13:21:23 +08:00
commit b957a53def
5423 changed files with 863745 additions and 0 deletions
@@ -0,0 +1,41 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Microsoft.Extensions.Logging;
+using Microsoft.SemanticKernel;
+using QualityCheckWithFilters.Models;
+using QualityCheckWithFilters.Services;
+
+namespace QualityCheckWithFilters.Filters;
+
+/// <summary>
+/// Filter which performs text summarization evaluation using BERTScore metric: https://huggingface.co/spaces/evaluate-metric/bertscore.
+/// Evaluation result contains three values: precision, recall and F1 score.
+/// The higher F1 score - the better the quality of the summary.
+/// </summary>
+internal sealed class BertSummarizationEvaluationFilter(
+    EvaluationService evaluationService,
+    ILogger logger,
+    double threshold) : IFunctionInvocationFilter
+{
+    public async Task OnFunctionInvocationAsync(FunctionInvocationContext context, Func<FunctionInvocationContext, Task> next)
+    {
+        await next(context);
+
+        var sourceText = context.Result.RenderedPrompt!;
+        var summary = context.Result.ToString();
+
+        var request = new SummarizationEvaluationRequest { Sources = [sourceText], Summaries = [summary] };
+        var response = await evaluationService.EvaluateAsync<SummarizationEvaluationRequest, BertSummarizationEvaluationResponse>(request);
+
+        var precision = Math.Round(response.Precision[0], 4);
+        var recall = Math.Round(response.Recall[0], 4);
+        var f1 = Math.Round(response.F1[0], 4);
+
+        logger.LogInformation("[BERT] Precision: {Precision}, Recall: {Recall}, F1: {F1}", precision, recall, f1);
+
+        if (f1 < threshold)
+        {
+            throw new KernelException($"BERT summary evaluation score ({f1}) is lower than threshold ({threshold})");
+        }
+    }
+}
@@ -0,0 +1,46 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Microsoft.Extensions.Logging;
+using Microsoft.SemanticKernel;
+using QualityCheckWithFilters.Models;
+using QualityCheckWithFilters.Services;
+
+namespace QualityCheckWithFilters.Filters;
+
+/// <summary>
+/// Filter which performs text summarization evaluation using BLEU metric: https://huggingface.co/spaces/evaluate-metric/bleu.
+/// Evaluation result contains values like score, precisions, brevity penalty and length ratio.
+/// The closer the score and precision values are to 1 - the better the quality of the summary.
+/// </summary>
+internal sealed class BleuSummarizationEvaluationFilter(
+    EvaluationService evaluationService,
+    ILogger logger,
+    double threshold) : IFunctionInvocationFilter
+{
+    public async Task OnFunctionInvocationAsync(FunctionInvocationContext context, Func<FunctionInvocationContext, Task> next)
+    {
+        await next(context);
+
+        var sourceText = context.Result.RenderedPrompt!;
+        var summary = context.Result.ToString();
+
+        var request = new SummarizationEvaluationRequest { Sources = [sourceText], Summaries = [summary] };
+        var response = await evaluationService.EvaluateAsync<SummarizationEvaluationRequest, BleuSummarizationEvaluationResponse>(request);
+
+        var score = Math.Round(response.Score, 4);
+        var precisions = response.Precisions.Select(l => Math.Round(l, 4)).ToList();
+        var brevityPenalty = Math.Round(response.BrevityPenalty, 4);
+        var lengthRatio = Math.Round(response.LengthRatio, 4);
+
+        logger.LogInformation("[BLEU] Score: {Score}, Precisions: {Precisions}, Brevity penalty: {BrevityPenalty}, Length Ratio: {LengthRatio}",
+            score,
+            string.Join(", ", precisions),
+            brevityPenalty,
+            lengthRatio);
+
+        if (precisions[0] < threshold)
+        {
+            throw new KernelException($"BLEU summary evaluation score ({precisions[0]}) is lower than threshold ({threshold})");
+        }
+    }
+}
@@ -0,0 +1,40 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Microsoft.Extensions.Logging;
+using Microsoft.SemanticKernel;
+using QualityCheckWithFilters.Models;
+using QualityCheckWithFilters.Services;
+
+namespace QualityCheckWithFilters.Filters;
+
+/// <summary>
+/// Filter which performs text translation evaluation using COMET metric: https://huggingface.co/Unbabel/wmt22-cometkiwi-da.
+/// COMET score ranges from 0 to 1, where higher values indicate better translation.
+/// </summary>
+internal sealed class CometTranslationEvaluationFilter(
+    EvaluationService evaluationService,
+    ILogger logger,
+    double threshold) : IFunctionInvocationFilter
+{
+    public async Task OnFunctionInvocationAsync(FunctionInvocationContext context, Func<FunctionInvocationContext, Task> next)
+    {
+        await next(context);
+
+        var sourceText = context.Result.RenderedPrompt!;
+        var translation = context.Result.ToString();
+
+        logger.LogInformation("Translation: {Translation}", translation);
+
+        var request = new TranslationEvaluationRequest { Sources = [sourceText], Translations = [translation] };
+        var response = await evaluationService.EvaluateAsync<TranslationEvaluationRequest, CometTranslationEvaluationResponse>(request);
+
+        var score = Math.Round(response.Scores[0], 4);
+
+        logger.LogInformation("[COMET] Score: {Score}", score);
+
+        if (score < threshold)
+        {
+            throw new KernelException($"COMET translation evaluation score ({score}) is lower than threshold ({threshold})");
+        }
+    }
+}
@@ -0,0 +1,25 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Microsoft.Extensions.Logging;
+using Microsoft.SemanticKernel;
+using QualityCheckWithFilters.Models;
+using QualityCheckWithFilters.Services;
+
+namespace QualityCheckWithFilters.Filters;
+
+/// <summary>
+/// Factory class for function invocation filters based on evaluation score type.
+/// </summary>
+internal sealed class FilterFactory
+{
+    private static readonly Dictionary<EvaluationScoreType, Func<EvaluationService, ILogger, double, IFunctionInvocationFilter>> s_filters = new()
+    {
+        [EvaluationScoreType.BERT] = (service, logger, threshold) => new BertSummarizationEvaluationFilter(service, logger, threshold),
+        [EvaluationScoreType.BLEU] = (service, logger, threshold) => new BleuSummarizationEvaluationFilter(service, logger, threshold),
+        [EvaluationScoreType.METEOR] = (service, logger, threshold) => new MeteorSummarizationEvaluationFilter(service, logger, threshold),
+        [EvaluationScoreType.COMET] = (service, logger, threshold) => new CometTranslationEvaluationFilter(service, logger, threshold),
+    };
+
+    public static IFunctionInvocationFilter Create(EvaluationScoreType type, EvaluationService evaluationService, ILogger logger, double threshold)
+        => s_filters[type].Invoke(evaluationService, logger, threshold);
+}
@@ -0,0 +1,38 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Microsoft.Extensions.Logging;
+using Microsoft.SemanticKernel;
+using QualityCheckWithFilters.Models;
+using QualityCheckWithFilters.Services;
+
+namespace QualityCheckWithFilters.Filters;
+
+/// <summary>
+/// Filter which performs text summarization evaluation using METEOR metric: https://huggingface.co/spaces/evaluate-metric/meteor.
+/// METEOR score ranges from 0 to 1, where higher values indicate better similarity between original text and generated summary.
+/// </summary>
+internal sealed class MeteorSummarizationEvaluationFilter(
+    EvaluationService evaluationService,
+    ILogger logger,
+    double threshold) : IFunctionInvocationFilter
+{
+    public async Task OnFunctionInvocationAsync(FunctionInvocationContext context, Func<FunctionInvocationContext, Task> next)
+    {
+        await next(context);
+
+        var sourceText = context.Result.RenderedPrompt!;
+        var summary = context.Result.ToString();
+
+        var request = new SummarizationEvaluationRequest { Sources = [sourceText], Summaries = [summary] };
+        var response = await evaluationService.EvaluateAsync<SummarizationEvaluationRequest, MeteorSummarizationEvaluationResponse>(request);
+
+        var score = Math.Round(response.Score, 4);
+
+        logger.LogInformation("[METEOR] Score: {Score}", score);
+
+        if (score < threshold)
+        {
+            throw new KernelException($"METEOR summary evaluation score ({score}) is lower than threshold ({threshold})");
+        }
+    }
+}
@@ -0,0 +1,26 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Text.Json.Serialization;
+
+namespace QualityCheckWithFilters.Models;
+
+/// <summary>Base request model with source texts.</summary>
+internal class EvaluationRequest
+{
+    [JsonPropertyName("sources")]
+    public List<string> Sources { get; set; }
+}
+
+/// <summary>Request model with generated summaries.</summary>
+internal sealed class SummarizationEvaluationRequest : EvaluationRequest
+{
+    [JsonPropertyName("summaries")]
+    public List<string> Summaries { get; set; }
+}
+
+/// <summary>Request model with generated translations.</summary>
+internal sealed class TranslationEvaluationRequest : EvaluationRequest
+{
+    [JsonPropertyName("translations")]
+    public List<string> Translations { get; set; }
+}
@@ -0,0 +1,51 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Text.Json.Serialization;
+
+namespace QualityCheckWithFilters.Models;
+
+/// <summary>Response model for BERTScore metric: https://huggingface.co/spaces/evaluate-metric/bertscore.</summary>
+internal sealed class BertSummarizationEvaluationResponse
+{
+    [JsonPropertyName("precision")]
+    public List<double> Precision { get; set; }
+
+    [JsonPropertyName("recall")]
+    public List<double> Recall { get; set; }
+
+    [JsonPropertyName("f1")]
+    public List<double> F1 { get; set; }
+}
+
+/// <summary>Response model for BLEU metric: https://huggingface.co/spaces/evaluate-metric/bleu.</summary>
+internal sealed class BleuSummarizationEvaluationResponse
+{
+    [JsonPropertyName("bleu")]
+    public double Score { get; set; }
+
+    [JsonPropertyName("precisions")]
+    public List<double> Precisions { get; set; }
+
+    [JsonPropertyName("brevity_penalty")]
+    public double BrevityPenalty { get; set; }
+
+    [JsonPropertyName("length_ratio")]
+    public double LengthRatio { get; set; }
+}
+
+/// <summary>Response model for METEOR metric: https://huggingface.co/spaces/evaluate-metric/meteor.</summary>
+internal sealed class MeteorSummarizationEvaluationResponse
+{
+    [JsonPropertyName("meteor")]
+    public double Score { get; set; }
+}
+
+/// <summary>Response model for COMET metric: https://huggingface.co/Unbabel/wmt22-cometkiwi-da.</summary>
+internal sealed class CometTranslationEvaluationResponse
+{
+    [JsonPropertyName("scores")]
+    public List<double> Scores { get; set; }
+
+    [JsonPropertyName("system_score")]
+    public double SystemScore { get; set; }
+}
@@ -0,0 +1,33 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Diagnostics.CodeAnalysis;
+
+namespace QualityCheckWithFilters.Models;
+
+/// <summary>
+/// Internal representation of evaluation score type to configure and run examples.
+/// </summary>
+internal readonly struct EvaluationScoreType(string endpoint) : IEquatable<EvaluationScoreType>
+{
+    public string Endpoint { get; } = endpoint;
+
+    public static EvaluationScoreType BERT = new("bert-score");
+    public static EvaluationScoreType BLEU = new("bleu-score");
+    public static EvaluationScoreType METEOR = new("meteor-score");
+    public static EvaluationScoreType COMET = new("comet-score");
+
+    public static bool operator ==(EvaluationScoreType left, EvaluationScoreType right) => left.Equals(right);
+    public static bool operator !=(EvaluationScoreType left, EvaluationScoreType right) => !(left == right);
+
+    /// <inheritdoc/>
+    public override bool Equals([NotNullWhen(true)] object? obj) => obj is EvaluationScoreType other && this == other;
+
+    /// <inheritdoc/>
+    public bool Equals(EvaluationScoreType other) => string.Equals(this.Endpoint, other.Endpoint, StringComparison.OrdinalIgnoreCase);
+
+    /// <inheritdoc/>
+    public override int GetHashCode() => StringComparer.OrdinalIgnoreCase.GetHashCode(this.Endpoint ?? string.Empty);
+
+    /// <inheritdoc/>
+    public override string ToString() => this.Endpoint ?? string.Empty;
+}
@@ -0,0 +1,213 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using Microsoft.SemanticKernel;
+using Microsoft.SemanticKernel.ChatCompletion;
+using QualityCheckWithFilters.Filters;
+using QualityCheckWithFilters.Models;
+using QualityCheckWithFilters.Services;
+
+namespace QualityCheckWithFilters;
+
+public class Program
+{
+    /// <summary>
+    /// This example demonstrates how to evaluate LLM results on tasks such as text summarization and translation
+    /// using following metrics:
+    /// - BERTScore: https://github.com/Tiiiger/bert_score
+    /// - BLEU (BiLingual Evaluation Understudy): https://en.wikipedia.org/wiki/BLEU
+    /// - METEOR (Metric for Evaluation of Translation with Explicit ORdering): https://en.wikipedia.org/wiki/METEOR
+    /// - COMET (Crosslingual Optimized Metric for Evaluation of Translation): https://unbabel.github.io/COMET
+    /// Semantic Kernel Filters are used to perform following tasks during function invocation:
+    /// 1. Get original text to summarize/translate.
+    /// 2. Get LLM result.
+    /// 3. Call evaluation server to get specific metric score.
+    /// 4. Compare metric score to configured threshold and throw an exception if score is lower.
+    /// </summary>
+    public static async Task Main()
+    {
+        await SummarizationEvaluationAsync(EvaluationScoreType.BERT, threshold: 0.85);
+
+        // Output:
+        // Extractive summary: [BERT] Precision: 0.9756, Recall: 0.9114, F1: 0.9424
+        // Abstractive summary: [BERT] Precision: 0.8953, Recall: 0.8656, F1: 0.8802
+        // Random summary: [BERT] Precision: 0.8433, Recall: 0.787, F1: 0.8142
+        // Exception occurred during function invocation: BERT summary evaluation score (0.8142) is lower than threshold (0.85)
+
+        await SummarizationEvaluationAsync(EvaluationScoreType.BLEU, threshold: 0.5);
+
+        // Output:
+        // Extractive summary: [BLEU] Score: 0.3281, Precisions: 1, 1, 0.9726, 0.9444, Brevity penalty: 0.3351, Length Ratio: 0.4777
+        // Abstractive summary: [BLEU] Score: 0, Precisions: 0.678, 0.1552, 0.0175, 0, Brevity penalty: 0.1899, Length Ratio: 0.3758
+        // Random summary: [BLEU] Score: 0, Precisions: 0.2, 0, 0, 0, Brevity penalty: 0, Length Ratio: 0.0318
+        // Exception occurred during function invocation: BLEU summary evaluation score (0.2) is lower than threshold (0.5)
+
+        await SummarizationEvaluationAsync(EvaluationScoreType.METEOR, threshold: 0.1);
+
+        // Output:
+        // Extractive summary: [METEOR] Score: 0.438
+        // Abstractive summary: [METEOR] Score: 0.1661
+        // Random summary: [METEOR] Score: 0.0035
+        // Exception occurred during function invocation: METEOR summary evaluation score (0.0035) is lower than threshold (0.1)
+
+        await TranslationEvaluationAsync(threshold: 0.4);
+
+        // Output:
+        // Text to translate: Berlin ist die Hauptstadt der Deutschland.
+        // Translation: Berlin is the capital of Germany - [COMET] Score: 0.8695
+        // Translation: Berlin capital Germany is of The - [COMET] Score: 0.4724
+        // Translation: This is random translation - [COMET] Score: 0.3525
+        // Exception occurred during function invocation: COMET translation evaluation score (0.3525) is lower than threshold (0.4)
+    }
+
+    #region Scenarios
+
+    /// <summary>
+    /// This method performs summarization evaluation and compare following types of summaries:
+    /// - Extractive summary: involves selecting and extracting key sentences, phrases, or segments directly from the original text to create a summary.
+    /// - Abstractive summary: involves generating new sentences that convey the key information from the original text.
+    /// - Random summary: unrelated text to original source for comparison purposes.
+    /// </summary>
+    private static async Task SummarizationEvaluationAsync(EvaluationScoreType scoreType, double threshold)
+    {
+        // Define text to summarize and possible LLM summaries.
+        const string TextToSummarize =
+            """
+            The sun rose over the horizon, casting a warm glow across the landscape.
+            Birds began to chirp, greeting the new day with their melodious songs.
+            The flowers in the garden slowly opened their petals, revealing vibrant colors and delicate fragrances.
+            A gentle breeze rustled through the trees, creating a soothing sound that complemented the morning stillness.
+            People started to emerge from their homes, ready to embark on their daily routines.
+            Some went for a morning jog, enjoying the fresh air and the peaceful surroundings.
+            Others sipped their coffee while reading the newspaper on their porches.
+            The streets gradually filled with the hum of cars and the chatter of pedestrians.
+            In the park, children played joyfully, their laughter echoing through the air.
+            As the day progressed, the town buzzed with activity, each moment bringing new opportunities and experiences.
+            """;
+
+        const string ExtractiveSummary =
+            """
+            The sun rose over the horizon, casting a warm glow across the landscape.
+            Birds began to chirp, greeting the new day with their melodious songs.
+            People started to emerge from their homes, ready to embark on their daily routines.
+            The streets gradually filled with the hum of cars and the chatter of pedestrians.
+            In the park, children played joyfully, their laughter echoing through the air.
+            """;
+
+        const string AbstractiveSummary =
+            """
+            As the sun rises, nature awakens with birds singing and flowers blooming.
+            People begin their day with various routines, from jogging to enjoying coffee.
+            The town gradually becomes lively with the sounds of traffic and children's laughter in the park,
+            marking the start of a bustling day filled with new activities and opportunities.
+            """;
+
+        const string RandomSummary =
+            """
+            This is random text.
+            """;
+
+        // Get kernel builder with initial configuration.
+        var builder = GetKernelBuilder(scoreType, threshold);
+
+        // It doesn't matter which LLM to use for text summarization, since the main goal is to demonstrate how to evaluate the result and compare metrics.
+        // For demonstration purposes, fake chat completion service is used to simulate LLM response with predefined summary.
+        builder.Services.AddSingleton<IChatCompletionService>(new FakeChatCompletionService("extractive-summary-model", ExtractiveSummary));
+        builder.Services.AddSingleton<IChatCompletionService>(new FakeChatCompletionService("abstractive-summary-model", AbstractiveSummary));
+        builder.Services.AddSingleton<IChatCompletionService>(new FakeChatCompletionService("random-summary-model", RandomSummary));
+
+        // Build kernel
+        var kernel = builder.Build();
+
+        // Invoke function to perform text summarization with predefined result, trigger function invocation filter and evaluate the result.
+        await InvokeAsync(kernel, TextToSummarize, "extractive-summary-model");
+        await InvokeAsync(kernel, TextToSummarize, "abstractive-summary-model");
+        await InvokeAsync(kernel, TextToSummarize, "random-summary-model");
+    }
+
+    /// <summary>
+    /// This method performs translation evaluation and compare the results.
+    /// </summary>
+    private static async Task TranslationEvaluationAsync(double threshold)
+    {
+        EvaluationScoreType scoreType = EvaluationScoreType.COMET;
+
+        // Define text to translate and possible LLM translations.
+        const string TextToTranslate = "Berlin ist die Hauptstadt der Deutschland.";
+        const string Translation1 = "Berlin is the capital of Germany.";
+        const string Translation2 = "Berlin capital Germany is of The.";
+        const string Translation3 = "This is random translation.";
+
+        // Get kernel builder with initial configuration.
+        var builder = GetKernelBuilder(scoreType, threshold);
+
+        // It doesn't matter which LLM to use for text translation, since the main goal is to demonstrate how to evaluate the result and compare metrics.
+        // For demonstration purposes, fake chat completion service is used to simulate LLM response with predefined translation.
+        builder.Services.AddSingleton<IChatCompletionService>(new FakeChatCompletionService("translation-1-model", Translation1));
+        builder.Services.AddSingleton<IChatCompletionService>(new FakeChatCompletionService("translation-2-model", Translation2));
+        builder.Services.AddSingleton<IChatCompletionService>(new FakeChatCompletionService("translation-3-model", Translation3));
+
+        // Build kernel
+        var kernel = builder.Build();
+
+        // Invoke function to perform text translation with predefined result, trigger function invocation filter and evaluate the result.
+        await InvokeAsync(kernel, TextToTranslate, "translation-1-model");
+        await InvokeAsync(kernel, TextToTranslate, "translation-2-model");
+        await InvokeAsync(kernel, TextToTranslate, "translation-3-model");
+    }
+
+    #endregion
+
+    #region Helpers
+
+    /// <summary>
+    /// Gets kernel builder with initial configuration.
+    /// </summary>
+    private static IKernelBuilder GetKernelBuilder(EvaluationScoreType scoreType, double threshold)
+    {
+        // Create kernel builder
+        var builder = Kernel.CreateBuilder();
+
+        // Add logging
+        builder.Services.AddLogging(loggingBuilder => loggingBuilder.AddConsole().SetMinimumLevel(LogLevel.Information));
+
+        // Add default HTTP client with base address to local evaluation server
+        builder.Services.AddHttpClient("default", client => { client.BaseAddress = new Uri("http://localhost:8080"); });
+
+        // Add service which performs HTTP requests to evaluation server
+        builder.Services.AddSingleton<EvaluationService>(
+            sp => new EvaluationService(
+                sp.GetRequiredService<IHttpClientFactory>().CreateClient("default"),
+                scoreType.Endpoint));
+
+        // Add function invocation filter to perform evaluation and compare metric score with configured threshold
+        builder.Services.AddSingleton<IFunctionInvocationFilter>(
+            sp => FilterFactory.Create(
+                scoreType,
+                sp.GetRequiredService<EvaluationService>(),
+                sp.GetRequiredService<ILogger<Program>>(),
+                threshold));
+
+        return builder;
+    }
+
+    /// <summary>
+    /// Invokes kernel function with provided input and model ID.
+    /// </summary>
+    private static async Task InvokeAsync(Kernel kernel, string input, string modelId)
+    {
+        var logger = kernel.Services.GetRequiredService<ILogger<Program>>();
+
+        try
+        {
+            await kernel.InvokePromptAsync(input, new(new PromptExecutionSettings { ModelId = modelId }));
+        }
+        catch (KernelException exception)
+        {
+            logger.LogError(exception, "Exception occurred during function invocation: {Message}", exception.Message);
+        }
+    }
+
+    #endregion
+}
@@ -0,0 +1,17 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net10.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <NoWarn>$(NoWarn);VSTHRD111,CA2007,CS8618,CS1591,CA1052,SKEXP0001</NoWarn>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Extensions.Http" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Console" />
+    <ProjectReference Include="..\..\..\..\src\SemanticKernel.Core\SemanticKernel.Core.csproj" />
+  </ItemGroup>
+
+</Project>
@@ -0,0 +1,28 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Text;
+using System.Text.Json;
+using QualityCheckWithFilters.Models;
+
+namespace QualityCheckWithFilters.Services;
+
+/// <summary>
+/// Service which performs HTTP requests to evaluation server.
+/// </summary>
+internal sealed class EvaluationService(HttpClient httpClient, string endpoint)
+{
+    public async Task<TResponse> EvaluateAsync<TRequest, TResponse>(TRequest request)
+        where TRequest : EvaluationRequest
+    {
+        var requestContent = new StringContent(JsonSerializer.Serialize(request), Encoding.UTF8, "application/json");
+
+        var response = await httpClient.PostAsync(new Uri(endpoint, UriKind.Relative), requestContent);
+
+        response.EnsureSuccessStatusCode();
+
+        var responseContent = await response.Content.ReadAsStringAsync();
+
+        return JsonSerializer.Deserialize<TResponse>(responseContent) ??
+            throw new Exception("Response is not available.");
+    }
+}
@@ -0,0 +1,28 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Runtime.CompilerServices;
+using Microsoft.SemanticKernel;
+using Microsoft.SemanticKernel.ChatCompletion;
+using Microsoft.SemanticKernel.Services;
+
+namespace QualityCheckWithFilters.Services;
+
+#pragma warning disable CS1998
+
+/// <summary>
+/// Fake chat completion service to simulate a call to LLM and return predefined result for demonstration purposes.
+/// </summary>
+internal sealed class FakeChatCompletionService(string modelId, string result) : IChatCompletionService
+{
+    public IReadOnlyDictionary<string, object?> Attributes => new Dictionary<string, object?> { [AIServiceExtensions.ModelIdKey] = modelId };
+
+    public Task<IReadOnlyList<ChatMessageContent>> GetChatMessageContentsAsync(ChatHistory chatHistory, PromptExecutionSettings? executionSettings = null, Kernel? kernel = null, CancellationToken cancellationToken = default)
+    {
+        return Task.FromResult<IReadOnlyList<ChatMessageContent>>([new(AuthorRole.Assistant, result)]);
+    }
+
+    public async IAsyncEnumerable<StreamingChatMessageContent> GetStreamingChatMessageContentsAsync(ChatHistory chatHistory, PromptExecutionSettings? executionSettings = null, Kernel? kernel = null, [EnumeratorCancellation] CancellationToken cancellationToken = default)
+    {
+        yield return new StreamingChatMessageContent(AuthorRole.Assistant, result);
+    }
+}
@@ -0,0 +1,106 @@
+# Quality Check with Filters
+
+This sample provides a practical demonstration how to perform quality check on LLM results for such tasks as text summarization and translation with Semantic Kernel Filters.
+
+Metrics used in this example:
+
+- [BERTScore](https://github.com/Tiiiger/bert_score) - leverages the pre-trained contextual embeddings from BERT and matches words in candidate and reference sentences by cosine similarity.
+- [BLEU](https://en.wikipedia.org/wiki/BLEU) (BiLingual Evaluation Understudy) - evaluates the quality of text which has been machine-translated from one natural language to another.
+- [METEOR](https://en.wikipedia.org/wiki/METEOR) (Metric for Evaluation of Translation with Explicit ORdering) - evaluates the similarity between the generated summary and the reference summary, taking into account grammar and semantics.
+- [COMET](https://unbabel.github.io/COMET) (Crosslingual Optimized Metric for Evaluation of Translation) - is an open-source framework used to train Machine Translation metrics that achieve high levels of correlation with different types of human judgments.
+
+In this example, SK Filters call dedicated [server](./python-server/) which is responsible for task evaluation using metrics described above. If evaluation score of specific metric doesn't meet configured threshold, an exception is thrown with evaluation details.
+
+[Hugging Face Evaluate Metric](https://github.com/huggingface/evaluate) library is used to evaluate summarization and translation results.
+
+## Prerequisites
+
+1. [Python 3.12](https://www.python.org/downloads/)
+2. Get [Hugging Face API token](https://huggingface.co/docs/api-inference/en/quicktour#get-your-api-token).
+3. Accept conditions to access [Unbabel/wmt22-cometkiwi-da](https://huggingface.co/Unbabel/wmt22-cometkiwi-da) model on Hugging Face portal.
+
+## Setup
+
+It's possible to run Python server for task evaluation directly or with Docker.
+
+### Run server
+
+1. Open Python server directory:
+
+```bash
+cd python-server
+```
+
+2. Create and active virtual environment:
+
+```bash
+python -m venv venv
+source venv/Scripts/activate # activate on Windows
+source venv/bin/activate # activate on Unix/MacOS
+```
+
+3. Setup Hugging Face API key:
+
+```bash
+pip install "huggingface_hub[cli]"
+huggingface-cli login --token <your_token>
+```
+
+4. Install dependencies:
+
+```bash
+pip install -r requirements.txt
+```
+
+5. Run server:
+
+```bash
+cd app
+uvicorn main:app --port 8080 --reload
+```
+
+6. Open `http://localhost:8080/docs` and check available endpoints.
+
+### Run server with Docker
+
+1. Open Python server directory:
+
+```bash
+cd python-server
+```
+
+2. Create following `Dockerfile`:
+
+```dockerfile
+# syntax=docker/dockerfile:1.2
+FROM python:3.12
+
+WORKDIR /code
+
+COPY ./requirements.txt /code/requirements.txt
+
+RUN pip install "huggingface_hub[cli]"
+RUN --mount=type=secret,id=hf_token \
+    huggingface-cli login --token $(cat /run/secrets/hf_token)
+
+RUN pip install cmake
+RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
+
+COPY ./app /code/app
+
+CMD ["fastapi", "run", "app/main.py", "--port", "80"]
+```
+
+3. Create `.env/hf_token.txt` file and put Hugging Face API token in it.
+
+4. Build image and run container:
+
+```bash
+docker-compose up --build
+```
+
+5. Open `http://localhost:8080/docs` and check available endpoints.
+
+## Testing
+
+Open and run `QualityCheckWithFilters/Program.cs` to experiment with different evaluation metrics, thresholds and input parameters.
@@ -0,0 +1,40 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+from typing import List
+from pydantic import BaseModel
+
+from fastapi import FastAPI
+from evaluate import load
+from comet import download_model, load_from_checkpoint
+
+app = FastAPI()
+
+class SummarizationEvaluationRequest(BaseModel):
+    sources: List[str]
+    summaries: List[str]
+
+class TranslationEvaluationRequest(BaseModel):
+    sources: List[str]
+    translations: List[str]
+
+@app.post("/bert-score/")
+def bert_score(request: SummarizationEvaluationRequest):
+    bertscore = load("bertscore")
+    return bertscore.compute(predictions=request.summaries, references=request.sources, lang="en")
+
+@app.post("/meteor-score/")
+def meteor_score(request: SummarizationEvaluationRequest):
+    meteor = load("meteor")
+    return meteor.compute(predictions=request.summaries, references=request.sources)
+
+@app.post("/bleu-score/")
+def bleu_score(request: SummarizationEvaluationRequest):
+    bleu = load("bleu")
+    return bleu.compute(predictions=request.summaries, references=request.sources)
+
+@app.post("/comet-score/")
+def comet_score(request: TranslationEvaluationRequest):
+    model_path = download_model("Unbabel/wmt22-cometkiwi-da")
+    model = load_from_checkpoint(model_path)
+    data = [{"src": src, "mt": mt} for src, mt in zip(request.sources, request.translations)]
+    return model.predict(data, accelerator="cpu")
@@ -0,0 +1,16 @@
+version: '3.8'
+
+services:
+  quality-check:
+    build:
+      context: .
+      dockerfile: Dockerfile
+      secrets:
+        - hf_token
+    ports: 
+      - "8080:80"
+    secrets:
+      - hf_token
+secrets:
+  hf_token:
+    file: .env/hf_token.txt
@@ -0,0 +1,8 @@
+fastapi
+uvicorn
+pydantic
+bert_score
+nltk
+evaluate
+cmake
+unbabel-comet