Implement BLEU score evaluation for NLP tests

eng/MSBuild/LegacySupport.props

@@ -74,4 +74,8 @@
   <ItemGroup Condition="'$(InjectPlatformAttributesOnLegacy)' == 'true' AND ('$(TargetFramework)' == 'net462' or '$(TargetFramework)' == 'netstandard2.0' or '$(TargetFramework)' == 'netcoreapp3.1')">
     <Compile Include="$(MSBuildThisFileDirectory)\..\..\src\LegacySupport\PlatformAttributes\*.cs" LinkBase="LegacySupport\PlatformAttributes" />
   </ItemGroup>
+
+  <ItemGroup Condition="'$(InjectCollectionBuilderAttributesOnLegacy)' == 'true' AND ('$(TargetFramework)' == 'net462' or '$(TargetFramework)' == 'netstandard2.0' or '$(TargetFramework)' == 'netcoreapp3.1')">
+    <Compile Include="$(MSBuildThisFileDirectory)\..\..\src\LegacySupport\CollectionBuilder\*.cs" LinkBase="LegacySupport\CollectionBuilder" />
+  </ItemGroup>
 </Project>

src/LegacySupport/CollectionBuilder/CollectionBuilderAttribute.cs

@@ -0,0 +1,17 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+namespace System.Runtime.CompilerServices;
+
+[AttributeUsage(AttributeTargets.Class | AttributeTargets.Struct | AttributeTargets.Interface)]
+internal sealed class CollectionBuilderAttribute : Attribute
+{
+    public CollectionBuilderAttribute(Type builderType, string methodName)
+    {
+        BuilderType = builderType;
+        MethodName = methodName;
+    }
+
+    public Type BuilderType { get; }
+    public string MethodName { get; }
+}

src/LegacySupport/CollectionBuilder/README.md

@@ -0,0 +1,7 @@
+To use this source in your project, add the following to your `.csproj` file:
+
+```xml
+<PropertyGroup>
+  <InjectCollectionBuilderAttributesOnLegacy>true</InjectCollectionBuilderAttributesOnLegacy>
+</PropertyGroup>
+```

src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/README.md

@@ -5,6 +5,8 @@
 * [`Microsoft.Extensions.AI.Evaluation`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation) - Defines core abstractions and types for supporting evaluation.
 * [`Microsoft.Extensions.AI.Evaluation.Quality`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Quality) - Contains evaluators that can be used to evaluate the quality of AI responses in your projects including Relevance, Truth, Completeness, Fluency, Coherence, Retrieval, Equivalence and Groundedness.
 * [`Microsoft.Extensions.AI.Evaluation.Safety`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Safety) - Contains a set of evaluators that are built atop the Azure AI Foundry Evaluation service that can be used to evaluate the content safety of AI responses in your projects including Protected Material, Groundedness Pro, Ungrounded Attributes, Hate and Unfairness, Self Harm, Violence, Sexual, Code Vulnerability and Indirect Attack.
+* [`Microsoft.Extensions.AI.Evaluation.NLP`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.NLP) - Contains a set of evaluators that implement common algorithms for evaluating machine translation and natural
+language processing tasks. Evaluators currently include BLEU score, with more planned.
 * [`Microsoft.Extensions.AI.Evaluation.Reporting`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Reporting) - Contains support for caching LLM responses, storing the results of evaluations and generating reports from that data.
 * [`Microsoft.Extensions.AI.Evaluation.Reporting.Azure`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Reporting.Azure) - Supports the `Microsoft.Extensions.AI.Evaluation.Reporting` library with an implementation for caching LLM responses and storing the evaluation results in an Azure Storage container.
 * [`Microsoft.Extensions.AI.Evaluation.Console`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Console) - A command line dotnet tool for generating reports and managing evaluation data.
@@ -18,6 +20,7 @@ dotnet add package Microsoft.Extensions.AI.Evaluation
 dotnet add package Microsoft.Extensions.AI.Evaluation.Quality
 dotnet add package Microsoft.Extensions.AI.Evaluation.Safety
 dotnet add package Microsoft.Extensions.AI.Evaluation.Reporting
+dotnet add package Microsoft.Extensions.AI.Evaluation.NLP
 ```
 
 Or directly in the C# project file:
@@ -28,6 +31,7 @@ Or directly in the C# project file:
   <PackageReference Include="Microsoft.Extensions.AI.Evaluation.Quality" Version="[CURRENTVERSION]" />
   <PackageReference Include="Microsoft.Extensions.AI.Evaluation.Safety" Version="[CURRENTVERSION]" />
   <PackageReference Include="Microsoft.Extensions.AI.Evaluation.Reporting" Version="[CURRENTVERSION]" />
+  <PackageReference Include="Microsoft.Extensions.AI.Evaluation.NLP" Version="[CURRENTVERSION]" />
 </ItemGroup>
 ```
 

src/Libraries/Microsoft.Extensions.AI.Evaluation.NLP/BLEUEvaluator.cs

@@ -0,0 +1,96 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections.Generic;
+using System.Globalization;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.AI.Evaluation.NLP.Common;
+using Microsoft.Extensions.AI.Evaluation.Utilities;
+using Microsoft.Shared.Diagnostics;
+
+namespace Microsoft.Extensions.AI.Evaluation.NLP;
+
+/// <summary>
+/// An <see cref="IEvaluator"/> that evaluates the quality of a response produced by an AI model by comparing
+/// it to a reference response using the BLEU (Bilingual Evaluation Understudy) algorithm. It is often used
+/// to evaluate the quality of machine translation or text generation tasks.
+/// </summary>
+/// <remarks>
+/// <para>
+/// The <see cref="BLEUEvaluator"/> computes the BLEU score of a response ("hypothesis") compared to a reference
+/// supplied via <see cref="BLEUEvaluatorContext.References"/>. The score is returned in a <see cref="NumericMetric"/>
+/// with a value between 0.0 and 1.0 where 0.0 represents no match at all and 1.0 indicates a perfect match.
+/// By default, the score is interpreted with a pass/fail cutoff of 0.5. So a score of 0.5 or higher is
+/// passing and a score below 0.5 is failing.
+/// </para>
+/// </remarks>
+public sealed class BLEUEvaluator : IEvaluator
+{
+    /// <summary>
+    /// Gets the <see cref="EvaluationMetric.Name"/> of the <see cref="NumericMetric"/> returned by
+    /// <see cref="BLEUEvaluator"/>.
+    /// </summary>
+    public static string BLEUMetricName => "BLEU";
+
+    /// <inheritdoc/>
+    public IReadOnlyCollection<string> EvaluationMetricNames { get; } = [BLEUMetricName];
+
+    /// <inheritdoc/>
+    public ValueTask<EvaluationResult> EvaluateAsync(
+        IEnumerable<ChatMessage> messages,
+        ChatResponse modelResponse,
+        ChatConfiguration? chatConfiguration = null,
+        IEnumerable<EvaluationContext>? additionalContext = null,
+        CancellationToken cancellationToken = default)
+    {
+        _ = Throw.IfNull(modelResponse);
+
+        var metric = new NumericMetric(BLEUMetricName);
+        var result = new EvaluationResult(metric);
+
+        if (string.IsNullOrWhiteSpace(modelResponse.Text))
+        {
+            metric.AddDiagnostics(
+                EvaluationDiagnostic.Error($"The {nameof(modelResponse)} supplied for evaluation was null or empty."));
+
+            return new ValueTask<EvaluationResult>(result);
+        }
+
+        if (additionalContext?.OfType<BLEUEvaluatorContext>().FirstOrDefault()
+                is not BLEUEvaluatorContext context)
+        {
+            metric.AddDiagnostics(
+                EvaluationDiagnostic.Error(
+                    $"A value of type '{nameof(BLEUEvaluatorContext)}' was not found in the '{nameof(additionalContext)}' collection."));
+
+            return new ValueTask<EvaluationResult>(result);
+        }
+
+        if (context.References.Count is 0)
+        {
+            metric.AddDiagnostics(
+                EvaluationDiagnostic.Error(
+                    $"Supplied '{nameof(BLEUEvaluatorContext)}' did not contain any '{nameof(BLEUEvaluatorContext.References)}'."));
+
+            return new ValueTask<EvaluationResult>(result);
+        }
+
+        var (score, duration) = TimingHelper.ExecuteWithTiming(() =>
+        {
+            var references = context.References.Select(reference => SimpleWordTokenizer.WordTokenize(reference));
+            var hypothesis = SimpleWordTokenizer.WordTokenize(modelResponse.Text);
+            return BLEUAlgorithm.SentenceBLEU(references, hypothesis, BLEUAlgorithm.DefaultBLEUWeights, SmoothingFunction.Method4);
+        });
+
+        metric.Value = score;
+        string durationText = $"{duration.TotalSeconds.ToString("F2", CultureInfo.InvariantCulture)} s";
+        metric.AddOrUpdateMetadata(name: "evaluation-duration", value: durationText);
+        metric.AddOrUpdateContext(context);
+        metric.Interpretation = NLPScoreInterpretation.Interpret(metric);
+
+        return new ValueTask<EvaluationResult>(result);
+    }
+
+}

src/Libraries/Microsoft.Extensions.AI.Evaluation.NLP/BLEUEvaluatorContext.cs

@@ -0,0 +1,62 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+#pragma warning disable S3604
+// S3604: Member initializer values should not be redundant.
+// We disable this warning because it is a false positive arising from the analyzer's lack of support for C#'s primary
+// constructor syntax.
+
+using System.Collections.Generic;
+using System.Linq;
+
+namespace Microsoft.Extensions.AI.Evaluation.NLP;
+
+/// <summary>
+/// Contextual information that the <see cref="BLEUEvaluator"/> uses to compute the BLEU score for a response.
+/// </summary>
+/// <remarks>
+/// <see cref="BLEUEvaluator"/> measures the BLEU score of a response compared to a reference. BLEU (Bilingual Evaluation Understudy)
+/// is a metric used to evaluate the quality of machine-generated text.
+/// </remarks>
+public sealed class BLEUEvaluatorContext : EvaluationContext
+{
+    /// <summary>
+    /// Gets the unique <see cref="EvaluationContext.Name"/> that is used for
+    /// <see cref="BLEUEvaluatorContext"/>.
+    /// </summary>
+    public static string BLEUContextName => "BLEU Context";
+
+    /// <summary>
+    /// Gets the reference responses against which the provided model response will be scored.
+    /// </summary>
+    /// <remarks>
+    /// The <see cref="BLEUEvaluator"/> measures the degree to which the response being evaluated is similar to
+    /// the response supplied via <see cref="References"/>. The metric will be reported as a BLEU score.
+    /// </remarks>
+    public IReadOnlyList<string> References { get; }
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="BLEUEvaluatorContext"/> class.
+    /// </summary>
+    /// <param name="references">
+    /// The reference responses against which the response that is being evaluated is compared.
+    /// </param>
+    public BLEUEvaluatorContext(params string[] references)
+        : this(references as IEnumerable<string>)
+    {
+    }
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="BLEUEvaluatorContext"/> class.
+    /// </summary>
+    /// <param name="references">
+    /// The reference responses against which the response that is being evaluated is compared.
+    /// </param>
+    public BLEUEvaluatorContext(IEnumerable<string> references)
+        : base(
+            name: BLEUContextName,
+            contents: [.. references.Select(c => new TextContent(c))])
+    {
+        References = [.. references];
+    }
+}