Skip to content

Commit dec7e30

Browse files
jozkeejozkee
authored
Add RawRepresentationFactory to other options types (#6433)
* Add RawRepresentationFactory to other options types * Undo changes in Azure.AI.Inference * Address documentation feedback
1 parent ec4a041 commit dec7e30

File tree

16 files changed

+346
-182
lines changed

16 files changed

+346
-182
lines changed

‎src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatOptions.cs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ public class ChatOptions
113113
public IList<AITool>? Tools { get; set; }
114114

115115
/// <summary>
116-
/// Gets or sets a callback responsible of creating the raw representation of the chat options from an underlying implementation.
116+
/// Gets or sets a callback responsible for creating the raw representation of the chat options from an underlying implementation.
117117
/// </summary>
118118
/// <remarks>
119119
/// The underlying <see cref="IChatClient" /> implementation may have its own representation of options.
@@ -124,8 +124,8 @@ public class ChatOptions
124124
/// implementation-specific options type may be returned by this callback, for the <see cref="IChatClient" />
125125
/// implementation to use instead of creating a new instance. Such implementations may mutate the supplied options
126126
/// instance further based on other settings supplied on this <see cref="ChatOptions" /> instance or from other inputs,
127-
/// like the enumerable of <see cref="ChatMessage"/>s, therefore, its **strongly recommended** to not return shared instances
128-
/// and instead make the callback return a new instance per each call.
127+
/// like the enumerable of <see cref="ChatMessage"/>s, therefore, it is <b>strongly recommended</b> to not return shared instances
128+
/// and instead make the callback return a new instance on each call.
129129
/// This is typically used to set an implementation-specific setting that isn't otherwise exposed from the strongly-typed
130130
/// properties on <see cref="ChatOptions" />.
131131
/// </remarks>

‎src/Libraries/Microsoft.Extensions.AI.Abstractions/Embeddings/EmbeddingGenerationOptions.cs‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
// Licensed to the .NET Foundation under one or more agreements.
22
// The .NET Foundation licenses this file to you under the MIT license.
33

4+
using System;
5+
using System.Text.Json.Serialization;
46
using Microsoft.Shared.Diagnostics;
57

68
namespace Microsoft.Extensions.AI;
@@ -31,6 +33,25 @@ public int? Dimensions
3133
/// <summary>Gets or sets additional properties for the embedding generation request.</summary>
3234
public AdditionalPropertiesDictionary? AdditionalProperties { get; set; }
3335

36+
/// <summary>
37+
/// Gets or sets a callback responsible for creating the raw representation of the embedding generation options from an underlying implementation.
38+
/// </summary>
39+
/// <remarks>
40+
/// The underlying <see cref="IEmbeddingGenerator" /> implementation may have its own representation of options.
41+
/// When <see cref="IEmbeddingGenerator{TInput, TEmbedding}.GenerateAsync" />
42+
/// is invoked with an <see cref="EmbeddingGenerationOptions" />, that implementation may convert the provided options into
43+
/// its own representation in order to use it while performing the operation. For situations where a consumer knows
44+
/// which concrete <see cref="IEmbeddingGenerator" /> is being used and how it represents options, a new instance of that
45+
/// implementation-specific options type may be returned by this callback, for the <see cref="IEmbeddingGenerator" />
46+
/// implementation to use instead of creating a new instance. Such implementations may mutate the supplied options
47+
/// instance further based on other settings supplied on this <see cref="EmbeddingGenerationOptions" /> instance or from other inputs,
48+
/// therefore, it is <b>strongly recommended</b> to not return shared instances and instead make the callback return a new instance on each call.
49+
/// This is typically used to set an implementation-specific setting that isn't otherwise exposed from the strongly-typed
50+
/// properties on <see cref="EmbeddingGenerationOptions" />.
51+
/// </remarks>
52+
[JsonIgnore]
53+
public Func<IEmbeddingGenerator, object?>? RawRepresentationFactory { get; set; }
54+
3455
/// <summary>Produces a clone of the current <see cref="EmbeddingGenerationOptions"/> instance.</summary>
3556
/// <returns>A clone of the current <see cref="EmbeddingGenerationOptions"/> instance.</returns>
3657
/// <remarks>

‎src/Libraries/Microsoft.Extensions.AI.Abstractions/SpeechToText/SpeechToTextOptions.cs‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
// Licensed to the .NET Foundation under one or more agreements.
22
// The .NET Foundation licenses this file to you under the MIT license.
33

4+
using System;
45
using System.Diagnostics.CodeAnalysis;
6+
using System.Text.Json.Serialization;
57

68
namespace Microsoft.Extensions.AI;
79

@@ -24,6 +26,25 @@ public class SpeechToTextOptions
2426
/// <summary>Gets or sets any additional properties associated with the options.</summary>
2527
public AdditionalPropertiesDictionary? AdditionalProperties { get; set; }
2628

29+
/// <summary>
30+
/// Gets or sets a callback responsible for creating the raw representation of the embedding generation options from an underlying implementation.
31+
/// </summary>
32+
/// <remarks>
33+
/// The underlying <see cref="ISpeechToTextClient" /> implementation may have its own representation of options.
34+
/// When <see cref="ISpeechToTextClient.GetTextAsync" /> or <see cref="ISpeechToTextClient.GetStreamingTextAsync"/>
35+
/// is invoked with an <see cref="SpeechToTextOptions" />, that implementation may convert the provided options into
36+
/// its own representation in order to use it while performing the operation. For situations where a consumer knows
37+
/// which concrete <see cref="ISpeechToTextClient" /> is being used and how it represents options, a new instance of that
38+
/// implementation-specific options type may be returned by this callback, for the <see cref="ISpeechToTextClient" />
39+
/// implementation to use instead of creating a new instance. Such implementations may mutate the supplied options
40+
/// instance further based on other settings supplied on this <see cref="SpeechToTextOptions" /> instance or from other inputs,
41+
/// therefore, it is <b>strongly recommended</b> to not return shared instances and instead make the callback return a new instance on each call.
42+
/// This is typically used to set an implementation-specific setting that isn't otherwise exposed from the strongly-typed
43+
/// properties on <see cref="SpeechToTextOptions" />.
44+
/// </remarks>
45+
[JsonIgnore]
46+
public Func<ISpeechToTextClient, object?>? RawRepresentationFactory { get; set; }
47+
2748
/// <summary>Produces a clone of the current <see cref="SpeechToTextOptions"/> instance.</summary>
2849
/// <returns>A clone of the current <see cref="SpeechToTextOptions"/> instance.</returns>
2950
public virtual SpeechToTextOptions Clone()

‎src/Libraries/Microsoft.Extensions.AI.AzureAIInference/AzureAIInferenceChatClient.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -289,7 +289,7 @@ private ChatCompletionsOptions CreateAzureAIOptions(IEnumerable<ChatMessage> cha
289289
throw new InvalidOperationException("No model id was provided when either constructing the client or in the chat options.")
290290
};
291291

292-
/// <summary>Converts an extensions options instance to an AzureAI options instance.</summary>
292+
/// <summary>Converts an extensions options instance to an Azure.AI.Inference options instance.</summary>
293293
private ChatCompletionsOptions ToAzureAIOptions(IEnumerable<ChatMessage> chatContents, ChatOptions? options)
294294
{
295295
if (options is null)

‎src/Libraries/Microsoft.Extensions.AI.AzureAIInference/AzureAIInferenceEmbeddingGenerator.cs‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@ public async Task<GeneratedEmbeddings<Embedding<float>>> GenerateAsync(
9191
{
9292
_ = Throw.IfNull(values);
9393

94-
var azureAIOptions = ToAzureAIOptions(values, options, EmbeddingEncodingFormat.Base64);
94+
var azureAIOptions = ToAzureAIOptions(values, options);
9595

9696
var embeddings = (await _embeddingsClient.EmbedAsync(azureAIOptions, cancellationToken).ConfigureAwait(false)).Value;
9797

@@ -164,13 +164,16 @@ static void ThrowInvalidData() =>
164164
}
165165

166166
/// <summary>Converts an extensions options instance to an Azure.AI.Inference options instance.</summary>
167-
private EmbeddingsOptions ToAzureAIOptions(IEnumerable<string> inputs, EmbeddingGenerationOptions? options, EmbeddingEncodingFormat format)
167+
/// <remarks>
168+
/// Note: we can't currently use RawRepresentationFactory due to https://github.com/Azure/azure-sdk-for-net/issues/50018.
169+
/// </remarks>
170+
private EmbeddingsOptions ToAzureAIOptions(IEnumerable<string> inputs, EmbeddingGenerationOptions? options)
168171
{
169172
EmbeddingsOptions result = new(inputs)
170173
{
171174
Dimensions = options?.Dimensions ?? _dimensions,
172175
Model = options?.ModelId ?? _metadata.DefaultModelId,
173-
EncodingFormat = format,
176+
EncodingFormat = EmbeddingEncodingFormat.Base64,
174177
};
175178

176179
if (options?.AdditionalProperties is { } props)

‎src/Libraries/Microsoft.Extensions.AI.AzureAIInference/AzureAIInferenceImageEmbeddingGenerator.cs‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ public async Task<GeneratedEmbeddings<Embedding<float>>> GenerateAsync(
8787
{
8888
_ = Throw.IfNull(values);
8989

90-
var azureAIOptions = ToAzureAIOptions(values, options, EmbeddingEncodingFormat.Base64);
90+
var azureAIOptions = ToAzureAIOptions(values, options);
9191

9292
var embeddings = (await _imageEmbeddingsClient.EmbedAsync(azureAIOptions, cancellationToken).ConfigureAwait(false)).Value;
9393

@@ -117,13 +117,16 @@ void IDisposable.Dispose()
117117
}
118118

119119
/// <summary>Converts an extensions options instance to an Azure.AI.Inference options instance.</summary>
120-
private ImageEmbeddingsOptions ToAzureAIOptions(IEnumerable<DataContent> inputs, EmbeddingGenerationOptions? options, EmbeddingEncodingFormat format)
120+
/// <remarks>
121+
/// Note: we can't currently use RawRepresentationFactory due to https://github.com/Azure/azure-sdk-for-net/issues/50018.
122+
/// </remarks>
123+
private ImageEmbeddingsOptions ToAzureAIOptions(IEnumerable<DataContent> inputs, EmbeddingGenerationOptions? options)
121124
{
122125
ImageEmbeddingsOptions result = new(inputs.Select(dc => new ImageEmbeddingInput(dc.Uri)))
123126
{
124127
Dimensions = options?.Dimensions ?? _dimensions,
125128
Model = options?.ModelId ?? _metadata.DefaultModelId,
126-
EncodingFormat = format,
129+
EncodingFormat = EmbeddingEncodingFormat.Base64,
127130
};
128131

129132
if (options?.AdditionalProperties is { } props)

‎src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIEmbeddingGenerator.cs‎

Lines changed: 5 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -107,21 +107,14 @@ private static EmbeddingGeneratorMetadata CreateMetadata(string providerName, st
107107
new(providerName, Uri.TryCreate(providerUrl, UriKind.Absolute, out Uri? providerUri) ? providerUri : null, defaultModelId, defaultModelDimensions);
108108

109109
/// <summary>Converts an extensions options instance to an OpenAI options instance.</summary>
110-
private OpenAI.Embeddings.EmbeddingGenerationOptions? ToOpenAIOptions(EmbeddingGenerationOptions? options)
110+
private OpenAI.Embeddings.EmbeddingGenerationOptions ToOpenAIOptions(EmbeddingGenerationOptions? options)
111111
{
112-
OpenAI.Embeddings.EmbeddingGenerationOptions openAIOptions = new()
112+
if (options?.RawRepresentationFactory?.Invoke(this) is not OpenAI.Embeddings.EmbeddingGenerationOptions result)
113113
{
114-
Dimensions = options?.Dimensions ?? _dimensions,
115-
};
116-
117-
if (options?.AdditionalProperties is { Count: > 0 } additionalProperties)
118-
{
119-
if (additionalProperties.TryGetValue(nameof(openAIOptions.EndUserId), out string? endUserId))
120-
{
121-
openAIOptions.EndUserId = endUserId;
122-
}
114+
result = new OpenAI.Embeddings.EmbeddingGenerationOptions();
123115
}
124116

125-
return openAIOptions;
117+
result.Dimensions ??= options?.Dimensions ?? _dimensions;
118+
return result;
126119
}
127120
}

‎src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponseChatClient.cs‎

Lines changed: 44 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
#pragma warning disable S1067 // Expressions should not be too complex
1919
#pragma warning disable S3011 // Reflection should not be used to increase accessibility of classes, methods, or fields
2020
#pragma warning disable S3604 // Member initializer values should not be redundant
21+
#pragma warning disable SA1204 // Static elements should appear before instance elements
2122

2223
namespace Microsoft.Extensions.AI;
2324

@@ -315,88 +316,59 @@ private static ChatRole ToChatRole(MessageRole? role) =>
315316
null;
316317

317318
/// <summary>Converts a <see cref="ChatOptions"/> to a <see cref="ResponseCreationOptions"/>.</summary>
318-
private static ResponseCreationOptions ToOpenAIResponseCreationOptions(ChatOptions? options)
319+
private ResponseCreationOptions ToOpenAIResponseCreationOptions(ChatOptions? options)
319320
{
320-
ResponseCreationOptions result = new();
321+
if (options is null)
322+
{
323+
return new ResponseCreationOptions();
324+
}
321325

322-
if (options is not null)
326+
if (options.RawRepresentationFactory?.Invoke(this) is not ResponseCreationOptions result)
323327
{
324-
// Handle strongly-typed properties.
325-
result.MaxOutputTokenCount = options.MaxOutputTokens;
326-
result.PreviousResponseId = options.ConversationId;
327-
result.TopP = options.TopP;
328-
result.Temperature = options.Temperature;
329-
result.ParallelToolCallsEnabled = options.AllowMultipleToolCalls;
330-
331-
// Handle loosely-typed properties from AdditionalProperties.
332-
if (options.AdditionalProperties is { Count: > 0 } additionalProperties)
333-
{
334-
if (additionalProperties.TryGetValue(nameof(result.EndUserId), out string? endUserId))
335-
{
336-
result.EndUserId = endUserId;
337-
}
328+
result = new ResponseCreationOptions();
329+
}
338330

339-
if (additionalProperties.TryGetValue(nameof(result.Instructions), out string? instructions))
340-
{
341-
result.Instructions = instructions;
342-
}
331+
// Handle strongly-typed properties.
332+
result.MaxOutputTokenCount ??= options.MaxOutputTokens;
333+
result.PreviousResponseId ??= options.ConversationId;
334+
result.TopP ??= options.TopP;
335+
result.Temperature ??= options.Temperature;
336+
result.ParallelToolCallsEnabled ??= options.AllowMultipleToolCalls;
343337

344-
if (additionalProperties.TryGetValue(nameof(result.Metadata), out IDictionary<string, string>? metadata))
338+
// Populate tools if there are any.
339+
if (options.Tools is { Count: > 0 } tools)
340+
{
341+
foreach (AITool tool in tools)
342+
{
343+
switch (tool)
345344
{
346-
foreach (KeyValuePair<string, string> kvp in metadata)
347-
{
348-
result.Metadata[kvp.Key] = kvp.Value;
349-
}
350-
}
345+
case AIFunction af:
346+
var oaitool = JsonSerializer.Deserialize(SchemaTransformCache.GetOrCreateTransformedSchema(af), ResponseClientJsonContext.Default.ResponseToolJson)!;
347+
var functionParameters = BinaryData.FromBytes(JsonSerializer.SerializeToUtf8Bytes(oaitool, ResponseClientJsonContext.Default.ResponseToolJson));
348+
result.Tools.Add(ResponseTool.CreateFunctionTool(af.Name, af.Description, functionParameters));
349+
break;
351350

352-
if (additionalProperties.TryGetValue(nameof(result.ReasoningOptions), out ResponseReasoningOptions? reasoningOptions))
353-
{
354-
result.ReasoningOptions = reasoningOptions;
355-
}
351+
case HostedWebSearchTool:
352+
WebSearchToolLocation? location = null;
353+
if (tool.AdditionalProperties.TryGetValue(nameof(WebSearchToolLocation), out object? objLocation))
354+
{
355+
location = objLocation as WebSearchToolLocation;
356+
}
356357

357-
if (additionalProperties.TryGetValue(nameof(result.StoredOutputEnabled), out bool storeOutputEnabled))
358-
{
359-
result.StoredOutputEnabled = storeOutputEnabled;
360-
}
358+
WebSearchToolContextSize? size = null;
359+
if (tool.AdditionalProperties.TryGetValue(nameof(WebSearchToolContextSize), out object? objSize) &&
360+
objSize is WebSearchToolContextSize)
361+
{
362+
size = (WebSearchToolContextSize)objSize;
363+
}
361364

362-
if (additionalProperties.TryGetValue(nameof(result.TruncationMode), out ResponseTruncationMode truncationMode))
363-
{
364-
result.TruncationMode = truncationMode;
365+
result.Tools.Add(ResponseTool.CreateWebSearchTool(location, size));
366+
break;
365367
}
366368
}
367369

368-
// Populate tools if there are any.
369-
if (options.Tools is { Count: > 0 } tools)
370+
if (result.ToolChoice is null && result.Tools.Count > 0)
370371
{
371-
foreach (AITool tool in tools)
372-
{
373-
switch (tool)
374-
{
375-
case AIFunction af:
376-
var oaitool = JsonSerializer.Deserialize(SchemaTransformCache.GetOrCreateTransformedSchema(af), ResponseClientJsonContext.Default.ResponseToolJson)!;
377-
var functionParameters = BinaryData.FromBytes(JsonSerializer.SerializeToUtf8Bytes(oaitool, ResponseClientJsonContext.Default.ResponseToolJson));
378-
result.Tools.Add(ResponseTool.CreateFunctionTool(af.Name, af.Description, functionParameters));
379-
break;
380-
381-
case HostedWebSearchTool:
382-
WebSearchToolLocation? location = null;
383-
if (tool.AdditionalProperties.TryGetValue(nameof(WebSearchToolLocation), out object? objLocation))
384-
{
385-
location = objLocation as WebSearchToolLocation;
386-
}
387-
388-
WebSearchToolContextSize? size = null;
389-
if (tool.AdditionalProperties.TryGetValue(nameof(WebSearchToolContextSize), out object? objSize) &&
390-
objSize is WebSearchToolContextSize)
391-
{
392-
size = (WebSearchToolContextSize)objSize;
393-
}
394-
395-
result.Tools.Add(ResponseTool.CreateWebSearchTool(location, size));
396-
break;
397-
}
398-
}
399-
400372
switch (options.ToolMode)
401373
{
402374
case NoneChatToolMode:
@@ -415,8 +387,10 @@ private static ResponseCreationOptions ToOpenAIResponseCreationOptions(ChatOptio
415387
break;
416388
}
417389
}
390+
}
418391

419-
// Handle response format.
392+
if (result.TextOptions is null)
393+
{
420394
if (options.ResponseFormat is ChatResponseFormatText)
421395
{
422396
result.TextOptions = new()

0 commit comments

Comments
 (0)