Skip to content

Commit f3f1017

Browse files
stephentoubrogerbarreto
stephentoub
and
rogerbarreto
authored
Add CodeInterpreterToolCall/ResultContent content types (#6964)
* Add CodeInterpreterToolCall/ResultContent content types - Adds new CodeInterpreterToolCallContent and CodeInterpreterToolResultContent types - Updates the OpenAI Assistants and Responses IChatClient implementations to produce them - Updates HostedFileContent with an optional MediaType and Name, matching the corresponding properties on DataContent and UriContent. - Updates ToChatResponse{Async} coalescing to handle these code interpreter types. - Updates DataContent's DebuggerDisplay to show text for "text/*" and "application/json" media types. * Update src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/CodeInterpreterToolCallContent.cs Co-authored-by: Roger Barreto <[email protected]> * Address PR feedback --------- Co-authored-by: Roger Barreto <[email protected]>
1 parent d57eb9d commit f3f1017

25 files changed

+1037
-25
lines changed

‎src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseExtensions.cs‎

Lines changed: 110 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,11 @@
55
using System.Collections.Generic;
66
using System.Diagnostics;
77
using System.Diagnostics.CodeAnalysis;
8+
using System.IO;
89
using System.Linq;
10+
#if !NET
11+
using System.Runtime.InteropServices;
12+
#endif
913
using System.Text;
1014
using System.Threading;
1115
using System.Threading.Tasks;
@@ -181,7 +185,7 @@ static async Task<ChatResponse> ToChatResponseAsync(
181185
}
182186

183187
/// <summary>Coalesces sequential <see cref="AIContent"/> content elements.</summary>
184-
internal static void CoalesceTextContent(IList<AIContent> contents)
188+
internal static void CoalesceContent(IList<AIContent> contents)
185189
{
186190
Coalesce<TextContent>(
187191
contents,
@@ -215,6 +219,110 @@ internal static void CoalesceTextContent(IList<AIContent> contents)
215219
return content;
216220
});
217221

222+
Coalesce<DataContent>(
223+
contents,
224+
mergeSingle: false,
225+
canMerge: static (r1, r2) => r1.MediaType == r2.MediaType && r1.HasTopLevelMediaType("text") && r1.Name == r2.Name,
226+
static (contents, start, end) =>
227+
{
228+
Debug.Assert(end - start > 1, "Expected multiple contents to merge");
229+
230+
MemoryStream ms = new();
231+
for (int i = start; i < end; i++)
232+
{
233+
var current = (DataContent)contents[i];
234+
#if NET
235+
ms.Write(current.Data.Span);
236+
#else
237+
if (!MemoryMarshal.TryGetArray(current.Data, out var segment))
238+
{
239+
segment = new(current.Data.ToArray());
240+
}
241+
242+
ms.Write(segment.Array!, segment.Offset, segment.Count);
243+
#endif
244+
}
245+
246+
var first = (DataContent)contents[start];
247+
return new DataContent(new ReadOnlyMemory<byte>(ms.GetBuffer(), 0, (int)ms.Length), first.MediaType) { Name = first.Name };
248+
});
249+
250+
Coalesce<CodeInterpreterToolCallContent>(
251+
contents,
252+
mergeSingle: true,
253+
canMerge: static (r1, r2) => r1.CallId == r2.CallId,
254+
static (contents, start, end) =>
255+
{
256+
var firstContent = (CodeInterpreterToolCallContent)contents[start];
257+
258+
if (start == end - 1)
259+
{
260+
if (firstContent.Inputs is not null)
261+
{
262+
CoalesceContent(firstContent.Inputs);
263+
}
264+
265+
return firstContent;
266+
}
267+
268+
List<AIContent>? inputs = null;
269+
270+
for (int i = start; i < end; i++)
271+
{
272+
(inputs ??= []).AddRange(((CodeInterpreterToolCallContent)contents[i]).Inputs ?? []);
273+
}
274+
275+
if (inputs is not null)
276+
{
277+
CoalesceContent(inputs);
278+
}
279+
280+
return new()
281+
{
282+
CallId = firstContent.CallId,
283+
Inputs = inputs,
284+
AdditionalProperties = firstContent.AdditionalProperties?.Clone(),
285+
};
286+
});
287+
288+
Coalesce<CodeInterpreterToolResultContent>(
289+
contents,
290+
mergeSingle: true,
291+
canMerge: static (r1, r2) => r1.CallId is not null && r2.CallId is not null && r1.CallId == r2.CallId,
292+
static (contents, start, end) =>
293+
{
294+
var firstContent = (CodeInterpreterToolResultContent)contents[start];
295+
296+
if (start == end - 1)
297+
{
298+
if (firstContent.Outputs is not null)
299+
{
300+
CoalesceContent(firstContent.Outputs);
301+
}
302+
303+
return firstContent;
304+
}
305+
306+
List<AIContent>? output = null;
307+
308+
for (int i = start; i < end; i++)
309+
{
310+
(output ??= []).AddRange(((CodeInterpreterToolResultContent)contents[i]).Outputs ?? []);
311+
}
312+
313+
if (output is not null)
314+
{
315+
CoalesceContent(output);
316+
}
317+
318+
return new()
319+
{
320+
CallId = firstContent.CallId,
321+
Outputs = output,
322+
AdditionalProperties = firstContent.AdditionalProperties?.Clone(),
323+
};
324+
});
325+
218326
static string MergeText(IList<AIContent> contents, int start, int end)
219327
{
220328
Debug.Assert(end - start > 1, "Expected multiple contents to merge");
@@ -318,7 +426,7 @@ private static void FinalizeResponse(ChatResponse response)
318426
int count = response.Messages.Count;
319427
for (int i = 0; i < count; i++)
320428
{
321-
CoalesceTextContent((List<AIContent>)response.Messages[i].Contents);
429+
CoalesceContent((List<AIContent>)response.Messages[i].Contents);
322430
}
323431
}
324432

‎src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/AIContent.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ namespace Microsoft.Extensions.AI;
3030
// [JsonDerivedType(typeof(McpServerToolResultContent), typeDiscriminator: "mcpServerToolResult")]
3131
// [JsonDerivedType(typeof(McpServerToolApprovalRequestContent), typeDiscriminator: "mcpServerToolApprovalRequest")]
3232
// [JsonDerivedType(typeof(McpServerToolApprovalResponseContent), typeDiscriminator: "mcpServerToolApprovalResponse")]
33+
// [JsonDerivedType(typeof(CodeInterpreterToolCallContent), typeDiscriminator: "codeInterpreterToolCall")]
34+
// [JsonDerivedType(typeof(CodeInterpreterToolResultContent), typeDiscriminator: "codeInterpreterToolResult")]
3335

3436
public class AIContent
3537
{

‎src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/CodeInterpreterToolCallContent.cs‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
4+
using System.Collections.Generic;
5+
using System.Diagnostics.CodeAnalysis;
6+
7+
namespace Microsoft.Extensions.AI;
8+
9+
/// <summary>
10+
/// Represents a code interpreter tool call invocation by a hosted service.
11+
/// </summary>
12+
/// <remarks>
13+
/// This content type represents when a hosted AI service invokes a code interpreter tool.
14+
/// It is informational only and represents the call itself, not the result.
15+
/// </remarks>
16+
[Experimental("MEAI001")]
17+
public sealed class CodeInterpreterToolCallContent : AIContent
18+
{
19+
/// <summary>
20+
/// Initializes a new instance of the <see cref="CodeInterpreterToolCallContent"/> class.
21+
/// </summary>
22+
public CodeInterpreterToolCallContent()
23+
{
24+
}
25+
26+
/// <summary>
27+
/// Gets or sets the tool call ID.
28+
/// </summary>
29+
public string? CallId { get; set; }
30+
31+
/// <summary>
32+
/// Gets or sets the inputs to the code interpreter tool.
33+
/// </summary>
34+
/// <remarks>
35+
/// Inputs can include various types of content such as <see cref="HostedFileContent"/> for files,
36+
/// <see cref="DataContent"/> for binary data, or other <see cref="AIContent"/> types as supported
37+
/// by the service. Typically <see cref="Inputs"/> includes a <see cref="DataContent"/> with a "text/x-python"
38+
/// media type representing the code for execution by the code interpreter tool.
39+
/// </remarks>
40+
public IList<AIContent>? Inputs { get; set; }
41+
}

‎src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/CodeInterpreterToolResultContent.cs‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
4+
using System.Collections.Generic;
5+
using System.Diagnostics.CodeAnalysis;
6+
7+
namespace Microsoft.Extensions.AI;
8+
9+
/// <summary>
10+
/// Represents the result of a code interpreter tool invocation by a hosted service.
11+
/// </summary>
12+
[Experimental("MEAI001")]
13+
public sealed class CodeInterpreterToolResultContent : AIContent
14+
{
15+
/// <summary>
16+
/// Initializes a new instance of the <see cref="CodeInterpreterToolResultContent"/> class.
17+
/// </summary>
18+
public CodeInterpreterToolResultContent()
19+
{
20+
}
21+
22+
/// <summary>
23+
/// Gets or sets the tool call ID that this result corresponds to.
24+
/// </summary>
25+
public string? CallId { get; set; }
26+
27+
/// <summary>
28+
/// Gets or sets the output of code interpreter tool.
29+
/// </summary>
30+
/// <remarks>
31+
/// Outputs can include various types of content such as <see cref="HostedFileContent"/> for files,
32+
/// <see cref="DataContent"/> for binary data, <see cref="TextContent"/> for standard output text,
33+
/// or other <see cref="AIContent"/> types as supported by the service.
34+
/// </remarks>
35+
public IList<AIContent>? Outputs { get; set; }
36+
}

‎src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/DataContent.cs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
#endif
1010
using System.Diagnostics;
1111
using System.Diagnostics.CodeAnalysis;
12+
using System.Text;
1213
#if !NET
1314
using System.Runtime.InteropServices;
1415
#endif
@@ -115,6 +116,7 @@ public DataContent([StringSyntax(StringSyntaxAttribute.Uri)] string uri, string?
115116
/// <param name="mediaType">The media type (also known as MIME type) represented by the content.</param>
116117
/// <exception cref="ArgumentNullException"><paramref name="mediaType"/> is <see langword="null"/>.</exception>
117118
/// <exception cref="ArgumentException"><paramref name="mediaType"/> is empty or composed entirely of whitespace.</exception>
119+
/// <exception cref="ArgumentException"><paramref name="mediaType"/> represents an invalid media type.</exception>
118120
public DataContent(ReadOnlyMemory<byte> data, string mediaType)
119121
{
120122
MediaType = DataUriParser.ThrowIfInvalidMediaType(mediaType);
@@ -236,6 +238,16 @@ private string DebuggerDisplay
236238
{
237239
get
238240
{
241+
if (HasTopLevelMediaType("text"))
242+
{
243+
return $"MediaType = {MediaType}, Text = \"{Encoding.UTF8.GetString(Data.ToArray())}\"";
244+
}
245+
246+
if ("application/json".Equals(MediaType, StringComparison.OrdinalIgnoreCase))
247+
{
248+
return $"JSON = {Encoding.UTF8.GetString(Data.ToArray())}";
249+
}
250+
239251
const int MaxLength = 80;
240252

241253
string uri = Uri;

‎src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/HostedFileContent.cs‎

Lines changed: 37 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,9 @@ namespace Microsoft.Extensions.AI;
1414
/// Unlike <see cref="DataContent"/> which contains the data for a file or blob, this class represents a file that is hosted
1515
/// by the AI service and referenced by an identifier. Such identifiers are specific to the provider.
1616
/// </remarks>
17-
[DebuggerDisplay("FileId = {FileId}")]
17+
[DebuggerDisplay("{DebuggerDisplay,nq}")]
1818
public sealed class HostedFileContent : AIContent
1919
{
20-
private string _fileId;
21-
2220
/// <summary>
2321
/// Initializes a new instance of the <see cref="HostedFileContent"/> class.
2422
/// </summary>
@@ -27,7 +25,7 @@ public sealed class HostedFileContent : AIContent
2725
/// <exception cref="ArgumentException"><paramref name="fileId"/> is empty or composed entirely of whitespace.</exception>
2826
public HostedFileContent(string fileId)
2927
{
30-
_fileId = Throw.IfNullOrWhitespace(fileId);
28+
FileId = fileId;
3129
}
3230

3331
/// <summary>
@@ -37,7 +35,40 @@ public HostedFileContent(string fileId)
3735
/// <exception cref="ArgumentException"><paramref name="value"/> is empty or composed entirely of whitespace.</exception>
3836
public string FileId
3937
{
40-
get => _fileId;
41-
set => _fileId = Throw.IfNullOrWhitespace(value);
38+
get => field;
39+
set => field = Throw.IfNullOrWhitespace(value);
40+
}
41+
42+
/// <summary>Gets or sets an optional media type (also known as MIME type) associated with the file.</summary>
43+
/// <exception cref="ArgumentException"><paramref name="value"/> represents an invalid media type.</exception>
44+
public string? MediaType
45+
{
46+
get;
47+
set => field = value is not null ? DataUriParser.ThrowIfInvalidMediaType(value) : value;
48+
}
49+
50+
/// <summary>Gets or sets an optional name associated with the file.</summary>
51+
public string? Name { get; set; }
52+
53+
/// <summary>Gets a string representing this instance to display in the debugger.</summary>
54+
[DebuggerBrowsable(DebuggerBrowsableState.Never)]
55+
private string DebuggerDisplay
56+
{
57+
get
58+
{
59+
string display = $"FileId = {FileId}";
60+
61+
if (MediaType is string mediaType)
62+
{
63+
display += $", MediaType = {mediaType}";
64+
}
65+
66+
if (Name is string name)
67+
{
68+
display += $", Name = \"{name}\"";
69+
}
70+
71+
return display;
72+
}
4273
}
4374
}

‎src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/UriContent.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ public Uri Uri
6767
}
6868

6969
/// <summary>Gets or sets the media type (also known as MIME type) for this content.</summary>
70+
/// <exception cref="ArgumentException"><paramref name="value"/> represents an invalid media type.</exception>
7071
public string MediaType
7172
{
7273
get => _mediaType;

‎src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2012,6 +2012,14 @@
20122012
{
20132013
"Member": "string Microsoft.Extensions.AI.HostedFileContent.FileId { get; set; }",
20142014
"Stage": "Stable"
2015+
},
2016+
{
2017+
"Member": "string? Microsoft.Extensions.AI.HostedFileContent.MediaType { get; set; }",
2018+
"Stage": "Stable"
2019+
},
2020+
{
2021+
"Member": "string? Microsoft.Extensions.AI.HostedFileContent.Name { get; set; }",
2022+
"Stage": "Stable"
20152023
}
20162024
]
20172025
},

‎src/Libraries/Microsoft.Extensions.AI.Abstractions/SpeechToText/SpeechToTextResponseUpdateExtensions.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ public static SpeechToTextResponse ToSpeechToTextResponse(
3030
ProcessUpdate(update, response);
3131
}
3232

33-
ChatResponseExtensions.CoalesceTextContent((List<AIContent>)response.Contents);
33+
ChatResponseExtensions.CoalesceContent((List<AIContent>)response.Contents);
3434

3535
return response;
3636
}
@@ -56,7 +56,7 @@ static async Task<SpeechToTextResponse> ToResponseAsync(
5656
ProcessUpdate(update, response);
5757
}
5858

59-
ChatResponseExtensions.CoalesceTextContent((List<AIContent>)response.Contents);
59+
ChatResponseExtensions.CoalesceContent((List<AIContent>)response.Contents);
6060

6161
return response;
6262
}

‎src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Defaults.cs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,8 @@ private static JsonSerializerOptions CreateDefaultOptions()
5757
AddAIContentType(options, typeof(McpServerToolResultContent), typeDiscriminatorId: "mcpServerToolResult", checkBuiltIn: false);
5858
AddAIContentType(options, typeof(McpServerToolApprovalRequestContent), typeDiscriminatorId: "mcpServerToolApprovalRequest", checkBuiltIn: false);
5959
AddAIContentType(options, typeof(McpServerToolApprovalResponseContent), typeDiscriminatorId: "mcpServerToolApprovalResponse", checkBuiltIn: false);
60+
AddAIContentType(options, typeof(CodeInterpreterToolCallContent), typeDiscriminatorId: "codeInterpreterToolCall", checkBuiltIn: false);
61+
AddAIContentType(options, typeof(CodeInterpreterToolResultContent), typeDiscriminatorId: "codeInterpreterToolResult", checkBuiltIn: false);
6062

6163
if (JsonSerializer.IsReflectionEnabledByDefault)
6264
{
@@ -129,6 +131,8 @@ private static JsonSerializerOptions CreateDefaultOptions()
129131
[JsonSerializable(typeof(McpServerToolResultContent))]
130132
[JsonSerializable(typeof(McpServerToolApprovalRequestContent))]
131133
[JsonSerializable(typeof(McpServerToolApprovalResponseContent))]
134+
[JsonSerializable(typeof(CodeInterpreterToolCallContent))]
135+
[JsonSerializable(typeof(CodeInterpreterToolResultContent))]
132136
[JsonSerializable(typeof(ResponseContinuationToken))]
133137

134138
// IEmbeddingGenerator

0 commit comments

Comments
 (0)