-
Notifications
You must be signed in to change notification settings - Fork 668
.NET: ADR: Supporting user approvals #209
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 41 commits
c73282b
9e3b98b
a9a6a97
61063c0
3a0dabf
786374c
eace4e7
4076ed3
23b4fe5
12817f1
84e2855
632635c
461c8ef
7de1787
aa3c14e
ab04590
aba7dc6
89ff266
629204e
3783074
d3a9319
74d1f5d
3c40615
c068247
d8626a4
45254f3
3300dce
c8e42de
97352a0
2c39648
5293568
c5bedd0
edcad34
0a61294
436c490
0d5fcbf
8beb0f6
9fbd0a7
302d43d
8ad466b
049c9d3
39ef6bd
14d4773
b288fed
fc0bf74
d1939ad
6d8b20e
2bc90af
d12cdf8
3420337
6a6f9dd
f05bf90
3105941
26eb742
0adf519
b2a4915
5e52c11
b1289e3
0b4405e
545c231
e6aeebb
a1b2973
906ad2f
4c59b25
66b904e
d3dd02e
21680be
a712408
c1da46b
091f754
7518b70
a349c79
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Large diffs are not rendered by default.
| Original file line number | Diff line number | Diff line change | ||||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
|
@@ -3,6 +3,8 @@ | |||||||||||||||||
| using System.ComponentModel; | ||||||||||||||||||
| using Microsoft.Extensions.AI; | ||||||||||||||||||
| using Microsoft.Extensions.AI.Agents; | ||||||||||||||||||
| using Microsoft.Extensions.AI.ModelContextProtocol; | ||||||||||||||||||
| using Microsoft.Extensions.DependencyInjection; | ||||||||||||||||||
|
|
||||||||||||||||||
| namespace Steps; | ||||||||||||||||||
|
|
||||||||||||||||||
|
|
@@ -114,6 +116,96 @@ async Task RunAgentAsync(string input) | |||||||||||||||||
| await base.AgentCleanUpAsync(provider, agent, thread); | ||||||||||||||||||
| } | ||||||||||||||||||
|
|
||||||||||||||||||
| [Theory] | ||||||||||||||||||
| [InlineData(ChatClientProviders.AzureOpenAI)] | ||||||||||||||||||
| [InlineData(ChatClientProviders.AzureAIAgentsPersistent)] | ||||||||||||||||||
| [InlineData(ChatClientProviders.OpenAIAssistant)] | ||||||||||||||||||
| [InlineData(ChatClientProviders.OpenAIChatCompletion)] | ||||||||||||||||||
| [InlineData(ChatClientProviders.OpenAIResponses)] | ||||||||||||||||||
| public async Task ApprovalsWithTools(ChatClientProviders provider) | ||||||||||||||||||
| { | ||||||||||||||||||
| // Creating a MenuTools instance to be used by the agent. | ||||||||||||||||||
| var menuTools = new MenuTools(); | ||||||||||||||||||
|
|
||||||||||||||||||
| // Define the options for the chat client agent. | ||||||||||||||||||
| var agentOptions = new ChatClientAgentOptions( | ||||||||||||||||||
| name: "Host", | ||||||||||||||||||
| instructions: "Answer questions about the menu", | ||||||||||||||||||
| tools: [ | ||||||||||||||||||
| AIFunctionFactory.Create(menuTools.GetMenu), | ||||||||||||||||||
| new ApprovalRequiredAIFunction(AIFunctionFactory.Create(menuTools.GetSpecials)), | ||||||||||||||||||
| AIFunctionFactory.Create(menuTools.GetItemPrice), | ||||||||||||||||||
| new HostedMcpServerTool("MyService", new Uri("https://mcp-server.example.com")) | ||||||||||||||||||
| { | ||||||||||||||||||
| AllowedTools = ["add"], | ||||||||||||||||||
| ApprovalMode = HostedMcpServerToolApprovalMode.AlwaysRequire, | ||||||||||||||||||
| } | ||||||||||||||||||
| ]); | ||||||||||||||||||
|
|
||||||||||||||||||
| // Create the server-side agent Id when applicable (depending on the provider). | ||||||||||||||||||
| agentOptions.Id = await base.AgentCreateAsync(provider, agentOptions); | ||||||||||||||||||
|
|
||||||||||||||||||
| // Get the chat client to use for the agent. | ||||||||||||||||||
| using var chatClient = base.GetChatClient(provider, agentOptions); | ||||||||||||||||||
|
|
||||||||||||||||||
| var chatBuilder = chatClient.AsBuilder(); | ||||||||||||||||||
| if (chatClient.GetService<HostedMCPChatClient>() is null) | ||||||||||||||||||
| { | ||||||||||||||||||
| chatBuilder.Use((IChatClient innerClient, IServiceProvider services) => | ||||||||||||||||||
| { | ||||||||||||||||||
| return new HostedMCPChatClient(innerClient); | ||||||||||||||||||
| }); | ||||||||||||||||||
| } | ||||||||||||||||||
| if (chatClient.GetService<FunctionInvokingChatClientWithBuiltInApprovals>() is null) | ||||||||||||||||||
| { | ||||||||||||||||||
| chatBuilder.Use((IChatClient innerClient, IServiceProvider services) => | ||||||||||||||||||
| { | ||||||||||||||||||
| return new FunctionInvokingChatClientWithBuiltInApprovals(innerClient, null, services); | ||||||||||||||||||
| }); | ||||||||||||||||||
| } | ||||||||||||||||||
| using var chatClientWithMCPAndApprovals = chatBuilder.Build(); | ||||||||||||||||||
|
|
||||||||||||||||||
| // Define the agent | ||||||||||||||||||
| var agent = new ChatClientAgent(chatClientWithMCPAndApprovals, agentOptions); | ||||||||||||||||||
|
|
||||||||||||||||||
| // Create the chat history thread to capture the agent interaction. | ||||||||||||||||||
| var thread = agent.GetNewThread(); | ||||||||||||||||||
|
|
||||||||||||||||||
| // Respond to user input, invoking functions where appropriate. | ||||||||||||||||||
| await RunAgentAsync("What is the special soup and its price?"); | ||||||||||||||||||
| await RunAgentAsync("What is the special drink?"); | ||||||||||||||||||
| await RunAgentAsync("What is 2 + 2?"); | ||||||||||||||||||
|
|
||||||||||||||||||
| async Task RunAgentAsync(string input) | ||||||||||||||||||
| { | ||||||||||||||||||
| this.WriteUserMessage(input); | ||||||||||||||||||
| var response = await agent.RunAsync(input, thread); | ||||||||||||||||||
| this.WriteResponseOutput(response); | ||||||||||||||||||
|
|
||||||||||||||||||
| var userInputRequests = response.UserInputRequests.ToList(); | ||||||||||||||||||
|
|
||||||||||||||||||
| // Loop until all user input requests are handled. | ||||||||||||||||||
| while (userInputRequests.Count > 0) | ||||||||||||||||||
| { | ||||||||||||||||||
| List<ChatMessage> nextIterationMessages = userInputRequests?.Select((request) => request switch | ||||||||||||||||||
| { | ||||||||||||||||||
| FunctionApprovalRequestContent functionApprovalRequest when functionApprovalRequest.FunctionCall.Name == "GetSpecials" || functionApprovalRequest.FunctionCall.Name == "add" => functionApprovalRequest.Approve(), | ||||||||||||||||||
|
||||||||||||||||||
| FunctionApprovalRequestContent functionApprovalRequest when functionApprovalRequest.FunctionCall.Name == "GetSpecials" || functionApprovalRequest.FunctionCall.Name == "add" => functionApprovalRequest.Approve(), | |
| TextContent rejectionReason = new("refused because xyz"); | |
| TextContent approvalReason = new("approved according to the provided evidence"); | |
| DataContent imageEvidence = new(byteArray, "image/jpg"); | |
| FunctionApprovalResponse approvalResponse = new(approve: true, [approvalReason, imageEvidence]); | |
| FunctionApprovalResponse rejectResponse = new(approve: false, [rejectionReason]); | |
| FunctionApprovalRequestContent functionApprovalRequest when functionApprovalRequest.FunctionCall.Name == "GetSpecials" || functionApprovalRequest.FunctionCall.Name == "add" => functionApprovalRequest.Approve(approvalResponse), |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
What do you have in mind in terms of use cases for this reason?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This PR for instance (automated by an agent). I can approve it with LGTM or reject with Needs more, etc. Many scenarios in approvals may require more than just yes/no.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
My question was more about what would use this text? For approval we are just executing the function and returning it's result to the LLM. For rejection we return a standard message to the LLM saying that the function call was not approved by the user.
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -6,6 +6,8 @@ | |
| #endif | ||
| using System.Collections.Generic; | ||
| using System.Diagnostics.CodeAnalysis; | ||
| using System.Linq; | ||
|
|
||
| #if NET9_0_OR_GREATER | ||
| using System.Text; | ||
| #endif | ||
|
|
@@ -82,6 +84,13 @@ public IList<ChatMessage> Messages | |
| [JsonIgnore] | ||
| public string Text => this._messages?.ConcatText() ?? string.Empty; | ||
|
|
||
| /// <summary>Gets or sets the user input requests associated with the response.</summary> | ||
| /// <remarks> | ||
| /// This property concatenates all <see cref="UserInputRequestContent"/> instances in the response. | ||
| /// </remarks> | ||
| [JsonIgnore] | ||
| public IEnumerable<UserInputRequestContent> UserInputRequests => this._messages?.SelectMany(x => x.Contents).OfType<UserInputRequestContent>() ?? Array.Empty<UserInputRequestContent>(); | ||
westey-m marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
westey-m marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||
|
|
||
| /// <summary>Gets or sets the ID of the agent that produced the response.</summary> | ||
| public string? AgentId { get; set; } | ||
|
|
||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -4,6 +4,7 @@ | |
| using System.Collections.Generic; | ||
| using System.Diagnostics; | ||
| using System.Diagnostics.CodeAnalysis; | ||
| using System.Linq; | ||
| using System.Text.Json.Serialization; | ||
| using Microsoft.Shared.Diagnostics; | ||
|
|
||
|
|
@@ -92,6 +93,13 @@ public string? AuthorName | |
| [JsonIgnore] | ||
| public string Text => this._contents is not null ? this._contents.ConcatText() : string.Empty; | ||
|
|
||
| /// <summary>Gets or sets the user input requests associated with the response.</summary> | ||
| /// <remarks> | ||
| /// This property concatenates all <see cref="UserInputRequestContent"/> instances in the response. | ||
| /// </remarks> | ||
| [JsonIgnore] | ||
| public IEnumerable<UserInputRequestContent> UserInputRequests => this._contents?.OfType<UserInputRequestContent>() ?? Array.Empty<UserInputRequestContent>(); | ||
|
||
|
|
||
| /// <summary>Gets or sets the agent run response update content items.</summary> | ||
| [AllowNull] | ||
| public IList<AIContent> Contents | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,45 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
|
|
||
| using Microsoft.Shared.Diagnostics; | ||
|
|
||
| namespace Microsoft.Extensions.AI; | ||
|
|
||
| /// <summary> | ||
| /// Represents a request for user approval of a function call. | ||
| /// </summary> | ||
| public class FunctionApprovalRequestContent : UserInputRequestContent | ||
westey-m marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||
| { | ||
| /// <summary> | ||
| /// Initializes a new instance of the <see cref="FunctionApprovalRequestContent"/> class. | ||
| /// </summary> | ||
| /// <param name="approvalId">The ID to uniquely identify the user input request/response pair.</param> | ||
| /// <param name="functionCall">The function call that requires user approval.</param> | ||
| public FunctionApprovalRequestContent(string approvalId, FunctionCallContent functionCall) | ||
westey-m marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||
| : base(approvalId) | ||
| { | ||
| this.FunctionCall = Throw.IfNull(functionCall); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Gets or sets the function call that pre-invoke approval is required for. | ||
| /// </summary> | ||
| public FunctionCallContent FunctionCall { get; } | ||
|
|
||
| /// <summary> | ||
| /// Creates a <see cref="ChatMessage"/> representing an approval response. | ||
| /// </summary> | ||
| /// <returns>The <see cref="ChatMessage"/> representing the approval response.</returns> | ||
| public ChatMessage Approve() | ||
| { | ||
| return new ChatMessage(ChatRole.User, [new FunctionApprovalResponseContent(this.Id, true, this.FunctionCall)]); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Creates a <see cref="ChatMessage"/> representing a rejection response. | ||
| /// </summary> | ||
| /// <returns>The <see cref="ChatMessage"/> representing the rejection response.</returns> | ||
| public ChatMessage Reject() | ||
| { | ||
| return new ChatMessage(ChatRole.User, [new FunctionApprovalResponseContent(this.Id, false, this.FunctionCall)]); | ||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,34 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
|
|
||
| using Microsoft.Shared.Diagnostics; | ||
|
|
||
| namespace Microsoft.Extensions.AI; | ||
|
|
||
| /// <summary> | ||
| /// Represents a response to a function approval request. | ||
| /// </summary> | ||
| public class FunctionApprovalResponseContent : UserInputResponseContent | ||
| { | ||
| /// <summary> | ||
| /// Initializes a new instance of the <see cref="FunctionApprovalResponseContent"/> class. | ||
| /// </summary> | ||
| /// <param name="approvalId">The ID to uniquely identify the user input request/response pair.</param> | ||
| /// <param name="approved">Indicates whether the request was approved.</param> | ||
| /// <param name="functionCall">The function call that requires user approval.</param> | ||
| public FunctionApprovalResponseContent(string approvalId, bool approved, FunctionCallContent functionCall) | ||
| : base(approvalId) | ||
| { | ||
| this.Approved = approved; | ||
| this.FunctionCall = Throw.IfNull(functionCall); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Gets or sets a value indicating whether the user approved the request. | ||
| /// </summary> | ||
| public bool Approved { get; set; } | ||
westey-m marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||
|
|
||
| /// <summary> | ||
| /// Gets or sets the function call that pre-invoke approval is required for. | ||
| /// </summary> | ||
| public FunctionCallContent FunctionCall { get; } | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
|
|
||
| using Microsoft.Shared.Diagnostics; | ||
|
|
||
| namespace Microsoft.Extensions.AI; | ||
|
|
||
| /// <summary> | ||
| /// Base class for user input request content. | ||
| /// </summary> | ||
| public abstract class UserInputRequestContent : AIContent | ||
| { | ||
| /// <summary> | ||
| /// Initializes a new instance of the <see cref="UserInputRequestContent"/> class. | ||
| /// </summary> | ||
| /// <param name="id">The ID to uniquely identify the user input request/response pair.</param> | ||
| protected UserInputRequestContent(string id) | ||
| { | ||
| Id = Throw.IfNullOrWhitespace(id); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Gets the ID to uniquely identify the user input request/response pair. | ||
| /// </summary> | ||
| public string Id { get; } | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
|
|
||
| using Microsoft.Shared.Diagnostics; | ||
|
|
||
| namespace Microsoft.Extensions.AI; | ||
|
|
||
| /// <summary> | ||
| /// Base class for user input response content. | ||
| /// </summary> | ||
| public abstract class UserInputResponseContent : AIContent | ||
| { | ||
| /// <summary> | ||
| /// Initializes a new instance of the <see cref="UserInputRequestContent"/> class. | ||
| /// </summary> | ||
| /// <param name="id">The ID to uniquely identify the user input request/response pair.</param> | ||
| protected UserInputResponseContent(string id) | ||
| { | ||
| Id = Throw.IfNullOrWhitespace(id); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Gets the ID to uniquely identify the user input request/response pair. | ||
| /// </summary> | ||
| public string Id { get; } | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,61 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
|
|
||
| using System; | ||
| using System.Collections.Generic; | ||
| using Microsoft.Shared.Diagnostics; | ||
|
|
||
| namespace Microsoft.Extensions.AI; | ||
|
|
||
| /// <summary> | ||
| /// Represents a hosted MCP server tool that can be specified to an AI service. | ||
| /// </summary> | ||
| public class HostedMcpServerTool : AITool | ||
westey-m marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| { | ||
| /// <summary> | ||
| /// Initializes a new instance of the <see cref="HostedMcpServerTool"/> class. | ||
| /// </summary> | ||
| /// <param name="serverName">The name of the remote MCP server.</param> | ||
| /// <param name="url">The URL of the remote MCP server.</param> | ||
| public HostedMcpServerTool(string serverName, Uri url) | ||
rogerbarreto marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| { | ||
| ServerName = Throw.IfNullOrWhitespace(serverName); | ||
| Url = Throw.IfNull(url); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Gets the name of the remote MCP server that is used to identify it. | ||
| /// </summary> | ||
| public string ServerName { get; } | ||
|
|
||
| /// <summary> | ||
| /// Gets the URL of the remote MCP server. | ||
| /// </summary> | ||
| public Uri Url { get; } | ||
|
|
||
| /// <summary> | ||
| /// Gets or sets the description of the remote MCP server, used to provide more context to the AI service. | ||
| /// </summary> | ||
| public string? ServerDescription { get; set; } | ||
|
|
||
| /// <summary> | ||
| /// Gets or sets the list of tools allowed to be used by the AI service. | ||
| /// </summary> | ||
| public IList<string>? AllowedTools { get; set; } | ||
|
|
||
| /// <summary> | ||
| /// Gets or sets the approval mode that indicates when the AI service should require user approval for tool calls to the remote MCP server. | ||
| /// </summary> | ||
| /// <remarks> | ||
| /// You can set this property to <see cref="HostedMcpServerToolApprovalMode.AlwaysRequire"/> to require approval for all tool calls, | ||
| /// or to <see cref="HostedMcpServerToolApprovalMode.NeverRequire"/> to never require approval. | ||
| /// </remarks> | ||
| public HostedMcpServerToolApprovalMode? ApprovalMode { get; set; } | ||
|
|
||
| /// <summary> | ||
| /// Gets or sets the HTTP headers that the AI service should use when making tool calls to the remote MCP server. | ||
| /// </summary> | ||
| /// <remarks> | ||
| /// This property is useful for specifying the authentication header or other headers required by the MCP server. | ||
| /// </remarks> | ||
| public IDictionary<string, string>? Headers { get; set; } | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
We may want to extend
AIFunctionFactoryto have a create withapprovalRequired:for convenience.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I want to keep the concepts isolated. I don't want approval being part of the base AIFunction, as it has no impact on how the function actually works and enforces nothing about how folks actually use it. Having it be a higher level type helps with that distinction.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@stephentoub, Not sure if I follow, this is an
Extensionmethod proposal for convenience, not to get rid of theApprovalRequiredAIFunction.The extension can live in high level packages (i.e Agents), don't necessarily needs to be in the
Abstractionspackage.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
As an extension static method requiring C# 14?
And what does it mean to require approval like that when a caller does:
?
Uh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Ops, obsessed with those future extension methods 😊
Whenever using this extension the type would be a approvalRequired specialization, the parameter would be required as well.
Additional to that, helper methods like below could be valid.